feat: kontinuierliche LXC↔Mac-Diagnose + Auto-Retry
Diagnose-Sichtbarkeit (Plan A):
- Job.step_started_at + last_heartbeat_at, Heartbeat-Task pingt alle 3 s
während laufender Mac-Calls
- Mac-Worker hält per X-Job-Id Header einen Log-Ringbuffer pro Job
(200 Zeilen, 1 h TTL); GET /api/jobs/{id}/log liefert ihn aus
- LXC-Endpoint GET /api/jobs/{id}/diag bündelt Job-Status, Stage-Sekunden,
Heartbeat-Alter und Mac-Worker-Log
- Frontend: Live-Timing im Status-Pill ("läuft seit Xs · letzter Ping vor Ys"),
klappbares Diagnose-Panel mit Mac-Log für FAILED + langlaufende Jobs
Robustheit (Plan B):
- MacClient: einmaliger Retry mit 3 s Backoff bei RemoteProtocolError /
ConnectError / ReadError für summarize/protocol/export
- Mac-Worker /api/preload heizt Ollama vor (keep_alive 30 m, num_predict 1);
Pipeline ruft Preload vor summarize, damit Modell-Reload nicht mitten
in der Generierung passiert (Hauptursache "Server disconnected")
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
root
@@ -0,0 +1,69 @@
|
||||
"""Per-Job Log-Ringbuffer.
|
||||
|
||||
Während ein HTTP-Request läuft, setzt eine Middleware die ContextVar `current_job`
|
||||
auf die `X-Job-Id` aus dem Header. Ein Logging-Handler greift diesen Wert ab und
|
||||
schreibt jede Logzeile zusätzlich in einen Ringbuffer für genau diesen Job.
|
||||
|
||||
Die LXC-Frontend kann den Buffer per `GET /api/jobs/{job_id}/log` abrufen und in
|
||||
der Diagnose-Anzeige rendern — der User sieht so ohne SSH-Zugang zum Mac, was
|
||||
während eines hängenden Calls passiert.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import time
|
||||
from collections import deque
|
||||
from contextvars import ContextVar
|
||||
from threading import Lock
|
||||
|
||||
# Per-Coroutine: welcher Job-ID gehört dem aktuellen Async-Kontext.
|
||||
current_job: ContextVar[int | None] = ContextVar("current_job", default=None)
|
||||
|
||||
_RING_MAX = 200 # Zeilen pro Job
|
||||
_TTL_SECONDS = 3600 # alte Buffers nach 1h löschen
|
||||
_buffers: dict[int, dict] = {}
|
||||
_lock = Lock()
|
||||
|
||||
|
||||
def append(job_id: int, line: str) -> None:
|
||||
now = time.time()
|
||||
with _lock:
|
||||
buf = _buffers.get(job_id)
|
||||
if buf is None:
|
||||
buf = _buffers[job_id] = {"lines": deque(maxlen=_RING_MAX), "last": now}
|
||||
ts = time.strftime("%H:%M:%S", time.localtime(now))
|
||||
buf["lines"].append(f"{ts} {line}")
|
||||
buf["last"] = now
|
||||
# Lazy GC: alte Buffer rauswerfen.
|
||||
for jid in list(_buffers):
|
||||
if now - _buffers[jid]["last"] > _TTL_SECONDS:
|
||||
_buffers.pop(jid, None)
|
||||
|
||||
|
||||
def get(job_id: int, last_n: int = 200) -> list[str]:
|
||||
with _lock:
|
||||
buf = _buffers.get(job_id)
|
||||
if buf is None:
|
||||
return []
|
||||
return list(buf["lines"])[-last_n:]
|
||||
|
||||
|
||||
class JobLogHandler(logging.Handler):
|
||||
"""Spiegelt jede Log-Zeile in den Ringbuffer des aktuellen Jobs."""
|
||||
|
||||
def emit(self, record: logging.LogRecord) -> None:
|
||||
jid = current_job.get()
|
||||
if jid is None:
|
||||
return
|
||||
try:
|
||||
msg = self.format(record)
|
||||
except Exception: # noqa: BLE001
|
||||
msg = record.getMessage()
|
||||
append(jid, msg)
|
||||
|
||||
|
||||
def install() -> None:
|
||||
"""An Root-Logger anhängen, damit alle Module abgegriffen werden."""
|
||||
h = JobLogHandler()
|
||||
h.setFormatter(logging.Formatter("[%(levelname)s] %(name)s: %(message)s"))
|
||||
logging.getLogger().addHandler(h)
|
||||
+65
-2
@@ -2,11 +2,12 @@ import logging
|
||||
import secrets
|
||||
from pathlib import Path
|
||||
|
||||
from fastapi import FastAPI, File, Form, HTTPException, UploadFile
|
||||
import httpx
|
||||
from fastapi import FastAPI, File, Form, HTTPException, Request, UploadFile
|
||||
from fastapi.responses import Response
|
||||
from pydantic import BaseModel
|
||||
|
||||
from app.core import ollama_client, profiles
|
||||
from app.core import joblog, ollama_client, profiles
|
||||
from app.core.config import settings
|
||||
from app.core.docx_export import build_docx
|
||||
from app.core.pdf_export import build_pdf
|
||||
@@ -15,9 +16,30 @@ from app.core.whisper_engine import engine as whisper
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(name)s: %(message)s")
|
||||
log = logging.getLogger("mac-worker")
|
||||
|
||||
# Per-Job Log-Spiegel installieren — fängt alle Logs während eines Requests
|
||||
# mit X-Job-Id Header und legt sie im Ringbuffer ab.
|
||||
joblog.install()
|
||||
|
||||
app = FastAPI(title="Voice-Agent Mac-Worker")
|
||||
|
||||
|
||||
@app.middleware("http")
|
||||
async def attach_job_id(request: Request, call_next):
|
||||
"""Bindet X-Job-Id-Header an die ContextVar, damit Logs zugeordnet werden."""
|
||||
raw = request.headers.get("x-job-id")
|
||||
token = None
|
||||
if raw:
|
||||
try:
|
||||
token = joblog.current_job.set(int(raw))
|
||||
except ValueError:
|
||||
token = None
|
||||
try:
|
||||
return await call_next(request)
|
||||
finally:
|
||||
if token is not None:
|
||||
joblog.current_job.reset(token)
|
||||
|
||||
|
||||
class SummarizeIn(BaseModel):
|
||||
transcript: str
|
||||
title: str = ""
|
||||
@@ -41,6 +63,10 @@ class PdfIn(BaseModel):
|
||||
profile: str | None = None
|
||||
|
||||
|
||||
class PreloadIn(BaseModel):
|
||||
profile: str | None = None
|
||||
|
||||
|
||||
@app.get("/health")
|
||||
async def health():
|
||||
info = {
|
||||
@@ -115,3 +141,40 @@ async def export_pdf(payload: PdfIn):
|
||||
media_type="application/pdf",
|
||||
headers={"Content-Disposition": 'attachment; filename="protocol.pdf"'},
|
||||
)
|
||||
|
||||
|
||||
@app.post("/api/preload")
|
||||
async def preload(payload: PreloadIn):
|
||||
"""Forciert Ollama-Modell-Load (keep_alive 30 m) mit Mini-Prompt.
|
||||
|
||||
Wird vom LXC-Frontend vor summarize aufgerufen, damit Ollama den
|
||||
Modell-Reload nicht *während* der eigentlichen Generierung macht
|
||||
(häufige Ursache von 'Server disconnected without sending a response').
|
||||
"""
|
||||
profile_obj = profiles.get(payload.profile)
|
||||
log.info("preload model=%s profile=%s", settings.ollama_model, profile_obj.name)
|
||||
url = f"{settings.ollama_url.rstrip('/')}/api/generate"
|
||||
body = {
|
||||
"model": settings.ollama_model,
|
||||
"prompt": "ping",
|
||||
"stream": False,
|
||||
"keep_alive": "30m",
|
||||
"options": {"num_predict": 1, "temperature": 0},
|
||||
}
|
||||
timeout = httpx.Timeout(connect=15.0, read=300.0, write=15.0, pool=10.0)
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=timeout) as c:
|
||||
r = await c.post(url, json=body)
|
||||
r.raise_for_status()
|
||||
data = r.json()
|
||||
except Exception as e: # noqa: BLE001
|
||||
log.warning("preload failed: %s", e)
|
||||
raise HTTPException(503, f"Ollama preload fehlgeschlagen: {e}") from e
|
||||
log.info("preload ok load_duration=%dms", int(data.get("load_duration", 0) / 1_000_000))
|
||||
return {"ok": True, "model": settings.ollama_model, "load_duration_ns": data.get("load_duration", 0)}
|
||||
|
||||
|
||||
@app.get("/api/jobs/{job_id}/log")
|
||||
async def job_log(job_id: int, last: int = 200):
|
||||
"""Liefert die letzten Log-Zeilen, die beim Bearbeiten dieses Jobs entstanden sind."""
|
||||
return {"job_id": job_id, "lines": joblog.get(job_id, last)}
|
||||
|
||||
Reference in New Issue
Block a user