feat: kontinuierliche LXC↔Mac-Diagnose + Auto-Retry
Diagnose-Sichtbarkeit (Plan A):
- Job.step_started_at + last_heartbeat_at, Heartbeat-Task pingt alle 3 s
während laufender Mac-Calls
- Mac-Worker hält per X-Job-Id Header einen Log-Ringbuffer pro Job
(200 Zeilen, 1 h TTL); GET /api/jobs/{id}/log liefert ihn aus
- LXC-Endpoint GET /api/jobs/{id}/diag bündelt Job-Status, Stage-Sekunden,
Heartbeat-Alter und Mac-Worker-Log
- Frontend: Live-Timing im Status-Pill ("läuft seit Xs · letzter Ping vor Ys"),
klappbares Diagnose-Panel mit Mac-Log für FAILED + langlaufende Jobs
Robustheit (Plan B):
- MacClient: einmaliger Retry mit 3 s Backoff bei RemoteProtocolError /
ConnectError / ReadError für summarize/protocol/export
- Mac-Worker /api/preload heizt Ollama vor (keep_alive 30 m, num_predict 1);
Pipeline ruft Preload vor summarize, damit Modell-Reload nicht mitten
in der Generierung passiert (Hauptursache "Server disconnected")
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
root
parent
354e4c57eb
commit
f408a555f8
@ -120,6 +120,8 @@ async def retry_job(
|
||||
job.status = JobStatus.QUEUED
|
||||
job.progress = 0
|
||||
job.error = ""
|
||||
job.step_started_at = None
|
||||
job.last_heartbeat_at = None
|
||||
session.add(job)
|
||||
session.commit()
|
||||
session.refresh(job)
|
||||
@ -128,6 +130,21 @@ async def retry_job(
|
||||
return _job_dict(job, user)
|
||||
|
||||
|
||||
@router.get("/{job_id}/diag")
|
||||
async def diag(job_id: int, session: Session = Depends(get_session), user: User = Depends(current_user)):
|
||||
"""Diagnose-Bündel: Job-Status + letzte Mac-Worker-Logzeilen für genau diesen Job."""
|
||||
from app.core.mac_client import MacClient
|
||||
job = _require_access(session.get(Job, job_id), user)
|
||||
client = MacClient()
|
||||
mac_log = await client.fetch_log(job_id, last=200)
|
||||
return {
|
||||
"job": _job_dict(job, user),
|
||||
"step_started_at": job.step_started_at.isoformat() if job.step_started_at else None,
|
||||
"last_heartbeat_at": job.last_heartbeat_at.isoformat() if job.last_heartbeat_at else None,
|
||||
"mac_log": mac_log,
|
||||
}
|
||||
|
||||
|
||||
@router.get("/{job_id}/download/{kind}")
|
||||
def download(job_id: int, kind: str, session: Session = Depends(get_session), user: User = Depends(current_user)):
|
||||
job = _require_access(session.get(Job, job_id), user)
|
||||
@ -158,6 +175,8 @@ def _job_dict(j: Job, user: User, owner_username: str | None = None) -> dict:
|
||||
"profile": j.profile or "meeting",
|
||||
"created_at": j.created_at.isoformat(),
|
||||
"updated_at": j.updated_at.isoformat(),
|
||||
"step_started_at": j.step_started_at.isoformat() if j.step_started_at else None,
|
||||
"last_heartbeat_at": j.last_heartbeat_at.isoformat() if j.last_heartbeat_at else None,
|
||||
"has": {
|
||||
"transcript": bool(j.transcript_path),
|
||||
"summary": bool(j.summary_path),
|
||||
|
||||
@ -1,13 +1,62 @@
|
||||
import asyncio
|
||||
import logging
|
||||
from pathlib import Path
|
||||
|
||||
import httpx
|
||||
|
||||
from .config import settings
|
||||
|
||||
log = logging.getLogger("voice-agent.mac_client")
|
||||
|
||||
# Transient-Fehler, bei denen ein einmaliger Retry sinnvoll ist
|
||||
# (typisch bei Ollama-Reload oder kurzem Mac-Worker-Restart).
|
||||
_RETRY_EXCEPTIONS = (
|
||||
httpx.RemoteProtocolError,
|
||||
httpx.ConnectError,
|
||||
httpx.ReadError,
|
||||
)
|
||||
|
||||
|
||||
class MacClient:
|
||||
def __init__(self, base_url: str | None = None, timeout: int | None = None):
|
||||
self.base_url = (base_url or settings.mac_api_url).rstrip("/")
|
||||
self.timeout = timeout or settings.mac_api_timeout
|
||||
|
||||
@staticmethod
|
||||
def _hdrs(job_id: int | None) -> dict[str, str] | None:
|
||||
if job_id is None:
|
||||
return None
|
||||
return {"X-Job-Id": str(job_id)}
|
||||
|
||||
async def _post_json_retry(self, url: str, payload: dict, job_id: int | None) -> dict:
|
||||
"""JSON-POST mit einmaligem Retry bei transienten Verbindungsabbrüchen."""
|
||||
for attempt in (0, 1):
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=self.timeout) as c:
|
||||
r = await c.post(url, json=payload, headers=self._hdrs(job_id))
|
||||
r.raise_for_status()
|
||||
return r.json()
|
||||
except _RETRY_EXCEPTIONS as e:
|
||||
if attempt == 0:
|
||||
log.warning("[job %s] %s bei %s — Retry in 3s", job_id, type(e).__name__, url)
|
||||
await asyncio.sleep(3.0)
|
||||
continue
|
||||
raise
|
||||
|
||||
async def _post_bytes_retry(self, url: str, payload: dict, job_id: int | None) -> bytes:
|
||||
for attempt in (0, 1):
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=self.timeout) as c:
|
||||
r = await c.post(url, json=payload, headers=self._hdrs(job_id))
|
||||
r.raise_for_status()
|
||||
return r.content
|
||||
except _RETRY_EXCEPTIONS as e:
|
||||
if attempt == 0:
|
||||
log.warning("[job %s] %s bei %s — Retry in 3s", job_id, type(e).__name__, url)
|
||||
await asyncio.sleep(3.0)
|
||||
continue
|
||||
raise
|
||||
|
||||
async def health(self) -> dict:
|
||||
async with httpx.AsyncClient(timeout=10) as c:
|
||||
r = await c.get(f"{self.base_url}/health")
|
||||
@ -20,47 +69,67 @@ class MacClient:
|
||||
r.raise_for_status()
|
||||
return r.json().get("profiles", [])
|
||||
|
||||
async def transcribe(self, audio_path: Path, language: str = "de") -> dict:
|
||||
async def fetch_log(self, job_id: int, last: int = 200) -> list[str]:
|
||||
"""Holt die letzten Mac-Worker-Logzeilen für einen Job (für die Diagnose-Anzeige)."""
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=8) as c:
|
||||
r = await c.get(f"{self.base_url}/api/jobs/{job_id}/log", params={"last": last})
|
||||
r.raise_for_status()
|
||||
return r.json().get("lines", [])
|
||||
except Exception as e: # noqa: BLE001
|
||||
log.warning("fetch_log job=%s failed: %s", job_id, e)
|
||||
return []
|
||||
|
||||
async def preload(self, profile: str | None = None, job_id: int | None = None) -> dict:
|
||||
"""Vor-Heizen: Ollama soll das Modell laden, bevor der echte Call läuft."""
|
||||
return await self._post_json_retry(
|
||||
f"{self.base_url}/api/preload",
|
||||
{"profile": profile},
|
||||
job_id,
|
||||
)
|
||||
|
||||
async def transcribe(self, audio_path: Path, language: str = "de", job_id: int | None = None) -> dict:
|
||||
# Kein Retry: Datei-Upload ist nicht idempotent, und die Transkription
|
||||
# ist der teuerste Schritt — Fehler hier sollen sichtbar bleiben.
|
||||
async with httpx.AsyncClient(timeout=self.timeout) as c:
|
||||
with audio_path.open("rb") as f:
|
||||
files = {"audio": (audio_path.name, f, "application/octet-stream")}
|
||||
data = {"language": language}
|
||||
r = await c.post(f"{self.base_url}/api/transcribe", files=files, data=data)
|
||||
r = await c.post(
|
||||
f"{self.base_url}/api/transcribe",
|
||||
files=files,
|
||||
data=data,
|
||||
headers=self._hdrs(job_id),
|
||||
)
|
||||
r.raise_for_status()
|
||||
return r.json()
|
||||
|
||||
async def summarize(self, transcript: str, title: str = "", profile: str | None = None) -> dict:
|
||||
async with httpx.AsyncClient(timeout=self.timeout) as c:
|
||||
r = await c.post(
|
||||
async def summarize(self, transcript: str, title: str = "", profile: str | None = None,
|
||||
job_id: int | None = None) -> dict:
|
||||
return await self._post_json_retry(
|
||||
f"{self.base_url}/api/summarize",
|
||||
json={"transcript": transcript, "title": title, "profile": profile},
|
||||
{"transcript": transcript, "title": title, "profile": profile},
|
||||
job_id,
|
||||
)
|
||||
r.raise_for_status()
|
||||
return r.json()
|
||||
|
||||
async def protocol(self, transcript: str, summary: dict, title: str = "", profile: str | None = None) -> dict:
|
||||
async with httpx.AsyncClient(timeout=self.timeout) as c:
|
||||
r = await c.post(
|
||||
async def protocol(self, transcript: str, summary: dict, title: str = "", profile: str | None = None,
|
||||
job_id: int | None = None) -> dict:
|
||||
return await self._post_json_retry(
|
||||
f"{self.base_url}/api/protocol",
|
||||
json={"transcript": transcript, "summary": summary, "title": title, "profile": profile},
|
||||
{"transcript": transcript, "summary": summary, "title": title, "profile": profile},
|
||||
job_id,
|
||||
)
|
||||
r.raise_for_status()
|
||||
return r.json()
|
||||
|
||||
async def export_docx(self, data: dict, profile: str | None = None) -> bytes:
|
||||
async with httpx.AsyncClient(timeout=self.timeout) as c:
|
||||
r = await c.post(
|
||||
async def export_docx(self, data: dict, profile: str | None = None, job_id: int | None = None) -> bytes:
|
||||
return await self._post_bytes_retry(
|
||||
f"{self.base_url}/api/export/docx",
|
||||
json={"data": data, "profile": profile},
|
||||
{"data": data, "profile": profile},
|
||||
job_id,
|
||||
)
|
||||
r.raise_for_status()
|
||||
return r.content
|
||||
|
||||
async def export_pdf(self, data: dict, profile: str | None = None) -> bytes:
|
||||
async with httpx.AsyncClient(timeout=self.timeout) as c:
|
||||
r = await c.post(
|
||||
async def export_pdf(self, data: dict, profile: str | None = None, job_id: int | None = None) -> bytes:
|
||||
return await self._post_bytes_retry(
|
||||
f"{self.base_url}/api/export/pdf",
|
||||
json={"data": data, "profile": profile},
|
||||
{"data": data, "profile": profile},
|
||||
job_id,
|
||||
)
|
||||
r.raise_for_status()
|
||||
return r.content
|
||||
|
||||
@ -46,6 +46,15 @@ def run_migrations() -> None:
|
||||
log.info("Migration: füge job.pdf_path hinzu")
|
||||
conn.execute(text("ALTER TABLE job ADD COLUMN pdf_path VARCHAR DEFAULT ''"))
|
||||
|
||||
# 5) Diagnose-Felder: step_started_at, last_heartbeat_at
|
||||
cols = _existing_columns(conn, "job")
|
||||
if "step_started_at" not in cols:
|
||||
log.info("Migration: füge job.step_started_at hinzu")
|
||||
conn.execute(text("ALTER TABLE job ADD COLUMN step_started_at DATETIME"))
|
||||
if "last_heartbeat_at" not in cols:
|
||||
log.info("Migration: füge job.last_heartbeat_at hinzu")
|
||||
conn.execute(text("ALTER TABLE job ADD COLUMN last_heartbeat_at DATETIME"))
|
||||
|
||||
|
||||
def assign_orphan_jobs_to(user_id: int) -> int:
|
||||
"""Weist alle Jobs ohne owner_id dem angegebenen User zu. Gibt Anzahl zurück."""
|
||||
|
||||
@ -28,5 +28,7 @@ class Job(SQLModel, table=True):
|
||||
protocol_path: str = ""
|
||||
docx_path: str = ""
|
||||
pdf_path: str = ""
|
||||
step_started_at: Optional[datetime] = Field(default=None)
|
||||
last_heartbeat_at: Optional[datetime] = Field(default=None)
|
||||
created_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc))
|
||||
updated_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc))
|
||||
|
||||
@ -1,7 +1,9 @@
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Awaitable, TypeVar
|
||||
|
||||
from sqlmodel import Session
|
||||
|
||||
@ -12,6 +14,10 @@ from app.models.job import Job, JobStatus
|
||||
|
||||
log = logging.getLogger("voice-agent.pipeline")
|
||||
|
||||
T = TypeVar("T")
|
||||
|
||||
HEARTBEAT_INTERVAL = 3.0 # Sekunden
|
||||
|
||||
|
||||
def _update(job_id: int, **fields) -> None:
|
||||
with Session(engine) as s:
|
||||
@ -32,8 +38,51 @@ def _existing(path_str: str) -> Path | None:
|
||||
return p if p.exists() and p.stat().st_size > 0 else None
|
||||
|
||||
|
||||
def _start_step(job_id: int, status: str, progress: int) -> None:
|
||||
"""Setzt status/progress + step_started_at + initialen Heartbeat."""
|
||||
now = datetime.now(timezone.utc)
|
||||
_update(
|
||||
job_id,
|
||||
status=status,
|
||||
progress=progress,
|
||||
step_started_at=now,
|
||||
last_heartbeat_at=now,
|
||||
error="",
|
||||
)
|
||||
|
||||
|
||||
async def _with_heartbeat(job_id: int, coro: Awaitable[T]) -> T:
|
||||
"""Führt `coro` aus und pingt parallel alle paar Sekunden last_heartbeat_at.
|
||||
|
||||
Damit sieht die UI live, ob ein Schritt noch lebt — auch wenn der Mac
|
||||
minutenlang an einer Ollama-Generierung arbeitet.
|
||||
"""
|
||||
stop = asyncio.Event()
|
||||
|
||||
async def beat() -> None:
|
||||
while not stop.is_set():
|
||||
try:
|
||||
_update(job_id, last_heartbeat_at=datetime.now(timezone.utc))
|
||||
except Exception: # noqa: BLE001
|
||||
pass # Heartbeat darf den Job nie killen
|
||||
try:
|
||||
await asyncio.wait_for(stop.wait(), timeout=HEARTBEAT_INTERVAL)
|
||||
except asyncio.TimeoutError:
|
||||
pass
|
||||
|
||||
beat_task = asyncio.create_task(beat())
|
||||
try:
|
||||
return await coro
|
||||
finally:
|
||||
stop.set()
|
||||
try:
|
||||
await beat_task
|
||||
except Exception: # noqa: BLE001
|
||||
pass
|
||||
|
||||
|
||||
async def run_pipeline(job_id: int) -> None:
|
||||
"""Pipeline: transcribe → summarize → protocol → DOCX.
|
||||
"""Pipeline: transcribe → preload → summarize → protocol → DOCX → PDF.
|
||||
|
||||
Idempotent: bereits vorhandene Zwischenergebnisse werden weiterverwendet,
|
||||
damit ein 'Erneut versuchen' nicht die teure Transkription wiederholt.
|
||||
@ -64,9 +113,9 @@ async def run_pipeline(job_id: int) -> None:
|
||||
tx = json.loads(existing_tx_path.read_text(encoding="utf-8"))
|
||||
_update(job_id, status=JobStatus.TRANSCRIBING, progress=45)
|
||||
else:
|
||||
_update(job_id, status=JobStatus.TRANSCRIBING, progress=10, error="")
|
||||
_start_step(job_id, JobStatus.TRANSCRIBING, 10)
|
||||
log.info("[job %s] transcribe (profile=%s)", job_id, profile)
|
||||
tx = await client.transcribe(audio_path, language="de")
|
||||
tx = await _with_heartbeat(job_id, client.transcribe(audio_path, language="de", job_id=job_id))
|
||||
transcript_path = job_dir / "transcript.json"
|
||||
transcript_path.write_text(json.dumps(tx, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
_update(job_id, transcript_path=str(transcript_path), progress=45)
|
||||
@ -75,15 +124,24 @@ async def run_pipeline(job_id: int) -> None:
|
||||
s.get("text", "") for s in tx.get("segments", [])
|
||||
)
|
||||
|
||||
# 2) SUMMARY
|
||||
# 2) SUMMARY — vorher Preload, damit Ollama nicht mitten drin lädt
|
||||
if existing_sum_path:
|
||||
log.info("[job %s] reuse summary %s", job_id, existing_sum_path.name)
|
||||
summary = json.loads(existing_sum_path.read_text(encoding="utf-8"))
|
||||
_update(job_id, status=JobStatus.SUMMARIZING, progress=70)
|
||||
else:
|
||||
_update(job_id, status=JobStatus.SUMMARIZING, progress=55)
|
||||
_start_step(job_id, JobStatus.SUMMARIZING, 50)
|
||||
log.info("[job %s] preload model", job_id)
|
||||
try:
|
||||
await _with_heartbeat(job_id, client.preload(profile=profile, job_id=job_id))
|
||||
except Exception as e: # noqa: BLE001
|
||||
log.warning("[job %s] preload failed (non-fatal): %s", job_id, e)
|
||||
_update(job_id, progress=55)
|
||||
log.info("[job %s] summarize", job_id)
|
||||
summary = await client.summarize(transcript_text, title=title, profile=profile)
|
||||
summary = await _with_heartbeat(
|
||||
job_id,
|
||||
client.summarize(transcript_text, title=title, profile=profile, job_id=job_id),
|
||||
)
|
||||
summary_path = job_dir / "summary.json"
|
||||
summary_path.write_text(json.dumps(summary, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
_update(job_id, summary_path=str(summary_path), progress=70)
|
||||
@ -94,9 +152,12 @@ async def run_pipeline(job_id: int) -> None:
|
||||
protocol = json.loads(existing_pro_path.read_text(encoding="utf-8"))
|
||||
_update(job_id, status=JobStatus.PROTOCOLLING, progress=85)
|
||||
else:
|
||||
_update(job_id, status=JobStatus.PROTOCOLLING, progress=75)
|
||||
_start_step(job_id, JobStatus.PROTOCOLLING, 75)
|
||||
log.info("[job %s] protocol", job_id)
|
||||
protocol = await client.protocol(transcript_text, summary, title=title, profile=profile)
|
||||
protocol = await _with_heartbeat(
|
||||
job_id,
|
||||
client.protocol(transcript_text, summary, title=title, profile=profile, job_id=job_id),
|
||||
)
|
||||
protocol_path = job_dir / "protocol.json"
|
||||
protocol_path.write_text(json.dumps(protocol, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
_update(job_id, protocol_path=str(protocol_path), progress=85)
|
||||
@ -105,9 +166,12 @@ async def run_pipeline(job_id: int) -> None:
|
||||
if existing_dx_path:
|
||||
log.info("[job %s] reuse docx %s", job_id, existing_dx_path.name)
|
||||
else:
|
||||
_update(job_id, status=JobStatus.EXPORTING, progress=90)
|
||||
_start_step(job_id, JobStatus.EXPORTING, 90)
|
||||
log.info("[job %s] export docx", job_id)
|
||||
docx_bytes = await client.export_docx(protocol, profile=profile)
|
||||
docx_bytes = await _with_heartbeat(
|
||||
job_id,
|
||||
client.export_docx(protocol, profile=profile, job_id=job_id),
|
||||
)
|
||||
docx_path = job_dir / "protocol.docx"
|
||||
docx_path.write_bytes(docx_bytes)
|
||||
_update(job_id, docx_path=str(docx_path), progress=95)
|
||||
@ -119,7 +183,10 @@ async def run_pipeline(job_id: int) -> None:
|
||||
else:
|
||||
_update(job_id, status=JobStatus.EXPORTING, progress=96)
|
||||
log.info("[job %s] export pdf", job_id)
|
||||
pdf_bytes = await client.export_pdf(protocol, profile=profile)
|
||||
pdf_bytes = await _with_heartbeat(
|
||||
job_id,
|
||||
client.export_pdf(protocol, profile=profile, job_id=job_id),
|
||||
)
|
||||
pdf_path = job_dir / "protocol.pdf"
|
||||
pdf_path.write_bytes(pdf_bytes)
|
||||
_update(job_id, pdf_path=str(pdf_path), status=JobStatus.DONE, progress=100, error="")
|
||||
|
||||
@ -76,6 +76,38 @@ function fmtDate(s) {
|
||||
return d.toLocaleString("de-DE", { dateStyle: "short", timeStyle: "medium" });
|
||||
}
|
||||
|
||||
function fmtSecs(sec) {
|
||||
if (!isFinite(sec) || sec < 0) return "?";
|
||||
if (sec < 60) return `${Math.floor(sec)}s`;
|
||||
const m = Math.floor(sec / 60), s = Math.floor(sec % 60);
|
||||
return `${m}m ${s}s`;
|
||||
}
|
||||
|
||||
const RUNNING_STATES = new Set(["queued", "transcribing", "summarizing", "protocolling", "exporting"]);
|
||||
const DIAG_STATES = new Set([...RUNNING_STATES, "failed"]);
|
||||
|
||||
// Welche Jobs haben aktuell ein offenes Diag-Panel? Über Re-Renders hinweg merken.
|
||||
const openDiagIds = new Set();
|
||||
|
||||
function liveTimingHtml(job) {
|
||||
if (!RUNNING_STATES.has(job.status) || !job.step_started_at) return "";
|
||||
const now = Date.now();
|
||||
const stepSec = (now - new Date(job.step_started_at).getTime()) / 1000;
|
||||
let parts = [`läuft seit ${fmtSecs(stepSec)}`];
|
||||
if (job.last_heartbeat_at) {
|
||||
const beatSec = (now - new Date(job.last_heartbeat_at).getTime()) / 1000;
|
||||
const stale = beatSec > 10;
|
||||
parts.push(`letzter Ping vor ${fmtSecs(beatSec)}${stale ? " ⚠" : ""}`);
|
||||
}
|
||||
return `<small class="muted live-timing">${parts.join(" · ")}</small>`;
|
||||
}
|
||||
|
||||
function diagButtonHtml(job) {
|
||||
if (!DIAG_STATES.has(job.status)) return "";
|
||||
const open = openDiagIds.has(String(job.id));
|
||||
return `<button type="button" class="btn-link btn-diag" data-id="${job.id}">Diagnose ${open ? "▴" : "▾"}</button>`;
|
||||
}
|
||||
|
||||
async function api(path, opts = {}) {
|
||||
const r = await fetch(path, {
|
||||
credentials: "same-origin",
|
||||
@ -294,10 +326,13 @@ function row(job) {
|
||||
const retryBtn = job.status !== "done"
|
||||
? `<button type="button" class="btn-link btn-retry" data-id="${job.id}" title="Verarbeitung wiederholen (Transkript bleibt erhalten)">↻ Erneut versuchen</button>`
|
||||
: "";
|
||||
const timing = liveTimingHtml(job);
|
||||
const statusCell = `
|
||||
<span class="status-pill status-${job.status}">${escapeHtml(job.status)}</span>
|
||||
${timing ? `<div class="status-timing">${timing}</div>` : ""}
|
||||
${job.error ? `<div class="status-error" title="${escapeHtml(job.error)}">${escapeHtml(job.error)}</div>` : ""}
|
||||
${retryBtn}
|
||||
${diagButtonHtml(job)}
|
||||
`;
|
||||
|
||||
tr.innerHTML = `
|
||||
@ -311,6 +346,15 @@ function row(job) {
|
||||
return tr;
|
||||
}
|
||||
|
||||
// Zweite Tabellenzeile direkt unter `tr`, in die das Diagnose-Panel rutscht.
|
||||
function diagRow(job) {
|
||||
const tr = document.createElement("tr");
|
||||
tr.className = "diag-row hidden";
|
||||
tr.dataset.diagId = String(job.id);
|
||||
tr.innerHTML = `<td colspan="6"><div class="diag-content"><div class="muted">Lade Diagnose …</div></div></td>`;
|
||||
return tr;
|
||||
}
|
||||
|
||||
function card(job) {
|
||||
const div = document.createElement("div");
|
||||
div.className = "jc";
|
||||
@ -328,6 +372,7 @@ function card(job) {
|
||||
return `<a class="btn-dl ${dis ? "disabled" : ""}" ${href} title="${escapeHtml(tip)}">${label}</a>`;
|
||||
}).join("");
|
||||
|
||||
const timing = liveTimingHtml(job);
|
||||
div.innerHTML = `
|
||||
<div class="jc-head">
|
||||
<div>
|
||||
@ -341,8 +386,15 @@ function card(job) {
|
||||
<div class="jc-progress"><div class="bar"><div class="bar-fill" style="width:${job.progress}%"></div></div></div>
|
||||
<small class="muted">${job.progress}%</small>
|
||||
</div>
|
||||
${timing ? `<div class="jc-timing">${timing}</div>` : ""}
|
||||
${job.error ? `<div class="jc-error" title="${escapeHtml(job.error)}">${escapeHtml(job.error)}</div>` : ""}
|
||||
<div class="jc-actions">
|
||||
${job.status !== "done" ? `<button type="button" class="btn-link btn-retry" data-id="${job.id}">↻ Erneut versuchen</button>` : ""}
|
||||
${diagButtonHtml(job)}
|
||||
</div>
|
||||
<div class="diag-panel hidden" data-diag-id="${job.id}">
|
||||
<div class="diag-content"><div class="muted">Lade Diagnose …</div></div>
|
||||
</div>
|
||||
<div class="jc-sub">aktualisiert: ${escapeHtml(fmtDate(job.updated_at))}</div>
|
||||
<div class="jc-dl">${dlButtons}</div>
|
||||
`;
|
||||
@ -364,13 +416,83 @@ async function loadJobs() {
|
||||
}
|
||||
jobs.forEach((j) => {
|
||||
jobsBody.appendChild(row(j));
|
||||
jobsBody.appendChild(diagRow(j));
|
||||
jobsCards.appendChild(card(j));
|
||||
});
|
||||
// Vorher offene Diagnosen wieder aufklappen + frisch nachladen.
|
||||
openDiagIds.forEach((id) => {
|
||||
document.querySelectorAll(`[data-diag-id="${id}"]`).forEach((el) => el.classList.remove("hidden"));
|
||||
refreshDiag(id);
|
||||
});
|
||||
} catch (e) {
|
||||
console.error(e);
|
||||
}
|
||||
}
|
||||
|
||||
function renderDiag(diag) {
|
||||
const job = diag.job;
|
||||
const lines = (diag.mac_log || []);
|
||||
const stepInfo = diag.step_started_at
|
||||
? `Stage <strong>${escapeHtml(job.status)}</strong> seit ${fmtSecs((Date.now() - new Date(diag.step_started_at).getTime()) / 1000)}`
|
||||
: "kein Stage-Start gespeichert";
|
||||
const beatInfo = diag.last_heartbeat_at
|
||||
? `Letzter Heartbeat vor ${fmtSecs((Date.now() - new Date(diag.last_heartbeat_at).getTime()) / 1000)}`
|
||||
: "noch kein Heartbeat";
|
||||
const errBlock = job.error
|
||||
? `<div class="diag-err"><strong>Fehler:</strong> ${escapeHtml(job.error)}</div>`
|
||||
: "";
|
||||
const logBlock = lines.length
|
||||
? `<pre class="diag-log">${escapeHtml(lines.join("\n"))}</pre>`
|
||||
: `<div class="muted">Keine Mac-Worker-Logzeilen für diesen Job (Buffer leer oder Mac nicht erreichbar).</div>`;
|
||||
return `
|
||||
<div class="diag-meta">
|
||||
<div>${stepInfo}</div>
|
||||
<div>${beatInfo}</div>
|
||||
<div>Profil: <code>${escapeHtml(job.profile)}</code></div>
|
||||
</div>
|
||||
${errBlock}
|
||||
<div class="diag-loghead">Mac-Worker-Log (letzte ${lines.length} Zeilen)</div>
|
||||
${logBlock}
|
||||
`;
|
||||
}
|
||||
|
||||
async function refreshDiag(jobId) {
|
||||
let data;
|
||||
try {
|
||||
const r = await api(`/api/jobs/${jobId}/diag`);
|
||||
if (!r.ok) throw new Error(`HTTP ${r.status}`);
|
||||
data = await r.json();
|
||||
} catch (e) {
|
||||
document.querySelectorAll(`[data-diag-id="${jobId}"] .diag-content`).forEach((el) => {
|
||||
el.innerHTML = `<div class="diag-err">Diagnose konnte nicht geladen werden: ${escapeHtml(e.message)}</div>`;
|
||||
});
|
||||
return;
|
||||
}
|
||||
const html = renderDiag(data);
|
||||
document.querySelectorAll(`[data-diag-id="${jobId}"] .diag-content`).forEach((el) => {
|
||||
el.innerHTML = html;
|
||||
});
|
||||
}
|
||||
|
||||
// Diagnose-Toggle (Tabelle und Karten gemeinsam, Event-Delegation)
|
||||
document.addEventListener("click", (ev) => {
|
||||
const btn = ev.target.closest(".btn-diag");
|
||||
if (!btn) return;
|
||||
ev.preventDefault();
|
||||
const id = btn.dataset.id;
|
||||
const wasOpen = openDiagIds.has(id);
|
||||
if (wasOpen) {
|
||||
openDiagIds.delete(id);
|
||||
document.querySelectorAll(`[data-diag-id="${id}"]`).forEach((el) => el.classList.add("hidden"));
|
||||
btn.textContent = "Diagnose ▾";
|
||||
} else {
|
||||
openDiagIds.add(id);
|
||||
document.querySelectorAll(`[data-diag-id="${id}"]`).forEach((el) => el.classList.remove("hidden"));
|
||||
btn.textContent = "Diagnose ▴";
|
||||
refreshDiag(id);
|
||||
}
|
||||
});
|
||||
|
||||
form.addEventListener("submit", (ev) => {
|
||||
ev.preventDefault();
|
||||
// Quelle: Aufnahme bevorzugt, sonst File-Input
|
||||
|
||||
@ -560,3 +560,93 @@ select {
|
||||
text-transform: lowercase;
|
||||
vertical-align: middle;
|
||||
}
|
||||
|
||||
/* Live-Timing direkt unter dem Status-Pill (läuft seit · letzter Ping vor) */
|
||||
.status-timing, .jc-timing {
|
||||
margin-top: 4px;
|
||||
font-size: 11px;
|
||||
line-height: 1.3;
|
||||
}
|
||||
.live-timing { color: var(--muted); }
|
||||
|
||||
/* Diagnose-Button – wie btn-retry, aber neutraler Ton */
|
||||
.btn-diag {
|
||||
display: inline-block;
|
||||
margin-top: 6px;
|
||||
margin-left: 4px;
|
||||
padding: 4px 10px;
|
||||
font-size: 12px;
|
||||
background: rgba(120,120,120,.10);
|
||||
border: 1px solid rgba(160,160,160,.35);
|
||||
color: var(--muted);
|
||||
border-radius: 5px;
|
||||
text-decoration: none;
|
||||
cursor: pointer;
|
||||
}
|
||||
.btn-diag:hover { background: rgba(160,160,160,.20); color: inherit; }
|
||||
|
||||
/* Aktions-Reihe in der mobilen Karte */
|
||||
.jc-actions {
|
||||
display: flex;
|
||||
flex-wrap: wrap;
|
||||
gap: 6px;
|
||||
margin-top: 4px;
|
||||
}
|
||||
|
||||
/* Diagnose-Panel: Tabelle (zweite Zeile, full-width) und Karte */
|
||||
.diag-row > td {
|
||||
background: rgba(255,255,255,.03);
|
||||
border-top: 0;
|
||||
padding: 10px 12px;
|
||||
}
|
||||
.diag-panel {
|
||||
margin-top: 10px;
|
||||
padding: 10px;
|
||||
background: rgba(255,255,255,.03);
|
||||
border: 1px solid rgba(255,255,255,.08);
|
||||
border-radius: 6px;
|
||||
}
|
||||
.diag-meta {
|
||||
display: flex;
|
||||
gap: 16px;
|
||||
flex-wrap: wrap;
|
||||
font-size: 12px;
|
||||
color: var(--muted);
|
||||
margin-bottom: 6px;
|
||||
}
|
||||
.diag-meta code {
|
||||
background: rgba(255,255,255,.08);
|
||||
padding: 1px 4px;
|
||||
border-radius: 3px;
|
||||
}
|
||||
.diag-err {
|
||||
margin: 6px 0;
|
||||
padding: 6px 8px;
|
||||
background: rgba(231,76,60,.10);
|
||||
border: 1px solid rgba(231,76,60,.35);
|
||||
border-radius: 4px;
|
||||
color: var(--err);
|
||||
font-size: 12px;
|
||||
word-break: break-word;
|
||||
}
|
||||
.diag-loghead {
|
||||
margin-top: 8px;
|
||||
font-size: 11px;
|
||||
text-transform: uppercase;
|
||||
letter-spacing: .05em;
|
||||
color: var(--muted);
|
||||
}
|
||||
.diag-log {
|
||||
margin: 4px 0 0 0;
|
||||
padding: 8px;
|
||||
background: rgba(0,0,0,.30);
|
||||
border: 1px solid rgba(255,255,255,.06);
|
||||
border-radius: 4px;
|
||||
max-height: 320px;
|
||||
overflow: auto;
|
||||
font-family: ui-monospace, "SF Mono", Menlo, Consolas, monospace;
|
||||
font-size: 11.5px;
|
||||
line-height: 1.45;
|
||||
white-space: pre-wrap;
|
||||
word-break: break-word;
|
||||
}
|
||||
|
||||
69
mac-worker/app/core/joblog.py
Normal file
69
mac-worker/app/core/joblog.py
Normal file
@ -0,0 +1,69 @@
|
||||
"""Per-Job Log-Ringbuffer.
|
||||
|
||||
Während ein HTTP-Request läuft, setzt eine Middleware die ContextVar `current_job`
|
||||
auf die `X-Job-Id` aus dem Header. Ein Logging-Handler greift diesen Wert ab und
|
||||
schreibt jede Logzeile zusätzlich in einen Ringbuffer für genau diesen Job.
|
||||
|
||||
Die LXC-Frontend kann den Buffer per `GET /api/jobs/{job_id}/log` abrufen und in
|
||||
der Diagnose-Anzeige rendern — der User sieht so ohne SSH-Zugang zum Mac, was
|
||||
während eines hängenden Calls passiert.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import time
|
||||
from collections import deque
|
||||
from contextvars import ContextVar
|
||||
from threading import Lock
|
||||
|
||||
# Per-Coroutine: welcher Job-ID gehört dem aktuellen Async-Kontext.
|
||||
current_job: ContextVar[int | None] = ContextVar("current_job", default=None)
|
||||
|
||||
_RING_MAX = 200 # Zeilen pro Job
|
||||
_TTL_SECONDS = 3600 # alte Buffers nach 1h löschen
|
||||
_buffers: dict[int, dict] = {}
|
||||
_lock = Lock()
|
||||
|
||||
|
||||
def append(job_id: int, line: str) -> None:
|
||||
now = time.time()
|
||||
with _lock:
|
||||
buf = _buffers.get(job_id)
|
||||
if buf is None:
|
||||
buf = _buffers[job_id] = {"lines": deque(maxlen=_RING_MAX), "last": now}
|
||||
ts = time.strftime("%H:%M:%S", time.localtime(now))
|
||||
buf["lines"].append(f"{ts} {line}")
|
||||
buf["last"] = now
|
||||
# Lazy GC: alte Buffer rauswerfen.
|
||||
for jid in list(_buffers):
|
||||
if now - _buffers[jid]["last"] > _TTL_SECONDS:
|
||||
_buffers.pop(jid, None)
|
||||
|
||||
|
||||
def get(job_id: int, last_n: int = 200) -> list[str]:
|
||||
with _lock:
|
||||
buf = _buffers.get(job_id)
|
||||
if buf is None:
|
||||
return []
|
||||
return list(buf["lines"])[-last_n:]
|
||||
|
||||
|
||||
class JobLogHandler(logging.Handler):
|
||||
"""Spiegelt jede Log-Zeile in den Ringbuffer des aktuellen Jobs."""
|
||||
|
||||
def emit(self, record: logging.LogRecord) -> None:
|
||||
jid = current_job.get()
|
||||
if jid is None:
|
||||
return
|
||||
try:
|
||||
msg = self.format(record)
|
||||
except Exception: # noqa: BLE001
|
||||
msg = record.getMessage()
|
||||
append(jid, msg)
|
||||
|
||||
|
||||
def install() -> None:
|
||||
"""An Root-Logger anhängen, damit alle Module abgegriffen werden."""
|
||||
h = JobLogHandler()
|
||||
h.setFormatter(logging.Formatter("[%(levelname)s] %(name)s: %(message)s"))
|
||||
logging.getLogger().addHandler(h)
|
||||
@ -2,11 +2,12 @@ import logging
|
||||
import secrets
|
||||
from pathlib import Path
|
||||
|
||||
from fastapi import FastAPI, File, Form, HTTPException, UploadFile
|
||||
import httpx
|
||||
from fastapi import FastAPI, File, Form, HTTPException, Request, UploadFile
|
||||
from fastapi.responses import Response
|
||||
from pydantic import BaseModel
|
||||
|
||||
from app.core import ollama_client, profiles
|
||||
from app.core import joblog, ollama_client, profiles
|
||||
from app.core.config import settings
|
||||
from app.core.docx_export import build_docx
|
||||
from app.core.pdf_export import build_pdf
|
||||
@ -15,9 +16,30 @@ from app.core.whisper_engine import engine as whisper
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(name)s: %(message)s")
|
||||
log = logging.getLogger("mac-worker")
|
||||
|
||||
# Per-Job Log-Spiegel installieren — fängt alle Logs während eines Requests
|
||||
# mit X-Job-Id Header und legt sie im Ringbuffer ab.
|
||||
joblog.install()
|
||||
|
||||
app = FastAPI(title="Voice-Agent Mac-Worker")
|
||||
|
||||
|
||||
@app.middleware("http")
|
||||
async def attach_job_id(request: Request, call_next):
|
||||
"""Bindet X-Job-Id-Header an die ContextVar, damit Logs zugeordnet werden."""
|
||||
raw = request.headers.get("x-job-id")
|
||||
token = None
|
||||
if raw:
|
||||
try:
|
||||
token = joblog.current_job.set(int(raw))
|
||||
except ValueError:
|
||||
token = None
|
||||
try:
|
||||
return await call_next(request)
|
||||
finally:
|
||||
if token is not None:
|
||||
joblog.current_job.reset(token)
|
||||
|
||||
|
||||
class SummarizeIn(BaseModel):
|
||||
transcript: str
|
||||
title: str = ""
|
||||
@ -41,6 +63,10 @@ class PdfIn(BaseModel):
|
||||
profile: str | None = None
|
||||
|
||||
|
||||
class PreloadIn(BaseModel):
|
||||
profile: str | None = None
|
||||
|
||||
|
||||
@app.get("/health")
|
||||
async def health():
|
||||
info = {
|
||||
@ -115,3 +141,40 @@ async def export_pdf(payload: PdfIn):
|
||||
media_type="application/pdf",
|
||||
headers={"Content-Disposition": 'attachment; filename="protocol.pdf"'},
|
||||
)
|
||||
|
||||
|
||||
@app.post("/api/preload")
|
||||
async def preload(payload: PreloadIn):
|
||||
"""Forciert Ollama-Modell-Load (keep_alive 30 m) mit Mini-Prompt.
|
||||
|
||||
Wird vom LXC-Frontend vor summarize aufgerufen, damit Ollama den
|
||||
Modell-Reload nicht *während* der eigentlichen Generierung macht
|
||||
(häufige Ursache von 'Server disconnected without sending a response').
|
||||
"""
|
||||
profile_obj = profiles.get(payload.profile)
|
||||
log.info("preload model=%s profile=%s", settings.ollama_model, profile_obj.name)
|
||||
url = f"{settings.ollama_url.rstrip('/')}/api/generate"
|
||||
body = {
|
||||
"model": settings.ollama_model,
|
||||
"prompt": "ping",
|
||||
"stream": False,
|
||||
"keep_alive": "30m",
|
||||
"options": {"num_predict": 1, "temperature": 0},
|
||||
}
|
||||
timeout = httpx.Timeout(connect=15.0, read=300.0, write=15.0, pool=10.0)
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=timeout) as c:
|
||||
r = await c.post(url, json=body)
|
||||
r.raise_for_status()
|
||||
data = r.json()
|
||||
except Exception as e: # noqa: BLE001
|
||||
log.warning("preload failed: %s", e)
|
||||
raise HTTPException(503, f"Ollama preload fehlgeschlagen: {e}") from e
|
||||
log.info("preload ok load_duration=%dms", int(data.get("load_duration", 0) / 1_000_000))
|
||||
return {"ok": True, "model": settings.ollama_model, "load_duration_ns": data.get("load_duration", 0)}
|
||||
|
||||
|
||||
@app.get("/api/jobs/{job_id}/log")
|
||||
async def job_log(job_id: int, last: int = 200):
|
||||
"""Liefert die letzten Log-Zeilen, die beim Bearbeiten dieses Jobs entstanden sind."""
|
||||
return {"job_id": job_id, "lines": joblog.get(job_id, last)}
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user