feat: kontinuierliche LXC↔Mac-Diagnose + Auto-Retry
Diagnose-Sichtbarkeit (Plan A):
- Job.step_started_at + last_heartbeat_at, Heartbeat-Task pingt alle 3 s
während laufender Mac-Calls
- Mac-Worker hält per X-Job-Id Header einen Log-Ringbuffer pro Job
(200 Zeilen, 1 h TTL); GET /api/jobs/{id}/log liefert ihn aus
- LXC-Endpoint GET /api/jobs/{id}/diag bündelt Job-Status, Stage-Sekunden,
Heartbeat-Alter und Mac-Worker-Log
- Frontend: Live-Timing im Status-Pill ("läuft seit Xs · letzter Ping vor Ys"),
klappbares Diagnose-Panel mit Mac-Log für FAILED + langlaufende Jobs
Robustheit (Plan B):
- MacClient: einmaliger Retry mit 3 s Backoff bei RemoteProtocolError /
ConnectError / ReadError für summarize/protocol/export
- Mac-Worker /api/preload heizt Ollama vor (keep_alive 30 m, num_predict 1);
Pipeline ruft Preload vor summarize, damit Modell-Reload nicht mitten
in der Generierung passiert (Hauptursache "Server disconnected")
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
root
@@ -76,6 +76,38 @@ function fmtDate(s) {
|
||||
return d.toLocaleString("de-DE", { dateStyle: "short", timeStyle: "medium" });
|
||||
}
|
||||
|
||||
function fmtSecs(sec) {
|
||||
if (!isFinite(sec) || sec < 0) return "?";
|
||||
if (sec < 60) return `${Math.floor(sec)}s`;
|
||||
const m = Math.floor(sec / 60), s = Math.floor(sec % 60);
|
||||
return `${m}m ${s}s`;
|
||||
}
|
||||
|
||||
const RUNNING_STATES = new Set(["queued", "transcribing", "summarizing", "protocolling", "exporting"]);
|
||||
const DIAG_STATES = new Set([...RUNNING_STATES, "failed"]);
|
||||
|
||||
// Welche Jobs haben aktuell ein offenes Diag-Panel? Über Re-Renders hinweg merken.
|
||||
const openDiagIds = new Set();
|
||||
|
||||
function liveTimingHtml(job) {
|
||||
if (!RUNNING_STATES.has(job.status) || !job.step_started_at) return "";
|
||||
const now = Date.now();
|
||||
const stepSec = (now - new Date(job.step_started_at).getTime()) / 1000;
|
||||
let parts = [`läuft seit ${fmtSecs(stepSec)}`];
|
||||
if (job.last_heartbeat_at) {
|
||||
const beatSec = (now - new Date(job.last_heartbeat_at).getTime()) / 1000;
|
||||
const stale = beatSec > 10;
|
||||
parts.push(`letzter Ping vor ${fmtSecs(beatSec)}${stale ? " ⚠" : ""}`);
|
||||
}
|
||||
return `<small class="muted live-timing">${parts.join(" · ")}</small>`;
|
||||
}
|
||||
|
||||
function diagButtonHtml(job) {
|
||||
if (!DIAG_STATES.has(job.status)) return "";
|
||||
const open = openDiagIds.has(String(job.id));
|
||||
return `<button type="button" class="btn-link btn-diag" data-id="${job.id}">Diagnose ${open ? "▴" : "▾"}</button>`;
|
||||
}
|
||||
|
||||
async function api(path, opts = {}) {
|
||||
const r = await fetch(path, {
|
||||
credentials: "same-origin",
|
||||
@@ -294,10 +326,13 @@ function row(job) {
|
||||
const retryBtn = job.status !== "done"
|
||||
? `<button type="button" class="btn-link btn-retry" data-id="${job.id}" title="Verarbeitung wiederholen (Transkript bleibt erhalten)">↻ Erneut versuchen</button>`
|
||||
: "";
|
||||
const timing = liveTimingHtml(job);
|
||||
const statusCell = `
|
||||
<span class="status-pill status-${job.status}">${escapeHtml(job.status)}</span>
|
||||
${timing ? `<div class="status-timing">${timing}</div>` : ""}
|
||||
${job.error ? `<div class="status-error" title="${escapeHtml(job.error)}">${escapeHtml(job.error)}</div>` : ""}
|
||||
${retryBtn}
|
||||
${diagButtonHtml(job)}
|
||||
`;
|
||||
|
||||
tr.innerHTML = `
|
||||
@@ -311,6 +346,15 @@ function row(job) {
|
||||
return tr;
|
||||
}
|
||||
|
||||
// Zweite Tabellenzeile direkt unter `tr`, in die das Diagnose-Panel rutscht.
|
||||
function diagRow(job) {
|
||||
const tr = document.createElement("tr");
|
||||
tr.className = "diag-row hidden";
|
||||
tr.dataset.diagId = String(job.id);
|
||||
tr.innerHTML = `<td colspan="6"><div class="diag-content"><div class="muted">Lade Diagnose …</div></div></td>`;
|
||||
return tr;
|
||||
}
|
||||
|
||||
function card(job) {
|
||||
const div = document.createElement("div");
|
||||
div.className = "jc";
|
||||
@@ -328,6 +372,7 @@ function card(job) {
|
||||
return `<a class="btn-dl ${dis ? "disabled" : ""}" ${href} title="${escapeHtml(tip)}">${label}</a>`;
|
||||
}).join("");
|
||||
|
||||
const timing = liveTimingHtml(job);
|
||||
div.innerHTML = `
|
||||
<div class="jc-head">
|
||||
<div>
|
||||
@@ -341,8 +386,15 @@ function card(job) {
|
||||
<div class="jc-progress"><div class="bar"><div class="bar-fill" style="width:${job.progress}%"></div></div></div>
|
||||
<small class="muted">${job.progress}%</small>
|
||||
</div>
|
||||
${timing ? `<div class="jc-timing">${timing}</div>` : ""}
|
||||
${job.error ? `<div class="jc-error" title="${escapeHtml(job.error)}">${escapeHtml(job.error)}</div>` : ""}
|
||||
${job.status !== "done" ? `<button type="button" class="btn-link btn-retry" data-id="${job.id}">↻ Erneut versuchen</button>` : ""}
|
||||
<div class="jc-actions">
|
||||
${job.status !== "done" ? `<button type="button" class="btn-link btn-retry" data-id="${job.id}">↻ Erneut versuchen</button>` : ""}
|
||||
${diagButtonHtml(job)}
|
||||
</div>
|
||||
<div class="diag-panel hidden" data-diag-id="${job.id}">
|
||||
<div class="diag-content"><div class="muted">Lade Diagnose …</div></div>
|
||||
</div>
|
||||
<div class="jc-sub">aktualisiert: ${escapeHtml(fmtDate(job.updated_at))}</div>
|
||||
<div class="jc-dl">${dlButtons}</div>
|
||||
`;
|
||||
@@ -364,13 +416,83 @@ async function loadJobs() {
|
||||
}
|
||||
jobs.forEach((j) => {
|
||||
jobsBody.appendChild(row(j));
|
||||
jobsBody.appendChild(diagRow(j));
|
||||
jobsCards.appendChild(card(j));
|
||||
});
|
||||
// Vorher offene Diagnosen wieder aufklappen + frisch nachladen.
|
||||
openDiagIds.forEach((id) => {
|
||||
document.querySelectorAll(`[data-diag-id="${id}"]`).forEach((el) => el.classList.remove("hidden"));
|
||||
refreshDiag(id);
|
||||
});
|
||||
} catch (e) {
|
||||
console.error(e);
|
||||
}
|
||||
}
|
||||
|
||||
function renderDiag(diag) {
|
||||
const job = diag.job;
|
||||
const lines = (diag.mac_log || []);
|
||||
const stepInfo = diag.step_started_at
|
||||
? `Stage <strong>${escapeHtml(job.status)}</strong> seit ${fmtSecs((Date.now() - new Date(diag.step_started_at).getTime()) / 1000)}`
|
||||
: "kein Stage-Start gespeichert";
|
||||
const beatInfo = diag.last_heartbeat_at
|
||||
? `Letzter Heartbeat vor ${fmtSecs((Date.now() - new Date(diag.last_heartbeat_at).getTime()) / 1000)}`
|
||||
: "noch kein Heartbeat";
|
||||
const errBlock = job.error
|
||||
? `<div class="diag-err"><strong>Fehler:</strong> ${escapeHtml(job.error)}</div>`
|
||||
: "";
|
||||
const logBlock = lines.length
|
||||
? `<pre class="diag-log">${escapeHtml(lines.join("\n"))}</pre>`
|
||||
: `<div class="muted">Keine Mac-Worker-Logzeilen für diesen Job (Buffer leer oder Mac nicht erreichbar).</div>`;
|
||||
return `
|
||||
<div class="diag-meta">
|
||||
<div>${stepInfo}</div>
|
||||
<div>${beatInfo}</div>
|
||||
<div>Profil: <code>${escapeHtml(job.profile)}</code></div>
|
||||
</div>
|
||||
${errBlock}
|
||||
<div class="diag-loghead">Mac-Worker-Log (letzte ${lines.length} Zeilen)</div>
|
||||
${logBlock}
|
||||
`;
|
||||
}
|
||||
|
||||
async function refreshDiag(jobId) {
|
||||
let data;
|
||||
try {
|
||||
const r = await api(`/api/jobs/${jobId}/diag`);
|
||||
if (!r.ok) throw new Error(`HTTP ${r.status}`);
|
||||
data = await r.json();
|
||||
} catch (e) {
|
||||
document.querySelectorAll(`[data-diag-id="${jobId}"] .diag-content`).forEach((el) => {
|
||||
el.innerHTML = `<div class="diag-err">Diagnose konnte nicht geladen werden: ${escapeHtml(e.message)}</div>`;
|
||||
});
|
||||
return;
|
||||
}
|
||||
const html = renderDiag(data);
|
||||
document.querySelectorAll(`[data-diag-id="${jobId}"] .diag-content`).forEach((el) => {
|
||||
el.innerHTML = html;
|
||||
});
|
||||
}
|
||||
|
||||
// Diagnose-Toggle (Tabelle und Karten gemeinsam, Event-Delegation)
|
||||
document.addEventListener("click", (ev) => {
|
||||
const btn = ev.target.closest(".btn-diag");
|
||||
if (!btn) return;
|
||||
ev.preventDefault();
|
||||
const id = btn.dataset.id;
|
||||
const wasOpen = openDiagIds.has(id);
|
||||
if (wasOpen) {
|
||||
openDiagIds.delete(id);
|
||||
document.querySelectorAll(`[data-diag-id="${id}"]`).forEach((el) => el.classList.add("hidden"));
|
||||
btn.textContent = "Diagnose ▾";
|
||||
} else {
|
||||
openDiagIds.add(id);
|
||||
document.querySelectorAll(`[data-diag-id="${id}"]`).forEach((el) => el.classList.remove("hidden"));
|
||||
btn.textContent = "Diagnose ▴";
|
||||
refreshDiag(id);
|
||||
}
|
||||
});
|
||||
|
||||
form.addEventListener("submit", (ev) => {
|
||||
ev.preventDefault();
|
||||
// Quelle: Aufnahme bevorzugt, sonst File-Input
|
||||
|
||||
@@ -560,3 +560,93 @@ select {
|
||||
text-transform: lowercase;
|
||||
vertical-align: middle;
|
||||
}
|
||||
|
||||
/* Live-Timing direkt unter dem Status-Pill (läuft seit · letzter Ping vor) */
|
||||
.status-timing, .jc-timing {
|
||||
margin-top: 4px;
|
||||
font-size: 11px;
|
||||
line-height: 1.3;
|
||||
}
|
||||
.live-timing { color: var(--muted); }
|
||||
|
||||
/* Diagnose-Button – wie btn-retry, aber neutraler Ton */
|
||||
.btn-diag {
|
||||
display: inline-block;
|
||||
margin-top: 6px;
|
||||
margin-left: 4px;
|
||||
padding: 4px 10px;
|
||||
font-size: 12px;
|
||||
background: rgba(120,120,120,.10);
|
||||
border: 1px solid rgba(160,160,160,.35);
|
||||
color: var(--muted);
|
||||
border-radius: 5px;
|
||||
text-decoration: none;
|
||||
cursor: pointer;
|
||||
}
|
||||
.btn-diag:hover { background: rgba(160,160,160,.20); color: inherit; }
|
||||
|
||||
/* Aktions-Reihe in der mobilen Karte */
|
||||
.jc-actions {
|
||||
display: flex;
|
||||
flex-wrap: wrap;
|
||||
gap: 6px;
|
||||
margin-top: 4px;
|
||||
}
|
||||
|
||||
/* Diagnose-Panel: Tabelle (zweite Zeile, full-width) und Karte */
|
||||
.diag-row > td {
|
||||
background: rgba(255,255,255,.03);
|
||||
border-top: 0;
|
||||
padding: 10px 12px;
|
||||
}
|
||||
.diag-panel {
|
||||
margin-top: 10px;
|
||||
padding: 10px;
|
||||
background: rgba(255,255,255,.03);
|
||||
border: 1px solid rgba(255,255,255,.08);
|
||||
border-radius: 6px;
|
||||
}
|
||||
.diag-meta {
|
||||
display: flex;
|
||||
gap: 16px;
|
||||
flex-wrap: wrap;
|
||||
font-size: 12px;
|
||||
color: var(--muted);
|
||||
margin-bottom: 6px;
|
||||
}
|
||||
.diag-meta code {
|
||||
background: rgba(255,255,255,.08);
|
||||
padding: 1px 4px;
|
||||
border-radius: 3px;
|
||||
}
|
||||
.diag-err {
|
||||
margin: 6px 0;
|
||||
padding: 6px 8px;
|
||||
background: rgba(231,76,60,.10);
|
||||
border: 1px solid rgba(231,76,60,.35);
|
||||
border-radius: 4px;
|
||||
color: var(--err);
|
||||
font-size: 12px;
|
||||
word-break: break-word;
|
||||
}
|
||||
.diag-loghead {
|
||||
margin-top: 8px;
|
||||
font-size: 11px;
|
||||
text-transform: uppercase;
|
||||
letter-spacing: .05em;
|
||||
color: var(--muted);
|
||||
}
|
||||
.diag-log {
|
||||
margin: 4px 0 0 0;
|
||||
padding: 8px;
|
||||
background: rgba(0,0,0,.30);
|
||||
border: 1px solid rgba(255,255,255,.06);
|
||||
border-radius: 4px;
|
||||
max-height: 320px;
|
||||
overflow: auto;
|
||||
font-family: ui-monospace, "SF Mono", Menlo, Consolas, monospace;
|
||||
font-size: 11.5px;
|
||||
line-height: 1.45;
|
||||
white-space: pre-wrap;
|
||||
word-break: break-word;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user