feat(dashboard): realtime status + graph render

fastapi_app.py

@@ -17,7 +17,7 @@ from pathlib import Path
 from urllib.parse import urlparse
 
 from fastapi import FastAPI, Form, Query, Request
-from fastapi.responses import HTMLResponse, JSONResponse, PlainTextResponse
+from fastapi.responses import HTMLResponse, JSONResponse, PlainTextResponse, StreamingResponse
 from fastapi.staticfiles import StaticFiles
 
 # ─── Config ──────────────────────────────────────────────────────────────────
@@ -121,6 +121,19 @@ def _systemd_unit_state(unit: str) -> dict:
     except Exception as e:
         return {"unit": unit, "error": str(e)}
 
+def _dir_size_bytes(path: Path) -> int:
+    total = 0
+    try:
+        for p in path.rglob("*"):
+            try:
+                if p.is_file():
+                    total += p.stat().st_size
+            except Exception:
+                pass
+    except Exception:
+        pass
+    return total
+
 # ─── API Endpoints ───────────────────────────────────────────────────────────
 
 @app.get("/healthz", response_class=PlainTextResponse)
@@ -146,6 +159,69 @@ def api_db_info():
         "mtime": datetime.fromtimestamp(st.st_mtime, tz=timezone.utc).isoformat(),
     }
 
+@app.get("/api/storage_stats")
+def api_storage_stats():
+    conn = get_db()
+    c = conn.cursor()
+    total = c.execute("SELECT COUNT(*) FROM engrams").fetchone()[0]
+    confirmed = c.execute(
+        "SELECT COUNT(*) FROM engrams WHERE json_extract(correctness_json, '$.confirmed') = 1"
+    ).fetchone()[0]
+    sources = {
+        r[0]: r[1]
+        for r in c.execute(
+            "SELECT json_extract(metadata_json, '$.source') AS src, COUNT(*) FROM engrams GROUP BY src ORDER BY COUNT(*) DESC"
+        ).fetchall()
+        if r[0] is not None
+    }
+    conn.close()
+
+    chroma_dir = WORKSPACE / "data" / "chroma"
+    emb_cache_dir = WORKSPACE / "data" / "embedding_cache"
+    vec_state_path = WORKSPACE / "data" / "vector_index_state.json"
+
+    vec_state = {}
+    if vec_state_path.exists():
+        try:
+            vec_state = json.loads(vec_state_path.read_text())
+        except Exception:
+            vec_state = {}
+
+    obsidian_cfg_path = WORKSPACE / "data" / "obsidian_config.json"
+    obsidian_cfg = None
+    if obsidian_cfg_path.exists():
+        try:
+            obsidian_cfg = json.loads(obsidian_cfg_path.read_text())
+        except Exception:
+            obsidian_cfg = {"raw": obsidian_cfg_path.read_text()[:2000]}
+
+    backup_files = sorted((WORKSPACE / "data").glob("backup_*.jsonl"))
+
+    return {
+        "sql": {
+            "total_engrams": total,
+            "confirmed": confirmed,
+            "pending": total - confirmed,
+            "by_source": sources,
+        },
+        "vector": {
+            "chroma_dir": str(chroma_dir),
+            "chroma_size_bytes": _dir_size_bytes(chroma_dir) if chroma_dir.exists() else 0,
+            "embedding_cache_dir": str(emb_cache_dir),
+            "embedding_cache_files": len(list(emb_cache_dir.glob("*.json"))) if emb_cache_dir.exists() else 0,
+            "vector_state": vec_state,
+        },
+        "obsidian": {
+            "config_path": str(obsidian_cfg_path),
+            "configured": bool(obsidian_cfg),
+            "config": obsidian_cfg,
+        },
+        "backups": {
+            "count": len(backup_files),
+            "latest": str(backup_files[-1]) if backup_files else None,
+        },
+    }
+
 @app.get("/api/jobs")
 def api_jobs():
     # Known units that influence "freshness" of the brain.
@@ -243,21 +319,25 @@ def api_graph(limit_nodes: int = Query(200, ge=50, le=1000)):
     nodes: dict[str, dict] = {}
     edges: list[dict] = []
 
-    def add_node(nid: str, kind: str):
+    def add_node(nid: str, kind: str, label: str | None = None, weight: float | None = None):
         if nid not in nodes:
             nodes[nid] = {"id": nid, "kind": kind}
+        if label is not None:
+            nodes[nid]["label"] = label
+        if weight is not None:
+            nodes[nid]["weight"] = weight
 
     for r in rows:
         eid = r["id"]
-        add_node(eid, "engram")
+        add_node(eid, "engram", label=eid[:8])
         for t in _safe_json_extract_tags(r["metadata_json"]):
             tid = f"tag:{t}"
-            add_node(tid, "tag")
+            add_node(tid, "tag", label=t)
             edges.append({"from": eid, "to": tid, "kind": "has_tag"})
         host = _host_from_meta(r["metadata_json"])
         if host:
             hid = f"host:{host}"
-            add_node(hid, "host")
+            add_node(hid, "host", label=host)
             edges.append({"from": eid, "to": hid, "kind": "grounded_at"})
 
     for fr, to in link_rows:
@@ -297,6 +377,27 @@ def api_graph(limit_nodes: int = Query(200, ge=50, le=1000)):
 
     return {"nodes": list(nodes.values()), "edges": edges}
 
+@app.get("/api/events")
+def api_events():
+    """
+    Server-Sent Events stream for lightweight real-time UI refresh.
+    """
+    import time
+
+    def gen():
+        while True:
+            payload = {
+                "ts": datetime.now(timezone.utc).isoformat(),
+                "stats": api_stats(),
+                "storage": api_storage_stats(),
+                "jobs": api_jobs(),
+                "insights": api_insights(limit=8),
+            }
+            yield f"data: {json.dumps(payload, ensure_ascii=False)}\n\n"
+            time.sleep(5)
+
+    return StreamingResponse(gen(), media_type="text/event-stream")
+
 
 @app.exception_handler(FileNotFoundError)
 def handle_file_not_found(request: Request, exc: FileNotFoundError):

static/style.css

@@ -27,6 +27,31 @@ body {
     top: 0;
     z-index: 50;
 }
+
+.tabs-bar{
+    display:flex;
+    gap:8px;
+    padding:8px 12px 10px;
+    background:#141419;
+    border-bottom:1px solid #252530;
+    position: sticky;
+    top: 52px;
+    z-index: 45;
+}
+.tabs-bar .tab-btn{
+    flex:1;
+    background:#1e1e28;
+    border:1px solid #2a2a3a;
+    border-radius: 12px;
+    padding:10px 10px;
+    color:#cfd3ff;
+    font-weight:700;
+    font-size:0.82rem;
+}
+.tabs-bar .tab-btn.active{
+    border-color:#6c8af5;
+    box-shadow:0 0 0 1px rgba(108,138,245,0.22) inset;
+}
 .stat {
     text-align: center;
     min-width: 60px;
@@ -55,26 +80,7 @@ body {
     background: #141419;
 }
 
-/* ─── View Tabs ───────────────────────────────────────────────────────────── */
+/* tab buttons styled via .tabs-bar */
-.view-tabs {
-    display: flex;
-    gap: 8px;
-    padding: 0 12px 10px;
-}
-.tab-btn {
-    flex: 1;
-    background: #1e1e28;
-    border: 1px solid #2a2a3a;
-    border-radius: 10px;
-    padding: 9px 10px;
-    color: #cfd3ff;
-    font-weight: 600;
-    font-size: 0.85rem;
-}
-.tab-btn.active {
-    border-color: #6c8af5;
-    box-shadow: 0 0 0 1px rgba(108,138,245,0.25) inset;
-}
 
 /* ─── Panels (Graph/Status) ──────────────────────────────────────────────── */
 .panel {
@@ -123,6 +129,16 @@ body {
     font-size: 0.8rem;
     margin-top: 6px;
 }
+.small { font-size: 0.75rem; }
+
+/* Graph canvas */
+#graphCanvas{
+    display:block;
+    margin: 8px auto 0;
+    background:#12121a;
+    border:1px solid #252533;
+    border-radius: 14px;
+}
 #searchInput {
     flex: 1;
     background: #1e1e28;

templates/dashboard.html

@@ -16,6 +16,12 @@
             <div class="stat"><span class="stat-num" id="statErrors">-</span><span class="stat-label">Err</span></div>
         </header>
 
+        <div class="tabs-bar">
+            <button class="tab-btn active" id="tabCards" onclick="setView('cards')">Cards</button>
+            <button class="tab-btn" id="tabGraph" onclick="setView('graph')">Graph</button>
+            <button class="tab-btn" id="tabStatus" onclick="setView('status')">Status</button>
+        </div>
+
         <!-- Search -->
         <div class="search-box">
             <input type="text" id="searchInput" placeholder="🔍 Suche..." />
@@ -27,13 +33,6 @@
             </select>
         </div>
 
-        <!-- Views -->
-        <div class="view-tabs">
-            <button class="tab-btn active" id="tabCards" onclick="setView('cards')">Cards</button>
-            <button class="tab-btn" id="tabGraph" onclick="setView('graph')">Graph</button>
-            <button class="tab-btn" id="tabStatus" onclick="setView('status')">Status</button>
-        </div>
-
         <!-- New Engram -->
         <div class="new-engram">
             <textarea id="newContent" placeholder="Neues Engramm..."></textarea>
@@ -45,7 +44,10 @@
         <div class="cards" id="cards"></div>
 
         <!-- Graph -->
-        <div class="graph" id="graph" style="display:none;"></div>
+        <div class="graph" id="graph" style="display:none;">
+            <canvas id="graphCanvas" width="440" height="520"></canvas>
+            <div class="muted small" id="graphHint">Lade Graph…</div>
+        </div>
 
         <!-- Status -->
         <div class="status" id="status" style="display:none;"></div>
@@ -81,6 +83,7 @@ let state = {
     search: '',
     autoRefresh: true,
     view: 'cards',
+    lastEvent: null,
 };
 
 // ─── Fetch ──────────────────────────────────────────────────────────────────
@@ -98,6 +101,15 @@ async function loadStats() {
     document.getElementById('statErrors').textContent = s.errors;
 }
 
+function updateStatsFromEvent(ev) {
+    if (!ev || !ev.stats) return;
+    const s = ev.stats;
+    document.getElementById('statTotal').textContent = s.total;
+    document.getElementById('statConfirmed').textContent = s.confirmed;
+    document.getElementById('statPending').textContent = s.pending;
+    document.getElementById('statErrors').textContent = s.errors;
+}
+
 function setView(view) {
     state.view = view;
     document.getElementById('tabCards').classList.toggle('active', view === 'cards');
@@ -129,11 +141,12 @@ async function loadCards() {
 }
 
 async function loadStatus() {
-    const [cfg, db, jobs, ins] = await Promise.all([
+    const [cfg, db, jobs, ins, stor] = await Promise.all([
         api('/api/config'),
         api('/api/db_info'),
         api('/api/jobs'),
         api('/api/insights?limit=8'),
+        api('/api/storage_stats'),
     ]);
 
     const el = document.getElementById('status');
@@ -146,6 +159,10 @@ async function loadStatus() {
 
     const topTags = (ins.top_tags || []).map(t => `<span class="pill">${t.key} (${t.count})</span>`).join(' ');
     const topHosts = (ins.top_hosts || []).map(t => `<span class="pill">${t.key} (${t.count})</span>`).join(' ');
+    const bySource = Object.entries((stor.sql && stor.sql.by_source) ? stor.sql.by_source : {})
+        .slice(0, 8)
+        .map(([k,v]) => `<span class="pill">${k}: ${v}</span>`)
+        .join(' ');
 
     el.innerHTML = `
         <div class="panel">
@@ -154,6 +171,13 @@ async function loadStatus() {
             <div class="kv-row"><div class="kv-key">db</div><div class="kv-val">${db.db_path}</div></div>
             <div class="kv-row"><div class="kv-key">db mtime</div><div class="kv-val">${new Date(db.mtime).toLocaleString()}</div></div>
         </div>
+        <div class="panel">
+            <div class="panel-title">Storage</div>
+            <div class="kv-row"><div class="kv-key">SQL</div><div class="kv-val">${stor.sql.total_engrams} engrams (ok ${stor.sql.confirmed}, pending ${stor.sql.pending})</div></div>
+            <div class="kv-row"><div class="kv-key">Vector</div><div class="kv-val">chroma ${(stor.vector.chroma_size_bytes/1024/1024).toFixed(1)} MB, cache ${stor.vector.embedding_cache_files} files</div></div>
+            <div class="kv-row"><div class="kv-key">Obsidian</div><div class="kv-val">${stor.obsidian.configured ? 'configured' : 'not configured'}</div></div>
+            <div class="kv-row"><div class="kv-key">By source</div><div class="kv-val">${bySource || '-'}</div></div>
+        </div>
         <div class="panel">
             <div class="panel-title">Jobs</div>
             ${jobsHtml || '<div class="muted">Keine Daten</div>'}
@@ -168,30 +192,135 @@ async function loadStatus() {
 }
 
 async function loadGraph() {
-    const g = await api('/api/graph?limit_nodes=250');
+    const g = await api('/api/graph?limit_nodes=200');
-    const nodes = g.nodes || [];
+    renderGraph(g.nodes || [], g.edges || []);
-    const edges = g.edges || [];
-
-    const countByKind = nodes.reduce((acc, n) => {
-        acc[n.kind] = (acc[n.kind] || 0) + 1;
-        return acc;
-    }, {});
-
-    const el = document.getElementById('graph');
-    el.innerHTML = `
-        <div class="panel">
-            <div class="panel-title">Graph (lightweight)</div>
-            <div class="kv-row"><div class="kv-key">nodes</div><div class="kv-val">${nodes.length} (${Object.entries(countByKind).map(([k,v])=>k+':'+v).join(', ')})</div></div>
-            <div class="kv-row"><div class="kv-key">edges</div><div class="kv-val">${edges.length}</div></div>
-            <div class="muted">Aktuell: Tag/Host Knoten + Engram-Links. Nächster Schritt: echte Tool↔API↔Endpoint Ontologie.</div>
-        </div>
-        <div class="panel">
-            <div class="panel-title">Beispiele</div>
-            <div class="muted">Tip: öffne ein Engram-Detail und folge Links.</div>
-        </div>
-    `;
 }
 
+function renderGraph(nodes, edges) {
+    const canvas = document.getElementById('graphCanvas');
+    const hint = document.getElementById('graphHint');
+    const ctx = canvas.getContext('2d');
+
+    // Fit canvas to container width (mobile)
+    const w = canvas.parentElement.clientWidth - 24;
+    canvas.width = Math.max(320, Math.min(520, w));
+    canvas.height = 520;
+
+    if (!nodes.length || !edges.length) {
+        hint.textContent = 'Graph: keine Kanten (noch keine Links/Tags/Hosts im Sample).';
+        ctx.clearRect(0,0,canvas.width,canvas.height);
+        return;
+    }
+
+    hint.textContent = `nodes=${nodes.length} edges=${edges.length}`;
+
+    const nodeById = new Map(nodes.map(n => [n.id, n]));
+    const sim = nodes.map(n => ({
+        id: n.id,
+        kind: n.kind,
+        label: n.label || n.id,
+        x: Math.random()*canvas.width,
+        y: Math.random()*canvas.height,
+        vx: 0, vy: 0,
+    }));
+    const simById = new Map(sim.map(n => [n.id, n]));
+
+    const links = edges
+        .map(e => ({a: simById.get(e.from), b: simById.get(e.to), kind: e.kind}))
+        .filter(l => l.a && l.b);
+
+    // Simple force layout (few iterations)
+    for (let iter=0; iter<180; iter++) {
+        // repulsion
+        for (let i=0; i<sim.length; i++) {
+            for (let j=i+1; j<sim.length; j++) {
+                const a = sim[i], b = sim[j];
+                const dx = a.x - b.x, dy = a.y - b.y;
+                const d2 = dx*dx + dy*dy + 0.01;
+                const f = 120 / d2;
+                a.vx += dx*f; a.vy += dy*f;
+                b.vx -= dx*f; b.vy -= dy*f;
+            }
+        }
+        // springs
+        for (const l of links) {
+            const a = l.a, b = l.b;
+            const dx = b.x - a.x, dy = b.y - a.y;
+            const dist = Math.sqrt(dx*dx + dy*dy) || 1;
+            const target = 60;
+            const k = 0.02;
+            const f = (dist - target) * k;
+            const fx = (dx/dist)*f, fy = (dy/dist)*f;
+            a.vx += fx; a.vy += fy;
+            b.vx -= fx; b.vy -= fy;
+        }
+        // integrate + bounds
+        for (const n of sim) {
+            n.vx *= 0.85; n.vy *= 0.85;
+            n.x += n.vx; n.y += n.vy;
+            n.x = Math.max(10, Math.min(canvas.width-10, n.x));
+            n.y = Math.max(10, Math.min(canvas.height-10, n.y));
+        }
+    }
+
+    ctx.clearRect(0,0,canvas.width,canvas.height);
+    // edges
+    ctx.globalAlpha = 0.5;
+    ctx.strokeStyle = '#3a3a55';
+    ctx.lineWidth = 1;
+    for (const l of links) {
+        ctx.beginPath();
+        ctx.moveTo(l.a.x, l.a.y);
+        ctx.lineTo(l.b.x, l.b.y);
+        ctx.stroke();
+    }
+    ctx.globalAlpha = 1.0;
+
+    // nodes
+    for (const n of sim) {
+        let r = 5;
+        let fill = '#6c8af5';
+        if (n.kind === 'tag') { fill = '#8a9aff'; r = 4; }
+        if (n.kind === 'host') { fill = '#f5b46c'; r = 4; }
+        if (n.kind === 'engram') { fill = '#6c8af5'; r = 5; }
+
+        ctx.beginPath();
+        ctx.fillStyle = fill;
+        ctx.arc(n.x, n.y, r, 0, Math.PI*2);
+        ctx.fill();
+    }
+}
+
+// Real-time updates via SSE
+function startEvents() {
+    try {
+        const es = new EventSource('/api/events');
+        es.onmessage = (msg) => {
+            try {
+                state.lastEvent = JSON.parse(msg.data);
+                updateStatsFromEvent(state.lastEvent);
+                if (state.view === 'status') {
+                    // refresh status panels without heavy re-render: just rerun loadStatus occasionally
+                    loadStatus();
+                }
+                if (state.view === 'graph') {
+                    // fetch graph less often (every ~15s)
+                    const t = Date.now();
+                    if (!state._lastGraphFetch || (t - state._lastGraphFetch) > 15000) {
+                        state._lastGraphFetch = t;
+                        loadGraph();
+                    }
+                }
+            } catch (e) {}
+        };
+        es.onerror = () => {
+            // keep UI usable even if SSE drops
+        };
+    } catch (e) {}
+}
+
+startEvents();
+
 function renderCards() {
     const el = document.getElementById('cards');
     el.innerHTML = state.items.map(item => `