feat(dashboard): realtime status + graph render

fastapi_app.py

@@ -17,7 +17,7 @@ from pathlib import Path
 from urllib.parse import urlparse
 
 from fastapi import FastAPI, Form, Query, Request
-from fastapi.responses import HTMLResponse, JSONResponse, PlainTextResponse
+from fastapi.responses import HTMLResponse, JSONResponse, PlainTextResponse, StreamingResponse
 from fastapi.staticfiles import StaticFiles
 
 # ─── Config ──────────────────────────────────────────────────────────────────
@@ -121,6 +121,19 @@ def _systemd_unit_state(unit: str) -> dict:
     except Exception as e:
         return {"unit": unit, "error": str(e)}
 
+def _dir_size_bytes(path: Path) -> int:
+    total = 0
+    try:
+        for p in path.rglob("*"):
+            try:
+                if p.is_file():
+                    total += p.stat().st_size
+            except Exception:
+                pass
+    except Exception:
+        pass
+    return total
+
 # ─── API Endpoints ───────────────────────────────────────────────────────────
 
 @app.get("/healthz", response_class=PlainTextResponse)
@@ -146,6 +159,69 @@ def api_db_info():
         "mtime": datetime.fromtimestamp(st.st_mtime, tz=timezone.utc).isoformat(),
     }
 
+@app.get("/api/storage_stats")
+def api_storage_stats():
+    conn = get_db()
+    c = conn.cursor()
+    total = c.execute("SELECT COUNT(*) FROM engrams").fetchone()[0]
+    confirmed = c.execute(
+        "SELECT COUNT(*) FROM engrams WHERE json_extract(correctness_json, '$.confirmed') = 1"
+    ).fetchone()[0]
+    sources = {
+        r[0]: r[1]
+        for r in c.execute(
+            "SELECT json_extract(metadata_json, '$.source') AS src, COUNT(*) FROM engrams GROUP BY src ORDER BY COUNT(*) DESC"
+        ).fetchall()
+        if r[0] is not None
+    }
+    conn.close()
+
+    chroma_dir = WORKSPACE / "data" / "chroma"
+    emb_cache_dir = WORKSPACE / "data" / "embedding_cache"
+    vec_state_path = WORKSPACE / "data" / "vector_index_state.json"
+
+    vec_state = {}
+    if vec_state_path.exists():
+        try:
+            vec_state = json.loads(vec_state_path.read_text())
+        except Exception:
+            vec_state = {}
+
+    obsidian_cfg_path = WORKSPACE / "data" / "obsidian_config.json"
+    obsidian_cfg = None
+    if obsidian_cfg_path.exists():
+        try:
+            obsidian_cfg = json.loads(obsidian_cfg_path.read_text())
+        except Exception:
+            obsidian_cfg = {"raw": obsidian_cfg_path.read_text()[:2000]}
+
+    backup_files = sorted((WORKSPACE / "data").glob("backup_*.jsonl"))
+
+    return {
+        "sql": {
+            "total_engrams": total,
+            "confirmed": confirmed,
+            "pending": total - confirmed,
+            "by_source": sources,
+        },
+        "vector": {
+            "chroma_dir": str(chroma_dir),
+            "chroma_size_bytes": _dir_size_bytes(chroma_dir) if chroma_dir.exists() else 0,
+            "embedding_cache_dir": str(emb_cache_dir),
+            "embedding_cache_files": len(list(emb_cache_dir.glob("*.json"))) if emb_cache_dir.exists() else 0,
+            "vector_state": vec_state,
+        },
+        "obsidian": {
+            "config_path": str(obsidian_cfg_path),
+            "configured": bool(obsidian_cfg),
+            "config": obsidian_cfg,
+        },
+        "backups": {
+            "count": len(backup_files),
+            "latest": str(backup_files[-1]) if backup_files else None,
+        },
+    }
+
 @app.get("/api/jobs")
 def api_jobs():
     # Known units that influence "freshness" of the brain.
@@ -243,21 +319,25 @@ def api_graph(limit_nodes: int = Query(200, ge=50, le=1000)):
     nodes: dict[str, dict] = {}
     edges: list[dict] = []
 
-    def add_node(nid: str, kind: str):
+    def add_node(nid: str, kind: str, label: str | None = None, weight: float | None = None):
         if nid not in nodes:
             nodes[nid] = {"id": nid, "kind": kind}
+        if label is not None:
+            nodes[nid]["label"] = label
+        if weight is not None:
+            nodes[nid]["weight"] = weight
 
     for r in rows:
         eid = r["id"]
-        add_node(eid, "engram")
+        add_node(eid, "engram", label=eid[:8])
         for t in _safe_json_extract_tags(r["metadata_json"]):
             tid = f"tag:{t}"
-            add_node(tid, "tag")
+            add_node(tid, "tag", label=t)
             edges.append({"from": eid, "to": tid, "kind": "has_tag"})
         host = _host_from_meta(r["metadata_json"])
         if host:
             hid = f"host:{host}"
-            add_node(hid, "host")
+            add_node(hid, "host", label=host)
             edges.append({"from": eid, "to": hid, "kind": "grounded_at"})
 
     for fr, to in link_rows:
@@ -297,6 +377,27 @@ def api_graph(limit_nodes: int = Query(200, ge=50, le=1000)):
 
     return {"nodes": list(nodes.values()), "edges": edges}
 
+@app.get("/api/events")
+def api_events():
+    """
+    Server-Sent Events stream for lightweight real-time UI refresh.
+    """
+    import time
+
+    def gen():
+        while True:
+            payload = {
+                "ts": datetime.now(timezone.utc).isoformat(),
+                "stats": api_stats(),
+                "storage": api_storage_stats(),
+                "jobs": api_jobs(),
+                "insights": api_insights(limit=8),
+            }
+            yield f"data: {json.dumps(payload, ensure_ascii=False)}\n\n"
+            time.sleep(5)
+
+    return StreamingResponse(gen(), media_type="text/event-stream")
+
 
 @app.exception_handler(FileNotFoundError)
 def handle_file_not_found(request: Request, exc: FileNotFoundError):