diff --git a/fastapi_app.py b/fastapi_app.py index a4e11e4..86a2357 100644 --- a/fastapi_app.py +++ b/fastapi_app.py @@ -17,7 +17,7 @@ from pathlib import Path from urllib.parse import urlparse from fastapi import FastAPI, Form, Query, Request -from fastapi.responses import HTMLResponse, JSONResponse, PlainTextResponse +from fastapi.responses import HTMLResponse, JSONResponse, PlainTextResponse, StreamingResponse from fastapi.staticfiles import StaticFiles # ─── Config ────────────────────────────────────────────────────────────────── @@ -121,6 +121,19 @@ def _systemd_unit_state(unit: str) -> dict: except Exception as e: return {"unit": unit, "error": str(e)} +def _dir_size_bytes(path: Path) -> int: + total = 0 + try: + for p in path.rglob("*"): + try: + if p.is_file(): + total += p.stat().st_size + except Exception: + pass + except Exception: + pass + return total + # ─── API Endpoints ─────────────────────────────────────────────────────────── @app.get("/healthz", response_class=PlainTextResponse) @@ -146,6 +159,69 @@ def api_db_info(): "mtime": datetime.fromtimestamp(st.st_mtime, tz=timezone.utc).isoformat(), } +@app.get("/api/storage_stats") +def api_storage_stats(): + conn = get_db() + c = conn.cursor() + total = c.execute("SELECT COUNT(*) FROM engrams").fetchone()[0] + confirmed = c.execute( + "SELECT COUNT(*) FROM engrams WHERE json_extract(correctness_json, '$.confirmed') = 1" + ).fetchone()[0] + sources = { + r[0]: r[1] + for r in c.execute( + "SELECT json_extract(metadata_json, '$.source') AS src, COUNT(*) FROM engrams GROUP BY src ORDER BY COUNT(*) DESC" + ).fetchall() + if r[0] is not None + } + conn.close() + + chroma_dir = WORKSPACE / "data" / "chroma" + emb_cache_dir = WORKSPACE / "data" / "embedding_cache" + vec_state_path = WORKSPACE / "data" / "vector_index_state.json" + + vec_state = {} + if vec_state_path.exists(): + try: + vec_state = json.loads(vec_state_path.read_text()) + except Exception: + vec_state = {} + + obsidian_cfg_path = WORKSPACE / "data" / "obsidian_config.json" + obsidian_cfg = None + if obsidian_cfg_path.exists(): + try: + obsidian_cfg = json.loads(obsidian_cfg_path.read_text()) + except Exception: + obsidian_cfg = {"raw": obsidian_cfg_path.read_text()[:2000]} + + backup_files = sorted((WORKSPACE / "data").glob("backup_*.jsonl")) + + return { + "sql": { + "total_engrams": total, + "confirmed": confirmed, + "pending": total - confirmed, + "by_source": sources, + }, + "vector": { + "chroma_dir": str(chroma_dir), + "chroma_size_bytes": _dir_size_bytes(chroma_dir) if chroma_dir.exists() else 0, + "embedding_cache_dir": str(emb_cache_dir), + "embedding_cache_files": len(list(emb_cache_dir.glob("*.json"))) if emb_cache_dir.exists() else 0, + "vector_state": vec_state, + }, + "obsidian": { + "config_path": str(obsidian_cfg_path), + "configured": bool(obsidian_cfg), + "config": obsidian_cfg, + }, + "backups": { + "count": len(backup_files), + "latest": str(backup_files[-1]) if backup_files else None, + }, + } + @app.get("/api/jobs") def api_jobs(): # Known units that influence "freshness" of the brain. @@ -243,21 +319,25 @@ def api_graph(limit_nodes: int = Query(200, ge=50, le=1000)): nodes: dict[str, dict] = {} edges: list[dict] = [] - def add_node(nid: str, kind: str): + def add_node(nid: str, kind: str, label: str | None = None, weight: float | None = None): if nid not in nodes: nodes[nid] = {"id": nid, "kind": kind} + if label is not None: + nodes[nid]["label"] = label + if weight is not None: + nodes[nid]["weight"] = weight for r in rows: eid = r["id"] - add_node(eid, "engram") + add_node(eid, "engram", label=eid[:8]) for t in _safe_json_extract_tags(r["metadata_json"]): tid = f"tag:{t}" - add_node(tid, "tag") + add_node(tid, "tag", label=t) edges.append({"from": eid, "to": tid, "kind": "has_tag"}) host = _host_from_meta(r["metadata_json"]) if host: hid = f"host:{host}" - add_node(hid, "host") + add_node(hid, "host", label=host) edges.append({"from": eid, "to": hid, "kind": "grounded_at"}) for fr, to in link_rows: @@ -297,6 +377,27 @@ def api_graph(limit_nodes: int = Query(200, ge=50, le=1000)): return {"nodes": list(nodes.values()), "edges": edges} +@app.get("/api/events") +def api_events(): + """ + Server-Sent Events stream for lightweight real-time UI refresh. + """ + import time + + def gen(): + while True: + payload = { + "ts": datetime.now(timezone.utc).isoformat(), + "stats": api_stats(), + "storage": api_storage_stats(), + "jobs": api_jobs(), + "insights": api_insights(limit=8), + } + yield f"data: {json.dumps(payload, ensure_ascii=False)}\n\n" + time.sleep(5) + + return StreamingResponse(gen(), media_type="text/event-stream") + @app.exception_handler(FileNotFoundError) def handle_file_not_found(request: Request, exc: FileNotFoundError): diff --git a/static/style.css b/static/style.css index 92790f4..6a3e8f3 100644 --- a/static/style.css +++ b/static/style.css @@ -27,6 +27,31 @@ body { top: 0; z-index: 50; } + +.tabs-bar{ + display:flex; + gap:8px; + padding:8px 12px 10px; + background:#141419; + border-bottom:1px solid #252530; + position: sticky; + top: 52px; + z-index: 45; +} +.tabs-bar .tab-btn{ + flex:1; + background:#1e1e28; + border:1px solid #2a2a3a; + border-radius: 12px; + padding:10px 10px; + color:#cfd3ff; + font-weight:700; + font-size:0.82rem; +} +.tabs-bar .tab-btn.active{ + border-color:#6c8af5; + box-shadow:0 0 0 1px rgba(108,138,245,0.22) inset; +} .stat { text-align: center; min-width: 60px; @@ -55,26 +80,7 @@ body { background: #141419; } -/* ─── View Tabs ───────────────────────────────────────────────────────────── */ -.view-tabs { - display: flex; - gap: 8px; - padding: 0 12px 10px; -} -.tab-btn { - flex: 1; - background: #1e1e28; - border: 1px solid #2a2a3a; - border-radius: 10px; - padding: 9px 10px; - color: #cfd3ff; - font-weight: 600; - font-size: 0.85rem; -} -.tab-btn.active { - border-color: #6c8af5; - box-shadow: 0 0 0 1px rgba(108,138,245,0.25) inset; -} +/* tab buttons styled via .tabs-bar */ /* ─── Panels (Graph/Status) ──────────────────────────────────────────────── */ .panel { @@ -123,6 +129,16 @@ body { font-size: 0.8rem; margin-top: 6px; } +.small { font-size: 0.75rem; } + +/* Graph canvas */ +#graphCanvas{ + display:block; + margin: 8px auto 0; + background:#12121a; + border:1px solid #252533; + border-radius: 14px; +} #searchInput { flex: 1; background: #1e1e28; diff --git a/templates/dashboard.html b/templates/dashboard.html index deb178d..4ebea86 100644 --- a/templates/dashboard.html +++ b/templates/dashboard.html @@ -16,6 +16,12 @@
-Err
+
+ + + +
+ - -
- - - -
-
@@ -45,7 +44,10 @@
- + @@ -81,6 +83,7 @@ let state = { search: '', autoRefresh: true, view: 'cards', + lastEvent: null, }; // ─── Fetch ────────────────────────────────────────────────────────────────── @@ -98,6 +101,15 @@ async function loadStats() { document.getElementById('statErrors').textContent = s.errors; } +function updateStatsFromEvent(ev) { + if (!ev || !ev.stats) return; + const s = ev.stats; + document.getElementById('statTotal').textContent = s.total; + document.getElementById('statConfirmed').textContent = s.confirmed; + document.getElementById('statPending').textContent = s.pending; + document.getElementById('statErrors').textContent = s.errors; +} + function setView(view) { state.view = view; document.getElementById('tabCards').classList.toggle('active', view === 'cards'); @@ -129,11 +141,12 @@ async function loadCards() { } async function loadStatus() { - const [cfg, db, jobs, ins] = await Promise.all([ + const [cfg, db, jobs, ins, stor] = await Promise.all([ api('/api/config'), api('/api/db_info'), api('/api/jobs'), api('/api/insights?limit=8'), + api('/api/storage_stats'), ]); const el = document.getElementById('status'); @@ -146,6 +159,10 @@ async function loadStatus() { const topTags = (ins.top_tags || []).map(t => `${t.key} (${t.count})`).join(' '); const topHosts = (ins.top_hosts || []).map(t => `${t.key} (${t.count})`).join(' '); + const bySource = Object.entries((stor.sql && stor.sql.by_source) ? stor.sql.by_source : {}) + .slice(0, 8) + .map(([k,v]) => `${k}: ${v}`) + .join(' '); el.innerHTML = `
@@ -154,6 +171,13 @@ async function loadStatus() {
db
${db.db_path}
db mtime
${new Date(db.mtime).toLocaleString()}
+
+
Storage
+
SQL
${stor.sql.total_engrams} engrams (ok ${stor.sql.confirmed}, pending ${stor.sql.pending})
+
Vector
chroma ${(stor.vector.chroma_size_bytes/1024/1024).toFixed(1)} MB, cache ${stor.vector.embedding_cache_files} files
+
Obsidian
${stor.obsidian.configured ? 'configured' : 'not configured'}
+
By source
${bySource || '-'}
+
Jobs
${jobsHtml || '
Keine Daten
'} @@ -168,30 +192,135 @@ async function loadStatus() { } async function loadGraph() { - const g = await api('/api/graph?limit_nodes=250'); - const nodes = g.nodes || []; - const edges = g.edges || []; - - const countByKind = nodes.reduce((acc, n) => { - acc[n.kind] = (acc[n.kind] || 0) + 1; - return acc; - }, {}); - - const el = document.getElementById('graph'); - el.innerHTML = ` -
-
Graph (lightweight)
-
nodes
${nodes.length} (${Object.entries(countByKind).map(([k,v])=>k+':'+v).join(', ')})
-
edges
${edges.length}
-
Aktuell: Tag/Host Knoten + Engram-Links. Nächster Schritt: echte Tool↔API↔Endpoint Ontologie.
-
-
-
Beispiele
-
Tip: öffne ein Engram-Detail und folge Links.
-
- `; + const g = await api('/api/graph?limit_nodes=200'); + renderGraph(g.nodes || [], g.edges || []); } +function renderGraph(nodes, edges) { + const canvas = document.getElementById('graphCanvas'); + const hint = document.getElementById('graphHint'); + const ctx = canvas.getContext('2d'); + + // Fit canvas to container width (mobile) + const w = canvas.parentElement.clientWidth - 24; + canvas.width = Math.max(320, Math.min(520, w)); + canvas.height = 520; + + if (!nodes.length || !edges.length) { + hint.textContent = 'Graph: keine Kanten (noch keine Links/Tags/Hosts im Sample).'; + ctx.clearRect(0,0,canvas.width,canvas.height); + return; + } + + hint.textContent = `nodes=${nodes.length} edges=${edges.length}`; + + const nodeById = new Map(nodes.map(n => [n.id, n])); + const sim = nodes.map(n => ({ + id: n.id, + kind: n.kind, + label: n.label || n.id, + x: Math.random()*canvas.width, + y: Math.random()*canvas.height, + vx: 0, vy: 0, + })); + const simById = new Map(sim.map(n => [n.id, n])); + + const links = edges + .map(e => ({a: simById.get(e.from), b: simById.get(e.to), kind: e.kind})) + .filter(l => l.a && l.b); + + // Simple force layout (few iterations) + for (let iter=0; iter<180; iter++) { + // repulsion + for (let i=0; i { + try { + state.lastEvent = JSON.parse(msg.data); + updateStatsFromEvent(state.lastEvent); + if (state.view === 'status') { + // refresh status panels without heavy re-render: just rerun loadStatus occasionally + loadStatus(); + } + if (state.view === 'graph') { + // fetch graph less often (every ~15s) + const t = Date.now(); + if (!state._lastGraphFetch || (t - state._lastGraphFetch) > 15000) { + state._lastGraphFetch = t; + loadGraph(); + } + } + } catch (e) {} + }; + es.onerror = () => { + // keep UI usable even if SSE drops + }; + } catch (e) {} +} + +startEvents(); + function renderCards() { const el = document.getElementById('cards'); el.innerHTML = state.items.map(item => `