diff --git a/fastapi_app.py b/fastapi_app.py index 67e0282..2d0c37c 100644 --- a/fastapi_app.py +++ b/fastapi_app.py @@ -591,6 +591,183 @@ def api_graph( return {"nodes": list(nodes.values()), "edges": edges} + +def _graph_payload_from_rows(rows: list[sqlite3.Row], link_rows: list[sqlite3.Row]) -> dict: + nodes: dict[str, dict] = {} + edges: list[dict] = [] + + def add_node(nid: str, kind: str, label: str | None = None, weight: float | None = None): + if nid not in nodes: + nodes[nid] = {"id": nid, "kind": kind} + if label is not None: + nodes[nid]["label"] = label + if weight is not None: + nodes[nid]["weight"] = weight + + def add_edge(fr: str, to: str, kind: str, weight: float): + if fr and to and fr != to: + edges.append({"from": fr, "to": to, "kind": kind, "weight": weight}) + + for r in rows: + eid = r["id"] + try: + meta = json.loads(r["metadata_json"] or "{}") + except Exception: + meta = {} + try: + corr = json.loads(r["correctness_json"] or "{}") + except Exception: + corr = {} + + verdict = corr.get("verdict") + if not isinstance(verdict, str) or not verdict: + if corr.get("confirmed", False): + verdict = "confirmed_true" + elif int(corr.get("rejections", 0) or 0) > 0: + verdict = "confirmed_false" + else: + verdict = "unknown" + + content = (r["content"] or "").strip() + source = str(meta.get("source", "unknown") or "unknown") + add_node(eid, "engram", label=(content[:54] or eid[:8]), weight=float(meta.get("access_count", 0) or 0)) + nodes[eid].update( + { + "source": source, + "confidence": float(meta.get("confidence", 0.0) or 0.0), + "created": meta.get("created", r["created_at"]), + "modified": meta.get("modified", r["modified_at"]), + "last_accessed": meta.get("last_accessed"), + "verdict": verdict, + "confirmed": bool(corr.get("confirmed", False)), + "rejections": int(corr.get("rejections", 0) or 0), + } + ) + + sid = f"source:{source}" + add_node(sid, "source", label=source, weight=5) + add_edge(eid, sid, "from_source", 0.45) + + for t in _safe_json_extract_tags(r["metadata_json"]): + tid = f"tag:{t}" + add_node(tid, "tag", label=t, weight=2) + add_edge(eid, tid, "has_tag", 0.35) + + host = _host_from_meta(r["metadata_json"]) + if host: + hid = f"host:{host}" + add_node(hid, "host", label=host, weight=3) + add_edge(eid, hid, "grounded_at", 0.25) + + for lr in link_rows: + fr = lr["from_id"] + to = lr["to_id"] + add_node(fr, "engram", label=fr[:8]) + add_node(to, "engram", label=to[:8]) + add_edge(fr, to, "link", 1.0) + + return {"nodes": list(nodes.values()), "edges": edges} + + +@app.get("/api/graph_chunk") +def api_graph_chunk( + offset: int = Query(0, ge=0), + limit: int = Query(600, ge=50, le=2500), +): + """ + Incremental graph payload. The dashboard can render immediately, then keep + adding chunks until every SQL/RAG/Obsidian-imported engram is visible. + """ + conn = get_db() + c = conn.cursor() + total = c.execute("SELECT COUNT(*) FROM engrams").fetchone()[0] + max_modified = c.execute("SELECT MAX(modified_at) FROM engrams").fetchone()[0] + rows = c.execute( + """ + SELECT id, content, metadata_json, correctness_json, created_at, modified_at + FROM engrams + ORDER BY created_at DESC + LIMIT ? OFFSET ? + """, + (limit, offset), + ).fetchall() + ids = [r["id"] for r in rows] + link_rows: list[sqlite3.Row] = [] + if ids: + placeholders = ",".join("?" * len(ids)) + link_rows = c.execute( + f""" + SELECT from_id, to_id FROM engrams_links + WHERE from_id IN ({placeholders}) OR to_id IN ({placeholders}) + LIMIT 20000 + """, + ids + ids, + ).fetchall() + conn.close() + payload = _graph_payload_from_rows(rows, link_rows) + payload.update( + { + "offset": offset, + "limit": limit, + "next_offset": offset + len(rows), + "done": offset + len(rows) >= total, + "total_engrams": total, + "max_modified": max_modified, + } + ) + return payload + + +@app.get("/api/graph_changes") +def api_graph_changes( + since: str = Query(""), + limit: int = Query(300, ge=20, le=2000), +): + """ + Lightweight live deltas for the graph. Called from SSE ticks so the canvas + updates without reloading the whole graph. + """ + conn = get_db() + c = conn.cursor() + if since: + rows = c.execute( + """ + SELECT id, content, metadata_json, correctness_json, created_at, modified_at + FROM engrams + WHERE modified_at > ? + ORDER BY modified_at DESC + LIMIT ? + """, + (since, limit), + ).fetchall() + else: + rows = c.execute( + """ + SELECT id, content, metadata_json, correctness_json, created_at, modified_at + FROM engrams + ORDER BY modified_at DESC + LIMIT ? + """, + (limit,), + ).fetchall() + ids = [r["id"] for r in rows] + link_rows: list[sqlite3.Row] = [] + if ids: + placeholders = ",".join("?" * len(ids)) + link_rows = c.execute( + f""" + SELECT from_id, to_id FROM engrams_links + WHERE from_id IN ({placeholders}) OR to_id IN ({placeholders}) + LIMIT 10000 + """, + ids + ids, + ).fetchall() + max_modified = c.execute("SELECT MAX(modified_at) FROM engrams").fetchone()[0] + conn.close() + payload = _graph_payload_from_rows(rows, link_rows) + payload.update({"count": len(rows), "max_modified": max_modified}) + return payload + @app.get("/api/events") def api_events(): """ diff --git a/static/style.css b/static/style.css index 8ae430d..07c60b2 100644 --- a/static/style.css +++ b/static/style.css @@ -206,8 +206,28 @@ body { .legend-dot{ width:10px; height:10px; border-radius:50%; display:inline-block; } .legend-dot.engram{ background:#6c8af5; } .legend-dot.tag{ background:#8a9aff; } +.legend-dot.source{ background:#14b8a6; } .legend-dot.match{ background:#f7d154; } .graph-hint{ padding: 4px 12px 10px; } +.graph-live{ + margin: 8px 12px 0; + padding: 10px 12px; + background:#101820; + border:1px solid #22303d; + border-radius: 10px; + color:#bfe8df; + font-size:0.78rem; +} +.graph-live-title{ + color:#e8fffb; + font-weight:700; + margin-bottom:4px; +} +.graph-live-feed{ + display:grid; + gap:3px; + min-height: 22px; +} #searchInput { width: 100%; flex: 1; diff --git a/templates/dashboard.html b/templates/dashboard.html index 221466c..0a90705 100644 --- a/templates/dashboard.html +++ b/templates/dashboard.html @@ -76,10 +76,15 @@
Lade Graph…
+
+
Live
+
+
Graph: Zoom per Pinch (2 Finger), Pan per Drag (1 Finger). Tap auf Engram öffnet Details, Tap auf Tag setzt Suche.
Engram
Tag
+
Quelle
Match (Suche)
@@ -329,17 +334,35 @@ async function loadStatus() { } async function loadGraph() { - const sel = document.getElementById('graphLimit'); - const fromSel = sel ? parseInt(sel.value || '0', 10) : NaN; - const fromStore = parseInt(localStorage.getItem('graphLimit') || '0', 10); - const q = (!Number.isNaN(fromSel)) ? fromSel : (Number.isNaN(fromStore) ? 0 : fromStore); - if (sel) sel.value = String(q); - if (sel) localStorage.setItem('graphLimit', String(q)); const hint = document.getElementById('graphHint'); - if (hint) hint.textContent = 'Lade Graph…'; + const sel = document.getElementById('graphLimit'); + const selected = sel ? parseInt(sel.value || '0', 10) : 0; + const maxEngrams = Number.isNaN(selected) ? 0 : selected; + if (sel) localStorage.setItem('graphLimit', String(maxEngrams)); + graphState.loadingToken = (graphState.loadingToken || 0) + 1; + const token = graphState.loadingToken; + _graphResetData(); + _graphResizeCanvas(); + _graphInitInteractions(); + if (hint) hint.textContent = 'Graph startet...'; + _graphDraw(); try { - const g = await api(`/api/graph?limit_nodes=${q}`); - renderGraph(g.nodes || [], g.edges || []); + let offset = 0; + const chunkSize = maxEngrams && maxEngrams <= 1000 ? 400 : 900; + while (token === graphState.loadingToken) { + const limit = maxEngrams ? Math.min(chunkSize, Math.max(0, maxEngrams - offset)) : chunkSize; + if (limit <= 0) break; + const g = await api(`/api/graph_chunk?offset=${offset}&limit=${limit}`); + _graphMergePayload(g, {progressive: true}); + graphState.loadedEngrams = Math.min(g.next_offset || offset, g.total_engrams || 0); + graphState.totalEngrams = g.total_engrams || graphState.totalEngrams || 0; + graphState.lastModified = g.max_modified || graphState.lastModified; + _graphDraw(); + if (g.done || (maxEngrams && graphState.loadedEngrams >= maxEngrams)) break; + offset = g.next_offset || (offset + limit); + await new Promise(resolve => setTimeout(resolve, 16)); + } + fitGraphView(); } catch (e) { if (hint) hint.textContent = `Graph-Fehler: ${e && e.message ? e.message : String(e)}`; const canvas = _graphCanvas(); @@ -350,6 +373,22 @@ async function loadGraph() { function reloadGraph() { loadGraph(); } +async function loadGraphChanges() { + if (!graphState.lastModified) return; + try { + const g = await api(`/api/graph_changes?since=${encodeURIComponent(graphState.lastModified)}&limit=600`); + if ((g.nodes || []).length || (g.edges || []).length) { + _graphMergePayload(g, {progressive: true}); + _graphPushLive(`Delta +${(g.nodes || []).filter(n => n.kind === 'engram').length} Einträge, +${(g.edges || []).length} Kanten`); + _graphDraw(); + } else { + graphState.lastModified = g.max_modified || graphState.lastModified; + } + } catch (e) { + _graphPushLive(`Delta-Fehler: ${e && e.message ? e.message : String(e)}`); + } +} + // ─── Graph Renderer (Canvas) ──────────────────────────────────────────────── let graphState = { nodes: [], @@ -376,14 +415,167 @@ let graphState = { pinchStartPan: null, down: null, // {pointerId, cx, cy, t} physicsStrength: parseInt(localStorage.getItem('physicsStrength') || '60', 10), + edgeKeys: new Set(), + loadingToken: 0, + totalEngrams: 0, + loadedEngrams: 0, + lastModified: null, + liveFeed: [], }; function _graphCanvas() { return document.getElementById('graphCanvas'); } function _graphCtx() { return _graphCanvas().getContext('2d'); } +function _graphResizeCanvas() { + const canvas = _graphCanvas(); + if (!canvas) return; + const w = canvas.parentElement.clientWidth - 24; + canvas.width = Math.max(320, w); + canvas.height = Math.max(560, Math.min(980, (window.innerHeight || 900) - 250)); +} + +function _graphResetData() { + if (graphState.raf) cancelAnimationFrame(graphState.raf); + graphState.nodes = []; + graphState.edges = []; + graphState.sim = []; + graphState.links = []; + graphState.nodeById = new Map(); + graphState.simById = new Map(); + graphState.degree = new Map(); + graphState.edgeKeys = new Set(); + graphState.totalEngrams = 0; + graphState.loadedEngrams = 0; + graphState.lastModified = null; + graphState.physicsOn = false; + graphState.panX = 0; + graphState.panY = 0; + graphState.zoom = 1; + const b = document.getElementById('btnGraphPhysics'); + if (b) { + b.textContent = 'Physics: off'; + b.classList.remove('primary'); + } +} + +function _graphHashUnit(s) { + let h = 2166136261; + const str = String(s || ''); + for (let i = 0; i < str.length; i++) { + h ^= str.charCodeAt(i); + h = Math.imul(h, 16777619); + } + return ((h >>> 0) / 4294967295); +} + +function _graphPlaceNode(n) { + const canvas = _graphCanvas(); + const idx = graphState.sim.length + 1; + if (n.kind === 'source') { + const angle = _graphHashUnit(n.id) * Math.PI * 2; + const ring = 70 + (_graphHashUnit(n.id + ':r') * 45); + return {x: Math.cos(angle) * ring, y: Math.sin(angle) * ring}; + } + if (n.kind === 'tag') { + const angle = _graphHashUnit(n.id) * Math.PI * 2; + const ring = 210 + (_graphHashUnit(n.id + ':r') * 170); + return {x: Math.cos(angle) * ring, y: Math.sin(angle) * ring}; + } + if (n.kind === 'host') { + const angle = _graphHashUnit(n.id) * Math.PI * 2; + const ring = 360 + (_graphHashUnit(n.id + ':r') * 160); + return {x: Math.cos(angle) * ring, y: Math.sin(angle) * ring}; + } + + // Obsidian-like "brain": a golden-angle cloud with source-aware lobes. + const sourceSeed = _graphHashUnit(n.source || 'unknown') * Math.PI * 2; + const angle = idx * 2.399963 + sourceSeed * 0.45; + const radius = 120 + Math.sqrt(idx) * 15 + _graphHashUnit(n.id) * 70; + const lobe = 1 + (_graphHashUnit(n.source || 'unknown') - 0.5) * 0.26; + return { + x: Math.cos(angle) * radius * lobe, + y: Math.sin(angle) * radius * (2 - lobe), + }; +} + +function _graphEnsureSimNode(n) { + const existing = graphState.simById.get(n.id); + if (existing) { + Object.assign(existing, { + kind: n.kind || existing.kind, + label: n.label || existing.label, + weight: n.weight ?? existing.weight, + verdict: n.verdict ?? existing.verdict, + confidence: n.confidence ?? existing.confidence, + created: n.created ?? existing.created, + modified: n.modified ?? existing.modified, + last_accessed: n.last_accessed ?? existing.last_accessed, + source: n.source ?? existing.source, + predict_locked: n.predict_locked ?? existing.predict_locked, + }); + graphState.nodeById.set(n.id, {...(graphState.nodeById.get(n.id) || {}), ...n}); + return existing; + } + const p = _graphPlaceNode(n); + const sim = { + id: n.id, + kind: n.kind, + label: n.label || n.id, + weight: n.weight, + verdict: n.verdict, + confidence: n.confidence, + created: n.created, + modified: n.modified, + last_accessed: n.last_accessed, + source: n.source, + predict_locked: n.predict_locked, + x: p.x, + y: p.y, + vx: 0, vy: 0, + }; + graphState.nodes.push(n); + graphState.nodeById.set(n.id, n); + graphState.sim.push(sim); + graphState.simById.set(n.id, sim); + return sim; +} + +function _graphMergePayload(payload, opts = {}) { + const incomingNodes = payload.nodes || []; + const incomingEdges = payload.edges || []; + for (const n of incomingNodes) _graphEnsureSimNode(n); + for (const e of incomingEdges) { + const a = graphState.simById.get(e.from); + const b = graphState.simById.get(e.to); + if (!a || !b) continue; + const key = `${e.from}\u0000${e.to}\u0000${e.kind || ''}`; + if (graphState.edgeKeys.has(key)) continue; + graphState.edgeKeys.add(key); + graphState.edges.push(e); + graphState.links.push({a, b, kind: e.kind, weight: e.weight || 1.0}); + graphState.degree.set(e.from, (graphState.degree.get(e.from) || 0) + 1); + graphState.degree.set(e.to, (graphState.degree.get(e.to) || 0) + 1); + } + graphState.lastModified = payload.max_modified || graphState.lastModified; + graphState.search = state.search || graphState.search || ''; + if (opts.progressive && graphState.sim.length < 2500) { + const iters = graphState.sim.length < 900 ? 12 : 3; + for (let i = 0; i < iters; i++) _graphStepPhysics(0.28); + } +} + +function _graphPushLive(text) { + const feed = document.getElementById('graphLiveFeed'); + if (!feed || !text) return; + const ts = new Date().toLocaleTimeString('de-DE', {hour: '2-digit', minute: '2-digit', second: '2-digit'}); + graphState.liveFeed.unshift(`${ts} ${text}`); + graphState.liveFeed = graphState.liveFeed.slice(0, 8); + feed.innerHTML = graphState.liveFeed.map(x => `
${escapeHtml(x)}
`).join(''); +} + function _graphNodeRadius(n) { const d = graphState.degree.get(n.id) || 0; - const base = n.kind === 'tag' ? 4 : (n.kind === 'host' ? 5 : 7); + const base = n.kind === 'tag' ? 4 : (n.kind === 'host' ? 5 : (n.kind === 'source' ? 11 : 7)); const w = (n.weight || 0); const bonus = Math.min(6, Math.sqrt(Math.max(0, w)) * 0.8); return Math.max(3, Math.min(18, base + Math.sqrt(d) + bonus)); @@ -402,6 +594,10 @@ function _graphNodeFill(n) { const [r,g,b] = mix([245, 158, 11]); return `rgb(${r},${g},${b})`; } + if (n.kind === 'source') { + const [r,g,b] = mix([20, 184, 166]); + return `rgb(${r},${g},${b})`; + } const verdict = (n.verdict || '').toString(); let base = [96, 165, 250]; // pending/unknown = blue @@ -926,7 +1122,8 @@ function _graphDraw() { } ctx.restore(); - hint.textContent = `nodes=${graphState.nodes.length} edges=${graphState.edges.length}` + (term ? ` | match=${matches}` : ''); + const loaded = graphState.totalEngrams ? ` | engrams=${graphState.loadedEngrams}/${graphState.totalEngrams}` : ''; + hint.textContent = `nodes=${graphState.nodes.length} edges=${graphState.edges.length}${loaded}` + (term ? ` | match=${matches}` : ''); } function _graphLoop() { @@ -982,8 +1179,8 @@ function fitGraphView() { const zx = (canvas.width - 40) / w; const zy = (canvas.height - 40) / h; graphState.zoom = Math.max(0.35, Math.min(2.5, Math.min(zx, zy))); - graphState.panX = canvas.width / 2; - graphState.panY = canvas.height / 2; + graphState.panX = canvas.width / 2 - ((minX + maxX) / 2) * graphState.zoom; + graphState.panY = canvas.height / 2 - ((minY + maxY) / 2) * graphState.zoom; _graphDraw(); } @@ -1005,11 +1202,13 @@ function startEvents() { loadStatus(); } if (state.view === 'graph') { - // fetch graph less often (every ~15s) const t = Date.now(); - if (!state._lastGraphFetch || (t - state._lastGraphFetch) > 15000) { - state._lastGraphFetch = t; - loadGraph(); + const stats = state.lastEvent.stats || {}; + const jobs = state.lastEvent.jobs || {}; + _graphPushLive(`Stats total=${stats.total ?? '-'} pending=${stats.pending ?? '-'} jobs=${Array.isArray(jobs.units) ? jobs.units.length : '-'}`); + if (!state._lastGraphDelta || (t - state._lastGraphDelta) > 5000) { + state._lastGraphDelta = t; + loadGraphChanges(); } } } catch (e) {}