Improve second brain live graph

fastapi_app.py

@@ -591,6 +591,183 @@ def api_graph(
 
     return {"nodes": list(nodes.values()), "edges": edges}
 
+
+def _graph_payload_from_rows(rows: list[sqlite3.Row], link_rows: list[sqlite3.Row]) -> dict:
+    nodes: dict[str, dict] = {}
+    edges: list[dict] = []
+
+    def add_node(nid: str, kind: str, label: str | None = None, weight: float | None = None):
+        if nid not in nodes:
+            nodes[nid] = {"id": nid, "kind": kind}
+        if label is not None:
+            nodes[nid]["label"] = label
+        if weight is not None:
+            nodes[nid]["weight"] = weight
+
+    def add_edge(fr: str, to: str, kind: str, weight: float):
+        if fr and to and fr != to:
+            edges.append({"from": fr, "to": to, "kind": kind, "weight": weight})
+
+    for r in rows:
+        eid = r["id"]
+        try:
+            meta = json.loads(r["metadata_json"] or "{}")
+        except Exception:
+            meta = {}
+        try:
+            corr = json.loads(r["correctness_json"] or "{}")
+        except Exception:
+            corr = {}
+
+        verdict = corr.get("verdict")
+        if not isinstance(verdict, str) or not verdict:
+            if corr.get("confirmed", False):
+                verdict = "confirmed_true"
+            elif int(corr.get("rejections", 0) or 0) > 0:
+                verdict = "confirmed_false"
+            else:
+                verdict = "unknown"
+
+        content = (r["content"] or "").strip()
+        source = str(meta.get("source", "unknown") or "unknown")
+        add_node(eid, "engram", label=(content[:54] or eid[:8]), weight=float(meta.get("access_count", 0) or 0))
+        nodes[eid].update(
+            {
+                "source": source,
+                "confidence": float(meta.get("confidence", 0.0) or 0.0),
+                "created": meta.get("created", r["created_at"]),
+                "modified": meta.get("modified", r["modified_at"]),
+                "last_accessed": meta.get("last_accessed"),
+                "verdict": verdict,
+                "confirmed": bool(corr.get("confirmed", False)),
+                "rejections": int(corr.get("rejections", 0) or 0),
+            }
+        )
+
+        sid = f"source:{source}"
+        add_node(sid, "source", label=source, weight=5)
+        add_edge(eid, sid, "from_source", 0.45)
+
+        for t in _safe_json_extract_tags(r["metadata_json"]):
+            tid = f"tag:{t}"
+            add_node(tid, "tag", label=t, weight=2)
+            add_edge(eid, tid, "has_tag", 0.35)
+
+        host = _host_from_meta(r["metadata_json"])
+        if host:
+            hid = f"host:{host}"
+            add_node(hid, "host", label=host, weight=3)
+            add_edge(eid, hid, "grounded_at", 0.25)
+
+    for lr in link_rows:
+        fr = lr["from_id"]
+        to = lr["to_id"]
+        add_node(fr, "engram", label=fr[:8])
+        add_node(to, "engram", label=to[:8])
+        add_edge(fr, to, "link", 1.0)
+
+    return {"nodes": list(nodes.values()), "edges": edges}
+
+
+@app.get("/api/graph_chunk")
+def api_graph_chunk(
+    offset: int = Query(0, ge=0),
+    limit: int = Query(600, ge=50, le=2500),
+):
+    """
+    Incremental graph payload. The dashboard can render immediately, then keep
+    adding chunks until every SQL/RAG/Obsidian-imported engram is visible.
+    """
+    conn = get_db()
+    c = conn.cursor()
+    total = c.execute("SELECT COUNT(*) FROM engrams").fetchone()[0]
+    max_modified = c.execute("SELECT MAX(modified_at) FROM engrams").fetchone()[0]
+    rows = c.execute(
+        """
+        SELECT id, content, metadata_json, correctness_json, created_at, modified_at
+        FROM engrams
+        ORDER BY created_at DESC
+        LIMIT ? OFFSET ?
+        """,
+        (limit, offset),
+    ).fetchall()
+    ids = [r["id"] for r in rows]
+    link_rows: list[sqlite3.Row] = []
+    if ids:
+        placeholders = ",".join("?" * len(ids))
+        link_rows = c.execute(
+            f"""
+            SELECT from_id, to_id FROM engrams_links
+            WHERE from_id IN ({placeholders}) OR to_id IN ({placeholders})
+            LIMIT 20000
+            """,
+            ids + ids,
+        ).fetchall()
+    conn.close()
+    payload = _graph_payload_from_rows(rows, link_rows)
+    payload.update(
+        {
+            "offset": offset,
+            "limit": limit,
+            "next_offset": offset + len(rows),
+            "done": offset + len(rows) >= total,
+            "total_engrams": total,
+            "max_modified": max_modified,
+        }
+    )
+    return payload
+
+
+@app.get("/api/graph_changes")
+def api_graph_changes(
+    since: str = Query(""),
+    limit: int = Query(300, ge=20, le=2000),
+):
+    """
+    Lightweight live deltas for the graph. Called from SSE ticks so the canvas
+    updates without reloading the whole graph.
+    """
+    conn = get_db()
+    c = conn.cursor()
+    if since:
+        rows = c.execute(
+            """
+            SELECT id, content, metadata_json, correctness_json, created_at, modified_at
+            FROM engrams
+            WHERE modified_at > ?
+            ORDER BY modified_at DESC
+            LIMIT ?
+            """,
+            (since, limit),
+        ).fetchall()
+    else:
+        rows = c.execute(
+            """
+            SELECT id, content, metadata_json, correctness_json, created_at, modified_at
+            FROM engrams
+            ORDER BY modified_at DESC
+            LIMIT ?
+            """,
+            (limit,),
+        ).fetchall()
+    ids = [r["id"] for r in rows]
+    link_rows: list[sqlite3.Row] = []
+    if ids:
+        placeholders = ",".join("?" * len(ids))
+        link_rows = c.execute(
+            f"""
+            SELECT from_id, to_id FROM engrams_links
+            WHERE from_id IN ({placeholders}) OR to_id IN ({placeholders})
+            LIMIT 10000
+            """,
+            ids + ids,
+        ).fetchall()
+    max_modified = c.execute("SELECT MAX(modified_at) FROM engrams").fetchone()[0]
+    conn.close()
+    payload = _graph_payload_from_rows(rows, link_rows)
+    payload.update({"count": len(rows), "max_modified": max_modified})
+    return payload
+
 @app.get("/api/events")
 def api_events():
     """

static/style.css

@@ -206,8 +206,28 @@ body {
 .legend-dot{ width:10px; height:10px; border-radius:50%; display:inline-block; }
 .legend-dot.engram{ background:#6c8af5; }
 .legend-dot.tag{ background:#8a9aff; }
+.legend-dot.source{ background:#14b8a6; }
 .legend-dot.match{ background:#f7d154; }
 .graph-hint{ padding: 4px 12px 10px; }
+.graph-live{
+    margin: 8px 12px 0;
+    padding: 10px 12px;
+    background:#101820;
+    border:1px solid #22303d;
+    border-radius: 10px;
+    color:#bfe8df;
+    font-size:0.78rem;
+}
+.graph-live-title{
+    color:#e8fffb;
+    font-weight:700;
+    margin-bottom:4px;
+}
+.graph-live-feed{
+    display:grid;
+    gap:3px;
+    min-height: 22px;
+}
 #searchInput {
     width: 100%;
     flex: 1;

templates/dashboard.html

@@ -76,10 +76,15 @@
             </div>
             <canvas id="graphCanvas" width="440" height="520"></canvas>
             <div class="graph-hint muted small" id="graphHint">Lade Graph…</div>
+            <div class="graph-live" id="graphLive">
+                <div class="graph-live-title">Live</div>
+                <div id="graphLiveFeed" class="graph-live-feed"></div>
+            </div>
             <div class="graph-legend">
                 <div><strong>Graph</strong>: Zoom per Pinch (2 Finger), Pan per Drag (1 Finger). Tap auf Engram öffnet Details, Tap auf Tag setzt Suche.</div>
                 <div class="legend-row"><span class="legend-dot engram"></span> Engram</div>
                 <div class="legend-row"><span class="legend-dot tag"></span> Tag</div>
+                <div class="legend-row"><span class="legend-dot source"></span> Quelle</div>
                 <div class="legend-row"><span class="legend-dot match"></span> Match (Suche)</div>
             </div>
         </div>
@@ -329,17 +334,35 @@ async function loadStatus() {
 }
 
 async function loadGraph() {
-    const sel = document.getElementById('graphLimit');
-    const fromSel = sel ? parseInt(sel.value || '0', 10) : NaN;
-    const fromStore = parseInt(localStorage.getItem('graphLimit') || '0', 10);
-    const q = (!Number.isNaN(fromSel)) ? fromSel : (Number.isNaN(fromStore) ? 0 : fromStore);
-    if (sel) sel.value = String(q);
-    if (sel) localStorage.setItem('graphLimit', String(q));
     const hint = document.getElementById('graphHint');
-    if (hint) hint.textContent = 'Lade Graph…';
+    const sel = document.getElementById('graphLimit');
+    const selected = sel ? parseInt(sel.value || '0', 10) : 0;
+    const maxEngrams = Number.isNaN(selected) ? 0 : selected;
+    if (sel) localStorage.setItem('graphLimit', String(maxEngrams));
+    graphState.loadingToken = (graphState.loadingToken || 0) + 1;
+    const token = graphState.loadingToken;
+    _graphResetData();
+    _graphResizeCanvas();
+    _graphInitInteractions();
+    if (hint) hint.textContent = 'Graph startet...';
+    _graphDraw();
     try {
-        const g = await api(`/api/graph?limit_nodes=${q}`);
+        let offset = 0;
-        renderGraph(g.nodes || [], g.edges || []);
+        const chunkSize = maxEngrams && maxEngrams <= 1000 ? 400 : 900;
+        while (token === graphState.loadingToken) {
+            const limit = maxEngrams ? Math.min(chunkSize, Math.max(0, maxEngrams - offset)) : chunkSize;
+            if (limit <= 0) break;
+            const g = await api(`/api/graph_chunk?offset=${offset}&limit=${limit}`);
+            _graphMergePayload(g, {progressive: true});
+            graphState.loadedEngrams = Math.min(g.next_offset || offset, g.total_engrams || 0);
+            graphState.totalEngrams = g.total_engrams || graphState.totalEngrams || 0;
+            graphState.lastModified = g.max_modified || graphState.lastModified;
+            _graphDraw();
+            if (g.done || (maxEngrams && graphState.loadedEngrams >= maxEngrams)) break;
+            offset = g.next_offset || (offset + limit);
+            await new Promise(resolve => setTimeout(resolve, 16));
+        }
+        fitGraphView();
     } catch (e) {
         if (hint) hint.textContent = `Graph-Fehler: ${e && e.message ? e.message : String(e)}`;
         const canvas = _graphCanvas();
@@ -350,6 +373,22 @@ async function loadGraph() {
 
 function reloadGraph() { loadGraph(); }
 
+async function loadGraphChanges() {
+    if (!graphState.lastModified) return;
+    try {
+        const g = await api(`/api/graph_changes?since=${encodeURIComponent(graphState.lastModified)}&limit=600`);
+        if ((g.nodes || []).length || (g.edges || []).length) {
+            _graphMergePayload(g, {progressive: true});
+            _graphPushLive(`Delta +${(g.nodes || []).filter(n => n.kind === 'engram').length} Einträge, +${(g.edges || []).length} Kanten`);
+            _graphDraw();
+        } else {
+            graphState.lastModified = g.max_modified || graphState.lastModified;
+        }
+    } catch (e) {
+        _graphPushLive(`Delta-Fehler: ${e && e.message ? e.message : String(e)}`);
+    }
+}
+
 // ─── Graph Renderer (Canvas) ────────────────────────────────────────────────
 let graphState = {
     nodes: [],
@@ -376,14 +415,167 @@ let graphState = {
     pinchStartPan: null,
     down: null, // {pointerId, cx, cy, t}
     physicsStrength: parseInt(localStorage.getItem('physicsStrength') || '60', 10),
+    edgeKeys: new Set(),
+    loadingToken: 0,
+    totalEngrams: 0,
+    loadedEngrams: 0,
+    lastModified: null,
+    liveFeed: [],
 };
 
 function _graphCanvas() { return document.getElementById('graphCanvas'); }
 function _graphCtx() { return _graphCanvas().getContext('2d'); }
 
+function _graphResizeCanvas() {
+    const canvas = _graphCanvas();
+    if (!canvas) return;
+    const w = canvas.parentElement.clientWidth - 24;
+    canvas.width = Math.max(320, w);
+    canvas.height = Math.max(560, Math.min(980, (window.innerHeight || 900) - 250));
+}
+
+function _graphResetData() {
+    if (graphState.raf) cancelAnimationFrame(graphState.raf);
+    graphState.nodes = [];
+    graphState.edges = [];
+    graphState.sim = [];
+    graphState.links = [];
+    graphState.nodeById = new Map();
+    graphState.simById = new Map();
+    graphState.degree = new Map();
+    graphState.edgeKeys = new Set();
+    graphState.totalEngrams = 0;
+    graphState.loadedEngrams = 0;
+    graphState.lastModified = null;
+    graphState.physicsOn = false;
+    graphState.panX = 0;
+    graphState.panY = 0;
+    graphState.zoom = 1;
+    const b = document.getElementById('btnGraphPhysics');
+    if (b) {
+        b.textContent = 'Physics: off';
+        b.classList.remove('primary');
+    }
+}
+
+function _graphHashUnit(s) {
+    let h = 2166136261;
+    const str = String(s || '');
+    for (let i = 0; i < str.length; i++) {
+        h ^= str.charCodeAt(i);
+        h = Math.imul(h, 16777619);
+    }
+    return ((h >>> 0) / 4294967295);
+}
+
+function _graphPlaceNode(n) {
+    const canvas = _graphCanvas();
+    const idx = graphState.sim.length + 1;
+    if (n.kind === 'source') {
+        const angle = _graphHashUnit(n.id) * Math.PI * 2;
+        const ring = 70 + (_graphHashUnit(n.id + ':r') * 45);
+        return {x: Math.cos(angle) * ring, y: Math.sin(angle) * ring};
+    }
+    if (n.kind === 'tag') {
+        const angle = _graphHashUnit(n.id) * Math.PI * 2;
+        const ring = 210 + (_graphHashUnit(n.id + ':r') * 170);
+        return {x: Math.cos(angle) * ring, y: Math.sin(angle) * ring};
+    }
+    if (n.kind === 'host') {
+        const angle = _graphHashUnit(n.id) * Math.PI * 2;
+        const ring = 360 + (_graphHashUnit(n.id + ':r') * 160);
+        return {x: Math.cos(angle) * ring, y: Math.sin(angle) * ring};
+    }
+
+    // Obsidian-like "brain": a golden-angle cloud with source-aware lobes.
+    const sourceSeed = _graphHashUnit(n.source || 'unknown') * Math.PI * 2;
+    const angle = idx * 2.399963 + sourceSeed * 0.45;
+    const radius = 120 + Math.sqrt(idx) * 15 + _graphHashUnit(n.id) * 70;
+    const lobe = 1 + (_graphHashUnit(n.source || 'unknown') - 0.5) * 0.26;
+    return {
+        x: Math.cos(angle) * radius * lobe,
+        y: Math.sin(angle) * radius * (2 - lobe),
+    };
+}
+
+function _graphEnsureSimNode(n) {
+    const existing = graphState.simById.get(n.id);
+    if (existing) {
+        Object.assign(existing, {
+            kind: n.kind || existing.kind,
+            label: n.label || existing.label,
+            weight: n.weight ?? existing.weight,
+            verdict: n.verdict ?? existing.verdict,
+            confidence: n.confidence ?? existing.confidence,
+            created: n.created ?? existing.created,
+            modified: n.modified ?? existing.modified,
+            last_accessed: n.last_accessed ?? existing.last_accessed,
+            source: n.source ?? existing.source,
+            predict_locked: n.predict_locked ?? existing.predict_locked,
+        });
+        graphState.nodeById.set(n.id, {...(graphState.nodeById.get(n.id) || {}), ...n});
+        return existing;
+    }
+    const p = _graphPlaceNode(n);
+    const sim = {
+        id: n.id,
+        kind: n.kind,
+        label: n.label || n.id,
+        weight: n.weight,
+        verdict: n.verdict,
+        confidence: n.confidence,
+        created: n.created,
+        modified: n.modified,
+        last_accessed: n.last_accessed,
+        source: n.source,
+        predict_locked: n.predict_locked,
+        x: p.x,
+        y: p.y,
+        vx: 0, vy: 0,
+    };
+    graphState.nodes.push(n);
+    graphState.nodeById.set(n.id, n);
+    graphState.sim.push(sim);
+    graphState.simById.set(n.id, sim);
+    return sim;
+}
+
+function _graphMergePayload(payload, opts = {}) {
+    const incomingNodes = payload.nodes || [];
+    const incomingEdges = payload.edges || [];
+    for (const n of incomingNodes) _graphEnsureSimNode(n);
+    for (const e of incomingEdges) {
+        const a = graphState.simById.get(e.from);
+        const b = graphState.simById.get(e.to);
+        if (!a || !b) continue;
+        const key = `${e.from}\u0000${e.to}\u0000${e.kind || ''}`;
+        if (graphState.edgeKeys.has(key)) continue;
+        graphState.edgeKeys.add(key);
+        graphState.edges.push(e);
+        graphState.links.push({a, b, kind: e.kind, weight: e.weight || 1.0});
+        graphState.degree.set(e.from, (graphState.degree.get(e.from) || 0) + 1);
+        graphState.degree.set(e.to, (graphState.degree.get(e.to) || 0) + 1);
+    }
+    graphState.lastModified = payload.max_modified || graphState.lastModified;
+    graphState.search = state.search || graphState.search || '';
+    if (opts.progressive && graphState.sim.length < 2500) {
+        const iters = graphState.sim.length < 900 ? 12 : 3;
+        for (let i = 0; i < iters; i++) _graphStepPhysics(0.28);
+    }
+}
+
+function _graphPushLive(text) {
+    const feed = document.getElementById('graphLiveFeed');
+    if (!feed || !text) return;
+    const ts = new Date().toLocaleTimeString('de-DE', {hour: '2-digit', minute: '2-digit', second: '2-digit'});
+    graphState.liveFeed.unshift(`${ts} ${text}`);
+    graphState.liveFeed = graphState.liveFeed.slice(0, 8);
+    feed.innerHTML = graphState.liveFeed.map(x => `<div>${escapeHtml(x)}</div>`).join('');
+}
+
 function _graphNodeRadius(n) {
     const d = graphState.degree.get(n.id) || 0;
-    const base = n.kind === 'tag' ? 4 : (n.kind === 'host' ? 5 : 7);
+    const base = n.kind === 'tag' ? 4 : (n.kind === 'host' ? 5 : (n.kind === 'source' ? 11 : 7));
     const w = (n.weight || 0);
     const bonus = Math.min(6, Math.sqrt(Math.max(0, w)) * 0.8);
     return Math.max(3, Math.min(18, base + Math.sqrt(d) + bonus));
@@ -402,6 +594,10 @@ function _graphNodeFill(n) {
         const [r,g,b] = mix([245, 158, 11]);
         return `rgb(${r},${g},${b})`;
     }
+    if (n.kind === 'source') {
+        const [r,g,b] = mix([20, 184, 166]);
+        return `rgb(${r},${g},${b})`;
+    }
 
     const verdict = (n.verdict || '').toString();
     let base = [96, 165, 250]; // pending/unknown = blue
@@ -926,7 +1122,8 @@ function _graphDraw() {
     }
 
     ctx.restore();
-    hint.textContent = `nodes=${graphState.nodes.length} edges=${graphState.edges.length}` + (term ? ` | match=${matches}` : '');
+    const loaded = graphState.totalEngrams ? ` | engrams=${graphState.loadedEngrams}/${graphState.totalEngrams}` : '';
+    hint.textContent = `nodes=${graphState.nodes.length} edges=${graphState.edges.length}${loaded}` + (term ? ` | match=${matches}` : '');
 }
 
 function _graphLoop() {
@@ -982,8 +1179,8 @@ function fitGraphView() {
     const zx = (canvas.width - 40) / w;
     const zy = (canvas.height - 40) / h;
     graphState.zoom = Math.max(0.35, Math.min(2.5, Math.min(zx, zy)));
-    graphState.panX = canvas.width / 2;
+    graphState.panX = canvas.width / 2 - ((minX + maxX) / 2) * graphState.zoom;
-    graphState.panY = canvas.height / 2;
+    graphState.panY = canvas.height / 2 - ((minY + maxY) / 2) * graphState.zoom;
     _graphDraw();
 }
 
@@ -1005,11 +1202,13 @@ function startEvents() {
                     loadStatus();
                 }
                 if (state.view === 'graph') {
-                    // fetch graph less often (every ~15s)
                     const t = Date.now();
-                    if (!state._lastGraphFetch || (t - state._lastGraphFetch) > 15000) {
+                    const stats = state.lastEvent.stats || {};
-                        state._lastGraphFetch = t;
+                    const jobs = state.lastEvent.jobs || {};
-                        loadGraph();
+                    _graphPushLive(`Stats total=${stats.total ?? '-'} pending=${stats.pending ?? '-'} jobs=${Array.isArray(jobs.units) ? jobs.units.length : '-'}`);
+                    if (!state._lastGraphDelta || (t - state._lastGraphDelta) > 5000) {
+                        state._lastGraphDelta = t;
+                        loadGraphChanges();
                     }
                 }
             } catch (e) {}