Improve second brain live graph
This commit is contained in:
177
fastapi_app.py
177
fastapi_app.py
@@ -591,6 +591,183 @@ def api_graph(
|
||||
|
||||
return {"nodes": list(nodes.values()), "edges": edges}
|
||||
|
||||
|
||||
def _graph_payload_from_rows(rows: list[sqlite3.Row], link_rows: list[sqlite3.Row]) -> dict:
|
||||
nodes: dict[str, dict] = {}
|
||||
edges: list[dict] = []
|
||||
|
||||
def add_node(nid: str, kind: str, label: str | None = None, weight: float | None = None):
|
||||
if nid not in nodes:
|
||||
nodes[nid] = {"id": nid, "kind": kind}
|
||||
if label is not None:
|
||||
nodes[nid]["label"] = label
|
||||
if weight is not None:
|
||||
nodes[nid]["weight"] = weight
|
||||
|
||||
def add_edge(fr: str, to: str, kind: str, weight: float):
|
||||
if fr and to and fr != to:
|
||||
edges.append({"from": fr, "to": to, "kind": kind, "weight": weight})
|
||||
|
||||
for r in rows:
|
||||
eid = r["id"]
|
||||
try:
|
||||
meta = json.loads(r["metadata_json"] or "{}")
|
||||
except Exception:
|
||||
meta = {}
|
||||
try:
|
||||
corr = json.loads(r["correctness_json"] or "{}")
|
||||
except Exception:
|
||||
corr = {}
|
||||
|
||||
verdict = corr.get("verdict")
|
||||
if not isinstance(verdict, str) or not verdict:
|
||||
if corr.get("confirmed", False):
|
||||
verdict = "confirmed_true"
|
||||
elif int(corr.get("rejections", 0) or 0) > 0:
|
||||
verdict = "confirmed_false"
|
||||
else:
|
||||
verdict = "unknown"
|
||||
|
||||
content = (r["content"] or "").strip()
|
||||
source = str(meta.get("source", "unknown") or "unknown")
|
||||
add_node(eid, "engram", label=(content[:54] or eid[:8]), weight=float(meta.get("access_count", 0) or 0))
|
||||
nodes[eid].update(
|
||||
{
|
||||
"source": source,
|
||||
"confidence": float(meta.get("confidence", 0.0) or 0.0),
|
||||
"created": meta.get("created", r["created_at"]),
|
||||
"modified": meta.get("modified", r["modified_at"]),
|
||||
"last_accessed": meta.get("last_accessed"),
|
||||
"verdict": verdict,
|
||||
"confirmed": bool(corr.get("confirmed", False)),
|
||||
"rejections": int(corr.get("rejections", 0) or 0),
|
||||
}
|
||||
)
|
||||
|
||||
sid = f"source:{source}"
|
||||
add_node(sid, "source", label=source, weight=5)
|
||||
add_edge(eid, sid, "from_source", 0.45)
|
||||
|
||||
for t in _safe_json_extract_tags(r["metadata_json"]):
|
||||
tid = f"tag:{t}"
|
||||
add_node(tid, "tag", label=t, weight=2)
|
||||
add_edge(eid, tid, "has_tag", 0.35)
|
||||
|
||||
host = _host_from_meta(r["metadata_json"])
|
||||
if host:
|
||||
hid = f"host:{host}"
|
||||
add_node(hid, "host", label=host, weight=3)
|
||||
add_edge(eid, hid, "grounded_at", 0.25)
|
||||
|
||||
for lr in link_rows:
|
||||
fr = lr["from_id"]
|
||||
to = lr["to_id"]
|
||||
add_node(fr, "engram", label=fr[:8])
|
||||
add_node(to, "engram", label=to[:8])
|
||||
add_edge(fr, to, "link", 1.0)
|
||||
|
||||
return {"nodes": list(nodes.values()), "edges": edges}
|
||||
|
||||
|
||||
@app.get("/api/graph_chunk")
|
||||
def api_graph_chunk(
|
||||
offset: int = Query(0, ge=0),
|
||||
limit: int = Query(600, ge=50, le=2500),
|
||||
):
|
||||
"""
|
||||
Incremental graph payload. The dashboard can render immediately, then keep
|
||||
adding chunks until every SQL/RAG/Obsidian-imported engram is visible.
|
||||
"""
|
||||
conn = get_db()
|
||||
c = conn.cursor()
|
||||
total = c.execute("SELECT COUNT(*) FROM engrams").fetchone()[0]
|
||||
max_modified = c.execute("SELECT MAX(modified_at) FROM engrams").fetchone()[0]
|
||||
rows = c.execute(
|
||||
"""
|
||||
SELECT id, content, metadata_json, correctness_json, created_at, modified_at
|
||||
FROM engrams
|
||||
ORDER BY created_at DESC
|
||||
LIMIT ? OFFSET ?
|
||||
""",
|
||||
(limit, offset),
|
||||
).fetchall()
|
||||
ids = [r["id"] for r in rows]
|
||||
link_rows: list[sqlite3.Row] = []
|
||||
if ids:
|
||||
placeholders = ",".join("?" * len(ids))
|
||||
link_rows = c.execute(
|
||||
f"""
|
||||
SELECT from_id, to_id FROM engrams_links
|
||||
WHERE from_id IN ({placeholders}) OR to_id IN ({placeholders})
|
||||
LIMIT 20000
|
||||
""",
|
||||
ids + ids,
|
||||
).fetchall()
|
||||
conn.close()
|
||||
payload = _graph_payload_from_rows(rows, link_rows)
|
||||
payload.update(
|
||||
{
|
||||
"offset": offset,
|
||||
"limit": limit,
|
||||
"next_offset": offset + len(rows),
|
||||
"done": offset + len(rows) >= total,
|
||||
"total_engrams": total,
|
||||
"max_modified": max_modified,
|
||||
}
|
||||
)
|
||||
return payload
|
||||
|
||||
|
||||
@app.get("/api/graph_changes")
|
||||
def api_graph_changes(
|
||||
since: str = Query(""),
|
||||
limit: int = Query(300, ge=20, le=2000),
|
||||
):
|
||||
"""
|
||||
Lightweight live deltas for the graph. Called from SSE ticks so the canvas
|
||||
updates without reloading the whole graph.
|
||||
"""
|
||||
conn = get_db()
|
||||
c = conn.cursor()
|
||||
if since:
|
||||
rows = c.execute(
|
||||
"""
|
||||
SELECT id, content, metadata_json, correctness_json, created_at, modified_at
|
||||
FROM engrams
|
||||
WHERE modified_at > ?
|
||||
ORDER BY modified_at DESC
|
||||
LIMIT ?
|
||||
""",
|
||||
(since, limit),
|
||||
).fetchall()
|
||||
else:
|
||||
rows = c.execute(
|
||||
"""
|
||||
SELECT id, content, metadata_json, correctness_json, created_at, modified_at
|
||||
FROM engrams
|
||||
ORDER BY modified_at DESC
|
||||
LIMIT ?
|
||||
""",
|
||||
(limit,),
|
||||
).fetchall()
|
||||
ids = [r["id"] for r in rows]
|
||||
link_rows: list[sqlite3.Row] = []
|
||||
if ids:
|
||||
placeholders = ",".join("?" * len(ids))
|
||||
link_rows = c.execute(
|
||||
f"""
|
||||
SELECT from_id, to_id FROM engrams_links
|
||||
WHERE from_id IN ({placeholders}) OR to_id IN ({placeholders})
|
||||
LIMIT 10000
|
||||
""",
|
||||
ids + ids,
|
||||
).fetchall()
|
||||
max_modified = c.execute("SELECT MAX(modified_at) FROM engrams").fetchone()[0]
|
||||
conn.close()
|
||||
payload = _graph_payload_from_rows(rows, link_rows)
|
||||
payload.update({"count": len(rows), "max_modified": max_modified})
|
||||
return payload
|
||||
|
||||
@app.get("/api/events")
|
||||
def api_events():
|
||||
"""
|
||||
|
||||
Reference in New Issue
Block a user