From e5061b317fc443308d3382bf2a5a57c9d6771dea Mon Sep 17 00:00:00 2001 From: Otto Date: Wed, 27 May 2026 00:11:44 +0200 Subject: [PATCH] feat(dashboard): add status+graph views --- fastapi_app.py | 214 +++++++++++++++++++++++++++++++++++++++ static/style.css | 69 +++++++++++++ templates/dashboard.html | 93 +++++++++++++++++ 3 files changed, 376 insertions(+) diff --git a/fastapi_app.py b/fastapi_app.py index 79ed4da..a4e11e4 100644 --- a/fastapi_app.py +++ b/fastapi_app.py @@ -11,8 +11,10 @@ Goals: import json import os import sqlite3 +import subprocess from datetime import datetime, timezone from pathlib import Path +from urllib.parse import urlparse from fastapi import FastAPI, Form, Query, Request from fastapi.responses import HTMLResponse, JSONResponse, PlainTextResponse @@ -68,6 +70,56 @@ def parse_engram(row: sqlite3.Row) -> dict: "grounding": meta.get("grounding", 0), } +def _safe_json_extract_tags(meta_json: str) -> list[str]: + try: + d = json.loads(meta_json or "{}") + tags = d.get("tags") or [] + return [t for t in tags if isinstance(t, str)] + except Exception: + return [] + +def _host_from_meta(meta_json: str) -> str | None: + try: + d = json.loads(meta_json or "{}") + grounding = d.get("grounding") + url = d.get("url") + if isinstance(grounding, dict) and isinstance(grounding.get("url"), str): + url = grounding.get("url") + if not isinstance(url, str): + return None + parsed = urlparse(url) + return parsed.hostname + except Exception: + return None + +def _systemd_unit_state(unit: str) -> dict: + """ + Best-effort systemd status snapshot for a known unit. + Never raises; returns minimal fields. + """ + try: + out = subprocess.check_output( + ["systemctl", "show", unit, "--no-page", "--property=ActiveState,SubState,Result,ExecMainStatus,ExecMainCode,ExecMainStartTimestamp,ExecMainExitTimestamp"], + text=True, + stderr=subprocess.STDOUT, + timeout=2, + ) + kv = {} + for line in out.splitlines(): + if "=" in line: + k, v = line.split("=", 1) + kv[k] = v + return { + "unit": unit, + "active": kv.get("ActiveState"), + "sub": kv.get("SubState"), + "result": kv.get("Result"), + "exit_status": kv.get("ExecMainStatus"), + "start_ts": kv.get("ExecMainStartTimestamp"), + "exit_ts": kv.get("ExecMainExitTimestamp"), + } + except Exception as e: + return {"unit": unit, "error": str(e)} # ─── API Endpoints ─────────────────────────────────────────────────────────── @@ -83,6 +135,168 @@ def api_config(): "db_path": str(DB_PATH), } +@app.get("/api/db_info") +def api_db_info(): + if not DB_PATH.exists(): + raise FileNotFoundError(f"DB not found: {DB_PATH}") + st = DB_PATH.stat() + return { + "db_path": str(DB_PATH), + "size_bytes": st.st_size, + "mtime": datetime.fromtimestamp(st.st_mtime, tz=timezone.utc).isoformat(), + } + +@app.get("/api/jobs") +def api_jobs(): + # Known units that influence "freshness" of the brain. + units = [ + "openclaw-secondbrain-ingest-memory.service", + "openclaw-secondbrain-index-vectors.service", + "openclaw-secondbrain-review.service", + "openclaw-secondbrain-heartbeat.service", + "openclaw-secondbrain-verify-pending.service", + ] + return {"items": [_systemd_unit_state(u) for u in units]} + +@app.get("/api/insights") +def api_insights(limit: int = Query(8, ge=1, le=50)): + conn = get_db() + c = conn.cursor() + rows = c.execute( + "SELECT id, metadata_json, correctness_json, created_at, modified_at FROM engrams ORDER BY created_at DESC LIMIT 2000" + ).fetchall() + total = c.execute("SELECT COUNT(*) FROM engrams").fetchone()[0] + confirmed = c.execute( + "SELECT COUNT(*) FROM engrams WHERE json_extract(correctness_json, '$.confirmed') = 1" + ).fetchone()[0] + pending = total - confirmed + + tag_counts: dict[str, int] = {} + source_counts: dict[str, int] = {} + host_counts: dict[str, int] = {} + active: list[dict] = [] + forgotten: list[dict] = [] + + for r in rows: + meta = json.loads(r["metadata_json"] or "{}") + src = meta.get("source", "unknown") + source_counts[src] = source_counts.get(src, 0) + 1 + for t in (meta.get("tags") or []): + if isinstance(t, str): + tag_counts[t] = tag_counts.get(t, 0) + 1 + host = _host_from_meta(r["metadata_json"]) + if host: + host_counts[host] = host_counts.get(host, 0) + 1 + + access_count = int(meta.get("access_count", 0) or 0) + created = meta.get("created", r["created_at"]) + if access_count >= 5 and len(active) < limit: + active.append( + { + "id": r["id"], + "access_count": access_count, + "source": src, + "created": created, + } + ) + if access_count == 0 and len(forgotten) < limit: + forgotten.append( + { + "id": r["id"], + "access_count": access_count, + "source": src, + "created": created, + } + ) + + def top_k(d: dict[str, int]) -> list[dict]: + return [ + {"key": k, "count": v} + for k, v in sorted(d.items(), key=lambda kv: kv[1], reverse=True)[:limit] + ] + + conn.close() + return { + "total": total, + "confirmed": confirmed, + "pending": pending, + "top_tags": top_k(tag_counts), + "top_sources": top_k(source_counts), + "top_hosts": top_k(host_counts), + "active_engrams": active, + "forgotten_engrams": forgotten, + } + +@app.get("/api/graph") +def api_graph(limit_nodes: int = Query(200, ge=50, le=1000)): + """ + Returns a lightweight graph view: + - Nodes: engrams + tag: + host: + - Edges: engram->tag and engram->host plus explicit engrams_links edges. + """ + conn = get_db() + c = conn.cursor() + rows = c.execute("SELECT id, metadata_json FROM engrams ORDER BY created_at DESC LIMIT 1000").fetchall() + link_rows = c.execute("SELECT from_id, to_id FROM engrams_links ORDER BY rowid DESC LIMIT 2000").fetchall() + conn.close() + + nodes: dict[str, dict] = {} + edges: list[dict] = [] + + def add_node(nid: str, kind: str): + if nid not in nodes: + nodes[nid] = {"id": nid, "kind": kind} + + for r in rows: + eid = r["id"] + add_node(eid, "engram") + for t in _safe_json_extract_tags(r["metadata_json"]): + tid = f"tag:{t}" + add_node(tid, "tag") + edges.append({"from": eid, "to": tid, "kind": "has_tag"}) + host = _host_from_meta(r["metadata_json"]) + if host: + hid = f"host:{host}" + add_node(hid, "host") + edges.append({"from": eid, "to": hid, "kind": "grounded_at"}) + + for fr, to in link_rows: + add_node(fr, "engram") + add_node(to, "engram") + edges.append({"from": fr, "to": to, "kind": "link"}) + + # Trim nodes to keep payload bounded (prefer engrams and connected tags/hosts) + if len(nodes) > limit_nodes: + # Keep a balanced subset: many engrams plus the most-connected non-engrams. + kept: dict[str, dict] = {} + engram_budget = int(limit_nodes * 0.7) + + # 1) Keep newest engrams first (they appear first in `rows` loop insertion order) + for r in rows: + eid = r["id"] + if eid in nodes: + kept[eid] = nodes[eid] + if len(kept) >= engram_budget: + break + + # 2) Rank remaining nodes by degree within current edge set + degree: dict[str, int] = {} + for e in edges: + degree[e["from"]] = degree.get(e["from"], 0) + 1 + degree[e["to"]] = degree.get(e["to"], 0) + 1 + + remaining = [nid for nid in nodes.keys() if nid not in kept] + remaining.sort(key=lambda nid: degree.get(nid, 0), reverse=True) + for nid in remaining: + kept[nid] = nodes[nid] + if len(kept) >= limit_nodes: + break + + nodes = kept + edges = [e for e in edges if e["from"] in nodes and e["to"] in nodes] + + return {"nodes": list(nodes.values()), "edges": edges} + @app.exception_handler(FileNotFoundError) def handle_file_not_found(request: Request, exc: FileNotFoundError): diff --git a/static/style.css b/static/style.css index b5d0b41..92790f4 100644 --- a/static/style.css +++ b/static/style.css @@ -54,6 +54,75 @@ body { padding: 10px 12px; background: #141419; } + +/* ─── View Tabs ───────────────────────────────────────────────────────────── */ +.view-tabs { + display: flex; + gap: 8px; + padding: 0 12px 10px; +} +.tab-btn { + flex: 1; + background: #1e1e28; + border: 1px solid #2a2a3a; + border-radius: 10px; + padding: 9px 10px; + color: #cfd3ff; + font-weight: 600; + font-size: 0.85rem; +} +.tab-btn.active { + border-color: #6c8af5; + box-shadow: 0 0 0 1px rgba(108,138,245,0.25) inset; +} + +/* ─── Panels (Graph/Status) ──────────────────────────────────────────────── */ +.panel { + margin: 8px 12px; + background: #1a1a24; + border: 1px solid #252533; + border-radius: 14px; + padding: 10px 12px; +} +.panel-title { + font-size: 0.8rem; + text-transform: uppercase; + letter-spacing: 0.5px; + color: #9aa3d9; + margin-bottom: 8px; +} +.kv-row { + display: flex; + gap: 10px; + padding: 6px 0; + border-bottom: 1px solid #20202a; +} +.kv-row:last-child { border-bottom: none; } +.kv-key { + width: 110px; + color: #888899; + font-size: 0.78rem; +} +.kv-val { + flex: 1; + color: #e8e8ee; + font-size: 0.85rem; + word-break: break-word; +} +.pill { + display: inline-block; + margin: 2px 4px 2px 0; + padding: 2px 8px; + border-radius: 999px; + background: #2a2a3a; + color: #8a9aff; + font-size: 0.72rem; +} +.muted { + color: #888899; + font-size: 0.8rem; + margin-top: 6px; +} #searchInput { flex: 1; background: #1e1e28; diff --git a/templates/dashboard.html b/templates/dashboard.html index 7bdc892..deb178d 100644 --- a/templates/dashboard.html +++ b/templates/dashboard.html @@ -27,6 +27,13 @@ + +
+ + + +
+
@@ -37,6 +44,12 @@
+ + + + + +