19 Commits

Author SHA1 Message Date
a261f5b9e1 docs: link workspace integration + gitea docs 2026-05-30 01:57:23 +02:00
e6e8eba8f6 chore: sync local workspace state 2026-05-30 00:38:57 +02:00
20098a3253 Merge pull request 'fix: graph touch UX + colors/legend (closes #23)' (#24) from fix/graph-touch-colors into master 2026-05-29 11:56:00 +02:00
fa2ba11b66 fix: improve graph touch controls and legend 2026-05-29 11:54:00 +02:00
7dfd9c4228 Merge pull request 'feat: verdict/evidence verification model (closes #17)' (#18) from feat/verdict-evidence-issue-17 into master 2026-05-29 11:35:09 +02:00
6d99c520e6 feat: add verdict/evidence verification model 2026-05-29 11:30:24 +02:00
f10a5b9f19 docs: add dashboard UI/UX design plan 2026-05-29 10:50:23 +02:00
6232f25cc9 fix(fastapi): remove duplicate confirm/reject routes
- api_confirm and api_reject were defined twice on same paths
- FastAPI only registers first definition, causing silent 404s
- Kept api_confirm_engram and api_reject_engram (use _update_correctness)
- Removed duplicate direct DB implementations
- Fixes dashboard confirm/reject buttons not working
2026-05-27 18:36:03 +02:00
6b0cf5889f fix(store): escape FTS5 special characters in search_text()
- FTS5 crashes on dots (IP addresses) and hyphens (dates)
- Add regex to strip non-alphanumeric chars before FTS5 MATCH
- Fixes: fts5 syntax error near '.' and no such column: 05

Files changed: src/store.py
2026-05-27 17:54:51 +02:00
021fd0e328 feat(dashboard): pending queue + confirm/reject endpoints 2026-05-27 01:17:32 +02:00
d52e3a7f74 feat(ingest): transcript direct to DB 2026-05-27 01:14:42 +02:00
1635ee8b03 fix(verify): scan all engrams 2026-05-27 01:11:59 +02:00
f8ac0af869 chore(timers): ingest memory every 5 min 2026-05-27 01:11:17 +02:00
9dd5e49e2a chore(timers): process pending every 5 min 2026-05-27 01:08:08 +02:00
b158b19208 feat(ingest): tail session transcript into memory 2026-05-27 01:04:47 +02:00
095e6a33f8 feat(dashboard): realtime status + graph render 2026-05-27 00:22:14 +02:00
e5061b317f feat(dashboard): add status+graph views 2026-05-27 00:11:44 +02:00
ec8870ea40 feat(verify): add pending external verifier 2026-05-27 00:05:51 +02:00
8f47151a48 docs(systemd): fix paths + add dashboard smoke + vendor units 2026-05-26 23:47:07 +02:00
44 changed files with 7952 additions and 88 deletions

View File

@@ -2,19 +2,32 @@
This is the operational quick-reference for the shipped systemd timers/services and the optional FastAPI dashboard backend.
Repository root (on host): `/root/.openclaw/workspace`
Repository root (on host): `/root/.openclaw/workspace/second-brain`
Integration + orchestration docs live in the workspace repo:
- `/root/.openclaw/workspace/docs/SECOND_BRAIN.md`
- `/root/.openclaw/workspace/docs/GITEA.md`
## Systemd units (cron jobs)
Unit files are shipped in `systemd/` (repo root). Install them into `/etc/systemd/system/` (symlink or copy), then reload:
Unit files are shipped in `systemd/` (this repo). Install them into `/etc/systemd/system/` (symlink or copy), then reload:
```bash
sudo ln -sf /root/.openclaw/workspace/systemd/openclaw-secondbrain-*.service /etc/systemd/system/
sudo ln -sf /root/.openclaw/workspace/systemd/openclaw-secondbrain-*.timer /etc/systemd/system/
sudo ln -sf /root/.openclaw/workspace/systemd/openclaw-memory-archive.* /etc/systemd/system/
sudo ln -sf /root/.openclaw/workspace/second-brain/systemd/openclaw-secondbrain-*.service /etc/systemd/system/
sudo ln -sf /root/.openclaw/workspace/second-brain/systemd/openclaw-secondbrain-*.timer /etc/systemd/system/
sudo ln -sf /root/.openclaw/workspace/second-brain/systemd/openclaw-memory-archive.* /etc/systemd/system/
sudo systemctl daemon-reload
```
Optional (verification hardening):
```bash
sudo ln -sf /root/.openclaw/workspace/second-brain/systemd/openclaw-secondbrain-verify-pending.* /etc/systemd/system/
sudo systemctl daemon-reload
sudo systemctl enable --now openclaw-secondbrain-verify-pending.timer
```
Enable timers:
```bash
@@ -61,7 +74,7 @@ python3 -m pip install -r second-brain/requirements-dashboard.txt
SECOND_BRAIN_WORKSPACE="/root/.openclaw/workspace/second-brain" python3 second-brain/fastapi_app.py
```
Default port is `8501` (same as Streamlit default). Do not run both on the same port.
Default port is `8501` (same as Streamlit default). You can override via `SECOND_BRAIN_PORT` (or `PORT`) when starting manually.
Endpoint smoke tests:

View File

@@ -0,0 +1,160 @@
#!/usr/bin/env python3
"""
Ingest OpenClaw session transcript JSONL directly into the Second-Brain DB.
State is tracked with byte offsets per transcript file.
Sources are configured via workspace/memory/session_sources.json.
"""
from __future__ import annotations
import json
import os
from dataclasses import dataclass
from datetime import datetime, timezone
from pathlib import Path
from typing import Any, Dict, Iterable, List
from uuid import NAMESPACE_URL, uuid5
WORKSPACE = Path("/root/.openclaw/workspace")
MEMORY_DIR = WORKSPACE / "memory"
SOURCES_PATH = MEMORY_DIR / "session_sources.json"
STATE_PATH = MEMORY_DIR / "session_db_ingest_state.json"
BRAIN_DIR = WORKSPACE / "second-brain"
DB_PATH = Path(os.environ.get("BRAIN_DB", str(BRAIN_DIR / "data" / "brain.sqlite")))
import sys
sys.path.insert(0, str(BRAIN_DIR))
from src.engram import Engram, Grounding # type: ignore
from src.store import EngramStore # type: ignore
def _now() -> str:
return datetime.now(timezone.utc).isoformat()
def _load_json(path: Path, default: Any) -> Any:
try:
if not path.exists():
return default
return json.loads(path.read_text(encoding="utf-8"))
except Exception:
return default
def _save_json(path: Path, payload: Any) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(json.dumps(payload, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
def _extract_text(content: Any) -> str:
if isinstance(content, str):
return content.strip()
if isinstance(content, list):
parts: List[str] = []
for c in content:
if isinstance(c, dict) and c.get("type") == "text" and isinstance(c.get("text"), str):
parts.append(c["text"])
elif isinstance(c, str):
parts.append(c)
return "\n".join(p.strip() for p in parts if p and p.strip()).strip()
if isinstance(content, dict) and isinstance(content.get("text"), str):
return content["text"].strip()
return str(content).strip()
@dataclass
class Source:
label: str
transcript_path: Path
def _load_sources() -> List[Source]:
payload = _load_json(SOURCES_PATH, {"sources": []})
sources: List[Source] = []
for item in payload.get("sources", []) if isinstance(payload, dict) else []:
if not isinstance(item, dict):
continue
label = str(item.get("label") or "session")
p = item.get("path")
if not isinstance(p, str) or not p:
continue
sources.append(Source(label=label, transcript_path=Path(p)))
return sources
def _iter_new_lines(path: Path, *, start_offset: int) -> Iterable[tuple[int, str]]:
with open(path, "rb") as f:
f.seek(max(0, int(start_offset)))
while True:
raw = f.readline()
if not raw:
break
line = raw.decode("utf-8", errors="ignore").strip()
if not line:
continue
yield (f.tell(), line)
def run() -> Dict[str, Any]:
if not DB_PATH.exists():
return {"success": False, "time": _now(), "error": f"db missing: {DB_PATH}"}
sources = _load_sources()
state = _load_json(STATE_PATH, {"offsets": {}})
offsets: Dict[str, int] = state.get("offsets", {}) if isinstance(state, dict) else {}
store = EngramStore(str(DB_PATH))
out = {"success": True, "time": _now(), "sources": len(sources), "messages_saved": 0, "messages_skipped": 0, "errors": []}
for src in sources:
try:
if not src.transcript_path.exists():
continue
key = str(src.transcript_path)
start = int(offsets.get(key, 0))
for new_off, line in _iter_new_lines(src.transcript_path, start_offset=start):
offsets[key] = new_off
obj = json.loads(line)
if not isinstance(obj, dict) or obj.get("type") != "message":
continue
msg = obj.get("message") if isinstance(obj.get("message"), dict) else {}
role = str(msg.get("role") or "unknown")
content = _extract_text(msg.get("content"))
if len(content.strip()) < 5:
continue
mid = str(obj.get("id") or msg.get("id") or msg.get("messageId") or msg.get("message_id") or "")
if not mid:
mid = str(uuid5(NAMESPACE_URL, f"openclaw-transcript:{src.label}:{role}:{content[:200]}"))
eid = str(uuid5(NAMESPACE_URL, f"openclaw-transcript:{src.label}:{mid}"))
if store.get(eid):
out["messages_skipped"] += 1
continue
eg = Engram.create(
content=f"[transcript:{src.label}] [{role}] [{mid}]\n\n{content}"[:4000],
source="session",
tags=["session", "openclaw", "transcript", f"role:{role}"],
session_id=src.label,
confidence=0.55,
grounding=Grounding.ASSUMPTION,
)
eg.id = uuid5(NAMESPACE_URL, eid)
eg.metadata["source"] = "session"
eg.metadata["session_id"] = src.label
eg.metadata["role"] = role
eg.metadata["message_id"] = mid
store.save(eg)
out["messages_saved"] += 1
except Exception as e:
out["success"] = False
out["errors"].append(f"{src.transcript_path}: {e}")
_save_json(STATE_PATH, {"offsets": offsets, "updated_at": out["time"]})
return out
if __name__ == "__main__":
print(json.dumps(run(), ensure_ascii=False, indent=2))

View File

@@ -0,0 +1,213 @@
#!/usr/bin/env python3
"""
Tail OpenClaw session transcript JSONL files and append new messages into
workspace/memory/YYYY-MM-DD.md so the existing ingest_memory pipeline can pick
them up.
Why: when chat_autosave hooks are missed/aborted, the "memory/*.md -> DB" ingest
doesn't see the latest conversation. This bridges transcript -> memory.
Safety: read-only access to transcript files; state stored in workspace/memory/.
"""
from __future__ import annotations
import json
import os
from dataclasses import dataclass
from datetime import datetime, timezone
from pathlib import Path
from typing import Any, Dict, Iterable, List, Optional
try:
from zoneinfo import ZoneInfo # py3.9+
except Exception: # pragma: no cover
ZoneInfo = None # type: ignore
WORKSPACE = Path("/root/.openclaw/workspace")
MEMORY_DIR = WORKSPACE / "memory"
SOURCES_PATH = MEMORY_DIR / "session_sources.json"
STATE_PATH = MEMORY_DIR / "session_ingest_state.json"
def _local_tz():
tz = os.environ.get("TZ") or "Europe/Berlin"
if ZoneInfo:
try:
return ZoneInfo(tz)
except Exception:
return timezone.utc
return timezone.utc
def _load_json(path: Path, default: Any) -> Any:
try:
if not path.exists():
return default
return json.loads(path.read_text(encoding="utf-8"))
except Exception:
return default
def _save_json(path: Path, payload: Any) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(json.dumps(payload, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
def _iso_to_dt(ts: str) -> Optional[datetime]:
try:
if ts.endswith("Z"):
ts = ts[:-1] + "+00:00"
return datetime.fromisoformat(ts)
except Exception:
return None
def _ms_to_dt(ms: int) -> datetime:
return datetime.fromtimestamp(ms / 1000, tz=timezone.utc)
def _extract_text(content: Any) -> str:
if isinstance(content, str):
return content.strip()
if isinstance(content, list):
parts: List[str] = []
for c in content:
if isinstance(c, dict) and c.get("type") == "text" and isinstance(c.get("text"), str):
parts.append(c["text"])
elif isinstance(c, str):
parts.append(c)
return "\n".join(p.strip() for p in parts if p and p.strip()).strip()
if isinstance(content, dict):
if isinstance(content.get("text"), str):
return content["text"].strip()
return str(content).strip()
@dataclass
class Source:
label: str
transcript_path: Path
def _load_sources() -> List[Source]:
"""
Sources file format:
{
"sources": [
{ "label": "telegram:263887248", "path": "/root/.openclaw/agents/main/sessions/<id>.jsonl" }
]
}
"""
payload = _load_json(SOURCES_PATH, {"sources": []})
sources: List[Source] = []
for item in payload.get("sources", []) if isinstance(payload, dict) else []:
if not isinstance(item, dict):
continue
label = str(item.get("label") or "session")
p = item.get("path")
if not isinstance(p, str) or not p:
continue
sources.append(Source(label=label, transcript_path=Path(p)))
return sources
def _memory_path_for(dt: datetime) -> Path:
tz = _local_tz()
local = dt.astimezone(tz)
MEMORY_DIR.mkdir(parents=True, exist_ok=True)
return MEMORY_DIR / f"{local.date().isoformat()}.md"
def _append_memory(dt: datetime, label: str, role: str, text: str) -> None:
if not text.strip():
return
tz = _local_tz()
local = dt.astimezone(tz)
mem = _memory_path_for(dt)
if not mem.exists():
mem.write_text(f"# {local.date().isoformat()}\n\n", encoding="utf-8")
header = f"## {local.strftime('%H:%M:%S')} - {label} ({role})"
body = text.strip()
mem.write_text(mem.read_text(encoding="utf-8") + f"{header}\n\n{body}\n\n", encoding="utf-8")
def _iter_new_lines(path: Path, *, start_offset: int) -> Iterable[tuple[int, str]]:
with open(path, "rb") as f:
f.seek(max(0, int(start_offset)))
while True:
raw = f.readline()
if not raw:
break
try:
line = raw.decode("utf-8", errors="ignore").strip()
except Exception:
line = ""
if not line:
continue
yield (f.tell(), line)
def run() -> Dict[str, Any]:
sources = _load_sources()
state = _load_json(STATE_PATH, {"offsets": {}})
offsets: Dict[str, int] = state.get("offsets", {}) if isinstance(state, dict) else {}
out = {
"success": True,
"time": datetime.now(timezone.utc).isoformat(),
"sources": len(sources),
"messages_appended": 0,
"errors": [],
}
for src in sources:
try:
if not src.transcript_path.exists():
continue
key = str(src.transcript_path)
start = int(offsets.get(key, 0))
for new_off, line in _iter_new_lines(src.transcript_path, start_offset=start):
try:
obj = json.loads(line)
except Exception:
offsets[key] = new_off
continue
if not isinstance(obj, dict) or obj.get("type") != "message":
offsets[key] = new_off
continue
msg = obj.get("message") if isinstance(obj.get("message"), dict) else {}
role = str(msg.get("role") or "unknown")
content = msg.get("content")
text = _extract_text(content)
dt = None
if isinstance(msg.get("timestamp"), (int, float)):
dt = _ms_to_dt(int(msg["timestamp"]))
elif isinstance(obj.get("timestamp"), str):
dt = _iso_to_dt(obj["timestamp"])
if dt is None:
dt = datetime.now(timezone.utc)
if len(text.strip()) < 5:
offsets[key] = new_off
continue
_append_memory(dt, src.label, role, text)
out["messages_appended"] += 1
offsets[key] = new_off
except Exception as e:
out["success"] = False
out["errors"].append(f"{src.transcript_path}: {e}")
_save_json(STATE_PATH, {"offsets": offsets, "updated_at": out["time"]})
return out
if __name__ == "__main__":
res = run()
print(json.dumps(res, ensure_ascii=False, indent=2))

View File

@@ -0,0 +1,141 @@
#!/usr/bin/env python3
"""
Verify pending (unconfirmed) engrams using lightweight external checks.
Policy (conservative):
- `openclaw-memory` is treated as internal ground-truth and is auto-confirmed
by the review job (see `cron_tasks/review_brain.py` in the workspace runtime).
- For `source=web`, confirm if the grounded URL responds with HTTP 2xx, reject on
4xx/5xx, and keep pending on timeouts/unknown.
- Reject obvious low-signal placeholders (e.g. session summary stubs).
"""
import json
import os
import sys
from pathlib import Path
from datetime import datetime, timezone
from typing import Any, Optional
WORKSPACE = Path(os.environ.get("SECOND_BRAIN_WORKSPACE", "/root/.openclaw/workspace/second-brain"))
DB_PATH = Path(os.environ.get("BRAIN_DB", str(WORKSPACE / "data" / "brain.sqlite"))).resolve()
sys.path.insert(0, str(WORKSPACE))
from src.store import EngramStore
from src.engram import ReviewEntry
OUTPUT_FILE = os.environ.get("CRON_OUTPUT_FILE", "/tmp/verify_pending_external.json")
def _now() -> str:
return datetime.now(timezone.utc).isoformat()
def _get_url(meta: dict[str, Any]) -> Optional[str]:
url = meta.get("url")
if isinstance(url, str) and url.startswith(("http://", "https://")):
return url
grounding = meta.get("grounding")
if isinstance(grounding, dict):
g_url = grounding.get("url")
if isinstance(g_url, str) and g_url.startswith(("http://", "https://")):
return g_url
return None
def _http_status(url: str, timeout_s: float = 6.0) -> Optional[int]:
try:
import urllib.request
req = urllib.request.Request(
url,
method="GET",
headers={"User-Agent": "openclaw-secondbrain/verify_pending_external"},
)
with urllib.request.urlopen(req, timeout=timeout_s) as resp:
return int(getattr(resp, "status", 200))
except Exception:
return None
def main() -> int:
if not DB_PATH.exists():
out = {"success": False, "error": f"db missing: {DB_PATH}", "time": _now()}
Path(OUTPUT_FILE).write_text(json.dumps(out, indent=2))
print(out["error"])
return 1
store = EngramStore(str(DB_PATH))
all_egs = []
offset = 0
while True:
batch = store.get_all(limit=2000, offset=offset)
if not batch:
break
all_egs.extend(batch)
offset += len(batch)
pending = [
eg
for eg in all_egs
if (not eg.correctness.is_final())
]
confirmed = 0
rejected = 0
still_pending = 0
checked = 0
for eg in pending:
checked += 1
src = eg.metadata.get("source")
content = (eg.content or "").strip()
if src == "session" and (
content.startswith("Session Summary (sess_") or content.startswith("Please remember ")
):
eg.correctness.reject("verify-pending", "Auto-reject: session placeholder")
store.save(eg)
rejected += 1
continue
if src == "web":
url = _get_url(eg.metadata)
if not url:
still_pending += 1
continue
status = _http_status(url)
if status is None:
still_pending += 1
continue
if 200 <= status < 300:
eg.correctness.confirm("verify-pending", f"Auto-confirm: web url ok ({status}) {url}")
store.save(eg)
confirmed += 1
else:
eg.correctness.reject("verify-pending", f"Auto-reject: web url status={status} {url}")
store.save(eg)
rejected += 1
continue
still_pending += 1
out = {
"success": True,
"time": _now(),
"total": len(all_egs),
"pending_before": len(pending),
"checked": checked,
"confirmed": confirmed,
"rejected": rejected,
"still_pending": still_pending,
}
Path(OUTPUT_FILE).write_text(json.dumps(out, indent=2))
print(
f"VERIFY: pending_before={out['pending_before']} confirmed={confirmed} rejected={rejected} still_pending={still_pending}"
)
return 0
if __name__ == "__main__":
raise SystemExit(main())

View File

@@ -1,6 +1,6 @@
# Second-Brain 2.0 — Release Checklist (Grundversion + FastAPI)
Scope: OpenClaw workspace at `/root/.openclaw/workspace` with `second-brain/` and the shipped `systemd/` unit files.
Scope: OpenClaw workspace at `/root/.openclaw/workspace` with `second-brain/` and the shipped `second-brain/systemd/` unit files.
This checklist is copy/paste friendly. Run as a user with `sudo` (systemd commands need it).
@@ -24,13 +24,21 @@ ls -la /etc/systemd/system/openclaw-memory-archive.* 2>/dev/null || true
If missing, install them (symlink is fine; copy is fine too):
```bash
sudo ln -sf /root/.openclaw/workspace/systemd/openclaw-secondbrain-*.service /etc/systemd/system/
sudo ln -sf /root/.openclaw/workspace/systemd/openclaw-secondbrain-*.timer /etc/systemd/system/
sudo ln -sf /root/.openclaw/workspace/systemd/openclaw-memory-archive.* /etc/systemd/system/
sudo ln -sf /root/.openclaw/workspace/systemd/openclaw-secondbrain-dashboard.service /etc/systemd/system/
sudo ln -sf /root/.openclaw/workspace/second-brain/systemd/openclaw-secondbrain-*.service /etc/systemd/system/
sudo ln -sf /root/.openclaw/workspace/second-brain/systemd/openclaw-secondbrain-*.timer /etc/systemd/system/
sudo ln -sf /root/.openclaw/workspace/second-brain/systemd/openclaw-memory-archive.* /etc/systemd/system/
sudo ln -sf /root/.openclaw/workspace/second-brain/systemd/openclaw-secondbrain-dashboard.service /etc/systemd/system/
sudo systemctl daemon-reload
```
Optional (verification hardening):
```bash
sudo ln -sf /root/.openclaw/workspace/second-brain/systemd/openclaw-secondbrain-verify-pending.* /etc/systemd/system/
sudo systemctl daemon-reload
sudo systemctl enable --now openclaw-secondbrain-verify-pending.timer
```
### Enable timers
```bash

View File

@@ -0,0 +1,215 @@
# Second-Brain Dashboard UI/UX Design Plan (mobile-first)
Basis: `second-brain/templates/dashboard.html`, `second-brain/static/style.css` (aktueller Stand) + neue Graph-Controls aus `/tmp/second-brain-staging/`.
## 1) Konkreter Design-Plan
### Ziele (UX)
- **Schnelles Triaging unterwegs:** Pending/Errors sofort sichtbar, 1-Hand-Bedienung.
- **Konsistentes Design-System:** einheitliche Buttons/Inputs/Panels statt Einzellösungen.
- **Graph als Diagnose-Tool:** klare Controls, Legende, nachvollziehbares Feedback (Loading/Empty/Errors).
### Farbschema (Dark, high-contrast, "indigo + emerald")
- **Background:** sehr dunkel (nahe #0f1117) für weniger Blendung.
- **Surface (Cards/Panels):** abgestufte Flächen (Surface-1, Surface-2) für Hierarchie.
- **Primary:** Indigo/Blue für interaktive Elemente und Highlights (bisher #6c8af5 bleibt als Basis).
- **Success:** Emerald für Confirm/OK (bisher #3a7d3a → etwas heller/satter).
- **Danger:** Red für Errors/Reject.
- **Warning:** Amber für Pending/Match/Attention.
- **Text:** fast-weiß, sekundär gedimmt.
### Typografie
- Systemfont-Stack (schnell, gut lesbar): `ui-sans-serif, system-ui, -apple-system, Segoe UI, Roboto, Arial`.
- Skala (mobile-first):
- `text-xs` 12px (Labels, Meta)
- `text-sm` 1314px (secondary)
- `text-md` 1516px (Body)
- `text-lg` 1820px (Titles)
- **Ziffern:** optional `font-variant-numeric: tabular-nums;` für Stats.
### Spacing/Rhythm
- 4px Grid; Standard-Gaps: 8/12/16.
- Container-Padding: 12px (mobile), 1620px (>=768px).
- Border-radius: 1216px (Cards/Modals), 10px (Inputs/Buttons), 999px (Pills).
### Komponenten (konkret)
#### A) Tabs
- Tabs bleiben als 3 Buttons, aber:
- **Active State** deutlicher (background + border + subtle glow).
- **Touch target** min. 44px Höhe.
- Sticky bleibt, aber mit leichter **Backdrop-Blurring** (wenn möglich) oder solid surface.
#### B) Search
- Search-Row wird als **kompakte Toolbar** gestaltet:
- Input + Filter in einer Zeile.
- Optional Quick-Chips darunter: `All / Pending / Confirmed / Errors` (klickbar) als Alternative zum Select.
- Clear-Button (×) im Input (per CSS `::-webkit-search-cancel-button` oder eigener Button) für Mobile.
#### C) Cards
- Karte als 3 Zonen: Header (badges/tags/date), Body (content), Footer (actions).
- Status wird stärker codiert:
- left-border + kleine **Status-Pill** (OK/Pending/Error) mit eindeutiger Farbe.
- Body: bessere Lesbarkeit via `line-height: 1.55` und max-height/clamp optional.
#### D) Modal
- Modal als **Bottom Sheet** auf Mobile (>=50vh) + klassisches Center-Modal auf Desktop.
- Close-Button größer + Tap-Area.
- Inhalt in Tabs/Sections (History/Meta/Content) optional später.
#### E) Graph + Controls (aus Staging)
- Controls als **Control Bar** oberhalb Canvas:
- Primary: Physics toggle.
- Secondary: Fit, Reset, Reload.
- Text labels kurz (z.B. `Physics` statt `Physics: off`, state als Badge).
- Canvas passt sich an Viewport an:
- `width: min(100%, 560px)`; Height: `min(65vh, 560px)` (CSS statt fester HTML Attribute, wenn möglich).
- Legende als einklappbares Panel (`Details`/`summary`) oder leichtes Panel unter Canvas.
#### F) Status Panels
- Status-View nutzt vorhandene `.panel`/`.kv-*`:
- Gruppen: System, Storage, Jobs, Insights, Pending Queue.
- Jede Gruppe als Panel mit klarer Title Row.
- Kritische Werte (Errors/Pending/Queue) farblich markieren.
## 2) CSS-Variablen (Theme-Tokens) + Mapping
### Token-Vorschlag (`:root`)
```css
:root {
/* typography */
--font-sans: ui-sans-serif, system-ui, -apple-system, Segoe UI, Roboto, Arial, sans-serif;
/* colors */
--bg: #0f1117;
--surface-1: #14151d;
--surface-2: #1a1b26;
--border: #2a2d3a;
--text: #e8e8ee;
--text-muted: #8b90a3;
--text-dim: #5c6276;
--primary: #6c8af5; /* existing */
--primary-2: #8aa1ff;
--success: #2fbf71;
--danger: #ef4444;
--warning: #f7d154; /* existing-ish */
/* shadows */
--shadow-1: 0 1px 0 rgba(0,0,0,.25), 0 8px 24px rgba(0,0,0,.35);
/* radius */
--r-sm: 10px;
--r-md: 14px;
--r-lg: 16px;
/* spacing */
--s-1: 4px;
--s-2: 8px;
--s-3: 12px;
--s-4: 16px;
--s-5: 20px;
/* control sizes */
--tap: 44px;
}
```
### Mapping auf existierende Selektoren
- `body`/`.app`
- `background: var(--bg)` statt `#141419`
- `font-family: var(--font-sans)`
- `color: var(--text)`
- `.stats-bar`, `.tabs-bar`, `.search-box`
- `background: var(--surface-1)` (Stats ggf. Gradient bleibt, aber auf Tokens)
- `border-bottom: 1px solid var(--border)`
- `.panel`, `.card`, `.modal-content`, `.graph-legend`
- `background: var(--surface-2)`
- `border: 1px solid var(--border)`
- `border-radius: var(--r-md)`
- `.stat-num`, `#pageNum`, `.tab-btn.active`, `#searchInput:focus`
- `color/border-color: var(--primary)`
- `.muted`, `.stat-label`, `.kv-key`, `.date`
- `color: var(--text-muted)` bzw. `var(--text-dim)`
- Buttons
- vereinheitlichen über `.btn` + Modifier: `.btn.primary`, `.btn.danger`, `.btn.success`, `.btn.ghost`
- `min-height: var(--tap)` für Touch
## 3) 510 priorisierte UI-Änderungen (mit Begründung)
1) **Design Tokens (CSS vars) einführen** → reduziert Farbmix, erleichtert spätere Themes/Anpassungen.
2) **Einheitliche Button-Komponente (`.btn`)** (inkl. `:active`, `:disabled`, `min-height`) → bessere Touch-UX, konsistente Interaktion.
3) **Graph-Controls + Legende aus Staging in den Main-Template-Stand ziehen** → Graph wird tatsächlich bedienbar/selbsterklärend.
4) **Responsive Graph-Canvas (CSS gesteuert)** statt fixer `width/height` → bessere Nutzung auf Phones, weniger Scroll.
5) **Search als Toolbar + Clear-Action** → schnelleres Filtern unterwegs, weniger Friktion.
6) **Modal als Bottom-Sheet auf Mobile** → angenehmer für längeren Content + History, weniger „winziges Fenster“.
7) **Status/Health Werte farblich akzentuieren** (pending/errors/warn) → schnelleres Erkennen von Problemen.
8) **Cards: Status-Pill + typografische Lesbarkeit** (line-height, spacing) → weniger „Textblock“, bessere Scanbarkeit.
9) **Accessibility-Basics**: Focus-Rings, Kontrast, `prefers-reduced-motion` → weniger „invisible focus“ und bessere Bedienbarkeit.
10) **Top-level Layout Max-Width für Desktop** (z.B. 560720px) → verhindert „zu breite“ Zeilen.
## 4) Optionale Patch-Vorschläge (Diff-Snippets, NICHT anwenden)
> Hinweis: Snippets sind bewusst klein gehalten. Gesamt < 120 Zeilen.
### Snippet A — Tokens + Button-System (style.css)
```diff
--- a/second-brain/static/style.css
+++ b/second-brain/static/style.css
@@
+:root {
+ --font-sans: ui-sans-serif, system-ui, -apple-system, Segoe UI, Roboto, Arial, sans-serif;
+ --bg:#0f1117; --surface-1:#14151d; --surface-2:#1a1b26; --border:#2a2d3a;
+ --text:#e8e8ee; --text-muted:#8b90a3; --text-dim:#5c6276;
+ --primary:#6c8af5; --success:#2fbf71; --danger:#ef4444; --warning:#f7d154;
+ --r-sm:10px; --r-md:14px; --r-lg:16px;
+ --s-2:8px; --s-3:12px; --s-4:16px;
+ --tap:44px;
+}
+
+body { font-family: var(--font-sans); background: var(--bg); color: var(--text); }
+
+.btn{
+ min-height: var(--tap);
+ background: #1e1e28;
+ border: 1px solid var(--border);
+ border-radius: var(--r-sm);
+ padding: 8px 12px;
+ color: #cfd3ff;
+ font-weight: 700;
+}
+.btn.primary{ border-color: var(--primary); box-shadow: 0 0 0 1px rgba(108,138,245,0.18) inset; }
+.btn.success{ background: rgba(47,191,113,.18); border-color: rgba(47,191,113,.35); }
+.btn.danger{ background: rgba(239,68,68,.16); border-color: rgba(239,68,68,.35); }
+.btn:active{ transform: scale(.98); }
+.btn:disabled{ opacity: .45; }
```
### Snippet B — Graph Controls + Legend übernehmen (dashboard.html)
```diff
--- a/second-brain/templates/dashboard.html
+++ b/second-brain/templates/dashboard.html
@@
- <div class="graph" id="graph" style="display:none;">
- <canvas id="graphCanvas" width="440" height="520"></canvas>
- <div class="muted small" id="graphHint">Lade Graph…</div>
- </div>
+ <div class="graph" id="graph" style="display:none;">
+ <div class="graph-controls">
+ <button class="btn primary" id="btnGraphPhysics" onclick="toggleGraphPhysics()">Physics</button>
+ <button class="btn" onclick="resetGraphView()">Reset</button>
+ <button class="btn" onclick="fitGraphView()">Fit</button>
+ <button class="btn" onclick="reloadGraph()">Reload</button>
+ </div>
+ <canvas id="graphCanvas" width="440" height="520"></canvas>
+ <div class="graph-hint muted small" id="graphHint">Lade Graph…</div>
+ <div class="graph-legend">
+ <div><strong>Graph</strong>: Zoom (Wheel/Pinch), Pan (Drag). Klick auf Engram öffnet Details, Klick auf Tag setzt Suche.</div>
+ <div class="legend-row"><span class="legend-dot engram"></span> Engram</div>
+ <div class="legend-row"><span class="legend-dot tag"></span> Tag</div>
+ <div class="legend-row"><span class="legend-dot match"></span> Match</div>
+ </div>
+ </div>
```

View File

@@ -11,11 +11,14 @@ Goals:
import json
import os
import sqlite3
import subprocess
from datetime import datetime, timezone
from pathlib import Path
from uuid import uuid4
from urllib.parse import urlparse
from fastapi import FastAPI, Form, Query, Request
from fastapi.responses import HTMLResponse, JSONResponse, PlainTextResponse
from fastapi.responses import HTMLResponse, JSONResponse, PlainTextResponse, StreamingResponse
from fastapi.staticfiles import StaticFiles
# ─── Config ──────────────────────────────────────────────────────────────────
@@ -51,11 +54,22 @@ def get_db():
def parse_engram(row: sqlite3.Row) -> dict:
meta = json.loads(row["metadata_json"] or "{}")
correctness = json.loads(row["correctness_json"] or "{}")
verdict = correctness.get("verdict")
if not isinstance(verdict, str) or not verdict:
# Back-compat inference for older rows
if correctness.get("confirmed", False):
verdict = "confirmed_true"
elif int(correctness.get("rejections", 0) or 0) > 0:
verdict = "confirmed_false"
else:
verdict = "unknown"
return {
"id": row["id"],
"content": row["content"],
"confidence": meta.get("confidence", 0.0),
"confirmed": correctness.get("confirmed", False),
"verdict": verdict,
"evidence": correctness.get("evidence", []),
"confirmations": correctness.get("confirmations", 0),
"rejections": correctness.get("rejections", 0),
"tags": meta.get("tags", []),
@@ -69,6 +83,158 @@ def parse_engram(row: sqlite3.Row) -> dict:
}
def _now_iso() -> str:
return datetime.now(timezone.utc).isoformat()
def _update_correctness(engram_id: str, *, action: str, reason: str | None = None) -> dict:
"""
Update correctness_json for an engram. action: confirm|reject
"""
conn = get_db()
c = conn.cursor()
row = c.execute("SELECT correctness_json FROM engrams WHERE id = ?", (engram_id,)).fetchone()
if not row:
conn.close()
raise FileNotFoundError(f"Engram not found: {engram_id}")
corr = json.loads(row["correctness_json"] or "{}")
corr.setdefault("verdict", None)
corr.setdefault("evidence", [])
corr.setdefault("confirmed", False)
corr.setdefault("confirmations", 0)
corr.setdefault("rejections", 0)
corr.setdefault("review_history", [])
corr["last_reviewed"] = _now_iso()
entry = {
"by": "dashboard",
"action": action,
"at": corr["last_reviewed"],
"note": (reason or "").strip(),
}
try:
corr["review_history"].append(entry)
except Exception:
corr["review_history"] = [entry]
if action == "confirm":
corr["verdict"] = "confirmed_true"
corr["confirmed"] = True
corr["confirmations"] = int(corr.get("confirmations", 0) or 0) + 1
elif action == "reject":
corr["verdict"] = "confirmed_false"
corr["rejections"] = int(corr.get("rejections", 0) or 0) + 1
corr["confirmed"] = False
# Store minimal evidence for dashboard-driven actions.
try:
ev = corr.get("evidence")
if not isinstance(ev, list):
ev = []
ev.append(
{
"kind": "human",
"by": "dashboard",
"at": corr["last_reviewed"],
"action": action,
}
)
corr["evidence"] = ev[-50:] # cap growth
except Exception:
pass
c.execute(
"UPDATE engrams SET correctness_json = ?, modified_at = ? WHERE id = ?",
(json.dumps(corr, ensure_ascii=False), corr["last_reviewed"], engram_id),
)
conn.commit()
conn.close()
return {"ok": True}
def _bump_access(engram_id: str) -> dict:
conn = get_db()
c = conn.cursor()
row = c.execute("SELECT metadata_json FROM engrams WHERE id = ?", (engram_id,)).fetchone()
if not row:
conn.close()
raise FileNotFoundError(f"Engram not found: {engram_id}")
meta = json.loads(row["metadata_json"] or "{}")
meta["access_count"] = int(meta.get("access_count", 0) or 0) + 1
meta["last_accessed"] = _now_iso()
c.execute(
"UPDATE engrams SET metadata_json = ?, modified_at = ? WHERE id = ?",
(json.dumps(meta, ensure_ascii=False), meta["last_accessed"], engram_id),
)
conn.commit()
conn.close()
return {"ok": True}
def _safe_json_extract_tags(meta_json: str) -> list[str]:
try:
d = json.loads(meta_json or "{}")
tags = d.get("tags") or []
return [t for t in tags if isinstance(t, str)]
except Exception:
return []
def _host_from_meta(meta_json: str) -> str | None:
try:
d = json.loads(meta_json or "{}")
grounding = d.get("grounding")
url = d.get("url")
if isinstance(grounding, dict) and isinstance(grounding.get("url"), str):
url = grounding.get("url")
if not isinstance(url, str):
return None
parsed = urlparse(url)
return parsed.hostname
except Exception:
return None
def _systemd_unit_state(unit: str) -> dict:
"""
Best-effort systemd status snapshot for a known unit.
Never raises; returns minimal fields.
"""
try:
out = subprocess.check_output(
["systemctl", "show", unit, "--no-page", "--property=ActiveState,SubState,Result,ExecMainStatus,ExecMainCode,ExecMainStartTimestamp,ExecMainExitTimestamp"],
text=True,
stderr=subprocess.STDOUT,
timeout=2,
)
kv = {}
for line in out.splitlines():
if "=" in line:
k, v = line.split("=", 1)
kv[k] = v
return {
"unit": unit,
"active": kv.get("ActiveState"),
"sub": kv.get("SubState"),
"result": kv.get("Result"),
"exit_status": kv.get("ExecMainStatus"),
"start_ts": kv.get("ExecMainStartTimestamp"),
"exit_ts": kv.get("ExecMainExitTimestamp"),
}
except Exception as e:
return {"unit": unit, "error": str(e)}
def _dir_size_bytes(path: Path) -> int:
total = 0
try:
for p in path.rglob("*"):
try:
if p.is_file():
total += p.stat().st_size
except Exception:
pass
except Exception:
pass
return total
# ─── API Endpoints ───────────────────────────────────────────────────────────
@app.get("/healthz", response_class=PlainTextResponse)
@@ -83,6 +249,363 @@ def api_config():
"db_path": str(DB_PATH),
}
@app.get("/api/db_info")
def api_db_info():
if not DB_PATH.exists():
raise FileNotFoundError(f"DB not found: {DB_PATH}")
st = DB_PATH.stat()
return {
"db_path": str(DB_PATH),
"size_bytes": st.st_size,
"mtime": datetime.fromtimestamp(st.st_mtime, tz=timezone.utc).isoformat(),
}
@app.get("/api/storage_stats")
def api_storage_stats():
conn = get_db()
c = conn.cursor()
total = c.execute("SELECT COUNT(*) FROM engrams").fetchone()[0]
confirmed_true = c.execute(
"""
SELECT COUNT(*) FROM engrams
WHERE (
json_extract(correctness_json, '$.verdict') = 'confirmed_true'
OR (json_extract(correctness_json, '$.verdict') IS NULL AND json_extract(correctness_json, '$.confirmed') = 1)
)
"""
).fetchone()[0]
confirmed_false = c.execute(
"""
SELECT COUNT(*) FROM engrams
WHERE (
json_extract(correctness_json, '$.verdict') = 'confirmed_false'
OR (json_extract(correctness_json, '$.verdict') IS NULL
AND json_extract(correctness_json, '$.confirmed') = 0
AND COALESCE(json_extract(correctness_json, '$.rejections'), 0) > 0)
)
"""
).fetchone()[0]
sources = {
r[0]: r[1]
for r in c.execute(
"SELECT json_extract(metadata_json, '$.source') AS src, COUNT(*) FROM engrams GROUP BY src ORDER BY COUNT(*) DESC"
).fetchall()
if r[0] is not None
}
conn.close()
chroma_dir = WORKSPACE / "data" / "chroma"
emb_cache_dir = WORKSPACE / "data" / "embedding_cache"
vec_state_path = WORKSPACE / "data" / "vector_index_state.json"
vec_state = {}
if vec_state_path.exists():
try:
vec_state = json.loads(vec_state_path.read_text())
except Exception:
vec_state = {}
obsidian_cfg_path = WORKSPACE / "data" / "obsidian_config.json"
obsidian_cfg = None
if obsidian_cfg_path.exists():
try:
obsidian_cfg = json.loads(obsidian_cfg_path.read_text())
except Exception:
obsidian_cfg = {"raw": obsidian_cfg_path.read_text()[:2000]}
backup_files = sorted((WORKSPACE / "data").glob("backup_*.jsonl"))
return {
"sql": {
"total_engrams": total,
"confirmed": confirmed_true,
"rejected": confirmed_false,
"pending": total - confirmed_true - confirmed_false,
"by_source": sources,
},
"vector": {
"chroma_dir": str(chroma_dir),
"chroma_size_bytes": _dir_size_bytes(chroma_dir) if chroma_dir.exists() else 0,
"embedding_cache_dir": str(emb_cache_dir),
"embedding_cache_files": len(list(emb_cache_dir.glob("*.json"))) if emb_cache_dir.exists() else 0,
"vector_state": vec_state,
},
"obsidian": {
"config_path": str(obsidian_cfg_path),
"configured": bool(obsidian_cfg),
"config": obsidian_cfg,
},
"backups": {
"count": len(backup_files),
"latest": str(backup_files[-1]) if backup_files else None,
},
}
@app.get("/api/jobs")
def api_jobs():
# Known units that influence "freshness" of the brain.
units = [
"openclaw-secondbrain-ingest-memory.service",
"openclaw-secondbrain-index-vectors.service",
"openclaw-secondbrain-review.service",
"openclaw-secondbrain-heartbeat.service",
"openclaw-secondbrain-verify-pending.service",
]
return {"items": [_systemd_unit_state(u) for u in units]}
@app.get("/api/insights")
def api_insights(limit: int = Query(8, ge=1, le=50)):
conn = get_db()
c = conn.cursor()
rows = c.execute(
"SELECT id, metadata_json, correctness_json, created_at, modified_at FROM engrams ORDER BY created_at DESC LIMIT 2000"
).fetchall()
total = c.execute("SELECT COUNT(*) FROM engrams").fetchone()[0]
confirmed_true = c.execute(
"""
SELECT COUNT(*) FROM engrams
WHERE (
json_extract(correctness_json, '$.verdict') = 'confirmed_true'
OR (json_extract(correctness_json, '$.verdict') IS NULL AND json_extract(correctness_json, '$.confirmed') = 1)
)
"""
).fetchone()[0]
confirmed_false = c.execute(
"""
SELECT COUNT(*) FROM engrams
WHERE (
json_extract(correctness_json, '$.verdict') = 'confirmed_false'
OR (json_extract(correctness_json, '$.verdict') IS NULL
AND json_extract(correctness_json, '$.confirmed') = 0
AND COALESCE(json_extract(correctness_json, '$.rejections'), 0) > 0)
)
"""
).fetchone()[0]
pending = total - confirmed_true - confirmed_false
tag_counts: dict[str, int] = {}
source_counts: dict[str, int] = {}
host_counts: dict[str, int] = {}
active: list[dict] = []
forgotten: list[dict] = []
for r in rows:
try:
meta = json.loads(r["metadata_json"] or "{}")
except Exception:
meta = {}
src = meta.get("source", "unknown")
source_counts[src] = source_counts.get(src, 0) + 1
for t in (meta.get("tags") or []):
if isinstance(t, str):
tag_counts[t] = tag_counts.get(t, 0) + 1
try:
host = _host_from_meta(r["metadata_json"])
except Exception:
host = None
if host:
host_counts[host] = host_counts.get(host, 0) + 1
access_count = int(meta.get("access_count", 0) or 0)
created = meta.get("created", r["created_at"])
if access_count >= 5 and len(active) < limit:
active.append(
{
"id": r["id"],
"access_count": access_count,
"source": src,
"created": created,
}
)
if access_count == 0 and len(forgotten) < limit:
forgotten.append(
{
"id": r["id"],
"access_count": access_count,
"source": src,
"created": created,
}
)
def top_k(d: dict[str, int]) -> list[dict]:
return [
{"key": k, "count": v}
for k, v in sorted(d.items(), key=lambda kv: kv[1], reverse=True)[:limit]
]
conn.close()
return {
"total": total,
"confirmed": confirmed_true,
"rejected": confirmed_false,
"pending": pending,
"top_tags": top_k(tag_counts),
"top_sources": top_k(source_counts),
"top_hosts": top_k(host_counts),
"active_engrams": active,
"forgotten_engrams": forgotten,
}
@app.get("/api/graph")
def api_graph(
limit_nodes: int = Query(0, ge=0, le=50000),
limit_edges: int = Query(0, ge=0, le=200000),
):
"""
Returns a lightweight graph view:
- Nodes: engrams + tag:<tag> + host:<hostname>
- Edges: engram->tag and engram->host plus explicit engrams_links edges.
"""
conn = get_db()
c = conn.cursor()
if limit_nodes > 0:
# Fetch a bigger window than the final node cap so trim can keep hubs + neighbors.
engram_fetch = min(50000, max(1000, int(limit_nodes * 3)))
else:
engram_fetch = None
if limit_edges > 0:
link_fetch = limit_edges
elif limit_nodes > 0:
link_fetch = min(200000, max(2000, int(limit_nodes * 10)))
else:
link_fetch = None
if engram_fetch is None:
rows = c.execute("SELECT id, metadata_json, correctness_json, created_at, modified_at FROM engrams ORDER BY created_at DESC").fetchall()
else:
rows = c.execute(
"SELECT id, metadata_json, correctness_json, created_at, modified_at FROM engrams ORDER BY created_at DESC LIMIT ?",
(engram_fetch,),
).fetchall()
if link_fetch is None:
link_rows = c.execute("SELECT from_id, to_id FROM engrams_links ORDER BY rowid DESC").fetchall()
else:
link_rows = c.execute(
"SELECT from_id, to_id FROM engrams_links ORDER BY rowid DESC LIMIT ?",
(link_fetch,),
).fetchall()
conn.close()
nodes: dict[str, dict] = {}
edges: list[dict] = []
def add_node(nid: str, kind: str, label: str | None = None, weight: float | None = None):
if nid not in nodes:
nodes[nid] = {"id": nid, "kind": kind}
if label is not None:
nodes[nid]["label"] = label
if weight is not None:
nodes[nid]["weight"] = weight
for r in rows:
eid = r["id"]
try:
meta = json.loads(r["metadata_json"] or "{}")
except Exception:
meta = {}
try:
corr = json.loads(r["correctness_json"] or "{}")
except Exception:
corr = {}
verdict = corr.get("verdict")
if not isinstance(verdict, str) or not verdict:
if corr.get("confirmed", False):
verdict = "confirmed_true"
elif int(corr.get("rejections", 0) or 0) > 0:
verdict = "confirmed_false"
else:
verdict = "unknown"
add_node(
eid,
"engram",
label=eid[:8],
weight=float(meta.get("access_count", 0) or 0),
)
nodes[eid].update(
{
"source": meta.get("source", "unknown"),
"confidence": float(meta.get("confidence", 0.0) or 0.0),
"created": meta.get("created", r["created_at"]),
"modified": meta.get("modified", r["modified_at"]),
"last_accessed": meta.get("last_accessed"),
"verdict": verdict,
"confirmed": bool(corr.get("confirmed", False)),
"rejections": int(corr.get("rejections", 0) or 0),
"grounding": meta.get("grounding", 0),
"predict_locked": bool(meta.get("predict_locked", False)),
}
)
for t in _safe_json_extract_tags(r["metadata_json"]):
tid = f"tag:{t}"
add_node(tid, "tag", label=t)
edges.append({"from": eid, "to": tid, "kind": "has_tag", "weight": 0.35})
host = _host_from_meta(r["metadata_json"])
if host:
hid = f"host:{host}"
add_node(hid, "host", label=host)
edges.append({"from": eid, "to": hid, "kind": "grounded_at", "weight": 0.25})
for fr, to in link_rows:
add_node(fr, "engram")
add_node(to, "engram")
edges.append({"from": fr, "to": to, "kind": "link", "weight": 1.0})
# Trim nodes to keep payload bounded (prefer engrams and connected tags/hosts)
if limit_nodes > 0 and len(nodes) > limit_nodes:
# Keep a balanced subset: many engrams plus the most-connected non-engrams.
kept: dict[str, dict] = {}
engram_budget = int(limit_nodes * 0.7)
# 1) Keep newest engrams first (they appear first in `rows` loop insertion order)
for r in rows:
eid = r["id"]
if eid in nodes:
kept[eid] = nodes[eid]
if len(kept) >= engram_budget:
break
# 2) Rank remaining nodes by degree within current edge set
degree: dict[str, int] = {}
for e in edges:
degree[e["from"]] = degree.get(e["from"], 0) + 1
degree[e["to"]] = degree.get(e["to"], 0) + 1
remaining = [nid for nid in nodes.keys() if nid not in kept]
remaining.sort(key=lambda nid: degree.get(nid, 0), reverse=True)
for nid in remaining:
kept[nid] = nodes[nid]
if len(kept) >= limit_nodes:
break
nodes = kept
edges = [e for e in edges if e["from"] in nodes and e["to"] in nodes]
return {"nodes": list(nodes.values()), "edges": edges}
@app.get("/api/events")
def api_events():
"""
Server-Sent Events stream for lightweight real-time UI refresh.
"""
import time
def gen():
while True:
payload = {
"ts": datetime.now(timezone.utc).isoformat(),
"stats": api_stats(),
"storage": api_storage_stats(),
"jobs": api_jobs(),
"insights": api_insights(limit=8),
}
yield f"data: {json.dumps(payload, ensure_ascii=False)}\n\n"
time.sleep(5)
return StreamingResponse(gen(), media_type="text/event-stream")
@app.exception_handler(FileNotFoundError)
def handle_file_not_found(request: Request, exc: FileNotFoundError):
@@ -105,10 +628,27 @@ def api_stats():
conn = get_db()
c = conn.cursor()
total = c.execute("SELECT COUNT(*) FROM engrams").fetchone()[0]
confirmed = c.execute(
"SELECT COUNT(*) FROM engrams WHERE json_extract(correctness_json, '$.confirmed') = 1"
confirmed_true = c.execute(
"""
SELECT COUNT(*) FROM engrams
WHERE (
json_extract(correctness_json, '$.verdict') = 'confirmed_true'
OR (json_extract(correctness_json, '$.verdict') IS NULL AND json_extract(correctness_json, '$.confirmed') = 1)
)
"""
).fetchone()[0]
pending = total - confirmed
confirmed_false = c.execute(
"""
SELECT COUNT(*) FROM engrams
WHERE (
json_extract(correctness_json, '$.verdict') = 'confirmed_false'
OR (json_extract(correctness_json, '$.verdict') IS NULL
AND json_extract(correctness_json, '$.confirmed') = 0
AND COALESCE(json_extract(correctness_json, '$.rejections'), 0) > 0)
)
"""
).fetchone()[0]
pending = total - confirmed_true - confirmed_false
errors = c.execute(
"SELECT COUNT(*) FROM engrams WHERE json_extract(metadata_json, '$.tags') LIKE '%error%'"
).fetchone()[0]
@@ -118,7 +658,8 @@ def api_stats():
conn.close()
return {
"total": total,
"confirmed": confirmed,
"confirmed": confirmed_true,
"rejected": confirmed_false,
"pending": pending,
"errors": errors,
"avg_confidence": round(avg_conf, 2),
@@ -131,6 +672,7 @@ def api_engrams(
offset: int = Query(0, ge=0),
tag: str = Query(None),
confirmed: bool = Query(None),
verdict: str = Query(None),
search: str = Query(None),
min_confidence: float = Query(0.0),
):
@@ -144,9 +686,30 @@ def api_engrams(
params.append(f'%"{tag}"%')
if confirmed is not None:
where_clauses.append(
f"json_extract(correctness_json, '$.confirmed') = {int(confirmed)}"
)
if confirmed:
# confirmed == statement is true (verdict confirmed_true)
where_clauses.append(
"("
"json_extract(correctness_json, '$.verdict') = 'confirmed_true' "
"OR (json_extract(correctness_json, '$.verdict') IS NULL AND json_extract(correctness_json, '$.confirmed') = 1)"
")"
)
else:
# pending/unresolved (unknown/probable) but exclude confirmed_false.
where_clauses.append(
"("
"json_extract(correctness_json, '$.verdict') IN ('unknown','probable_true','probable_false') "
"OR (json_extract(correctness_json, '$.verdict') IS NULL "
" AND json_extract(correctness_json, '$.confirmed') = 0 "
" AND COALESCE(json_extract(correctness_json, '$.rejections'), 0) = 0)"
")"
)
if verdict:
v = verdict.strip()
if v in ("unknown", "probable_true", "probable_false", "confirmed_true", "confirmed_false"):
where_clauses.append("json_extract(correctness_json, '$.verdict') = ?")
params.append(v)
if search:
# Use FTS
@@ -204,70 +767,107 @@ def api_engram_detail(engram_id: str):
return result
@app.post("/api/engrams/{engram_id}/confirm")
def api_confirm(engram_id: str, reason: str = Form("")):
@app.get("/api/pending")
def api_pending(
limit: int = Query(20, ge=1, le=200),
offset: int = Query(0, ge=0),
source: str | None = Query(None),
):
conn = get_db()
c = conn.cursor()
row = c.execute(
"SELECT correctness_json FROM engrams WHERE id = ?", (engram_id,)
).fetchone()
if not row:
conn.close()
return JSONResponse({"error": "Not found"}, status_code=404)
where = ["json_extract(correctness_json, '$.confirmed') = 0"]
params: list = []
if source:
where.append("json_extract(metadata_json, '$.source') = ?")
params.append(source)
correctness = json.loads(row["correctness_json"] or "{}")
correctness["confirmed"] = True
correctness["confirmations"] = correctness.get("confirmations", 0) + 1
correctness["last_reviewed"] = datetime.now(timezone.utc).isoformat()
review_history = correctness.get("review_history", [])
review_history.append({
"by": "web",
"action": "confirm",
"at": datetime.now(timezone.utc).isoformat(),
"note": reason or "confirmed via dashboard",
})
correctness["review_history"] = review_history
rows = c.execute(
f"""
SELECT * FROM engrams
WHERE {' AND '.join(where)}
ORDER BY created_at DESC
LIMIT ? OFFSET ?
""",
params + [limit, offset],
).fetchall()
items = [parse_engram(r) for r in rows]
conn.close()
return {"items": items, "limit": limit, "offset": offset}
@app.post("/api/engrams")
def api_create_engram(content: str = Form(...), tags: str = Form("")):
content = (content or "").strip()
if not content:
return JSONResponse({"error": "content required"}, status_code=400)
tag_list = [t.strip() for t in (tags or "").split(",") if t.strip()]
now = _now_iso()
engram_id = str(uuid4())
meta = {
"source": "user",
"confidence": 0.7,
"created": now,
"modified": now,
"access_count": 0,
"last_accessed": now,
"tags": tag_list,
"session_id": None,
"agent_id": None,
"grounding": 0,
}
corr = {
"confirmed": False,
"confirmations": 0,
"rejections": 0,
"last_reviewed": None,
"review_history": [],
}
conn = get_db()
c = conn.cursor()
c.execute(
"UPDATE engrams SET correctness_json = ?, modified_at = ? WHERE id = ?",
(json.dumps(correctness), datetime.now(timezone.utc).isoformat(), engram_id),
"""
INSERT INTO engrams (id, content, metadata_json, correctness_json, links_json, hierarchy_json, embedding_json, created_at, modified_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
""",
(
engram_id,
content,
json.dumps(meta, ensure_ascii=False),
json.dumps(corr, ensure_ascii=False),
"[]",
"{}",
None,
now,
now,
),
)
conn.commit()
conn.close()
return {"success": True, "engram_id": engram_id}
return {"id": engram_id}
@app.post("/api/engrams/{engram_id}/confirm")
def api_confirm_engram(engram_id: str, reason: str = Form("")):
try:
return _update_correctness(engram_id, action="confirm", reason=reason)
except FileNotFoundError as e:
return JSONResponse({"error": str(e)}, status_code=404)
@app.post("/api/engrams/{engram_id}/reject")
def api_reject(engram_id: str, reason: str = Form("")):
conn = get_db()
c = conn.cursor()
row = c.execute(
"SELECT correctness_json FROM engrams WHERE id = ?", (engram_id,)
).fetchone()
if not row:
conn.close()
return JSONResponse({"error": "Not found"}, status_code=404)
def api_reject_engram(engram_id: str, reason: str = Form("")):
try:
return _update_correctness(engram_id, action="reject", reason=reason)
except FileNotFoundError as e:
return JSONResponse({"error": str(e)}, status_code=404)
correctness = json.loads(row["correctness_json"] or "{}")
correctness["confirmed"] = False
correctness["rejections"] = correctness.get("rejections", 0) + 1
correctness["last_reviewed"] = datetime.now(timezone.utc).isoformat()
review_history = correctness.get("review_history", [])
review_history.append({
"by": "web",
"action": "reject",
"at": datetime.now(timezone.utc).isoformat(),
"note": reason or "rejected via dashboard",
})
correctness["review_history"] = review_history
c.execute(
"UPDATE engrams SET correctness_json = ?, modified_at = ? WHERE id = ?",
(json.dumps(correctness), datetime.now(timezone.utc).isoformat(), engram_id),
)
conn.commit()
conn.close()
return {"success": True, "engram_id": engram_id}
@app.post("/api/engrams/{engram_id}/refresh")
def api_refresh_engram(engram_id: str):
try:
return _bump_access(engram_id)
except FileNotFoundError as e:
return JSONResponse({"error": str(e)}, status_code=404)
@app.post("/api/engrams/{engram_id}/refresh")
@@ -320,6 +920,8 @@ def api_create_engram(content: str = Form(...), tags: str = Form(""), source: st
"hash": "",
}
correctness = {
"verdict": "unknown",
"evidence": [],
"confirmed": False,
"confirmations": 0,
"rejections": 0,
@@ -347,7 +949,12 @@ def api_pending(limit: int = Query(20, ge=1, le=100), offset: int = Query(0, ge=
rows = c.execute(
"""
SELECT * FROM engrams
WHERE json_extract(correctness_json, '$.confirmed') = 0
WHERE (
json_extract(correctness_json, '$.verdict') IN ('unknown','probable_true','probable_false')
OR (json_extract(correctness_json, '$.verdict') IS NULL
AND json_extract(correctness_json, '$.confirmed') = 0
AND COALESCE(json_extract(correctness_json, '$.rejections'), 0) = 0)
)
ORDER BY created_at DESC
LIMIT ? OFFSET ?
""",

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,191 @@
#!/usr/bin/env python3
import argparse
import json
import hashlib
from datetime import datetime, timezone
from pathlib import Path
from typing import Any, Dict, Iterable, List, Optional, Tuple
from src.engram import Engram, Grounding
from src.store import EngramStore
def _now_utc_iso() -> str:
return datetime.now(timezone.utc).isoformat()
def _hash16(text: str) -> str:
return hashlib.sha256(text.encode("utf-8")).hexdigest()[:16]
def _iter_jsonl(path: Path) -> Iterable[Dict[str, Any]]:
with path.open("r", encoding="utf-8") as f:
for line_no, line in enumerate(f, start=1):
line = line.strip()
if not line:
continue
try:
obj = json.loads(line)
except Exception:
raise SystemExit(f"Invalid JSON at {path}:{line_no}")
if not isinstance(obj, dict):
continue
yield obj
def _marker_to_content(marker_obj: Dict[str, Any]) -> Tuple[str, List[Dict[str, Any]]]:
marker = str(marker_obj.get("marker", "")).strip()
details = str(marker_obj.get("details", "")).strip()
checks = marker_obj.get("checks") or []
sources = marker_obj.get("sources") or []
if not marker:
raise ValueError("missing marker")
evidence: List[Dict[str, Any]] = []
for src in sources:
if not isinstance(src, dict):
continue
url = (src.get("url") or "").strip()
title = (src.get("title") or "").strip()
if not url:
continue
evidence.append({"url": url, "title": title})
lines: List[str] = []
lines.append(f"WEBDEV_MARKER: {marker}")
if details:
lines.append("")
lines.append(f"Details: {details}")
if isinstance(checks, list) and checks:
lines.append("")
lines.append("Checks:")
for c in checks[:8]:
c = str(c).strip()
if c:
lines.append(f"- {c}")
if evidence:
lines.append("")
lines.append("Sources:")
for ev in evidence[:12]:
title = (ev.get("title") or "").strip()
url = (ev.get("url") or "").strip()
if title:
lines.append(f"- {title}: {url}")
else:
lines.append(f"- {url}")
return "\n".join(lines).strip(), evidence
def _tags_for(marker_obj: Dict[str, Any]) -> List[str]:
tags = ["web_design", "web_development", "mobile"]
area = str(marker_obj.get("area", "")).strip()
if area:
tags.append(area)
return tags
def import_markers(
db_path: Path,
jsonl_paths: List[Path],
source: str,
verdict: str,
agent_id: str,
dry_run: bool,
) -> Dict[str, int]:
store = EngramStore(str(db_path))
stats = {"seen": 0, "imported": 0, "skipped_dup": 0, "skipped_invalid": 0}
seen_hashes: set[str] = set()
# Preload existing hashes (fast-ish; avoids duplicate spam).
existing_hashes: set[str] = set()
try:
cur = store._conn.execute("SELECT metadata_json FROM engrams") # noqa: SLF001
for row in cur.fetchall():
try:
meta = json.loads(row["metadata_json"])
h = meta.get("hash")
if isinstance(h, str) and h:
existing_hashes.add(h)
except Exception:
continue
except Exception:
# If this fails (schema mismatch), proceed without preload.
existing_hashes = set()
for path in jsonl_paths:
for marker_obj in _iter_jsonl(path):
if (marker_obj.get("kind") or "") != "web_design_marker":
continue
stats["seen"] += 1
try:
content, evidence = _marker_to_content(marker_obj)
except Exception:
stats["skipped_invalid"] += 1
continue
h = _hash16(content)
if h in seen_hashes or h in existing_hashes:
stats["skipped_dup"] += 1
continue
seen_hashes.add(h)
eg = Engram.create(
content=content,
source=source,
confidence=0.75,
tags=_tags_for(marker_obj),
session_id=None,
agent_id=agent_id or str(marker_obj.get("agent_id") or ""),
grounding=Grounding.SOURCED,
)
# Overwrite hash to exactly match our content representation.
eg.metadata["hash"] = h
eg.metadata["modified"] = _now_utc_iso()
eg.metadata["created"] = marker_obj.get("created_at") or eg.metadata["created"]
eg.correctness.set_verdict(
by=agent_id or "importer",
verdict=verdict,
note=f"Imported from {path.name}",
evidence=evidence,
)
if not dry_run:
store.save(eg)
stats["imported"] += 1
return stats
def main() -> None:
p = argparse.ArgumentParser(description="Import web_design_marker JSONL files into brain.sqlite")
p.add_argument("--db", default="second-brain/data/brain.sqlite", help="Path to brain.sqlite")
p.add_argument("--glob", default="/tmp/web_design_markers_*.jsonl", help="Glob for marker JSONL files")
p.add_argument("--source", default="web_research", help="Engram source")
p.add_argument("--verdict", default="probable_true", help="Correctness verdict")
p.add_argument("--agent-id", default="web_research_import", help="Agent id to record")
p.add_argument("--dry-run", action="store_true", help="Parse/dedupe but do not write to DB")
args = p.parse_args()
db_path = Path(args.db)
jsonl_paths = sorted(Path("/").glob(args.glob.lstrip("/"))) if args.glob.startswith("/") else sorted(Path(".").glob(args.glob))
if not jsonl_paths:
raise SystemExit(f"No files match glob: {args.glob}")
stats = import_markers(
db_path=db_path,
jsonl_paths=jsonl_paths,
source=args.source,
verdict=args.verdict,
agent_id=args.agent_id,
dry_run=bool(args.dry_run),
)
print(json.dumps({"db": str(db_path), "files": [str(p) for p in jsonl_paths], "stats": stats}, ensure_ascii=False, indent=2))
if __name__ == "__main__":
main()

View File

@@ -0,0 +1,163 @@
#!/usr/bin/env python3
"""
Process pending second brain engrams.
- For unconfirmed, unrejected engrams: evaluate confidence
- If confidence > 0.8: confirm
- If confidence < 0.3: reject
- Otherwise: mark for review (leave as is)
- Check for stale topics and archive if needed
- Produce summary report
"""
import sys
import json
from datetime import datetime, timezone
from pathlib import Path
# Add src to path and set PYTHONPATH for proper module resolution
base_dir = Path(__file__).parent.parent
sys.path.insert(0, str(base_dir / "src"))
# Import using absolute module paths
from src.store import EngramStore
from src.engram import Engram, Grounding
DB_PATH = Path(__file__).parent.parent / "data" / "brain.sqlite"
def is_stale(engram: Engram, days_threshold: int = 90) -> bool:
"""Check if an engram is stale (old and rarely accessed)."""
created = engram.metadata.get("created", "")
access_count = engram.metadata.get("access_count", 0)
last_accessed = engram.metadata.get("last_accessed", created)
try:
created_dt = datetime.fromisoformat(created.replace("Z", "+00:00"))
last_accessed_dt = datetime.fromisoformat(last_accessed.replace("Z", "+00:00"))
age_days = (datetime.now(timezone.utc) - created_dt).total_seconds() / 86400
days_since_access = (datetime.now(timezone.utc) - last_accessed_dt).total_seconds() / 86400
# Stale if: old (>90 days) AND rarely accessed (<3 times) AND not accessed recently (>60 days)
if age_days > days_threshold and access_count < 3 and days_since_access > 60:
return True
except Exception:
pass
return False
def process_pending_engrams():
"""Main processing function."""
store = EngramStore(str(DB_PATH))
# Get all engrams
all_engrams = store.get_all(limit=10000)
print(f"Total engrams in database: {len(all_engrams)}")
# Filter pending (unconfirmed and unrejected)
# Unconfirmed: not confirmed_true, not confirmed_false
pending = []
for eg in all_engrams:
verdict = eg.correctness.verdict
if verdict not in ("confirmed_true", "confirmed_false"):
pending.append(eg)
print(f"Pending engrams (unconfirmed/unrejected): {len(pending)}")
actions = {
"confirmed": 0,
"rejected": 0,
"left_for_review": 0,
"archived_stale": 0,
"errors": 0
}
details = []
for eg in pending:
try:
confidence = eg.compute_confidence()
engram_id = str(eg.id)
content_preview = eg.content[:80] + ("..." if len(eg.content) > 80 else "")
# Check if stale and should be archived
if is_stale(eg):
# For stale engrams, we'll mark them in metadata for archiving
# Instead of deleting, we'll add an "archived" tag and lower their priority
tags = eg.metadata.get("tags", [])
if "archived" not in tags:
tags.append("archived")
eg.metadata["tags"] = tags
eg.metadata["archived_at"] = datetime.now(timezone.utc).isoformat()
store.save(eg)
actions["archived_stale"] += 1
details.append(f"📦 Archived stale: [{engram_id[:8]}] {content_preview} (conf: {confidence:.2f})")
# Even if stale, we still evaluate confidence for reporting
# But we don't confirm/reject stale ones automatically unless confidence is extreme
# Actually, the task says to check for stale topics and archive if needed. We've done that.
# We still need to apply confidence thresholds to non-stale or all pending?
# Let's continue to evaluate all pending, including stale, but maybe skip confirm/reject for stale?
# The task: "For each pending engram... evaluate... If >0.8 confirm, <0.3 reject, otherwise mark for review"
# It doesn't say to skip stale ones. So we'll still apply thresholds.
# But we already archived it. We can still confirm/reject it if confidence is extreme.
# Let's continue.
# Apply confidence thresholds
if confidence > 0.8:
eg.correctness.confirm(by="auto_processor", note=f"Auto-confirmed: confidence {confidence:.2f}")
store.save(eg)
actions["confirmed"] += 1
details.append(f"✅ Confirmed: [{engram_id[:8]}] {content_preview} (conf: {confidence:.2f})")
elif confidence < 0.3:
eg.correctness.reject(by="auto_processor", note=f"Auto-rejected: confidence {confidence:.2f}")
store.save(eg)
actions["rejected"] += 1
details.append(f"❌ Rejected: [{engram_id[:8]}] {content_preview} (conf: {confidence:.2f})")
else:
actions["left_for_review"] += 1
details.append(f"⏳ Review later: [{engram_id[:8]}] {content_preview} (conf: {confidence:.2f})")
except Exception as e:
actions["errors"] += 1
details.append(f"⚠️ Error processing engram: {str(e)}")
# Generate summary report
report_lines = []
report_lines.append("=" * 60)
report_lines.append("PENDING ENGRAMS PROCESSING REPORT")
report_lines.append("=" * 60)
report_lines.append(f"Timestamp: {datetime.now(timezone.utc).isoformat()}")
report_lines.append(f"Total engrams: {len(all_engrams)}")
report_lines.append(f"Pending engrams processed: {len(pending)}")
report_lines.append("")
report_lines.append("ACTIONS TAKEN:")
report_lines.append(f" ✅ Auto-confirmed (confidence > 0.8): {actions['confirmed']}")
report_lines.append(f" ❌ Auto-rejected (confidence < 0.3): {actions['rejected']}")
report_lines.append(f" ⏳ Left for review (0.3 ≤ confidence ≤ 0.8): {actions['left_for_review']}")
report_lines.append(f" 📦 Archived stale topics: {actions['archived_stale']}")
report_lines.append(f" ⚠️ Errors: {actions['errors']}")
report_lines.append("")
report_lines.append("DETAILS:")
report_lines.extend(details)
report_lines.append("")
report_lines.append("=" * 60)
report = "\n".join(report_lines)
# Print to stdout
print("\n" + report)
# Save report to file
report_dir = Path(__file__).parent.parent / "reports"
report_dir.mkdir(parents=True, exist_ok=True)
report_file = report_dir / f"pending_engrams_{datetime.now().strftime('%Y%m%d_%H%M%S')}.txt"
report_file.write_text(report, encoding="utf-8")
print(f"\n📄 Report saved to: {report_file}")
store.close()
return actions
if __name__ == "__main__":
result = process_pending_engrams()
print("\nProcessing complete.")

20
scripts/smoke_dashboard.sh Executable file
View File

@@ -0,0 +1,20 @@
#!/usr/bin/env bash
set -euo pipefail
HOST="${SECOND_BRAIN_HOST:-127.0.0.1}"
PORT="${SECOND_BRAIN_PORT:-${PORT:-8501}}"
BASE_URL="http://${HOST}:${PORT}"
if command -v systemctl >/dev/null 2>&1; then
if ! systemctl is-active --quiet openclaw-secondbrain-dashboard.service; then
echo "ERROR: openclaw-secondbrain-dashboard.service is not active" >&2
exit 1
fi
fi
curl -fsS "${BASE_URL}/healthz" >/dev/null
stats_json="$(curl -fsS "${BASE_URL}/api/stats")"
python3 -c 'import json,sys; json.load(sys.stdin)' <<<"$stats_json"
echo "OK: dashboard smoke test passed (${BASE_URL})"

View File

@@ -40,26 +40,60 @@ class ReviewEntry:
@dataclass
class Correctness:
"""Verfolgt die Korrektheit eines Engramms über Zeit."""
# verdict model (not only binary confirm/reject)
# Values:
# - unknown
# - probable_true / probable_false
# - confirmed_true / confirmed_false
verdict: str = "unknown"
evidence: List[Dict[str, Any]] = field(default_factory=list)
confirmed: bool = False
confirmations: int = 0
rejections: int = 0
last_reviewed: Optional[str] = None
review_history: List[ReviewEntry] = field(default_factory=list)
def is_final(self) -> bool:
return self.verdict in ("confirmed_true", "confirmed_false")
def set_verdict(self, by: str, verdict: str, note: str = "", evidence: Optional[List[Dict[str, Any]]] = None) -> None:
verdict = (verdict or "").strip()
if verdict not in ("unknown", "probable_true", "probable_false", "confirmed_true", "confirmed_false"):
verdict = "unknown"
self.verdict = verdict
# Keep backward-compatible boolean in sync:
# historically, confirmed=True meant "this statement is correct".
self.confirmed = verdict == "confirmed_true"
self.last_reviewed = _now()
if evidence:
try:
self.evidence.extend([e for e in evidence if isinstance(e, dict)])
except Exception:
pass
self.review_history.append(ReviewEntry(by, "set_verdict", self.last_reviewed, f"{verdict}: {note}".strip()))
def confirm(self, by: str, note: str = "") -> None:
self.confirmations += 1
self.confirmed = True
self.last_reviewed = _now()
self.set_verdict(by, "confirmed_true", note)
# Preserve historic action tag too
self.review_history.append(ReviewEntry(by, "confirm", self.last_reviewed, note))
def reject(self, by: str, note: str = "") -> None:
self.rejections += 1
self.confirmed = False
self.last_reviewed = _now()
self.set_verdict(by, "confirmed_false", note)
self.review_history.append(ReviewEntry(by, "reject", self.last_reviewed, note))
def score(self) -> float:
"""Confidence-Score aus Korrekturhistorie."""
# verdict-first scoring (explicit, non-binary)
if self.verdict == "confirmed_true":
return 1.0
if self.verdict == "confirmed_false":
return 0.0
if self.verdict == "probable_true":
return 0.75
if self.verdict == "probable_false":
return 0.25
total = self.confirmations + self.rejections
if total == 0:
return 0.5 # Unbestimmt
@@ -74,6 +108,8 @@ class Correctness:
else:
review_history.append(entry.to_dict())
return {
"verdict": self.verdict,
"evidence": self.evidence,
"confirmed": self.confirmed,
"confirmations": self.confirmations,
"rejections": self.rejections,
@@ -84,11 +120,30 @@ class Correctness:
@classmethod
def from_dict(cls, d: dict) -> "Correctness":
c = cls()
verdict = d.get("verdict")
if isinstance(verdict, str) and verdict.strip():
c.verdict = verdict.strip()
c.confirmed = d.get("confirmed", False)
c.confirmations = d.get("confirmations", 0)
c.rejections = d.get("rejections", 0)
c.last_reviewed = d.get("last_reviewed")
ev = d.get("evidence", [])
if isinstance(ev, list):
c.evidence = [e for e in ev if isinstance(e, dict)]
c.review_history = [ReviewEntry.from_dict(r) for r in d.get("review_history", [])]
# Backfill verdict if missing/invalid.
if c.verdict not in ("unknown", "probable_true", "probable_false", "confirmed_true", "confirmed_false"):
if c.confirmed:
c.verdict = "confirmed_true"
elif c.rejections > 0:
c.verdict = "confirmed_false"
else:
c.verdict = "unknown"
# Ensure boolean stays consistent for older mixed data.
if c.verdict == "confirmed_true":
c.confirmed = True
elif c.verdict == "confirmed_false":
c.confirmed = False
return c

View File

@@ -134,11 +134,19 @@ def _node_size(access_count: int) -> float:
def generate_graph_html(store: EngramStore, output_path: str) -> str:
"""Generiert interaktive HTML-Graph-Visualisierung."""
engrams = store.get_all()
# store.get_all() defaults to 1000; paginate so the graph can include all nodes.
engrams = []
offset = 0
while True:
batch = store.get_all(limit=2000, offset=offset)
if not batch:
break
engrams.extend(batch)
offset += len(batch)
nodes = []
edges = []
node_ids = set()
node_ids = set(str(e.id) for e in engrams)
for eg in engrams:
eid = str(eg.id)
@@ -156,22 +164,19 @@ def generate_graph_html(store: EngramStore, output_path: str) -> str:
"size": size,
"confidence": conf,
"confirmed": eg.correctness.confirmed,
"verdict": getattr(eg.correctness, "verdict", "unknown"),
"source": eg.metadata.get("source", "?"),
"tags": tags,
}
})
node_ids.add(eid)
# Add edges after all nodes are known (otherwise early nodes miss links).
for eg in engrams:
eid = str(eg.id)
for lid in eg.links:
lid_s = str(lid)
if lid_s in node_ids:
edges.append({
"data": {
"id": f"{eid}_{lid_s}",
"source": eid,
"target": lid_s,
}
})
edges.append({"data": {"id": f"{eid}_{lid_s}", "source": eid, "target": lid_s}})
elements = {"nodes": nodes, "edges": edges}
html = _HTML_TEMPLATE.format(elements_json=json.dumps(elements, ensure_ascii=False))

View File

@@ -12,6 +12,7 @@ import os
import sys
import json
import traceback
import re
from pathlib import Path
from datetime import datetime, timezone
from typing import Optional, List, Dict, Any
@@ -48,6 +49,45 @@ except Exception:
# --- Konfiguration ---
BRAIN_DB = Path(__file__).parent.parent / "data" / "brain.sqlite"
_UUID_RE = re.compile(r"\b[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\b", re.I)
_SHORT_ID_RE = re.compile(r"\b[0-9a-f]{8}\b", re.I)
def _detect_feedback(content: str) -> dict | None:
"""
Heuristik: erkennt kurze Korrektur-/Feedback-Nachrichten in Chats.
Returns:
{"kind":"confirm"|"reject"|"stop", "target": <id or short-id or None>, "raw": <normalized>}
"""
if not isinstance(content, str):
return None
raw = content.strip()
if not raw:
return None
norm = raw.lower().strip()
target = None
m = _UUID_RE.search(raw)
if m:
target = m.group(0)
else:
m2 = _SHORT_ID_RE.search(raw)
if m2:
target = m2.group(0)
if norm in {"stop", "stopp", "halt"}:
return {"kind": "stop", "target": target, "raw": norm}
if norm in {"nein", "no", "falsch", "wrong"}:
return {"kind": "reject", "target": target, "raw": norm}
if norm in {"ja", "yes", "richtig", "korrekt", "stimmt"}:
return {"kind": "confirm", "target": target, "raw": norm}
if norm.startswith(("korrigiert", "korrektur", "correction")):
if "richtig" in norm or "korrekt" in norm:
return {"kind": "confirm", "target": target, "raw": norm}
return {"kind": "reject", "target": target, "raw": norm}
return None
def get_brain() -> EngramStore:
"""Gibt initialisierten Brain-Store."""
@@ -77,14 +117,43 @@ def save_session_learned(
)
"""
store = get_brain()
tags = tags or []
fb = _detect_feedback(content) if source == "session" else None
fb_target_id: str | None = None
if fb:
tags = list(dict.fromkeys(tags + ["feedback", fb["kind"]]))
confidence = min(confidence, 0.2)
grounding = Grounding.ASSUMPTION
if session_id:
recent = store.get_latest_by_session_id(session_id, limit=10, exclude_tags=["feedback"])
if recent:
fb_target_id = str(recent[0].id)
eg = Engram.create(
content=content,
source=source,
tags=tags or [],
tags=tags,
session_id=session_id,
confidence=confidence,
grounding=grounding,
)
if fb and fb_target_id:
target = store.get(fb_target_id)
if target:
try:
# Link both ways for graphing/traceability
eg.links.append(target.id)
if eg.id not in target.links:
target.links.append(eg.id)
# Lock target so auto-review does not keep "re-deciding" after a correction signal.
target.metadata["predict_locked"] = True
target.metadata["predict_locked_reason"] = f"feedback:{fb['raw']}"
target.metadata["predict_locked_at"] = datetime.now(timezone.utc).isoformat()
store.save(target)
except Exception:
pass
store.save(eg)
return eg

View File

@@ -6,6 +6,7 @@ Keine externen Abhängigkeiten außer sqlite3 (stdlib).
import json
import sqlite3
import os
import re
from pathlib import Path
from typing import List, Optional, Dict, Any
from uuid import UUID
@@ -135,6 +136,32 @@ class EngramStore:
).fetchall()
return [self._row_to_engram(r) for r in rows]
def get_latest_by_session_id(
self,
session_id: str,
*,
limit: int = 5,
exclude_tags: Optional[List[str]] = None,
) -> List[Engram]:
"""
Lädt die neuesten Engramme für eine OpenClaw-Session-ID.
Hinweis: session_id liegt im `metadata_json`; wir nutzen eine robuste
LIKE-Suche, damit auch Legacy-Records gefunden werden.
"""
if not session_id:
return []
rows = self._conn.execute(
"SELECT * FROM engrams WHERE metadata_json LIKE ? ORDER BY created_at DESC LIMIT ?",
(f'%"session_id": "{session_id}"%', limit),
).fetchall()
engrams = [self._row_to_engram(r) for r in rows]
if exclude_tags:
ex = set(t for t in exclude_tags if isinstance(t, str))
if ex:
engrams = [e for e in engrams if not (ex & set(e.metadata.get("tags", []) or []))]
return engrams
def delete(self, engram_id: str) -> bool:
"""Löscht ein Engramm und alle Verknüpfungen."""
rowid = self._conn.execute(
@@ -158,7 +185,12 @@ class EngramStore:
def search_text(self, query: str, limit: int = 10) -> List[Engram]:
"""Full-Text-Suche über Engramm-Inhalt via SQLite FTS5 (OR-Verknüpfung)."""
# FTS5-Syntax: Wörter mit OR verbinden für bessere Ergebnisse
words = [w.strip() for w in query.replace("'", "''").split() if w.strip()]
words = []
for word in query.split():
# Nur alphanumerische Zeichen als FTS5-Tokens akzeptieren
clean_word = re.sub(r'[^a-zA-Z0-9]+', '', word)
if clean_word:
words.append(clean_word)
safe_query = " OR ".join(words) if len(words) > 1 else (words[0] if words else "*")
sql = """
SELECT e.* FROM engrams e

View File

@@ -27,6 +27,31 @@ body {
top: 0;
z-index: 50;
}
.tabs-bar{
display:flex;
gap:8px;
padding:8px 12px 10px;
background:#141419;
border-bottom:1px solid #252530;
position: sticky;
top: 52px;
z-index: 45;
}
.tabs-bar .tab-btn{
flex:1;
background:#1e1e28;
border:1px solid #2a2a3a;
border-radius: 12px;
padding:10px 10px;
color:#cfd3ff;
font-weight:700;
font-size:0.82rem;
}
.tabs-bar .tab-btn.active{
border-color:#6c8af5;
box-shadow:0 0 0 1px rgba(108,138,245,0.22) inset;
}
.stat {
text-align: center;
min-width: 60px;
@@ -54,6 +79,122 @@ body {
padding: 10px 12px;
background: #141419;
}
/* tab buttons styled via .tabs-bar */
/* ─── Panels (Graph/Status) ──────────────────────────────────────────────── */
.panel {
margin: 8px 12px;
background: #1a1a24;
border: 1px solid #252533;
border-radius: 14px;
padding: 10px 12px;
}
.panel-title {
font-size: 0.8rem;
text-transform: uppercase;
letter-spacing: 0.5px;
color: #9aa3d9;
margin-bottom: 8px;
}
.kv-row {
display: flex;
gap: 10px;
padding: 6px 0;
border-bottom: 1px solid #20202a;
}
.kv-row:last-child { border-bottom: none; }
.kv-key {
width: 110px;
color: #888899;
font-size: 0.78rem;
}
.kv-val {
flex: 1;
color: #e8e8ee;
font-size: 0.85rem;
word-break: break-word;
}
.pill {
display: inline-block;
margin: 2px 4px 2px 0;
padding: 2px 8px;
border-radius: 999px;
background: #2a2a3a;
color: #8a9aff;
font-size: 0.72rem;
}
.verdict-pill{
display:inline-block;
margin: 2px 6px 2px 0;
padding: 2px 8px;
border-radius: 999px;
font-size: 0.72rem;
font-weight: 800;
letter-spacing: 0.4px;
border: 1px solid #2a2a3a;
background: #1e1e28;
color: #cfd3ff;
}
.verdict-pill.v-true{ border-color:#2f6b3f; color:#aaf0b6; }
.verdict-pill.v-false{ border-color:#7a2c2c; color:#ffb3b3; }
.verdict-pill.v-prob-true{ border-color:#6c8af5; color:#cfd9ff; }
.verdict-pill.v-prob-false{ border-color:#b08a2a; color:#ffe2a3; }
.verdict-pill.v-unknown{ border-color:#3a3a55; color:#b9b9c9; }
.muted {
color: #888899;
font-size: 0.8rem;
margin-top: 6px;
}
.small { font-size: 0.75rem; }
/* Graph canvas */
#graphCanvas{
display:block;
margin: 8px auto 0;
background:#12121a;
border:1px solid #252533;
border-radius: 14px;
touch-action: none;
}
.graph-controls{
display:flex;
gap:8px;
padding: 10px 12px 0;
align-items:center;
flex-wrap: wrap;
}
.graph-controls .btn{
background:#1e1e28;
border:1px solid #2a2a3a;
border-radius: 10px;
padding: 8px 10px;
color:#cfd3ff;
font-weight:700;
font-size:0.82rem;
}
.graph-controls .btn.primary{
border-color:#6c8af5;
box-shadow:0 0 0 1px rgba(108,138,245,0.18) inset;
}
.graph-legend{
margin: 8px 12px 0;
padding: 10px 12px;
background:#1a1a24;
border:1px solid #252533;
border-radius: 14px;
color:#b9b9c9;
font-size:0.8rem;
line-height:1.4;
}
.legend-row{ display:flex; align-items:center; gap:8px; margin-top:6px; }
.legend-dot{ width:10px; height:10px; border-radius:50%; display:inline-block; }
.legend-dot.engram{ background:#6c8af5; }
.legend-dot.tag{ background:#8a9aff; }
.legend-dot.match{ background:#f7d154; }
.graph-hint{ padding: 4px 12px 10px; }
#searchInput {
flex: 1;
background: #1e1e28;

View File

@@ -0,0 +1,9 @@
[Unit]
Description=OpenClaw archive memory/*.md older than 7 days
OnFailure=openclaw-secondbrain-notify@%n.service
[Service]
Type=oneshot
WorkingDirectory=/root/.openclaw/workspace
ExecStart=/bin/bash -lc 'flock -n /tmp/%n.lock /usr/bin/python3 /root/.openclaw/workspace/openclaw_cron_wrapper.py archive_memory_md'

View File

@@ -0,0 +1,10 @@
[Unit]
Description=OpenClaw archive memory/*.md daily
[Timer]
OnCalendar=*-*-* 03:30:00
Persistent=true
[Install]
WantedBy=timers.target

View File

@@ -0,0 +1,8 @@
[Unit]
Description=OpenClaw Second-Brain backup_secondbrain
OnFailure=openclaw-secondbrain-notify@%n.service
[Service]
Type=oneshot
WorkingDirectory=/root/.openclaw/workspace
ExecStart=/bin/bash -lc 'flock -n /tmp/%n.lock /usr/bin/python3 /root/.openclaw/workspace/openclaw_cron_wrapper.py backup_secondbrain'

View File

@@ -0,0 +1,10 @@
[Unit]
Description=OpenClaw Second-Brain backup_secondbrain (daily 02:00)
[Timer]
OnCalendar=*-*-* 02:00:00
Persistent=true
[Install]
WantedBy=timers.target

View File

@@ -0,0 +1,15 @@
[Unit]
Description=OpenClaw Second-Brain Dashboard (FastAPI)
After=network.target
[Service]
Type=simple
WorkingDirectory=/root/.openclaw/workspace/second-brain
Environment=SECOND_BRAIN_WORKSPACE=/root/.openclaw/workspace/second-brain
Environment=SECOND_BRAIN_PORT=8501
ExecStart=/root/.openclaw/workspace/second-brain/.venv/bin/uvicorn fastapi_app:app --host 0.0.0.0 --port 8501
Restart=always
RestartSec=2
[Install]
WantedBy=multi-user.target

View File

@@ -0,0 +1,8 @@
[Unit]
Description=OpenClaw Second-Brain export_obsidian
OnFailure=openclaw-secondbrain-notify@%n.service
[Service]
Type=oneshot
WorkingDirectory=/root/.openclaw/workspace
ExecStart=/bin/bash -lc 'flock -n /tmp/%n.lock /usr/bin/python3 /root/.openclaw/workspace/openclaw_cron_wrapper.py export_obsidian'

View File

@@ -0,0 +1,10 @@
[Unit]
Description=OpenClaw Second-Brain export_obsidian (hourly)
[Timer]
OnCalendar=hourly
Persistent=true
[Install]
WantedBy=timers.target

View File

@@ -0,0 +1,8 @@
[Unit]
Description=OpenClaw Second-Brain heartbeat_secondbrain
OnFailure=openclaw-secondbrain-notify@%n.service
[Service]
Type=oneshot
WorkingDirectory=/root/.openclaw/workspace
ExecStart=/bin/bash -lc 'flock -n /tmp/%n.lock /usr/bin/python3 /root/.openclaw/workspace/openclaw_cron_wrapper.py heartbeat_secondbrain'

View File

@@ -0,0 +1,11 @@
[Unit]
Description=OpenClaw Second-Brain heartbeat_secondbrain (every 6h)
[Timer]
OnCalendar=*-*-* 00,06,12,18:00:00
RandomizedDelaySec=300
Persistent=true
[Install]
WantedBy=timers.target

View File

@@ -0,0 +1,10 @@
[Unit]
Description=OpenClaw Second-Brain index_vectors
OnFailure=openclaw-secondbrain-notify@%n.service
[Service]
Type=oneshot
WorkingDirectory=/root/.openclaw/workspace
Environment=HF_HOME=/root/.openclaw/workspace/second-brain/data/hf_cache
Environment=SENTENCE_TRANSFORMERS_HOME=/root/.openclaw/workspace/second-brain/data/st_cache
ExecStart=/bin/bash -lc 'flock -n /tmp/%n.lock /usr/bin/python3 /root/.openclaw/workspace/openclaw_cron_wrapper.py index_vectors'

View File

@@ -0,0 +1,10 @@
[Unit]
Description=OpenClaw Second-Brain index_vectors (every 30 min)
[Timer]
OnCalendar=*:0/30
Persistent=true
[Install]
WantedBy=timers.target

View File

@@ -0,0 +1,8 @@
[Unit]
Description=OpenClaw Second-Brain ingest_memory
OnFailure=openclaw-secondbrain-notify@%n.service
[Service]
Type=oneshot
WorkingDirectory=/root/.openclaw/workspace
ExecStart=/bin/bash -lc 'flock -n /tmp/%n.lock /usr/bin/python3 /root/.openclaw/workspace/openclaw_cron_wrapper.py ingest_memory'

View File

@@ -0,0 +1,9 @@
[Unit]
Description=OpenClaw Second-Brain ingest_memory (every 5 min)
[Timer]
OnCalendar=*:0/5
Persistent=true
[Install]
WantedBy=timers.target

View File

@@ -0,0 +1,8 @@
[Unit]
Description=OpenClaw Second-Brain ingest_obsidian
OnFailure=openclaw-secondbrain-notify@%n.service
[Service]
Type=oneshot
WorkingDirectory=/root/.openclaw/workspace
ExecStart=/bin/bash -lc 'flock -n /tmp/%n.lock /usr/bin/python3 /root/.openclaw/workspace/openclaw_cron_wrapper.py ingest_obsidian'

View File

@@ -0,0 +1,10 @@
[Unit]
Description=OpenClaw Second-Brain ingest_obsidian (every 15 min)
[Timer]
OnCalendar=*:0/15
Persistent=true
[Install]
WantedBy=timers.target

View File

@@ -0,0 +1,9 @@
[Unit]
Description=OpenClaw Second-Brain ingest transcript -> DB
OnFailure=openclaw-secondbrain-notify@%n.service
[Service]
Type=oneshot
WorkingDirectory=/root/.openclaw/workspace
ExecStart=/bin/bash -lc 'flock -n /tmp/%n.lock /usr/bin/python3 /root/.openclaw/workspace/openclaw_cron_wrapper.py ingest_transcript_to_db'

View File

@@ -0,0 +1,12 @@
[Unit]
Description=OpenClaw Second-Brain ingest transcript -> DB (every 5 min)
[Timer]
OnBootSec=90s
OnUnitActiveSec=300s
Unit=openclaw-secondbrain-ingest-transcript-to-db.service
Persistent=true
[Install]
WantedBy=timers.target

View File

@@ -0,0 +1,8 @@
[Unit]
Description=OpenClaw Second-Brain ingest transcript -> memory/*.md
OnFailure=openclaw-secondbrain-notify@%n.service
[Service]
Type=oneshot
WorkingDirectory=/root/.openclaw/workspace
ExecStart=/bin/bash -lc 'flock -n /tmp/%n.lock /usr/bin/python3 /root/.openclaw/workspace/second-brain/cron_tasks/ingest_transcript_to_memory.py'

View File

@@ -0,0 +1,11 @@
[Unit]
Description=OpenClaw Second-Brain ingest transcript -> memory (every 1 min)
[Timer]
OnBootSec=30s
OnUnitActiveSec=60s
Unit=openclaw-secondbrain-ingest-transcript-to-memory.service
[Install]
WantedBy=timers.target

View File

@@ -0,0 +1,8 @@
[Unit]
Description=OpenClaw Second-Brain failure notify (%i)
[Service]
Type=oneshot
WorkingDirectory=/root/.openclaw/workspace
ExecStart=/bin/bash -lc '/root/.openclaw/workspace/notify-telegram.sh "❌ Second-Brain job failed: %i. Check: journalctl -u %i -n 50 --no-pager"'

View File

@@ -0,0 +1,10 @@
[Unit]
Description=OpenClaw Second-Brain proactive_search_wrapper
Wants=network-online.target
After=network-online.target
OnFailure=openclaw-secondbrain-notify@%n.service
[Service]
Type=oneshot
WorkingDirectory=/root/.openclaw/workspace
ExecStart=/bin/bash -lc 'flock -n /tmp/%n.lock /usr/bin/python3 /root/.openclaw/workspace/openclaw_cron_wrapper.py proactive_search_wrapper'

View File

@@ -0,0 +1,11 @@
[Unit]
Description=OpenClaw Second-Brain proactive_search_wrapper (every 4h)
[Timer]
OnCalendar=*-*-* 01,05,09,13,17,21:10:00
RandomizedDelaySec=600
Persistent=true
[Install]
WantedBy=timers.target

View File

@@ -0,0 +1,8 @@
[Unit]
Description=OpenClaw Second-Brain review_brain
OnFailure=openclaw-secondbrain-notify@%n.service
[Service]
Type=oneshot
WorkingDirectory=/root/.openclaw/workspace
ExecStart=/bin/bash -lc 'flock -n /tmp/%n.lock /usr/bin/python3 /root/.openclaw/workspace/openclaw_cron_wrapper.py review_brain'

View File

@@ -0,0 +1,9 @@
[Unit]
Description=OpenClaw Second-Brain review_brain (every 5 min)
[Timer]
OnCalendar=*:0/5
Persistent=true
[Install]
WantedBy=timers.target

View File

@@ -0,0 +1,13 @@
[Unit]
Description=OpenClaw Second-Brain task (%i)
Wants=network-online.target
After=network-online.target
[Service]
Type=oneshot
WorkingDirectory=/root/.openclaw/workspace
ExecStart=/usr/bin/python3 /root/.openclaw/workspace/openclaw_cron_wrapper.py %i
Nice=10
IOSchedulingClass=best-effort
IOSchedulingPriority=6

View File

@@ -0,0 +1,9 @@
[Unit]
Description=OpenClaw Second-Brain verify_pending_external
OnFailure=openclaw-secondbrain-notify@%n.service
[Service]
Type=oneshot
WorkingDirectory=/root/.openclaw/workspace
ExecStart=/bin/bash -lc 'flock -n /tmp/%n.lock /usr/bin/python3 /root/.openclaw/workspace/openclaw_cron_wrapper.py verify_pending_external'

View File

@@ -0,0 +1,10 @@
[Unit]
Description=OpenClaw Second-Brain periodic verify_pending_external
[Timer]
OnBootSec=2min
OnUnitActiveSec=5min
Persistent=true
[Install]
WantedBy=timers.target

View File

@@ -16,6 +16,12 @@
<div class="stat"><span class="stat-num" id="statErrors">-</span><span class="stat-label">Err</span></div>
</header>
<div class="tabs-bar">
<button class="tab-btn active" id="tabCards" onclick="setView('cards')">Cards</button>
<button class="tab-btn" id="tabGraph" onclick="setView('graph')">Graph</button>
<button class="tab-btn" id="tabStatus" onclick="setView('status')">Status</button>
</div>
<!-- Search -->
<div class="search-box">
<input type="text" id="searchInput" placeholder="🔍 Suche..." />
@@ -23,6 +29,7 @@
<option value="all">Alle</option>
<option value="pending">Pending</option>
<option value="confirmed">Confirmed</option>
<option value="rejected">Rejected</option>
<option value="errors">Errors</option>
</select>
</div>
@@ -37,6 +44,38 @@
<!-- Cards -->
<div class="cards" id="cards"></div>
<!-- Graph -->
<div class="graph" id="graph" style="display:none;">
<div class="graph-controls">
<button class="btn primary" id="btnGraphPhysics" onclick="toggleGraphPhysics()">Physics: off</button>
<button class="btn" onclick="resetGraphView()">Reset view</button>
<button class="btn" onclick="fitGraphView()">Fit</button>
<label class="muted small" style="display:flex;align-items:center;gap:8px">
<span>Physics</span>
<input id="physicsStrength" type="range" min="0" max="100" value="60" oninput="setPhysicsStrength(this.value)" style="width:140px">
<span id="physicsStrengthVal">60</span>
</label>
<select class="btn" id="graphLimit" onchange="reloadGraph()" title="Wie viele Knoten laden? 0=all">
<option value="0">Nodes: all</option>
<option value="200">Nodes: 200</option>
<option value="1000">Nodes: 1000</option>
<option value="5000">Nodes: 5000</option>
</select>
<button class="btn" onclick="reloadGraph()">Reload</button>
</div>
<canvas id="graphCanvas" width="440" height="520"></canvas>
<div class="graph-hint muted small" id="graphHint">Lade Graph…</div>
<div class="graph-legend">
<div><strong>Graph</strong>: Zoom per Pinch (2 Finger), Pan per Drag (1 Finger). Tap auf Engram öffnet Details, Tap auf Tag setzt Suche.</div>
<div class="legend-row"><span class="legend-dot engram"></span> Engram</div>
<div class="legend-row"><span class="legend-dot tag"></span> Tag</div>
<div class="legend-row"><span class="legend-dot match"></span> Match (Suche)</div>
</div>
</div>
<!-- Status -->
<div class="status" id="status" style="display:none;"></div>
<!-- Pagination -->
<div class="pagination" id="pagination">
<button id="btnPrev" onclick="prevPage()"></button>
@@ -67,12 +106,25 @@ let state = {
filter: 'all',
search: '',
autoRefresh: true,
view: 'cards',
lastEvent: null,
};
// ─── Fetch ──────────────────────────────────────────────────────────────────
async function api(path, opts = {}) {
const r = await fetch(path, opts);
if (!r.ok) throw new Error((await r.json()).error || r.statusText);
if (!r.ok) {
let msg = r.statusText;
try {
const j = await r.json();
msg = j.error || j.detail || msg;
if (Array.isArray(msg)) msg = JSON.stringify(msg);
if (typeof msg !== 'string') msg = JSON.stringify(msg);
} catch (e) {
// ignore JSON parse errors
}
throw new Error(msg);
}
return r.json();
}
@@ -84,11 +136,36 @@ async function loadStats() {
document.getElementById('statErrors').textContent = s.errors;
}
function updateStatsFromEvent(ev) {
if (!ev || !ev.stats) return;
const s = ev.stats;
document.getElementById('statTotal').textContent = s.total;
document.getElementById('statConfirmed').textContent = s.confirmed;
document.getElementById('statPending').textContent = s.pending;
document.getElementById('statErrors').textContent = s.errors;
}
function setView(view) {
state.view = view;
document.getElementById('tabCards').classList.toggle('active', view === 'cards');
document.getElementById('tabGraph').classList.toggle('active', view === 'graph');
document.getElementById('tabStatus').classList.toggle('active', view === 'status');
document.getElementById('cards').style.display = view === 'cards' ? '' : 'none';
document.getElementById('pagination').style.display = view === 'cards' ? '' : 'none';
document.getElementById('graph').style.display = view === 'graph' ? '' : 'none';
document.getElementById('status').style.display = view === 'status' ? '' : 'none';
if (view === 'graph') loadGraph();
if (view === 'status') loadStatus();
}
async function loadCards() {
let url = `/api/engrams?limit=${state.limit}&offset=${state.offset}`;
if (state.search) url += `&search=${encodeURIComponent(state.search)}`;
if (state.filter === 'confirmed') url += '&confirmed=1';
if (state.filter === 'pending') url += '&confirmed=0';
if (state.filter === 'rejected') url += '&verdict=confirmed_false';
if (state.filter === 'errors') url += '&tag=error';
const data = await api(url);
@@ -99,12 +176,788 @@ async function loadCards() {
document.getElementById('btnNext').disabled = data.items.length < state.limit;
}
async function loadStatus() {
const reqs = await Promise.allSettled([
api('/api/config'),
api('/api/db_info'),
api('/api/jobs'),
api('/api/insights?limit=8'),
api('/api/storage_stats'),
api('/api/pending?limit=20&offset=0'),
]);
const pick = (i, fallback) => (reqs[i].status === 'fulfilled' ? reqs[i].value : fallback);
const err = (i) => (reqs[i].status === 'rejected' ? (reqs[i].reason && reqs[i].reason.message ? reqs[i].reason.message : String(reqs[i].reason)) : null);
const cfg = pick(0, { workspace: '-', db_path: '-' });
const db = pick(1, { db_path: '-', mtime: null });
const jobs = pick(2, { items: [], error: err(2) });
const ins = pick(3, { pending: '-', top_tags: [], top_hosts: [], error: err(3) });
const stor = pick(4, { sql: { total_engrams: '-', confirmed: '-', pending: '-', by_source: {} }, vector: { chroma_size_bytes: 0, embedding_cache_files: 0 }, obsidian: { configured: false } });
const pend = pick(5, { items: [], error: err(5) });
const el = document.getElementById('status');
const jobsHtml = (jobs.items || []).map(j => `
<div class="kv-row">
<div class="kv-key">${j.unit}</div>
<div class="kv-val">${j.error ? ('ERR: ' + j.error) : (j.active + '/' + j.sub)}</div>
</div>
`).join('');
const topTags = (ins.top_tags || []).map(t => `<span class="pill">${t.key} (${t.count})</span>`).join(' ');
const topHosts = (ins.top_hosts || []).map(t => `<span class="pill">${t.key} (${t.count})</span>`).join(' ');
const bySource = Object.entries((stor.sql && stor.sql.by_source) ? stor.sql.by_source : {})
.slice(0, 8)
.map(([k,v]) => `<span class="pill">${k}: ${v}</span>`)
.join(' ');
const pendItems = (pend.items || []);
const pendHtml = pendItems.map(p => `
<div class="kv-row" onclick="showDetail('${p.id}')">
<div class="kv-key">${(p.source||'').slice(0,12)}</div>
<div class="kv-val">
<span class="pill">${p.id.substring(0,8)}</span>
${escapeHtml((p.content||'').substring(0,120))}${(p.content||'').length>120?'…':''}
<div class="actions" style="margin-top:6px" onclick="event.stopPropagation()">
<button class="btn-ok" onclick="confirm('${p.id}', event)">✅</button>
<button class="btn-no" onclick="reject('${p.id}', event)">❌</button>
</div>
</div>
</div>
`).join('');
el.innerHTML = `
<div class="panel">
<div class="panel-title">Config</div>
<div class="kv-row"><div class="kv-key">workspace</div><div class="kv-val">${cfg.workspace}</div></div>
<div class="kv-row"><div class="kv-key">db</div><div class="kv-val">${db.db_path}</div></div>
<div class="kv-row"><div class="kv-key">db mtime</div><div class="kv-val">${new Date(db.mtime).toLocaleString()}</div></div>
</div>
<div class="panel">
<div class="panel-title">Storage</div>
<div class="kv-row"><div class="kv-key">SQL</div><div class="kv-val">${stor.sql.total_engrams} engrams (ok ${stor.sql.confirmed}, pending ${stor.sql.pending})</div></div>
<div class="kv-row"><div class="kv-key">Vector</div><div class="kv-val">chroma ${(stor.vector.chroma_size_bytes/1024/1024).toFixed(1)} MB, cache ${stor.vector.embedding_cache_files} files</div></div>
<div class="kv-row"><div class="kv-key">Obsidian</div><div class="kv-val">${stor.obsidian.configured ? 'configured' : 'not configured'}</div></div>
<div class="kv-row"><div class="kv-key">By source</div><div class="kv-val">${bySource || '-'}</div></div>
</div>
<div class="panel">
<div class="panel-title">Jobs</div>
${jobs.error ? `<div class="muted">Fehler: ${escapeHtml(jobs.error)}</div>` : (jobsHtml || '<div class="muted">Keine Daten</div>')}
</div>
<div class="panel">
<div class="panel-title">Insights</div>
${ins.error ? `<div class="muted">Fehler: ${escapeHtml(ins.error)}</div>` : ''}
<div class="kv-row"><div class="kv-key">pending</div><div class="kv-val">${ins.pending}</div></div>
<div class="kv-row"><div class="kv-key">top tags</div><div class="kv-val">${topTags || '-'}</div></div>
<div class="kv-row"><div class="kv-key">top hosts</div><div class="kv-val">${topHosts || '-'}</div></div>
</div>
<div class="panel">
<div class="panel-title">Pending Queue (latest)</div>
${pend.error ? `<div class="muted">Fehler: ${escapeHtml(pend.error)}</div>` : (pendHtml || '<div class="muted">Keine Pendings</div>')}
</div>
`;
}
async function loadGraph() {
const sel = document.getElementById('graphLimit');
const fromSel = sel ? parseInt(sel.value || '0', 10) : NaN;
const fromStore = parseInt(localStorage.getItem('graphLimit') || '0', 10);
const q = (!Number.isNaN(fromSel)) ? fromSel : (Number.isNaN(fromStore) ? 0 : fromStore);
if (sel) sel.value = String(q);
if (sel) localStorage.setItem('graphLimit', String(q));
const hint = document.getElementById('graphHint');
if (hint) hint.textContent = 'Lade Graph…';
try {
const g = await api(`/api/graph?limit_nodes=${q}`);
renderGraph(g.nodes || [], g.edges || []);
} catch (e) {
if (hint) hint.textContent = `Graph-Fehler: ${e && e.message ? e.message : String(e)}`;
const canvas = _graphCanvas();
const ctx = _graphCtx();
if (canvas && ctx) ctx.clearRect(0,0,canvas.width,canvas.height);
}
}
function reloadGraph() { loadGraph(); }
// ─── Graph Renderer (Canvas) ────────────────────────────────────────────────
let graphState = {
nodes: [],
edges: [],
sim: [],
links: [],
nodeById: new Map(),
simById: new Map(),
degree: new Map(),
physicsOn: false,
draggingId: null,
selectedId: null,
panning: false,
lastX: 0,
lastY: 0,
panX: 0,
panY: 0,
zoom: 1,
raf: null,
search: '',
pointers: new Map(),
pinchStartDist: null,
pinchStartZoom: null,
pinchStartPan: null,
down: null, // {pointerId, cx, cy, t}
physicsStrength: parseInt(localStorage.getItem('physicsStrength') || '60', 10),
};
function _graphCanvas() { return document.getElementById('graphCanvas'); }
function _graphCtx() { return _graphCanvas().getContext('2d'); }
function _graphNodeRadius(n) {
const d = graphState.degree.get(n.id) || 0;
const base = n.kind === 'tag' ? 4 : (n.kind === 'host' ? 5 : 7);
const w = (n.weight || 0);
const bonus = Math.min(6, Math.sqrt(Math.max(0, w)) * 0.8);
return Math.max(3, Math.min(18, base + Math.sqrt(d) + bonus));
}
function _graphNodeFill(n) {
const d = graphState.degree.get(n.id) || 0;
const t = Math.max(0, Math.min(0.55, d / 18)); // higher degree -> brighter
const mix = (rgb) => rgb.map(c => Math.round(c + (255 - c) * t));
if (n.kind === 'tag') {
const [r,g,b] = mix([167, 139, 250]);
return `rgb(${r},${g},${b})`;
}
if (n.kind === 'host') {
const [r,g,b] = mix([245, 158, 11]);
return `rgb(${r},${g},${b})`;
}
const verdict = (n.verdict || '').toString();
let base = [96, 165, 250]; // pending/unknown = blue
if (verdict === 'confirmed_true') base = [34, 197, 94]; // green
else if (verdict === 'confirmed_false') base = [239, 68, 68]; // red
else if (verdict === 'probable_true') base = [52, 211, 153]; // teal
else if (verdict === 'probable_false') base = [251, 191, 36]; // amber
// Recency brightening (brand new pops)
const now = Date.now();
const created = Date.parse(n.created || '') || 0;
const ageMin = created ? (now - created) / 60000 : 999999;
const rec = Math.max(0, Math.min(0.35, (30 - ageMin) / 30 * 0.35));
const bump = (c) => Math.round(c + (255 - c) * rec);
const [r,g,b] = mix(base).map(bump);
return `rgb(${r},${g},${b})`;
}
function _graphMatches(n, term) {
const t = (term || '').trim().toLowerCase();
if (!t) return false;
const id = (n.id || '').toLowerCase();
const label = (n.label || '').toLowerCase();
return id.includes(t) || label.includes(t);
}
function _graphWorldFromScreen(cx, cy) {
return {
x: (cx - graphState.panX) / graphState.zoom,
y: (cy - graphState.panY) / graphState.zoom,
};
}
function _graphHitTest(wx, wy) {
for (let i = graphState.sim.length - 1; i >= 0; i--) {
const n = graphState.sim[i];
const r = _graphNodeRadius(n) + 2;
const dx = wx - n.x;
const dy = wy - n.y;
if ((dx*dx + dy*dy) <= r*r) return n;
}
return null;
}
function _graphInitInteractions() {
const canvas = _graphCanvas();
if (canvas._graphBound) return;
canvas._graphBound = true;
const toCanvasXY = (ev) => {
const rect = canvas.getBoundingClientRect();
return { cx: ev.clientX - rect.left, cy: ev.clientY - rect.top };
};
const pinchMidpoint = () => {
const pts = Array.from(graphState.pointers.values());
if (pts.length < 2) return null;
return { cx: (pts[0].cx + pts[1].cx) / 2, cy: (pts[0].cy + pts[1].cy) / 2 };
};
const pinchDistance = () => {
const pts = Array.from(graphState.pointers.values());
if (pts.length < 2) return null;
const dx = pts[0].cx - pts[1].cx;
const dy = pts[0].cy - pts[1].cy;
return Math.sqrt(dx*dx + dy*dy);
};
canvas.addEventListener('pointerdown', (ev) => {
canvas.setPointerCapture(ev.pointerId);
const pos = toCanvasXY(ev);
graphState.pointers.set(ev.pointerId, pos);
graphState.down = { pointerId: ev.pointerId, cx: pos.cx, cy: pos.cy, t: Date.now() };
// When 2 pointers: start pinch
if (graphState.pointers.size === 2) {
graphState.panning = false;
graphState.draggingId = null;
graphState.down = null;
graphState.pinchStartDist = pinchDistance();
graphState.pinchStartZoom = graphState.zoom;
graphState.pinchStartPan = { panX: graphState.panX, panY: graphState.panY };
_graphDraw();
return;
}
// Single pointer: pan or drag node
const { cx, cy } = pos;
const w = _graphWorldFromScreen(cx, cy);
const hit = _graphHitTest(w.x, w.y);
graphState.lastX = cx; graphState.lastY = cy;
if (hit) {
graphState.draggingId = hit.id;
graphState.selectedId = hit.id;
} else {
graphState.panning = true;
graphState.selectedId = null;
}
_graphDraw();
});
canvas.addEventListener('pointermove', (ev) => {
if (!graphState.pointers.has(ev.pointerId)) return;
const pos = toCanvasXY(ev);
graphState.pointers.set(ev.pointerId, pos);
// Pinch zoom (2 pointers)
if (graphState.pointers.size >= 2 && graphState.pinchStartDist) {
const dist = pinchDistance();
const mid = pinchMidpoint();
if (!dist || !mid) return;
const zoomOld = graphState.zoom;
const wx = (mid.cx - graphState.panX) / zoomOld;
const wy = (mid.cy - graphState.panY) / zoomOld;
const factor = dist / graphState.pinchStartDist;
const next = Math.max(0.35, Math.min(3.0, graphState.pinchStartZoom * factor));
graphState.zoom = next;
graphState.panX = mid.cx - wx * graphState.zoom;
graphState.panY = mid.cy - wy * graphState.zoom;
_graphDraw();
return;
}
// Single pointer
const { cx, cy } = pos;
const dx = cx - graphState.lastX;
const dy = cy - graphState.lastY;
graphState.lastX = cx; graphState.lastY = cy;
if (graphState.draggingId) {
const w = _graphWorldFromScreen(cx, cy);
const n = graphState.simById.get(graphState.draggingId);
if (n) { n.x = w.x; n.y = w.y; n.vx = 0; n.vy = 0; }
_graphDraw();
return;
}
if (graphState.panning) {
graphState.panX += dx;
graphState.panY += dy;
_graphDraw();
}
});
const endPointer = (ev) => {
if (graphState.pointers.has(ev.pointerId)) graphState.pointers.delete(ev.pointerId);
try { canvas.releasePointerCapture(ev.pointerId); } catch {}
if (graphState.pointers.size < 2) {
graphState.pinchStartDist = null;
graphState.pinchStartZoom = null;
graphState.pinchStartPan = null;
}
const pos = toCanvasXY(ev);
const down = graphState.down && graphState.down.pointerId === ev.pointerId ? graphState.down : null;
graphState.down = null;
const moved = down ? Math.hypot(pos.cx - down.cx, pos.cy - down.cy) : 999;
const isTap = !!down && moved <= 8 && (Date.now() - down.t) <= 500;
const w = _graphWorldFromScreen(pos.cx, pos.cy);
const hit = _graphHitTest(w.x, w.y);
graphState.draggingId = null;
graphState.panning = false;
if (isTap && hit) {
graphState.selectedId = hit.id;
_graphDraw();
if (hit.kind === 'engram') {
showDetail(hit.id);
} else if (hit.kind === 'tag') {
const label = (hit.label || '').replace(/^tag:/, '');
const inp = document.getElementById('searchInput');
inp.value = label;
inp.dispatchEvent(new Event('input'));
}
return;
}
_graphDraw();
};
canvas.addEventListener('pointerup', endPointer);
canvas.addEventListener('pointercancel', endPointer);
}
function renderGraph(nodes, edges) {
const canvas = _graphCanvas();
const hint = document.getElementById('graphHint');
const ctx = _graphCtx();
// sync physics slider
const slider = document.getElementById('physicsStrength');
const sliderVal = document.getElementById('physicsStrengthVal');
const s = Math.max(0, Math.min(100, parseInt(graphState.physicsStrength || 60, 10)));
graphState.physicsStrength = s;
if (slider) slider.value = String(s);
if (sliderVal) sliderVal.textContent = String(s);
const w = canvas.parentElement.clientWidth - 24;
canvas.width = Math.max(320, w);
canvas.height = Math.max(520, Math.min(900, (window.innerHeight || 900) - 260));
graphState.nodes = nodes || [];
graphState.edges = edges || [];
graphState.nodeById = new Map(graphState.nodes.map(n => [n.id, n]));
graphState.degree = new Map();
for (const e of graphState.edges) {
graphState.degree.set(e.from, (graphState.degree.get(e.from) || 0) + 1);
graphState.degree.set(e.to, (graphState.degree.get(e.to) || 0) + 1);
}
if (!graphState.nodes.length) {
hint.textContent = 'Graph: keine Nodes.';
ctx.clearRect(0,0,canvas.width,canvas.height);
return;
}
hint.textContent = `nodes=${graphState.nodes.length} edges=${graphState.edges.length}`;
// Build adjacency for deterministic layout (O(n+e))
const adj = new Map();
const addAdj = (a, b) => {
if (!adj.has(a)) adj.set(a, []);
adj.get(a).push(b);
};
for (const e of graphState.edges) {
addAdj(e.from, e.to);
addAdj(e.to, e.from);
}
const degree = graphState.degree;
const visited = new Set();
const comps = [];
for (const n of graphState.nodes) {
if (visited.has(n.id)) continue;
const q = [n.id];
visited.add(n.id);
const comp = [];
while (q.length) {
const cur = q.pop();
comp.push(cur);
const neigh = adj.get(cur) || [];
for (const nb of neigh) {
if (!visited.has(nb)) {
visited.add(nb);
q.push(nb);
}
}
}
comps.push(comp);
}
comps.sort((a, b) => b.length - a.length);
// component centers in a loose grid/spiral
const centers = [];
const gap = 240;
const cols = Math.max(1, Math.floor(canvas.width / gap));
for (let i = 0; i < comps.length; i++) {
const cx = (i % cols) * gap - ((cols - 1) * gap) / 2;
const cy = Math.floor(i / cols) * gap - (Math.floor((comps.length - 1) / cols) * gap) / 2;
centers.push({ cx, cy });
}
const pos = new Map();
const R = 46;
for (let ci = 0; ci < comps.length; ci++) {
const comp = comps[ci];
const center = centers[ci] || { cx: 0, cy: 0 };
// hub: highest degree engram preferred
let hub = comp[0];
let hubScore = -1;
for (const id of comp) {
const n = graphState.nodeById.get(id) || {};
const score = (degree.get(id) || 0) + (n.kind === 'engram' ? 3 : 0);
if (score > hubScore) { hubScore = score; hub = id; }
}
// BFS layers from hub
const layer = new Map();
layer.set(hub, 0);
const qq = [hub];
while (qq.length) {
const cur = qq.shift();
const l = layer.get(cur) || 0;
const neigh = adj.get(cur) || [];
for (const nb of neigh) {
if (!layer.has(nb)) {
layer.set(nb, l + 1);
qq.push(nb);
}
}
}
const buckets = new Map();
for (const id of comp) {
const l = layer.get(id);
const ll = (typeof l === 'number') ? l : 99;
if (!buckets.has(ll)) buckets.set(ll, []);
buckets.get(ll).push(id);
}
// Place hubs first, then rings
pos.set(hub, { x: center.cx, y: center.cy });
const maxLayer = Math.max(...Array.from(buckets.keys()));
for (let l = 1; l <= maxLayer; l++) {
const ids = buckets.get(l) || [];
if (!ids.length) continue;
// Stable order: engrams first, then by degree desc
ids.sort((a, b) => {
const na = graphState.nodeById.get(a) || {};
const nb = graphState.nodeById.get(b) || {};
const ka = na.kind === 'engram' ? 0 : (na.kind === 'tag' ? 1 : 2);
const kb = nb.kind === 'engram' ? 0 : (nb.kind === 'tag' ? 1 : 2);
if (ka !== kb) return ka - kb;
return (degree.get(b) || 0) - (degree.get(a) || 0);
});
const rad = l * R;
for (let i = 0; i < ids.length; i++) {
const ang = (i / ids.length) * Math.PI * 2;
pos.set(ids[i], {
x: center.cx + Math.cos(ang) * rad,
y: center.cy + Math.sin(ang) * rad,
});
}
}
}
graphState.sim = graphState.nodes.map(n => {
const p = pos.get(n.id) || { x: (Math.random() - 0.5) * canvas.width, y: (Math.random() - 0.5) * canvas.height };
return {
id: n.id,
kind: n.kind,
label: n.label || n.id,
weight: n.weight,
verdict: n.verdict,
confidence: n.confidence,
created: n.created,
modified: n.modified,
last_accessed: n.last_accessed,
predict_locked: n.predict_locked,
x: p.x,
y: p.y,
vx: 0, vy: 0,
};
});
graphState.simById = new Map(graphState.sim.map(n => [n.id, n]));
graphState.links = graphState.edges
.map(e => ({a: graphState.simById.get(e.from), b: graphState.simById.get(e.to), kind: e.kind, weight: e.weight || 1.0}))
.filter(l => l.a && l.b);
graphState.panX = canvas.width / 2;
graphState.panY = canvas.height / 2;
graphState.zoom = 1;
graphState.search = state.search || '';
graphState.selectedId = null;
_graphInitInteractions();
// Physics is expensive; pre-relax a little but keep it optional for big graphs.
const relaxIters = graphState.sim.length <= 700 ? 70 : 12;
for (let i = 0; i < relaxIters; i++) _graphStepPhysics(0.85);
_graphDraw();
}
function _graphStepPhysics(alpha = 1.0) {
const canvas = _graphCanvas();
const repulsion = (graphState.sim.length > 700) ? 120 : 180;
const damping = 0.86;
const target = 80;
const springK = 0.018;
const sim = graphState.sim;
if (sim.length > 700) {
// Fast approximate repulsion via spatial hashing (checks only local neighbor cells).
const cell = 120;
const grid = new Map();
const key = (cx, cy) => `${cx},${cy}`;
for (let i = 0; i < sim.length; i++) {
const n = sim[i];
const cx = Math.floor(n.x / cell);
const cy = Math.floor(n.y / cell);
const k = key(cx, cy);
if (!grid.has(k)) grid.set(k, []);
grid.get(k).push(i);
}
for (let i = 0; i < sim.length; i++) {
const a = sim[i];
const cx = Math.floor(a.x / cell);
const cy = Math.floor(a.y / cell);
for (let ox = -1; ox <= 1; ox++) {
for (let oy = -1; oy <= 1; oy++) {
const idxs = grid.get(key(cx + ox, cy + oy));
if (!idxs) continue;
for (const j of idxs) {
if (j === i) continue;
const b = sim[j];
const dx = a.x - b.x, dy = a.y - b.y;
const d2 = dx*dx + dy*dy + 0.12;
const f = (repulsion / d2) * alpha * 0.55;
a.vx += dx*f; a.vy += dy*f;
}
}
}
}
} else {
for (let i = 0; i < sim.length; i++) {
for (let j = i + 1; j < sim.length; j++) {
const a = sim[i], b = sim[j];
const dx = a.x - b.x, dy = a.y - b.y;
const d2 = dx*dx + dy*dy + 0.02;
const f = (repulsion / d2) * alpha;
a.vx += dx*f; a.vy += dy*f;
b.vx -= dx*f; b.vy -= dy*f;
}
}
}
for (const l of graphState.links) {
const a = l.a, b = l.b;
const dx = b.x - a.x, dy = b.y - a.y;
const dist = Math.sqrt(dx*dx + dy*dy) || 1;
const f = (dist - target) * springK * alpha;
const fx = (dx/dist) * f, fy = (dy/dist) * f;
a.vx += fx; a.vy += fy;
b.vx -= fx; b.vy -= fy;
}
for (const n of sim) {
n.vx *= damping; n.vy *= damping;
n.x += n.vx; n.y += n.vy;
const pad = 10;
const bx = canvas.width / graphState.zoom;
const by = canvas.height / graphState.zoom;
n.x = Math.max(-bx/2 + pad, Math.min(bx/2 - pad, n.x));
n.y = Math.max(-by/2 + pad, Math.min(by/2 - pad, n.y));
}
}
function _graphEdgeColor(kind) {
const k = (kind || '').toString().toLowerCase();
if (k.includes('tag')) return '#7c3aed';
if (k.includes('host')) return '#f59e0b';
if (k.includes('ref')) return '#10b981';
return '#3a3a55';
}
function _graphDraw() {
const canvas = _graphCanvas();
const ctx = _graphCtx();
const hint = document.getElementById('graphHint');
ctx.clearRect(0,0,canvas.width,canvas.height);
ctx.save();
ctx.translate(graphState.panX, graphState.panY);
ctx.scale(graphState.zoom, graphState.zoom);
for (const l of graphState.links) {
const term = (graphState.search || '').trim();
const isMatchEdge = term && (_graphMatches(l.a, term) || _graphMatches(l.b, term));
const w = Math.max(0.2, Math.min(3.0, (l.weight || 1.0)));
ctx.lineWidth = (0.6 + w) / graphState.zoom;
ctx.globalAlpha = isMatchEdge ? 0.85 : (0.25 + Math.min(0.35, w * 0.18));
ctx.strokeStyle = isMatchEdge ? '#f7d154' : _graphEdgeColor(l.kind);
ctx.beginPath();
ctx.moveTo(l.a.x, l.a.y);
ctx.lineTo(l.b.x, l.b.y);
ctx.stroke();
}
ctx.globalAlpha = 1.0;
const term = (graphState.search || '').trim();
let matches = 0;
for (const n of graphState.sim) {
const r = _graphNodeRadius(n);
const isMatch = _graphMatches(n, term);
if (isMatch) matches++;
if (isMatch) {
ctx.beginPath();
ctx.strokeStyle = '#f7d154';
ctx.lineWidth = 3 / graphState.zoom;
ctx.arc(n.x, n.y, r + 3, 0, Math.PI*2);
ctx.stroke();
}
if (graphState.selectedId === n.id) {
ctx.beginPath();
ctx.strokeStyle = '#ffffff';
ctx.lineWidth = 2 / graphState.zoom;
ctx.arc(n.x, n.y, r + 2, 0, Math.PI*2);
ctx.stroke();
}
ctx.beginPath();
ctx.fillStyle = _graphNodeFill(n);
ctx.arc(n.x, n.y, r, 0, Math.PI*2);
ctx.fill();
// status/recency/lock border
let stroke = null;
const v = (n.verdict || '').toString();
if (v === 'confirmed_true') stroke = '#bbf7d0';
else if (v === 'confirmed_false') stroke = '#fecaca';
else if (v) stroke = '#c7d2fe';
const now = Date.now();
const created = Date.parse(n.created || '') || 0;
const modified = Date.parse(n.modified || '') || 0;
const isNew = created && (now - created) < (30 * 60 * 1000);
const isHot = modified && (now - modified) < (10 * 60 * 1000);
if (isNew) stroke = '#f7d154';
if (isHot) stroke = '#ffffff';
if (n.predict_locked) stroke = '#a78bfa';
if (stroke) {
ctx.beginPath();
ctx.strokeStyle = stroke;
ctx.lineWidth = (n.predict_locked ? 3 : 1.5) / graphState.zoom;
ctx.arc(n.x, n.y, r + 0.8, 0, Math.PI*2);
ctx.stroke();
}
}
ctx.restore();
hint.textContent = `nodes=${graphState.nodes.length} edges=${graphState.edges.length}` + (term ? ` | match=${matches}` : '');
}
function _graphLoop() {
if (!graphState.physicsOn) return;
const speed = 0.25 + (Math.max(0, Math.min(100, graphState.physicsStrength || 60)) / 100) * 0.95;
_graphStepPhysics(speed);
_graphDraw();
graphState.raf = requestAnimationFrame(_graphLoop);
}
function setPhysicsStrength(v) {
const n = Math.max(0, Math.min(100, parseInt(v || '0', 10)));
graphState.physicsStrength = n;
localStorage.setItem('physicsStrength', String(n));
const el = document.getElementById('physicsStrengthVal');
if (el) el.textContent = String(n);
}
function toggleGraphPhysics() {
graphState.physicsOn = !graphState.physicsOn;
const b = document.getElementById('btnGraphPhysics');
const fast = (graphState.sim || []).length > 700;
b.textContent = `Physics: ${graphState.physicsOn ? ('on' + (fast ? ' (fast)' : '')) : 'off'}`;
b.classList.toggle('primary', graphState.physicsOn);
if (graphState.physicsOn) {
if (graphState.raf) cancelAnimationFrame(graphState.raf);
graphState.raf = requestAnimationFrame(_graphLoop);
} else {
if (graphState.raf) cancelAnimationFrame(graphState.raf);
graphState.raf = null;
_graphDraw();
}
}
function resetGraphView() {
const canvas = _graphCanvas();
graphState.panX = canvas.width / 2;
graphState.panY = canvas.height / 2;
graphState.zoom = 1;
_graphDraw();
}
function fitGraphView() {
const canvas = _graphCanvas();
if (!graphState.sim.length) return;
let minX = Infinity, minY = Infinity, maxX = -Infinity, maxY = -Infinity;
for (const n of graphState.sim) {
minX = Math.min(minX, n.x); minY = Math.min(minY, n.y);
maxX = Math.max(maxX, n.x); maxY = Math.max(maxY, n.y);
}
const w = Math.max(1, maxX - minX);
const h = Math.max(1, maxY - minY);
const zx = (canvas.width - 40) / w;
const zy = (canvas.height - 40) / h;
graphState.zoom = Math.max(0.35, Math.min(2.5, Math.min(zx, zy)));
graphState.panX = canvas.width / 2;
graphState.panY = canvas.height / 2;
_graphDraw();
}
function graphApplySearch(term) {
graphState.search = term || '';
_graphDraw();
}
// Real-time updates via SSE
function startEvents() {
try {
const es = new EventSource('/api/events');
es.onmessage = (msg) => {
try {
state.lastEvent = JSON.parse(msg.data);
updateStatsFromEvent(state.lastEvent);
if (state.view === 'status') {
// refresh status panels without heavy re-render: just rerun loadStatus occasionally
loadStatus();
}
if (state.view === 'graph') {
// fetch graph less often (every ~15s)
const t = Date.now();
if (!state._lastGraphFetch || (t - state._lastGraphFetch) > 15000) {
state._lastGraphFetch = t;
loadGraph();
}
}
} catch (e) {}
};
es.onerror = () => {
// keep UI usable even if SSE drops
};
} catch (e) {}
}
startEvents();
function renderCards() {
const el = document.getElementById('cards');
el.innerHTML = state.items.map(item => `
<div class="card ${item.confirmed ? 'confirmed' : ''} ${item.rejections > 0 ? 'rejected' : ''}" data-id="${item.id}">
<div class="card-header">
<span class="conf-badge" style="background:hsl(${item.confidence*120},70%,40%)">${Math.round(item.confidence*100)}%</span>
${renderVerdictPill(item)}
<span class="tags">${item.tags.map(t => '<span class="tag">'+t+'</span>').join('')}</span>
<span class="date">${fmtDate(item.created)}</span>
</div>
@@ -123,6 +976,19 @@ function renderCards() {
`).join('');
}
function renderVerdictPill(item) {
const v = (item.verdict || '').toString();
if (!v) return '';
let cls = 'v-unknown';
let label = v;
if (v === 'confirmed_true') { cls = 'v-true'; label = 'TRUE'; }
else if (v === 'confirmed_false') { cls = 'v-false'; label = 'FALSE'; }
else if (v === 'probable_true') { cls = 'v-prob-true'; label = 'LIKELY'; }
else if (v === 'probable_false') { cls = 'v-prob-false'; label = 'UNLIKELY'; }
else if (v === 'unknown') { cls = 'v-unknown'; label = 'UNKNOWN'; }
return `<span class="verdict-pill ${cls}">${label}</span>`;
}
function fmtDate(iso) {
const d = new Date(iso);
return `${d.getDate().toString().padStart(2,'0')}.${(d.getMonth()+1).toString().padStart(2,'0')} ${d.getHours().toString().padStart(2,'0')}:${d.getMinutes().toString().padStart(2,'0')}`;
@@ -144,6 +1010,8 @@ async function confirm(id, ev) {
body: new URLSearchParams({reason})
});
await loadCards(); await loadStats();
if (state.view === 'graph') loadGraph();
if (state.view === 'status') loadStatus();
}
async function reject(id, ev) {
@@ -155,6 +1023,8 @@ async function reject(id, ev) {
body: new URLSearchParams({reason})
});
await loadCards(); await loadStats();
if (state.view === 'graph') loadGraph();
if (state.view === 'status') loadStatus();
}
async function refresh(id, ev) {
@@ -175,6 +1045,8 @@ async function createEngram() {
document.getElementById('newContent').value = '';
document.getElementById('newTags').value = '';
await loadCards(); await loadStats();
if (state.view === 'graph') loadGraph();
if (state.view === 'status') loadStatus();
}
async function showDetail(id) {
@@ -220,6 +1092,7 @@ document.getElementById('searchInput').addEventListener('input', (e) => {
state.search = e.target.value;
state.offset = 0;
loadCards();
if (state.view === 'graph') graphApplySearch(state.search);
});
document.getElementById('filterSelect').addEventListener('change', (e) => {