Compare commits
19 Commits
v2.0.0-mvp
...
a261f5b9e1
| Author | SHA1 | Date | |
|---|---|---|---|
| a261f5b9e1 | |||
| e6e8eba8f6 | |||
| 20098a3253 | |||
| fa2ba11b66 | |||
| 7dfd9c4228 | |||
| 6d99c520e6 | |||
| f10a5b9f19 | |||
| 6232f25cc9 | |||
| 6b0cf5889f | |||
| 021fd0e328 | |||
| d52e3a7f74 | |||
| 1635ee8b03 | |||
| f8ac0af869 | |||
| 9dd5e49e2a | |||
| b158b19208 | |||
| 095e6a33f8 | |||
| e5061b317f | |||
| ec8870ea40 | |||
| 8f47151a48 |
25
RUNBOOK.md
25
RUNBOOK.md
@@ -2,19 +2,32 @@
|
|||||||
|
|
||||||
This is the operational quick-reference for the shipped systemd timers/services and the optional FastAPI dashboard backend.
|
This is the operational quick-reference for the shipped systemd timers/services and the optional FastAPI dashboard backend.
|
||||||
|
|
||||||
Repository root (on host): `/root/.openclaw/workspace`
|
Repository root (on host): `/root/.openclaw/workspace/second-brain`
|
||||||
|
|
||||||
|
Integration + orchestration docs live in the workspace repo:
|
||||||
|
|
||||||
|
- `/root/.openclaw/workspace/docs/SECOND_BRAIN.md`
|
||||||
|
- `/root/.openclaw/workspace/docs/GITEA.md`
|
||||||
|
|
||||||
## Systemd units (cron jobs)
|
## Systemd units (cron jobs)
|
||||||
|
|
||||||
Unit files are shipped in `systemd/` (repo root). Install them into `/etc/systemd/system/` (symlink or copy), then reload:
|
Unit files are shipped in `systemd/` (this repo). Install them into `/etc/systemd/system/` (symlink or copy), then reload:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
sudo ln -sf /root/.openclaw/workspace/systemd/openclaw-secondbrain-*.service /etc/systemd/system/
|
sudo ln -sf /root/.openclaw/workspace/second-brain/systemd/openclaw-secondbrain-*.service /etc/systemd/system/
|
||||||
sudo ln -sf /root/.openclaw/workspace/systemd/openclaw-secondbrain-*.timer /etc/systemd/system/
|
sudo ln -sf /root/.openclaw/workspace/second-brain/systemd/openclaw-secondbrain-*.timer /etc/systemd/system/
|
||||||
sudo ln -sf /root/.openclaw/workspace/systemd/openclaw-memory-archive.* /etc/systemd/system/
|
sudo ln -sf /root/.openclaw/workspace/second-brain/systemd/openclaw-memory-archive.* /etc/systemd/system/
|
||||||
sudo systemctl daemon-reload
|
sudo systemctl daemon-reload
|
||||||
```
|
```
|
||||||
|
|
||||||
|
Optional (verification hardening):
|
||||||
|
|
||||||
|
```bash
|
||||||
|
sudo ln -sf /root/.openclaw/workspace/second-brain/systemd/openclaw-secondbrain-verify-pending.* /etc/systemd/system/
|
||||||
|
sudo systemctl daemon-reload
|
||||||
|
sudo systemctl enable --now openclaw-secondbrain-verify-pending.timer
|
||||||
|
```
|
||||||
|
|
||||||
Enable timers:
|
Enable timers:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
@@ -61,7 +74,7 @@ python3 -m pip install -r second-brain/requirements-dashboard.txt
|
|||||||
SECOND_BRAIN_WORKSPACE="/root/.openclaw/workspace/second-brain" python3 second-brain/fastapi_app.py
|
SECOND_BRAIN_WORKSPACE="/root/.openclaw/workspace/second-brain" python3 second-brain/fastapi_app.py
|
||||||
```
|
```
|
||||||
|
|
||||||
Default port is `8501` (same as Streamlit default). Do not run both on the same port.
|
Default port is `8501` (same as Streamlit default). You can override via `SECOND_BRAIN_PORT` (or `PORT`) when starting manually.
|
||||||
|
|
||||||
Endpoint smoke tests:
|
Endpoint smoke tests:
|
||||||
|
|
||||||
|
|||||||
160
cron_tasks/ingest_transcript_to_db.py
Normal file
160
cron_tasks/ingest_transcript_to_db.py
Normal file
@@ -0,0 +1,160 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""
|
||||||
|
Ingest OpenClaw session transcript JSONL directly into the Second-Brain DB.
|
||||||
|
|
||||||
|
State is tracked with byte offsets per transcript file.
|
||||||
|
Sources are configured via workspace/memory/session_sources.json.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Any, Dict, Iterable, List
|
||||||
|
from uuid import NAMESPACE_URL, uuid5
|
||||||
|
|
||||||
|
WORKSPACE = Path("/root/.openclaw/workspace")
|
||||||
|
MEMORY_DIR = WORKSPACE / "memory"
|
||||||
|
SOURCES_PATH = MEMORY_DIR / "session_sources.json"
|
||||||
|
STATE_PATH = MEMORY_DIR / "session_db_ingest_state.json"
|
||||||
|
|
||||||
|
BRAIN_DIR = WORKSPACE / "second-brain"
|
||||||
|
DB_PATH = Path(os.environ.get("BRAIN_DB", str(BRAIN_DIR / "data" / "brain.sqlite")))
|
||||||
|
|
||||||
|
import sys
|
||||||
|
|
||||||
|
sys.path.insert(0, str(BRAIN_DIR))
|
||||||
|
from src.engram import Engram, Grounding # type: ignore
|
||||||
|
from src.store import EngramStore # type: ignore
|
||||||
|
|
||||||
|
|
||||||
|
def _now() -> str:
|
||||||
|
return datetime.now(timezone.utc).isoformat()
|
||||||
|
|
||||||
|
|
||||||
|
def _load_json(path: Path, default: Any) -> Any:
|
||||||
|
try:
|
||||||
|
if not path.exists():
|
||||||
|
return default
|
||||||
|
return json.loads(path.read_text(encoding="utf-8"))
|
||||||
|
except Exception:
|
||||||
|
return default
|
||||||
|
|
||||||
|
|
||||||
|
def _save_json(path: Path, payload: Any) -> None:
|
||||||
|
path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
path.write_text(json.dumps(payload, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
|
||||||
|
|
||||||
|
|
||||||
|
def _extract_text(content: Any) -> str:
|
||||||
|
if isinstance(content, str):
|
||||||
|
return content.strip()
|
||||||
|
if isinstance(content, list):
|
||||||
|
parts: List[str] = []
|
||||||
|
for c in content:
|
||||||
|
if isinstance(c, dict) and c.get("type") == "text" and isinstance(c.get("text"), str):
|
||||||
|
parts.append(c["text"])
|
||||||
|
elif isinstance(c, str):
|
||||||
|
parts.append(c)
|
||||||
|
return "\n".join(p.strip() for p in parts if p and p.strip()).strip()
|
||||||
|
if isinstance(content, dict) and isinstance(content.get("text"), str):
|
||||||
|
return content["text"].strip()
|
||||||
|
return str(content).strip()
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class Source:
|
||||||
|
label: str
|
||||||
|
transcript_path: Path
|
||||||
|
|
||||||
|
|
||||||
|
def _load_sources() -> List[Source]:
|
||||||
|
payload = _load_json(SOURCES_PATH, {"sources": []})
|
||||||
|
sources: List[Source] = []
|
||||||
|
for item in payload.get("sources", []) if isinstance(payload, dict) else []:
|
||||||
|
if not isinstance(item, dict):
|
||||||
|
continue
|
||||||
|
label = str(item.get("label") or "session")
|
||||||
|
p = item.get("path")
|
||||||
|
if not isinstance(p, str) or not p:
|
||||||
|
continue
|
||||||
|
sources.append(Source(label=label, transcript_path=Path(p)))
|
||||||
|
return sources
|
||||||
|
|
||||||
|
|
||||||
|
def _iter_new_lines(path: Path, *, start_offset: int) -> Iterable[tuple[int, str]]:
|
||||||
|
with open(path, "rb") as f:
|
||||||
|
f.seek(max(0, int(start_offset)))
|
||||||
|
while True:
|
||||||
|
raw = f.readline()
|
||||||
|
if not raw:
|
||||||
|
break
|
||||||
|
line = raw.decode("utf-8", errors="ignore").strip()
|
||||||
|
if not line:
|
||||||
|
continue
|
||||||
|
yield (f.tell(), line)
|
||||||
|
|
||||||
|
|
||||||
|
def run() -> Dict[str, Any]:
|
||||||
|
if not DB_PATH.exists():
|
||||||
|
return {"success": False, "time": _now(), "error": f"db missing: {DB_PATH}"}
|
||||||
|
|
||||||
|
sources = _load_sources()
|
||||||
|
state = _load_json(STATE_PATH, {"offsets": {}})
|
||||||
|
offsets: Dict[str, int] = state.get("offsets", {}) if isinstance(state, dict) else {}
|
||||||
|
|
||||||
|
store = EngramStore(str(DB_PATH))
|
||||||
|
out = {"success": True, "time": _now(), "sources": len(sources), "messages_saved": 0, "messages_skipped": 0, "errors": []}
|
||||||
|
|
||||||
|
for src in sources:
|
||||||
|
try:
|
||||||
|
if not src.transcript_path.exists():
|
||||||
|
continue
|
||||||
|
key = str(src.transcript_path)
|
||||||
|
start = int(offsets.get(key, 0))
|
||||||
|
for new_off, line in _iter_new_lines(src.transcript_path, start_offset=start):
|
||||||
|
offsets[key] = new_off
|
||||||
|
obj = json.loads(line)
|
||||||
|
if not isinstance(obj, dict) or obj.get("type") != "message":
|
||||||
|
continue
|
||||||
|
msg = obj.get("message") if isinstance(obj.get("message"), dict) else {}
|
||||||
|
role = str(msg.get("role") or "unknown")
|
||||||
|
content = _extract_text(msg.get("content"))
|
||||||
|
if len(content.strip()) < 5:
|
||||||
|
continue
|
||||||
|
mid = str(obj.get("id") or msg.get("id") or msg.get("messageId") or msg.get("message_id") or "")
|
||||||
|
if not mid:
|
||||||
|
mid = str(uuid5(NAMESPACE_URL, f"openclaw-transcript:{src.label}:{role}:{content[:200]}"))
|
||||||
|
eid = str(uuid5(NAMESPACE_URL, f"openclaw-transcript:{src.label}:{mid}"))
|
||||||
|
if store.get(eid):
|
||||||
|
out["messages_skipped"] += 1
|
||||||
|
continue
|
||||||
|
eg = Engram.create(
|
||||||
|
content=f"[transcript:{src.label}] [{role}] [{mid}]\n\n{content}"[:4000],
|
||||||
|
source="session",
|
||||||
|
tags=["session", "openclaw", "transcript", f"role:{role}"],
|
||||||
|
session_id=src.label,
|
||||||
|
confidence=0.55,
|
||||||
|
grounding=Grounding.ASSUMPTION,
|
||||||
|
)
|
||||||
|
eg.id = uuid5(NAMESPACE_URL, eid)
|
||||||
|
eg.metadata["source"] = "session"
|
||||||
|
eg.metadata["session_id"] = src.label
|
||||||
|
eg.metadata["role"] = role
|
||||||
|
eg.metadata["message_id"] = mid
|
||||||
|
store.save(eg)
|
||||||
|
out["messages_saved"] += 1
|
||||||
|
except Exception as e:
|
||||||
|
out["success"] = False
|
||||||
|
out["errors"].append(f"{src.transcript_path}: {e}")
|
||||||
|
|
||||||
|
_save_json(STATE_PATH, {"offsets": offsets, "updated_at": out["time"]})
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
print(json.dumps(run(), ensure_ascii=False, indent=2))
|
||||||
|
|
||||||
213
cron_tasks/ingest_transcript_to_memory.py
Normal file
213
cron_tasks/ingest_transcript_to_memory.py
Normal file
@@ -0,0 +1,213 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""
|
||||||
|
Tail OpenClaw session transcript JSONL files and append new messages into
|
||||||
|
workspace/memory/YYYY-MM-DD.md so the existing ingest_memory pipeline can pick
|
||||||
|
them up.
|
||||||
|
|
||||||
|
Why: when chat_autosave hooks are missed/aborted, the "memory/*.md -> DB" ingest
|
||||||
|
doesn't see the latest conversation. This bridges transcript -> memory.
|
||||||
|
|
||||||
|
Safety: read-only access to transcript files; state stored in workspace/memory/.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Any, Dict, Iterable, List, Optional
|
||||||
|
|
||||||
|
try:
|
||||||
|
from zoneinfo import ZoneInfo # py3.9+
|
||||||
|
except Exception: # pragma: no cover
|
||||||
|
ZoneInfo = None # type: ignore
|
||||||
|
|
||||||
|
|
||||||
|
WORKSPACE = Path("/root/.openclaw/workspace")
|
||||||
|
MEMORY_DIR = WORKSPACE / "memory"
|
||||||
|
SOURCES_PATH = MEMORY_DIR / "session_sources.json"
|
||||||
|
STATE_PATH = MEMORY_DIR / "session_ingest_state.json"
|
||||||
|
|
||||||
|
|
||||||
|
def _local_tz():
|
||||||
|
tz = os.environ.get("TZ") or "Europe/Berlin"
|
||||||
|
if ZoneInfo:
|
||||||
|
try:
|
||||||
|
return ZoneInfo(tz)
|
||||||
|
except Exception:
|
||||||
|
return timezone.utc
|
||||||
|
return timezone.utc
|
||||||
|
|
||||||
|
|
||||||
|
def _load_json(path: Path, default: Any) -> Any:
|
||||||
|
try:
|
||||||
|
if not path.exists():
|
||||||
|
return default
|
||||||
|
return json.loads(path.read_text(encoding="utf-8"))
|
||||||
|
except Exception:
|
||||||
|
return default
|
||||||
|
|
||||||
|
|
||||||
|
def _save_json(path: Path, payload: Any) -> None:
|
||||||
|
path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
path.write_text(json.dumps(payload, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
|
||||||
|
|
||||||
|
|
||||||
|
def _iso_to_dt(ts: str) -> Optional[datetime]:
|
||||||
|
try:
|
||||||
|
if ts.endswith("Z"):
|
||||||
|
ts = ts[:-1] + "+00:00"
|
||||||
|
return datetime.fromisoformat(ts)
|
||||||
|
except Exception:
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _ms_to_dt(ms: int) -> datetime:
|
||||||
|
return datetime.fromtimestamp(ms / 1000, tz=timezone.utc)
|
||||||
|
|
||||||
|
|
||||||
|
def _extract_text(content: Any) -> str:
|
||||||
|
if isinstance(content, str):
|
||||||
|
return content.strip()
|
||||||
|
if isinstance(content, list):
|
||||||
|
parts: List[str] = []
|
||||||
|
for c in content:
|
||||||
|
if isinstance(c, dict) and c.get("type") == "text" and isinstance(c.get("text"), str):
|
||||||
|
parts.append(c["text"])
|
||||||
|
elif isinstance(c, str):
|
||||||
|
parts.append(c)
|
||||||
|
return "\n".join(p.strip() for p in parts if p and p.strip()).strip()
|
||||||
|
if isinstance(content, dict):
|
||||||
|
if isinstance(content.get("text"), str):
|
||||||
|
return content["text"].strip()
|
||||||
|
return str(content).strip()
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class Source:
|
||||||
|
label: str
|
||||||
|
transcript_path: Path
|
||||||
|
|
||||||
|
|
||||||
|
def _load_sources() -> List[Source]:
|
||||||
|
"""
|
||||||
|
Sources file format:
|
||||||
|
{
|
||||||
|
"sources": [
|
||||||
|
{ "label": "telegram:263887248", "path": "/root/.openclaw/agents/main/sessions/<id>.jsonl" }
|
||||||
|
]
|
||||||
|
}
|
||||||
|
"""
|
||||||
|
payload = _load_json(SOURCES_PATH, {"sources": []})
|
||||||
|
sources: List[Source] = []
|
||||||
|
for item in payload.get("sources", []) if isinstance(payload, dict) else []:
|
||||||
|
if not isinstance(item, dict):
|
||||||
|
continue
|
||||||
|
label = str(item.get("label") or "session")
|
||||||
|
p = item.get("path")
|
||||||
|
if not isinstance(p, str) or not p:
|
||||||
|
continue
|
||||||
|
sources.append(Source(label=label, transcript_path=Path(p)))
|
||||||
|
return sources
|
||||||
|
|
||||||
|
|
||||||
|
def _memory_path_for(dt: datetime) -> Path:
|
||||||
|
tz = _local_tz()
|
||||||
|
local = dt.astimezone(tz)
|
||||||
|
MEMORY_DIR.mkdir(parents=True, exist_ok=True)
|
||||||
|
return MEMORY_DIR / f"{local.date().isoformat()}.md"
|
||||||
|
|
||||||
|
|
||||||
|
def _append_memory(dt: datetime, label: str, role: str, text: str) -> None:
|
||||||
|
if not text.strip():
|
||||||
|
return
|
||||||
|
tz = _local_tz()
|
||||||
|
local = dt.astimezone(tz)
|
||||||
|
mem = _memory_path_for(dt)
|
||||||
|
if not mem.exists():
|
||||||
|
mem.write_text(f"# {local.date().isoformat()}\n\n", encoding="utf-8")
|
||||||
|
|
||||||
|
header = f"## {local.strftime('%H:%M:%S')} - {label} ({role})"
|
||||||
|
body = text.strip()
|
||||||
|
mem.write_text(mem.read_text(encoding="utf-8") + f"{header}\n\n{body}\n\n", encoding="utf-8")
|
||||||
|
|
||||||
|
|
||||||
|
def _iter_new_lines(path: Path, *, start_offset: int) -> Iterable[tuple[int, str]]:
|
||||||
|
with open(path, "rb") as f:
|
||||||
|
f.seek(max(0, int(start_offset)))
|
||||||
|
while True:
|
||||||
|
raw = f.readline()
|
||||||
|
if not raw:
|
||||||
|
break
|
||||||
|
try:
|
||||||
|
line = raw.decode("utf-8", errors="ignore").strip()
|
||||||
|
except Exception:
|
||||||
|
line = ""
|
||||||
|
if not line:
|
||||||
|
continue
|
||||||
|
yield (f.tell(), line)
|
||||||
|
|
||||||
|
|
||||||
|
def run() -> Dict[str, Any]:
|
||||||
|
sources = _load_sources()
|
||||||
|
state = _load_json(STATE_PATH, {"offsets": {}})
|
||||||
|
offsets: Dict[str, int] = state.get("offsets", {}) if isinstance(state, dict) else {}
|
||||||
|
|
||||||
|
out = {
|
||||||
|
"success": True,
|
||||||
|
"time": datetime.now(timezone.utc).isoformat(),
|
||||||
|
"sources": len(sources),
|
||||||
|
"messages_appended": 0,
|
||||||
|
"errors": [],
|
||||||
|
}
|
||||||
|
|
||||||
|
for src in sources:
|
||||||
|
try:
|
||||||
|
if not src.transcript_path.exists():
|
||||||
|
continue
|
||||||
|
key = str(src.transcript_path)
|
||||||
|
start = int(offsets.get(key, 0))
|
||||||
|
for new_off, line in _iter_new_lines(src.transcript_path, start_offset=start):
|
||||||
|
try:
|
||||||
|
obj = json.loads(line)
|
||||||
|
except Exception:
|
||||||
|
offsets[key] = new_off
|
||||||
|
continue
|
||||||
|
|
||||||
|
if not isinstance(obj, dict) or obj.get("type") != "message":
|
||||||
|
offsets[key] = new_off
|
||||||
|
continue
|
||||||
|
|
||||||
|
msg = obj.get("message") if isinstance(obj.get("message"), dict) else {}
|
||||||
|
role = str(msg.get("role") or "unknown")
|
||||||
|
content = msg.get("content")
|
||||||
|
text = _extract_text(content)
|
||||||
|
|
||||||
|
dt = None
|
||||||
|
if isinstance(msg.get("timestamp"), (int, float)):
|
||||||
|
dt = _ms_to_dt(int(msg["timestamp"]))
|
||||||
|
elif isinstance(obj.get("timestamp"), str):
|
||||||
|
dt = _iso_to_dt(obj["timestamp"])
|
||||||
|
if dt is None:
|
||||||
|
dt = datetime.now(timezone.utc)
|
||||||
|
|
||||||
|
if len(text.strip()) < 5:
|
||||||
|
offsets[key] = new_off
|
||||||
|
continue
|
||||||
|
|
||||||
|
_append_memory(dt, src.label, role, text)
|
||||||
|
out["messages_appended"] += 1
|
||||||
|
offsets[key] = new_off
|
||||||
|
except Exception as e:
|
||||||
|
out["success"] = False
|
||||||
|
out["errors"].append(f"{src.transcript_path}: {e}")
|
||||||
|
|
||||||
|
_save_json(STATE_PATH, {"offsets": offsets, "updated_at": out["time"]})
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
res = run()
|
||||||
|
print(json.dumps(res, ensure_ascii=False, indent=2))
|
||||||
141
cron_tasks/verify_pending_external.py
Executable file
141
cron_tasks/verify_pending_external.py
Executable file
@@ -0,0 +1,141 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""
|
||||||
|
Verify pending (unconfirmed) engrams using lightweight external checks.
|
||||||
|
|
||||||
|
Policy (conservative):
|
||||||
|
- `openclaw-memory` is treated as internal ground-truth and is auto-confirmed
|
||||||
|
by the review job (see `cron_tasks/review_brain.py` in the workspace runtime).
|
||||||
|
- For `source=web`, confirm if the grounded URL responds with HTTP 2xx, reject on
|
||||||
|
4xx/5xx, and keep pending on timeouts/unknown.
|
||||||
|
- Reject obvious low-signal placeholders (e.g. session summary stubs).
|
||||||
|
"""
|
||||||
|
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
from pathlib import Path
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
from typing import Any, Optional
|
||||||
|
|
||||||
|
|
||||||
|
WORKSPACE = Path(os.environ.get("SECOND_BRAIN_WORKSPACE", "/root/.openclaw/workspace/second-brain"))
|
||||||
|
DB_PATH = Path(os.environ.get("BRAIN_DB", str(WORKSPACE / "data" / "brain.sqlite"))).resolve()
|
||||||
|
|
||||||
|
sys.path.insert(0, str(WORKSPACE))
|
||||||
|
from src.store import EngramStore
|
||||||
|
from src.engram import ReviewEntry
|
||||||
|
|
||||||
|
OUTPUT_FILE = os.environ.get("CRON_OUTPUT_FILE", "/tmp/verify_pending_external.json")
|
||||||
|
|
||||||
|
|
||||||
|
def _now() -> str:
|
||||||
|
return datetime.now(timezone.utc).isoformat()
|
||||||
|
|
||||||
|
|
||||||
|
def _get_url(meta: dict[str, Any]) -> Optional[str]:
|
||||||
|
url = meta.get("url")
|
||||||
|
if isinstance(url, str) and url.startswith(("http://", "https://")):
|
||||||
|
return url
|
||||||
|
grounding = meta.get("grounding")
|
||||||
|
if isinstance(grounding, dict):
|
||||||
|
g_url = grounding.get("url")
|
||||||
|
if isinstance(g_url, str) and g_url.startswith(("http://", "https://")):
|
||||||
|
return g_url
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _http_status(url: str, timeout_s: float = 6.0) -> Optional[int]:
|
||||||
|
try:
|
||||||
|
import urllib.request
|
||||||
|
|
||||||
|
req = urllib.request.Request(
|
||||||
|
url,
|
||||||
|
method="GET",
|
||||||
|
headers={"User-Agent": "openclaw-secondbrain/verify_pending_external"},
|
||||||
|
)
|
||||||
|
with urllib.request.urlopen(req, timeout=timeout_s) as resp:
|
||||||
|
return int(getattr(resp, "status", 200))
|
||||||
|
except Exception:
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> int:
|
||||||
|
if not DB_PATH.exists():
|
||||||
|
out = {"success": False, "error": f"db missing: {DB_PATH}", "time": _now()}
|
||||||
|
Path(OUTPUT_FILE).write_text(json.dumps(out, indent=2))
|
||||||
|
print(out["error"])
|
||||||
|
return 1
|
||||||
|
|
||||||
|
store = EngramStore(str(DB_PATH))
|
||||||
|
all_egs = []
|
||||||
|
offset = 0
|
||||||
|
while True:
|
||||||
|
batch = store.get_all(limit=2000, offset=offset)
|
||||||
|
if not batch:
|
||||||
|
break
|
||||||
|
all_egs.extend(batch)
|
||||||
|
offset += len(batch)
|
||||||
|
pending = [
|
||||||
|
eg
|
||||||
|
for eg in all_egs
|
||||||
|
if (not eg.correctness.is_final())
|
||||||
|
]
|
||||||
|
|
||||||
|
confirmed = 0
|
||||||
|
rejected = 0
|
||||||
|
still_pending = 0
|
||||||
|
checked = 0
|
||||||
|
|
||||||
|
for eg in pending:
|
||||||
|
checked += 1
|
||||||
|
src = eg.metadata.get("source")
|
||||||
|
content = (eg.content or "").strip()
|
||||||
|
|
||||||
|
if src == "session" and (
|
||||||
|
content.startswith("Session Summary (sess_") or content.startswith("Please remember ")
|
||||||
|
):
|
||||||
|
eg.correctness.reject("verify-pending", "Auto-reject: session placeholder")
|
||||||
|
store.save(eg)
|
||||||
|
rejected += 1
|
||||||
|
continue
|
||||||
|
|
||||||
|
if src == "web":
|
||||||
|
url = _get_url(eg.metadata)
|
||||||
|
if not url:
|
||||||
|
still_pending += 1
|
||||||
|
continue
|
||||||
|
status = _http_status(url)
|
||||||
|
if status is None:
|
||||||
|
still_pending += 1
|
||||||
|
continue
|
||||||
|
if 200 <= status < 300:
|
||||||
|
eg.correctness.confirm("verify-pending", f"Auto-confirm: web url ok ({status}) {url}")
|
||||||
|
store.save(eg)
|
||||||
|
confirmed += 1
|
||||||
|
else:
|
||||||
|
eg.correctness.reject("verify-pending", f"Auto-reject: web url status={status} {url}")
|
||||||
|
store.save(eg)
|
||||||
|
rejected += 1
|
||||||
|
continue
|
||||||
|
|
||||||
|
still_pending += 1
|
||||||
|
|
||||||
|
out = {
|
||||||
|
"success": True,
|
||||||
|
"time": _now(),
|
||||||
|
"total": len(all_egs),
|
||||||
|
"pending_before": len(pending),
|
||||||
|
"checked": checked,
|
||||||
|
"confirmed": confirmed,
|
||||||
|
"rejected": rejected,
|
||||||
|
"still_pending": still_pending,
|
||||||
|
}
|
||||||
|
Path(OUTPUT_FILE).write_text(json.dumps(out, indent=2))
|
||||||
|
print(
|
||||||
|
f"VERIFY: pending_before={out['pending_before']} confirmed={confirmed} rejected={rejected} still_pending={still_pending}"
|
||||||
|
)
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
raise SystemExit(main())
|
||||||
@@ -1,6 +1,6 @@
|
|||||||
# Second-Brain 2.0 — Release Checklist (Grundversion + FastAPI)
|
# Second-Brain 2.0 — Release Checklist (Grundversion + FastAPI)
|
||||||
|
|
||||||
Scope: OpenClaw workspace at `/root/.openclaw/workspace` with `second-brain/` and the shipped `systemd/` unit files.
|
Scope: OpenClaw workspace at `/root/.openclaw/workspace` with `second-brain/` and the shipped `second-brain/systemd/` unit files.
|
||||||
|
|
||||||
This checklist is copy/paste friendly. Run as a user with `sudo` (systemd commands need it).
|
This checklist is copy/paste friendly. Run as a user with `sudo` (systemd commands need it).
|
||||||
|
|
||||||
@@ -24,13 +24,21 @@ ls -la /etc/systemd/system/openclaw-memory-archive.* 2>/dev/null || true
|
|||||||
If missing, install them (symlink is fine; copy is fine too):
|
If missing, install them (symlink is fine; copy is fine too):
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
sudo ln -sf /root/.openclaw/workspace/systemd/openclaw-secondbrain-*.service /etc/systemd/system/
|
sudo ln -sf /root/.openclaw/workspace/second-brain/systemd/openclaw-secondbrain-*.service /etc/systemd/system/
|
||||||
sudo ln -sf /root/.openclaw/workspace/systemd/openclaw-secondbrain-*.timer /etc/systemd/system/
|
sudo ln -sf /root/.openclaw/workspace/second-brain/systemd/openclaw-secondbrain-*.timer /etc/systemd/system/
|
||||||
sudo ln -sf /root/.openclaw/workspace/systemd/openclaw-memory-archive.* /etc/systemd/system/
|
sudo ln -sf /root/.openclaw/workspace/second-brain/systemd/openclaw-memory-archive.* /etc/systemd/system/
|
||||||
sudo ln -sf /root/.openclaw/workspace/systemd/openclaw-secondbrain-dashboard.service /etc/systemd/system/
|
sudo ln -sf /root/.openclaw/workspace/second-brain/systemd/openclaw-secondbrain-dashboard.service /etc/systemd/system/
|
||||||
sudo systemctl daemon-reload
|
sudo systemctl daemon-reload
|
||||||
```
|
```
|
||||||
|
|
||||||
|
Optional (verification hardening):
|
||||||
|
|
||||||
|
```bash
|
||||||
|
sudo ln -sf /root/.openclaw/workspace/second-brain/systemd/openclaw-secondbrain-verify-pending.* /etc/systemd/system/
|
||||||
|
sudo systemctl daemon-reload
|
||||||
|
sudo systemctl enable --now openclaw-secondbrain-verify-pending.timer
|
||||||
|
```
|
||||||
|
|
||||||
### Enable timers
|
### Enable timers
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
|
|||||||
215
docs/dashboard-ui-ux-design.md
Normal file
215
docs/dashboard-ui-ux-design.md
Normal file
@@ -0,0 +1,215 @@
|
|||||||
|
# Second-Brain Dashboard UI/UX Design Plan (mobile-first)
|
||||||
|
|
||||||
|
Basis: `second-brain/templates/dashboard.html`, `second-brain/static/style.css` (aktueller Stand) + neue Graph-Controls aus `/tmp/second-brain-staging/`.
|
||||||
|
|
||||||
|
## 1) Konkreter Design-Plan
|
||||||
|
|
||||||
|
### Ziele (UX)
|
||||||
|
- **Schnelles Triaging unterwegs:** Pending/Errors sofort sichtbar, 1-Hand-Bedienung.
|
||||||
|
- **Konsistentes Design-System:** einheitliche Buttons/Inputs/Panels statt Einzellösungen.
|
||||||
|
- **Graph als Diagnose-Tool:** klare Controls, Legende, nachvollziehbares Feedback (Loading/Empty/Errors).
|
||||||
|
|
||||||
|
### Farbschema (Dark, high-contrast, "indigo + emerald")
|
||||||
|
- **Background:** sehr dunkel (nahe #0f1117) für weniger Blendung.
|
||||||
|
- **Surface (Cards/Panels):** abgestufte Flächen (Surface-1, Surface-2) für Hierarchie.
|
||||||
|
- **Primary:** Indigo/Blue für interaktive Elemente und Highlights (bisher #6c8af5 bleibt als Basis).
|
||||||
|
- **Success:** Emerald für Confirm/OK (bisher #3a7d3a → etwas heller/satter).
|
||||||
|
- **Danger:** Red für Errors/Reject.
|
||||||
|
- **Warning:** Amber für Pending/Match/Attention.
|
||||||
|
- **Text:** fast-weiß, sekundär gedimmt.
|
||||||
|
|
||||||
|
### Typografie
|
||||||
|
- Systemfont-Stack (schnell, gut lesbar): `ui-sans-serif, system-ui, -apple-system, Segoe UI, Roboto, Arial`.
|
||||||
|
- Skala (mobile-first):
|
||||||
|
- `text-xs` 12px (Labels, Meta)
|
||||||
|
- `text-sm` 13–14px (secondary)
|
||||||
|
- `text-md` 15–16px (Body)
|
||||||
|
- `text-lg` 18–20px (Titles)
|
||||||
|
- **Ziffern:** optional `font-variant-numeric: tabular-nums;` für Stats.
|
||||||
|
|
||||||
|
### Spacing/Rhythm
|
||||||
|
- 4px Grid; Standard-Gaps: 8/12/16.
|
||||||
|
- Container-Padding: 12px (mobile), 16–20px (>=768px).
|
||||||
|
- Border-radius: 12–16px (Cards/Modals), 10px (Inputs/Buttons), 999px (Pills).
|
||||||
|
|
||||||
|
### Komponenten (konkret)
|
||||||
|
|
||||||
|
#### A) Tabs
|
||||||
|
- Tabs bleiben als 3 Buttons, aber:
|
||||||
|
- **Active State** deutlicher (background + border + subtle glow).
|
||||||
|
- **Touch target** min. 44px Höhe.
|
||||||
|
- Sticky bleibt, aber mit leichter **Backdrop-Blurring** (wenn möglich) oder solid surface.
|
||||||
|
|
||||||
|
#### B) Search
|
||||||
|
- Search-Row wird als **kompakte Toolbar** gestaltet:
|
||||||
|
- Input + Filter in einer Zeile.
|
||||||
|
- Optional Quick-Chips darunter: `All / Pending / Confirmed / Errors` (klickbar) als Alternative zum Select.
|
||||||
|
- Clear-Button (×) im Input (per CSS `::-webkit-search-cancel-button` oder eigener Button) für Mobile.
|
||||||
|
|
||||||
|
#### C) Cards
|
||||||
|
- Karte als 3 Zonen: Header (badges/tags/date), Body (content), Footer (actions).
|
||||||
|
- Status wird stärker codiert:
|
||||||
|
- left-border + kleine **Status-Pill** (OK/Pending/Error) mit eindeutiger Farbe.
|
||||||
|
- Body: bessere Lesbarkeit via `line-height: 1.55` und max-height/clamp optional.
|
||||||
|
|
||||||
|
#### D) Modal
|
||||||
|
- Modal als **Bottom Sheet** auf Mobile (>=50vh) + klassisches Center-Modal auf Desktop.
|
||||||
|
- Close-Button größer + Tap-Area.
|
||||||
|
- Inhalt in Tabs/Sections (History/Meta/Content) optional später.
|
||||||
|
|
||||||
|
#### E) Graph + Controls (aus Staging)
|
||||||
|
- Controls als **Control Bar** oberhalb Canvas:
|
||||||
|
- Primary: Physics toggle.
|
||||||
|
- Secondary: Fit, Reset, Reload.
|
||||||
|
- Text labels kurz (z.B. `Physics` statt `Physics: off`, state als Badge).
|
||||||
|
- Canvas passt sich an Viewport an:
|
||||||
|
- `width: min(100%, 560px)`; Height: `min(65vh, 560px)` (CSS statt fester HTML Attribute, wenn möglich).
|
||||||
|
- Legende als einklappbares Panel (`Details`/`summary`) oder leichtes Panel unter Canvas.
|
||||||
|
|
||||||
|
#### F) Status Panels
|
||||||
|
- Status-View nutzt vorhandene `.panel`/`.kv-*`:
|
||||||
|
- Gruppen: System, Storage, Jobs, Insights, Pending Queue.
|
||||||
|
- Jede Gruppe als Panel mit klarer Title Row.
|
||||||
|
- Kritische Werte (Errors/Pending/Queue) farblich markieren.
|
||||||
|
|
||||||
|
## 2) CSS-Variablen (Theme-Tokens) + Mapping
|
||||||
|
|
||||||
|
### Token-Vorschlag (`:root`)
|
||||||
|
```css
|
||||||
|
:root {
|
||||||
|
/* typography */
|
||||||
|
--font-sans: ui-sans-serif, system-ui, -apple-system, Segoe UI, Roboto, Arial, sans-serif;
|
||||||
|
|
||||||
|
/* colors */
|
||||||
|
--bg: #0f1117;
|
||||||
|
--surface-1: #14151d;
|
||||||
|
--surface-2: #1a1b26;
|
||||||
|
--border: #2a2d3a;
|
||||||
|
|
||||||
|
--text: #e8e8ee;
|
||||||
|
--text-muted: #8b90a3;
|
||||||
|
--text-dim: #5c6276;
|
||||||
|
|
||||||
|
--primary: #6c8af5; /* existing */
|
||||||
|
--primary-2: #8aa1ff;
|
||||||
|
--success: #2fbf71;
|
||||||
|
--danger: #ef4444;
|
||||||
|
--warning: #f7d154; /* existing-ish */
|
||||||
|
|
||||||
|
/* shadows */
|
||||||
|
--shadow-1: 0 1px 0 rgba(0,0,0,.25), 0 8px 24px rgba(0,0,0,.35);
|
||||||
|
|
||||||
|
/* radius */
|
||||||
|
--r-sm: 10px;
|
||||||
|
--r-md: 14px;
|
||||||
|
--r-lg: 16px;
|
||||||
|
|
||||||
|
/* spacing */
|
||||||
|
--s-1: 4px;
|
||||||
|
--s-2: 8px;
|
||||||
|
--s-3: 12px;
|
||||||
|
--s-4: 16px;
|
||||||
|
--s-5: 20px;
|
||||||
|
|
||||||
|
/* control sizes */
|
||||||
|
--tap: 44px;
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
### Mapping auf existierende Selektoren
|
||||||
|
- `body`/`.app`
|
||||||
|
- `background: var(--bg)` statt `#141419`
|
||||||
|
- `font-family: var(--font-sans)`
|
||||||
|
- `color: var(--text)`
|
||||||
|
- `.stats-bar`, `.tabs-bar`, `.search-box`
|
||||||
|
- `background: var(--surface-1)` (Stats ggf. Gradient bleibt, aber auf Tokens)
|
||||||
|
- `border-bottom: 1px solid var(--border)`
|
||||||
|
- `.panel`, `.card`, `.modal-content`, `.graph-legend`
|
||||||
|
- `background: var(--surface-2)`
|
||||||
|
- `border: 1px solid var(--border)`
|
||||||
|
- `border-radius: var(--r-md)`
|
||||||
|
- `.stat-num`, `#pageNum`, `.tab-btn.active`, `#searchInput:focus`
|
||||||
|
- `color/border-color: var(--primary)`
|
||||||
|
- `.muted`, `.stat-label`, `.kv-key`, `.date`
|
||||||
|
- `color: var(--text-muted)` bzw. `var(--text-dim)`
|
||||||
|
- Buttons
|
||||||
|
- vereinheitlichen über `.btn` + Modifier: `.btn.primary`, `.btn.danger`, `.btn.success`, `.btn.ghost`
|
||||||
|
- `min-height: var(--tap)` für Touch
|
||||||
|
|
||||||
|
## 3) 5–10 priorisierte UI-Änderungen (mit Begründung)
|
||||||
|
|
||||||
|
1) **Design Tokens (CSS vars) einführen** → reduziert Farbmix, erleichtert spätere Themes/Anpassungen.
|
||||||
|
2) **Einheitliche Button-Komponente (`.btn`)** (inkl. `:active`, `:disabled`, `min-height`) → bessere Touch-UX, konsistente Interaktion.
|
||||||
|
3) **Graph-Controls + Legende aus Staging in den Main-Template-Stand ziehen** → Graph wird tatsächlich bedienbar/selbsterklärend.
|
||||||
|
4) **Responsive Graph-Canvas (CSS gesteuert)** statt fixer `width/height` → bessere Nutzung auf Phones, weniger Scroll.
|
||||||
|
5) **Search als Toolbar + Clear-Action** → schnelleres Filtern unterwegs, weniger Friktion.
|
||||||
|
6) **Modal als Bottom-Sheet auf Mobile** → angenehmer für längeren Content + History, weniger „winziges Fenster“.
|
||||||
|
7) **Status/Health Werte farblich akzentuieren** (pending/errors/warn) → schnelleres Erkennen von Problemen.
|
||||||
|
8) **Cards: Status-Pill + typografische Lesbarkeit** (line-height, spacing) → weniger „Textblock“, bessere Scanbarkeit.
|
||||||
|
9) **Accessibility-Basics**: Focus-Rings, Kontrast, `prefers-reduced-motion` → weniger „invisible focus“ und bessere Bedienbarkeit.
|
||||||
|
10) **Top-level Layout Max-Width für Desktop** (z.B. 560–720px) → verhindert „zu breite“ Zeilen.
|
||||||
|
|
||||||
|
## 4) Optionale Patch-Vorschläge (Diff-Snippets, NICHT anwenden)
|
||||||
|
|
||||||
|
> Hinweis: Snippets sind bewusst klein gehalten. Gesamt < 120 Zeilen.
|
||||||
|
|
||||||
|
### Snippet A — Tokens + Button-System (style.css)
|
||||||
|
```diff
|
||||||
|
--- a/second-brain/static/style.css
|
||||||
|
+++ b/second-brain/static/style.css
|
||||||
|
@@
|
||||||
|
+:root {
|
||||||
|
+ --font-sans: ui-sans-serif, system-ui, -apple-system, Segoe UI, Roboto, Arial, sans-serif;
|
||||||
|
+ --bg:#0f1117; --surface-1:#14151d; --surface-2:#1a1b26; --border:#2a2d3a;
|
||||||
|
+ --text:#e8e8ee; --text-muted:#8b90a3; --text-dim:#5c6276;
|
||||||
|
+ --primary:#6c8af5; --success:#2fbf71; --danger:#ef4444; --warning:#f7d154;
|
||||||
|
+ --r-sm:10px; --r-md:14px; --r-lg:16px;
|
||||||
|
+ --s-2:8px; --s-3:12px; --s-4:16px;
|
||||||
|
+ --tap:44px;
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+body { font-family: var(--font-sans); background: var(--bg); color: var(--text); }
|
||||||
|
+
|
||||||
|
+.btn{
|
||||||
|
+ min-height: var(--tap);
|
||||||
|
+ background: #1e1e28;
|
||||||
|
+ border: 1px solid var(--border);
|
||||||
|
+ border-radius: var(--r-sm);
|
||||||
|
+ padding: 8px 12px;
|
||||||
|
+ color: #cfd3ff;
|
||||||
|
+ font-weight: 700;
|
||||||
|
+}
|
||||||
|
+.btn.primary{ border-color: var(--primary); box-shadow: 0 0 0 1px rgba(108,138,245,0.18) inset; }
|
||||||
|
+.btn.success{ background: rgba(47,191,113,.18); border-color: rgba(47,191,113,.35); }
|
||||||
|
+.btn.danger{ background: rgba(239,68,68,.16); border-color: rgba(239,68,68,.35); }
|
||||||
|
+.btn:active{ transform: scale(.98); }
|
||||||
|
+.btn:disabled{ opacity: .45; }
|
||||||
|
```
|
||||||
|
|
||||||
|
### Snippet B — Graph Controls + Legend übernehmen (dashboard.html)
|
||||||
|
```diff
|
||||||
|
--- a/second-brain/templates/dashboard.html
|
||||||
|
+++ b/second-brain/templates/dashboard.html
|
||||||
|
@@
|
||||||
|
- <div class="graph" id="graph" style="display:none;">
|
||||||
|
- <canvas id="graphCanvas" width="440" height="520"></canvas>
|
||||||
|
- <div class="muted small" id="graphHint">Lade Graph…</div>
|
||||||
|
- </div>
|
||||||
|
+ <div class="graph" id="graph" style="display:none;">
|
||||||
|
+ <div class="graph-controls">
|
||||||
|
+ <button class="btn primary" id="btnGraphPhysics" onclick="toggleGraphPhysics()">Physics</button>
|
||||||
|
+ <button class="btn" onclick="resetGraphView()">Reset</button>
|
||||||
|
+ <button class="btn" onclick="fitGraphView()">Fit</button>
|
||||||
|
+ <button class="btn" onclick="reloadGraph()">Reload</button>
|
||||||
|
+ </div>
|
||||||
|
+ <canvas id="graphCanvas" width="440" height="520"></canvas>
|
||||||
|
+ <div class="graph-hint muted small" id="graphHint">Lade Graph…</div>
|
||||||
|
+ <div class="graph-legend">
|
||||||
|
+ <div><strong>Graph</strong>: Zoom (Wheel/Pinch), Pan (Drag). Klick auf Engram öffnet Details, Klick auf Tag setzt Suche.</div>
|
||||||
|
+ <div class="legend-row"><span class="legend-dot engram"></span> Engram</div>
|
||||||
|
+ <div class="legend-row"><span class="legend-dot tag"></span> Tag</div>
|
||||||
|
+ <div class="legend-row"><span class="legend-dot match"></span> Match</div>
|
||||||
|
+ </div>
|
||||||
|
+ </div>
|
||||||
|
```
|
||||||
|
|
||||||
727
fastapi_app.py
727
fastapi_app.py
@@ -11,11 +11,14 @@ Goals:
|
|||||||
import json
|
import json
|
||||||
import os
|
import os
|
||||||
import sqlite3
|
import sqlite3
|
||||||
|
import subprocess
|
||||||
from datetime import datetime, timezone
|
from datetime import datetime, timezone
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
from uuid import uuid4
|
||||||
|
from urllib.parse import urlparse
|
||||||
|
|
||||||
from fastapi import FastAPI, Form, Query, Request
|
from fastapi import FastAPI, Form, Query, Request
|
||||||
from fastapi.responses import HTMLResponse, JSONResponse, PlainTextResponse
|
from fastapi.responses import HTMLResponse, JSONResponse, PlainTextResponse, StreamingResponse
|
||||||
from fastapi.staticfiles import StaticFiles
|
from fastapi.staticfiles import StaticFiles
|
||||||
|
|
||||||
# ─── Config ──────────────────────────────────────────────────────────────────
|
# ─── Config ──────────────────────────────────────────────────────────────────
|
||||||
@@ -51,11 +54,22 @@ def get_db():
|
|||||||
def parse_engram(row: sqlite3.Row) -> dict:
|
def parse_engram(row: sqlite3.Row) -> dict:
|
||||||
meta = json.loads(row["metadata_json"] or "{}")
|
meta = json.loads(row["metadata_json"] or "{}")
|
||||||
correctness = json.loads(row["correctness_json"] or "{}")
|
correctness = json.loads(row["correctness_json"] or "{}")
|
||||||
|
verdict = correctness.get("verdict")
|
||||||
|
if not isinstance(verdict, str) or not verdict:
|
||||||
|
# Back-compat inference for older rows
|
||||||
|
if correctness.get("confirmed", False):
|
||||||
|
verdict = "confirmed_true"
|
||||||
|
elif int(correctness.get("rejections", 0) or 0) > 0:
|
||||||
|
verdict = "confirmed_false"
|
||||||
|
else:
|
||||||
|
verdict = "unknown"
|
||||||
return {
|
return {
|
||||||
"id": row["id"],
|
"id": row["id"],
|
||||||
"content": row["content"],
|
"content": row["content"],
|
||||||
"confidence": meta.get("confidence", 0.0),
|
"confidence": meta.get("confidence", 0.0),
|
||||||
"confirmed": correctness.get("confirmed", False),
|
"confirmed": correctness.get("confirmed", False),
|
||||||
|
"verdict": verdict,
|
||||||
|
"evidence": correctness.get("evidence", []),
|
||||||
"confirmations": correctness.get("confirmations", 0),
|
"confirmations": correctness.get("confirmations", 0),
|
||||||
"rejections": correctness.get("rejections", 0),
|
"rejections": correctness.get("rejections", 0),
|
||||||
"tags": meta.get("tags", []),
|
"tags": meta.get("tags", []),
|
||||||
@@ -69,6 +83,158 @@ def parse_engram(row: sqlite3.Row) -> dict:
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _now_iso() -> str:
|
||||||
|
return datetime.now(timezone.utc).isoformat()
|
||||||
|
|
||||||
|
|
||||||
|
def _update_correctness(engram_id: str, *, action: str, reason: str | None = None) -> dict:
|
||||||
|
"""
|
||||||
|
Update correctness_json for an engram. action: confirm|reject
|
||||||
|
"""
|
||||||
|
conn = get_db()
|
||||||
|
c = conn.cursor()
|
||||||
|
row = c.execute("SELECT correctness_json FROM engrams WHERE id = ?", (engram_id,)).fetchone()
|
||||||
|
if not row:
|
||||||
|
conn.close()
|
||||||
|
raise FileNotFoundError(f"Engram not found: {engram_id}")
|
||||||
|
|
||||||
|
corr = json.loads(row["correctness_json"] or "{}")
|
||||||
|
corr.setdefault("verdict", None)
|
||||||
|
corr.setdefault("evidence", [])
|
||||||
|
corr.setdefault("confirmed", False)
|
||||||
|
corr.setdefault("confirmations", 0)
|
||||||
|
corr.setdefault("rejections", 0)
|
||||||
|
corr.setdefault("review_history", [])
|
||||||
|
corr["last_reviewed"] = _now_iso()
|
||||||
|
|
||||||
|
entry = {
|
||||||
|
"by": "dashboard",
|
||||||
|
"action": action,
|
||||||
|
"at": corr["last_reviewed"],
|
||||||
|
"note": (reason or "").strip(),
|
||||||
|
}
|
||||||
|
try:
|
||||||
|
corr["review_history"].append(entry)
|
||||||
|
except Exception:
|
||||||
|
corr["review_history"] = [entry]
|
||||||
|
|
||||||
|
if action == "confirm":
|
||||||
|
corr["verdict"] = "confirmed_true"
|
||||||
|
corr["confirmed"] = True
|
||||||
|
corr["confirmations"] = int(corr.get("confirmations", 0) or 0) + 1
|
||||||
|
elif action == "reject":
|
||||||
|
corr["verdict"] = "confirmed_false"
|
||||||
|
corr["rejections"] = int(corr.get("rejections", 0) or 0) + 1
|
||||||
|
corr["confirmed"] = False
|
||||||
|
|
||||||
|
# Store minimal evidence for dashboard-driven actions.
|
||||||
|
try:
|
||||||
|
ev = corr.get("evidence")
|
||||||
|
if not isinstance(ev, list):
|
||||||
|
ev = []
|
||||||
|
ev.append(
|
||||||
|
{
|
||||||
|
"kind": "human",
|
||||||
|
"by": "dashboard",
|
||||||
|
"at": corr["last_reviewed"],
|
||||||
|
"action": action,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
corr["evidence"] = ev[-50:] # cap growth
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
c.execute(
|
||||||
|
"UPDATE engrams SET correctness_json = ?, modified_at = ? WHERE id = ?",
|
||||||
|
(json.dumps(corr, ensure_ascii=False), corr["last_reviewed"], engram_id),
|
||||||
|
)
|
||||||
|
conn.commit()
|
||||||
|
conn.close()
|
||||||
|
return {"ok": True}
|
||||||
|
|
||||||
|
|
||||||
|
def _bump_access(engram_id: str) -> dict:
|
||||||
|
conn = get_db()
|
||||||
|
c = conn.cursor()
|
||||||
|
row = c.execute("SELECT metadata_json FROM engrams WHERE id = ?", (engram_id,)).fetchone()
|
||||||
|
if not row:
|
||||||
|
conn.close()
|
||||||
|
raise FileNotFoundError(f"Engram not found: {engram_id}")
|
||||||
|
meta = json.loads(row["metadata_json"] or "{}")
|
||||||
|
meta["access_count"] = int(meta.get("access_count", 0) or 0) + 1
|
||||||
|
meta["last_accessed"] = _now_iso()
|
||||||
|
c.execute(
|
||||||
|
"UPDATE engrams SET metadata_json = ?, modified_at = ? WHERE id = ?",
|
||||||
|
(json.dumps(meta, ensure_ascii=False), meta["last_accessed"], engram_id),
|
||||||
|
)
|
||||||
|
conn.commit()
|
||||||
|
conn.close()
|
||||||
|
return {"ok": True}
|
||||||
|
|
||||||
|
def _safe_json_extract_tags(meta_json: str) -> list[str]:
|
||||||
|
try:
|
||||||
|
d = json.loads(meta_json or "{}")
|
||||||
|
tags = d.get("tags") or []
|
||||||
|
return [t for t in tags if isinstance(t, str)]
|
||||||
|
except Exception:
|
||||||
|
return []
|
||||||
|
|
||||||
|
def _host_from_meta(meta_json: str) -> str | None:
|
||||||
|
try:
|
||||||
|
d = json.loads(meta_json or "{}")
|
||||||
|
grounding = d.get("grounding")
|
||||||
|
url = d.get("url")
|
||||||
|
if isinstance(grounding, dict) and isinstance(grounding.get("url"), str):
|
||||||
|
url = grounding.get("url")
|
||||||
|
if not isinstance(url, str):
|
||||||
|
return None
|
||||||
|
parsed = urlparse(url)
|
||||||
|
return parsed.hostname
|
||||||
|
except Exception:
|
||||||
|
return None
|
||||||
|
|
||||||
|
def _systemd_unit_state(unit: str) -> dict:
|
||||||
|
"""
|
||||||
|
Best-effort systemd status snapshot for a known unit.
|
||||||
|
Never raises; returns minimal fields.
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
out = subprocess.check_output(
|
||||||
|
["systemctl", "show", unit, "--no-page", "--property=ActiveState,SubState,Result,ExecMainStatus,ExecMainCode,ExecMainStartTimestamp,ExecMainExitTimestamp"],
|
||||||
|
text=True,
|
||||||
|
stderr=subprocess.STDOUT,
|
||||||
|
timeout=2,
|
||||||
|
)
|
||||||
|
kv = {}
|
||||||
|
for line in out.splitlines():
|
||||||
|
if "=" in line:
|
||||||
|
k, v = line.split("=", 1)
|
||||||
|
kv[k] = v
|
||||||
|
return {
|
||||||
|
"unit": unit,
|
||||||
|
"active": kv.get("ActiveState"),
|
||||||
|
"sub": kv.get("SubState"),
|
||||||
|
"result": kv.get("Result"),
|
||||||
|
"exit_status": kv.get("ExecMainStatus"),
|
||||||
|
"start_ts": kv.get("ExecMainStartTimestamp"),
|
||||||
|
"exit_ts": kv.get("ExecMainExitTimestamp"),
|
||||||
|
}
|
||||||
|
except Exception as e:
|
||||||
|
return {"unit": unit, "error": str(e)}
|
||||||
|
|
||||||
|
def _dir_size_bytes(path: Path) -> int:
|
||||||
|
total = 0
|
||||||
|
try:
|
||||||
|
for p in path.rglob("*"):
|
||||||
|
try:
|
||||||
|
if p.is_file():
|
||||||
|
total += p.stat().st_size
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
return total
|
||||||
|
|
||||||
# ─── API Endpoints ───────────────────────────────────────────────────────────
|
# ─── API Endpoints ───────────────────────────────────────────────────────────
|
||||||
|
|
||||||
@app.get("/healthz", response_class=PlainTextResponse)
|
@app.get("/healthz", response_class=PlainTextResponse)
|
||||||
@@ -83,6 +249,363 @@ def api_config():
|
|||||||
"db_path": str(DB_PATH),
|
"db_path": str(DB_PATH),
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@app.get("/api/db_info")
|
||||||
|
def api_db_info():
|
||||||
|
if not DB_PATH.exists():
|
||||||
|
raise FileNotFoundError(f"DB not found: {DB_PATH}")
|
||||||
|
st = DB_PATH.stat()
|
||||||
|
return {
|
||||||
|
"db_path": str(DB_PATH),
|
||||||
|
"size_bytes": st.st_size,
|
||||||
|
"mtime": datetime.fromtimestamp(st.st_mtime, tz=timezone.utc).isoformat(),
|
||||||
|
}
|
||||||
|
|
||||||
|
@app.get("/api/storage_stats")
|
||||||
|
def api_storage_stats():
|
||||||
|
conn = get_db()
|
||||||
|
c = conn.cursor()
|
||||||
|
total = c.execute("SELECT COUNT(*) FROM engrams").fetchone()[0]
|
||||||
|
confirmed_true = c.execute(
|
||||||
|
"""
|
||||||
|
SELECT COUNT(*) FROM engrams
|
||||||
|
WHERE (
|
||||||
|
json_extract(correctness_json, '$.verdict') = 'confirmed_true'
|
||||||
|
OR (json_extract(correctness_json, '$.verdict') IS NULL AND json_extract(correctness_json, '$.confirmed') = 1)
|
||||||
|
)
|
||||||
|
"""
|
||||||
|
).fetchone()[0]
|
||||||
|
confirmed_false = c.execute(
|
||||||
|
"""
|
||||||
|
SELECT COUNT(*) FROM engrams
|
||||||
|
WHERE (
|
||||||
|
json_extract(correctness_json, '$.verdict') = 'confirmed_false'
|
||||||
|
OR (json_extract(correctness_json, '$.verdict') IS NULL
|
||||||
|
AND json_extract(correctness_json, '$.confirmed') = 0
|
||||||
|
AND COALESCE(json_extract(correctness_json, '$.rejections'), 0) > 0)
|
||||||
|
)
|
||||||
|
"""
|
||||||
|
).fetchone()[0]
|
||||||
|
sources = {
|
||||||
|
r[0]: r[1]
|
||||||
|
for r in c.execute(
|
||||||
|
"SELECT json_extract(metadata_json, '$.source') AS src, COUNT(*) FROM engrams GROUP BY src ORDER BY COUNT(*) DESC"
|
||||||
|
).fetchall()
|
||||||
|
if r[0] is not None
|
||||||
|
}
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
chroma_dir = WORKSPACE / "data" / "chroma"
|
||||||
|
emb_cache_dir = WORKSPACE / "data" / "embedding_cache"
|
||||||
|
vec_state_path = WORKSPACE / "data" / "vector_index_state.json"
|
||||||
|
|
||||||
|
vec_state = {}
|
||||||
|
if vec_state_path.exists():
|
||||||
|
try:
|
||||||
|
vec_state = json.loads(vec_state_path.read_text())
|
||||||
|
except Exception:
|
||||||
|
vec_state = {}
|
||||||
|
|
||||||
|
obsidian_cfg_path = WORKSPACE / "data" / "obsidian_config.json"
|
||||||
|
obsidian_cfg = None
|
||||||
|
if obsidian_cfg_path.exists():
|
||||||
|
try:
|
||||||
|
obsidian_cfg = json.loads(obsidian_cfg_path.read_text())
|
||||||
|
except Exception:
|
||||||
|
obsidian_cfg = {"raw": obsidian_cfg_path.read_text()[:2000]}
|
||||||
|
|
||||||
|
backup_files = sorted((WORKSPACE / "data").glob("backup_*.jsonl"))
|
||||||
|
|
||||||
|
return {
|
||||||
|
"sql": {
|
||||||
|
"total_engrams": total,
|
||||||
|
"confirmed": confirmed_true,
|
||||||
|
"rejected": confirmed_false,
|
||||||
|
"pending": total - confirmed_true - confirmed_false,
|
||||||
|
"by_source": sources,
|
||||||
|
},
|
||||||
|
"vector": {
|
||||||
|
"chroma_dir": str(chroma_dir),
|
||||||
|
"chroma_size_bytes": _dir_size_bytes(chroma_dir) if chroma_dir.exists() else 0,
|
||||||
|
"embedding_cache_dir": str(emb_cache_dir),
|
||||||
|
"embedding_cache_files": len(list(emb_cache_dir.glob("*.json"))) if emb_cache_dir.exists() else 0,
|
||||||
|
"vector_state": vec_state,
|
||||||
|
},
|
||||||
|
"obsidian": {
|
||||||
|
"config_path": str(obsidian_cfg_path),
|
||||||
|
"configured": bool(obsidian_cfg),
|
||||||
|
"config": obsidian_cfg,
|
||||||
|
},
|
||||||
|
"backups": {
|
||||||
|
"count": len(backup_files),
|
||||||
|
"latest": str(backup_files[-1]) if backup_files else None,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
@app.get("/api/jobs")
|
||||||
|
def api_jobs():
|
||||||
|
# Known units that influence "freshness" of the brain.
|
||||||
|
units = [
|
||||||
|
"openclaw-secondbrain-ingest-memory.service",
|
||||||
|
"openclaw-secondbrain-index-vectors.service",
|
||||||
|
"openclaw-secondbrain-review.service",
|
||||||
|
"openclaw-secondbrain-heartbeat.service",
|
||||||
|
"openclaw-secondbrain-verify-pending.service",
|
||||||
|
]
|
||||||
|
return {"items": [_systemd_unit_state(u) for u in units]}
|
||||||
|
|
||||||
|
@app.get("/api/insights")
|
||||||
|
def api_insights(limit: int = Query(8, ge=1, le=50)):
|
||||||
|
conn = get_db()
|
||||||
|
c = conn.cursor()
|
||||||
|
rows = c.execute(
|
||||||
|
"SELECT id, metadata_json, correctness_json, created_at, modified_at FROM engrams ORDER BY created_at DESC LIMIT 2000"
|
||||||
|
).fetchall()
|
||||||
|
total = c.execute("SELECT COUNT(*) FROM engrams").fetchone()[0]
|
||||||
|
confirmed_true = c.execute(
|
||||||
|
"""
|
||||||
|
SELECT COUNT(*) FROM engrams
|
||||||
|
WHERE (
|
||||||
|
json_extract(correctness_json, '$.verdict') = 'confirmed_true'
|
||||||
|
OR (json_extract(correctness_json, '$.verdict') IS NULL AND json_extract(correctness_json, '$.confirmed') = 1)
|
||||||
|
)
|
||||||
|
"""
|
||||||
|
).fetchone()[0]
|
||||||
|
confirmed_false = c.execute(
|
||||||
|
"""
|
||||||
|
SELECT COUNT(*) FROM engrams
|
||||||
|
WHERE (
|
||||||
|
json_extract(correctness_json, '$.verdict') = 'confirmed_false'
|
||||||
|
OR (json_extract(correctness_json, '$.verdict') IS NULL
|
||||||
|
AND json_extract(correctness_json, '$.confirmed') = 0
|
||||||
|
AND COALESCE(json_extract(correctness_json, '$.rejections'), 0) > 0)
|
||||||
|
)
|
||||||
|
"""
|
||||||
|
).fetchone()[0]
|
||||||
|
pending = total - confirmed_true - confirmed_false
|
||||||
|
|
||||||
|
tag_counts: dict[str, int] = {}
|
||||||
|
source_counts: dict[str, int] = {}
|
||||||
|
host_counts: dict[str, int] = {}
|
||||||
|
active: list[dict] = []
|
||||||
|
forgotten: list[dict] = []
|
||||||
|
|
||||||
|
for r in rows:
|
||||||
|
try:
|
||||||
|
meta = json.loads(r["metadata_json"] or "{}")
|
||||||
|
except Exception:
|
||||||
|
meta = {}
|
||||||
|
src = meta.get("source", "unknown")
|
||||||
|
source_counts[src] = source_counts.get(src, 0) + 1
|
||||||
|
for t in (meta.get("tags") or []):
|
||||||
|
if isinstance(t, str):
|
||||||
|
tag_counts[t] = tag_counts.get(t, 0) + 1
|
||||||
|
try:
|
||||||
|
host = _host_from_meta(r["metadata_json"])
|
||||||
|
except Exception:
|
||||||
|
host = None
|
||||||
|
if host:
|
||||||
|
host_counts[host] = host_counts.get(host, 0) + 1
|
||||||
|
|
||||||
|
access_count = int(meta.get("access_count", 0) or 0)
|
||||||
|
created = meta.get("created", r["created_at"])
|
||||||
|
if access_count >= 5 and len(active) < limit:
|
||||||
|
active.append(
|
||||||
|
{
|
||||||
|
"id": r["id"],
|
||||||
|
"access_count": access_count,
|
||||||
|
"source": src,
|
||||||
|
"created": created,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
if access_count == 0 and len(forgotten) < limit:
|
||||||
|
forgotten.append(
|
||||||
|
{
|
||||||
|
"id": r["id"],
|
||||||
|
"access_count": access_count,
|
||||||
|
"source": src,
|
||||||
|
"created": created,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
def top_k(d: dict[str, int]) -> list[dict]:
|
||||||
|
return [
|
||||||
|
{"key": k, "count": v}
|
||||||
|
for k, v in sorted(d.items(), key=lambda kv: kv[1], reverse=True)[:limit]
|
||||||
|
]
|
||||||
|
|
||||||
|
conn.close()
|
||||||
|
return {
|
||||||
|
"total": total,
|
||||||
|
"confirmed": confirmed_true,
|
||||||
|
"rejected": confirmed_false,
|
||||||
|
"pending": pending,
|
||||||
|
"top_tags": top_k(tag_counts),
|
||||||
|
"top_sources": top_k(source_counts),
|
||||||
|
"top_hosts": top_k(host_counts),
|
||||||
|
"active_engrams": active,
|
||||||
|
"forgotten_engrams": forgotten,
|
||||||
|
}
|
||||||
|
|
||||||
|
@app.get("/api/graph")
|
||||||
|
def api_graph(
|
||||||
|
limit_nodes: int = Query(0, ge=0, le=50000),
|
||||||
|
limit_edges: int = Query(0, ge=0, le=200000),
|
||||||
|
):
|
||||||
|
"""
|
||||||
|
Returns a lightweight graph view:
|
||||||
|
- Nodes: engrams + tag:<tag> + host:<hostname>
|
||||||
|
- Edges: engram->tag and engram->host plus explicit engrams_links edges.
|
||||||
|
"""
|
||||||
|
conn = get_db()
|
||||||
|
c = conn.cursor()
|
||||||
|
if limit_nodes > 0:
|
||||||
|
# Fetch a bigger window than the final node cap so trim can keep hubs + neighbors.
|
||||||
|
engram_fetch = min(50000, max(1000, int(limit_nodes * 3)))
|
||||||
|
else:
|
||||||
|
engram_fetch = None
|
||||||
|
|
||||||
|
if limit_edges > 0:
|
||||||
|
link_fetch = limit_edges
|
||||||
|
elif limit_nodes > 0:
|
||||||
|
link_fetch = min(200000, max(2000, int(limit_nodes * 10)))
|
||||||
|
else:
|
||||||
|
link_fetch = None
|
||||||
|
|
||||||
|
if engram_fetch is None:
|
||||||
|
rows = c.execute("SELECT id, metadata_json, correctness_json, created_at, modified_at FROM engrams ORDER BY created_at DESC").fetchall()
|
||||||
|
else:
|
||||||
|
rows = c.execute(
|
||||||
|
"SELECT id, metadata_json, correctness_json, created_at, modified_at FROM engrams ORDER BY created_at DESC LIMIT ?",
|
||||||
|
(engram_fetch,),
|
||||||
|
).fetchall()
|
||||||
|
|
||||||
|
if link_fetch is None:
|
||||||
|
link_rows = c.execute("SELECT from_id, to_id FROM engrams_links ORDER BY rowid DESC").fetchall()
|
||||||
|
else:
|
||||||
|
link_rows = c.execute(
|
||||||
|
"SELECT from_id, to_id FROM engrams_links ORDER BY rowid DESC LIMIT ?",
|
||||||
|
(link_fetch,),
|
||||||
|
).fetchall()
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
nodes: dict[str, dict] = {}
|
||||||
|
edges: list[dict] = []
|
||||||
|
|
||||||
|
def add_node(nid: str, kind: str, label: str | None = None, weight: float | None = None):
|
||||||
|
if nid not in nodes:
|
||||||
|
nodes[nid] = {"id": nid, "kind": kind}
|
||||||
|
if label is not None:
|
||||||
|
nodes[nid]["label"] = label
|
||||||
|
if weight is not None:
|
||||||
|
nodes[nid]["weight"] = weight
|
||||||
|
|
||||||
|
for r in rows:
|
||||||
|
eid = r["id"]
|
||||||
|
try:
|
||||||
|
meta = json.loads(r["metadata_json"] or "{}")
|
||||||
|
except Exception:
|
||||||
|
meta = {}
|
||||||
|
try:
|
||||||
|
corr = json.loads(r["correctness_json"] or "{}")
|
||||||
|
except Exception:
|
||||||
|
corr = {}
|
||||||
|
verdict = corr.get("verdict")
|
||||||
|
if not isinstance(verdict, str) or not verdict:
|
||||||
|
if corr.get("confirmed", False):
|
||||||
|
verdict = "confirmed_true"
|
||||||
|
elif int(corr.get("rejections", 0) or 0) > 0:
|
||||||
|
verdict = "confirmed_false"
|
||||||
|
else:
|
||||||
|
verdict = "unknown"
|
||||||
|
|
||||||
|
add_node(
|
||||||
|
eid,
|
||||||
|
"engram",
|
||||||
|
label=eid[:8],
|
||||||
|
weight=float(meta.get("access_count", 0) or 0),
|
||||||
|
)
|
||||||
|
nodes[eid].update(
|
||||||
|
{
|
||||||
|
"source": meta.get("source", "unknown"),
|
||||||
|
"confidence": float(meta.get("confidence", 0.0) or 0.0),
|
||||||
|
"created": meta.get("created", r["created_at"]),
|
||||||
|
"modified": meta.get("modified", r["modified_at"]),
|
||||||
|
"last_accessed": meta.get("last_accessed"),
|
||||||
|
"verdict": verdict,
|
||||||
|
"confirmed": bool(corr.get("confirmed", False)),
|
||||||
|
"rejections": int(corr.get("rejections", 0) or 0),
|
||||||
|
"grounding": meta.get("grounding", 0),
|
||||||
|
"predict_locked": bool(meta.get("predict_locked", False)),
|
||||||
|
}
|
||||||
|
)
|
||||||
|
for t in _safe_json_extract_tags(r["metadata_json"]):
|
||||||
|
tid = f"tag:{t}"
|
||||||
|
add_node(tid, "tag", label=t)
|
||||||
|
edges.append({"from": eid, "to": tid, "kind": "has_tag", "weight": 0.35})
|
||||||
|
host = _host_from_meta(r["metadata_json"])
|
||||||
|
if host:
|
||||||
|
hid = f"host:{host}"
|
||||||
|
add_node(hid, "host", label=host)
|
||||||
|
edges.append({"from": eid, "to": hid, "kind": "grounded_at", "weight": 0.25})
|
||||||
|
|
||||||
|
for fr, to in link_rows:
|
||||||
|
add_node(fr, "engram")
|
||||||
|
add_node(to, "engram")
|
||||||
|
edges.append({"from": fr, "to": to, "kind": "link", "weight": 1.0})
|
||||||
|
|
||||||
|
# Trim nodes to keep payload bounded (prefer engrams and connected tags/hosts)
|
||||||
|
if limit_nodes > 0 and len(nodes) > limit_nodes:
|
||||||
|
# Keep a balanced subset: many engrams plus the most-connected non-engrams.
|
||||||
|
kept: dict[str, dict] = {}
|
||||||
|
engram_budget = int(limit_nodes * 0.7)
|
||||||
|
|
||||||
|
# 1) Keep newest engrams first (they appear first in `rows` loop insertion order)
|
||||||
|
for r in rows:
|
||||||
|
eid = r["id"]
|
||||||
|
if eid in nodes:
|
||||||
|
kept[eid] = nodes[eid]
|
||||||
|
if len(kept) >= engram_budget:
|
||||||
|
break
|
||||||
|
|
||||||
|
# 2) Rank remaining nodes by degree within current edge set
|
||||||
|
degree: dict[str, int] = {}
|
||||||
|
for e in edges:
|
||||||
|
degree[e["from"]] = degree.get(e["from"], 0) + 1
|
||||||
|
degree[e["to"]] = degree.get(e["to"], 0) + 1
|
||||||
|
|
||||||
|
remaining = [nid for nid in nodes.keys() if nid not in kept]
|
||||||
|
remaining.sort(key=lambda nid: degree.get(nid, 0), reverse=True)
|
||||||
|
for nid in remaining:
|
||||||
|
kept[nid] = nodes[nid]
|
||||||
|
if len(kept) >= limit_nodes:
|
||||||
|
break
|
||||||
|
|
||||||
|
nodes = kept
|
||||||
|
edges = [e for e in edges if e["from"] in nodes and e["to"] in nodes]
|
||||||
|
|
||||||
|
return {"nodes": list(nodes.values()), "edges": edges}
|
||||||
|
|
||||||
|
@app.get("/api/events")
|
||||||
|
def api_events():
|
||||||
|
"""
|
||||||
|
Server-Sent Events stream for lightweight real-time UI refresh.
|
||||||
|
"""
|
||||||
|
import time
|
||||||
|
|
||||||
|
def gen():
|
||||||
|
while True:
|
||||||
|
payload = {
|
||||||
|
"ts": datetime.now(timezone.utc).isoformat(),
|
||||||
|
"stats": api_stats(),
|
||||||
|
"storage": api_storage_stats(),
|
||||||
|
"jobs": api_jobs(),
|
||||||
|
"insights": api_insights(limit=8),
|
||||||
|
}
|
||||||
|
yield f"data: {json.dumps(payload, ensure_ascii=False)}\n\n"
|
||||||
|
time.sleep(5)
|
||||||
|
|
||||||
|
return StreamingResponse(gen(), media_type="text/event-stream")
|
||||||
|
|
||||||
|
|
||||||
@app.exception_handler(FileNotFoundError)
|
@app.exception_handler(FileNotFoundError)
|
||||||
def handle_file_not_found(request: Request, exc: FileNotFoundError):
|
def handle_file_not_found(request: Request, exc: FileNotFoundError):
|
||||||
@@ -105,10 +628,27 @@ def api_stats():
|
|||||||
conn = get_db()
|
conn = get_db()
|
||||||
c = conn.cursor()
|
c = conn.cursor()
|
||||||
total = c.execute("SELECT COUNT(*) FROM engrams").fetchone()[0]
|
total = c.execute("SELECT COUNT(*) FROM engrams").fetchone()[0]
|
||||||
confirmed = c.execute(
|
confirmed_true = c.execute(
|
||||||
"SELECT COUNT(*) FROM engrams WHERE json_extract(correctness_json, '$.confirmed') = 1"
|
"""
|
||||||
|
SELECT COUNT(*) FROM engrams
|
||||||
|
WHERE (
|
||||||
|
json_extract(correctness_json, '$.verdict') = 'confirmed_true'
|
||||||
|
OR (json_extract(correctness_json, '$.verdict') IS NULL AND json_extract(correctness_json, '$.confirmed') = 1)
|
||||||
|
)
|
||||||
|
"""
|
||||||
).fetchone()[0]
|
).fetchone()[0]
|
||||||
pending = total - confirmed
|
confirmed_false = c.execute(
|
||||||
|
"""
|
||||||
|
SELECT COUNT(*) FROM engrams
|
||||||
|
WHERE (
|
||||||
|
json_extract(correctness_json, '$.verdict') = 'confirmed_false'
|
||||||
|
OR (json_extract(correctness_json, '$.verdict') IS NULL
|
||||||
|
AND json_extract(correctness_json, '$.confirmed') = 0
|
||||||
|
AND COALESCE(json_extract(correctness_json, '$.rejections'), 0) > 0)
|
||||||
|
)
|
||||||
|
"""
|
||||||
|
).fetchone()[0]
|
||||||
|
pending = total - confirmed_true - confirmed_false
|
||||||
errors = c.execute(
|
errors = c.execute(
|
||||||
"SELECT COUNT(*) FROM engrams WHERE json_extract(metadata_json, '$.tags') LIKE '%error%'"
|
"SELECT COUNT(*) FROM engrams WHERE json_extract(metadata_json, '$.tags') LIKE '%error%'"
|
||||||
).fetchone()[0]
|
).fetchone()[0]
|
||||||
@@ -118,7 +658,8 @@ def api_stats():
|
|||||||
conn.close()
|
conn.close()
|
||||||
return {
|
return {
|
||||||
"total": total,
|
"total": total,
|
||||||
"confirmed": confirmed,
|
"confirmed": confirmed_true,
|
||||||
|
"rejected": confirmed_false,
|
||||||
"pending": pending,
|
"pending": pending,
|
||||||
"errors": errors,
|
"errors": errors,
|
||||||
"avg_confidence": round(avg_conf, 2),
|
"avg_confidence": round(avg_conf, 2),
|
||||||
@@ -131,6 +672,7 @@ def api_engrams(
|
|||||||
offset: int = Query(0, ge=0),
|
offset: int = Query(0, ge=0),
|
||||||
tag: str = Query(None),
|
tag: str = Query(None),
|
||||||
confirmed: bool = Query(None),
|
confirmed: bool = Query(None),
|
||||||
|
verdict: str = Query(None),
|
||||||
search: str = Query(None),
|
search: str = Query(None),
|
||||||
min_confidence: float = Query(0.0),
|
min_confidence: float = Query(0.0),
|
||||||
):
|
):
|
||||||
@@ -144,9 +686,30 @@ def api_engrams(
|
|||||||
params.append(f'%"{tag}"%')
|
params.append(f'%"{tag}"%')
|
||||||
|
|
||||||
if confirmed is not None:
|
if confirmed is not None:
|
||||||
where_clauses.append(
|
if confirmed:
|
||||||
f"json_extract(correctness_json, '$.confirmed') = {int(confirmed)}"
|
# confirmed == statement is true (verdict confirmed_true)
|
||||||
)
|
where_clauses.append(
|
||||||
|
"("
|
||||||
|
"json_extract(correctness_json, '$.verdict') = 'confirmed_true' "
|
||||||
|
"OR (json_extract(correctness_json, '$.verdict') IS NULL AND json_extract(correctness_json, '$.confirmed') = 1)"
|
||||||
|
")"
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
# pending/unresolved (unknown/probable) but exclude confirmed_false.
|
||||||
|
where_clauses.append(
|
||||||
|
"("
|
||||||
|
"json_extract(correctness_json, '$.verdict') IN ('unknown','probable_true','probable_false') "
|
||||||
|
"OR (json_extract(correctness_json, '$.verdict') IS NULL "
|
||||||
|
" AND json_extract(correctness_json, '$.confirmed') = 0 "
|
||||||
|
" AND COALESCE(json_extract(correctness_json, '$.rejections'), 0) = 0)"
|
||||||
|
")"
|
||||||
|
)
|
||||||
|
|
||||||
|
if verdict:
|
||||||
|
v = verdict.strip()
|
||||||
|
if v in ("unknown", "probable_true", "probable_false", "confirmed_true", "confirmed_false"):
|
||||||
|
where_clauses.append("json_extract(correctness_json, '$.verdict') = ?")
|
||||||
|
params.append(v)
|
||||||
|
|
||||||
if search:
|
if search:
|
||||||
# Use FTS
|
# Use FTS
|
||||||
@@ -204,70 +767,107 @@ def api_engram_detail(engram_id: str):
|
|||||||
return result
|
return result
|
||||||
|
|
||||||
|
|
||||||
@app.post("/api/engrams/{engram_id}/confirm")
|
@app.get("/api/pending")
|
||||||
def api_confirm(engram_id: str, reason: str = Form("")):
|
def api_pending(
|
||||||
|
limit: int = Query(20, ge=1, le=200),
|
||||||
|
offset: int = Query(0, ge=0),
|
||||||
|
source: str | None = Query(None),
|
||||||
|
):
|
||||||
conn = get_db()
|
conn = get_db()
|
||||||
c = conn.cursor()
|
c = conn.cursor()
|
||||||
row = c.execute(
|
where = ["json_extract(correctness_json, '$.confirmed') = 0"]
|
||||||
"SELECT correctness_json FROM engrams WHERE id = ?", (engram_id,)
|
params: list = []
|
||||||
).fetchone()
|
if source:
|
||||||
if not row:
|
where.append("json_extract(metadata_json, '$.source') = ?")
|
||||||
conn.close()
|
params.append(source)
|
||||||
return JSONResponse({"error": "Not found"}, status_code=404)
|
|
||||||
|
|
||||||
correctness = json.loads(row["correctness_json"] or "{}")
|
rows = c.execute(
|
||||||
correctness["confirmed"] = True
|
f"""
|
||||||
correctness["confirmations"] = correctness.get("confirmations", 0) + 1
|
SELECT * FROM engrams
|
||||||
correctness["last_reviewed"] = datetime.now(timezone.utc).isoformat()
|
WHERE {' AND '.join(where)}
|
||||||
review_history = correctness.get("review_history", [])
|
ORDER BY created_at DESC
|
||||||
review_history.append({
|
LIMIT ? OFFSET ?
|
||||||
"by": "web",
|
""",
|
||||||
"action": "confirm",
|
params + [limit, offset],
|
||||||
"at": datetime.now(timezone.utc).isoformat(),
|
).fetchall()
|
||||||
"note": reason or "confirmed via dashboard",
|
items = [parse_engram(r) for r in rows]
|
||||||
})
|
conn.close()
|
||||||
correctness["review_history"] = review_history
|
return {"items": items, "limit": limit, "offset": offset}
|
||||||
|
|
||||||
|
|
||||||
|
@app.post("/api/engrams")
|
||||||
|
def api_create_engram(content: str = Form(...), tags: str = Form("")):
|
||||||
|
content = (content or "").strip()
|
||||||
|
if not content:
|
||||||
|
return JSONResponse({"error": "content required"}, status_code=400)
|
||||||
|
tag_list = [t.strip() for t in (tags or "").split(",") if t.strip()]
|
||||||
|
now = _now_iso()
|
||||||
|
engram_id = str(uuid4())
|
||||||
|
meta = {
|
||||||
|
"source": "user",
|
||||||
|
"confidence": 0.7,
|
||||||
|
"created": now,
|
||||||
|
"modified": now,
|
||||||
|
"access_count": 0,
|
||||||
|
"last_accessed": now,
|
||||||
|
"tags": tag_list,
|
||||||
|
"session_id": None,
|
||||||
|
"agent_id": None,
|
||||||
|
"grounding": 0,
|
||||||
|
}
|
||||||
|
corr = {
|
||||||
|
"confirmed": False,
|
||||||
|
"confirmations": 0,
|
||||||
|
"rejections": 0,
|
||||||
|
"last_reviewed": None,
|
||||||
|
"review_history": [],
|
||||||
|
}
|
||||||
|
conn = get_db()
|
||||||
|
c = conn.cursor()
|
||||||
c.execute(
|
c.execute(
|
||||||
"UPDATE engrams SET correctness_json = ?, modified_at = ? WHERE id = ?",
|
"""
|
||||||
(json.dumps(correctness), datetime.now(timezone.utc).isoformat(), engram_id),
|
INSERT INTO engrams (id, content, metadata_json, correctness_json, links_json, hierarchy_json, embedding_json, created_at, modified_at)
|
||||||
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||||
|
""",
|
||||||
|
(
|
||||||
|
engram_id,
|
||||||
|
content,
|
||||||
|
json.dumps(meta, ensure_ascii=False),
|
||||||
|
json.dumps(corr, ensure_ascii=False),
|
||||||
|
"[]",
|
||||||
|
"{}",
|
||||||
|
None,
|
||||||
|
now,
|
||||||
|
now,
|
||||||
|
),
|
||||||
)
|
)
|
||||||
conn.commit()
|
conn.commit()
|
||||||
conn.close()
|
conn.close()
|
||||||
return {"success": True, "engram_id": engram_id}
|
return {"id": engram_id}
|
||||||
|
|
||||||
|
|
||||||
|
@app.post("/api/engrams/{engram_id}/confirm")
|
||||||
|
def api_confirm_engram(engram_id: str, reason: str = Form("")):
|
||||||
|
try:
|
||||||
|
return _update_correctness(engram_id, action="confirm", reason=reason)
|
||||||
|
except FileNotFoundError as e:
|
||||||
|
return JSONResponse({"error": str(e)}, status_code=404)
|
||||||
|
|
||||||
|
|
||||||
@app.post("/api/engrams/{engram_id}/reject")
|
@app.post("/api/engrams/{engram_id}/reject")
|
||||||
def api_reject(engram_id: str, reason: str = Form("")):
|
def api_reject_engram(engram_id: str, reason: str = Form("")):
|
||||||
conn = get_db()
|
try:
|
||||||
c = conn.cursor()
|
return _update_correctness(engram_id, action="reject", reason=reason)
|
||||||
row = c.execute(
|
except FileNotFoundError as e:
|
||||||
"SELECT correctness_json FROM engrams WHERE id = ?", (engram_id,)
|
return JSONResponse({"error": str(e)}, status_code=404)
|
||||||
).fetchone()
|
|
||||||
if not row:
|
|
||||||
conn.close()
|
|
||||||
return JSONResponse({"error": "Not found"}, status_code=404)
|
|
||||||
|
|
||||||
correctness = json.loads(row["correctness_json"] or "{}")
|
|
||||||
correctness["confirmed"] = False
|
|
||||||
correctness["rejections"] = correctness.get("rejections", 0) + 1
|
|
||||||
correctness["last_reviewed"] = datetime.now(timezone.utc).isoformat()
|
|
||||||
review_history = correctness.get("review_history", [])
|
|
||||||
review_history.append({
|
|
||||||
"by": "web",
|
|
||||||
"action": "reject",
|
|
||||||
"at": datetime.now(timezone.utc).isoformat(),
|
|
||||||
"note": reason or "rejected via dashboard",
|
|
||||||
})
|
|
||||||
correctness["review_history"] = review_history
|
|
||||||
|
|
||||||
c.execute(
|
@app.post("/api/engrams/{engram_id}/refresh")
|
||||||
"UPDATE engrams SET correctness_json = ?, modified_at = ? WHERE id = ?",
|
def api_refresh_engram(engram_id: str):
|
||||||
(json.dumps(correctness), datetime.now(timezone.utc).isoformat(), engram_id),
|
try:
|
||||||
)
|
return _bump_access(engram_id)
|
||||||
conn.commit()
|
except FileNotFoundError as e:
|
||||||
conn.close()
|
return JSONResponse({"error": str(e)}, status_code=404)
|
||||||
return {"success": True, "engram_id": engram_id}
|
|
||||||
|
|
||||||
|
|
||||||
@app.post("/api/engrams/{engram_id}/refresh")
|
@app.post("/api/engrams/{engram_id}/refresh")
|
||||||
@@ -320,6 +920,8 @@ def api_create_engram(content: str = Form(...), tags: str = Form(""), source: st
|
|||||||
"hash": "",
|
"hash": "",
|
||||||
}
|
}
|
||||||
correctness = {
|
correctness = {
|
||||||
|
"verdict": "unknown",
|
||||||
|
"evidence": [],
|
||||||
"confirmed": False,
|
"confirmed": False,
|
||||||
"confirmations": 0,
|
"confirmations": 0,
|
||||||
"rejections": 0,
|
"rejections": 0,
|
||||||
@@ -347,7 +949,12 @@ def api_pending(limit: int = Query(20, ge=1, le=100), offset: int = Query(0, ge=
|
|||||||
rows = c.execute(
|
rows = c.execute(
|
||||||
"""
|
"""
|
||||||
SELECT * FROM engrams
|
SELECT * FROM engrams
|
||||||
WHERE json_extract(correctness_json, '$.confirmed') = 0
|
WHERE (
|
||||||
|
json_extract(correctness_json, '$.verdict') IN ('unknown','probable_true','probable_false')
|
||||||
|
OR (json_extract(correctness_json, '$.verdict') IS NULL
|
||||||
|
AND json_extract(correctness_json, '$.confirmed') = 0
|
||||||
|
AND COALESCE(json_extract(correctness_json, '$.rejections'), 0) = 0)
|
||||||
|
)
|
||||||
ORDER BY created_at DESC
|
ORDER BY created_at DESC
|
||||||
LIMIT ? OFFSET ?
|
LIMIT ? OFFSET ?
|
||||||
""",
|
""",
|
||||||
|
|||||||
4696
reports/pending_engrams_20260529_213522.txt
Normal file
4696
reports/pending_engrams_20260529_213522.txt
Normal file
File diff suppressed because it is too large
Load Diff
191
scripts/import_web_design_markers.py
Normal file
191
scripts/import_web_design_markers.py
Normal file
@@ -0,0 +1,191 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
import argparse
|
||||||
|
import json
|
||||||
|
import hashlib
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Any, Dict, Iterable, List, Optional, Tuple
|
||||||
|
|
||||||
|
from src.engram import Engram, Grounding
|
||||||
|
from src.store import EngramStore
|
||||||
|
|
||||||
|
|
||||||
|
def _now_utc_iso() -> str:
|
||||||
|
return datetime.now(timezone.utc).isoformat()
|
||||||
|
|
||||||
|
|
||||||
|
def _hash16(text: str) -> str:
|
||||||
|
return hashlib.sha256(text.encode("utf-8")).hexdigest()[:16]
|
||||||
|
|
||||||
|
|
||||||
|
def _iter_jsonl(path: Path) -> Iterable[Dict[str, Any]]:
|
||||||
|
with path.open("r", encoding="utf-8") as f:
|
||||||
|
for line_no, line in enumerate(f, start=1):
|
||||||
|
line = line.strip()
|
||||||
|
if not line:
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
obj = json.loads(line)
|
||||||
|
except Exception:
|
||||||
|
raise SystemExit(f"Invalid JSON at {path}:{line_no}")
|
||||||
|
if not isinstance(obj, dict):
|
||||||
|
continue
|
||||||
|
yield obj
|
||||||
|
|
||||||
|
|
||||||
|
def _marker_to_content(marker_obj: Dict[str, Any]) -> Tuple[str, List[Dict[str, Any]]]:
|
||||||
|
marker = str(marker_obj.get("marker", "")).strip()
|
||||||
|
details = str(marker_obj.get("details", "")).strip()
|
||||||
|
checks = marker_obj.get("checks") or []
|
||||||
|
sources = marker_obj.get("sources") or []
|
||||||
|
|
||||||
|
if not marker:
|
||||||
|
raise ValueError("missing marker")
|
||||||
|
|
||||||
|
evidence: List[Dict[str, Any]] = []
|
||||||
|
for src in sources:
|
||||||
|
if not isinstance(src, dict):
|
||||||
|
continue
|
||||||
|
url = (src.get("url") or "").strip()
|
||||||
|
title = (src.get("title") or "").strip()
|
||||||
|
if not url:
|
||||||
|
continue
|
||||||
|
evidence.append({"url": url, "title": title})
|
||||||
|
|
||||||
|
lines: List[str] = []
|
||||||
|
lines.append(f"WEBDEV_MARKER: {marker}")
|
||||||
|
if details:
|
||||||
|
lines.append("")
|
||||||
|
lines.append(f"Details: {details}")
|
||||||
|
if isinstance(checks, list) and checks:
|
||||||
|
lines.append("")
|
||||||
|
lines.append("Checks:")
|
||||||
|
for c in checks[:8]:
|
||||||
|
c = str(c).strip()
|
||||||
|
if c:
|
||||||
|
lines.append(f"- {c}")
|
||||||
|
if evidence:
|
||||||
|
lines.append("")
|
||||||
|
lines.append("Sources:")
|
||||||
|
for ev in evidence[:12]:
|
||||||
|
title = (ev.get("title") or "").strip()
|
||||||
|
url = (ev.get("url") or "").strip()
|
||||||
|
if title:
|
||||||
|
lines.append(f"- {title}: {url}")
|
||||||
|
else:
|
||||||
|
lines.append(f"- {url}")
|
||||||
|
return "\n".join(lines).strip(), evidence
|
||||||
|
|
||||||
|
|
||||||
|
def _tags_for(marker_obj: Dict[str, Any]) -> List[str]:
|
||||||
|
tags = ["web_design", "web_development", "mobile"]
|
||||||
|
area = str(marker_obj.get("area", "")).strip()
|
||||||
|
if area:
|
||||||
|
tags.append(area)
|
||||||
|
return tags
|
||||||
|
|
||||||
|
|
||||||
|
def import_markers(
|
||||||
|
db_path: Path,
|
||||||
|
jsonl_paths: List[Path],
|
||||||
|
source: str,
|
||||||
|
verdict: str,
|
||||||
|
agent_id: str,
|
||||||
|
dry_run: bool,
|
||||||
|
) -> Dict[str, int]:
|
||||||
|
store = EngramStore(str(db_path))
|
||||||
|
|
||||||
|
stats = {"seen": 0, "imported": 0, "skipped_dup": 0, "skipped_invalid": 0}
|
||||||
|
seen_hashes: set[str] = set()
|
||||||
|
|
||||||
|
# Preload existing hashes (fast-ish; avoids duplicate spam).
|
||||||
|
existing_hashes: set[str] = set()
|
||||||
|
try:
|
||||||
|
cur = store._conn.execute("SELECT metadata_json FROM engrams") # noqa: SLF001
|
||||||
|
for row in cur.fetchall():
|
||||||
|
try:
|
||||||
|
meta = json.loads(row["metadata_json"])
|
||||||
|
h = meta.get("hash")
|
||||||
|
if isinstance(h, str) and h:
|
||||||
|
existing_hashes.add(h)
|
||||||
|
except Exception:
|
||||||
|
continue
|
||||||
|
except Exception:
|
||||||
|
# If this fails (schema mismatch), proceed without preload.
|
||||||
|
existing_hashes = set()
|
||||||
|
|
||||||
|
for path in jsonl_paths:
|
||||||
|
for marker_obj in _iter_jsonl(path):
|
||||||
|
if (marker_obj.get("kind") or "") != "web_design_marker":
|
||||||
|
continue
|
||||||
|
stats["seen"] += 1
|
||||||
|
try:
|
||||||
|
content, evidence = _marker_to_content(marker_obj)
|
||||||
|
except Exception:
|
||||||
|
stats["skipped_invalid"] += 1
|
||||||
|
continue
|
||||||
|
|
||||||
|
h = _hash16(content)
|
||||||
|
if h in seen_hashes or h in existing_hashes:
|
||||||
|
stats["skipped_dup"] += 1
|
||||||
|
continue
|
||||||
|
seen_hashes.add(h)
|
||||||
|
|
||||||
|
eg = Engram.create(
|
||||||
|
content=content,
|
||||||
|
source=source,
|
||||||
|
confidence=0.75,
|
||||||
|
tags=_tags_for(marker_obj),
|
||||||
|
session_id=None,
|
||||||
|
agent_id=agent_id or str(marker_obj.get("agent_id") or ""),
|
||||||
|
grounding=Grounding.SOURCED,
|
||||||
|
)
|
||||||
|
# Overwrite hash to exactly match our content representation.
|
||||||
|
eg.metadata["hash"] = h
|
||||||
|
eg.metadata["modified"] = _now_utc_iso()
|
||||||
|
eg.metadata["created"] = marker_obj.get("created_at") or eg.metadata["created"]
|
||||||
|
|
||||||
|
eg.correctness.set_verdict(
|
||||||
|
by=agent_id or "importer",
|
||||||
|
verdict=verdict,
|
||||||
|
note=f"Imported from {path.name}",
|
||||||
|
evidence=evidence,
|
||||||
|
)
|
||||||
|
|
||||||
|
if not dry_run:
|
||||||
|
store.save(eg)
|
||||||
|
stats["imported"] += 1
|
||||||
|
|
||||||
|
return stats
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> None:
|
||||||
|
p = argparse.ArgumentParser(description="Import web_design_marker JSONL files into brain.sqlite")
|
||||||
|
p.add_argument("--db", default="second-brain/data/brain.sqlite", help="Path to brain.sqlite")
|
||||||
|
p.add_argument("--glob", default="/tmp/web_design_markers_*.jsonl", help="Glob for marker JSONL files")
|
||||||
|
p.add_argument("--source", default="web_research", help="Engram source")
|
||||||
|
p.add_argument("--verdict", default="probable_true", help="Correctness verdict")
|
||||||
|
p.add_argument("--agent-id", default="web_research_import", help="Agent id to record")
|
||||||
|
p.add_argument("--dry-run", action="store_true", help="Parse/dedupe but do not write to DB")
|
||||||
|
args = p.parse_args()
|
||||||
|
|
||||||
|
db_path = Path(args.db)
|
||||||
|
jsonl_paths = sorted(Path("/").glob(args.glob.lstrip("/"))) if args.glob.startswith("/") else sorted(Path(".").glob(args.glob))
|
||||||
|
if not jsonl_paths:
|
||||||
|
raise SystemExit(f"No files match glob: {args.glob}")
|
||||||
|
|
||||||
|
stats = import_markers(
|
||||||
|
db_path=db_path,
|
||||||
|
jsonl_paths=jsonl_paths,
|
||||||
|
source=args.source,
|
||||||
|
verdict=args.verdict,
|
||||||
|
agent_id=args.agent_id,
|
||||||
|
dry_run=bool(args.dry_run),
|
||||||
|
)
|
||||||
|
|
||||||
|
print(json.dumps({"db": str(db_path), "files": [str(p) for p in jsonl_paths], "stats": stats}, ensure_ascii=False, indent=2))
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
|
|
||||||
163
scripts/process_pending_engrams.py
Normal file
163
scripts/process_pending_engrams.py
Normal file
@@ -0,0 +1,163 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""
|
||||||
|
Process pending second brain engrams.
|
||||||
|
- For unconfirmed, unrejected engrams: evaluate confidence
|
||||||
|
- If confidence > 0.8: confirm
|
||||||
|
- If confidence < 0.3: reject
|
||||||
|
- Otherwise: mark for review (leave as is)
|
||||||
|
- Check for stale topics and archive if needed
|
||||||
|
- Produce summary report
|
||||||
|
"""
|
||||||
|
|
||||||
|
import sys
|
||||||
|
import json
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
# Add src to path and set PYTHONPATH for proper module resolution
|
||||||
|
base_dir = Path(__file__).parent.parent
|
||||||
|
sys.path.insert(0, str(base_dir / "src"))
|
||||||
|
|
||||||
|
# Import using absolute module paths
|
||||||
|
from src.store import EngramStore
|
||||||
|
from src.engram import Engram, Grounding
|
||||||
|
|
||||||
|
DB_PATH = Path(__file__).parent.parent / "data" / "brain.sqlite"
|
||||||
|
|
||||||
|
|
||||||
|
def is_stale(engram: Engram, days_threshold: int = 90) -> bool:
|
||||||
|
"""Check if an engram is stale (old and rarely accessed)."""
|
||||||
|
created = engram.metadata.get("created", "")
|
||||||
|
access_count = engram.metadata.get("access_count", 0)
|
||||||
|
last_accessed = engram.metadata.get("last_accessed", created)
|
||||||
|
|
||||||
|
try:
|
||||||
|
created_dt = datetime.fromisoformat(created.replace("Z", "+00:00"))
|
||||||
|
last_accessed_dt = datetime.fromisoformat(last_accessed.replace("Z", "+00:00"))
|
||||||
|
age_days = (datetime.now(timezone.utc) - created_dt).total_seconds() / 86400
|
||||||
|
days_since_access = (datetime.now(timezone.utc) - last_accessed_dt).total_seconds() / 86400
|
||||||
|
|
||||||
|
# Stale if: old (>90 days) AND rarely accessed (<3 times) AND not accessed recently (>60 days)
|
||||||
|
if age_days > days_threshold and access_count < 3 and days_since_access > 60:
|
||||||
|
return True
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def process_pending_engrams():
|
||||||
|
"""Main processing function."""
|
||||||
|
store = EngramStore(str(DB_PATH))
|
||||||
|
|
||||||
|
# Get all engrams
|
||||||
|
all_engrams = store.get_all(limit=10000)
|
||||||
|
print(f"Total engrams in database: {len(all_engrams)}")
|
||||||
|
|
||||||
|
# Filter pending (unconfirmed and unrejected)
|
||||||
|
# Unconfirmed: not confirmed_true, not confirmed_false
|
||||||
|
pending = []
|
||||||
|
for eg in all_engrams:
|
||||||
|
verdict = eg.correctness.verdict
|
||||||
|
if verdict not in ("confirmed_true", "confirmed_false"):
|
||||||
|
pending.append(eg)
|
||||||
|
|
||||||
|
print(f"Pending engrams (unconfirmed/unrejected): {len(pending)}")
|
||||||
|
|
||||||
|
actions = {
|
||||||
|
"confirmed": 0,
|
||||||
|
"rejected": 0,
|
||||||
|
"left_for_review": 0,
|
||||||
|
"archived_stale": 0,
|
||||||
|
"errors": 0
|
||||||
|
}
|
||||||
|
|
||||||
|
details = []
|
||||||
|
|
||||||
|
for eg in pending:
|
||||||
|
try:
|
||||||
|
confidence = eg.compute_confidence()
|
||||||
|
engram_id = str(eg.id)
|
||||||
|
content_preview = eg.content[:80] + ("..." if len(eg.content) > 80 else "")
|
||||||
|
|
||||||
|
# Check if stale and should be archived
|
||||||
|
if is_stale(eg):
|
||||||
|
# For stale engrams, we'll mark them in metadata for archiving
|
||||||
|
# Instead of deleting, we'll add an "archived" tag and lower their priority
|
||||||
|
tags = eg.metadata.get("tags", [])
|
||||||
|
if "archived" not in tags:
|
||||||
|
tags.append("archived")
|
||||||
|
eg.metadata["tags"] = tags
|
||||||
|
eg.metadata["archived_at"] = datetime.now(timezone.utc).isoformat()
|
||||||
|
store.save(eg)
|
||||||
|
actions["archived_stale"] += 1
|
||||||
|
details.append(f"📦 Archived stale: [{engram_id[:8]}] {content_preview} (conf: {confidence:.2f})")
|
||||||
|
# Even if stale, we still evaluate confidence for reporting
|
||||||
|
# But we don't confirm/reject stale ones automatically unless confidence is extreme
|
||||||
|
# Actually, the task says to check for stale topics and archive if needed. We've done that.
|
||||||
|
# We still need to apply confidence thresholds to non-stale or all pending?
|
||||||
|
# Let's continue to evaluate all pending, including stale, but maybe skip confirm/reject for stale?
|
||||||
|
# The task: "For each pending engram... evaluate... If >0.8 confirm, <0.3 reject, otherwise mark for review"
|
||||||
|
# It doesn't say to skip stale ones. So we'll still apply thresholds.
|
||||||
|
# But we already archived it. We can still confirm/reject it if confidence is extreme.
|
||||||
|
# Let's continue.
|
||||||
|
|
||||||
|
# Apply confidence thresholds
|
||||||
|
if confidence > 0.8:
|
||||||
|
eg.correctness.confirm(by="auto_processor", note=f"Auto-confirmed: confidence {confidence:.2f}")
|
||||||
|
store.save(eg)
|
||||||
|
actions["confirmed"] += 1
|
||||||
|
details.append(f"✅ Confirmed: [{engram_id[:8]}] {content_preview} (conf: {confidence:.2f})")
|
||||||
|
elif confidence < 0.3:
|
||||||
|
eg.correctness.reject(by="auto_processor", note=f"Auto-rejected: confidence {confidence:.2f}")
|
||||||
|
store.save(eg)
|
||||||
|
actions["rejected"] += 1
|
||||||
|
details.append(f"❌ Rejected: [{engram_id[:8]}] {content_preview} (conf: {confidence:.2f})")
|
||||||
|
else:
|
||||||
|
actions["left_for_review"] += 1
|
||||||
|
details.append(f"⏳ Review later: [{engram_id[:8]}] {content_preview} (conf: {confidence:.2f})")
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
actions["errors"] += 1
|
||||||
|
details.append(f"⚠️ Error processing engram: {str(e)}")
|
||||||
|
|
||||||
|
# Generate summary report
|
||||||
|
report_lines = []
|
||||||
|
report_lines.append("=" * 60)
|
||||||
|
report_lines.append("PENDING ENGRAMS PROCESSING REPORT")
|
||||||
|
report_lines.append("=" * 60)
|
||||||
|
report_lines.append(f"Timestamp: {datetime.now(timezone.utc).isoformat()}")
|
||||||
|
report_lines.append(f"Total engrams: {len(all_engrams)}")
|
||||||
|
report_lines.append(f"Pending engrams processed: {len(pending)}")
|
||||||
|
report_lines.append("")
|
||||||
|
report_lines.append("ACTIONS TAKEN:")
|
||||||
|
report_lines.append(f" ✅ Auto-confirmed (confidence > 0.8): {actions['confirmed']}")
|
||||||
|
report_lines.append(f" ❌ Auto-rejected (confidence < 0.3): {actions['rejected']}")
|
||||||
|
report_lines.append(f" ⏳ Left for review (0.3 ≤ confidence ≤ 0.8): {actions['left_for_review']}")
|
||||||
|
report_lines.append(f" 📦 Archived stale topics: {actions['archived_stale']}")
|
||||||
|
report_lines.append(f" ⚠️ Errors: {actions['errors']}")
|
||||||
|
report_lines.append("")
|
||||||
|
report_lines.append("DETAILS:")
|
||||||
|
report_lines.extend(details)
|
||||||
|
report_lines.append("")
|
||||||
|
report_lines.append("=" * 60)
|
||||||
|
|
||||||
|
report = "\n".join(report_lines)
|
||||||
|
|
||||||
|
# Print to stdout
|
||||||
|
print("\n" + report)
|
||||||
|
|
||||||
|
# Save report to file
|
||||||
|
report_dir = Path(__file__).parent.parent / "reports"
|
||||||
|
report_dir.mkdir(parents=True, exist_ok=True)
|
||||||
|
report_file = report_dir / f"pending_engrams_{datetime.now().strftime('%Y%m%d_%H%M%S')}.txt"
|
||||||
|
report_file.write_text(report, encoding="utf-8")
|
||||||
|
print(f"\n📄 Report saved to: {report_file}")
|
||||||
|
|
||||||
|
store.close()
|
||||||
|
return actions
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
result = process_pending_engrams()
|
||||||
|
print("\nProcessing complete.")
|
||||||
20
scripts/smoke_dashboard.sh
Executable file
20
scripts/smoke_dashboard.sh
Executable file
@@ -0,0 +1,20 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
HOST="${SECOND_BRAIN_HOST:-127.0.0.1}"
|
||||||
|
PORT="${SECOND_BRAIN_PORT:-${PORT:-8501}}"
|
||||||
|
BASE_URL="http://${HOST}:${PORT}"
|
||||||
|
|
||||||
|
if command -v systemctl >/dev/null 2>&1; then
|
||||||
|
if ! systemctl is-active --quiet openclaw-secondbrain-dashboard.service; then
|
||||||
|
echo "ERROR: openclaw-secondbrain-dashboard.service is not active" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
curl -fsS "${BASE_URL}/healthz" >/dev/null
|
||||||
|
|
||||||
|
stats_json="$(curl -fsS "${BASE_URL}/api/stats")"
|
||||||
|
python3 -c 'import json,sys; json.load(sys.stdin)' <<<"$stats_json"
|
||||||
|
|
||||||
|
echo "OK: dashboard smoke test passed (${BASE_URL})"
|
||||||
@@ -40,26 +40,60 @@ class ReviewEntry:
|
|||||||
@dataclass
|
@dataclass
|
||||||
class Correctness:
|
class Correctness:
|
||||||
"""Verfolgt die Korrektheit eines Engramms über Zeit."""
|
"""Verfolgt die Korrektheit eines Engramms über Zeit."""
|
||||||
|
# verdict model (not only binary confirm/reject)
|
||||||
|
# Values:
|
||||||
|
# - unknown
|
||||||
|
# - probable_true / probable_false
|
||||||
|
# - confirmed_true / confirmed_false
|
||||||
|
verdict: str = "unknown"
|
||||||
|
evidence: List[Dict[str, Any]] = field(default_factory=list)
|
||||||
confirmed: bool = False
|
confirmed: bool = False
|
||||||
confirmations: int = 0
|
confirmations: int = 0
|
||||||
rejections: int = 0
|
rejections: int = 0
|
||||||
last_reviewed: Optional[str] = None
|
last_reviewed: Optional[str] = None
|
||||||
review_history: List[ReviewEntry] = field(default_factory=list)
|
review_history: List[ReviewEntry] = field(default_factory=list)
|
||||||
|
|
||||||
|
def is_final(self) -> bool:
|
||||||
|
return self.verdict in ("confirmed_true", "confirmed_false")
|
||||||
|
|
||||||
|
def set_verdict(self, by: str, verdict: str, note: str = "", evidence: Optional[List[Dict[str, Any]]] = None) -> None:
|
||||||
|
verdict = (verdict or "").strip()
|
||||||
|
if verdict not in ("unknown", "probable_true", "probable_false", "confirmed_true", "confirmed_false"):
|
||||||
|
verdict = "unknown"
|
||||||
|
self.verdict = verdict
|
||||||
|
# Keep backward-compatible boolean in sync:
|
||||||
|
# historically, confirmed=True meant "this statement is correct".
|
||||||
|
self.confirmed = verdict == "confirmed_true"
|
||||||
|
self.last_reviewed = _now()
|
||||||
|
if evidence:
|
||||||
|
try:
|
||||||
|
self.evidence.extend([e for e in evidence if isinstance(e, dict)])
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
self.review_history.append(ReviewEntry(by, "set_verdict", self.last_reviewed, f"{verdict}: {note}".strip()))
|
||||||
|
|
||||||
def confirm(self, by: str, note: str = "") -> None:
|
def confirm(self, by: str, note: str = "") -> None:
|
||||||
self.confirmations += 1
|
self.confirmations += 1
|
||||||
self.confirmed = True
|
self.set_verdict(by, "confirmed_true", note)
|
||||||
self.last_reviewed = _now()
|
# Preserve historic action tag too
|
||||||
self.review_history.append(ReviewEntry(by, "confirm", self.last_reviewed, note))
|
self.review_history.append(ReviewEntry(by, "confirm", self.last_reviewed, note))
|
||||||
|
|
||||||
def reject(self, by: str, note: str = "") -> None:
|
def reject(self, by: str, note: str = "") -> None:
|
||||||
self.rejections += 1
|
self.rejections += 1
|
||||||
self.confirmed = False
|
self.set_verdict(by, "confirmed_false", note)
|
||||||
self.last_reviewed = _now()
|
|
||||||
self.review_history.append(ReviewEntry(by, "reject", self.last_reviewed, note))
|
self.review_history.append(ReviewEntry(by, "reject", self.last_reviewed, note))
|
||||||
|
|
||||||
def score(self) -> float:
|
def score(self) -> float:
|
||||||
"""Confidence-Score aus Korrekturhistorie."""
|
"""Confidence-Score aus Korrekturhistorie."""
|
||||||
|
# verdict-first scoring (explicit, non-binary)
|
||||||
|
if self.verdict == "confirmed_true":
|
||||||
|
return 1.0
|
||||||
|
if self.verdict == "confirmed_false":
|
||||||
|
return 0.0
|
||||||
|
if self.verdict == "probable_true":
|
||||||
|
return 0.75
|
||||||
|
if self.verdict == "probable_false":
|
||||||
|
return 0.25
|
||||||
total = self.confirmations + self.rejections
|
total = self.confirmations + self.rejections
|
||||||
if total == 0:
|
if total == 0:
|
||||||
return 0.5 # Unbestimmt
|
return 0.5 # Unbestimmt
|
||||||
@@ -74,6 +108,8 @@ class Correctness:
|
|||||||
else:
|
else:
|
||||||
review_history.append(entry.to_dict())
|
review_history.append(entry.to_dict())
|
||||||
return {
|
return {
|
||||||
|
"verdict": self.verdict,
|
||||||
|
"evidence": self.evidence,
|
||||||
"confirmed": self.confirmed,
|
"confirmed": self.confirmed,
|
||||||
"confirmations": self.confirmations,
|
"confirmations": self.confirmations,
|
||||||
"rejections": self.rejections,
|
"rejections": self.rejections,
|
||||||
@@ -84,11 +120,30 @@ class Correctness:
|
|||||||
@classmethod
|
@classmethod
|
||||||
def from_dict(cls, d: dict) -> "Correctness":
|
def from_dict(cls, d: dict) -> "Correctness":
|
||||||
c = cls()
|
c = cls()
|
||||||
|
verdict = d.get("verdict")
|
||||||
|
if isinstance(verdict, str) and verdict.strip():
|
||||||
|
c.verdict = verdict.strip()
|
||||||
c.confirmed = d.get("confirmed", False)
|
c.confirmed = d.get("confirmed", False)
|
||||||
c.confirmations = d.get("confirmations", 0)
|
c.confirmations = d.get("confirmations", 0)
|
||||||
c.rejections = d.get("rejections", 0)
|
c.rejections = d.get("rejections", 0)
|
||||||
c.last_reviewed = d.get("last_reviewed")
|
c.last_reviewed = d.get("last_reviewed")
|
||||||
|
ev = d.get("evidence", [])
|
||||||
|
if isinstance(ev, list):
|
||||||
|
c.evidence = [e for e in ev if isinstance(e, dict)]
|
||||||
c.review_history = [ReviewEntry.from_dict(r) for r in d.get("review_history", [])]
|
c.review_history = [ReviewEntry.from_dict(r) for r in d.get("review_history", [])]
|
||||||
|
# Backfill verdict if missing/invalid.
|
||||||
|
if c.verdict not in ("unknown", "probable_true", "probable_false", "confirmed_true", "confirmed_false"):
|
||||||
|
if c.confirmed:
|
||||||
|
c.verdict = "confirmed_true"
|
||||||
|
elif c.rejections > 0:
|
||||||
|
c.verdict = "confirmed_false"
|
||||||
|
else:
|
||||||
|
c.verdict = "unknown"
|
||||||
|
# Ensure boolean stays consistent for older mixed data.
|
||||||
|
if c.verdict == "confirmed_true":
|
||||||
|
c.confirmed = True
|
||||||
|
elif c.verdict == "confirmed_false":
|
||||||
|
c.confirmed = False
|
||||||
return c
|
return c
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -134,11 +134,19 @@ def _node_size(access_count: int) -> float:
|
|||||||
|
|
||||||
def generate_graph_html(store: EngramStore, output_path: str) -> str:
|
def generate_graph_html(store: EngramStore, output_path: str) -> str:
|
||||||
"""Generiert interaktive HTML-Graph-Visualisierung."""
|
"""Generiert interaktive HTML-Graph-Visualisierung."""
|
||||||
engrams = store.get_all()
|
# store.get_all() defaults to 1000; paginate so the graph can include all nodes.
|
||||||
|
engrams = []
|
||||||
|
offset = 0
|
||||||
|
while True:
|
||||||
|
batch = store.get_all(limit=2000, offset=offset)
|
||||||
|
if not batch:
|
||||||
|
break
|
||||||
|
engrams.extend(batch)
|
||||||
|
offset += len(batch)
|
||||||
|
|
||||||
nodes = []
|
nodes = []
|
||||||
edges = []
|
edges = []
|
||||||
node_ids = set()
|
node_ids = set(str(e.id) for e in engrams)
|
||||||
|
|
||||||
for eg in engrams:
|
for eg in engrams:
|
||||||
eid = str(eg.id)
|
eid = str(eg.id)
|
||||||
@@ -156,22 +164,19 @@ def generate_graph_html(store: EngramStore, output_path: str) -> str:
|
|||||||
"size": size,
|
"size": size,
|
||||||
"confidence": conf,
|
"confidence": conf,
|
||||||
"confirmed": eg.correctness.confirmed,
|
"confirmed": eg.correctness.confirmed,
|
||||||
|
"verdict": getattr(eg.correctness, "verdict", "unknown"),
|
||||||
"source": eg.metadata.get("source", "?"),
|
"source": eg.metadata.get("source", "?"),
|
||||||
"tags": tags,
|
"tags": tags,
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
node_ids.add(eid)
|
|
||||||
|
|
||||||
|
# Add edges after all nodes are known (otherwise early nodes miss links).
|
||||||
|
for eg in engrams:
|
||||||
|
eid = str(eg.id)
|
||||||
for lid in eg.links:
|
for lid in eg.links:
|
||||||
lid_s = str(lid)
|
lid_s = str(lid)
|
||||||
if lid_s in node_ids:
|
if lid_s in node_ids:
|
||||||
edges.append({
|
edges.append({"data": {"id": f"{eid}_{lid_s}", "source": eid, "target": lid_s}})
|
||||||
"data": {
|
|
||||||
"id": f"{eid}_{lid_s}",
|
|
||||||
"source": eid,
|
|
||||||
"target": lid_s,
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
elements = {"nodes": nodes, "edges": edges}
|
elements = {"nodes": nodes, "edges": edges}
|
||||||
html = _HTML_TEMPLATE.format(elements_json=json.dumps(elements, ensure_ascii=False))
|
html = _HTML_TEMPLATE.format(elements_json=json.dumps(elements, ensure_ascii=False))
|
||||||
|
|||||||
@@ -12,6 +12,7 @@ import os
|
|||||||
import sys
|
import sys
|
||||||
import json
|
import json
|
||||||
import traceback
|
import traceback
|
||||||
|
import re
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from datetime import datetime, timezone
|
from datetime import datetime, timezone
|
||||||
from typing import Optional, List, Dict, Any
|
from typing import Optional, List, Dict, Any
|
||||||
@@ -48,6 +49,45 @@ except Exception:
|
|||||||
# --- Konfiguration ---
|
# --- Konfiguration ---
|
||||||
BRAIN_DB = Path(__file__).parent.parent / "data" / "brain.sqlite"
|
BRAIN_DB = Path(__file__).parent.parent / "data" / "brain.sqlite"
|
||||||
|
|
||||||
|
_UUID_RE = re.compile(r"\b[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\b", re.I)
|
||||||
|
_SHORT_ID_RE = re.compile(r"\b[0-9a-f]{8}\b", re.I)
|
||||||
|
|
||||||
|
|
||||||
|
def _detect_feedback(content: str) -> dict | None:
|
||||||
|
"""
|
||||||
|
Heuristik: erkennt kurze Korrektur-/Feedback-Nachrichten in Chats.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
{"kind":"confirm"|"reject"|"stop", "target": <id or short-id or None>, "raw": <normalized>}
|
||||||
|
"""
|
||||||
|
if not isinstance(content, str):
|
||||||
|
return None
|
||||||
|
raw = content.strip()
|
||||||
|
if not raw:
|
||||||
|
return None
|
||||||
|
norm = raw.lower().strip()
|
||||||
|
|
||||||
|
target = None
|
||||||
|
m = _UUID_RE.search(raw)
|
||||||
|
if m:
|
||||||
|
target = m.group(0)
|
||||||
|
else:
|
||||||
|
m2 = _SHORT_ID_RE.search(raw)
|
||||||
|
if m2:
|
||||||
|
target = m2.group(0)
|
||||||
|
|
||||||
|
if norm in {"stop", "stopp", "halt"}:
|
||||||
|
return {"kind": "stop", "target": target, "raw": norm}
|
||||||
|
if norm in {"nein", "no", "falsch", "wrong"}:
|
||||||
|
return {"kind": "reject", "target": target, "raw": norm}
|
||||||
|
if norm in {"ja", "yes", "richtig", "korrekt", "stimmt"}:
|
||||||
|
return {"kind": "confirm", "target": target, "raw": norm}
|
||||||
|
if norm.startswith(("korrigiert", "korrektur", "correction")):
|
||||||
|
if "richtig" in norm or "korrekt" in norm:
|
||||||
|
return {"kind": "confirm", "target": target, "raw": norm}
|
||||||
|
return {"kind": "reject", "target": target, "raw": norm}
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
def get_brain() -> EngramStore:
|
def get_brain() -> EngramStore:
|
||||||
"""Gibt initialisierten Brain-Store."""
|
"""Gibt initialisierten Brain-Store."""
|
||||||
@@ -77,14 +117,43 @@ def save_session_learned(
|
|||||||
)
|
)
|
||||||
"""
|
"""
|
||||||
store = get_brain()
|
store = get_brain()
|
||||||
|
tags = tags or []
|
||||||
|
|
||||||
|
fb = _detect_feedback(content) if source == "session" else None
|
||||||
|
fb_target_id: str | None = None
|
||||||
|
if fb:
|
||||||
|
tags = list(dict.fromkeys(tags + ["feedback", fb["kind"]]))
|
||||||
|
confidence = min(confidence, 0.2)
|
||||||
|
grounding = Grounding.ASSUMPTION
|
||||||
|
if session_id:
|
||||||
|
recent = store.get_latest_by_session_id(session_id, limit=10, exclude_tags=["feedback"])
|
||||||
|
if recent:
|
||||||
|
fb_target_id = str(recent[0].id)
|
||||||
|
|
||||||
eg = Engram.create(
|
eg = Engram.create(
|
||||||
content=content,
|
content=content,
|
||||||
source=source,
|
source=source,
|
||||||
tags=tags or [],
|
tags=tags,
|
||||||
session_id=session_id,
|
session_id=session_id,
|
||||||
confidence=confidence,
|
confidence=confidence,
|
||||||
grounding=grounding,
|
grounding=grounding,
|
||||||
)
|
)
|
||||||
|
if fb and fb_target_id:
|
||||||
|
target = store.get(fb_target_id)
|
||||||
|
if target:
|
||||||
|
try:
|
||||||
|
# Link both ways for graphing/traceability
|
||||||
|
eg.links.append(target.id)
|
||||||
|
if eg.id not in target.links:
|
||||||
|
target.links.append(eg.id)
|
||||||
|
|
||||||
|
# Lock target so auto-review does not keep "re-deciding" after a correction signal.
|
||||||
|
target.metadata["predict_locked"] = True
|
||||||
|
target.metadata["predict_locked_reason"] = f"feedback:{fb['raw']}"
|
||||||
|
target.metadata["predict_locked_at"] = datetime.now(timezone.utc).isoformat()
|
||||||
|
store.save(target)
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
store.save(eg)
|
store.save(eg)
|
||||||
return eg
|
return eg
|
||||||
|
|
||||||
|
|||||||
34
src/store.py
34
src/store.py
@@ -6,6 +6,7 @@ Keine externen Abhängigkeiten außer sqlite3 (stdlib).
|
|||||||
import json
|
import json
|
||||||
import sqlite3
|
import sqlite3
|
||||||
import os
|
import os
|
||||||
|
import re
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import List, Optional, Dict, Any
|
from typing import List, Optional, Dict, Any
|
||||||
from uuid import UUID
|
from uuid import UUID
|
||||||
@@ -135,6 +136,32 @@ class EngramStore:
|
|||||||
).fetchall()
|
).fetchall()
|
||||||
return [self._row_to_engram(r) for r in rows]
|
return [self._row_to_engram(r) for r in rows]
|
||||||
|
|
||||||
|
def get_latest_by_session_id(
|
||||||
|
self,
|
||||||
|
session_id: str,
|
||||||
|
*,
|
||||||
|
limit: int = 5,
|
||||||
|
exclude_tags: Optional[List[str]] = None,
|
||||||
|
) -> List[Engram]:
|
||||||
|
"""
|
||||||
|
Lädt die neuesten Engramme für eine OpenClaw-Session-ID.
|
||||||
|
|
||||||
|
Hinweis: session_id liegt im `metadata_json`; wir nutzen eine robuste
|
||||||
|
LIKE-Suche, damit auch Legacy-Records gefunden werden.
|
||||||
|
"""
|
||||||
|
if not session_id:
|
||||||
|
return []
|
||||||
|
rows = self._conn.execute(
|
||||||
|
"SELECT * FROM engrams WHERE metadata_json LIKE ? ORDER BY created_at DESC LIMIT ?",
|
||||||
|
(f'%"session_id": "{session_id}"%', limit),
|
||||||
|
).fetchall()
|
||||||
|
engrams = [self._row_to_engram(r) for r in rows]
|
||||||
|
if exclude_tags:
|
||||||
|
ex = set(t for t in exclude_tags if isinstance(t, str))
|
||||||
|
if ex:
|
||||||
|
engrams = [e for e in engrams if not (ex & set(e.metadata.get("tags", []) or []))]
|
||||||
|
return engrams
|
||||||
|
|
||||||
def delete(self, engram_id: str) -> bool:
|
def delete(self, engram_id: str) -> bool:
|
||||||
"""Löscht ein Engramm und alle Verknüpfungen."""
|
"""Löscht ein Engramm und alle Verknüpfungen."""
|
||||||
rowid = self._conn.execute(
|
rowid = self._conn.execute(
|
||||||
@@ -158,7 +185,12 @@ class EngramStore:
|
|||||||
def search_text(self, query: str, limit: int = 10) -> List[Engram]:
|
def search_text(self, query: str, limit: int = 10) -> List[Engram]:
|
||||||
"""Full-Text-Suche über Engramm-Inhalt via SQLite FTS5 (OR-Verknüpfung)."""
|
"""Full-Text-Suche über Engramm-Inhalt via SQLite FTS5 (OR-Verknüpfung)."""
|
||||||
# FTS5-Syntax: Wörter mit OR verbinden für bessere Ergebnisse
|
# FTS5-Syntax: Wörter mit OR verbinden für bessere Ergebnisse
|
||||||
words = [w.strip() for w in query.replace("'", "''").split() if w.strip()]
|
words = []
|
||||||
|
for word in query.split():
|
||||||
|
# Nur alphanumerische Zeichen als FTS5-Tokens akzeptieren
|
||||||
|
clean_word = re.sub(r'[^a-zA-Z0-9]+', '', word)
|
||||||
|
if clean_word:
|
||||||
|
words.append(clean_word)
|
||||||
safe_query = " OR ".join(words) if len(words) > 1 else (words[0] if words else "*")
|
safe_query = " OR ".join(words) if len(words) > 1 else (words[0] if words else "*")
|
||||||
sql = """
|
sql = """
|
||||||
SELECT e.* FROM engrams e
|
SELECT e.* FROM engrams e
|
||||||
|
|||||||
141
static/style.css
141
static/style.css
@@ -27,6 +27,31 @@ body {
|
|||||||
top: 0;
|
top: 0;
|
||||||
z-index: 50;
|
z-index: 50;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
.tabs-bar{
|
||||||
|
display:flex;
|
||||||
|
gap:8px;
|
||||||
|
padding:8px 12px 10px;
|
||||||
|
background:#141419;
|
||||||
|
border-bottom:1px solid #252530;
|
||||||
|
position: sticky;
|
||||||
|
top: 52px;
|
||||||
|
z-index: 45;
|
||||||
|
}
|
||||||
|
.tabs-bar .tab-btn{
|
||||||
|
flex:1;
|
||||||
|
background:#1e1e28;
|
||||||
|
border:1px solid #2a2a3a;
|
||||||
|
border-radius: 12px;
|
||||||
|
padding:10px 10px;
|
||||||
|
color:#cfd3ff;
|
||||||
|
font-weight:700;
|
||||||
|
font-size:0.82rem;
|
||||||
|
}
|
||||||
|
.tabs-bar .tab-btn.active{
|
||||||
|
border-color:#6c8af5;
|
||||||
|
box-shadow:0 0 0 1px rgba(108,138,245,0.22) inset;
|
||||||
|
}
|
||||||
.stat {
|
.stat {
|
||||||
text-align: center;
|
text-align: center;
|
||||||
min-width: 60px;
|
min-width: 60px;
|
||||||
@@ -54,6 +79,122 @@ body {
|
|||||||
padding: 10px 12px;
|
padding: 10px 12px;
|
||||||
background: #141419;
|
background: #141419;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* tab buttons styled via .tabs-bar */
|
||||||
|
|
||||||
|
/* ─── Panels (Graph/Status) ──────────────────────────────────────────────── */
|
||||||
|
.panel {
|
||||||
|
margin: 8px 12px;
|
||||||
|
background: #1a1a24;
|
||||||
|
border: 1px solid #252533;
|
||||||
|
border-radius: 14px;
|
||||||
|
padding: 10px 12px;
|
||||||
|
}
|
||||||
|
.panel-title {
|
||||||
|
font-size: 0.8rem;
|
||||||
|
text-transform: uppercase;
|
||||||
|
letter-spacing: 0.5px;
|
||||||
|
color: #9aa3d9;
|
||||||
|
margin-bottom: 8px;
|
||||||
|
}
|
||||||
|
.kv-row {
|
||||||
|
display: flex;
|
||||||
|
gap: 10px;
|
||||||
|
padding: 6px 0;
|
||||||
|
border-bottom: 1px solid #20202a;
|
||||||
|
}
|
||||||
|
.kv-row:last-child { border-bottom: none; }
|
||||||
|
.kv-key {
|
||||||
|
width: 110px;
|
||||||
|
color: #888899;
|
||||||
|
font-size: 0.78rem;
|
||||||
|
}
|
||||||
|
.kv-val {
|
||||||
|
flex: 1;
|
||||||
|
color: #e8e8ee;
|
||||||
|
font-size: 0.85rem;
|
||||||
|
word-break: break-word;
|
||||||
|
}
|
||||||
|
.pill {
|
||||||
|
display: inline-block;
|
||||||
|
margin: 2px 4px 2px 0;
|
||||||
|
padding: 2px 8px;
|
||||||
|
border-radius: 999px;
|
||||||
|
background: #2a2a3a;
|
||||||
|
color: #8a9aff;
|
||||||
|
font-size: 0.72rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
.verdict-pill{
|
||||||
|
display:inline-block;
|
||||||
|
margin: 2px 6px 2px 0;
|
||||||
|
padding: 2px 8px;
|
||||||
|
border-radius: 999px;
|
||||||
|
font-size: 0.72rem;
|
||||||
|
font-weight: 800;
|
||||||
|
letter-spacing: 0.4px;
|
||||||
|
border: 1px solid #2a2a3a;
|
||||||
|
background: #1e1e28;
|
||||||
|
color: #cfd3ff;
|
||||||
|
}
|
||||||
|
.verdict-pill.v-true{ border-color:#2f6b3f; color:#aaf0b6; }
|
||||||
|
.verdict-pill.v-false{ border-color:#7a2c2c; color:#ffb3b3; }
|
||||||
|
.verdict-pill.v-prob-true{ border-color:#6c8af5; color:#cfd9ff; }
|
||||||
|
.verdict-pill.v-prob-false{ border-color:#b08a2a; color:#ffe2a3; }
|
||||||
|
.verdict-pill.v-unknown{ border-color:#3a3a55; color:#b9b9c9; }
|
||||||
|
.muted {
|
||||||
|
color: #888899;
|
||||||
|
font-size: 0.8rem;
|
||||||
|
margin-top: 6px;
|
||||||
|
}
|
||||||
|
.small { font-size: 0.75rem; }
|
||||||
|
|
||||||
|
/* Graph canvas */
|
||||||
|
#graphCanvas{
|
||||||
|
display:block;
|
||||||
|
margin: 8px auto 0;
|
||||||
|
background:#12121a;
|
||||||
|
border:1px solid #252533;
|
||||||
|
border-radius: 14px;
|
||||||
|
touch-action: none;
|
||||||
|
}
|
||||||
|
|
||||||
|
.graph-controls{
|
||||||
|
display:flex;
|
||||||
|
gap:8px;
|
||||||
|
padding: 10px 12px 0;
|
||||||
|
align-items:center;
|
||||||
|
flex-wrap: wrap;
|
||||||
|
}
|
||||||
|
.graph-controls .btn{
|
||||||
|
background:#1e1e28;
|
||||||
|
border:1px solid #2a2a3a;
|
||||||
|
border-radius: 10px;
|
||||||
|
padding: 8px 10px;
|
||||||
|
color:#cfd3ff;
|
||||||
|
font-weight:700;
|
||||||
|
font-size:0.82rem;
|
||||||
|
}
|
||||||
|
.graph-controls .btn.primary{
|
||||||
|
border-color:#6c8af5;
|
||||||
|
box-shadow:0 0 0 1px rgba(108,138,245,0.18) inset;
|
||||||
|
}
|
||||||
|
.graph-legend{
|
||||||
|
margin: 8px 12px 0;
|
||||||
|
padding: 10px 12px;
|
||||||
|
background:#1a1a24;
|
||||||
|
border:1px solid #252533;
|
||||||
|
border-radius: 14px;
|
||||||
|
color:#b9b9c9;
|
||||||
|
font-size:0.8rem;
|
||||||
|
line-height:1.4;
|
||||||
|
}
|
||||||
|
.legend-row{ display:flex; align-items:center; gap:8px; margin-top:6px; }
|
||||||
|
.legend-dot{ width:10px; height:10px; border-radius:50%; display:inline-block; }
|
||||||
|
.legend-dot.engram{ background:#6c8af5; }
|
||||||
|
.legend-dot.tag{ background:#8a9aff; }
|
||||||
|
.legend-dot.match{ background:#f7d154; }
|
||||||
|
.graph-hint{ padding: 4px 12px 10px; }
|
||||||
#searchInput {
|
#searchInput {
|
||||||
flex: 1;
|
flex: 1;
|
||||||
background: #1e1e28;
|
background: #1e1e28;
|
||||||
|
|||||||
9
systemd/openclaw-memory-archive.service
Normal file
9
systemd/openclaw-memory-archive.service
Normal file
@@ -0,0 +1,9 @@
|
|||||||
|
[Unit]
|
||||||
|
Description=OpenClaw archive memory/*.md older than 7 days
|
||||||
|
OnFailure=openclaw-secondbrain-notify@%n.service
|
||||||
|
|
||||||
|
[Service]
|
||||||
|
Type=oneshot
|
||||||
|
WorkingDirectory=/root/.openclaw/workspace
|
||||||
|
ExecStart=/bin/bash -lc 'flock -n /tmp/%n.lock /usr/bin/python3 /root/.openclaw/workspace/openclaw_cron_wrapper.py archive_memory_md'
|
||||||
|
|
||||||
10
systemd/openclaw-memory-archive.timer
Normal file
10
systemd/openclaw-memory-archive.timer
Normal file
@@ -0,0 +1,10 @@
|
|||||||
|
[Unit]
|
||||||
|
Description=OpenClaw archive memory/*.md daily
|
||||||
|
|
||||||
|
[Timer]
|
||||||
|
OnCalendar=*-*-* 03:30:00
|
||||||
|
Persistent=true
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=timers.target
|
||||||
|
|
||||||
8
systemd/openclaw-secondbrain-backup.service
Normal file
8
systemd/openclaw-secondbrain-backup.service
Normal file
@@ -0,0 +1,8 @@
|
|||||||
|
[Unit]
|
||||||
|
Description=OpenClaw Second-Brain backup_secondbrain
|
||||||
|
OnFailure=openclaw-secondbrain-notify@%n.service
|
||||||
|
|
||||||
|
[Service]
|
||||||
|
Type=oneshot
|
||||||
|
WorkingDirectory=/root/.openclaw/workspace
|
||||||
|
ExecStart=/bin/bash -lc 'flock -n /tmp/%n.lock /usr/bin/python3 /root/.openclaw/workspace/openclaw_cron_wrapper.py backup_secondbrain'
|
||||||
10
systemd/openclaw-secondbrain-backup.timer
Normal file
10
systemd/openclaw-secondbrain-backup.timer
Normal file
@@ -0,0 +1,10 @@
|
|||||||
|
[Unit]
|
||||||
|
Description=OpenClaw Second-Brain backup_secondbrain (daily 02:00)
|
||||||
|
|
||||||
|
[Timer]
|
||||||
|
OnCalendar=*-*-* 02:00:00
|
||||||
|
Persistent=true
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=timers.target
|
||||||
|
|
||||||
15
systemd/openclaw-secondbrain-dashboard.service
Normal file
15
systemd/openclaw-secondbrain-dashboard.service
Normal file
@@ -0,0 +1,15 @@
|
|||||||
|
[Unit]
|
||||||
|
Description=OpenClaw Second-Brain Dashboard (FastAPI)
|
||||||
|
After=network.target
|
||||||
|
|
||||||
|
[Service]
|
||||||
|
Type=simple
|
||||||
|
WorkingDirectory=/root/.openclaw/workspace/second-brain
|
||||||
|
Environment=SECOND_BRAIN_WORKSPACE=/root/.openclaw/workspace/second-brain
|
||||||
|
Environment=SECOND_BRAIN_PORT=8501
|
||||||
|
ExecStart=/root/.openclaw/workspace/second-brain/.venv/bin/uvicorn fastapi_app:app --host 0.0.0.0 --port 8501
|
||||||
|
Restart=always
|
||||||
|
RestartSec=2
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=multi-user.target
|
||||||
8
systemd/openclaw-secondbrain-export-obsidian.service
Normal file
8
systemd/openclaw-secondbrain-export-obsidian.service
Normal file
@@ -0,0 +1,8 @@
|
|||||||
|
[Unit]
|
||||||
|
Description=OpenClaw Second-Brain export_obsidian
|
||||||
|
OnFailure=openclaw-secondbrain-notify@%n.service
|
||||||
|
|
||||||
|
[Service]
|
||||||
|
Type=oneshot
|
||||||
|
WorkingDirectory=/root/.openclaw/workspace
|
||||||
|
ExecStart=/bin/bash -lc 'flock -n /tmp/%n.lock /usr/bin/python3 /root/.openclaw/workspace/openclaw_cron_wrapper.py export_obsidian'
|
||||||
10
systemd/openclaw-secondbrain-export-obsidian.timer
Normal file
10
systemd/openclaw-secondbrain-export-obsidian.timer
Normal file
@@ -0,0 +1,10 @@
|
|||||||
|
[Unit]
|
||||||
|
Description=OpenClaw Second-Brain export_obsidian (hourly)
|
||||||
|
|
||||||
|
[Timer]
|
||||||
|
OnCalendar=hourly
|
||||||
|
Persistent=true
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=timers.target
|
||||||
|
|
||||||
8
systemd/openclaw-secondbrain-heartbeat.service
Normal file
8
systemd/openclaw-secondbrain-heartbeat.service
Normal file
@@ -0,0 +1,8 @@
|
|||||||
|
[Unit]
|
||||||
|
Description=OpenClaw Second-Brain heartbeat_secondbrain
|
||||||
|
OnFailure=openclaw-secondbrain-notify@%n.service
|
||||||
|
|
||||||
|
[Service]
|
||||||
|
Type=oneshot
|
||||||
|
WorkingDirectory=/root/.openclaw/workspace
|
||||||
|
ExecStart=/bin/bash -lc 'flock -n /tmp/%n.lock /usr/bin/python3 /root/.openclaw/workspace/openclaw_cron_wrapper.py heartbeat_secondbrain'
|
||||||
11
systemd/openclaw-secondbrain-heartbeat.timer
Normal file
11
systemd/openclaw-secondbrain-heartbeat.timer
Normal file
@@ -0,0 +1,11 @@
|
|||||||
|
[Unit]
|
||||||
|
Description=OpenClaw Second-Brain heartbeat_secondbrain (every 6h)
|
||||||
|
|
||||||
|
[Timer]
|
||||||
|
OnCalendar=*-*-* 00,06,12,18:00:00
|
||||||
|
RandomizedDelaySec=300
|
||||||
|
Persistent=true
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=timers.target
|
||||||
|
|
||||||
10
systemd/openclaw-secondbrain-index-vectors.service
Normal file
10
systemd/openclaw-secondbrain-index-vectors.service
Normal file
@@ -0,0 +1,10 @@
|
|||||||
|
[Unit]
|
||||||
|
Description=OpenClaw Second-Brain index_vectors
|
||||||
|
OnFailure=openclaw-secondbrain-notify@%n.service
|
||||||
|
|
||||||
|
[Service]
|
||||||
|
Type=oneshot
|
||||||
|
WorkingDirectory=/root/.openclaw/workspace
|
||||||
|
Environment=HF_HOME=/root/.openclaw/workspace/second-brain/data/hf_cache
|
||||||
|
Environment=SENTENCE_TRANSFORMERS_HOME=/root/.openclaw/workspace/second-brain/data/st_cache
|
||||||
|
ExecStart=/bin/bash -lc 'flock -n /tmp/%n.lock /usr/bin/python3 /root/.openclaw/workspace/openclaw_cron_wrapper.py index_vectors'
|
||||||
10
systemd/openclaw-secondbrain-index-vectors.timer
Normal file
10
systemd/openclaw-secondbrain-index-vectors.timer
Normal file
@@ -0,0 +1,10 @@
|
|||||||
|
[Unit]
|
||||||
|
Description=OpenClaw Second-Brain index_vectors (every 30 min)
|
||||||
|
|
||||||
|
[Timer]
|
||||||
|
OnCalendar=*:0/30
|
||||||
|
Persistent=true
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=timers.target
|
||||||
|
|
||||||
8
systemd/openclaw-secondbrain-ingest-memory.service
Normal file
8
systemd/openclaw-secondbrain-ingest-memory.service
Normal file
@@ -0,0 +1,8 @@
|
|||||||
|
[Unit]
|
||||||
|
Description=OpenClaw Second-Brain ingest_memory
|
||||||
|
OnFailure=openclaw-secondbrain-notify@%n.service
|
||||||
|
|
||||||
|
[Service]
|
||||||
|
Type=oneshot
|
||||||
|
WorkingDirectory=/root/.openclaw/workspace
|
||||||
|
ExecStart=/bin/bash -lc 'flock -n /tmp/%n.lock /usr/bin/python3 /root/.openclaw/workspace/openclaw_cron_wrapper.py ingest_memory'
|
||||||
9
systemd/openclaw-secondbrain-ingest-memory.timer
Normal file
9
systemd/openclaw-secondbrain-ingest-memory.timer
Normal file
@@ -0,0 +1,9 @@
|
|||||||
|
[Unit]
|
||||||
|
Description=OpenClaw Second-Brain ingest_memory (every 5 min)
|
||||||
|
|
||||||
|
[Timer]
|
||||||
|
OnCalendar=*:0/5
|
||||||
|
Persistent=true
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=timers.target
|
||||||
8
systemd/openclaw-secondbrain-ingest-obsidian.service
Normal file
8
systemd/openclaw-secondbrain-ingest-obsidian.service
Normal file
@@ -0,0 +1,8 @@
|
|||||||
|
[Unit]
|
||||||
|
Description=OpenClaw Second-Brain ingest_obsidian
|
||||||
|
OnFailure=openclaw-secondbrain-notify@%n.service
|
||||||
|
|
||||||
|
[Service]
|
||||||
|
Type=oneshot
|
||||||
|
WorkingDirectory=/root/.openclaw/workspace
|
||||||
|
ExecStart=/bin/bash -lc 'flock -n /tmp/%n.lock /usr/bin/python3 /root/.openclaw/workspace/openclaw_cron_wrapper.py ingest_obsidian'
|
||||||
10
systemd/openclaw-secondbrain-ingest-obsidian.timer
Normal file
10
systemd/openclaw-secondbrain-ingest-obsidian.timer
Normal file
@@ -0,0 +1,10 @@
|
|||||||
|
[Unit]
|
||||||
|
Description=OpenClaw Second-Brain ingest_obsidian (every 15 min)
|
||||||
|
|
||||||
|
[Timer]
|
||||||
|
OnCalendar=*:0/15
|
||||||
|
Persistent=true
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=timers.target
|
||||||
|
|
||||||
@@ -0,0 +1,9 @@
|
|||||||
|
[Unit]
|
||||||
|
Description=OpenClaw Second-Brain ingest transcript -> DB
|
||||||
|
OnFailure=openclaw-secondbrain-notify@%n.service
|
||||||
|
|
||||||
|
[Service]
|
||||||
|
Type=oneshot
|
||||||
|
WorkingDirectory=/root/.openclaw/workspace
|
||||||
|
ExecStart=/bin/bash -lc 'flock -n /tmp/%n.lock /usr/bin/python3 /root/.openclaw/workspace/openclaw_cron_wrapper.py ingest_transcript_to_db'
|
||||||
|
|
||||||
12
systemd/openclaw-secondbrain-ingest-transcript-to-db.timer
Normal file
12
systemd/openclaw-secondbrain-ingest-transcript-to-db.timer
Normal file
@@ -0,0 +1,12 @@
|
|||||||
|
[Unit]
|
||||||
|
Description=OpenClaw Second-Brain ingest transcript -> DB (every 5 min)
|
||||||
|
|
||||||
|
[Timer]
|
||||||
|
OnBootSec=90s
|
||||||
|
OnUnitActiveSec=300s
|
||||||
|
Unit=openclaw-secondbrain-ingest-transcript-to-db.service
|
||||||
|
Persistent=true
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=timers.target
|
||||||
|
|
||||||
@@ -0,0 +1,8 @@
|
|||||||
|
[Unit]
|
||||||
|
Description=OpenClaw Second-Brain ingest transcript -> memory/*.md
|
||||||
|
OnFailure=openclaw-secondbrain-notify@%n.service
|
||||||
|
|
||||||
|
[Service]
|
||||||
|
Type=oneshot
|
||||||
|
WorkingDirectory=/root/.openclaw/workspace
|
||||||
|
ExecStart=/bin/bash -lc 'flock -n /tmp/%n.lock /usr/bin/python3 /root/.openclaw/workspace/second-brain/cron_tasks/ingest_transcript_to_memory.py'
|
||||||
@@ -0,0 +1,11 @@
|
|||||||
|
[Unit]
|
||||||
|
Description=OpenClaw Second-Brain ingest transcript -> memory (every 1 min)
|
||||||
|
|
||||||
|
[Timer]
|
||||||
|
OnBootSec=30s
|
||||||
|
OnUnitActiveSec=60s
|
||||||
|
Unit=openclaw-secondbrain-ingest-transcript-to-memory.service
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=timers.target
|
||||||
|
|
||||||
8
systemd/openclaw-secondbrain-notify@.service
Normal file
8
systemd/openclaw-secondbrain-notify@.service
Normal file
@@ -0,0 +1,8 @@
|
|||||||
|
[Unit]
|
||||||
|
Description=OpenClaw Second-Brain failure notify (%i)
|
||||||
|
|
||||||
|
[Service]
|
||||||
|
Type=oneshot
|
||||||
|
WorkingDirectory=/root/.openclaw/workspace
|
||||||
|
ExecStart=/bin/bash -lc '/root/.openclaw/workspace/notify-telegram.sh "❌ Second-Brain job failed: %i. Check: journalctl -u %i -n 50 --no-pager"'
|
||||||
|
|
||||||
10
systemd/openclaw-secondbrain-proactive-search.service
Normal file
10
systemd/openclaw-secondbrain-proactive-search.service
Normal file
@@ -0,0 +1,10 @@
|
|||||||
|
[Unit]
|
||||||
|
Description=OpenClaw Second-Brain proactive_search_wrapper
|
||||||
|
Wants=network-online.target
|
||||||
|
After=network-online.target
|
||||||
|
OnFailure=openclaw-secondbrain-notify@%n.service
|
||||||
|
|
||||||
|
[Service]
|
||||||
|
Type=oneshot
|
||||||
|
WorkingDirectory=/root/.openclaw/workspace
|
||||||
|
ExecStart=/bin/bash -lc 'flock -n /tmp/%n.lock /usr/bin/python3 /root/.openclaw/workspace/openclaw_cron_wrapper.py proactive_search_wrapper'
|
||||||
11
systemd/openclaw-secondbrain-proactive-search.timer
Normal file
11
systemd/openclaw-secondbrain-proactive-search.timer
Normal file
@@ -0,0 +1,11 @@
|
|||||||
|
[Unit]
|
||||||
|
Description=OpenClaw Second-Brain proactive_search_wrapper (every 4h)
|
||||||
|
|
||||||
|
[Timer]
|
||||||
|
OnCalendar=*-*-* 01,05,09,13,17,21:10:00
|
||||||
|
RandomizedDelaySec=600
|
||||||
|
Persistent=true
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=timers.target
|
||||||
|
|
||||||
8
systemd/openclaw-secondbrain-review.service
Normal file
8
systemd/openclaw-secondbrain-review.service
Normal file
@@ -0,0 +1,8 @@
|
|||||||
|
[Unit]
|
||||||
|
Description=OpenClaw Second-Brain review_brain
|
||||||
|
OnFailure=openclaw-secondbrain-notify@%n.service
|
||||||
|
|
||||||
|
[Service]
|
||||||
|
Type=oneshot
|
||||||
|
WorkingDirectory=/root/.openclaw/workspace
|
||||||
|
ExecStart=/bin/bash -lc 'flock -n /tmp/%n.lock /usr/bin/python3 /root/.openclaw/workspace/openclaw_cron_wrapper.py review_brain'
|
||||||
9
systemd/openclaw-secondbrain-review.timer
Normal file
9
systemd/openclaw-secondbrain-review.timer
Normal file
@@ -0,0 +1,9 @@
|
|||||||
|
[Unit]
|
||||||
|
Description=OpenClaw Second-Brain review_brain (every 5 min)
|
||||||
|
|
||||||
|
[Timer]
|
||||||
|
OnCalendar=*:0/5
|
||||||
|
Persistent=true
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=timers.target
|
||||||
13
systemd/openclaw-secondbrain-task@.service
Normal file
13
systemd/openclaw-secondbrain-task@.service
Normal file
@@ -0,0 +1,13 @@
|
|||||||
|
[Unit]
|
||||||
|
Description=OpenClaw Second-Brain task (%i)
|
||||||
|
Wants=network-online.target
|
||||||
|
After=network-online.target
|
||||||
|
|
||||||
|
[Service]
|
||||||
|
Type=oneshot
|
||||||
|
WorkingDirectory=/root/.openclaw/workspace
|
||||||
|
ExecStart=/usr/bin/python3 /root/.openclaw/workspace/openclaw_cron_wrapper.py %i
|
||||||
|
Nice=10
|
||||||
|
IOSchedulingClass=best-effort
|
||||||
|
IOSchedulingPriority=6
|
||||||
|
|
||||||
9
systemd/openclaw-secondbrain-verify-pending.service
Normal file
9
systemd/openclaw-secondbrain-verify-pending.service
Normal file
@@ -0,0 +1,9 @@
|
|||||||
|
[Unit]
|
||||||
|
Description=OpenClaw Second-Brain verify_pending_external
|
||||||
|
OnFailure=openclaw-secondbrain-notify@%n.service
|
||||||
|
|
||||||
|
[Service]
|
||||||
|
Type=oneshot
|
||||||
|
WorkingDirectory=/root/.openclaw/workspace
|
||||||
|
ExecStart=/bin/bash -lc 'flock -n /tmp/%n.lock /usr/bin/python3 /root/.openclaw/workspace/openclaw_cron_wrapper.py verify_pending_external'
|
||||||
|
|
||||||
10
systemd/openclaw-secondbrain-verify-pending.timer
Normal file
10
systemd/openclaw-secondbrain-verify-pending.timer
Normal file
@@ -0,0 +1,10 @@
|
|||||||
|
[Unit]
|
||||||
|
Description=OpenClaw Second-Brain periodic verify_pending_external
|
||||||
|
|
||||||
|
[Timer]
|
||||||
|
OnBootSec=2min
|
||||||
|
OnUnitActiveSec=5min
|
||||||
|
Persistent=true
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=timers.target
|
||||||
@@ -16,6 +16,12 @@
|
|||||||
<div class="stat"><span class="stat-num" id="statErrors">-</span><span class="stat-label">Err</span></div>
|
<div class="stat"><span class="stat-num" id="statErrors">-</span><span class="stat-label">Err</span></div>
|
||||||
</header>
|
</header>
|
||||||
|
|
||||||
|
<div class="tabs-bar">
|
||||||
|
<button class="tab-btn active" id="tabCards" onclick="setView('cards')">Cards</button>
|
||||||
|
<button class="tab-btn" id="tabGraph" onclick="setView('graph')">Graph</button>
|
||||||
|
<button class="tab-btn" id="tabStatus" onclick="setView('status')">Status</button>
|
||||||
|
</div>
|
||||||
|
|
||||||
<!-- Search -->
|
<!-- Search -->
|
||||||
<div class="search-box">
|
<div class="search-box">
|
||||||
<input type="text" id="searchInput" placeholder="🔍 Suche..." />
|
<input type="text" id="searchInput" placeholder="🔍 Suche..." />
|
||||||
@@ -23,6 +29,7 @@
|
|||||||
<option value="all">Alle</option>
|
<option value="all">Alle</option>
|
||||||
<option value="pending">Pending</option>
|
<option value="pending">Pending</option>
|
||||||
<option value="confirmed">Confirmed</option>
|
<option value="confirmed">Confirmed</option>
|
||||||
|
<option value="rejected">Rejected</option>
|
||||||
<option value="errors">Errors</option>
|
<option value="errors">Errors</option>
|
||||||
</select>
|
</select>
|
||||||
</div>
|
</div>
|
||||||
@@ -37,6 +44,38 @@
|
|||||||
<!-- Cards -->
|
<!-- Cards -->
|
||||||
<div class="cards" id="cards"></div>
|
<div class="cards" id="cards"></div>
|
||||||
|
|
||||||
|
<!-- Graph -->
|
||||||
|
<div class="graph" id="graph" style="display:none;">
|
||||||
|
<div class="graph-controls">
|
||||||
|
<button class="btn primary" id="btnGraphPhysics" onclick="toggleGraphPhysics()">Physics: off</button>
|
||||||
|
<button class="btn" onclick="resetGraphView()">Reset view</button>
|
||||||
|
<button class="btn" onclick="fitGraphView()">Fit</button>
|
||||||
|
<label class="muted small" style="display:flex;align-items:center;gap:8px">
|
||||||
|
<span>Physics</span>
|
||||||
|
<input id="physicsStrength" type="range" min="0" max="100" value="60" oninput="setPhysicsStrength(this.value)" style="width:140px">
|
||||||
|
<span id="physicsStrengthVal">60</span>
|
||||||
|
</label>
|
||||||
|
<select class="btn" id="graphLimit" onchange="reloadGraph()" title="Wie viele Knoten laden? 0=all">
|
||||||
|
<option value="0">Nodes: all</option>
|
||||||
|
<option value="200">Nodes: 200</option>
|
||||||
|
<option value="1000">Nodes: 1000</option>
|
||||||
|
<option value="5000">Nodes: 5000</option>
|
||||||
|
</select>
|
||||||
|
<button class="btn" onclick="reloadGraph()">Reload</button>
|
||||||
|
</div>
|
||||||
|
<canvas id="graphCanvas" width="440" height="520"></canvas>
|
||||||
|
<div class="graph-hint muted small" id="graphHint">Lade Graph…</div>
|
||||||
|
<div class="graph-legend">
|
||||||
|
<div><strong>Graph</strong>: Zoom per Pinch (2 Finger), Pan per Drag (1 Finger). Tap auf Engram öffnet Details, Tap auf Tag setzt Suche.</div>
|
||||||
|
<div class="legend-row"><span class="legend-dot engram"></span> Engram</div>
|
||||||
|
<div class="legend-row"><span class="legend-dot tag"></span> Tag</div>
|
||||||
|
<div class="legend-row"><span class="legend-dot match"></span> Match (Suche)</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<!-- Status -->
|
||||||
|
<div class="status" id="status" style="display:none;"></div>
|
||||||
|
|
||||||
<!-- Pagination -->
|
<!-- Pagination -->
|
||||||
<div class="pagination" id="pagination">
|
<div class="pagination" id="pagination">
|
||||||
<button id="btnPrev" onclick="prevPage()">◀</button>
|
<button id="btnPrev" onclick="prevPage()">◀</button>
|
||||||
@@ -67,12 +106,25 @@ let state = {
|
|||||||
filter: 'all',
|
filter: 'all',
|
||||||
search: '',
|
search: '',
|
||||||
autoRefresh: true,
|
autoRefresh: true,
|
||||||
|
view: 'cards',
|
||||||
|
lastEvent: null,
|
||||||
};
|
};
|
||||||
|
|
||||||
// ─── Fetch ──────────────────────────────────────────────────────────────────
|
// ─── Fetch ──────────────────────────────────────────────────────────────────
|
||||||
async function api(path, opts = {}) {
|
async function api(path, opts = {}) {
|
||||||
const r = await fetch(path, opts);
|
const r = await fetch(path, opts);
|
||||||
if (!r.ok) throw new Error((await r.json()).error || r.statusText);
|
if (!r.ok) {
|
||||||
|
let msg = r.statusText;
|
||||||
|
try {
|
||||||
|
const j = await r.json();
|
||||||
|
msg = j.error || j.detail || msg;
|
||||||
|
if (Array.isArray(msg)) msg = JSON.stringify(msg);
|
||||||
|
if (typeof msg !== 'string') msg = JSON.stringify(msg);
|
||||||
|
} catch (e) {
|
||||||
|
// ignore JSON parse errors
|
||||||
|
}
|
||||||
|
throw new Error(msg);
|
||||||
|
}
|
||||||
return r.json();
|
return r.json();
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -84,11 +136,36 @@ async function loadStats() {
|
|||||||
document.getElementById('statErrors').textContent = s.errors;
|
document.getElementById('statErrors').textContent = s.errors;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function updateStatsFromEvent(ev) {
|
||||||
|
if (!ev || !ev.stats) return;
|
||||||
|
const s = ev.stats;
|
||||||
|
document.getElementById('statTotal').textContent = s.total;
|
||||||
|
document.getElementById('statConfirmed').textContent = s.confirmed;
|
||||||
|
document.getElementById('statPending').textContent = s.pending;
|
||||||
|
document.getElementById('statErrors').textContent = s.errors;
|
||||||
|
}
|
||||||
|
|
||||||
|
function setView(view) {
|
||||||
|
state.view = view;
|
||||||
|
document.getElementById('tabCards').classList.toggle('active', view === 'cards');
|
||||||
|
document.getElementById('tabGraph').classList.toggle('active', view === 'graph');
|
||||||
|
document.getElementById('tabStatus').classList.toggle('active', view === 'status');
|
||||||
|
|
||||||
|
document.getElementById('cards').style.display = view === 'cards' ? '' : 'none';
|
||||||
|
document.getElementById('pagination').style.display = view === 'cards' ? '' : 'none';
|
||||||
|
document.getElementById('graph').style.display = view === 'graph' ? '' : 'none';
|
||||||
|
document.getElementById('status').style.display = view === 'status' ? '' : 'none';
|
||||||
|
|
||||||
|
if (view === 'graph') loadGraph();
|
||||||
|
if (view === 'status') loadStatus();
|
||||||
|
}
|
||||||
|
|
||||||
async function loadCards() {
|
async function loadCards() {
|
||||||
let url = `/api/engrams?limit=${state.limit}&offset=${state.offset}`;
|
let url = `/api/engrams?limit=${state.limit}&offset=${state.offset}`;
|
||||||
if (state.search) url += `&search=${encodeURIComponent(state.search)}`;
|
if (state.search) url += `&search=${encodeURIComponent(state.search)}`;
|
||||||
if (state.filter === 'confirmed') url += '&confirmed=1';
|
if (state.filter === 'confirmed') url += '&confirmed=1';
|
||||||
if (state.filter === 'pending') url += '&confirmed=0';
|
if (state.filter === 'pending') url += '&confirmed=0';
|
||||||
|
if (state.filter === 'rejected') url += '&verdict=confirmed_false';
|
||||||
if (state.filter === 'errors') url += '&tag=error';
|
if (state.filter === 'errors') url += '&tag=error';
|
||||||
|
|
||||||
const data = await api(url);
|
const data = await api(url);
|
||||||
@@ -99,12 +176,788 @@ async function loadCards() {
|
|||||||
document.getElementById('btnNext').disabled = data.items.length < state.limit;
|
document.getElementById('btnNext').disabled = data.items.length < state.limit;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async function loadStatus() {
|
||||||
|
const reqs = await Promise.allSettled([
|
||||||
|
api('/api/config'),
|
||||||
|
api('/api/db_info'),
|
||||||
|
api('/api/jobs'),
|
||||||
|
api('/api/insights?limit=8'),
|
||||||
|
api('/api/storage_stats'),
|
||||||
|
api('/api/pending?limit=20&offset=0'),
|
||||||
|
]);
|
||||||
|
const pick = (i, fallback) => (reqs[i].status === 'fulfilled' ? reqs[i].value : fallback);
|
||||||
|
const err = (i) => (reqs[i].status === 'rejected' ? (reqs[i].reason && reqs[i].reason.message ? reqs[i].reason.message : String(reqs[i].reason)) : null);
|
||||||
|
|
||||||
|
const cfg = pick(0, { workspace: '-', db_path: '-' });
|
||||||
|
const db = pick(1, { db_path: '-', mtime: null });
|
||||||
|
const jobs = pick(2, { items: [], error: err(2) });
|
||||||
|
const ins = pick(3, { pending: '-', top_tags: [], top_hosts: [], error: err(3) });
|
||||||
|
const stor = pick(4, { sql: { total_engrams: '-', confirmed: '-', pending: '-', by_source: {} }, vector: { chroma_size_bytes: 0, embedding_cache_files: 0 }, obsidian: { configured: false } });
|
||||||
|
const pend = pick(5, { items: [], error: err(5) });
|
||||||
|
|
||||||
|
const el = document.getElementById('status');
|
||||||
|
const jobsHtml = (jobs.items || []).map(j => `
|
||||||
|
<div class="kv-row">
|
||||||
|
<div class="kv-key">${j.unit}</div>
|
||||||
|
<div class="kv-val">${j.error ? ('ERR: ' + j.error) : (j.active + '/' + j.sub)}</div>
|
||||||
|
</div>
|
||||||
|
`).join('');
|
||||||
|
|
||||||
|
const topTags = (ins.top_tags || []).map(t => `<span class="pill">${t.key} (${t.count})</span>`).join(' ');
|
||||||
|
const topHosts = (ins.top_hosts || []).map(t => `<span class="pill">${t.key} (${t.count})</span>`).join(' ');
|
||||||
|
const bySource = Object.entries((stor.sql && stor.sql.by_source) ? stor.sql.by_source : {})
|
||||||
|
.slice(0, 8)
|
||||||
|
.map(([k,v]) => `<span class="pill">${k}: ${v}</span>`)
|
||||||
|
.join(' ');
|
||||||
|
|
||||||
|
const pendItems = (pend.items || []);
|
||||||
|
const pendHtml = pendItems.map(p => `
|
||||||
|
<div class="kv-row" onclick="showDetail('${p.id}')">
|
||||||
|
<div class="kv-key">${(p.source||'').slice(0,12)}</div>
|
||||||
|
<div class="kv-val">
|
||||||
|
<span class="pill">${p.id.substring(0,8)}</span>
|
||||||
|
${escapeHtml((p.content||'').substring(0,120))}${(p.content||'').length>120?'…':''}
|
||||||
|
<div class="actions" style="margin-top:6px" onclick="event.stopPropagation()">
|
||||||
|
<button class="btn-ok" onclick="confirm('${p.id}', event)">✅</button>
|
||||||
|
<button class="btn-no" onclick="reject('${p.id}', event)">❌</button>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
`).join('');
|
||||||
|
|
||||||
|
el.innerHTML = `
|
||||||
|
<div class="panel">
|
||||||
|
<div class="panel-title">Config</div>
|
||||||
|
<div class="kv-row"><div class="kv-key">workspace</div><div class="kv-val">${cfg.workspace}</div></div>
|
||||||
|
<div class="kv-row"><div class="kv-key">db</div><div class="kv-val">${db.db_path}</div></div>
|
||||||
|
<div class="kv-row"><div class="kv-key">db mtime</div><div class="kv-val">${new Date(db.mtime).toLocaleString()}</div></div>
|
||||||
|
</div>
|
||||||
|
<div class="panel">
|
||||||
|
<div class="panel-title">Storage</div>
|
||||||
|
<div class="kv-row"><div class="kv-key">SQL</div><div class="kv-val">${stor.sql.total_engrams} engrams (ok ${stor.sql.confirmed}, pending ${stor.sql.pending})</div></div>
|
||||||
|
<div class="kv-row"><div class="kv-key">Vector</div><div class="kv-val">chroma ${(stor.vector.chroma_size_bytes/1024/1024).toFixed(1)} MB, cache ${stor.vector.embedding_cache_files} files</div></div>
|
||||||
|
<div class="kv-row"><div class="kv-key">Obsidian</div><div class="kv-val">${stor.obsidian.configured ? 'configured' : 'not configured'}</div></div>
|
||||||
|
<div class="kv-row"><div class="kv-key">By source</div><div class="kv-val">${bySource || '-'}</div></div>
|
||||||
|
</div>
|
||||||
|
<div class="panel">
|
||||||
|
<div class="panel-title">Jobs</div>
|
||||||
|
${jobs.error ? `<div class="muted">Fehler: ${escapeHtml(jobs.error)}</div>` : (jobsHtml || '<div class="muted">Keine Daten</div>')}
|
||||||
|
</div>
|
||||||
|
<div class="panel">
|
||||||
|
<div class="panel-title">Insights</div>
|
||||||
|
${ins.error ? `<div class="muted">Fehler: ${escapeHtml(ins.error)}</div>` : ''}
|
||||||
|
<div class="kv-row"><div class="kv-key">pending</div><div class="kv-val">${ins.pending}</div></div>
|
||||||
|
<div class="kv-row"><div class="kv-key">top tags</div><div class="kv-val">${topTags || '-'}</div></div>
|
||||||
|
<div class="kv-row"><div class="kv-key">top hosts</div><div class="kv-val">${topHosts || '-'}</div></div>
|
||||||
|
</div>
|
||||||
|
<div class="panel">
|
||||||
|
<div class="panel-title">Pending Queue (latest)</div>
|
||||||
|
${pend.error ? `<div class="muted">Fehler: ${escapeHtml(pend.error)}</div>` : (pendHtml || '<div class="muted">Keine Pendings</div>')}
|
||||||
|
</div>
|
||||||
|
`;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function loadGraph() {
|
||||||
|
const sel = document.getElementById('graphLimit');
|
||||||
|
const fromSel = sel ? parseInt(sel.value || '0', 10) : NaN;
|
||||||
|
const fromStore = parseInt(localStorage.getItem('graphLimit') || '0', 10);
|
||||||
|
const q = (!Number.isNaN(fromSel)) ? fromSel : (Number.isNaN(fromStore) ? 0 : fromStore);
|
||||||
|
if (sel) sel.value = String(q);
|
||||||
|
if (sel) localStorage.setItem('graphLimit', String(q));
|
||||||
|
const hint = document.getElementById('graphHint');
|
||||||
|
if (hint) hint.textContent = 'Lade Graph…';
|
||||||
|
try {
|
||||||
|
const g = await api(`/api/graph?limit_nodes=${q}`);
|
||||||
|
renderGraph(g.nodes || [], g.edges || []);
|
||||||
|
} catch (e) {
|
||||||
|
if (hint) hint.textContent = `Graph-Fehler: ${e && e.message ? e.message : String(e)}`;
|
||||||
|
const canvas = _graphCanvas();
|
||||||
|
const ctx = _graphCtx();
|
||||||
|
if (canvas && ctx) ctx.clearRect(0,0,canvas.width,canvas.height);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function reloadGraph() { loadGraph(); }
|
||||||
|
|
||||||
|
// ─── Graph Renderer (Canvas) ────────────────────────────────────────────────
|
||||||
|
let graphState = {
|
||||||
|
nodes: [],
|
||||||
|
edges: [],
|
||||||
|
sim: [],
|
||||||
|
links: [],
|
||||||
|
nodeById: new Map(),
|
||||||
|
simById: new Map(),
|
||||||
|
degree: new Map(),
|
||||||
|
physicsOn: false,
|
||||||
|
draggingId: null,
|
||||||
|
selectedId: null,
|
||||||
|
panning: false,
|
||||||
|
lastX: 0,
|
||||||
|
lastY: 0,
|
||||||
|
panX: 0,
|
||||||
|
panY: 0,
|
||||||
|
zoom: 1,
|
||||||
|
raf: null,
|
||||||
|
search: '',
|
||||||
|
pointers: new Map(),
|
||||||
|
pinchStartDist: null,
|
||||||
|
pinchStartZoom: null,
|
||||||
|
pinchStartPan: null,
|
||||||
|
down: null, // {pointerId, cx, cy, t}
|
||||||
|
physicsStrength: parseInt(localStorage.getItem('physicsStrength') || '60', 10),
|
||||||
|
};
|
||||||
|
|
||||||
|
function _graphCanvas() { return document.getElementById('graphCanvas'); }
|
||||||
|
function _graphCtx() { return _graphCanvas().getContext('2d'); }
|
||||||
|
|
||||||
|
function _graphNodeRadius(n) {
|
||||||
|
const d = graphState.degree.get(n.id) || 0;
|
||||||
|
const base = n.kind === 'tag' ? 4 : (n.kind === 'host' ? 5 : 7);
|
||||||
|
const w = (n.weight || 0);
|
||||||
|
const bonus = Math.min(6, Math.sqrt(Math.max(0, w)) * 0.8);
|
||||||
|
return Math.max(3, Math.min(18, base + Math.sqrt(d) + bonus));
|
||||||
|
}
|
||||||
|
|
||||||
|
function _graphNodeFill(n) {
|
||||||
|
const d = graphState.degree.get(n.id) || 0;
|
||||||
|
const t = Math.max(0, Math.min(0.55, d / 18)); // higher degree -> brighter
|
||||||
|
const mix = (rgb) => rgb.map(c => Math.round(c + (255 - c) * t));
|
||||||
|
|
||||||
|
if (n.kind === 'tag') {
|
||||||
|
const [r,g,b] = mix([167, 139, 250]);
|
||||||
|
return `rgb(${r},${g},${b})`;
|
||||||
|
}
|
||||||
|
if (n.kind === 'host') {
|
||||||
|
const [r,g,b] = mix([245, 158, 11]);
|
||||||
|
return `rgb(${r},${g},${b})`;
|
||||||
|
}
|
||||||
|
|
||||||
|
const verdict = (n.verdict || '').toString();
|
||||||
|
let base = [96, 165, 250]; // pending/unknown = blue
|
||||||
|
if (verdict === 'confirmed_true') base = [34, 197, 94]; // green
|
||||||
|
else if (verdict === 'confirmed_false') base = [239, 68, 68]; // red
|
||||||
|
else if (verdict === 'probable_true') base = [52, 211, 153]; // teal
|
||||||
|
else if (verdict === 'probable_false') base = [251, 191, 36]; // amber
|
||||||
|
|
||||||
|
// Recency brightening (brand new pops)
|
||||||
|
const now = Date.now();
|
||||||
|
const created = Date.parse(n.created || '') || 0;
|
||||||
|
const ageMin = created ? (now - created) / 60000 : 999999;
|
||||||
|
const rec = Math.max(0, Math.min(0.35, (30 - ageMin) / 30 * 0.35));
|
||||||
|
const bump = (c) => Math.round(c + (255 - c) * rec);
|
||||||
|
const [r,g,b] = mix(base).map(bump);
|
||||||
|
return `rgb(${r},${g},${b})`;
|
||||||
|
}
|
||||||
|
|
||||||
|
function _graphMatches(n, term) {
|
||||||
|
const t = (term || '').trim().toLowerCase();
|
||||||
|
if (!t) return false;
|
||||||
|
const id = (n.id || '').toLowerCase();
|
||||||
|
const label = (n.label || '').toLowerCase();
|
||||||
|
return id.includes(t) || label.includes(t);
|
||||||
|
}
|
||||||
|
|
||||||
|
function _graphWorldFromScreen(cx, cy) {
|
||||||
|
return {
|
||||||
|
x: (cx - graphState.panX) / graphState.zoom,
|
||||||
|
y: (cy - graphState.panY) / graphState.zoom,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function _graphHitTest(wx, wy) {
|
||||||
|
for (let i = graphState.sim.length - 1; i >= 0; i--) {
|
||||||
|
const n = graphState.sim[i];
|
||||||
|
const r = _graphNodeRadius(n) + 2;
|
||||||
|
const dx = wx - n.x;
|
||||||
|
const dy = wy - n.y;
|
||||||
|
if ((dx*dx + dy*dy) <= r*r) return n;
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
function _graphInitInteractions() {
|
||||||
|
const canvas = _graphCanvas();
|
||||||
|
if (canvas._graphBound) return;
|
||||||
|
canvas._graphBound = true;
|
||||||
|
|
||||||
|
const toCanvasXY = (ev) => {
|
||||||
|
const rect = canvas.getBoundingClientRect();
|
||||||
|
return { cx: ev.clientX - rect.left, cy: ev.clientY - rect.top };
|
||||||
|
};
|
||||||
|
|
||||||
|
const pinchMidpoint = () => {
|
||||||
|
const pts = Array.from(graphState.pointers.values());
|
||||||
|
if (pts.length < 2) return null;
|
||||||
|
return { cx: (pts[0].cx + pts[1].cx) / 2, cy: (pts[0].cy + pts[1].cy) / 2 };
|
||||||
|
};
|
||||||
|
|
||||||
|
const pinchDistance = () => {
|
||||||
|
const pts = Array.from(graphState.pointers.values());
|
||||||
|
if (pts.length < 2) return null;
|
||||||
|
const dx = pts[0].cx - pts[1].cx;
|
||||||
|
const dy = pts[0].cy - pts[1].cy;
|
||||||
|
return Math.sqrt(dx*dx + dy*dy);
|
||||||
|
};
|
||||||
|
|
||||||
|
canvas.addEventListener('pointerdown', (ev) => {
|
||||||
|
canvas.setPointerCapture(ev.pointerId);
|
||||||
|
const pos = toCanvasXY(ev);
|
||||||
|
graphState.pointers.set(ev.pointerId, pos);
|
||||||
|
graphState.down = { pointerId: ev.pointerId, cx: pos.cx, cy: pos.cy, t: Date.now() };
|
||||||
|
|
||||||
|
// When 2 pointers: start pinch
|
||||||
|
if (graphState.pointers.size === 2) {
|
||||||
|
graphState.panning = false;
|
||||||
|
graphState.draggingId = null;
|
||||||
|
graphState.down = null;
|
||||||
|
graphState.pinchStartDist = pinchDistance();
|
||||||
|
graphState.pinchStartZoom = graphState.zoom;
|
||||||
|
graphState.pinchStartPan = { panX: graphState.panX, panY: graphState.panY };
|
||||||
|
_graphDraw();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Single pointer: pan or drag node
|
||||||
|
const { cx, cy } = pos;
|
||||||
|
const w = _graphWorldFromScreen(cx, cy);
|
||||||
|
const hit = _graphHitTest(w.x, w.y);
|
||||||
|
graphState.lastX = cx; graphState.lastY = cy;
|
||||||
|
if (hit) {
|
||||||
|
graphState.draggingId = hit.id;
|
||||||
|
graphState.selectedId = hit.id;
|
||||||
|
} else {
|
||||||
|
graphState.panning = true;
|
||||||
|
graphState.selectedId = null;
|
||||||
|
}
|
||||||
|
_graphDraw();
|
||||||
|
});
|
||||||
|
|
||||||
|
canvas.addEventListener('pointermove', (ev) => {
|
||||||
|
if (!graphState.pointers.has(ev.pointerId)) return;
|
||||||
|
const pos = toCanvasXY(ev);
|
||||||
|
graphState.pointers.set(ev.pointerId, pos);
|
||||||
|
|
||||||
|
// Pinch zoom (2 pointers)
|
||||||
|
if (graphState.pointers.size >= 2 && graphState.pinchStartDist) {
|
||||||
|
const dist = pinchDistance();
|
||||||
|
const mid = pinchMidpoint();
|
||||||
|
if (!dist || !mid) return;
|
||||||
|
|
||||||
|
const zoomOld = graphState.zoom;
|
||||||
|
const wx = (mid.cx - graphState.panX) / zoomOld;
|
||||||
|
const wy = (mid.cy - graphState.panY) / zoomOld;
|
||||||
|
|
||||||
|
const factor = dist / graphState.pinchStartDist;
|
||||||
|
const next = Math.max(0.35, Math.min(3.0, graphState.pinchStartZoom * factor));
|
||||||
|
graphState.zoom = next;
|
||||||
|
graphState.panX = mid.cx - wx * graphState.zoom;
|
||||||
|
graphState.panY = mid.cy - wy * graphState.zoom;
|
||||||
|
_graphDraw();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Single pointer
|
||||||
|
const { cx, cy } = pos;
|
||||||
|
const dx = cx - graphState.lastX;
|
||||||
|
const dy = cy - graphState.lastY;
|
||||||
|
graphState.lastX = cx; graphState.lastY = cy;
|
||||||
|
|
||||||
|
if (graphState.draggingId) {
|
||||||
|
const w = _graphWorldFromScreen(cx, cy);
|
||||||
|
const n = graphState.simById.get(graphState.draggingId);
|
||||||
|
if (n) { n.x = w.x; n.y = w.y; n.vx = 0; n.vy = 0; }
|
||||||
|
_graphDraw();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (graphState.panning) {
|
||||||
|
graphState.panX += dx;
|
||||||
|
graphState.panY += dy;
|
||||||
|
_graphDraw();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
const endPointer = (ev) => {
|
||||||
|
if (graphState.pointers.has(ev.pointerId)) graphState.pointers.delete(ev.pointerId);
|
||||||
|
try { canvas.releasePointerCapture(ev.pointerId); } catch {}
|
||||||
|
|
||||||
|
if (graphState.pointers.size < 2) {
|
||||||
|
graphState.pinchStartDist = null;
|
||||||
|
graphState.pinchStartZoom = null;
|
||||||
|
graphState.pinchStartPan = null;
|
||||||
|
}
|
||||||
|
|
||||||
|
const pos = toCanvasXY(ev);
|
||||||
|
const down = graphState.down && graphState.down.pointerId === ev.pointerId ? graphState.down : null;
|
||||||
|
graphState.down = null;
|
||||||
|
|
||||||
|
const moved = down ? Math.hypot(pos.cx - down.cx, pos.cy - down.cy) : 999;
|
||||||
|
const isTap = !!down && moved <= 8 && (Date.now() - down.t) <= 500;
|
||||||
|
|
||||||
|
const w = _graphWorldFromScreen(pos.cx, pos.cy);
|
||||||
|
const hit = _graphHitTest(w.x, w.y);
|
||||||
|
|
||||||
|
graphState.draggingId = null;
|
||||||
|
graphState.panning = false;
|
||||||
|
|
||||||
|
if (isTap && hit) {
|
||||||
|
graphState.selectedId = hit.id;
|
||||||
|
_graphDraw();
|
||||||
|
if (hit.kind === 'engram') {
|
||||||
|
showDetail(hit.id);
|
||||||
|
} else if (hit.kind === 'tag') {
|
||||||
|
const label = (hit.label || '').replace(/^tag:/, '');
|
||||||
|
const inp = document.getElementById('searchInput');
|
||||||
|
inp.value = label;
|
||||||
|
inp.dispatchEvent(new Event('input'));
|
||||||
|
}
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
_graphDraw();
|
||||||
|
};
|
||||||
|
canvas.addEventListener('pointerup', endPointer);
|
||||||
|
canvas.addEventListener('pointercancel', endPointer);
|
||||||
|
}
|
||||||
|
|
||||||
|
function renderGraph(nodes, edges) {
|
||||||
|
const canvas = _graphCanvas();
|
||||||
|
const hint = document.getElementById('graphHint');
|
||||||
|
const ctx = _graphCtx();
|
||||||
|
|
||||||
|
// sync physics slider
|
||||||
|
const slider = document.getElementById('physicsStrength');
|
||||||
|
const sliderVal = document.getElementById('physicsStrengthVal');
|
||||||
|
const s = Math.max(0, Math.min(100, parseInt(graphState.physicsStrength || 60, 10)));
|
||||||
|
graphState.physicsStrength = s;
|
||||||
|
if (slider) slider.value = String(s);
|
||||||
|
if (sliderVal) sliderVal.textContent = String(s);
|
||||||
|
|
||||||
|
const w = canvas.parentElement.clientWidth - 24;
|
||||||
|
canvas.width = Math.max(320, w);
|
||||||
|
canvas.height = Math.max(520, Math.min(900, (window.innerHeight || 900) - 260));
|
||||||
|
|
||||||
|
graphState.nodes = nodes || [];
|
||||||
|
graphState.edges = edges || [];
|
||||||
|
graphState.nodeById = new Map(graphState.nodes.map(n => [n.id, n]));
|
||||||
|
graphState.degree = new Map();
|
||||||
|
for (const e of graphState.edges) {
|
||||||
|
graphState.degree.set(e.from, (graphState.degree.get(e.from) || 0) + 1);
|
||||||
|
graphState.degree.set(e.to, (graphState.degree.get(e.to) || 0) + 1);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!graphState.nodes.length) {
|
||||||
|
hint.textContent = 'Graph: keine Nodes.';
|
||||||
|
ctx.clearRect(0,0,canvas.width,canvas.height);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
hint.textContent = `nodes=${graphState.nodes.length} edges=${graphState.edges.length}`;
|
||||||
|
|
||||||
|
// Build adjacency for deterministic layout (O(n+e))
|
||||||
|
const adj = new Map();
|
||||||
|
const addAdj = (a, b) => {
|
||||||
|
if (!adj.has(a)) adj.set(a, []);
|
||||||
|
adj.get(a).push(b);
|
||||||
|
};
|
||||||
|
for (const e of graphState.edges) {
|
||||||
|
addAdj(e.from, e.to);
|
||||||
|
addAdj(e.to, e.from);
|
||||||
|
}
|
||||||
|
|
||||||
|
const degree = graphState.degree;
|
||||||
|
const visited = new Set();
|
||||||
|
const comps = [];
|
||||||
|
for (const n of graphState.nodes) {
|
||||||
|
if (visited.has(n.id)) continue;
|
||||||
|
const q = [n.id];
|
||||||
|
visited.add(n.id);
|
||||||
|
const comp = [];
|
||||||
|
while (q.length) {
|
||||||
|
const cur = q.pop();
|
||||||
|
comp.push(cur);
|
||||||
|
const neigh = adj.get(cur) || [];
|
||||||
|
for (const nb of neigh) {
|
||||||
|
if (!visited.has(nb)) {
|
||||||
|
visited.add(nb);
|
||||||
|
q.push(nb);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
comps.push(comp);
|
||||||
|
}
|
||||||
|
comps.sort((a, b) => b.length - a.length);
|
||||||
|
|
||||||
|
// component centers in a loose grid/spiral
|
||||||
|
const centers = [];
|
||||||
|
const gap = 240;
|
||||||
|
const cols = Math.max(1, Math.floor(canvas.width / gap));
|
||||||
|
for (let i = 0; i < comps.length; i++) {
|
||||||
|
const cx = (i % cols) * gap - ((cols - 1) * gap) / 2;
|
||||||
|
const cy = Math.floor(i / cols) * gap - (Math.floor((comps.length - 1) / cols) * gap) / 2;
|
||||||
|
centers.push({ cx, cy });
|
||||||
|
}
|
||||||
|
|
||||||
|
const pos = new Map();
|
||||||
|
const R = 46;
|
||||||
|
for (let ci = 0; ci < comps.length; ci++) {
|
||||||
|
const comp = comps[ci];
|
||||||
|
const center = centers[ci] || { cx: 0, cy: 0 };
|
||||||
|
|
||||||
|
// hub: highest degree engram preferred
|
||||||
|
let hub = comp[0];
|
||||||
|
let hubScore = -1;
|
||||||
|
for (const id of comp) {
|
||||||
|
const n = graphState.nodeById.get(id) || {};
|
||||||
|
const score = (degree.get(id) || 0) + (n.kind === 'engram' ? 3 : 0);
|
||||||
|
if (score > hubScore) { hubScore = score; hub = id; }
|
||||||
|
}
|
||||||
|
|
||||||
|
// BFS layers from hub
|
||||||
|
const layer = new Map();
|
||||||
|
layer.set(hub, 0);
|
||||||
|
const qq = [hub];
|
||||||
|
while (qq.length) {
|
||||||
|
const cur = qq.shift();
|
||||||
|
const l = layer.get(cur) || 0;
|
||||||
|
const neigh = adj.get(cur) || [];
|
||||||
|
for (const nb of neigh) {
|
||||||
|
if (!layer.has(nb)) {
|
||||||
|
layer.set(nb, l + 1);
|
||||||
|
qq.push(nb);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const buckets = new Map();
|
||||||
|
for (const id of comp) {
|
||||||
|
const l = layer.get(id);
|
||||||
|
const ll = (typeof l === 'number') ? l : 99;
|
||||||
|
if (!buckets.has(ll)) buckets.set(ll, []);
|
||||||
|
buckets.get(ll).push(id);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Place hubs first, then rings
|
||||||
|
pos.set(hub, { x: center.cx, y: center.cy });
|
||||||
|
const maxLayer = Math.max(...Array.from(buckets.keys()));
|
||||||
|
for (let l = 1; l <= maxLayer; l++) {
|
||||||
|
const ids = buckets.get(l) || [];
|
||||||
|
if (!ids.length) continue;
|
||||||
|
// Stable order: engrams first, then by degree desc
|
||||||
|
ids.sort((a, b) => {
|
||||||
|
const na = graphState.nodeById.get(a) || {};
|
||||||
|
const nb = graphState.nodeById.get(b) || {};
|
||||||
|
const ka = na.kind === 'engram' ? 0 : (na.kind === 'tag' ? 1 : 2);
|
||||||
|
const kb = nb.kind === 'engram' ? 0 : (nb.kind === 'tag' ? 1 : 2);
|
||||||
|
if (ka !== kb) return ka - kb;
|
||||||
|
return (degree.get(b) || 0) - (degree.get(a) || 0);
|
||||||
|
});
|
||||||
|
const rad = l * R;
|
||||||
|
for (let i = 0; i < ids.length; i++) {
|
||||||
|
const ang = (i / ids.length) * Math.PI * 2;
|
||||||
|
pos.set(ids[i], {
|
||||||
|
x: center.cx + Math.cos(ang) * rad,
|
||||||
|
y: center.cy + Math.sin(ang) * rad,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
graphState.sim = graphState.nodes.map(n => {
|
||||||
|
const p = pos.get(n.id) || { x: (Math.random() - 0.5) * canvas.width, y: (Math.random() - 0.5) * canvas.height };
|
||||||
|
return {
|
||||||
|
id: n.id,
|
||||||
|
kind: n.kind,
|
||||||
|
label: n.label || n.id,
|
||||||
|
weight: n.weight,
|
||||||
|
verdict: n.verdict,
|
||||||
|
confidence: n.confidence,
|
||||||
|
created: n.created,
|
||||||
|
modified: n.modified,
|
||||||
|
last_accessed: n.last_accessed,
|
||||||
|
predict_locked: n.predict_locked,
|
||||||
|
x: p.x,
|
||||||
|
y: p.y,
|
||||||
|
vx: 0, vy: 0,
|
||||||
|
};
|
||||||
|
});
|
||||||
|
graphState.simById = new Map(graphState.sim.map(n => [n.id, n]));
|
||||||
|
graphState.links = graphState.edges
|
||||||
|
.map(e => ({a: graphState.simById.get(e.from), b: graphState.simById.get(e.to), kind: e.kind, weight: e.weight || 1.0}))
|
||||||
|
.filter(l => l.a && l.b);
|
||||||
|
|
||||||
|
graphState.panX = canvas.width / 2;
|
||||||
|
graphState.panY = canvas.height / 2;
|
||||||
|
graphState.zoom = 1;
|
||||||
|
graphState.search = state.search || '';
|
||||||
|
graphState.selectedId = null;
|
||||||
|
|
||||||
|
_graphInitInteractions();
|
||||||
|
// Physics is expensive; pre-relax a little but keep it optional for big graphs.
|
||||||
|
const relaxIters = graphState.sim.length <= 700 ? 70 : 12;
|
||||||
|
for (let i = 0; i < relaxIters; i++) _graphStepPhysics(0.85);
|
||||||
|
_graphDraw();
|
||||||
|
}
|
||||||
|
|
||||||
|
function _graphStepPhysics(alpha = 1.0) {
|
||||||
|
const canvas = _graphCanvas();
|
||||||
|
const repulsion = (graphState.sim.length > 700) ? 120 : 180;
|
||||||
|
const damping = 0.86;
|
||||||
|
const target = 80;
|
||||||
|
const springK = 0.018;
|
||||||
|
|
||||||
|
const sim = graphState.sim;
|
||||||
|
if (sim.length > 700) {
|
||||||
|
// Fast approximate repulsion via spatial hashing (checks only local neighbor cells).
|
||||||
|
const cell = 120;
|
||||||
|
const grid = new Map();
|
||||||
|
const key = (cx, cy) => `${cx},${cy}`;
|
||||||
|
for (let i = 0; i < sim.length; i++) {
|
||||||
|
const n = sim[i];
|
||||||
|
const cx = Math.floor(n.x / cell);
|
||||||
|
const cy = Math.floor(n.y / cell);
|
||||||
|
const k = key(cx, cy);
|
||||||
|
if (!grid.has(k)) grid.set(k, []);
|
||||||
|
grid.get(k).push(i);
|
||||||
|
}
|
||||||
|
for (let i = 0; i < sim.length; i++) {
|
||||||
|
const a = sim[i];
|
||||||
|
const cx = Math.floor(a.x / cell);
|
||||||
|
const cy = Math.floor(a.y / cell);
|
||||||
|
for (let ox = -1; ox <= 1; ox++) {
|
||||||
|
for (let oy = -1; oy <= 1; oy++) {
|
||||||
|
const idxs = grid.get(key(cx + ox, cy + oy));
|
||||||
|
if (!idxs) continue;
|
||||||
|
for (const j of idxs) {
|
||||||
|
if (j === i) continue;
|
||||||
|
const b = sim[j];
|
||||||
|
const dx = a.x - b.x, dy = a.y - b.y;
|
||||||
|
const d2 = dx*dx + dy*dy + 0.12;
|
||||||
|
const f = (repulsion / d2) * alpha * 0.55;
|
||||||
|
a.vx += dx*f; a.vy += dy*f;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
for (let i = 0; i < sim.length; i++) {
|
||||||
|
for (let j = i + 1; j < sim.length; j++) {
|
||||||
|
const a = sim[i], b = sim[j];
|
||||||
|
const dx = a.x - b.x, dy = a.y - b.y;
|
||||||
|
const d2 = dx*dx + dy*dy + 0.02;
|
||||||
|
const f = (repulsion / d2) * alpha;
|
||||||
|
a.vx += dx*f; a.vy += dy*f;
|
||||||
|
b.vx -= dx*f; b.vy -= dy*f;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for (const l of graphState.links) {
|
||||||
|
const a = l.a, b = l.b;
|
||||||
|
const dx = b.x - a.x, dy = b.y - a.y;
|
||||||
|
const dist = Math.sqrt(dx*dx + dy*dy) || 1;
|
||||||
|
const f = (dist - target) * springK * alpha;
|
||||||
|
const fx = (dx/dist) * f, fy = (dy/dist) * f;
|
||||||
|
a.vx += fx; a.vy += fy;
|
||||||
|
b.vx -= fx; b.vy -= fy;
|
||||||
|
}
|
||||||
|
for (const n of sim) {
|
||||||
|
n.vx *= damping; n.vy *= damping;
|
||||||
|
n.x += n.vx; n.y += n.vy;
|
||||||
|
const pad = 10;
|
||||||
|
const bx = canvas.width / graphState.zoom;
|
||||||
|
const by = canvas.height / graphState.zoom;
|
||||||
|
n.x = Math.max(-bx/2 + pad, Math.min(bx/2 - pad, n.x));
|
||||||
|
n.y = Math.max(-by/2 + pad, Math.min(by/2 - pad, n.y));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function _graphEdgeColor(kind) {
|
||||||
|
const k = (kind || '').toString().toLowerCase();
|
||||||
|
if (k.includes('tag')) return '#7c3aed';
|
||||||
|
if (k.includes('host')) return '#f59e0b';
|
||||||
|
if (k.includes('ref')) return '#10b981';
|
||||||
|
return '#3a3a55';
|
||||||
|
}
|
||||||
|
|
||||||
|
function _graphDraw() {
|
||||||
|
const canvas = _graphCanvas();
|
||||||
|
const ctx = _graphCtx();
|
||||||
|
const hint = document.getElementById('graphHint');
|
||||||
|
|
||||||
|
ctx.clearRect(0,0,canvas.width,canvas.height);
|
||||||
|
ctx.save();
|
||||||
|
ctx.translate(graphState.panX, graphState.panY);
|
||||||
|
ctx.scale(graphState.zoom, graphState.zoom);
|
||||||
|
|
||||||
|
for (const l of graphState.links) {
|
||||||
|
const term = (graphState.search || '').trim();
|
||||||
|
const isMatchEdge = term && (_graphMatches(l.a, term) || _graphMatches(l.b, term));
|
||||||
|
const w = Math.max(0.2, Math.min(3.0, (l.weight || 1.0)));
|
||||||
|
ctx.lineWidth = (0.6 + w) / graphState.zoom;
|
||||||
|
ctx.globalAlpha = isMatchEdge ? 0.85 : (0.25 + Math.min(0.35, w * 0.18));
|
||||||
|
ctx.strokeStyle = isMatchEdge ? '#f7d154' : _graphEdgeColor(l.kind);
|
||||||
|
ctx.beginPath();
|
||||||
|
ctx.moveTo(l.a.x, l.a.y);
|
||||||
|
ctx.lineTo(l.b.x, l.b.y);
|
||||||
|
ctx.stroke();
|
||||||
|
}
|
||||||
|
ctx.globalAlpha = 1.0;
|
||||||
|
|
||||||
|
const term = (graphState.search || '').trim();
|
||||||
|
let matches = 0;
|
||||||
|
for (const n of graphState.sim) {
|
||||||
|
const r = _graphNodeRadius(n);
|
||||||
|
const isMatch = _graphMatches(n, term);
|
||||||
|
if (isMatch) matches++;
|
||||||
|
|
||||||
|
if (isMatch) {
|
||||||
|
ctx.beginPath();
|
||||||
|
ctx.strokeStyle = '#f7d154';
|
||||||
|
ctx.lineWidth = 3 / graphState.zoom;
|
||||||
|
ctx.arc(n.x, n.y, r + 3, 0, Math.PI*2);
|
||||||
|
ctx.stroke();
|
||||||
|
}
|
||||||
|
|
||||||
|
if (graphState.selectedId === n.id) {
|
||||||
|
ctx.beginPath();
|
||||||
|
ctx.strokeStyle = '#ffffff';
|
||||||
|
ctx.lineWidth = 2 / graphState.zoom;
|
||||||
|
ctx.arc(n.x, n.y, r + 2, 0, Math.PI*2);
|
||||||
|
ctx.stroke();
|
||||||
|
}
|
||||||
|
|
||||||
|
ctx.beginPath();
|
||||||
|
ctx.fillStyle = _graphNodeFill(n);
|
||||||
|
ctx.arc(n.x, n.y, r, 0, Math.PI*2);
|
||||||
|
ctx.fill();
|
||||||
|
|
||||||
|
// status/recency/lock border
|
||||||
|
let stroke = null;
|
||||||
|
const v = (n.verdict || '').toString();
|
||||||
|
if (v === 'confirmed_true') stroke = '#bbf7d0';
|
||||||
|
else if (v === 'confirmed_false') stroke = '#fecaca';
|
||||||
|
else if (v) stroke = '#c7d2fe';
|
||||||
|
|
||||||
|
const now = Date.now();
|
||||||
|
const created = Date.parse(n.created || '') || 0;
|
||||||
|
const modified = Date.parse(n.modified || '') || 0;
|
||||||
|
const isNew = created && (now - created) < (30 * 60 * 1000);
|
||||||
|
const isHot = modified && (now - modified) < (10 * 60 * 1000);
|
||||||
|
if (isNew) stroke = '#f7d154';
|
||||||
|
if (isHot) stroke = '#ffffff';
|
||||||
|
if (n.predict_locked) stroke = '#a78bfa';
|
||||||
|
|
||||||
|
if (stroke) {
|
||||||
|
ctx.beginPath();
|
||||||
|
ctx.strokeStyle = stroke;
|
||||||
|
ctx.lineWidth = (n.predict_locked ? 3 : 1.5) / graphState.zoom;
|
||||||
|
ctx.arc(n.x, n.y, r + 0.8, 0, Math.PI*2);
|
||||||
|
ctx.stroke();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
ctx.restore();
|
||||||
|
hint.textContent = `nodes=${graphState.nodes.length} edges=${graphState.edges.length}` + (term ? ` | match=${matches}` : '');
|
||||||
|
}
|
||||||
|
|
||||||
|
function _graphLoop() {
|
||||||
|
if (!graphState.physicsOn) return;
|
||||||
|
const speed = 0.25 + (Math.max(0, Math.min(100, graphState.physicsStrength || 60)) / 100) * 0.95;
|
||||||
|
_graphStepPhysics(speed);
|
||||||
|
_graphDraw();
|
||||||
|
graphState.raf = requestAnimationFrame(_graphLoop);
|
||||||
|
}
|
||||||
|
|
||||||
|
function setPhysicsStrength(v) {
|
||||||
|
const n = Math.max(0, Math.min(100, parseInt(v || '0', 10)));
|
||||||
|
graphState.physicsStrength = n;
|
||||||
|
localStorage.setItem('physicsStrength', String(n));
|
||||||
|
const el = document.getElementById('physicsStrengthVal');
|
||||||
|
if (el) el.textContent = String(n);
|
||||||
|
}
|
||||||
|
|
||||||
|
function toggleGraphPhysics() {
|
||||||
|
graphState.physicsOn = !graphState.physicsOn;
|
||||||
|
const b = document.getElementById('btnGraphPhysics');
|
||||||
|
const fast = (graphState.sim || []).length > 700;
|
||||||
|
b.textContent = `Physics: ${graphState.physicsOn ? ('on' + (fast ? ' (fast)' : '')) : 'off'}`;
|
||||||
|
b.classList.toggle('primary', graphState.physicsOn);
|
||||||
|
if (graphState.physicsOn) {
|
||||||
|
if (graphState.raf) cancelAnimationFrame(graphState.raf);
|
||||||
|
graphState.raf = requestAnimationFrame(_graphLoop);
|
||||||
|
} else {
|
||||||
|
if (graphState.raf) cancelAnimationFrame(graphState.raf);
|
||||||
|
graphState.raf = null;
|
||||||
|
_graphDraw();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function resetGraphView() {
|
||||||
|
const canvas = _graphCanvas();
|
||||||
|
graphState.panX = canvas.width / 2;
|
||||||
|
graphState.panY = canvas.height / 2;
|
||||||
|
graphState.zoom = 1;
|
||||||
|
_graphDraw();
|
||||||
|
}
|
||||||
|
|
||||||
|
function fitGraphView() {
|
||||||
|
const canvas = _graphCanvas();
|
||||||
|
if (!graphState.sim.length) return;
|
||||||
|
let minX = Infinity, minY = Infinity, maxX = -Infinity, maxY = -Infinity;
|
||||||
|
for (const n of graphState.sim) {
|
||||||
|
minX = Math.min(minX, n.x); minY = Math.min(minY, n.y);
|
||||||
|
maxX = Math.max(maxX, n.x); maxY = Math.max(maxY, n.y);
|
||||||
|
}
|
||||||
|
const w = Math.max(1, maxX - minX);
|
||||||
|
const h = Math.max(1, maxY - minY);
|
||||||
|
const zx = (canvas.width - 40) / w;
|
||||||
|
const zy = (canvas.height - 40) / h;
|
||||||
|
graphState.zoom = Math.max(0.35, Math.min(2.5, Math.min(zx, zy)));
|
||||||
|
graphState.panX = canvas.width / 2;
|
||||||
|
graphState.panY = canvas.height / 2;
|
||||||
|
_graphDraw();
|
||||||
|
}
|
||||||
|
|
||||||
|
function graphApplySearch(term) {
|
||||||
|
graphState.search = term || '';
|
||||||
|
_graphDraw();
|
||||||
|
}
|
||||||
|
|
||||||
|
// Real-time updates via SSE
|
||||||
|
function startEvents() {
|
||||||
|
try {
|
||||||
|
const es = new EventSource('/api/events');
|
||||||
|
es.onmessage = (msg) => {
|
||||||
|
try {
|
||||||
|
state.lastEvent = JSON.parse(msg.data);
|
||||||
|
updateStatsFromEvent(state.lastEvent);
|
||||||
|
if (state.view === 'status') {
|
||||||
|
// refresh status panels without heavy re-render: just rerun loadStatus occasionally
|
||||||
|
loadStatus();
|
||||||
|
}
|
||||||
|
if (state.view === 'graph') {
|
||||||
|
// fetch graph less often (every ~15s)
|
||||||
|
const t = Date.now();
|
||||||
|
if (!state._lastGraphFetch || (t - state._lastGraphFetch) > 15000) {
|
||||||
|
state._lastGraphFetch = t;
|
||||||
|
loadGraph();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} catch (e) {}
|
||||||
|
};
|
||||||
|
es.onerror = () => {
|
||||||
|
// keep UI usable even if SSE drops
|
||||||
|
};
|
||||||
|
} catch (e) {}
|
||||||
|
}
|
||||||
|
|
||||||
|
startEvents();
|
||||||
|
|
||||||
function renderCards() {
|
function renderCards() {
|
||||||
const el = document.getElementById('cards');
|
const el = document.getElementById('cards');
|
||||||
el.innerHTML = state.items.map(item => `
|
el.innerHTML = state.items.map(item => `
|
||||||
<div class="card ${item.confirmed ? 'confirmed' : ''} ${item.rejections > 0 ? 'rejected' : ''}" data-id="${item.id}">
|
<div class="card ${item.confirmed ? 'confirmed' : ''} ${item.rejections > 0 ? 'rejected' : ''}" data-id="${item.id}">
|
||||||
<div class="card-header">
|
<div class="card-header">
|
||||||
<span class="conf-badge" style="background:hsl(${item.confidence*120},70%,40%)">${Math.round(item.confidence*100)}%</span>
|
<span class="conf-badge" style="background:hsl(${item.confidence*120},70%,40%)">${Math.round(item.confidence*100)}%</span>
|
||||||
|
${renderVerdictPill(item)}
|
||||||
<span class="tags">${item.tags.map(t => '<span class="tag">'+t+'</span>').join('')}</span>
|
<span class="tags">${item.tags.map(t => '<span class="tag">'+t+'</span>').join('')}</span>
|
||||||
<span class="date">${fmtDate(item.created)}</span>
|
<span class="date">${fmtDate(item.created)}</span>
|
||||||
</div>
|
</div>
|
||||||
@@ -123,6 +976,19 @@ function renderCards() {
|
|||||||
`).join('');
|
`).join('');
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function renderVerdictPill(item) {
|
||||||
|
const v = (item.verdict || '').toString();
|
||||||
|
if (!v) return '';
|
||||||
|
let cls = 'v-unknown';
|
||||||
|
let label = v;
|
||||||
|
if (v === 'confirmed_true') { cls = 'v-true'; label = 'TRUE'; }
|
||||||
|
else if (v === 'confirmed_false') { cls = 'v-false'; label = 'FALSE'; }
|
||||||
|
else if (v === 'probable_true') { cls = 'v-prob-true'; label = 'LIKELY'; }
|
||||||
|
else if (v === 'probable_false') { cls = 'v-prob-false'; label = 'UNLIKELY'; }
|
||||||
|
else if (v === 'unknown') { cls = 'v-unknown'; label = 'UNKNOWN'; }
|
||||||
|
return `<span class="verdict-pill ${cls}">${label}</span>`;
|
||||||
|
}
|
||||||
|
|
||||||
function fmtDate(iso) {
|
function fmtDate(iso) {
|
||||||
const d = new Date(iso);
|
const d = new Date(iso);
|
||||||
return `${d.getDate().toString().padStart(2,'0')}.${(d.getMonth()+1).toString().padStart(2,'0')} ${d.getHours().toString().padStart(2,'0')}:${d.getMinutes().toString().padStart(2,'0')}`;
|
return `${d.getDate().toString().padStart(2,'0')}.${(d.getMonth()+1).toString().padStart(2,'0')} ${d.getHours().toString().padStart(2,'0')}:${d.getMinutes().toString().padStart(2,'0')}`;
|
||||||
@@ -144,6 +1010,8 @@ async function confirm(id, ev) {
|
|||||||
body: new URLSearchParams({reason})
|
body: new URLSearchParams({reason})
|
||||||
});
|
});
|
||||||
await loadCards(); await loadStats();
|
await loadCards(); await loadStats();
|
||||||
|
if (state.view === 'graph') loadGraph();
|
||||||
|
if (state.view === 'status') loadStatus();
|
||||||
}
|
}
|
||||||
|
|
||||||
async function reject(id, ev) {
|
async function reject(id, ev) {
|
||||||
@@ -155,6 +1023,8 @@ async function reject(id, ev) {
|
|||||||
body: new URLSearchParams({reason})
|
body: new URLSearchParams({reason})
|
||||||
});
|
});
|
||||||
await loadCards(); await loadStats();
|
await loadCards(); await loadStats();
|
||||||
|
if (state.view === 'graph') loadGraph();
|
||||||
|
if (state.view === 'status') loadStatus();
|
||||||
}
|
}
|
||||||
|
|
||||||
async function refresh(id, ev) {
|
async function refresh(id, ev) {
|
||||||
@@ -175,6 +1045,8 @@ async function createEngram() {
|
|||||||
document.getElementById('newContent').value = '';
|
document.getElementById('newContent').value = '';
|
||||||
document.getElementById('newTags').value = '';
|
document.getElementById('newTags').value = '';
|
||||||
await loadCards(); await loadStats();
|
await loadCards(); await loadStats();
|
||||||
|
if (state.view === 'graph') loadGraph();
|
||||||
|
if (state.view === 'status') loadStatus();
|
||||||
}
|
}
|
||||||
|
|
||||||
async function showDetail(id) {
|
async function showDetail(id) {
|
||||||
@@ -220,6 +1092,7 @@ document.getElementById('searchInput').addEventListener('input', (e) => {
|
|||||||
state.search = e.target.value;
|
state.search = e.target.value;
|
||||||
state.offset = 0;
|
state.offset = 0;
|
||||||
loadCards();
|
loadCards();
|
||||||
|
if (state.view === 'graph') graphApplySearch(state.search);
|
||||||
});
|
});
|
||||||
|
|
||||||
document.getElementById('filterSelect').addEventListener('change', (e) => {
|
document.getElementById('filterSelect').addEventListener('change', (e) => {
|
||||||
|
|||||||
Reference in New Issue
Block a user