12 Commits

Author SHA1 Message Date
6232f25cc9 fix(fastapi): remove duplicate confirm/reject routes
- api_confirm and api_reject were defined twice on same paths
- FastAPI only registers first definition, causing silent 404s
- Kept api_confirm_engram and api_reject_engram (use _update_correctness)
- Removed duplicate direct DB implementations
- Fixes dashboard confirm/reject buttons not working
2026-05-27 18:36:03 +02:00
6b0cf5889f fix(store): escape FTS5 special characters in search_text()
- FTS5 crashes on dots (IP addresses) and hyphens (dates)
- Add regex to strip non-alphanumeric chars before FTS5 MATCH
- Fixes: fts5 syntax error near '.' and no such column: 05

Files changed: src/store.py
2026-05-27 17:54:51 +02:00
021fd0e328 feat(dashboard): pending queue + confirm/reject endpoints 2026-05-27 01:17:32 +02:00
d52e3a7f74 feat(ingest): transcript direct to DB 2026-05-27 01:14:42 +02:00
1635ee8b03 fix(verify): scan all engrams 2026-05-27 01:11:59 +02:00
f8ac0af869 chore(timers): ingest memory every 5 min 2026-05-27 01:11:17 +02:00
9dd5e49e2a chore(timers): process pending every 5 min 2026-05-27 01:08:08 +02:00
b158b19208 feat(ingest): tail session transcript into memory 2026-05-27 01:04:47 +02:00
095e6a33f8 feat(dashboard): realtime status + graph render 2026-05-27 00:22:14 +02:00
e5061b317f feat(dashboard): add status+graph views 2026-05-27 00:11:44 +02:00
ec8870ea40 feat(verify): add pending external verifier 2026-05-27 00:05:51 +02:00
8f47151a48 docs(systemd): fix paths + add dashboard smoke + vendor units 2026-05-26 23:47:07 +02:00
37 changed files with 1657 additions and 64 deletions

View File

@@ -2,19 +2,27 @@
This is the operational quick-reference for the shipped systemd timers/services and the optional FastAPI dashboard backend. This is the operational quick-reference for the shipped systemd timers/services and the optional FastAPI dashboard backend.
Repository root (on host): `/root/.openclaw/workspace` Repository root (on host): `/root/.openclaw/workspace/second-brain`
## Systemd units (cron jobs) ## Systemd units (cron jobs)
Unit files are shipped in `systemd/` (repo root). Install them into `/etc/systemd/system/` (symlink or copy), then reload: Unit files are shipped in `systemd/` (this repo). Install them into `/etc/systemd/system/` (symlink or copy), then reload:
```bash ```bash
sudo ln -sf /root/.openclaw/workspace/systemd/openclaw-secondbrain-*.service /etc/systemd/system/ sudo ln -sf /root/.openclaw/workspace/second-brain/systemd/openclaw-secondbrain-*.service /etc/systemd/system/
sudo ln -sf /root/.openclaw/workspace/systemd/openclaw-secondbrain-*.timer /etc/systemd/system/ sudo ln -sf /root/.openclaw/workspace/second-brain/systemd/openclaw-secondbrain-*.timer /etc/systemd/system/
sudo ln -sf /root/.openclaw/workspace/systemd/openclaw-memory-archive.* /etc/systemd/system/ sudo ln -sf /root/.openclaw/workspace/second-brain/systemd/openclaw-memory-archive.* /etc/systemd/system/
sudo systemctl daemon-reload sudo systemctl daemon-reload
``` ```
Optional (verification hardening):
```bash
sudo ln -sf /root/.openclaw/workspace/second-brain/systemd/openclaw-secondbrain-verify-pending.* /etc/systemd/system/
sudo systemctl daemon-reload
sudo systemctl enable --now openclaw-secondbrain-verify-pending.timer
```
Enable timers: Enable timers:
```bash ```bash
@@ -61,7 +69,7 @@ python3 -m pip install -r second-brain/requirements-dashboard.txt
SECOND_BRAIN_WORKSPACE="/root/.openclaw/workspace/second-brain" python3 second-brain/fastapi_app.py SECOND_BRAIN_WORKSPACE="/root/.openclaw/workspace/second-brain" python3 second-brain/fastapi_app.py
``` ```
Default port is `8501` (same as Streamlit default). Do not run both on the same port. Default port is `8501` (same as Streamlit default). You can override via `SECOND_BRAIN_PORT` (or `PORT`) when starting manually.
Endpoint smoke tests: Endpoint smoke tests:

View File

@@ -0,0 +1,160 @@
#!/usr/bin/env python3
"""
Ingest OpenClaw session transcript JSONL directly into the Second-Brain DB.
State is tracked with byte offsets per transcript file.
Sources are configured via workspace/memory/session_sources.json.
"""
from __future__ import annotations
import json
import os
from dataclasses import dataclass
from datetime import datetime, timezone
from pathlib import Path
from typing import Any, Dict, Iterable, List
from uuid import NAMESPACE_URL, uuid5
WORKSPACE = Path("/root/.openclaw/workspace")
MEMORY_DIR = WORKSPACE / "memory"
SOURCES_PATH = MEMORY_DIR / "session_sources.json"
STATE_PATH = MEMORY_DIR / "session_db_ingest_state.json"
BRAIN_DIR = WORKSPACE / "second-brain"
DB_PATH = Path(os.environ.get("BRAIN_DB", str(BRAIN_DIR / "data" / "brain.sqlite")))
import sys
sys.path.insert(0, str(BRAIN_DIR))
from src.engram import Engram, Grounding # type: ignore
from src.store import EngramStore # type: ignore
def _now() -> str:
return datetime.now(timezone.utc).isoformat()
def _load_json(path: Path, default: Any) -> Any:
try:
if not path.exists():
return default
return json.loads(path.read_text(encoding="utf-8"))
except Exception:
return default
def _save_json(path: Path, payload: Any) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(json.dumps(payload, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
def _extract_text(content: Any) -> str:
if isinstance(content, str):
return content.strip()
if isinstance(content, list):
parts: List[str] = []
for c in content:
if isinstance(c, dict) and c.get("type") == "text" and isinstance(c.get("text"), str):
parts.append(c["text"])
elif isinstance(c, str):
parts.append(c)
return "\n".join(p.strip() for p in parts if p and p.strip()).strip()
if isinstance(content, dict) and isinstance(content.get("text"), str):
return content["text"].strip()
return str(content).strip()
@dataclass
class Source:
label: str
transcript_path: Path
def _load_sources() -> List[Source]:
payload = _load_json(SOURCES_PATH, {"sources": []})
sources: List[Source] = []
for item in payload.get("sources", []) if isinstance(payload, dict) else []:
if not isinstance(item, dict):
continue
label = str(item.get("label") or "session")
p = item.get("path")
if not isinstance(p, str) or not p:
continue
sources.append(Source(label=label, transcript_path=Path(p)))
return sources
def _iter_new_lines(path: Path, *, start_offset: int) -> Iterable[tuple[int, str]]:
with open(path, "rb") as f:
f.seek(max(0, int(start_offset)))
while True:
raw = f.readline()
if not raw:
break
line = raw.decode("utf-8", errors="ignore").strip()
if not line:
continue
yield (f.tell(), line)
def run() -> Dict[str, Any]:
if not DB_PATH.exists():
return {"success": False, "time": _now(), "error": f"db missing: {DB_PATH}"}
sources = _load_sources()
state = _load_json(STATE_PATH, {"offsets": {}})
offsets: Dict[str, int] = state.get("offsets", {}) if isinstance(state, dict) else {}
store = EngramStore(str(DB_PATH))
out = {"success": True, "time": _now(), "sources": len(sources), "messages_saved": 0, "messages_skipped": 0, "errors": []}
for src in sources:
try:
if not src.transcript_path.exists():
continue
key = str(src.transcript_path)
start = int(offsets.get(key, 0))
for new_off, line in _iter_new_lines(src.transcript_path, start_offset=start):
offsets[key] = new_off
obj = json.loads(line)
if not isinstance(obj, dict) or obj.get("type") != "message":
continue
msg = obj.get("message") if isinstance(obj.get("message"), dict) else {}
role = str(msg.get("role") or "unknown")
content = _extract_text(msg.get("content"))
if len(content.strip()) < 5:
continue
mid = str(obj.get("id") or msg.get("id") or msg.get("messageId") or msg.get("message_id") or "")
if not mid:
mid = str(uuid5(NAMESPACE_URL, f"openclaw-transcript:{src.label}:{role}:{content[:200]}"))
eid = str(uuid5(NAMESPACE_URL, f"openclaw-transcript:{src.label}:{mid}"))
if store.get(eid):
out["messages_skipped"] += 1
continue
eg = Engram.create(
content=f"[transcript:{src.label}] [{role}] [{mid}]\n\n{content}"[:4000],
source="session",
tags=["session", "openclaw", "transcript", f"role:{role}"],
session_id=src.label,
confidence=0.55,
grounding=Grounding.ASSUMPTION,
)
eg.id = uuid5(NAMESPACE_URL, eid)
eg.metadata["source"] = "session"
eg.metadata["session_id"] = src.label
eg.metadata["role"] = role
eg.metadata["message_id"] = mid
store.save(eg)
out["messages_saved"] += 1
except Exception as e:
out["success"] = False
out["errors"].append(f"{src.transcript_path}: {e}")
_save_json(STATE_PATH, {"offsets": offsets, "updated_at": out["time"]})
return out
if __name__ == "__main__":
print(json.dumps(run(), ensure_ascii=False, indent=2))

View File

@@ -0,0 +1,213 @@
#!/usr/bin/env python3
"""
Tail OpenClaw session transcript JSONL files and append new messages into
workspace/memory/YYYY-MM-DD.md so the existing ingest_memory pipeline can pick
them up.
Why: when chat_autosave hooks are missed/aborted, the "memory/*.md -> DB" ingest
doesn't see the latest conversation. This bridges transcript -> memory.
Safety: read-only access to transcript files; state stored in workspace/memory/.
"""
from __future__ import annotations
import json
import os
from dataclasses import dataclass
from datetime import datetime, timezone
from pathlib import Path
from typing import Any, Dict, Iterable, List, Optional
try:
from zoneinfo import ZoneInfo # py3.9+
except Exception: # pragma: no cover
ZoneInfo = None # type: ignore
WORKSPACE = Path("/root/.openclaw/workspace")
MEMORY_DIR = WORKSPACE / "memory"
SOURCES_PATH = MEMORY_DIR / "session_sources.json"
STATE_PATH = MEMORY_DIR / "session_ingest_state.json"
def _local_tz():
tz = os.environ.get("TZ") or "Europe/Berlin"
if ZoneInfo:
try:
return ZoneInfo(tz)
except Exception:
return timezone.utc
return timezone.utc
def _load_json(path: Path, default: Any) -> Any:
try:
if not path.exists():
return default
return json.loads(path.read_text(encoding="utf-8"))
except Exception:
return default
def _save_json(path: Path, payload: Any) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(json.dumps(payload, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
def _iso_to_dt(ts: str) -> Optional[datetime]:
try:
if ts.endswith("Z"):
ts = ts[:-1] + "+00:00"
return datetime.fromisoformat(ts)
except Exception:
return None
def _ms_to_dt(ms: int) -> datetime:
return datetime.fromtimestamp(ms / 1000, tz=timezone.utc)
def _extract_text(content: Any) -> str:
if isinstance(content, str):
return content.strip()
if isinstance(content, list):
parts: List[str] = []
for c in content:
if isinstance(c, dict) and c.get("type") == "text" and isinstance(c.get("text"), str):
parts.append(c["text"])
elif isinstance(c, str):
parts.append(c)
return "\n".join(p.strip() for p in parts if p and p.strip()).strip()
if isinstance(content, dict):
if isinstance(content.get("text"), str):
return content["text"].strip()
return str(content).strip()
@dataclass
class Source:
label: str
transcript_path: Path
def _load_sources() -> List[Source]:
"""
Sources file format:
{
"sources": [
{ "label": "telegram:263887248", "path": "/root/.openclaw/agents/main/sessions/<id>.jsonl" }
]
}
"""
payload = _load_json(SOURCES_PATH, {"sources": []})
sources: List[Source] = []
for item in payload.get("sources", []) if isinstance(payload, dict) else []:
if not isinstance(item, dict):
continue
label = str(item.get("label") or "session")
p = item.get("path")
if not isinstance(p, str) or not p:
continue
sources.append(Source(label=label, transcript_path=Path(p)))
return sources
def _memory_path_for(dt: datetime) -> Path:
tz = _local_tz()
local = dt.astimezone(tz)
MEMORY_DIR.mkdir(parents=True, exist_ok=True)
return MEMORY_DIR / f"{local.date().isoformat()}.md"
def _append_memory(dt: datetime, label: str, role: str, text: str) -> None:
if not text.strip():
return
tz = _local_tz()
local = dt.astimezone(tz)
mem = _memory_path_for(dt)
if not mem.exists():
mem.write_text(f"# {local.date().isoformat()}\n\n", encoding="utf-8")
header = f"## {local.strftime('%H:%M:%S')} - {label} ({role})"
body = text.strip()
mem.write_text(mem.read_text(encoding="utf-8") + f"{header}\n\n{body}\n\n", encoding="utf-8")
def _iter_new_lines(path: Path, *, start_offset: int) -> Iterable[tuple[int, str]]:
with open(path, "rb") as f:
f.seek(max(0, int(start_offset)))
while True:
raw = f.readline()
if not raw:
break
try:
line = raw.decode("utf-8", errors="ignore").strip()
except Exception:
line = ""
if not line:
continue
yield (f.tell(), line)
def run() -> Dict[str, Any]:
sources = _load_sources()
state = _load_json(STATE_PATH, {"offsets": {}})
offsets: Dict[str, int] = state.get("offsets", {}) if isinstance(state, dict) else {}
out = {
"success": True,
"time": datetime.now(timezone.utc).isoformat(),
"sources": len(sources),
"messages_appended": 0,
"errors": [],
}
for src in sources:
try:
if not src.transcript_path.exists():
continue
key = str(src.transcript_path)
start = int(offsets.get(key, 0))
for new_off, line in _iter_new_lines(src.transcript_path, start_offset=start):
try:
obj = json.loads(line)
except Exception:
offsets[key] = new_off
continue
if not isinstance(obj, dict) or obj.get("type") != "message":
offsets[key] = new_off
continue
msg = obj.get("message") if isinstance(obj.get("message"), dict) else {}
role = str(msg.get("role") or "unknown")
content = msg.get("content")
text = _extract_text(content)
dt = None
if isinstance(msg.get("timestamp"), (int, float)):
dt = _ms_to_dt(int(msg["timestamp"]))
elif isinstance(obj.get("timestamp"), str):
dt = _iso_to_dt(obj["timestamp"])
if dt is None:
dt = datetime.now(timezone.utc)
if len(text.strip()) < 5:
offsets[key] = new_off
continue
_append_memory(dt, src.label, role, text)
out["messages_appended"] += 1
offsets[key] = new_off
except Exception as e:
out["success"] = False
out["errors"].append(f"{src.transcript_path}: {e}")
_save_json(STATE_PATH, {"offsets": offsets, "updated_at": out["time"]})
return out
if __name__ == "__main__":
res = run()
print(json.dumps(res, ensure_ascii=False, indent=2))

View File

@@ -0,0 +1,169 @@
#!/usr/bin/env python3
"""
Verify pending (unconfirmed) engrams using lightweight external checks.
Policy (conservative):
- `openclaw-memory` is treated as internal ground-truth and is auto-confirmed
by the review job (see `cron_tasks/review_brain.py` in the workspace runtime).
- For `source=web`, confirm if the grounded URL responds with HTTP 2xx, reject on
4xx/5xx, and keep pending on timeouts/unknown.
- Reject obvious low-signal placeholders (e.g. session summary stubs).
"""
import json
import os
import sys
from pathlib import Path
from datetime import datetime, timezone
from typing import Any, Optional
WORKSPACE = Path(os.environ.get("SECOND_BRAIN_WORKSPACE", "/root/.openclaw/workspace/second-brain"))
DB_PATH = Path(os.environ.get("BRAIN_DB", str(WORKSPACE / "data" / "brain.sqlite"))).resolve()
sys.path.insert(0, str(WORKSPACE))
from src.store import EngramStore
from src.engram import ReviewEntry
OUTPUT_FILE = os.environ.get("CRON_OUTPUT_FILE", "/tmp/verify_pending_external.json")
def _now() -> str:
return datetime.now(timezone.utc).isoformat()
def _get_url(meta: dict[str, Any]) -> Optional[str]:
url = meta.get("url")
if isinstance(url, str) and url.startswith(("http://", "https://")):
return url
grounding = meta.get("grounding")
if isinstance(grounding, dict):
g_url = grounding.get("url")
if isinstance(g_url, str) and g_url.startswith(("http://", "https://")):
return g_url
return None
def _http_status(url: str, timeout_s: float = 6.0) -> Optional[int]:
try:
import urllib.request
req = urllib.request.Request(
url,
method="GET",
headers={"User-Agent": "openclaw-secondbrain/verify_pending_external"},
)
with urllib.request.urlopen(req, timeout=timeout_s) as resp:
return int(getattr(resp, "status", 200))
except Exception:
return None
def main() -> int:
if not DB_PATH.exists():
out = {"success": False, "error": f"db missing: {DB_PATH}", "time": _now()}
Path(OUTPUT_FILE).write_text(json.dumps(out, indent=2))
print(out["error"])
return 1
store = EngramStore(str(DB_PATH))
all_egs = []
offset = 0
while True:
batch = store.get_all(limit=2000, offset=offset)
if not batch:
break
all_egs.extend(batch)
offset += len(batch)
pending = [
eg
for eg in all_egs
if (not eg.correctness.confirmed and eg.correctness.rejections == 0)
]
confirmed = 0
rejected = 0
still_pending = 0
checked = 0
for eg in pending:
checked += 1
src = eg.metadata.get("source")
content = (eg.content or "").strip()
if src == "session" and (
content.startswith("Session Summary (sess_") or content.startswith("Please remember ")
):
eg.correctness.rejections += 1
eg.correctness.last_reviewed = _now()
eg.correctness.review_history.append(
ReviewEntry(
by="verify-pending",
action="reject",
at=_now(),
note="Auto-reject: session placeholder",
)
)
store.save(eg)
rejected += 1
continue
if src == "web":
url = _get_url(eg.metadata)
if not url:
still_pending += 1
continue
status = _http_status(url)
if status is None:
still_pending += 1
continue
if 200 <= status < 300:
eg.correctness.confirmed = True
eg.correctness.confirmations += 1
eg.correctness.last_reviewed = _now()
eg.correctness.review_history.append(
ReviewEntry(
by="verify-pending",
action="confirm",
at=_now(),
note=f"Auto-confirm: web url ok ({status}) {url}",
)
)
store.save(eg)
confirmed += 1
else:
eg.correctness.rejections += 1
eg.correctness.last_reviewed = _now()
eg.correctness.review_history.append(
ReviewEntry(
by="verify-pending",
action="reject",
at=_now(),
note=f"Auto-reject: web url status={status} {url}",
)
)
store.save(eg)
rejected += 1
continue
still_pending += 1
out = {
"success": True,
"time": _now(),
"total": len(all_egs),
"pending_before": len(pending),
"checked": checked,
"confirmed": confirmed,
"rejected": rejected,
"still_pending": still_pending,
}
Path(OUTPUT_FILE).write_text(json.dumps(out, indent=2))
print(
f"VERIFY: pending_before={out['pending_before']} confirmed={confirmed} rejected={rejected} still_pending={still_pending}"
)
return 0
if __name__ == "__main__":
raise SystemExit(main())

View File

@@ -1,6 +1,6 @@
# Second-Brain 2.0 — Release Checklist (Grundversion + FastAPI) # Second-Brain 2.0 — Release Checklist (Grundversion + FastAPI)
Scope: OpenClaw workspace at `/root/.openclaw/workspace` with `second-brain/` and the shipped `systemd/` unit files. Scope: OpenClaw workspace at `/root/.openclaw/workspace` with `second-brain/` and the shipped `second-brain/systemd/` unit files.
This checklist is copy/paste friendly. Run as a user with `sudo` (systemd commands need it). This checklist is copy/paste friendly. Run as a user with `sudo` (systemd commands need it).
@@ -24,13 +24,21 @@ ls -la /etc/systemd/system/openclaw-memory-archive.* 2>/dev/null || true
If missing, install them (symlink is fine; copy is fine too): If missing, install them (symlink is fine; copy is fine too):
```bash ```bash
sudo ln -sf /root/.openclaw/workspace/systemd/openclaw-secondbrain-*.service /etc/systemd/system/ sudo ln -sf /root/.openclaw/workspace/second-brain/systemd/openclaw-secondbrain-*.service /etc/systemd/system/
sudo ln -sf /root/.openclaw/workspace/systemd/openclaw-secondbrain-*.timer /etc/systemd/system/ sudo ln -sf /root/.openclaw/workspace/second-brain/systemd/openclaw-secondbrain-*.timer /etc/systemd/system/
sudo ln -sf /root/.openclaw/workspace/systemd/openclaw-memory-archive.* /etc/systemd/system/ sudo ln -sf /root/.openclaw/workspace/second-brain/systemd/openclaw-memory-archive.* /etc/systemd/system/
sudo ln -sf /root/.openclaw/workspace/systemd/openclaw-secondbrain-dashboard.service /etc/systemd/system/ sudo ln -sf /root/.openclaw/workspace/second-brain/systemd/openclaw-secondbrain-dashboard.service /etc/systemd/system/
sudo systemctl daemon-reload sudo systemctl daemon-reload
``` ```
Optional (verification hardening):
```bash
sudo ln -sf /root/.openclaw/workspace/second-brain/systemd/openclaw-secondbrain-verify-pending.* /etc/systemd/system/
sudo systemctl daemon-reload
sudo systemctl enable --now openclaw-secondbrain-verify-pending.timer
```
### Enable timers ### Enable timers
```bash ```bash

View File

@@ -11,11 +11,14 @@ Goals:
import json import json
import os import os
import sqlite3 import sqlite3
import subprocess
from datetime import datetime, timezone from datetime import datetime, timezone
from pathlib import Path from pathlib import Path
from uuid import uuid4
from urllib.parse import urlparse
from fastapi import FastAPI, Form, Query, Request from fastapi import FastAPI, Form, Query, Request
from fastapi.responses import HTMLResponse, JSONResponse, PlainTextResponse from fastapi.responses import HTMLResponse, JSONResponse, PlainTextResponse, StreamingResponse
from fastapi.staticfiles import StaticFiles from fastapi.staticfiles import StaticFiles
# ─── Config ────────────────────────────────────────────────────────────────── # ─── Config ──────────────────────────────────────────────────────────────────
@@ -69,6 +72,136 @@ def parse_engram(row: sqlite3.Row) -> dict:
} }
def _now_iso() -> str:
return datetime.now(timezone.utc).isoformat()
def _update_correctness(engram_id: str, *, action: str, reason: str | None = None) -> dict:
"""
Update correctness_json for an engram. action: confirm|reject
"""
conn = get_db()
c = conn.cursor()
row = c.execute("SELECT correctness_json FROM engrams WHERE id = ?", (engram_id,)).fetchone()
if not row:
conn.close()
raise FileNotFoundError(f"Engram not found: {engram_id}")
corr = json.loads(row["correctness_json"] or "{}")
corr.setdefault("confirmed", False)
corr.setdefault("confirmations", 0)
corr.setdefault("rejections", 0)
corr.setdefault("review_history", [])
corr["last_reviewed"] = _now_iso()
entry = {
"by": "dashboard",
"action": action,
"at": corr["last_reviewed"],
"note": (reason or "").strip(),
}
try:
corr["review_history"].append(entry)
except Exception:
corr["review_history"] = [entry]
if action == "confirm":
corr["confirmed"] = True
corr["confirmations"] = int(corr.get("confirmations", 0) or 0) + 1
elif action == "reject":
corr["rejections"] = int(corr.get("rejections", 0) or 0) + 1
c.execute(
"UPDATE engrams SET correctness_json = ?, modified_at = ? WHERE id = ?",
(json.dumps(corr, ensure_ascii=False), corr["last_reviewed"], engram_id),
)
conn.commit()
conn.close()
return {"ok": True}
def _bump_access(engram_id: str) -> dict:
conn = get_db()
c = conn.cursor()
row = c.execute("SELECT metadata_json FROM engrams WHERE id = ?", (engram_id,)).fetchone()
if not row:
conn.close()
raise FileNotFoundError(f"Engram not found: {engram_id}")
meta = json.loads(row["metadata_json"] or "{}")
meta["access_count"] = int(meta.get("access_count", 0) or 0) + 1
meta["last_accessed"] = _now_iso()
c.execute(
"UPDATE engrams SET metadata_json = ?, modified_at = ? WHERE id = ?",
(json.dumps(meta, ensure_ascii=False), meta["last_accessed"], engram_id),
)
conn.commit()
conn.close()
return {"ok": True}
def _safe_json_extract_tags(meta_json: str) -> list[str]:
try:
d = json.loads(meta_json or "{}")
tags = d.get("tags") or []
return [t for t in tags if isinstance(t, str)]
except Exception:
return []
def _host_from_meta(meta_json: str) -> str | None:
try:
d = json.loads(meta_json or "{}")
grounding = d.get("grounding")
url = d.get("url")
if isinstance(grounding, dict) and isinstance(grounding.get("url"), str):
url = grounding.get("url")
if not isinstance(url, str):
return None
parsed = urlparse(url)
return parsed.hostname
except Exception:
return None
def _systemd_unit_state(unit: str) -> dict:
"""
Best-effort systemd status snapshot for a known unit.
Never raises; returns minimal fields.
"""
try:
out = subprocess.check_output(
["systemctl", "show", unit, "--no-page", "--property=ActiveState,SubState,Result,ExecMainStatus,ExecMainCode,ExecMainStartTimestamp,ExecMainExitTimestamp"],
text=True,
stderr=subprocess.STDOUT,
timeout=2,
)
kv = {}
for line in out.splitlines():
if "=" in line:
k, v = line.split("=", 1)
kv[k] = v
return {
"unit": unit,
"active": kv.get("ActiveState"),
"sub": kv.get("SubState"),
"result": kv.get("Result"),
"exit_status": kv.get("ExecMainStatus"),
"start_ts": kv.get("ExecMainStartTimestamp"),
"exit_ts": kv.get("ExecMainExitTimestamp"),
}
except Exception as e:
return {"unit": unit, "error": str(e)}
def _dir_size_bytes(path: Path) -> int:
total = 0
try:
for p in path.rglob("*"):
try:
if p.is_file():
total += p.stat().st_size
except Exception:
pass
except Exception:
pass
return total
# ─── API Endpoints ─────────────────────────────────────────────────────────── # ─── API Endpoints ───────────────────────────────────────────────────────────
@app.get("/healthz", response_class=PlainTextResponse) @app.get("/healthz", response_class=PlainTextResponse)
@@ -83,6 +216,256 @@ def api_config():
"db_path": str(DB_PATH), "db_path": str(DB_PATH),
} }
@app.get("/api/db_info")
def api_db_info():
if not DB_PATH.exists():
raise FileNotFoundError(f"DB not found: {DB_PATH}")
st = DB_PATH.stat()
return {
"db_path": str(DB_PATH),
"size_bytes": st.st_size,
"mtime": datetime.fromtimestamp(st.st_mtime, tz=timezone.utc).isoformat(),
}
@app.get("/api/storage_stats")
def api_storage_stats():
conn = get_db()
c = conn.cursor()
total = c.execute("SELECT COUNT(*) FROM engrams").fetchone()[0]
confirmed = c.execute(
"SELECT COUNT(*) FROM engrams WHERE json_extract(correctness_json, '$.confirmed') = 1"
).fetchone()[0]
sources = {
r[0]: r[1]
for r in c.execute(
"SELECT json_extract(metadata_json, '$.source') AS src, COUNT(*) FROM engrams GROUP BY src ORDER BY COUNT(*) DESC"
).fetchall()
if r[0] is not None
}
conn.close()
chroma_dir = WORKSPACE / "data" / "chroma"
emb_cache_dir = WORKSPACE / "data" / "embedding_cache"
vec_state_path = WORKSPACE / "data" / "vector_index_state.json"
vec_state = {}
if vec_state_path.exists():
try:
vec_state = json.loads(vec_state_path.read_text())
except Exception:
vec_state = {}
obsidian_cfg_path = WORKSPACE / "data" / "obsidian_config.json"
obsidian_cfg = None
if obsidian_cfg_path.exists():
try:
obsidian_cfg = json.loads(obsidian_cfg_path.read_text())
except Exception:
obsidian_cfg = {"raw": obsidian_cfg_path.read_text()[:2000]}
backup_files = sorted((WORKSPACE / "data").glob("backup_*.jsonl"))
return {
"sql": {
"total_engrams": total,
"confirmed": confirmed,
"pending": total - confirmed,
"by_source": sources,
},
"vector": {
"chroma_dir": str(chroma_dir),
"chroma_size_bytes": _dir_size_bytes(chroma_dir) if chroma_dir.exists() else 0,
"embedding_cache_dir": str(emb_cache_dir),
"embedding_cache_files": len(list(emb_cache_dir.glob("*.json"))) if emb_cache_dir.exists() else 0,
"vector_state": vec_state,
},
"obsidian": {
"config_path": str(obsidian_cfg_path),
"configured": bool(obsidian_cfg),
"config": obsidian_cfg,
},
"backups": {
"count": len(backup_files),
"latest": str(backup_files[-1]) if backup_files else None,
},
}
@app.get("/api/jobs")
def api_jobs():
# Known units that influence "freshness" of the brain.
units = [
"openclaw-secondbrain-ingest-memory.service",
"openclaw-secondbrain-index-vectors.service",
"openclaw-secondbrain-review.service",
"openclaw-secondbrain-heartbeat.service",
"openclaw-secondbrain-verify-pending.service",
]
return {"items": [_systemd_unit_state(u) for u in units]}
@app.get("/api/insights")
def api_insights(limit: int = Query(8, ge=1, le=50)):
conn = get_db()
c = conn.cursor()
rows = c.execute(
"SELECT id, metadata_json, correctness_json, created_at, modified_at FROM engrams ORDER BY created_at DESC LIMIT 2000"
).fetchall()
total = c.execute("SELECT COUNT(*) FROM engrams").fetchone()[0]
confirmed = c.execute(
"SELECT COUNT(*) FROM engrams WHERE json_extract(correctness_json, '$.confirmed') = 1"
).fetchone()[0]
pending = total - confirmed
tag_counts: dict[str, int] = {}
source_counts: dict[str, int] = {}
host_counts: dict[str, int] = {}
active: list[dict] = []
forgotten: list[dict] = []
for r in rows:
meta = json.loads(r["metadata_json"] or "{}")
src = meta.get("source", "unknown")
source_counts[src] = source_counts.get(src, 0) + 1
for t in (meta.get("tags") or []):
if isinstance(t, str):
tag_counts[t] = tag_counts.get(t, 0) + 1
host = _host_from_meta(r["metadata_json"])
if host:
host_counts[host] = host_counts.get(host, 0) + 1
access_count = int(meta.get("access_count", 0) or 0)
created = meta.get("created", r["created_at"])
if access_count >= 5 and len(active) < limit:
active.append(
{
"id": r["id"],
"access_count": access_count,
"source": src,
"created": created,
}
)
if access_count == 0 and len(forgotten) < limit:
forgotten.append(
{
"id": r["id"],
"access_count": access_count,
"source": src,
"created": created,
}
)
def top_k(d: dict[str, int]) -> list[dict]:
return [
{"key": k, "count": v}
for k, v in sorted(d.items(), key=lambda kv: kv[1], reverse=True)[:limit]
]
conn.close()
return {
"total": total,
"confirmed": confirmed,
"pending": pending,
"top_tags": top_k(tag_counts),
"top_sources": top_k(source_counts),
"top_hosts": top_k(host_counts),
"active_engrams": active,
"forgotten_engrams": forgotten,
}
@app.get("/api/graph")
def api_graph(limit_nodes: int = Query(200, ge=50, le=1000)):
"""
Returns a lightweight graph view:
- Nodes: engrams + tag:<tag> + host:<hostname>
- Edges: engram->tag and engram->host plus explicit engrams_links edges.
"""
conn = get_db()
c = conn.cursor()
rows = c.execute("SELECT id, metadata_json FROM engrams ORDER BY created_at DESC LIMIT 1000").fetchall()
link_rows = c.execute("SELECT from_id, to_id FROM engrams_links ORDER BY rowid DESC LIMIT 2000").fetchall()
conn.close()
nodes: dict[str, dict] = {}
edges: list[dict] = []
def add_node(nid: str, kind: str, label: str | None = None, weight: float | None = None):
if nid not in nodes:
nodes[nid] = {"id": nid, "kind": kind}
if label is not None:
nodes[nid]["label"] = label
if weight is not None:
nodes[nid]["weight"] = weight
for r in rows:
eid = r["id"]
add_node(eid, "engram", label=eid[:8])
for t in _safe_json_extract_tags(r["metadata_json"]):
tid = f"tag:{t}"
add_node(tid, "tag", label=t)
edges.append({"from": eid, "to": tid, "kind": "has_tag"})
host = _host_from_meta(r["metadata_json"])
if host:
hid = f"host:{host}"
add_node(hid, "host", label=host)
edges.append({"from": eid, "to": hid, "kind": "grounded_at"})
for fr, to in link_rows:
add_node(fr, "engram")
add_node(to, "engram")
edges.append({"from": fr, "to": to, "kind": "link"})
# Trim nodes to keep payload bounded (prefer engrams and connected tags/hosts)
if len(nodes) > limit_nodes:
# Keep a balanced subset: many engrams plus the most-connected non-engrams.
kept: dict[str, dict] = {}
engram_budget = int(limit_nodes * 0.7)
# 1) Keep newest engrams first (they appear first in `rows` loop insertion order)
for r in rows:
eid = r["id"]
if eid in nodes:
kept[eid] = nodes[eid]
if len(kept) >= engram_budget:
break
# 2) Rank remaining nodes by degree within current edge set
degree: dict[str, int] = {}
for e in edges:
degree[e["from"]] = degree.get(e["from"], 0) + 1
degree[e["to"]] = degree.get(e["to"], 0) + 1
remaining = [nid for nid in nodes.keys() if nid not in kept]
remaining.sort(key=lambda nid: degree.get(nid, 0), reverse=True)
for nid in remaining:
kept[nid] = nodes[nid]
if len(kept) >= limit_nodes:
break
nodes = kept
edges = [e for e in edges if e["from"] in nodes and e["to"] in nodes]
return {"nodes": list(nodes.values()), "edges": edges}
@app.get("/api/events")
def api_events():
"""
Server-Sent Events stream for lightweight real-time UI refresh.
"""
import time
def gen():
while True:
payload = {
"ts": datetime.now(timezone.utc).isoformat(),
"stats": api_stats(),
"storage": api_storage_stats(),
"jobs": api_jobs(),
"insights": api_insights(limit=8),
}
yield f"data: {json.dumps(payload, ensure_ascii=False)}\n\n"
time.sleep(5)
return StreamingResponse(gen(), media_type="text/event-stream")
@app.exception_handler(FileNotFoundError) @app.exception_handler(FileNotFoundError)
def handle_file_not_found(request: Request, exc: FileNotFoundError): def handle_file_not_found(request: Request, exc: FileNotFoundError):
@@ -204,70 +587,107 @@ def api_engram_detail(engram_id: str):
return result return result
@app.post("/api/engrams/{engram_id}/confirm") @app.get("/api/pending")
def api_confirm(engram_id: str, reason: str = Form("")): def api_pending(
limit: int = Query(20, ge=1, le=200),
offset: int = Query(0, ge=0),
source: str | None = Query(None),
):
conn = get_db() conn = get_db()
c = conn.cursor() c = conn.cursor()
row = c.execute( where = ["json_extract(correctness_json, '$.confirmed') = 0"]
"SELECT correctness_json FROM engrams WHERE id = ?", (engram_id,) params: list = []
).fetchone() if source:
if not row: where.append("json_extract(metadata_json, '$.source') = ?")
params.append(source)
rows = c.execute(
f"""
SELECT * FROM engrams
WHERE {' AND '.join(where)}
ORDER BY created_at DESC
LIMIT ? OFFSET ?
""",
params + [limit, offset],
).fetchall()
items = [parse_engram(r) for r in rows]
conn.close() conn.close()
return JSONResponse({"error": "Not found"}, status_code=404) return {"items": items, "limit": limit, "offset": offset}
correctness = json.loads(row["correctness_json"] or "{}")
correctness["confirmed"] = True
correctness["confirmations"] = correctness.get("confirmations", 0) + 1
correctness["last_reviewed"] = datetime.now(timezone.utc).isoformat()
review_history = correctness.get("review_history", [])
review_history.append({
"by": "web",
"action": "confirm",
"at": datetime.now(timezone.utc).isoformat(),
"note": reason or "confirmed via dashboard",
})
correctness["review_history"] = review_history
@app.post("/api/engrams")
def api_create_engram(content: str = Form(...), tags: str = Form("")):
content = (content or "").strip()
if not content:
return JSONResponse({"error": "content required"}, status_code=400)
tag_list = [t.strip() for t in (tags or "").split(",") if t.strip()]
now = _now_iso()
engram_id = str(uuid4())
meta = {
"source": "user",
"confidence": 0.7,
"created": now,
"modified": now,
"access_count": 0,
"last_accessed": now,
"tags": tag_list,
"session_id": None,
"agent_id": None,
"grounding": 0,
}
corr = {
"confirmed": False,
"confirmations": 0,
"rejections": 0,
"last_reviewed": None,
"review_history": [],
}
conn = get_db()
c = conn.cursor()
c.execute( c.execute(
"UPDATE engrams SET correctness_json = ?, modified_at = ? WHERE id = ?", """
(json.dumps(correctness), datetime.now(timezone.utc).isoformat(), engram_id), INSERT INTO engrams (id, content, metadata_json, correctness_json, links_json, hierarchy_json, embedding_json, created_at, modified_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
""",
(
engram_id,
content,
json.dumps(meta, ensure_ascii=False),
json.dumps(corr, ensure_ascii=False),
"[]",
"{}",
None,
now,
now,
),
) )
conn.commit() conn.commit()
conn.close() conn.close()
return {"success": True, "engram_id": engram_id} return {"id": engram_id}
@app.post("/api/engrams/{engram_id}/confirm")
def api_confirm_engram(engram_id: str, reason: str = Form("")):
try:
return _update_correctness(engram_id, action="confirm", reason=reason)
except FileNotFoundError as e:
return JSONResponse({"error": str(e)}, status_code=404)
@app.post("/api/engrams/{engram_id}/reject") @app.post("/api/engrams/{engram_id}/reject")
def api_reject(engram_id: str, reason: str = Form("")): def api_reject_engram(engram_id: str, reason: str = Form("")):
conn = get_db() try:
c = conn.cursor() return _update_correctness(engram_id, action="reject", reason=reason)
row = c.execute( except FileNotFoundError as e:
"SELECT correctness_json FROM engrams WHERE id = ?", (engram_id,) return JSONResponse({"error": str(e)}, status_code=404)
).fetchone()
if not row:
conn.close()
return JSONResponse({"error": "Not found"}, status_code=404)
correctness = json.loads(row["correctness_json"] or "{}")
correctness["confirmed"] = False
correctness["rejections"] = correctness.get("rejections", 0) + 1
correctness["last_reviewed"] = datetime.now(timezone.utc).isoformat()
review_history = correctness.get("review_history", [])
review_history.append({
"by": "web",
"action": "reject",
"at": datetime.now(timezone.utc).isoformat(),
"note": reason or "rejected via dashboard",
})
correctness["review_history"] = review_history
c.execute( @app.post("/api/engrams/{engram_id}/refresh")
"UPDATE engrams SET correctness_json = ?, modified_at = ? WHERE id = ?", def api_refresh_engram(engram_id: str):
(json.dumps(correctness), datetime.now(timezone.utc).isoformat(), engram_id), try:
) return _bump_access(engram_id)
conn.commit() except FileNotFoundError as e:
conn.close() return JSONResponse({"error": str(e)}, status_code=404)
return {"success": True, "engram_id": engram_id}
@app.post("/api/engrams/{engram_id}/refresh") @app.post("/api/engrams/{engram_id}/refresh")

20
scripts/smoke_dashboard.sh Executable file
View File

@@ -0,0 +1,20 @@
#!/usr/bin/env bash
set -euo pipefail
HOST="${SECOND_BRAIN_HOST:-127.0.0.1}"
PORT="${SECOND_BRAIN_PORT:-${PORT:-8501}}"
BASE_URL="http://${HOST}:${PORT}"
if command -v systemctl >/dev/null 2>&1; then
if ! systemctl is-active --quiet openclaw-secondbrain-dashboard.service; then
echo "ERROR: openclaw-secondbrain-dashboard.service is not active" >&2
exit 1
fi
fi
curl -fsS "${BASE_URL}/healthz" >/dev/null
stats_json="$(curl -fsS "${BASE_URL}/api/stats")"
python3 -c 'import json,sys; json.load(sys.stdin)' <<<"$stats_json"
echo "OK: dashboard smoke test passed (${BASE_URL})"

View File

@@ -6,6 +6,7 @@ Keine externen Abhängigkeiten außer sqlite3 (stdlib).
import json import json
import sqlite3 import sqlite3
import os import os
import re
from pathlib import Path from pathlib import Path
from typing import List, Optional, Dict, Any from typing import List, Optional, Dict, Any
from uuid import UUID from uuid import UUID
@@ -158,7 +159,12 @@ class EngramStore:
def search_text(self, query: str, limit: int = 10) -> List[Engram]: def search_text(self, query: str, limit: int = 10) -> List[Engram]:
"""Full-Text-Suche über Engramm-Inhalt via SQLite FTS5 (OR-Verknüpfung).""" """Full-Text-Suche über Engramm-Inhalt via SQLite FTS5 (OR-Verknüpfung)."""
# FTS5-Syntax: Wörter mit OR verbinden für bessere Ergebnisse # FTS5-Syntax: Wörter mit OR verbinden für bessere Ergebnisse
words = [w.strip() for w in query.replace("'", "''").split() if w.strip()] words = []
for word in query.split():
# Nur alphanumerische Zeichen als FTS5-Tokens akzeptieren
clean_word = re.sub(r'[^a-zA-Z0-9]+', '', word)
if clean_word:
words.append(clean_word)
safe_query = " OR ".join(words) if len(words) > 1 else (words[0] if words else "*") safe_query = " OR ".join(words) if len(words) > 1 else (words[0] if words else "*")
sql = """ sql = """
SELECT e.* FROM engrams e SELECT e.* FROM engrams e

View File

@@ -27,6 +27,31 @@ body {
top: 0; top: 0;
z-index: 50; z-index: 50;
} }
.tabs-bar{
display:flex;
gap:8px;
padding:8px 12px 10px;
background:#141419;
border-bottom:1px solid #252530;
position: sticky;
top: 52px;
z-index: 45;
}
.tabs-bar .tab-btn{
flex:1;
background:#1e1e28;
border:1px solid #2a2a3a;
border-radius: 12px;
padding:10px 10px;
color:#cfd3ff;
font-weight:700;
font-size:0.82rem;
}
.tabs-bar .tab-btn.active{
border-color:#6c8af5;
box-shadow:0 0 0 1px rgba(108,138,245,0.22) inset;
}
.stat { .stat {
text-align: center; text-align: center;
min-width: 60px; min-width: 60px;
@@ -54,6 +79,66 @@ body {
padding: 10px 12px; padding: 10px 12px;
background: #141419; background: #141419;
} }
/* tab buttons styled via .tabs-bar */
/* ─── Panels (Graph/Status) ──────────────────────────────────────────────── */
.panel {
margin: 8px 12px;
background: #1a1a24;
border: 1px solid #252533;
border-radius: 14px;
padding: 10px 12px;
}
.panel-title {
font-size: 0.8rem;
text-transform: uppercase;
letter-spacing: 0.5px;
color: #9aa3d9;
margin-bottom: 8px;
}
.kv-row {
display: flex;
gap: 10px;
padding: 6px 0;
border-bottom: 1px solid #20202a;
}
.kv-row:last-child { border-bottom: none; }
.kv-key {
width: 110px;
color: #888899;
font-size: 0.78rem;
}
.kv-val {
flex: 1;
color: #e8e8ee;
font-size: 0.85rem;
word-break: break-word;
}
.pill {
display: inline-block;
margin: 2px 4px 2px 0;
padding: 2px 8px;
border-radius: 999px;
background: #2a2a3a;
color: #8a9aff;
font-size: 0.72rem;
}
.muted {
color: #888899;
font-size: 0.8rem;
margin-top: 6px;
}
.small { font-size: 0.75rem; }
/* Graph canvas */
#graphCanvas{
display:block;
margin: 8px auto 0;
background:#12121a;
border:1px solid #252533;
border-radius: 14px;
}
#searchInput { #searchInput {
flex: 1; flex: 1;
background: #1e1e28; background: #1e1e28;

View File

@@ -0,0 +1,9 @@
[Unit]
Description=OpenClaw archive memory/*.md older than 7 days
OnFailure=openclaw-secondbrain-notify@%n.service
[Service]
Type=oneshot
WorkingDirectory=/root/.openclaw/workspace
ExecStart=/bin/bash -lc 'flock -n /tmp/%n.lock /usr/bin/python3 /root/.openclaw/workspace/openclaw_cron_wrapper.py archive_memory_md'

View File

@@ -0,0 +1,10 @@
[Unit]
Description=OpenClaw archive memory/*.md daily
[Timer]
OnCalendar=*-*-* 03:30:00
Persistent=true
[Install]
WantedBy=timers.target

View File

@@ -0,0 +1,8 @@
[Unit]
Description=OpenClaw Second-Brain backup_secondbrain
OnFailure=openclaw-secondbrain-notify@%n.service
[Service]
Type=oneshot
WorkingDirectory=/root/.openclaw/workspace
ExecStart=/bin/bash -lc 'flock -n /tmp/%n.lock /usr/bin/python3 /root/.openclaw/workspace/openclaw_cron_wrapper.py backup_secondbrain'

View File

@@ -0,0 +1,10 @@
[Unit]
Description=OpenClaw Second-Brain backup_secondbrain (daily 02:00)
[Timer]
OnCalendar=*-*-* 02:00:00
Persistent=true
[Install]
WantedBy=timers.target

View File

@@ -0,0 +1,15 @@
[Unit]
Description=OpenClaw Second-Brain Dashboard (FastAPI)
After=network.target
[Service]
Type=simple
WorkingDirectory=/root/.openclaw/workspace/second-brain
Environment=SECOND_BRAIN_WORKSPACE=/root/.openclaw/workspace/second-brain
Environment=SECOND_BRAIN_PORT=8501
ExecStart=/root/.openclaw/workspace/second-brain/.venv/bin/uvicorn fastapi_app:app --host 0.0.0.0 --port 8501
Restart=always
RestartSec=2
[Install]
WantedBy=multi-user.target

View File

@@ -0,0 +1,8 @@
[Unit]
Description=OpenClaw Second-Brain export_obsidian
OnFailure=openclaw-secondbrain-notify@%n.service
[Service]
Type=oneshot
WorkingDirectory=/root/.openclaw/workspace
ExecStart=/bin/bash -lc 'flock -n /tmp/%n.lock /usr/bin/python3 /root/.openclaw/workspace/openclaw_cron_wrapper.py export_obsidian'

View File

@@ -0,0 +1,10 @@
[Unit]
Description=OpenClaw Second-Brain export_obsidian (hourly)
[Timer]
OnCalendar=hourly
Persistent=true
[Install]
WantedBy=timers.target

View File

@@ -0,0 +1,8 @@
[Unit]
Description=OpenClaw Second-Brain heartbeat_secondbrain
OnFailure=openclaw-secondbrain-notify@%n.service
[Service]
Type=oneshot
WorkingDirectory=/root/.openclaw/workspace
ExecStart=/bin/bash -lc 'flock -n /tmp/%n.lock /usr/bin/python3 /root/.openclaw/workspace/openclaw_cron_wrapper.py heartbeat_secondbrain'

View File

@@ -0,0 +1,11 @@
[Unit]
Description=OpenClaw Second-Brain heartbeat_secondbrain (every 6h)
[Timer]
OnCalendar=*-*-* 00,06,12,18:00:00
RandomizedDelaySec=300
Persistent=true
[Install]
WantedBy=timers.target

View File

@@ -0,0 +1,10 @@
[Unit]
Description=OpenClaw Second-Brain index_vectors
OnFailure=openclaw-secondbrain-notify@%n.service
[Service]
Type=oneshot
WorkingDirectory=/root/.openclaw/workspace
Environment=HF_HOME=/root/.openclaw/workspace/second-brain/data/hf_cache
Environment=SENTENCE_TRANSFORMERS_HOME=/root/.openclaw/workspace/second-brain/data/st_cache
ExecStart=/bin/bash -lc 'flock -n /tmp/%n.lock /usr/bin/python3 /root/.openclaw/workspace/openclaw_cron_wrapper.py index_vectors'

View File

@@ -0,0 +1,10 @@
[Unit]
Description=OpenClaw Second-Brain index_vectors (every 30 min)
[Timer]
OnCalendar=*:0/30
Persistent=true
[Install]
WantedBy=timers.target

View File

@@ -0,0 +1,8 @@
[Unit]
Description=OpenClaw Second-Brain ingest_memory
OnFailure=openclaw-secondbrain-notify@%n.service
[Service]
Type=oneshot
WorkingDirectory=/root/.openclaw/workspace
ExecStart=/bin/bash -lc 'flock -n /tmp/%n.lock /usr/bin/python3 /root/.openclaw/workspace/openclaw_cron_wrapper.py ingest_memory'

View File

@@ -0,0 +1,9 @@
[Unit]
Description=OpenClaw Second-Brain ingest_memory (every 5 min)
[Timer]
OnCalendar=*:0/5
Persistent=true
[Install]
WantedBy=timers.target

View File

@@ -0,0 +1,8 @@
[Unit]
Description=OpenClaw Second-Brain ingest_obsidian
OnFailure=openclaw-secondbrain-notify@%n.service
[Service]
Type=oneshot
WorkingDirectory=/root/.openclaw/workspace
ExecStart=/bin/bash -lc 'flock -n /tmp/%n.lock /usr/bin/python3 /root/.openclaw/workspace/openclaw_cron_wrapper.py ingest_obsidian'

View File

@@ -0,0 +1,10 @@
[Unit]
Description=OpenClaw Second-Brain ingest_obsidian (every 15 min)
[Timer]
OnCalendar=*:0/15
Persistent=true
[Install]
WantedBy=timers.target

View File

@@ -0,0 +1,9 @@
[Unit]
Description=OpenClaw Second-Brain ingest transcript -> DB
OnFailure=openclaw-secondbrain-notify@%n.service
[Service]
Type=oneshot
WorkingDirectory=/root/.openclaw/workspace
ExecStart=/bin/bash -lc 'flock -n /tmp/%n.lock /usr/bin/python3 /root/.openclaw/workspace/openclaw_cron_wrapper.py ingest_transcript_to_db'

View File

@@ -0,0 +1,12 @@
[Unit]
Description=OpenClaw Second-Brain ingest transcript -> DB (every 5 min)
[Timer]
OnBootSec=90s
OnUnitActiveSec=300s
Unit=openclaw-secondbrain-ingest-transcript-to-db.service
Persistent=true
[Install]
WantedBy=timers.target

View File

@@ -0,0 +1,8 @@
[Unit]
Description=OpenClaw Second-Brain ingest transcript -> memory/*.md
OnFailure=openclaw-secondbrain-notify@%n.service
[Service]
Type=oneshot
WorkingDirectory=/root/.openclaw/workspace
ExecStart=/bin/bash -lc 'flock -n /tmp/%n.lock /usr/bin/python3 /root/.openclaw/workspace/second-brain/cron_tasks/ingest_transcript_to_memory.py'

View File

@@ -0,0 +1,11 @@
[Unit]
Description=OpenClaw Second-Brain ingest transcript -> memory (every 1 min)
[Timer]
OnBootSec=30s
OnUnitActiveSec=60s
Unit=openclaw-secondbrain-ingest-transcript-to-memory.service
[Install]
WantedBy=timers.target

View File

@@ -0,0 +1,8 @@
[Unit]
Description=OpenClaw Second-Brain failure notify (%i)
[Service]
Type=oneshot
WorkingDirectory=/root/.openclaw/workspace
ExecStart=/bin/bash -lc '/root/.openclaw/workspace/notify-telegram.sh "❌ Second-Brain job failed: %i. Check: journalctl -u %i -n 50 --no-pager"'

View File

@@ -0,0 +1,10 @@
[Unit]
Description=OpenClaw Second-Brain proactive_search_wrapper
Wants=network-online.target
After=network-online.target
OnFailure=openclaw-secondbrain-notify@%n.service
[Service]
Type=oneshot
WorkingDirectory=/root/.openclaw/workspace
ExecStart=/bin/bash -lc 'flock -n /tmp/%n.lock /usr/bin/python3 /root/.openclaw/workspace/openclaw_cron_wrapper.py proactive_search_wrapper'

View File

@@ -0,0 +1,11 @@
[Unit]
Description=OpenClaw Second-Brain proactive_search_wrapper (every 4h)
[Timer]
OnCalendar=*-*-* 01,05,09,13,17,21:10:00
RandomizedDelaySec=600
Persistent=true
[Install]
WantedBy=timers.target

View File

@@ -0,0 +1,8 @@
[Unit]
Description=OpenClaw Second-Brain review_brain
OnFailure=openclaw-secondbrain-notify@%n.service
[Service]
Type=oneshot
WorkingDirectory=/root/.openclaw/workspace
ExecStart=/bin/bash -lc 'flock -n /tmp/%n.lock /usr/bin/python3 /root/.openclaw/workspace/openclaw_cron_wrapper.py review_brain'

View File

@@ -0,0 +1,9 @@
[Unit]
Description=OpenClaw Second-Brain review_brain (every 5 min)
[Timer]
OnCalendar=*:0/5
Persistent=true
[Install]
WantedBy=timers.target

View File

@@ -0,0 +1,13 @@
[Unit]
Description=OpenClaw Second-Brain task (%i)
Wants=network-online.target
After=network-online.target
[Service]
Type=oneshot
WorkingDirectory=/root/.openclaw/workspace
ExecStart=/usr/bin/python3 /root/.openclaw/workspace/openclaw_cron_wrapper.py %i
Nice=10
IOSchedulingClass=best-effort
IOSchedulingPriority=6

View File

@@ -0,0 +1,9 @@
[Unit]
Description=OpenClaw Second-Brain verify_pending_external
OnFailure=openclaw-secondbrain-notify@%n.service
[Service]
Type=oneshot
WorkingDirectory=/root/.openclaw/workspace
ExecStart=/bin/bash -lc 'flock -n /tmp/%n.lock /usr/bin/python3 /root/.openclaw/workspace/openclaw_cron_wrapper.py verify_pending_external'

View File

@@ -0,0 +1,10 @@
[Unit]
Description=OpenClaw Second-Brain periodic verify_pending_external
[Timer]
OnBootSec=2min
OnUnitActiveSec=5min
Persistent=true
[Install]
WantedBy=timers.target

View File

@@ -16,6 +16,12 @@
<div class="stat"><span class="stat-num" id="statErrors">-</span><span class="stat-label">Err</span></div> <div class="stat"><span class="stat-num" id="statErrors">-</span><span class="stat-label">Err</span></div>
</header> </header>
<div class="tabs-bar">
<button class="tab-btn active" id="tabCards" onclick="setView('cards')">Cards</button>
<button class="tab-btn" id="tabGraph" onclick="setView('graph')">Graph</button>
<button class="tab-btn" id="tabStatus" onclick="setView('status')">Status</button>
</div>
<!-- Search --> <!-- Search -->
<div class="search-box"> <div class="search-box">
<input type="text" id="searchInput" placeholder="🔍 Suche..." /> <input type="text" id="searchInput" placeholder="🔍 Suche..." />
@@ -37,6 +43,15 @@
<!-- Cards --> <!-- Cards -->
<div class="cards" id="cards"></div> <div class="cards" id="cards"></div>
<!-- Graph -->
<div class="graph" id="graph" style="display:none;">
<canvas id="graphCanvas" width="440" height="520"></canvas>
<div class="muted small" id="graphHint">Lade Graph…</div>
</div>
<!-- Status -->
<div class="status" id="status" style="display:none;"></div>
<!-- Pagination --> <!-- Pagination -->
<div class="pagination" id="pagination"> <div class="pagination" id="pagination">
<button id="btnPrev" onclick="prevPage()"></button> <button id="btnPrev" onclick="prevPage()"></button>
@@ -67,6 +82,8 @@ let state = {
filter: 'all', filter: 'all',
search: '', search: '',
autoRefresh: true, autoRefresh: true,
view: 'cards',
lastEvent: null,
}; };
// ─── Fetch ────────────────────────────────────────────────────────────────── // ─── Fetch ──────────────────────────────────────────────────────────────────
@@ -84,6 +101,30 @@ async function loadStats() {
document.getElementById('statErrors').textContent = s.errors; document.getElementById('statErrors').textContent = s.errors;
} }
function updateStatsFromEvent(ev) {
if (!ev || !ev.stats) return;
const s = ev.stats;
document.getElementById('statTotal').textContent = s.total;
document.getElementById('statConfirmed').textContent = s.confirmed;
document.getElementById('statPending').textContent = s.pending;
document.getElementById('statErrors').textContent = s.errors;
}
function setView(view) {
state.view = view;
document.getElementById('tabCards').classList.toggle('active', view === 'cards');
document.getElementById('tabGraph').classList.toggle('active', view === 'graph');
document.getElementById('tabStatus').classList.toggle('active', view === 'status');
document.getElementById('cards').style.display = view === 'cards' ? '' : 'none';
document.getElementById('pagination').style.display = view === 'cards' ? '' : 'none';
document.getElementById('graph').style.display = view === 'graph' ? '' : 'none';
document.getElementById('status').style.display = view === 'status' ? '' : 'none';
if (view === 'graph') loadGraph();
if (view === 'status') loadStatus();
}
async function loadCards() { async function loadCards() {
let url = `/api/engrams?limit=${state.limit}&offset=${state.offset}`; let url = `/api/engrams?limit=${state.limit}&offset=${state.offset}`;
if (state.search) url += `&search=${encodeURIComponent(state.search)}`; if (state.search) url += `&search=${encodeURIComponent(state.search)}`;
@@ -99,6 +140,207 @@ async function loadCards() {
document.getElementById('btnNext').disabled = data.items.length < state.limit; document.getElementById('btnNext').disabled = data.items.length < state.limit;
} }
async function loadStatus() {
const [cfg, db, jobs, ins, stor] = await Promise.all([
api('/api/config'),
api('/api/db_info'),
api('/api/jobs'),
api('/api/insights?limit=8'),
api('/api/storage_stats'),
]);
const pend = await api('/api/pending?limit=20&offset=0');
const el = document.getElementById('status');
const jobsHtml = (jobs.items || []).map(j => `
<div class="kv-row">
<div class="kv-key">${j.unit}</div>
<div class="kv-val">${j.error ? ('ERR: ' + j.error) : (j.active + '/' + j.sub)}</div>
</div>
`).join('');
const topTags = (ins.top_tags || []).map(t => `<span class="pill">${t.key} (${t.count})</span>`).join(' ');
const topHosts = (ins.top_hosts || []).map(t => `<span class="pill">${t.key} (${t.count})</span>`).join(' ');
const bySource = Object.entries((stor.sql && stor.sql.by_source) ? stor.sql.by_source : {})
.slice(0, 8)
.map(([k,v]) => `<span class="pill">${k}: ${v}</span>`)
.join(' ');
const pendItems = (pend.items || []);
const pendHtml = pendItems.map(p => `
<div class="kv-row" onclick="showDetail('${p.id}')">
<div class="kv-key">${(p.source||'').slice(0,12)}</div>
<div class="kv-val">
<span class="pill">${p.id.substring(0,8)}</span>
${escapeHtml((p.content||'').substring(0,120))}${(p.content||'').length>120?'…':''}
<div class="actions" style="margin-top:6px" onclick="event.stopPropagation()">
<button class="btn-ok" onclick="confirm('${p.id}', event)">✅</button>
<button class="btn-no" onclick="reject('${p.id}', event)">❌</button>
</div>
</div>
</div>
`).join('');
el.innerHTML = `
<div class="panel">
<div class="panel-title">Config</div>
<div class="kv-row"><div class="kv-key">workspace</div><div class="kv-val">${cfg.workspace}</div></div>
<div class="kv-row"><div class="kv-key">db</div><div class="kv-val">${db.db_path}</div></div>
<div class="kv-row"><div class="kv-key">db mtime</div><div class="kv-val">${new Date(db.mtime).toLocaleString()}</div></div>
</div>
<div class="panel">
<div class="panel-title">Storage</div>
<div class="kv-row"><div class="kv-key">SQL</div><div class="kv-val">${stor.sql.total_engrams} engrams (ok ${stor.sql.confirmed}, pending ${stor.sql.pending})</div></div>
<div class="kv-row"><div class="kv-key">Vector</div><div class="kv-val">chroma ${(stor.vector.chroma_size_bytes/1024/1024).toFixed(1)} MB, cache ${stor.vector.embedding_cache_files} files</div></div>
<div class="kv-row"><div class="kv-key">Obsidian</div><div class="kv-val">${stor.obsidian.configured ? 'configured' : 'not configured'}</div></div>
<div class="kv-row"><div class="kv-key">By source</div><div class="kv-val">${bySource || '-'}</div></div>
</div>
<div class="panel">
<div class="panel-title">Jobs</div>
${jobsHtml || '<div class="muted">Keine Daten</div>'}
</div>
<div class="panel">
<div class="panel-title">Insights</div>
<div class="kv-row"><div class="kv-key">pending</div><div class="kv-val">${ins.pending}</div></div>
<div class="kv-row"><div class="kv-key">top tags</div><div class="kv-val">${topTags || '-'}</div></div>
<div class="kv-row"><div class="kv-key">top hosts</div><div class="kv-val">${topHosts || '-'}</div></div>
</div>
<div class="panel">
<div class="panel-title">Pending Queue (latest)</div>
${pendHtml || '<div class="muted">Keine Pendings</div>'}
</div>
`;
}
async function loadGraph() {
const g = await api('/api/graph?limit_nodes=200');
renderGraph(g.nodes || [], g.edges || []);
}
function renderGraph(nodes, edges) {
const canvas = document.getElementById('graphCanvas');
const hint = document.getElementById('graphHint');
const ctx = canvas.getContext('2d');
// Fit canvas to container width (mobile)
const w = canvas.parentElement.clientWidth - 24;
canvas.width = Math.max(320, Math.min(520, w));
canvas.height = 520;
if (!nodes.length || !edges.length) {
hint.textContent = 'Graph: keine Kanten (noch keine Links/Tags/Hosts im Sample).';
ctx.clearRect(0,0,canvas.width,canvas.height);
return;
}
hint.textContent = `nodes=${nodes.length} edges=${edges.length}`;
const nodeById = new Map(nodes.map(n => [n.id, n]));
const sim = nodes.map(n => ({
id: n.id,
kind: n.kind,
label: n.label || n.id,
x: Math.random()*canvas.width,
y: Math.random()*canvas.height,
vx: 0, vy: 0,
}));
const simById = new Map(sim.map(n => [n.id, n]));
const links = edges
.map(e => ({a: simById.get(e.from), b: simById.get(e.to), kind: e.kind}))
.filter(l => l.a && l.b);
// Simple force layout (few iterations)
for (let iter=0; iter<180; iter++) {
// repulsion
for (let i=0; i<sim.length; i++) {
for (let j=i+1; j<sim.length; j++) {
const a = sim[i], b = sim[j];
const dx = a.x - b.x, dy = a.y - b.y;
const d2 = dx*dx + dy*dy + 0.01;
const f = 120 / d2;
a.vx += dx*f; a.vy += dy*f;
b.vx -= dx*f; b.vy -= dy*f;
}
}
// springs
for (const l of links) {
const a = l.a, b = l.b;
const dx = b.x - a.x, dy = b.y - a.y;
const dist = Math.sqrt(dx*dx + dy*dy) || 1;
const target = 60;
const k = 0.02;
const f = (dist - target) * k;
const fx = (dx/dist)*f, fy = (dy/dist)*f;
a.vx += fx; a.vy += fy;
b.vx -= fx; b.vy -= fy;
}
// integrate + bounds
for (const n of sim) {
n.vx *= 0.85; n.vy *= 0.85;
n.x += n.vx; n.y += n.vy;
n.x = Math.max(10, Math.min(canvas.width-10, n.x));
n.y = Math.max(10, Math.min(canvas.height-10, n.y));
}
}
ctx.clearRect(0,0,canvas.width,canvas.height);
// edges
ctx.globalAlpha = 0.5;
ctx.strokeStyle = '#3a3a55';
ctx.lineWidth = 1;
for (const l of links) {
ctx.beginPath();
ctx.moveTo(l.a.x, l.a.y);
ctx.lineTo(l.b.x, l.b.y);
ctx.stroke();
}
ctx.globalAlpha = 1.0;
// nodes
for (const n of sim) {
let r = 5;
let fill = '#6c8af5';
if (n.kind === 'tag') { fill = '#8a9aff'; r = 4; }
if (n.kind === 'host') { fill = '#f5b46c'; r = 4; }
if (n.kind === 'engram') { fill = '#6c8af5'; r = 5; }
ctx.beginPath();
ctx.fillStyle = fill;
ctx.arc(n.x, n.y, r, 0, Math.PI*2);
ctx.fill();
}
}
// Real-time updates via SSE
function startEvents() {
try {
const es = new EventSource('/api/events');
es.onmessage = (msg) => {
try {
state.lastEvent = JSON.parse(msg.data);
updateStatsFromEvent(state.lastEvent);
if (state.view === 'status') {
// refresh status panels without heavy re-render: just rerun loadStatus occasionally
loadStatus();
}
if (state.view === 'graph') {
// fetch graph less often (every ~15s)
const t = Date.now();
if (!state._lastGraphFetch || (t - state._lastGraphFetch) > 15000) {
state._lastGraphFetch = t;
loadGraph();
}
}
} catch (e) {}
};
es.onerror = () => {
// keep UI usable even if SSE drops
};
} catch (e) {}
}
startEvents();
function renderCards() { function renderCards() {
const el = document.getElementById('cards'); const el = document.getElementById('cards');
el.innerHTML = state.items.map(item => ` el.innerHTML = state.items.map(item => `