9 Commits

Author SHA1 Message Date
7814fa4a65 feat: free-first reaktivierung Teil 2 — torch-freier embedder, Live-Aktivitäten im Graph, systemd-Pfadkorrektur
- src/embedder.py: torch/sentence-transformers durch Hash-Fallback ersetzt
  (kein numpy/ph torch nötig; 384-dim Vektor per sha256+Positional Hash)
- src/store.py: _touch_access() tracked reads für Live-Graph-Aktivität
- fastapi_app.py: SSE-Event-Stream optimiert (2s Takt, Aktivitäts-Tracking)
- templates/dashboard.html: Graph 2.1 mit Live-Aktivitäts-Pulsringen,
  Canvas 16:10, linearem Gradienten, Legendeneinträgen für Lesen/Schreiben/Bewerten
- static/style.css: Grafik-Radius 6px, Activity-Legendenfarben
- openclaw_cron_wrapper.py: CRON_TASKS_DIR ins second-brain-Verzeichnis
- systemd/*.service: Pfade von workspace/ nach second-brain/ korrigiert
- systemd/openclaw-secondbrain.target: neuer Multi-User-Target für Second Brain
- .gitignore: backups/ hinzugefügt

Embedder-Smoke-Test bestanden: encode/encode_batch/similar funktionieren
ohne externe ML-Abhängigkeit.

Refs: #36
2026-06-25 03:03:21 +02:00
f803942914 fix: CLI lazy NeuralScorer import (torch only on demand), health_check use subprocess.run for systemctl 2026-06-25 02:16:20 +02:00
f58c342829 Add Obsidian ingest and export tasks 2026-06-20 14:19:02 +02:00
857d47b4f3 backup: limit local second-brain retention 2026-06-20 11:16:49 +02:00
89dd603629 Build second brain graph 2.0 view 2026-06-05 09:43:24 +02:00
5ebb87db41 Make second brain graph viewport circular 2026-06-05 09:25:13 +02:00
f6edf7cdf2 Fix second brain graph content scaling 2026-06-05 09:11:09 +02:00
51762611c5 Style second brain graph like live cluster map 2026-06-05 09:02:39 +02:00
2f61e900b8 Improve second brain live graph 2026-06-05 08:57:17 +02:00
25 changed files with 1220 additions and 148 deletions

1
.gitignore vendored
View File

@@ -2,3 +2,4 @@ __pycache__/
*.pyc *.pyc
.venv/ .venv/
data/ data/
backups/

View File

@@ -1,5 +1,11 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
"""Backup-Task für Second Brain - isoliert, persistent.""" """Backup-Task fuer Second Brain - isoliert, persistent.
Local retention is intentionally small on the OpenClaw host: keep only the
newest two JSONL exports. Longer-term/off-host retention should be handled by
an encrypted artifact/package workflow, not by committing raw brain exports to
Git.
"""
import json, os, sys import json, os, sys
from pathlib import Path from pathlib import Path
from datetime import datetime, timezone from datetime import datetime, timezone
@@ -8,14 +14,47 @@ BRAIN_DIR = Path("/root/.openclaw/workspace/second-brain")
sys.path.insert(0, str(BRAIN_DIR)) sys.path.insert(0, str(BRAIN_DIR))
from src.store import EngramStore from src.store import EngramStore
DEFAULT_KEEP_LOCAL_BACKUPS = 2
def backup_sort_key(path: Path) -> tuple[float, str]:
return (path.stat().st_mtime, path.name)
def prune_local_backups(data_dir: Path, keep: int = DEFAULT_KEEP_LOCAL_BACKUPS) -> list[str]:
"""Keep only the newest local backup files.
Both uncompressed `.jsonl` and compressed `.jsonl.gz` exports are counted,
so historical compressed backups do not silently accumulate again.
"""
keep = max(1, keep)
backups = sorted(
list(data_dir.glob("backup_*.jsonl")) + list(data_dir.glob("backup_*.jsonl.gz")),
key=backup_sort_key,
reverse=True,
)
removed: list[str] = []
for backup in backups[keep:]:
backup.unlink()
removed.append(str(backup))
return removed
def main(): def main():
brain_db = os.environ.get("BRAIN_DB", str(BRAIN_DIR / "data" / "brain.sqlite")) brain_db = os.environ.get("BRAIN_DB", str(BRAIN_DIR / "data" / "brain.sqlite"))
keep_local = int(os.environ.get("SECOND_BRAIN_BACKUP_KEEP_LOCAL", str(DEFAULT_KEEP_LOCAL_BACKUPS)))
store = EngramStore(brain_db) store = EngramStore(brain_db)
ts = datetime.now(timezone.utc).strftime("%Y%m%d_%H%M%S") ts = datetime.now(timezone.utc).strftime("%Y%m%d_%H%M%S")
backup_path = Path(brain_db).parent / f"backup_{ts}.jsonl" backup_path = Path(brain_db).parent / f"backup_{ts}.jsonl"
count = store.export_jsonl(str(backup_path)) count = store.export_jsonl(str(backup_path))
result = {"timestamp": datetime.now(timezone.utc).isoformat(), "backup_path": str(backup_path), "count": count, "success": True} removed = prune_local_backups(Path(brain_db).parent, keep_local)
result = {
"timestamp": datetime.now(timezone.utc).isoformat(),
"backup_path": str(backup_path),
"count": count,
"removed_old_backups": removed,
"keep_local": keep_local,
"success": True,
}
print(f"BACKUP: {count} Engramme -> {backup_path}") print(f"BACKUP: {count} Engramme -> {backup_path}")
print(json.dumps(result, ensure_ascii=True))
return 0 return 0
if __name__ == "__main__": if __name__ == "__main__":

View File

@@ -0,0 +1,109 @@
#!/usr/bin/env python3
"""Export Second-Brain engrams as Markdown notes into a configured Obsidian vault."""
import hashlib
import json
import os
import re
import sys
from pathlib import Path
BRAIN_DIR = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(BRAIN_DIR))
from src.store import EngramStore
DATA_DIR = BRAIN_DIR / "data"
CONFIG_PATH = DATA_DIR / "obsidian_config.json"
DEFAULT_STATE_PATH = DATA_DIR / "obsidian_export_state.json"
def load_json(path: Path, default: dict) -> dict:
if not path.exists():
return default
try:
return json.loads(path.read_text(encoding="utf-8"))
except Exception:
return default
def write_json(path: Path, data: dict) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(json.dumps(data, ensure_ascii=False, indent=2), encoding="utf-8")
def sha256_text(text: str) -> str:
return hashlib.sha256(text.encode("utf-8")).hexdigest()
def slugify(value: str) -> str:
slug = re.sub(r"[^A-Za-z0-9._-]+", "-", value).strip("-._")
return slug[:80] or "engram"
def render_markdown(eg) -> str:
tags = eg.metadata.get("tags", []) or []
tag_line = " ".join(f"#{slugify(str(t))}" for t in tags if str(t).strip())
return (
"---\n"
f"id: {eg.id}\n"
f"source: {json.dumps(eg.metadata.get('source', ''), ensure_ascii=False)}\n"
f"created: {eg.metadata.get('created', '')}\n"
f"modified: {eg.metadata.get('modified', '')}\n"
f"confidence: {eg.compute_confidence():.3f}\n"
f"tags: {json.dumps(tags, ensure_ascii=False)}\n"
"---\n\n"
f"{tag_line}\n\n"
f"{eg.content.strip()}\n"
)
def main() -> int:
cfg = load_json(CONFIG_PATH, {})
if not cfg.get("enabled", {}).get("export", False):
print(json.dumps({"success": True, "skipped": "export disabled"}))
return 0
vault = Path(cfg.get("vault_path") or "")
if not vault.exists() or not vault.is_dir():
print(json.dumps({"success": True, "skipped": "vault_path missing or invalid", "vault_path": str(vault)}))
return 0
if cfg.get("require_obsidian_dir", True) and not (vault / ".obsidian").is_dir():
print(json.dumps({"success": True, "skipped": "vault is missing .obsidian", "vault_path": str(vault)}))
return 0
export_cfg = cfg.get("export", {})
subdir = export_cfg.get("subdir", "SecondBrain")
max_per_run = int(export_cfg.get("max_per_run", 2000))
state_path_raw = export_cfg.get("state_path")
state_path = (BRAIN_DIR.parent / state_path_raw).resolve() if state_path_raw else DEFAULT_STATE_PATH
state = load_json(state_path, {"version": 1, "hash_by_id": {}})
hashes = dict(state.get("hash_by_id", {}))
out_dir = vault / subdir
out_dir.mkdir(parents=True, exist_ok=True)
store = EngramStore(os.environ.get("BRAIN_DB") or str(DATA_DIR / "brain.sqlite"))
exported = 0
unchanged = 0
for eg in store.get_all(limit=max_per_run):
text = render_markdown(eg)
digest = sha256_text(text)
eg_id = str(eg.id)
if hashes.get(eg_id) == digest:
unchanged += 1
continue
title = slugify(eg.content.splitlines()[0][:60] if eg.content else eg_id)
path = out_dir / f"{title}-{eg_id[:8]}.md"
path.write_text(text, encoding="utf-8")
hashes[eg_id] = digest
exported += 1
write_json(state_path, {"version": 1, "hash_by_id": hashes})
print(json.dumps({"success": True, "exported": exported, "unchanged": unchanged, "target": str(out_dir)}))
return 0
if __name__ == "__main__":
raise SystemExit(main())

View File

@@ -49,17 +49,14 @@ def get_backup_status():
def get_job_status(): def get_job_status():
units = [ units = [
"openclaw-secondbrain-ingest-memory.service", "openclaw-worker@secondbrain_manager.service", # aktueller Worker-Dienst
"openclaw-secondbrain-index-vectors.service", "secondbrain-dashboard.service", # Dashboard
"openclaw-secondbrain-review.service",
"openclaw-secondbrain-heartbeat.service",
"openclaw-secondbrain-verify-pending.service",
] ]
status = {} status = {}
for u in units: for u in units:
try: try:
out = subprocess.check_output(["systemctl", "is-active", u], text=True, stderr=subprocess.DEVNULL).strip() out = subprocess.run(["systemctl", "is-active", u], capture_output=True, text=True, check=False)
status[u] = out status[u] = out.stdout.strip() or "inactive"
except Exception: except Exception:
status[u] = "unknown" status[u] = "unknown"
return status return status

View File

@@ -0,0 +1,122 @@
#!/usr/bin/env python3
"""Import Markdown notes from a configured Obsidian vault into Second-Brain."""
import hashlib
import json
import os
import sys
from pathlib import Path
BRAIN_DIR = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(BRAIN_DIR))
from src.engram import Engram, Grounding
from src.store import EngramStore
DATA_DIR = BRAIN_DIR / "data"
CONFIG_PATH = DATA_DIR / "obsidian_config.json"
STATE_PATH = DATA_DIR / "obsidian_ingest_state.json"
def load_json(path: Path, default: dict) -> dict:
if not path.exists():
return default
try:
return json.loads(path.read_text(encoding="utf-8"))
except Exception:
return default
def write_json(path: Path, data: dict) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(json.dumps(data, ensure_ascii=False, indent=2), encoding="utf-8")
def sha256_text(text: str) -> str:
return hashlib.sha256(text.encode("utf-8")).hexdigest()
def is_excluded(relative: str, patterns: list[str]) -> bool:
p = Path(relative)
for pattern in patterns:
if p.match(pattern) or relative.startswith(pattern.rstrip("*")):
return True
return False
def main() -> int:
cfg = load_json(CONFIG_PATH, {})
if not cfg.get("enabled", {}).get("ingest", False):
print(json.dumps({"success": True, "skipped": "ingest disabled"}))
return 0
vault = Path(cfg.get("vault_path") or "")
if not vault.exists() or not vault.is_dir():
print(json.dumps({"success": True, "skipped": "vault_path missing or invalid", "vault_path": str(vault)}))
return 0
if cfg.get("require_obsidian_dir", True) and not (vault / ".obsidian").is_dir():
print(json.dumps({"success": True, "skipped": "vault is missing .obsidian", "vault_path": str(vault)}))
return 0
ingest_cfg = cfg.get("ingest", {})
include_glob = ingest_cfg.get("include_glob", "**/*.md")
exclude_globs = ingest_cfg.get("exclude_globs", [".obsidian/**", ".trash/**", "SecondBrain/**"])
min_chars = int(ingest_cfg.get("min_chars", 20))
max_files = int(ingest_cfg.get("max_files_per_run", 2000))
max_content_chars = int(ingest_cfg.get("max_content_chars", 4000))
state_max_seen = int(ingest_cfg.get("state_max_seen", 5000))
state = load_json(STATE_PATH, {"version": 2, "files": {}})
seen = dict(state.get("files", {}))
store = EngramStore(os.environ.get("BRAIN_DB") or str(DATA_DIR / "brain.sqlite"))
imported = 0
unchanged = 0
skipped = 0
processed = 0
for path in sorted(vault.glob(include_glob)):
if processed >= max_files:
break
if not path.is_file():
continue
rel = path.relative_to(vault).as_posix()
if is_excluded(rel, exclude_globs):
skipped += 1
continue
processed += 1
try:
text = path.read_text(encoding="utf-8", errors="replace").strip()
except Exception:
skipped += 1
continue
if len(text) < min_chars:
skipped += 1
continue
digest = sha256_text(text)
if seen.get(rel) == digest:
unchanged += 1
continue
content = text[:max_content_chars]
eg = Engram.create(
content=content,
source=f"obsidian:{rel}",
tags=["obsidian", "imported"],
grounding=Grounding.SOURCED,
)
eg.metadata["obsidian_path"] = rel
eg.metadata["obsidian_hash"] = digest
store.save(eg)
seen[rel] = digest
imported += 1
if len(seen) > state_max_seen:
seen = dict(list(seen.items())[-state_max_seen:])
write_json(STATE_PATH, {"version": 2, "files": seen})
print(json.dumps({"success": True, "imported": imported, "unchanged": unchanged, "skipped": skipped, "processed": processed}))
return 0
if __name__ == "__main__":
raise SystemExit(main())

View File

@@ -42,6 +42,9 @@ def create_app() -> FastAPI:
app = create_app() app = create_app()
_ACTIVITY_EVENTS: list[dict] = []
_ACTIVITY_MAX = 250
# ─── Helpers ───────────────────────────────────────────────────────────────── # ─── Helpers ─────────────────────────────────────────────────────────────────
def get_db(): def get_db():
if not DB_PATH.exists(): if not DB_PATH.exists():
@@ -93,6 +96,24 @@ def _now_iso() -> str:
return datetime.now(timezone.utc).isoformat() return datetime.now(timezone.utc).isoformat()
def _record_activity(action: str, engram_id: str | None = None, detail: str | None = None) -> None:
event = {
"ts": _now_iso(),
"action": action,
"engram_id": engram_id,
"detail": detail or "",
}
_ACTIVITY_EVENTS.append(event)
del _ACTIVITY_EVENTS[:-_ACTIVITY_MAX]
def _recent_activity_snapshot(since: str | None = None, limit: int = 25) -> list[dict]:
events = _ACTIVITY_EVENTS[-limit:]
if since:
events = [e for e in events if str(e.get("ts", "")) > since]
return events[-limit:]
def _update_correctness(engram_id: str, *, action: str, reason: str | None = None) -> dict: def _update_correctness(engram_id: str, *, action: str, reason: str | None = None) -> dict:
""" """
Update correctness_json for an engram. action: confirm|reject Update correctness_json for an engram. action: confirm|reject
@@ -156,6 +177,7 @@ def _update_correctness(engram_id: str, *, action: str, reason: str | None = Non
) )
conn.commit() conn.commit()
conn.close() conn.close()
_record_activity("review_confirm" if action == "confirm" else "review_reject", engram_id, reason)
return {"ok": True} return {"ok": True}
@@ -175,6 +197,7 @@ def _bump_access(engram_id: str) -> dict:
) )
conn.commit() conn.commit()
conn.close() conn.close()
_record_activity("read", engram_id, "detail/access")
return {"ok": True} return {"ok": True}
def _safe_json_extract_tags(meta_json: str) -> list[str]: def _safe_json_extract_tags(meta_json: str) -> list[str]:
@@ -591,6 +614,190 @@ def api_graph(
return {"nodes": list(nodes.values()), "edges": edges} return {"nodes": list(nodes.values()), "edges": edges}
def _graph_payload_from_rows(rows: list[sqlite3.Row], link_rows: list[sqlite3.Row]) -> dict:
nodes: dict[str, dict] = {}
edges: list[dict] = []
def add_node(nid: str, kind: str, label: str | None = None, weight: float | None = None):
if nid not in nodes:
nodes[nid] = {"id": nid, "kind": kind}
if label is not None:
nodes[nid]["label"] = label
if weight is not None:
nodes[nid]["weight"] = weight
def add_edge(fr: str, to: str, kind: str, weight: float):
if fr and to and fr != to:
edges.append({"from": fr, "to": to, "kind": kind, "weight": weight})
for r in rows:
eid = r["id"]
try:
meta = json.loads(r["metadata_json"] or "{}")
except Exception:
meta = {}
try:
corr = json.loads(r["correctness_json"] or "{}")
except Exception:
corr = {}
verdict = corr.get("verdict")
if not isinstance(verdict, str) or not verdict:
if corr.get("confirmed", False):
verdict = "confirmed_true"
elif int(corr.get("rejections", 0) or 0) > 0:
verdict = "confirmed_false"
else:
verdict = "unknown"
content = (r["content"] or "").strip()
source = str(meta.get("source", "unknown") or "unknown")
tags = [t for t in _safe_json_extract_tags(r["metadata_json"]) if t.strip()]
primary_cluster = tags[0] if tags else source
add_node(eid, "engram", label=(content[:54] or eid[:8]), weight=float(meta.get("access_count", 0) or 0))
nodes[eid].update(
{
"source": source,
"cluster": primary_cluster,
"tags": tags[:8],
"confidence": float(meta.get("confidence", 0.0) or 0.0),
"created": meta.get("created", r["created_at"]),
"modified": meta.get("modified", r["modified_at"]),
"last_accessed": meta.get("last_accessed"),
"verdict": verdict,
"confirmed": bool(corr.get("confirmed", False)),
"rejections": int(corr.get("rejections", 0) or 0),
}
)
sid = f"source:{source}"
add_node(sid, "source", label=source, weight=5)
nodes[sid]["cluster"] = source
add_edge(eid, sid, "from_source", 0.45)
for t in tags:
tid = f"tag:{t}"
add_node(tid, "tag", label=t, weight=2)
nodes[tid]["cluster"] = t
add_edge(eid, tid, "has_tag", 0.35)
host = _host_from_meta(r["metadata_json"])
if host:
hid = f"host:{host}"
add_node(hid, "host", label=host, weight=3)
nodes[hid]["cluster"] = source
add_edge(eid, hid, "grounded_at", 0.25)
for lr in link_rows:
fr = lr["from_id"]
to = lr["to_id"]
add_node(fr, "engram", label=fr[:8])
add_node(to, "engram", label=to[:8])
add_edge(fr, to, "link", 1.0)
return {"nodes": list(nodes.values()), "edges": edges}
@app.get("/api/graph_chunk")
def api_graph_chunk(
offset: int = Query(0, ge=0),
limit: int = Query(600, ge=50, le=2500),
):
"""
Incremental graph payload. The dashboard can render immediately, then keep
adding chunks until every SQL/RAG/Obsidian-imported engram is visible.
"""
conn = get_db()
c = conn.cursor()
total = c.execute("SELECT COUNT(*) FROM engrams").fetchone()[0]
max_modified = c.execute("SELECT MAX(modified_at) FROM engrams").fetchone()[0]
rows = c.execute(
"""
SELECT id, content, metadata_json, correctness_json, created_at, modified_at
FROM engrams
ORDER BY created_at DESC
LIMIT ? OFFSET ?
""",
(limit, offset),
).fetchall()
ids = [r["id"] for r in rows]
link_rows: list[sqlite3.Row] = []
if ids:
placeholders = ",".join("?" * len(ids))
link_rows = c.execute(
f"""
SELECT from_id, to_id FROM engrams_links
WHERE from_id IN ({placeholders}) OR to_id IN ({placeholders})
LIMIT 20000
""",
ids + ids,
).fetchall()
conn.close()
payload = _graph_payload_from_rows(rows, link_rows)
payload.update(
{
"offset": offset,
"limit": limit,
"next_offset": offset + len(rows),
"done": offset + len(rows) >= total,
"total_engrams": total,
"max_modified": max_modified,
}
)
return payload
@app.get("/api/graph_changes")
def api_graph_changes(
since: str = Query(""),
limit: int = Query(300, ge=20, le=2000),
):
"""
Lightweight live deltas for the graph. Called from SSE ticks so the canvas
updates without reloading the whole graph.
"""
conn = get_db()
c = conn.cursor()
if since:
rows = c.execute(
"""
SELECT id, content, metadata_json, correctness_json, created_at, modified_at
FROM engrams
WHERE modified_at > ?
ORDER BY modified_at DESC
LIMIT ?
""",
(since, limit),
).fetchall()
else:
rows = c.execute(
"""
SELECT id, content, metadata_json, correctness_json, created_at, modified_at
FROM engrams
ORDER BY modified_at DESC
LIMIT ?
""",
(limit,),
).fetchall()
ids = [r["id"] for r in rows]
link_rows: list[sqlite3.Row] = []
if ids:
placeholders = ",".join("?" * len(ids))
link_rows = c.execute(
f"""
SELECT from_id, to_id FROM engrams_links
WHERE from_id IN ({placeholders}) OR to_id IN ({placeholders})
LIMIT 10000
""",
ids + ids,
).fetchall()
max_modified = c.execute("SELECT MAX(modified_at) FROM engrams").fetchone()[0]
conn.close()
payload = _graph_payload_from_rows(rows, link_rows)
payload.update({"count": len(rows), "max_modified": max_modified})
return payload
@app.get("/api/events") @app.get("/api/events")
def api_events(): def api_events():
""" """
@@ -599,16 +806,21 @@ def api_events():
import time import time
def gen(): def gen():
tick = 0
while True: while True:
tick += 1
payload = { payload = {
"ts": datetime.now(timezone.utc).isoformat(), "ts": datetime.now(timezone.utc).isoformat(),
"stats": api_stats(), "stats": api_stats(),
"storage": api_storage_stats(), "activity": _recent_activity_snapshot(limit=25),
"jobs": api_jobs(),
"insights": api_insights(limit=8),
} }
# Systemd and storage snapshots are useful but too expensive to run
# every live tick on large brains. Send them at a slower cadence.
if tick == 1 or tick % 8 == 0:
payload["jobs"] = api_jobs()
payload["insights"] = api_insights(limit=8)
yield f"data: {json.dumps(payload, ensure_ascii=False)}\n\n" yield f"data: {json.dumps(payload, ensure_ascii=False)}\n\n"
time.sleep(5) time.sleep(2)
return StreamingResponse(gen(), media_type="text/event-stream") return StreamingResponse(gen(), media_type="text/event-stream")
@@ -770,6 +982,7 @@ def api_engram_detail(engram_id: str):
result = parse_engram(row) result = parse_engram(row)
result["links"] = [r[0] for r in links] result["links"] = [r[0] for r in links]
conn.close() conn.close()
_record_activity("read", engram_id, "detail")
return result return result
@@ -849,6 +1062,7 @@ def api_create_engram(content: str = Form(...), tags: str = Form("")):
) )
conn.commit() conn.commit()
conn.close() conn.close()
_record_activity("write", engram_id, content[:80])
return {"id": engram_id} return {"id": engram_id}
@@ -905,6 +1119,7 @@ def api_refresh(engram_id: str):
) )
conn.commit() conn.commit()
conn.close() conn.close()
_record_activity("score_refresh", engram_id, f"confidence {round(conf, 2)}")
return {"success": True, "new_confidence": round(conf, 2)} return {"success": True, "new_confidence": round(conf, 2)}
@@ -930,6 +1145,7 @@ def api_accept_link(from_id: str = Form(...), to_id: str = Form(...)):
) )
conn.commit() conn.commit()
conn.close() conn.close()
_record_activity("write_link", from_id, to_id)
return {"ok": True} return {"ok": True}
@@ -970,6 +1186,7 @@ def api_create_engram(content: str = Form(...), tags: str = Form(""), source: st
) )
conn.commit() conn.commit()
conn.close() conn.close()
_record_activity("write", engram_id, content[:80])
return {"success": True, "engram_id": engram_id} return {"success": True, "engram_id": engram_id}

View File

@@ -14,9 +14,9 @@ from datetime import datetime, timezone
# --- Konfiguration (persistent) --- # --- Konfiguration (persistent) ---
WORKSPACE = Path("/root/.openclaw/workspace") WORKSPACE = Path("/root/.openclaw/workspace")
CRON_TASKS_DIR = WORKSPACE / "cron_tasks"
LOG_FILE = WORKSPACE / "cron_wrapper.log"
BRAIN_DIR = WORKSPACE / "second-brain" BRAIN_DIR = WORKSPACE / "second-brain"
CRON_TASKS_DIR = BRAIN_DIR / "cron_tasks"
LOG_FILE = WORKSPACE / "cron_wrapper.log"
def log(msg: str): def log(msg: str):

View File

@@ -30,7 +30,6 @@ from .engram import Engram, Grounding
from .retriever import Retriever from .retriever import Retriever
from .chroma_store import ChromaStore from .chroma_store import ChromaStore
from .graph_view import generate_graph_html from .graph_view import generate_graph_html
from .neural_scorer import NeuralScorer
from .loop_detector import LoopDetector from .loop_detector import LoopDetector
from .error_healer import ErrorHealer from .error_healer import ErrorHealer
@@ -209,6 +208,7 @@ def cmd_heal(args):
def cmd_neural_train(args): def cmd_neural_train(args):
from .neural_scorer import NeuralScorer # lazy: needs torch
store = get_store() store = get_store()
scorer = NeuralScorer() scorer = NeuralScorer()
egs = store.get_all(limit=10000) egs = store.get_all(limit=10000)

View File

@@ -4,22 +4,22 @@ Offlined-fähig, cached auf Disk.
""" """
import json import json
import hashlib import hashlib
import os import math
from pathlib import Path from pathlib import Path
from typing import List, Optional from typing import List, Optional
import numpy as np
from sentence_transformers import SentenceTransformer
_MODEL_NAME = "all-MiniLM-L6-v2" _MODEL_NAME = "all-MiniLM-L6-v2"
_EMBED_DIM = 384 _EMBED_DIM = 384
_CACHE_DIR = Path(__file__).resolve().parent.parent / "data" / "embedding_cache" _CACHE_DIR = Path(__file__).resolve().parent.parent / "data" / "embedding_cache"
__model: Optional[SentenceTransformer] = None __model = None
__fallback_warned = False
def _get_model() -> SentenceTransformer: def _get_model():
global __model global __model
if __model is None: if __model is None:
from sentence_transformers import SentenceTransformer
__model = SentenceTransformer(_MODEL_NAME) __model = SentenceTransformer(_MODEL_NAME)
return __model return __model
@@ -33,6 +33,39 @@ def _cache_path(h: str) -> Path:
return _CACHE_DIR / f"{h}.json" return _CACHE_DIR / f"{h}.json"
def _normalize(vec: List[float]) -> List[float]:
norm = math.sqrt(sum(x * x for x in vec))
if norm <= 0:
return vec
return [x / norm for x in vec]
def _hash_encode(text: str, normalize: bool = True) -> List[float]:
"""
Lightweight deterministic fallback for small/edge hosts where torch is not
installed. It is weaker than a transformer embedding, but keeps vector
indexing and graph/search plumbing alive without pulling GPU-sized wheels.
"""
vec = [0.0] * _EMBED_DIM
words = [w for w in text.lower().split() if w]
if not words:
words = [text]
for word in words:
digest = hashlib.sha256(word.encode("utf-8")).digest()
for i, b in enumerate(digest):
idx = (b + i * 131) % _EMBED_DIM
sign = 1.0 if (b & 1) else -1.0
vec[idx] += sign
return _normalize(vec) if normalize else vec
def _warn_fallback(message: str) -> None:
global __fallback_warned
if not __fallback_warned:
print(message)
__fallback_warned = True
def encode(text: str, cache: bool = True, normalize: bool = True) -> Optional[List[float]]: def encode(text: str, cache: bool = True, normalize: bool = True) -> Optional[List[float]]:
"""Embeddiert einen Text. Gibt None zurück wenn Modell nicht verfügbar.""" """Embeddiert einen Text. Gibt None zurück wenn Modell nicht verfügbar."""
try: try:
@@ -43,13 +76,15 @@ def encode(text: str, cache: bool = True, normalize: bool = True) -> Optional[Li
data = json.load(f) data = json.load(f)
return data["embedding"] return data["embedding"]
model = _get_model() try:
vec = model.encode(text, convert_to_numpy=True) model = _get_model()
if normalize: vec = model.encode(text, convert_to_numpy=True)
norm = np.linalg.norm(vec) vec_list = vec.tolist()
if norm > 0: if normalize:
vec = vec / norm vec_list = _normalize([float(x) for x in vec_list])
vec_list = vec.tolist() except Exception as e:
_warn_fallback(f"[embedder] transformer unavailable, using hash fallback: {e}")
vec_list = _hash_encode(text, normalize=normalize)
if cache: if cache:
with open(cp, "w", encoding="utf-8") as f: with open(cp, "w", encoding="utf-8") as f:
@@ -81,14 +116,16 @@ def encode_batch(texts: List[str], cache: bool = True, normalize: bool = True) -
idx_map.append(i) idx_map.append(i)
if to_encode: if to_encode:
model = _get_model() try:
vecs = model.encode(to_encode, convert_to_numpy=True) model = _get_model()
if normalize: vecs = model.encode(to_encode, convert_to_numpy=True)
norms = np.linalg.norm(vecs, axis=1, keepdims=True) encoded = [vec.tolist() for vec in vecs]
norms[norms == 0] = 1 if normalize:
vecs = vecs / norms encoded = [_normalize([float(x) for x in vec]) for vec in encoded]
for m, vec in zip(idx_map, vecs): except Exception as e:
vec_list = vec.tolist() _warn_fallback(f"[embedder] batch transformer unavailable, using hash fallback: {e}")
encoded = [_hash_encode(text, normalize=normalize) for text in to_encode]
for m, vec_list in zip(idx_map, encoded):
results[m] = vec_list results[m] = vec_list
if cache: if cache:
h = _text_hash(texts[m]) h = _text_hash(texts[m])
@@ -104,13 +141,12 @@ def encode_batch(texts: List[str], cache: bool = True, normalize: bool = True) -
def similar(query: str, candidates: List[str], top_k: int = 5) -> List[tuple]: def similar(query: str, candidates: List[str], top_k: int = 5) -> List[tuple]:
"""Gibt die top-k besten Kandidaten für eine Query zurück.""" """Gibt die top-k besten Kandidaten für eine Query zurück."""
q_vec = np.array(encode(query)) q_vec = encode(query) or []
c_vecs = encode_batch(candidates) c_vecs = encode_batch(candidates)
scores = [] scores = []
for i, c_vec in enumerate(c_vecs): for i, c_vec in enumerate(c_vecs):
if c_vec is not None: if c_vec is not None:
c_arr = np.array(c_vec) score = float(sum(a * b for a, b in zip(q_vec, c_vec)))
score = float(np.dot(q_vec, c_arr))
scores.append((i, score)) scores.append((i, score))
scores.sort(key=lambda x: x[1], reverse=True) scores.sort(key=lambda x: x[1], reverse=True)
return [(candidates[i], s) for i, s in scores[:top_k]] return [(candidates[i], s) for i, s in scores[:top_k]]

View File

@@ -161,6 +161,7 @@ class EngramStore:
).fetchone() ).fetchone()
if not row: if not row:
return None return None
self._touch_access([engram_id])
return self._row_to_engram(row) return self._row_to_engram(row)
def get_all(self, limit: int = 1000, offset: int = 0) -> List[Engram]: def get_all(self, limit: int = 1000, offset: int = 0) -> List[Engram]:
@@ -276,6 +277,7 @@ class EngramStore:
LIMIT ? LIMIT ?
""" """
rows = self._conn.execute(sql, (safe_query, limit)).fetchall() rows = self._conn.execute(sql, (safe_query, limit)).fetchall()
self._touch_access([r["id"] for r in rows])
return [self._row_to_engram(r) for r in rows] return [self._row_to_engram(r) for r in rows]
def search_tag(self, tag: str, limit: int = 50) -> List[Engram]: def search_tag(self, tag: str, limit: int = 50) -> List[Engram]:
@@ -285,6 +287,7 @@ class EngramStore:
"SELECT * FROM engrams WHERE metadata_json LIKE ? ORDER BY created_at DESC LIMIT ?", "SELECT * FROM engrams WHERE metadata_json LIKE ? ORDER BY created_at DESC LIMIT ?",
(f'%"{tag}"%', limit) (f'%"{tag}"%', limit)
).fetchall() ).fetchall()
self._touch_access([r["id"] for r in rows])
return [self._row_to_engram(r) for r in rows] return [self._row_to_engram(r) for r in rows]
def search_source(self, source: str, limit: int = 50) -> List[Engram]: def search_source(self, source: str, limit: int = 50) -> List[Engram]:
@@ -293,6 +296,7 @@ class EngramStore:
"SELECT * FROM engrams WHERE metadata_json LIKE ? ORDER BY created_at DESC LIMIT ?", "SELECT * FROM engrams WHERE metadata_json LIKE ? ORDER BY created_at DESC LIMIT ?",
(f'%"source": "{source}"%', limit) (f'%"source": "{source}"%', limit)
).fetchall() ).fetchall()
self._touch_access([r["id"] for r in rows])
return [self._row_to_engram(r) for r in rows] return [self._row_to_engram(r) for r in rows]
# ---- Stats ---- # ---- Stats ----
@@ -367,6 +371,32 @@ class EngramStore:
d["embedding"] = json.loads(emb) d["embedding"] = json.loads(emb)
return Engram.from_dict(d) return Engram.from_dict(d)
def _touch_access(self, engram_ids: List[str]) -> None:
"""Mark reads so live graph deltas can show agent/user activity."""
ids = [str(x) for x in engram_ids if x]
if not ids:
return
now = _now()
for engram_id in ids[:100]:
row = self._conn.execute(
"SELECT metadata_json FROM engrams WHERE id=?",
(engram_id,),
).fetchone()
if not row:
continue
try:
meta = json.loads(row["metadata_json"] or "{}")
except Exception:
meta = {}
meta["access_count"] = int(meta.get("access_count", 0) or 0) + 1
meta["last_accessed"] = now
meta["modified"] = now
self._conn.execute(
"UPDATE engrams SET metadata_json=?, modified_at=? WHERE id=?",
(json.dumps(meta, ensure_ascii=False), now, engram_id),
)
self._conn.commit()
def close(self) -> None: def close(self) -> None:
self._conn.close() self._conn.close()

View File

@@ -10,10 +10,10 @@ body {
} }
.app { .app {
max-width: 480px; max-width: 1180px;
margin: 0 auto; margin: 0 auto;
min-height: 100vh; min-height: 100vh;
background: #141419; background: radial-gradient(circle at 50% 12%, #162238 0%, #11131d 42%, #0b0d13 100%);
width: 100%; width: 100%;
} }
@@ -165,10 +165,18 @@ body {
/* Graph canvas */ /* Graph canvas */
#graphCanvas{ #graphCanvas{
display:block; display:block;
margin: 8px auto 0; margin: 10px auto 0;
background:#12121a; background:#01030a;
border:1px solid #252533; border:0;
border-radius: 14px; border-radius: 6px;
width: min(100%, 1180px);
max-width: calc(100vw - 24px);
height: auto;
aspect-ratio: 16 / 10;
box-shadow:
0 0 48px rgba(34,211,238,0.14),
0 0 120px rgba(236,72,153,0.08),
inset 0 0 56px rgba(124,58,237,0.10);
touch-action: none; touch-action: none;
} }
@@ -180,8 +188,8 @@ body {
flex-wrap: wrap; flex-wrap: wrap;
} }
.graph-controls .btn{ .graph-controls .btn{
background:#1e1e28; background:rgba(12,18,31,0.88);
border:1px solid #2a2a3a; border:1px solid rgba(74,94,130,0.55);
border-radius: 10px; border-radius: 10px;
padding: 8px 10px; padding: 8px 10px;
color:#cfd3ff; color:#cfd3ff;
@@ -192,11 +200,20 @@ body {
border-color:#6c8af5; border-color:#6c8af5;
box-shadow:0 0 0 1px rgba(108,138,245,0.18) inset; box-shadow:0 0 0 1px rgba(108,138,245,0.18) inset;
} }
.graph-mode{
color:#a7f3d0;
font-size:0.78rem;
font-weight:700;
padding: 6px 8px;
border:1px solid rgba(52,211,153,0.35);
background:rgba(4,18,20,0.82);
border-radius: 8px;
}
.graph-legend{ .graph-legend{
margin: 8px 12px 0; margin: 8px 12px 0;
padding: 10px 12px; padding: 10px 12px;
background:#1a1a24; background:rgba(11,15,25,0.72);
border:1px solid #252533; border:1px solid rgba(65,78,112,0.4);
border-radius: 14px; border-radius: 14px;
color:#b9b9c9; color:#b9b9c9;
font-size:0.8rem; font-size:0.8rem;
@@ -206,8 +223,64 @@ body {
.legend-dot{ width:10px; height:10px; border-radius:50%; display:inline-block; } .legend-dot{ width:10px; height:10px; border-radius:50%; display:inline-block; }
.legend-dot.engram{ background:#6c8af5; } .legend-dot.engram{ background:#6c8af5; }
.legend-dot.tag{ background:#8a9aff; } .legend-dot.tag{ background:#8a9aff; }
.legend-dot.source{ background:#14b8a6; }
.legend-dot.match{ background:#f7d154; } .legend-dot.match{ background:#f7d154; }
.legend-dot.read{ background:#38bdf8; }
.legend-dot.write{ background:#4ade80; }
.legend-dot.review{ background:#f7d154; }
.graph-hint{ padding: 4px 12px 10px; } .graph-hint{ padding: 4px 12px 10px; }
.graph-live{
margin: 8px 12px 0;
padding: 10px 12px;
background:rgba(5,12,20,0.78);
border:1px solid rgba(56,189,248,0.22);
border-radius: 10px;
color:#bfe8df;
font-size:0.78rem;
}
.graph-insights{
display:grid;
grid-template-columns: repeat(auto-fit, minmax(120px, 1fr));
gap: 8px;
margin: 8px 12px 0;
}
.graph-chip{
min-height: 44px;
padding: 8px 10px;
border: 1px solid rgba(80,96,130,0.42);
border-radius: 8px;
background: rgba(10,15,26,0.72);
color: #dbeafe;
overflow: hidden;
}
.graph-chip b{
display:block;
font-size: 0.78rem;
white-space: nowrap;
overflow: hidden;
text-overflow: ellipsis;
}
.graph-chip span{
display:block;
color:#8aa0c7;
font-size:0.72rem;
margin-top:2px;
}
@media (max-width: 560px) {
.app { max-width: 100%; }
.graph-controls .btn { padding: 8px 9px; }
}
.graph-live-title{
color:#e8fffb;
font-weight:700;
margin-bottom:4px;
}
.graph-live-feed{
display:grid;
gap:3px;
min-height: 22px;
}
#searchInput { #searchInput {
width: 100%; width: 100%;
flex: 1; flex: 1;
@@ -385,7 +458,7 @@ body {
.actions button { .actions button {
width: 34px; width: 34px;
height: 34px; height: 34px;
border-radius: 50%; border-radius: 10px;
border: none; border: none;
font-size: 1rem; font-size: 1rem;
cursor: pointer; cursor: pointer;
@@ -436,7 +509,7 @@ body {
.refresh-btn { .refresh-btn {
background: #252535; background: #252535;
border: none; border: none;
border-radius: 50%; border-radius: 10px;
width: 36px; width: 36px;
height: 36px; height: 36px;
color: #8888aa; color: #8888aa;

View File

@@ -5,5 +5,5 @@ OnFailure=openclaw-secondbrain-notify@%n.service
[Service] [Service]
Type=oneshot Type=oneshot
WorkingDirectory=/root/.openclaw/workspace WorkingDirectory=/root/.openclaw/workspace
ExecStart=/bin/bash -lc 'flock -n /tmp/%n.lock /usr/bin/python3 /root/.openclaw/workspace/openclaw_cron_wrapper.py archive_memory_md' ExecStart=/bin/bash -lc 'flock -n /tmp/%n.lock /usr/bin/python3 /root/.openclaw/workspace/second-brain/openclaw_cron_wrapper.py archive_memory_md'

View File

@@ -5,4 +5,4 @@ OnFailure=openclaw-secondbrain-notify@%n.service
[Service] [Service]
Type=oneshot Type=oneshot
WorkingDirectory=/root/.openclaw/workspace WorkingDirectory=/root/.openclaw/workspace
ExecStart=/bin/bash -lc 'flock -n /tmp/%n.lock /usr/bin/python3 /root/.openclaw/workspace/openclaw_cron_wrapper.py backup_secondbrain' ExecStart=/bin/bash -lc 'flock -n /tmp/%n.lock /usr/bin/python3 /root/.openclaw/workspace/second-brain/openclaw_cron_wrapper.py backup_secondbrain'

View File

@@ -5,4 +5,4 @@ OnFailure=openclaw-secondbrain-notify@%n.service
[Service] [Service]
Type=oneshot Type=oneshot
WorkingDirectory=/root/.openclaw/workspace WorkingDirectory=/root/.openclaw/workspace
ExecStart=/bin/bash -lc 'flock -n /tmp/%n.lock /usr/bin/python3 /root/.openclaw/workspace/openclaw_cron_wrapper.py export_obsidian' ExecStart=/bin/bash -lc 'flock -n /tmp/%n.lock /usr/bin/python3 /root/.openclaw/workspace/second-brain/openclaw_cron_wrapper.py export_obsidian'

View File

@@ -5,4 +5,4 @@ OnFailure=openclaw-secondbrain-notify@%n.service
[Service] [Service]
Type=oneshot Type=oneshot
WorkingDirectory=/root/.openclaw/workspace WorkingDirectory=/root/.openclaw/workspace
ExecStart=/bin/bash -lc 'flock -n /tmp/%n.lock /usr/bin/python3 /root/.openclaw/workspace/openclaw_cron_wrapper.py heartbeat_secondbrain' ExecStart=/bin/bash -lc 'flock -n /tmp/%n.lock /usr/bin/python3 /root/.openclaw/workspace/second-brain/openclaw_cron_wrapper.py heartbeat_secondbrain'

View File

@@ -7,4 +7,4 @@ Type=oneshot
WorkingDirectory=/root/.openclaw/workspace WorkingDirectory=/root/.openclaw/workspace
Environment=HF_HOME=/root/.openclaw/workspace/second-brain/data/hf_cache Environment=HF_HOME=/root/.openclaw/workspace/second-brain/data/hf_cache
Environment=SENTENCE_TRANSFORMERS_HOME=/root/.openclaw/workspace/second-brain/data/st_cache Environment=SENTENCE_TRANSFORMERS_HOME=/root/.openclaw/workspace/second-brain/data/st_cache
ExecStart=/bin/bash -lc 'flock -n /tmp/%n.lock /usr/bin/python3 /root/.openclaw/workspace/openclaw_cron_wrapper.py index_vectors' ExecStart=/bin/bash -lc 'flock -n /tmp/%n.lock /root/.openclaw/workspace/second-brain/.venv/bin/python /root/.openclaw/workspace/second-brain/openclaw_cron_wrapper.py index_vectors'

View File

@@ -5,4 +5,4 @@ OnFailure=openclaw-secondbrain-notify@%n.service
[Service] [Service]
Type=oneshot Type=oneshot
WorkingDirectory=/root/.openclaw/workspace WorkingDirectory=/root/.openclaw/workspace
ExecStart=/bin/bash -lc 'flock -n /tmp/%n.lock /usr/bin/python3 /root/.openclaw/workspace/openclaw_cron_wrapper.py ingest_memory; code=$?; [ "$code" -eq 1 ] && exit 0; exit "$code"' ExecStart=/bin/bash -lc 'flock -n /tmp/%n.lock /usr/bin/python3 /root/.openclaw/workspace/second-brain/openclaw_cron_wrapper.py ingest_memory; code=$?; [ "$code" -eq 1 ] && exit 0; exit "$code"'

View File

@@ -5,4 +5,4 @@ OnFailure=openclaw-secondbrain-notify@%n.service
[Service] [Service]
Type=oneshot Type=oneshot
WorkingDirectory=/root/.openclaw/workspace WorkingDirectory=/root/.openclaw/workspace
ExecStart=/bin/bash -lc 'flock -n /tmp/%n.lock /usr/bin/python3 /root/.openclaw/workspace/openclaw_cron_wrapper.py ingest_obsidian' ExecStart=/bin/bash -lc 'flock -n /tmp/%n.lock /usr/bin/python3 /root/.openclaw/workspace/second-brain/openclaw_cron_wrapper.py ingest_obsidian'

View File

@@ -5,5 +5,5 @@ OnFailure=openclaw-secondbrain-notify@%n.service
[Service] [Service]
Type=oneshot Type=oneshot
WorkingDirectory=/root/.openclaw/workspace WorkingDirectory=/root/.openclaw/workspace
ExecStart=/bin/bash -lc 'flock -n /tmp/%n.lock /usr/bin/python3 /root/.openclaw/workspace/openclaw_cron_wrapper.py ingest_transcript_to_db' ExecStart=/bin/bash -lc 'flock -n /tmp/%n.lock /usr/bin/python3 /root/.openclaw/workspace/second-brain/openclaw_cron_wrapper.py ingest_transcript_to_db'

View File

@@ -8,4 +8,4 @@ OnFailure=openclaw-secondbrain-notify@%n.service
Type=oneshot Type=oneshot
WorkingDirectory=/root/.openclaw/workspace WorkingDirectory=/root/.openclaw/workspace
Environment=LLM_PROXY_MODEL=stepfun-ai/Step-3.5-Flash Environment=LLM_PROXY_MODEL=stepfun-ai/Step-3.5-Flash
ExecStart=/bin/bash -lc 'flock -n /tmp/%n.lock /usr/bin/python3 /root/.openclaw/workspace/openclaw_cron_wrapper.py proactive_search_wrapper' ExecStart=/bin/bash -lc 'flock -n /tmp/%n.lock /usr/bin/python3 /root/.openclaw/workspace/second-brain/openclaw_cron_wrapper.py proactive_search_wrapper'

View File

@@ -5,4 +5,4 @@ OnFailure=openclaw-secondbrain-notify@%n.service
[Service] [Service]
Type=oneshot Type=oneshot
WorkingDirectory=/root/.openclaw/workspace WorkingDirectory=/root/.openclaw/workspace
ExecStart=/bin/bash -lc 'flock -n /tmp/%n.lock /usr/bin/python3 /root/.openclaw/workspace/openclaw_cron_wrapper.py review_brain' ExecStart=/bin/bash -lc 'flock -n /tmp/%n.lock /usr/bin/python3 /root/.openclaw/workspace/second-brain/openclaw_cron_wrapper.py auto_assign_review'

View File

@@ -6,7 +6,7 @@ After=network-online.target
[Service] [Service]
Type=oneshot Type=oneshot
WorkingDirectory=/root/.openclaw/workspace WorkingDirectory=/root/.openclaw/workspace
ExecStart=/usr/bin/python3 /root/.openclaw/workspace/openclaw_cron_wrapper.py %i ExecStart=/usr/bin/python3 /root/.openclaw/workspace/second-brain/openclaw_cron_wrapper.py %i
Nice=10 Nice=10
IOSchedulingClass=best-effort IOSchedulingClass=best-effort
IOSchedulingPriority=6 IOSchedulingPriority=6

View File

@@ -5,5 +5,5 @@ OnFailure=openclaw-secondbrain-notify@%n.service
[Service] [Service]
Type=oneshot Type=oneshot
WorkingDirectory=/root/.openclaw/workspace WorkingDirectory=/root/.openclaw/workspace
ExecStart=/bin/bash -lc 'flock -n /tmp/%n.lock /usr/bin/python3 /root/.openclaw/workspace/openclaw_cron_wrapper.py verify_pending_external' ExecStart=/bin/bash -lc 'flock -n /tmp/%n.lock /usr/bin/python3 /root/.openclaw/workspace/second-brain/openclaw_cron_wrapper.py verify_pending_external'

View File

@@ -0,0 +1,7 @@
[Unit]
Description=OpenClaw Second-Brain services
Wants=openclaw-secondbrain-dashboard.service
After=network-online.target
[Install]
WantedBy=multi-user.target

View File

@@ -4,7 +4,7 @@
<meta charset="UTF-8"> <meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0, maximum-scale=1.0, user-scalable=no"> <meta name="viewport" content="width=device-width, initial-scale=1.0, maximum-scale=1.0, user-scalable=no">
<title>🧠 Second Brain</title> <title>🧠 Second Brain</title>
<link rel="stylesheet" href="/static/style.css"> <link rel="stylesheet" href="/static/style.css?v=20260610-live-graph">
</head> </head>
<body> <body>
<div class="app"> <div class="app">
@@ -58,14 +58,8 @@
<!-- Graph --> <!-- Graph -->
<div class="graph" id="graph" style="display:none;"> <div class="graph" id="graph" style="display:none;">
<div class="graph-controls"> <div class="graph-controls">
<button class="btn primary" id="btnGraphPhysics" onclick="toggleGraphPhysics()">Physics: off</button>
<button class="btn" onclick="resetGraphView()">Reset view</button> <button class="btn" onclick="resetGraphView()">Reset view</button>
<button class="btn" onclick="fitGraphView()">Fit</button> <button class="btn" onclick="fitGraphView()">Fit</button>
<label class="muted small" style="display:flex;align-items:center;gap:8px">
<span>Physics</span>
<input id="physicsStrength" type="range" min="0" max="100" value="60" oninput="setPhysicsStrength(this.value)" style="width:140px">
<span id="physicsStrengthVal">60</span>
</label>
<select class="btn" id="graphLimit" onchange="reloadGraph()" title="Wie viele Knoten laden? 0=all"> <select class="btn" id="graphLimit" onchange="reloadGraph()" title="Wie viele Knoten laden? 0=all">
<option value="0">Nodes: all</option> <option value="0">Nodes: all</option>
<option value="200">Nodes: 200</option> <option value="200">Nodes: 200</option>
@@ -73,14 +67,24 @@
<option value="5000">Nodes: 5000</option> <option value="5000">Nodes: 5000</option>
</select> </select>
<button class="btn" onclick="reloadGraph()">Reload</button> <button class="btn" onclick="reloadGraph()">Reload</button>
<span class="graph-mode" id="graphMode">Graph 2.1 live</span>
</div> </div>
<canvas id="graphCanvas" width="440" height="520"></canvas> <canvas id="graphCanvas" width="440" height="520"></canvas>
<div class="graph-hint muted small" id="graphHint">Lade Graph…</div> <div class="graph-hint muted small" id="graphHint">Lade Graph…</div>
<div class="graph-insights" id="graphInsights"></div>
<div class="graph-live" id="graphLive">
<div class="graph-live-title">Live</div>
<div id="graphLiveFeed" class="graph-live-feed"></div>
</div>
<div class="graph-legend"> <div class="graph-legend">
<div><strong>Graph</strong>: Zoom per Pinch (2 Finger), Pan per Drag (1 Finger). Tap auf Engram öffnet Details, Tap auf Tag setzt Suche.</div> <div><strong>Graph 2.0</strong>: Topic-Cluster, Brücken und Live-Deltas. Zoom per Pinch, Pan per Drag. Tap auf Engram öffnet Details, Tap auf Tag setzt Suche.</div>
<div class="legend-row"><span class="legend-dot engram"></span> Engram</div> <div class="legend-row"><span class="legend-dot engram"></span> Engram</div>
<div class="legend-row"><span class="legend-dot tag"></span> Tag</div> <div class="legend-row"><span class="legend-dot tag"></span> Tag</div>
<div class="legend-row"><span class="legend-dot source"></span> Quelle</div>
<div class="legend-row"><span class="legend-dot match"></span> Match (Suche)</div> <div class="legend-row"><span class="legend-dot match"></span> Match (Suche)</div>
<div class="legend-row"><span class="legend-dot read"></span> Lesen live</div>
<div class="legend-row"><span class="legend-dot write"></span> Schreiben live</div>
<div class="legend-row"><span class="legend-dot review"></span> Bewerten live</div>
</div> </div>
</div> </div>
@@ -172,7 +176,12 @@ function setView(view) {
document.getElementById('graph').style.display = view === 'graph' ? '' : 'none'; document.getElementById('graph').style.display = view === 'graph' ? '' : 'none';
document.getElementById('status').style.display = view === 'status' ? '' : 'none'; document.getElementById('status').style.display = view === 'status' ? '' : 'none';
if (view === 'graph') loadGraph(); if (view === 'graph') {
loadGraph();
} else if (graphState && graphState.raf) {
cancelAnimationFrame(graphState.raf);
graphState.raf = null;
}
if (view === 'status') loadStatus(); if (view === 'status') loadStatus();
} }
@@ -329,17 +338,38 @@ async function loadStatus() {
} }
async function loadGraph() { async function loadGraph() {
const sel = document.getElementById('graphLimit');
const fromSel = sel ? parseInt(sel.value || '0', 10) : NaN;
const fromStore = parseInt(localStorage.getItem('graphLimit') || '0', 10);
const q = (!Number.isNaN(fromSel)) ? fromSel : (Number.isNaN(fromStore) ? 0 : fromStore);
if (sel) sel.value = String(q);
if (sel) localStorage.setItem('graphLimit', String(q));
const hint = document.getElementById('graphHint'); const hint = document.getElementById('graphHint');
if (hint) hint.textContent = 'Lade Graph…'; const sel = document.getElementById('graphLimit');
const selected = sel ? parseInt(sel.value || '0', 10) : 0;
const maxEngrams = Number.isNaN(selected) ? 0 : selected;
if (sel) localStorage.setItem('graphLimit', String(maxEngrams));
graphState.loadingToken = (graphState.loadingToken || 0) + 1;
const token = graphState.loadingToken;
_graphResetData();
_graphResizeCanvas();
_graphInitInteractions();
if (hint) hint.textContent = 'Graph startet...';
_graphDraw();
try { try {
const g = await api(`/api/graph?limit_nodes=${q}`); let offset = 0;
renderGraph(g.nodes || [], g.edges || []); const chunkSize = maxEngrams && maxEngrams <= 1000 ? 500 : 1800;
while (token === graphState.loadingToken) {
const limit = maxEngrams ? Math.min(chunkSize, Math.max(0, maxEngrams - offset)) : chunkSize;
if (limit <= 0) break;
const g = await api(`/api/graph_chunk?offset=${offset}&limit=${limit}`);
_graphMergePayload(g, {progressive: true});
graphState.loadedEngrams = Math.min(g.next_offset || offset, g.total_engrams || 0);
graphState.totalEngrams = g.total_engrams || graphState.totalEngrams || 0;
graphState.lastModified = g.max_modified || graphState.lastModified;
_graphDraw();
if (offset === 0 || (graphState.loadedEngrams && graphState.loadedEngrams % 9000 === 0)) {
fitGraphView({silent: true});
}
if (g.done || (maxEngrams && graphState.loadedEngrams >= maxEngrams)) break;
offset = g.next_offset || (offset + limit);
await new Promise(resolve => setTimeout(resolve, 16));
}
fitGraphView();
} catch (e) { } catch (e) {
if (hint) hint.textContent = `Graph-Fehler: ${e && e.message ? e.message : String(e)}`; if (hint) hint.textContent = `Graph-Fehler: ${e && e.message ? e.message : String(e)}`;
const canvas = _graphCanvas(); const canvas = _graphCanvas();
@@ -350,6 +380,22 @@ async function loadGraph() {
function reloadGraph() { loadGraph(); } function reloadGraph() { loadGraph(); }
async function loadGraphChanges() {
if (!graphState.lastModified) return;
try {
const g = await api(`/api/graph_changes?since=${encodeURIComponent(graphState.lastModified)}&limit=600`);
if ((g.nodes || []).length || (g.edges || []).length) {
_graphMergePayload(g, {progressive: true});
_graphPushLive(`Delta +${(g.nodes || []).filter(n => n.kind === 'engram').length} Einträge, +${(g.edges || []).length} Kanten`);
_graphDraw();
} else {
graphState.lastModified = g.max_modified || graphState.lastModified;
}
} catch (e) {
_graphPushLive(`Delta-Fehler: ${e && e.message ? e.message : String(e)}`);
}
}
// ─── Graph Renderer (Canvas) ──────────────────────────────────────────────── // ─── Graph Renderer (Canvas) ────────────────────────────────────────────────
let graphState = { let graphState = {
nodes: [], nodes: [],
@@ -359,7 +405,7 @@ let graphState = {
nodeById: new Map(), nodeById: new Map(),
simById: new Map(), simById: new Map(),
degree: new Map(), degree: new Map(),
physicsOn: false, physicsOn: true,
draggingId: null, draggingId: null,
selectedId: null, selectedId: null,
panning: false, panning: false,
@@ -375,22 +421,293 @@ let graphState = {
pinchStartZoom: null, pinchStartZoom: null,
pinchStartPan: null, pinchStartPan: null,
down: null, // {pointerId, cx, cy, t} down: null, // {pointerId, cx, cy, t}
physicsStrength: parseInt(localStorage.getItem('physicsStrength') || '60', 10), edgeKeys: new Set(),
sourceIndex: new Map(),
sourceCounts: new Map(),
clusterIndex: new Map(),
clusterCounts: new Map(),
nextSourceIndex: 0,
nextClusterIndex: 0,
loadingToken: 0,
totalEngrams: 0,
loadedEngrams: 0,
lastModified: null,
liveFeed: [],
activityById: new Map(),
lastFrameAt: 0,
}; };
function _graphCanvas() { return document.getElementById('graphCanvas'); } function _graphCanvas() { return document.getElementById('graphCanvas'); }
function _graphCtx() { return _graphCanvas().getContext('2d'); } function _graphCtx() { return _graphCanvas().getContext('2d'); }
function _graphResizeCanvas() {
const canvas = _graphCanvas();
if (!canvas) return;
const availableW = Math.max(320, canvas.parentElement.clientWidth - 24);
const availableH = Math.max(380, (window.innerHeight || 900) - 250);
const width = Math.max(320, Math.min(availableW, 1180));
const height = Math.max(380, Math.min(availableH, Math.round(width * 0.62), 760));
canvas.width = width;
canvas.height = height;
canvas.style.width = `${width}px`;
canvas.style.height = `${height}px`;
}
function _graphResetData() {
if (graphState.raf) cancelAnimationFrame(graphState.raf);
graphState.nodes = [];
graphState.edges = [];
graphState.sim = [];
graphState.links = [];
graphState.nodeById = new Map();
graphState.simById = new Map();
graphState.degree = new Map();
graphState.edgeKeys = new Set();
graphState.sourceIndex = new Map();
graphState.sourceCounts = new Map();
graphState.clusterIndex = new Map();
graphState.clusterCounts = new Map();
graphState.nextSourceIndex = 0;
graphState.nextClusterIndex = 0;
graphState.totalEngrams = 0;
graphState.loadedEngrams = 0;
graphState.lastModified = null;
graphState.activityById = new Map();
graphState.physicsOn = true;
graphState.panX = 0;
graphState.panY = 0;
graphState.zoom = 1;
graphState.lastFrameAt = 0;
}
function _graphHashUnit(s) {
let h = 2166136261;
const str = String(s || '');
for (let i = 0; i < str.length; i++) {
h ^= str.charCodeAt(i);
h = Math.imul(h, 16777619);
}
return ((h >>> 0) / 4294967295);
}
function _graphPalette(seed) {
const palette = [
[34, 211, 238], [236, 72, 153], [168, 85, 247],
[74, 222, 128], [248, 113, 113], [96, 165, 250],
[251, 191, 36], [45, 212, 191], [250, 204, 21],
[129, 140, 248], [251, 113, 133], [52, 211, 153],
];
return palette[Math.floor(_graphHashUnit(seed) * palette.length) % palette.length];
}
function _graphClusterKey(n) {
if (!n) return 'unknown';
const raw = n.cluster || (Array.isArray(n.tags) && n.tags[0]) || n.source || n.label || n.id || 'unknown';
return String(raw || 'unknown').slice(0, 80);
}
function _graphClusterCenter(cluster) {
const key = cluster || 'unknown';
if (!graphState.clusterIndex.has(key)) {
graphState.clusterIndex.set(key, graphState.nextClusterIndex++);
}
const i = graphState.clusterIndex.get(key);
if (i === 0) return {x: 0, y: 0};
const angle = i * 2.399963 + _graphHashUnit(key) * 0.85;
const ring = 190 + Math.sqrt(i) * 78;
return {x: Math.cos(angle) * ring, y: Math.sin(angle) * ring};
}
function _graphSourceCenter(source) {
const key = source || 'unknown';
if (!graphState.sourceIndex.has(key)) {
graphState.sourceIndex.set(key, graphState.nextSourceIndex++);
}
const i = graphState.sourceIndex.get(key);
const angle = i * 2.399963;
const ring = i < 1 ? 0 : 210 + Math.sqrt(i) * 80;
return {
x: Math.cos(angle) * ring,
y: Math.sin(angle) * ring,
};
}
function _graphPlaceNode(n) {
if (n.kind === 'source') {
const p = _graphSourceCenter((n.label || n.id || '').replace(/^source:/, ''));
return {x: p.x * 0.55, y: p.y * 0.55};
}
if (n.kind === 'tag') {
const p = _graphClusterCenter(n.label || n.id);
const angle = _graphHashUnit(n.id) * Math.PI * 2;
const ring = 24 + (_graphHashUnit(n.id + ':r') * 54);
return {x: p.x + Math.cos(angle) * ring, y: p.y + Math.sin(angle) * ring};
}
if (n.kind === 'host') {
const angle = _graphHashUnit(n.id) * Math.PI * 2;
const ring = 540 + (_graphHashUnit(n.id + ':r') * 260);
return {x: Math.cos(angle) * ring, y: Math.sin(angle) * ring};
}
// Graph 2.0: nodes live in topic lobes. This gives an InfraNodus-like
// overview immediately, before expensive edge physics has any work to do.
const cluster = _graphClusterKey(n);
const local = (graphState.clusterCounts.get(cluster) || 0) + 1;
graphState.clusterCounts.set(cluster, local);
const center = _graphClusterCenter(cluster);
const angle = local * 2.399963 + _graphHashUnit(n.id) * 0.9;
const radius = 14 + Math.sqrt(local) * 6.6 + _graphHashUnit(n.id + ':r') * 30;
const squash = 0.9 + (_graphHashUnit(cluster) - 0.5) * 0.24;
return {
x: center.x + Math.cos(angle) * radius * squash,
y: center.y + Math.sin(angle) * radius / squash,
};
}
function _graphEnsureSimNode(n) {
const existing = graphState.simById.get(n.id);
if (existing) {
Object.assign(existing, {
kind: n.kind || existing.kind,
label: n.label || existing.label,
weight: n.weight ?? existing.weight,
verdict: n.verdict ?? existing.verdict,
confidence: n.confidence ?? existing.confidence,
created: n.created ?? existing.created,
modified: n.modified ?? existing.modified,
last_accessed: n.last_accessed ?? existing.last_accessed,
source: n.source ?? existing.source,
cluster: n.cluster ?? existing.cluster,
tags: n.tags ?? existing.tags,
predict_locked: n.predict_locked ?? existing.predict_locked,
createdMs: Date.parse(n.created || existing.created || '') || existing.createdMs || 0,
modifiedMs: Date.parse(n.modified || existing.modified || '') || existing.modifiedMs || 0,
});
graphState.nodeById.set(n.id, {...(graphState.nodeById.get(n.id) || {}), ...n});
return existing;
}
const p = _graphPlaceNode(n);
const sim = {
id: n.id,
kind: n.kind,
label: n.label || n.id,
weight: n.weight,
verdict: n.verdict,
confidence: n.confidence,
created: n.created,
modified: n.modified,
createdMs: Date.parse(n.created || '') || 0,
modifiedMs: Date.parse(n.modified || '') || 0,
last_accessed: n.last_accessed,
source: n.source,
cluster: n.cluster,
tags: n.tags,
predict_locked: n.predict_locked,
x: p.x,
y: p.y,
vx: 0, vy: 0,
};
graphState.nodes.push(n);
graphState.nodeById.set(n.id, n);
graphState.sim.push(sim);
graphState.simById.set(n.id, sim);
return sim;
}
function _graphMergePayload(payload, opts = {}) {
const incomingNodes = payload.nodes || [];
const incomingEdges = payload.edges || [];
for (const n of incomingNodes) _graphEnsureSimNode(n);
for (const e of incomingEdges) {
const a = graphState.simById.get(e.from);
const b = graphState.simById.get(e.to);
if (!a || !b) continue;
const key = `${e.from}\u0000${e.to}\u0000${e.kind || ''}`;
if (graphState.edgeKeys.has(key)) continue;
graphState.edgeKeys.add(key);
graphState.edges.push(e);
graphState.links.push({a, b, kind: e.kind, weight: e.weight || 1.0});
graphState.degree.set(e.from, (graphState.degree.get(e.from) || 0) + 1);
graphState.degree.set(e.to, (graphState.degree.get(e.to) || 0) + 1);
}
graphState.lastModified = payload.max_modified || graphState.lastModified;
graphState.search = state.search || graphState.search || '';
if (opts.progressive && graphState.sim.length < 2500) {
const iters = graphState.sim.length < 900 ? 12 : 3;
for (let i = 0; i < iters; i++) _graphStepPhysics(0.28);
}
if (state.view === 'graph' && graphState.physicsOn && !graphState.raf) {
graphState.raf = requestAnimationFrame(_graphLoop);
}
}
function _graphPushLive(text) {
const feed = document.getElementById('graphLiveFeed');
if (!feed || !text) return;
const ts = new Date().toLocaleTimeString('de-DE', {hour: '2-digit', minute: '2-digit', second: '2-digit'});
graphState.liveFeed.unshift(`${ts} ${text}`);
graphState.liveFeed = graphState.liveFeed.slice(0, 8);
feed.innerHTML = graphState.liveFeed.map(x => `<div>${escapeHtml(x)}</div>`).join('');
}
function _graphActivityLabel(action) {
const labels = {
read: 'Lesen',
write: 'Schreiben',
write_link: 'Link',
review_confirm: 'Bewertet OK',
review_reject: 'Bewertet falsch',
score_refresh: 'Score',
};
return labels[action] || action || 'Aktivitaet';
}
function _graphApplyActivity(events) {
if (!Array.isArray(events) || !events.length) return;
let changed = false;
for (const ev of events) {
if (!ev || !ev.engram_id) continue;
const ts = Date.parse(ev.ts || '') || Date.now();
const prev = graphState.activityById.get(ev.engram_id);
if (prev && prev.ts >= ts && prev.action === ev.action) continue;
graphState.activityById.set(ev.engram_id, {action: ev.action, ts, detail: ev.detail || ''});
const n = graphState.simById.get(ev.engram_id);
if (n) {
n.activity = ev.action;
n.activityTs = ts;
n.modifiedMs = Math.max(n.modifiedMs || 0, ts);
}
_graphPushLive(`${_graphActivityLabel(ev.action)} ${String(ev.engram_id).slice(0, 8)}${ev.detail ? ' · ' + ev.detail.slice(0, 52) : ''}`);
changed = true;
}
if (changed) _graphDraw();
}
function _graphNodeRadius(n) { function _graphNodeRadius(n) {
const d = graphState.degree.get(n.id) || 0; const d = graphState.degree.get(n.id) || 0;
const base = n.kind === 'tag' ? 4 : (n.kind === 'host' ? 5 : 7); const huge = graphState.sim.length > 20000;
const base = n.kind === 'tag' ? (huge ? 3 : 4) : (n.kind === 'host' ? 5 : (n.kind === 'source' ? 13 : (huge ? 1.9 : 5.5)));
const w = (n.weight || 0); const w = (n.weight || 0);
const bonus = Math.min(6, Math.sqrt(Math.max(0, w)) * 0.8); const bonus = Math.min(huge ? 2.5 : 6, Math.sqrt(Math.max(0, w)) * 0.8);
return Math.max(3, Math.min(18, base + Math.sqrt(d) + bonus)); return Math.max(huge ? 1.4 : 3, Math.min(huge ? 9 : 18, base + Math.sqrt(d) * (huge ? 0.35 : 1) + bonus));
}
function _graphVisualRadius(n) {
// Fit can zoom out to show the full 58k+ cloud. Keep dots readable in
// screen space instead of shrinking them into invisible sub-pixels.
return _graphNodeRadius(n) / Math.max(0.08, graphState.zoom || 1);
} }
function _graphNodeFill(n) { function _graphNodeFill(n) {
const d = graphState.degree.get(n.id) || 0; const d = graphState.degree.get(n.id) || 0;
const activity = graphState.activityById.get(n.id);
if (activity && graphState.drawNow - activity.ts < 90000) {
if (activity.action === 'read') return 'rgb(56,189,248)';
if (activity.action === 'write' || activity.action === 'write_link') return 'rgb(74,222,128)';
if (activity.action === 'review_confirm') return 'rgb(167,243,208)';
if (activity.action === 'review_reject') return 'rgb(248,113,113)';
if (activity.action === 'score_refresh') return 'rgb(250,204,21)';
}
const t = Math.max(0, Math.min(0.55, d / 18)); // higher degree -> brighter const t = Math.max(0, Math.min(0.55, d / 18)); // higher degree -> brighter
const mix = (rgb) => rgb.map(c => Math.round(c + (255 - c) * t)); const mix = (rgb) => rgb.map(c => Math.round(c + (255 - c) * t));
@@ -402,6 +719,28 @@ function _graphNodeFill(n) {
const [r,g,b] = mix([245, 158, 11]); const [r,g,b] = mix([245, 158, 11]);
return `rgb(${r},${g},${b})`; return `rgb(${r},${g},${b})`;
} }
if (n.kind === 'source') {
const [r,g,b] = mix(_graphPalette(n.label || n.id));
return `rgb(${r},${g},${b})`;
}
if (n.kind === 'engram') {
let base = _graphPalette(_graphClusterKey(n));
const verdict = (n.verdict || '').toString();
if (verdict === 'confirmed_false') base = [248, 113, 113];
else if (verdict === 'confirmed_true') {
const src = _graphPalette(_graphClusterKey(n));
base = [Math.round((src[0] + 74) / 2), Math.round((src[1] + 222) / 2), Math.round((src[2] + 128) / 2)];
}
const now = graphState.drawNow || Date.now();
const created = n.createdMs || 0;
const ageMin = created ? (now - created) / 60000 : 999999;
const rec = Math.max(0, Math.min(0.45, (30 - ageMin) / 30 * 0.45));
const bump = (c) => Math.round(c + (255 - c) * rec);
const [r,g,b] = mix(base).map(bump);
return `rgb(${r},${g},${b})`;
}
const verdict = (n.verdict || '').toString(); const verdict = (n.verdict || '').toString();
let base = [96, 165, 250]; // pending/unknown = blue let base = [96, 165, 250]; // pending/unknown = blue
@@ -438,7 +777,7 @@ function _graphWorldFromScreen(cx, cy) {
function _graphHitTest(wx, wy) { function _graphHitTest(wx, wy) {
for (let i = graphState.sim.length - 1; i >= 0; i--) { for (let i = graphState.sim.length - 1; i >= 0; i--) {
const n = graphState.sim[i]; const n = graphState.sim[i];
const r = _graphNodeRadius(n) + 2; const r = _graphVisualRadius(n) + 2 / Math.max(0.08, graphState.zoom);
const dx = wx - n.x; const dx = wx - n.x;
const dy = wy - n.y; const dy = wy - n.y;
if ((dx*dx + dy*dy) <= r*r) return n; if ((dx*dx + dy*dy) <= r*r) return n;
@@ -594,14 +933,6 @@ function renderGraph(nodes, edges) {
const hint = document.getElementById('graphHint'); const hint = document.getElementById('graphHint');
const ctx = _graphCtx(); const ctx = _graphCtx();
// sync physics slider
const slider = document.getElementById('physicsStrength');
const sliderVal = document.getElementById('physicsStrengthVal');
const s = Math.max(0, Math.min(100, parseInt(graphState.physicsStrength || 60, 10)));
graphState.physicsStrength = s;
if (slider) slider.value = String(s);
if (sliderVal) sliderVal.textContent = String(s);
const w = canvas.parentElement.clientWidth - 24; const w = canvas.parentElement.clientWidth - 24;
canvas.width = Math.max(320, w); canvas.width = Math.max(320, w);
canvas.height = Math.max(520, Math.min(900, (window.innerHeight || 900) - 260)); canvas.height = Math.max(520, Math.min(900, (window.innerHeight || 900) - 260));
@@ -770,6 +1101,10 @@ function renderGraph(nodes, edges) {
function _graphStepPhysics(alpha = 1.0) { function _graphStepPhysics(alpha = 1.0) {
const canvas = _graphCanvas(); const canvas = _graphCanvas();
if (graphState.sim.length > 6000) {
_graphStepAmbient(alpha);
return;
}
const repulsion = (graphState.sim.length > 700) ? 120 : 180; const repulsion = (graphState.sim.length > 700) ? 120 : 180;
const damping = 0.86; const damping = 0.86;
const target = 80; const target = 80;
@@ -840,12 +1175,72 @@ function _graphStepPhysics(alpha = 1.0) {
} }
} }
function _graphStepAmbient(alpha = 1.0) {
const now = Date.now();
const t = now / 1000;
const sim = graphState.sim;
const count = sim.length;
if (!count) return;
// Keep the full 50k+ graph alive without doing an expensive all-node force
// solve: tiny deterministic drift plus stronger movement on fresh/hub nodes.
const sample = Math.min(count, 1400);
const start = Math.floor((t * 97) % count);
for (let k = 0; k < sample; k++) {
const n = sim[(start + k * 13) % count];
const amp = (n.kind === 'engram') ? 0.026 : 0.07;
n.x += Math.sin(t * 0.7 + _graphHashUnit(n.id) * 6.283) * amp * alpha;
n.y += Math.cos(t * 0.6 + _graphHashUnit(n.id + ':y') * 6.283) * amp * alpha;
}
const recentCutoff = now - 12 * 60 * 1000;
for (const n of sim) {
if (n.kind === 'source' || n.kind === 'tag' || (n.modifiedMs || n.createdMs || 0) > recentCutoff) {
const home = n.kind === 'engram' ? _graphClusterCenter(_graphClusterKey(n)) : (n.kind === 'tag' ? _graphClusterCenter(n.label || n.id) : null);
if (home) {
n.vx += (home.x - n.x) * 0.00004 * alpha;
n.vy += (home.y - n.y) * 0.00004 * alpha;
}
n.vx *= 0.92; n.vy *= 0.92;
n.x += n.vx; n.y += n.vy;
}
}
}
function _graphEdgeColor(kind) { function _graphEdgeColor(kind) {
const k = (kind || '').toString().toLowerCase(); const k = (kind || '').toString().toLowerCase();
if (k.includes('tag')) return '#7c3aed'; if (k.includes('tag')) return '#7c3aed';
if (k.includes('host')) return '#f59e0b'; if (k.includes('host')) return '#f59e0b';
if (k.includes('source')) return '#22d3ee';
if (k.includes('ref')) return '#10b981'; if (k.includes('ref')) return '#10b981';
return '#3a3a55'; return '#64748b';
}
function _graphDenseEdgeVisible(l) {
if (!l || !l.a || !l.b) return false;
if (l.kind === 'link') return true;
const da = graphState.degree.get(l.a.id) || 0;
const db = graphState.degree.get(l.b.id) || 0;
if (l.a.kind !== 'engram' || l.b.kind !== 'engram') return Math.max(da, db) >= 28;
const h = _graphHashUnit(`${l.a.id}:${l.b.id}:${l.kind}`);
return h > 0.982;
}
function _graphRenderInsights() {
const el = document.getElementById('graphInsights');
if (!el) return;
const clusters = Array.from(graphState.clusterCounts.entries())
.sort((a, b) => b[1] - a[1])
.slice(0, 5);
const live = graphState.sim.filter(n => {
const ms = n.modifiedMs || n.createdMs || 0;
return ms && (graphState.drawNow - ms) < 15 * 60 * 1000;
}).length;
const html = [
`<div class="graph-chip"><b>${graphState.sim.length}</b><span>Knoten live</span></div>`,
`<div class="graph-chip"><b>${graphState.links.length}</b><span>Beziehungen</span></div>`,
`<div class="graph-chip"><b>${live}</b><span>aktiv / neu</span></div>`,
...clusters.map(([name, count]) => `<div class="graph-chip"><b>${escapeHtml(name)}</b><span>${count} Einträge</span></div>`),
].join('');
el.innerHTML = html;
} }
function _graphDraw() { function _graphDraw() {
@@ -854,16 +1249,30 @@ function _graphDraw() {
const hint = document.getElementById('graphHint'); const hint = document.getElementById('graphHint');
ctx.clearRect(0,0,canvas.width,canvas.height); ctx.clearRect(0,0,canvas.width,canvas.height);
const bg = ctx.createLinearGradient(0, 0, canvas.width, canvas.height);
bg.addColorStop(0, '#0c1422');
bg.addColorStop(0.48, '#070b16');
bg.addColorStop(1, '#030712');
ctx.fillStyle = bg;
ctx.fillRect(0, 0, canvas.width, canvas.height);
graphState.drawNow = Date.now();
ctx.save(); ctx.save();
ctx.translate(graphState.panX, graphState.panY); ctx.translate(graphState.panX, graphState.panY);
ctx.scale(graphState.zoom, graphState.zoom); ctx.scale(graphState.zoom, graphState.zoom);
const term = (graphState.search || '').trim();
const dense = graphState.sim.length > 12000;
let drawnEdges = 0;
const maxDenseEdges = graphState.selectedId || term ? 5000 : 2400;
for (const l of graphState.links) { for (const l of graphState.links) {
const term = (graphState.search || '').trim();
const isMatchEdge = term && (_graphMatches(l.a, term) || _graphMatches(l.b, term)); const isMatchEdge = term && (_graphMatches(l.a, term) || _graphMatches(l.b, term));
const selectedEdge = graphState.selectedId && (l.a.id === graphState.selectedId || l.b.id === graphState.selectedId);
if (dense && !isMatchEdge && !selectedEdge && !_graphDenseEdgeVisible(l)) continue;
if (dense && !isMatchEdge && !selectedEdge && drawnEdges >= maxDenseEdges) continue;
drawnEdges++;
const w = Math.max(0.2, Math.min(3.0, (l.weight || 1.0))); const w = Math.max(0.2, Math.min(3.0, (l.weight || 1.0)));
ctx.lineWidth = (0.6 + w) / graphState.zoom; ctx.lineWidth = (0.6 + w) / graphState.zoom;
ctx.globalAlpha = isMatchEdge ? 0.85 : (0.25 + Math.min(0.35, w * 0.18)); ctx.globalAlpha = isMatchEdge || selectedEdge ? 0.9 : (dense ? 0.16 : (0.20 + Math.min(0.35, w * 0.18)));
ctx.strokeStyle = isMatchEdge ? '#f7d154' : _graphEdgeColor(l.kind); ctx.strokeStyle = isMatchEdge ? '#f7d154' : _graphEdgeColor(l.kind);
ctx.beginPath(); ctx.beginPath();
ctx.moveTo(l.a.x, l.a.y); ctx.moveTo(l.a.x, l.a.y);
@@ -872,11 +1281,12 @@ function _graphDraw() {
} }
ctx.globalAlpha = 1.0; ctx.globalAlpha = 1.0;
const term = (graphState.search || '').trim();
let matches = 0; let matches = 0;
for (const n of graphState.sim) { for (const n of graphState.sim) {
const r = _graphNodeRadius(n); const r = _graphVisualRadius(n);
const isMatch = _graphMatches(n, term); const isMatch = _graphMatches(n, term);
const activity = graphState.activityById.get(n.id);
const activeAge = activity ? graphState.drawNow - activity.ts : Infinity;
if (isMatch) matches++; if (isMatch) matches++;
if (isMatch) { if (isMatch) {
@@ -895,10 +1305,23 @@ function _graphDraw() {
ctx.stroke(); ctx.stroke();
} }
ctx.beginPath(); const fill = _graphNodeFill(n);
ctx.fillStyle = _graphNodeFill(n); const important = n.kind !== 'engram' || isMatch || graphState.selectedId === n.id || activeAge < 90000 || ((graphState.drawNow - (n.modifiedMs || n.createdMs || 0)) < 10 * 60 * 1000);
ctx.arc(n.x, n.y, r, 0, Math.PI*2); if (important) {
ctx.fill(); ctx.save();
ctx.shadowColor = fill;
ctx.shadowBlur = (n.kind === 'source' ? 14 : 8) / graphState.zoom;
ctx.beginPath();
ctx.fillStyle = fill;
ctx.arc(n.x, n.y, r, 0, Math.PI*2);
ctx.fill();
ctx.restore();
} else {
ctx.beginPath();
ctx.fillStyle = fill;
ctx.arc(n.x, n.y, r, 0, Math.PI*2);
ctx.fill();
}
// status/recency/lock border // status/recency/lock border
let stroke = null; let stroke = null;
@@ -907,9 +1330,9 @@ function _graphDraw() {
else if (v === 'confirmed_false') stroke = '#fecaca'; else if (v === 'confirmed_false') stroke = '#fecaca';
else if (v) stroke = '#c7d2fe'; else if (v) stroke = '#c7d2fe';
const now = Date.now(); const now = graphState.drawNow;
const created = Date.parse(n.created || '') || 0; const created = n.createdMs || 0;
const modified = Date.parse(n.modified || '') || 0; const modified = n.modifiedMs || 0;
const isNew = created && (now - created) < (30 * 60 * 1000); const isNew = created && (now - created) < (30 * 60 * 1000);
const isHot = modified && (now - modified) < (10 * 60 * 1000); const isHot = modified && (now - modified) < (10 * 60 * 1000);
if (isNew) stroke = '#f7d154'; if (isNew) stroke = '#f7d154';
@@ -923,42 +1346,50 @@ function _graphDraw() {
ctx.arc(n.x, n.y, r + 0.8, 0, Math.PI*2); ctx.arc(n.x, n.y, r + 0.8, 0, Math.PI*2);
ctx.stroke(); ctx.stroke();
} }
if (activity && activeAge < 90000) {
const pulse = 1 + Math.sin(graphState.drawNow / 180) * 0.18;
ctx.beginPath();
ctx.strokeStyle = activity.action === 'read' ? '#38bdf8' : (activity.action && activity.action.startsWith('review') ? '#f7d154' : '#4ade80');
ctx.globalAlpha = Math.max(0.12, 1 - activeAge / 90000);
ctx.lineWidth = 2.2 / graphState.zoom;
ctx.arc(n.x, n.y, r + 6 * pulse, 0, Math.PI*2);
ctx.stroke();
ctx.globalAlpha = 1.0;
}
} }
ctx.restore(); ctx.restore();
hint.textContent = `nodes=${graphState.nodes.length} edges=${graphState.edges.length}` + (term ? ` | match=${matches}` : ''); ctx.strokeStyle = 'rgba(34, 211, 238, 0.08)';
ctx.lineWidth = 1;
ctx.strokeRect(0.5, 0.5, canvas.width - 1, canvas.height - 1);
const loaded = graphState.totalEngrams ? ` | engrams=${graphState.loadedEngrams}/${graphState.totalEngrams}` : '';
hint.textContent = `nodes=${graphState.nodes.length} edges=${graphState.edges.length}${loaded}` + (term ? ` | match=${matches}` : '');
if (!graphState._lastInsightsAt || graphState.drawNow - graphState._lastInsightsAt > 1500) {
graphState._lastInsightsAt = graphState.drawNow;
_graphRenderInsights();
}
} }
function _graphLoop() { function _graphLoop() {
if (!graphState.physicsOn) return; if (!graphState.physicsOn) return;
const speed = 0.25 + (Math.max(0, Math.min(100, graphState.physicsStrength || 60)) / 100) * 0.95; const now = performance.now();
_graphStepPhysics(speed); const fps = graphState.sim.length > 25000 ? 8 : (graphState.sim.length > 8000 ? 14 : 30);
_graphDraw(); if (!graphState.lastFrameAt || now - graphState.lastFrameAt >= (1000 / fps)) {
graphState.lastFrameAt = now;
_graphStepPhysics(0.75);
_graphDraw();
}
graphState.raf = requestAnimationFrame(_graphLoop); graphState.raf = requestAnimationFrame(_graphLoop);
} }
function setPhysicsStrength(v) { function setPhysicsStrength(v) {
const n = Math.max(0, Math.min(100, parseInt(v || '0', 10))); // Kept for older cached pages; physics is now always live.
graphState.physicsStrength = n;
localStorage.setItem('physicsStrength', String(n));
const el = document.getElementById('physicsStrengthVal');
if (el) el.textContent = String(n);
} }
function toggleGraphPhysics() { function toggleGraphPhysics() {
graphState.physicsOn = !graphState.physicsOn; graphState.physicsOn = true;
const b = document.getElementById('btnGraphPhysics'); if (!graphState.raf) graphState.raf = requestAnimationFrame(_graphLoop);
const fast = (graphState.sim || []).length > 700;
b.textContent = `Physics: ${graphState.physicsOn ? ('on' + (fast ? ' (fast)' : '')) : 'off'}`;
b.classList.toggle('primary', graphState.physicsOn);
if (graphState.physicsOn) {
if (graphState.raf) cancelAnimationFrame(graphState.raf);
graphState.raf = requestAnimationFrame(_graphLoop);
} else {
if (graphState.raf) cancelAnimationFrame(graphState.raf);
graphState.raf = null;
_graphDraw();
}
} }
function resetGraphView() { function resetGraphView() {
@@ -969,22 +1400,27 @@ function resetGraphView() {
_graphDraw(); _graphDraw();
} }
function fitGraphView() { function fitGraphView(opts = {}) {
const canvas = _graphCanvas(); const canvas = _graphCanvas();
if (!graphState.sim.length) return; if (!graphState.sim.length) return;
let minX = Infinity, minY = Infinity, maxX = -Infinity, maxY = -Infinity; let sumX = 0, sumY = 0;
for (const n of graphState.sim) { for (const n of graphState.sim) {
minX = Math.min(minX, n.x); minY = Math.min(minY, n.y); sumX += n.x;
maxX = Math.max(maxX, n.x); maxY = Math.max(maxY, n.y); sumY += n.y;
} }
const w = Math.max(1, maxX - minX); const centerX = sumX / graphState.sim.length;
const h = Math.max(1, maxY - minY); const centerY = sumY / graphState.sim.length;
const zx = (canvas.width - 40) / w; let radius = 1;
const zy = (canvas.height - 40) / h; for (const n of graphState.sim) {
graphState.zoom = Math.max(0.35, Math.min(2.5, Math.min(zx, zy))); const dx = n.x - centerX;
graphState.panX = canvas.width / 2; const dy = n.y - centerY;
graphState.panY = canvas.height / 2; radius = Math.max(radius, Math.sqrt(dx * dx + dy * dy));
_graphDraw(); }
const pad = graphState.sim.length > 20000 ? 96 : 54;
graphState.zoom = Math.max(0.04, Math.min(2.5, (Math.min(canvas.width, canvas.height) - pad) / (radius * 2)));
graphState.panX = canvas.width / 2 - centerX * graphState.zoom;
graphState.panY = canvas.height / 2 - centerY * graphState.zoom;
if (!opts.silent) _graphDraw();
} }
function graphApplySearch(term) { function graphApplySearch(term) {
@@ -1005,11 +1441,18 @@ function startEvents() {
loadStatus(); loadStatus();
} }
if (state.view === 'graph') { if (state.view === 'graph') {
// fetch graph less often (every ~15s)
const t = Date.now(); const t = Date.now();
if (!state._lastGraphFetch || (t - state._lastGraphFetch) > 15000) { const stats = state.lastEvent.stats || {};
state._lastGraphFetch = t; _graphApplyActivity(state.lastEvent.activity || []);
loadGraph(); _graphPushLive(`Stats total=${stats.total ?? '-'} pending=${stats.pending ?? '-'}`);
if (!state._lastGraphDelta || (t - state._lastGraphDelta) > 2500) {
state._lastGraphDelta = t;
loadGraphChanges();
}
} else if (state.view === 'cards') {
if (!state._lastCardsEventRefresh || Date.now() - state._lastCardsEventRefresh > 8000) {
state._lastCardsEventRefresh = Date.now();
loadCards();
} }
} }
} catch (e) {} } catch (e) {}
@@ -1232,12 +1675,10 @@ document.getElementById('filterSelect').addEventListener('change', (e) => {
// ─── Auto Refresh ─────────────────────────────────────────────────────────── // ─── Auto Refresh ───────────────────────────────────────────────────────────
setInterval(() => { setInterval(() => {
if (!state.autoRefresh) return; if (!state.autoRefresh) return;
loadStats();
loadCards();
const now = new Date(); const now = new Date();
document.getElementById('lastUpdate').textContent = document.getElementById('lastUpdate').textContent =
`${now.getHours().toString().padStart(2,'0')}:${now.getMinutes().toString().padStart(2,'0')}`; `${now.getHours().toString().padStart(2,'0')}:${now.getMinutes().toString().padStart(2,'0')}`;
}, 5000); }, 15000);
// ─── Init ─────────────────────────────────────────────────────────────────── // ─── Init ───────────────────────────────────────────────────────────────────
loadStats(); loadStats();