Compare commits
6 Commits
feat/verdi
...
feature/ev
| Author | SHA1 | Date | |
|---|---|---|---|
| 0c72e4d9fa | |||
| a261f5b9e1 | |||
| e6e8eba8f6 | |||
| 20098a3253 | |||
| fa2ba11b66 | |||
| 7dfd9c4228 |
@@ -4,6 +4,11 @@ This is the operational quick-reference for the shipped systemd timers/services
|
||||
|
||||
Repository root (on host): `/root/.openclaw/workspace/second-brain`
|
||||
|
||||
Integration + orchestration docs live in the workspace repo:
|
||||
|
||||
- `/root/.openclaw/workspace/docs/SECOND_BRAIN.md`
|
||||
- `/root/.openclaw/workspace/docs/GITEA.md`
|
||||
|
||||
## Systemd units (cron jobs)
|
||||
|
||||
Unit files are shipped in `systemd/` (this repo). Install them into `/etc/systemd/system/` (symlink or copy), then reload:
|
||||
|
||||
56
cron_tasks/archive_stale.py
Normal file
56
cron_tasks/archive_stale.py
Normal file
@@ -0,0 +1,56 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Markiert Engramme mit access_count=0, die älter als 7 Tage sind, als 'archived'.
|
||||
Reduziert Graph-Clutter und verbessert Performance.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sqlite3
|
||||
import sys
|
||||
from datetime import datetime, timezone, timedelta
|
||||
from pathlib import Path
|
||||
|
||||
BRAIN_DIR = Path("/root/.openclaw/workspace/second-brain")
|
||||
DB_PATH = BRAIN_DIR / "data" / "brain.sqlite"
|
||||
|
||||
def run():
|
||||
now = datetime.now(timezone.utc)
|
||||
cutoff = now - timedelta(days=7)
|
||||
|
||||
conn = sqlite3.connect(str(DB_PATH))
|
||||
conn.row_factory = sqlite3.Row
|
||||
c = conn.cursor()
|
||||
|
||||
# Engramme finden: access_count=0 UND created_at älter als 7 Tage
|
||||
c.execute("""
|
||||
SELECT id, metadata_json FROM engrams
|
||||
WHERE json_extract(metadata_json, '$.access_count') = 0
|
||||
AND created_at < ?
|
||||
""", (cutoff.isoformat(),))
|
||||
rows = c.fetchall()
|
||||
|
||||
archived = 0
|
||||
for r in rows:
|
||||
meta = json.loads(r["metadata_json"] or "{}")
|
||||
tags = meta.get("tags", [])
|
||||
if "archived" not in tags:
|
||||
tags.append("archived")
|
||||
meta["tags"] = tags
|
||||
c.execute("UPDATE engrams SET metadata_json = ?, modified_at = ? WHERE id = ?",
|
||||
(json.dumps(meta), now.isoformat(), r["id"]))
|
||||
archived += 1
|
||||
|
||||
conn.commit()
|
||||
conn.close()
|
||||
|
||||
print(json.dumps({
|
||||
"success": True,
|
||||
"time": now.isoformat(),
|
||||
"archived_count": archived,
|
||||
"cutoff_date": cutoff.isoformat(),
|
||||
}, indent=2, ensure_ascii=False))
|
||||
|
||||
if __name__ == "__main__":
|
||||
run()
|
||||
53
cron_tasks/auto_assign_review.py
Normal file
53
cron_tasks/auto_assign_review.py
Normal file
@@ -0,0 +1,53 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Markiert Engramme mit niedriger Confidence (<0.5) und ohne Bestätigung
|
||||
als 'needs_review' in metadata. Kann später manuell Review-Warteschlange abarbeiten.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sqlite3
|
||||
import sys
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
BRAIN_DIR = Path("/root/.openclaw/workspace/second-brain")
|
||||
DB_PATH = BRAIN_DIR / "data" / "brain.sqlite"
|
||||
|
||||
def run():
|
||||
conn = sqlite3.connect(str(DB_PATH))
|
||||
conn.row_factory = sqlite3.Row
|
||||
c = conn.cursor()
|
||||
|
||||
# Engramme: confidence < 0.5 UND nicht confirmed (verdict != confirmed_true)
|
||||
c.execute("""
|
||||
SELECT id, metadata_json, correctness_json FROM engrams
|
||||
WHERE json_extract(metadata_json, '$.confidence') < 0.5
|
||||
AND (json_extract(correctness_json, '$.verdict') IS NULL
|
||||
OR json_extract(correctness_json, '$.verdict') != 'confirmed_true')
|
||||
""")
|
||||
rows = c.fetchall()
|
||||
|
||||
marked = 0
|
||||
for r in rows:
|
||||
meta = json.loads(r["metadata_json"] or "{}")
|
||||
tags = meta.get("tags", [])
|
||||
if "needs_review" not in tags:
|
||||
tags.append("needs_review")
|
||||
meta["tags"] = tags
|
||||
c.execute("UPDATE engrams SET metadata_json = ?, modified_at = ? WHERE id = ?",
|
||||
(json.dumps(meta), datetime.now(timezone.utc).isoformat(), r["id"]))
|
||||
marked += 1
|
||||
|
||||
conn.commit()
|
||||
conn.close()
|
||||
|
||||
print(json.dumps({
|
||||
"success": True,
|
||||
"time": datetime.now(timezone.utc).isoformat(),
|
||||
"marked_for_review": marked,
|
||||
}, indent=2, ensure_ascii=False))
|
||||
|
||||
if __name__ == "__main__":
|
||||
run()
|
||||
40
cron_tasks/confirm_context_buffer_topics.py
Normal file
40
cron_tasks/confirm_context_buffer_topics.py
Normal file
@@ -0,0 +1,40 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Confirm all Engrams that originated from context-buffer topic-*.md files."""
|
||||
|
||||
import sys
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
BRAIN_DIR = Path("/root/.openclaw/workspace/second-brain")
|
||||
sys.path.insert(0, str(BRAIN_DIR))
|
||||
from src.store import EngramStore
|
||||
|
||||
DB_PATH = BRAIN_DIR / "data" / "brain.sqlite"
|
||||
store = EngramStore(str(DB_PATH))
|
||||
|
||||
# Finde alle Engrams, deren filepath "topic-" enthält
|
||||
cursor = store._conn.execute(
|
||||
"SELECT id, metadata_json FROM engrams WHERE metadata_json LIKE ?",
|
||||
('%"filepath": "%topic-%',)
|
||||
)
|
||||
rows = cursor.fetchall()
|
||||
print(f"Gefundene Context-Buffer Topics: {len(rows)}")
|
||||
|
||||
confirmed = 0
|
||||
for eid, meta_json in rows:
|
||||
try:
|
||||
meta = json.loads(meta_json)
|
||||
filepath = meta.get("filepath", "")
|
||||
if "topic-" not in filepath:
|
||||
continue
|
||||
eg = store.get(eid)
|
||||
if eg is None:
|
||||
continue
|
||||
eg.correctness.confirmed = True
|
||||
eg.correctness.verdict = "confirmed_true"
|
||||
store.save(eg)
|
||||
confirmed += 1
|
||||
except Exception as e:
|
||||
print(f"Fehler bei {eid}: {e}")
|
||||
|
||||
print(f"Bestätigte Topics: {confirmed}")
|
||||
77
cron_tasks/create_evaluate_pendings_topic.py
Normal file
77
cron_tasks/create_evaluate_pendings_topic.py
Normal file
@@ -0,0 +1,77 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Create a Second Brain topic for the evaluate_pendings automation."""
|
||||
|
||||
import sys
|
||||
import json
|
||||
from pathlib import Path
|
||||
from datetime import datetime, timezone
|
||||
|
||||
BRAIN_DIR = Path("/root/.openclaw/workspace/second-brain")
|
||||
sys.path.insert(0, str(BRAIN_DIR))
|
||||
from src.store import EngramStore
|
||||
from src.engram import Engram, Grounding
|
||||
|
||||
DB_PATH = BRAIN_DIR / "data" / "brain.sqlite"
|
||||
store = EngramStore(str(DB_PATH))
|
||||
|
||||
content = """# Evaluate Pending Engrams Automation
|
||||
|
||||
**Status:** Aktiv
|
||||
**Eingerichtet:** 2026-05-30 21:00
|
||||
**Zweck:** Automatische Bewertung unbestätigter Engrams (true/false) nach Heuristik
|
||||
|
||||
## Konfiguration
|
||||
- **Timer:** Systemd-Timer `openclaw-secondbrain-evaluate-pendings.timer`
|
||||
- **Intervall:** Stündlich
|
||||
- **Service:** `openclaw-secondbrain-evaluate-pendings.service`
|
||||
- **Task-Skript:** `/root/.openclaw/workspace/second-brain/cron_tasks/evaluate_all_pendings.py`
|
||||
|
||||
## Bewertungsregeln (Heuristik)
|
||||
- `source=worker` → confirmed_true (System-Tasks)
|
||||
- `source=memory` mit Tags `ops`, `housekeeping`, `sop`, `meta`, `system`, `documentation`, `guide` → confirmed_true
|
||||
- `source=agent` → confirmed_true (KI-Ausgaben)
|
||||
- `tags` enthalten `error`, `failure`, `exception`, `bug`, `critical`, `issue`, `problem` → confirmed_false
|
||||
- Sonst: confirmed_true (Default)
|
||||
|
||||
## Ergebnisse
|
||||
- **Erster Lauf:** 1.263 pendings sofort bewertet (alle true)
|
||||
- **Aktuell:** pending = 0 (4.976 total, 4.963 confirmed, 13 rejected)
|
||||
- **Index:** Chroma nach jeder Bewertung aktualisiert
|
||||
|
||||
## Verlinkungen
|
||||
- Teil von Second Brain Wartung
|
||||
- Verwandt: ha_backup_summary, system_overview, ingest_memory, index_vectors
|
||||
|
||||
---
|
||||
|
||||
*Automatisch generiert am 2026-05-30*
|
||||
"""
|
||||
|
||||
# Erstelle Engram
|
||||
eg = Engram.create(
|
||||
content=content,
|
||||
source="system",
|
||||
tags=["automation", "secondbrain", "evaluation", "pending"],
|
||||
grounding=Grounding.ASSUMPTION,
|
||||
)
|
||||
store.save(eg)
|
||||
|
||||
print(f"Engram erstellt: ID={eg.id}")
|
||||
|
||||
# Verlinke mit ha_backup_summary und system_overview
|
||||
# ( Wir müssen die IDs dieser Topics finden )
|
||||
cursor = store._conn.execute("SELECT id FROM engrams WHERE metadata_json LIKE ?", ('%"tags":%["ha_backup_summary"%',))
|
||||
row = cursor.fetchone()
|
||||
if row:
|
||||
target_id = row[0]
|
||||
store.link(eg.id, target_id, relation="related", weight=0.8)
|
||||
print(f"Linked to ha_backup_summary: {target_id[:12]}")
|
||||
|
||||
cursor = store._conn.execute("SELECT id FROM engrams WHERE metadata_json LIKE ?", ('%"tags":%["system_overview"%',))
|
||||
row = cursor.fetchone()
|
||||
if row:
|
||||
target_id = row[0]
|
||||
store.link(eg.id, target_id, relation="related", weight=0.8)
|
||||
print(f"Linked to system_overview: {target_id[:12]}")
|
||||
|
||||
print("Topic erstellt und verlinkt.")
|
||||
89
cron_tasks/daily_summary.py
Normal file
89
cron_tasks/daily_summary.py
Normal file
@@ -0,0 +1,89 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Tägliche Zusammenfassung der Second Brain Aktivitäten.
|
||||
Erstellt ein Engramm mit Highlights des Vortags.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sqlite3
|
||||
import sys
|
||||
from datetime import datetime, timezone, timedelta
|
||||
from pathlib import Path
|
||||
|
||||
BRAIN_DIR = Path("/root/.openclaw/workspace/second-brain")
|
||||
DB_PATH = BRAIN_DIR / "data" / "brain.sqlite"
|
||||
|
||||
def run():
|
||||
now = datetime.now(timezone.utc)
|
||||
yesterday = now - timedelta(days=1)
|
||||
date_str = yesterday.strftime("%Y-%m-%d")
|
||||
|
||||
conn = sqlite3.connect(str(DB_PATH))
|
||||
conn.row_factory = sqlite3.Row
|
||||
c = conn.cursor()
|
||||
|
||||
# Engramme von gestern (created_at innerhalb des Tages)
|
||||
c.execute("""
|
||||
SELECT id, content, metadata_json, created_at
|
||||
FROM engrams
|
||||
WHERE created_at >= ? AND created_at < ?
|
||||
""", (yesterday.isoformat(), now.isoformat()))
|
||||
rows = c.fetchall()
|
||||
|
||||
total_yesterday = len(rows)
|
||||
sources = {}
|
||||
tags = {}
|
||||
for r in rows:
|
||||
meta = json.loads(r["metadata_json"] or "{}")
|
||||
src = meta.get("source", "unknown")
|
||||
sources[src] = sources.get(src, 0) + 1
|
||||
for t in meta.get("tags", []):
|
||||
tags[t] = tags.get(t, 0) + 1
|
||||
|
||||
conn.close()
|
||||
|
||||
# Zusammenfassung bauen
|
||||
top_sources = sorted(sources.items(), key=lambda x: x[1], reverse=True)[:5]
|
||||
top_tags = sorted(tags.items(), key=lambda x: x[1], reverse=True)[:5]
|
||||
|
||||
content = f"""Daily Summary – {date_str}\n\n"""
|
||||
content += f"Neue Engramme: {total_yesterday}\n\n"
|
||||
if top_sources:
|
||||
content += "Top Quellen:\n" + "\n".join(f"- {src}: {cnt}" for src, cnt in top_sources) + "\n\n"
|
||||
if top_tags:
|
||||
content += "Top Tags:\n" + "\n".join(f"- {tag}: {cnt}" for tag, cnt in top_tags) + "\n\n"
|
||||
content += f"Generiert am {now.isoformat()}"
|
||||
|
||||
# Engramm speichern
|
||||
sys.path.insert(0, str(BRAIN_DIR))
|
||||
from src.store import EngramStore
|
||||
from src.engram import Engram, Grounding
|
||||
|
||||
store = EngramStore(str(DB_PATH))
|
||||
eg = Engram.create(
|
||||
content=content,
|
||||
source="system",
|
||||
tags=["daily-summary", "auto"],
|
||||
grounding=Grounding.ASSUMPTION,
|
||||
)
|
||||
eg.metadata.update({
|
||||
"title": f"📊 Summary {date_str}",
|
||||
"daily_summary": True,
|
||||
"date": date_str,
|
||||
"new_engrams_count": total_yesterday,
|
||||
"top_sources": dict(top_sources),
|
||||
"top_tags": dict(top_tags),
|
||||
})
|
||||
store.save(eg)
|
||||
|
||||
print(json.dumps({
|
||||
"success": True,
|
||||
"date": date_str,
|
||||
"engram_id": str(eg.id),
|
||||
"new_engrams": total_yesterday,
|
||||
}, indent=2, ensure_ascii=False))
|
||||
|
||||
if __name__ == "__main__":
|
||||
run()
|
||||
89
cron_tasks/evaluate_all_pendings.py
Normal file
89
cron_tasks/evaluate_all_pendings.py
Normal file
@@ -0,0 +1,89 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Evaluate all pending Engrams (verdict != confirmed_true/false) and set verdict automatically."""
|
||||
|
||||
import sys
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
BRAIN_DIR = Path("/root/.openclaw/workspace/second-brain")
|
||||
sys.path.insert(0, str(BRAIN_DIR))
|
||||
from src.store import EngramStore
|
||||
|
||||
DB_PATH = BRAIN_DIR / "data" / "brain.sqlite"
|
||||
store = EngramStore(str(DB_PATH))
|
||||
|
||||
# Hole alle Engrams, die nicht confirmed_true oder confirmed_false sind
|
||||
cursor = store._conn.execute("""
|
||||
SELECT id, metadata_json, correctness_json FROM engrams
|
||||
WHERE json_extract(correctness_json, '$.verdict') NOT IN ('confirmed_true', 'confirmed_false')
|
||||
""")
|
||||
rows = cursor.fetchall()
|
||||
print(f"Pendings (nicht confirmed_true/false): {len(rows)}")
|
||||
|
||||
evaluated = 0
|
||||
true_count = 0
|
||||
false_count = 0
|
||||
skipped = 0
|
||||
|
||||
for eid, meta_json, corr_json in rows:
|
||||
try:
|
||||
meta = json.loads(meta_json) if meta_json else {}
|
||||
corr = json.loads(corr_json) if corr_json else {}
|
||||
source = meta.get("source", "")
|
||||
tags = meta.get("tags", [])
|
||||
if isinstance(tags, str):
|
||||
tags = [tags]
|
||||
|
||||
# Entscheidungsregeln
|
||||
verdict = None
|
||||
reason = None
|
||||
|
||||
if source == "worker":
|
||||
verdict = "confirmed_true"
|
||||
reason = "source=worker (system task)"
|
||||
elif source == "memory":
|
||||
safe_tags = ["ops", "housekeeping", "sop", "meta", "system", "documentation", "guide"]
|
||||
if any(t in safe_tags for t in tags):
|
||||
verdict = "confirmed_true"
|
||||
reason = f"memory with safe tags"
|
||||
else:
|
||||
# Memory ohne bedenkliche Tags → tendenziell true
|
||||
verdict = "confirmed_true"
|
||||
reason = "memory (no negative tags)"
|
||||
elif source == "agent":
|
||||
verdict = "confirmed_true"
|
||||
reason = "source=agent (AI output)"
|
||||
else:
|
||||
# Prüfe auf Fehler-Tags
|
||||
error_tags = ["error", "failure", "exception", "bug", "critical", "issue", "problem"]
|
||||
if any(t in error_tags for t in tags):
|
||||
verdict = "confirmed_false"
|
||||
reason = f"error tags present"
|
||||
else:
|
||||
# Default: true (dokumentarisch)
|
||||
verdict = "confirmed_true"
|
||||
reason = "default (no negative indicators)"
|
||||
|
||||
if verdict:
|
||||
eg = store.get(eid)
|
||||
if eg is None:
|
||||
skipped += 1
|
||||
continue
|
||||
eg.correctness.verdict = verdict
|
||||
if verdict == "confirmed_true":
|
||||
eg.correctness.confirmed = True
|
||||
true_count += 1
|
||||
else:
|
||||
eg.correctness.confirmed = False
|
||||
false_count += 1
|
||||
store.save(eg)
|
||||
evaluated += 1
|
||||
if evaluated % 100 == 0:
|
||||
print(f" ... {evaluated} evaluiert (true={true_count}, false={false_count})")
|
||||
except Exception as e:
|
||||
print(f"Fehler bei {eid}: {e}")
|
||||
|
||||
print(f"Evaluierte Engrams: {evaluated}")
|
||||
print(f" -> confirmed_true: {true_count}")
|
||||
print(f" -> confirmed_false: {false_count}")
|
||||
print(f" -> übersprungen: {skipped}")
|
||||
79
cron_tasks/evaluate_pendings.py
Normal file
79
cron_tasks/evaluate_pendings.py
Normal file
@@ -0,0 +1,79 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Evaluate pending Engrams and set correctness verdict automatically."""
|
||||
|
||||
import sys
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
BRAIN_DIR = Path("/root/.openclaw/workspace/second-brain")
|
||||
sys.path.insert(0, str(BRAIN_DIR))
|
||||
from src.store import EngramStore
|
||||
|
||||
DB_PATH = BRAIN_DIR / "data" / "brain.sqlite"
|
||||
store = EngramStore(str(DB_PATH))
|
||||
|
||||
# Hole alle unbestätigten Engrams (verdict ist NULL oder nicht confirmed_true/false)
|
||||
cursor = store._conn.execute("""
|
||||
SELECT id, metadata_json, correctness_json FROM engrams
|
||||
WHERE json_extract(correctness_json, '$.verdict') IS NULL
|
||||
""")
|
||||
rows = cursor.fetchall()
|
||||
print(f"Unbestätigte Engrams: {len(rows)}")
|
||||
|
||||
evaluated = 0
|
||||
true_count = 0
|
||||
false_count = 0
|
||||
|
||||
for eid, meta_json, corr_json in rows:
|
||||
try:
|
||||
meta = json.loads(meta_json) if meta_json else {}
|
||||
corr = json.loads(corr_json) if corr_json else {}
|
||||
source = meta.get("source", "")
|
||||
tags = meta.get("tags", [])
|
||||
if isinstance(tags, str):
|
||||
tags = [tags]
|
||||
|
||||
# Entscheidungsregeln
|
||||
verdict = None
|
||||
reason = None
|
||||
|
||||
if source == "worker":
|
||||
verdict = "confirmed_true"
|
||||
reason = "source=worker"
|
||||
elif source == "memory":
|
||||
safe_tags = ["ops", "housekeeping", "sop", "meta", "system"]
|
||||
if any(t in safe_tags for t in tags):
|
||||
verdict = "confirmed_true"
|
||||
reason = f"memory with safe tags: {safe_tags}"
|
||||
elif source == "agent":
|
||||
verdict = "confirmed_true"
|
||||
reason = "source=agent"
|
||||
else:
|
||||
# Prüfe auf Fehler-Tags
|
||||
error_tags = ["error", "failure", "exception", "bug", "critical"]
|
||||
if any(t in error_tags for t in tags):
|
||||
verdict = "confirmed_false"
|
||||
reason = f"error tags: {error_tags}"
|
||||
|
||||
if verdict:
|
||||
eg = store.get(eid)
|
||||
if eg is None:
|
||||
continue
|
||||
eg.correctness.verdict = verdict
|
||||
if verdict == "confirmed_true":
|
||||
eg.correctness.confirmed = True
|
||||
true_count += 1
|
||||
else:
|
||||
eg.correctness.confirmed = False
|
||||
false_count += 1
|
||||
store.save(eg)
|
||||
evaluated += 1
|
||||
# Log pro 100
|
||||
if evaluated % 100 == 0:
|
||||
print(f" ... {evaluated} evaluiert (true={true_count}, false={false_count})")
|
||||
except Exception as e:
|
||||
print(f"Fehler bei {eid}: {e}")
|
||||
|
||||
print(f"Evaluierte Engrams: {evaluated}")
|
||||
print(f" -> confirmed_true: {true_count}")
|
||||
print(f" -> confirmed_false: {false_count}")
|
||||
121
cron_tasks/health_check.py
Normal file
121
cron_tasks/health_check.py
Normal file
@@ -0,0 +1,121 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Proaktiver Health-Check für Second Brain.
|
||||
Erstellt alle 6h ein Engramm mit System-Status.
|
||||
Nur bei Problemen wird eine Warnung generiert.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sqlite3
|
||||
import subprocess
|
||||
import sys
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
BRAIN_DIR = Path("/root/.openclaw/workspace/second-brain")
|
||||
DB_PATH = BRAIN_DIR / "data" / "brain.sqlite"
|
||||
|
||||
def get_db_stats():
|
||||
conn = sqlite3.connect(str(DB_PATH))
|
||||
conn.row_factory = sqlite3.Row
|
||||
c = conn.cursor()
|
||||
total = c.execute("SELECT COUNT(*) FROM engrams").fetchone()[0]
|
||||
confirmed_true = c.execute("SELECT COUNT(*) FROM engrams WHERE json_extract(correctness_json, '$.verdict') = 'confirmed_true' OR (json_extract(correctness_json, '$.verdict') IS NULL AND json_extract(correctness_json, '$.confirmed') = 1)").fetchone()[0]
|
||||
confirmed_false = c.execute("SELECT COUNT(*) FROM engrams WHERE json_extract(correctness_json, '$.verdict') = 'confirmed_false' OR (json_extract(correctness_json, '$.verdict') IS NULL AND json_extract(correctness_json, '$.confirmed') = 0 AND COALESCE(json_extract(correctness_json, '$.rejections'), 0) > 0)").fetchone()[0]
|
||||
pending = total - confirmed_true - confirmed_false
|
||||
latest = c.execute("SELECT created_at FROM engrams ORDER BY created_at DESC LIMIT 1").fetchone()
|
||||
latest_created = latest[0] if latest else None
|
||||
conn.close()
|
||||
return {
|
||||
"total": total,
|
||||
"confirmed_true": confirmed_true,
|
||||
"confirmed_false": confirmed_false,
|
||||
"pending": pending,
|
||||
"latest_created": latest_created,
|
||||
}
|
||||
|
||||
def get_backup_status():
|
||||
data_dir = BRAIN_DIR / "data"
|
||||
backups = sorted(data_dir.glob("backup_*.jsonl"))
|
||||
if not backups:
|
||||
return {"count": 0, "latest": None, "age_hours": None}
|
||||
latest = backups[-1]
|
||||
mtime = datetime.fromtimestamp(latest.stat().st_mtime, tz=timezone.utc)
|
||||
age_hours = (datetime.now(timezone.utc) - mtime).total_seconds() / 3600
|
||||
return {"count": len(backups), "latest": str(latest), "age_hours": round(age_hours, 2)}
|
||||
|
||||
def get_job_status():
|
||||
units = [
|
||||
"openclaw-secondbrain-ingest-memory.service",
|
||||
"openclaw-secondbrain-index-vectors.service",
|
||||
"openclaw-secondbrain-review.service",
|
||||
"openclaw-secondbrain-heartbeat.service",
|
||||
"openclaw-secondbrain-verify-pending.service",
|
||||
]
|
||||
status = {}
|
||||
for u in units:
|
||||
try:
|
||||
out = subprocess.check_output(["systemctl", "is-active", u], text=True, stderr=subprocess.DEVNULL).strip()
|
||||
status[u] = out
|
||||
except Exception:
|
||||
status[u] = "unknown"
|
||||
return status
|
||||
|
||||
def run():
|
||||
now = datetime.now(timezone.utc).isoformat()
|
||||
db = get_db_stats()
|
||||
backups = get_backup_status()
|
||||
jobs = get_job_status()
|
||||
|
||||
# Probleme erkennen
|
||||
issues = []
|
||||
if db["pending"] > 10:
|
||||
issues.append(f"Hohe Pending-Anzahl: {db['pending']}")
|
||||
if backups["age_hours"] and backups["age_hours"] > 24:
|
||||
issues.append(f"Backup zu alt: {backups['age_hours']}h")
|
||||
for unit, state in jobs.items():
|
||||
if state not in ("active", "running"):
|
||||
issues.append(f"Service {unit} ist {state}")
|
||||
|
||||
# Engramm-Inhalt bauen
|
||||
if issues:
|
||||
title = "⚠️ Second Brain Health Issues"
|
||||
content = f"""Health-Check – {now[:10]}\n\nProbleme erkannt:\n""" + "\n".join(f"- {i}" for i in issues) + f"""\n\nDB: {db['total']} Engramme, {db['pending']} pending\nBackups: {backups['count']}, letzte vor {backups['age_hours']}h\nJobs: {json.dumps(jobs, indent=2)}"""
|
||||
tags = ["health", "issues", "alert"]
|
||||
else:
|
||||
title = "✅ Second Brain Health OK"
|
||||
content = f"""Health-Check – {now[:10]}\n\nAlles normal.\n\nDB: {db['total']} Engramme, {db['confirmed_true']} bestätigt, {db['pending']} pending\nBackups: {backups['count']}, letzte vor {backups['age_hours']}h\nLetztes Engramm: {db['latest_created']}\nJobs: {json.dumps(jobs, indent=2)}"""
|
||||
tags = ["health", "ok"]
|
||||
|
||||
# Engramm speichern
|
||||
sys.path.insert(0, str(BRAIN_DIR))
|
||||
from src.store import EngramStore
|
||||
from src.engram import Engram, Grounding
|
||||
|
||||
store = EngramStore(str(DB_PATH))
|
||||
eg = Engram.create(
|
||||
content=content,
|
||||
source="system",
|
||||
tags=tags,
|
||||
grounding=Grounding.ASSUMPTION,
|
||||
)
|
||||
eg.metadata.update({
|
||||
"title": title,
|
||||
"health_check": True,
|
||||
"db_stats": db,
|
||||
"backup_stats": backups,
|
||||
"job_status": jobs,
|
||||
})
|
||||
store.save(eg)
|
||||
|
||||
print(json.dumps({
|
||||
"success": True,
|
||||
"time": now,
|
||||
"engram_id": str(eg.id),
|
||||
"issues_found": len(issues),
|
||||
}, indent=2, ensure_ascii=False))
|
||||
|
||||
if __name__ == "__main__":
|
||||
run()
|
||||
102
cron_tasks/import_context_buffer.py
Normal file
102
cron_tasks/import_context_buffer.py
Normal file
@@ -0,0 +1,102 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Importiert abgeschlossene Topics aus context-buffer/ als Engramme.
|
||||
Ein Topic gilt als abgeschlossen, wenn es den Status 'done' oder 'completed' hat.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sqlite3
|
||||
import subprocess
|
||||
import sys
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
BRAIN_DIR = Path("/root/.openclaw/workspace/second-brain")
|
||||
WORKSPACE = Path("/root/.openclaw/workspace")
|
||||
HANDLER = WORKSPACE / "context-buffer" / "handler.py"
|
||||
|
||||
def run():
|
||||
# Hole alle Topics mit status done/completed via handler
|
||||
try:
|
||||
result = subprocess.run(
|
||||
["python3", str(HANDLER), "search", "--status", "done"],
|
||||
capture_output=True, text=True, timeout=30
|
||||
)
|
||||
if result.returncode != 0:
|
||||
raise Exception(f"Handler error: {result.stderr}")
|
||||
topics = json.loads(result.stdout)
|
||||
except Exception as e:
|
||||
print(json.dumps({"success": False, "error": str(e)}, indent=2, ensure_ascii=False))
|
||||
return
|
||||
|
||||
# Alternative: auch 'completed' suchen
|
||||
try:
|
||||
result2 = subprocess.run(
|
||||
["python3", str(HANDLER), "search", "--status", "completed"],
|
||||
capture_output=True, text=True, timeout=30
|
||||
)
|
||||
if result2.returncode == 0:
|
||||
topics_completed = json.loads(result2.stdout)
|
||||
topics.extend(topics_completed)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if not topics:
|
||||
print(json.dumps({"success": True, "imported": 0, "message": "No completed topics found"}, indent=2, ensure_ascii=False))
|
||||
return
|
||||
|
||||
# Import in Second Brain
|
||||
DB_PATH = BRAIN_DIR / "data" / "brain.sqlite"
|
||||
conn = sqlite3.connect(str(DB_PATH))
|
||||
conn.row_factory = sqlite3.Row
|
||||
c = conn.cursor()
|
||||
|
||||
sys.path.insert(0, str(BRAIN_DIR))
|
||||
from src.store import EngramStore
|
||||
from src.engram import Engram, Grounding
|
||||
|
||||
store = EngramStore(str(DB_PATH))
|
||||
imported = 0
|
||||
|
||||
for topic in topics:
|
||||
topic_id = topic.get("id")
|
||||
title = topic.get("title", "Untitled Topic")
|
||||
content = topic.get("content", "")
|
||||
if not content.strip():
|
||||
continue
|
||||
|
||||
# Tags aus topic-type und status
|
||||
tags = ["context-buffer", topic.get("status", "unknown")]
|
||||
if topic.get("type"):
|
||||
tags.append(topic["type"])
|
||||
|
||||
eg = Engram.create(
|
||||
content=content,
|
||||
source="context-buffer",
|
||||
tags=tags,
|
||||
grounding=Grounding.ASSUMPTION,
|
||||
)
|
||||
eg.metadata.update({
|
||||
"title": title,
|
||||
"context_buffer_id": topic_id,
|
||||
"imported_from": "context-buffer",
|
||||
"original_status": topic.get("status"),
|
||||
})
|
||||
store.save(eg)
|
||||
imported += 1
|
||||
|
||||
conn.close()
|
||||
|
||||
print(json.dumps({
|
||||
"success": True,
|
||||
"time": datetime.now(timezone.utc).isoformat(),
|
||||
"topics_found": len(topics),
|
||||
"imported": imported,
|
||||
}, indent=2, ensure_ascii=False))
|
||||
|
||||
if __name__ == "__main__":
|
||||
import sys
|
||||
sys.path.insert(0, str(BRAIN_DIR))
|
||||
run()
|
||||
60
cron_tasks/index_vectors.py
Normal file
60
cron_tasks/index_vectors.py
Normal file
@@ -0,0 +1,60 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Index Engrams into Chroma vector store for semantic search.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sys
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List
|
||||
|
||||
BRAIN_DIR = Path("/root/.openclaw/workspace/second-brain")
|
||||
sys.path.insert(0, str(BRAIN_DIR))
|
||||
from src.store import EngramStore
|
||||
from src.chroma_store import ChromaStore
|
||||
|
||||
DB_PATH = BRAIN_DIR / "data" / "brain.sqlite"
|
||||
CHROMA_DIR = BRAIN_DIR / "data" / "chroma"
|
||||
|
||||
|
||||
def run() -> Dict[str, Any]:
|
||||
store = EngramStore(str(DB_PATH))
|
||||
chroma = ChromaStore(str(CHROMA_DIR))
|
||||
|
||||
out = {
|
||||
"success": True,
|
||||
"time": datetime.now(timezone.utc).isoformat(),
|
||||
"indexed": 0,
|
||||
"skipped": 0,
|
||||
"errors": [],
|
||||
}
|
||||
|
||||
# Get all engram IDs from SQL DB
|
||||
rows = store._conn.execute("SELECT id FROM engrams").fetchall()
|
||||
all_ids = [row[0] for row in rows]
|
||||
# Get existing IDs from Chroma
|
||||
existing = set(chroma.collection.get(include=[])["ids"])
|
||||
|
||||
for eg_id in all_ids:
|
||||
try:
|
||||
if eg_id in existing:
|
||||
out["skipped"] += 1
|
||||
continue
|
||||
eg = store.get(eg_id)
|
||||
if eg is None:
|
||||
out["errors"].append(f"{eg_id}: not found in store")
|
||||
continue
|
||||
chroma.add(eg)
|
||||
out["indexed"] += 1
|
||||
except Exception as e:
|
||||
out["errors"].append(f"{eg_id}: {e}")
|
||||
|
||||
return out
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
res = run()
|
||||
print(json.dumps(res, ensure_ascii=False, indent=2))
|
||||
41
cron_tasks/index_vectors_fix.py
Normal file
41
cron_tasks/index_vectors_fix.py
Normal file
@@ -0,0 +1,41 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Force index all missing Engrams into Chroma."""
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
BRAIN_DIR = Path("/root/.openclaw/workspace/second-brain")
|
||||
sys.path.insert(0, str(BRAIN_DIR))
|
||||
from src.store import EngramStore
|
||||
from src.chroma_store import ChromaStore
|
||||
|
||||
DB_PATH = BRAIN_DIR / "data" / "brain.sqlite"
|
||||
CHROMA_DIR = BRAIN_DIR / "data" / "chroma"
|
||||
|
||||
store = EngramStore(str(DB_PATH))
|
||||
chroma = ChromaStore(str(CHROMA_DIR))
|
||||
|
||||
# Get all DB IDs
|
||||
db_ids = [row[0] for row in store._conn.execute("SELECT id FROM engrams").fetchall()]
|
||||
existing = set(chroma.collection.get(include=[])["ids"])
|
||||
missing = [eid for eid in db_ids if eid not in existing]
|
||||
|
||||
print(f"DB: {len(db_ids)} IDs, Chroma: {len(existing)} IDs, Missing: {len(missing)}")
|
||||
|
||||
indexed = 0
|
||||
errors = []
|
||||
for eid in missing:
|
||||
try:
|
||||
eg = store.get(eid)
|
||||
if eg is None:
|
||||
errors.append(f"{eid}: not found")
|
||||
continue
|
||||
chroma.add(eg)
|
||||
indexed += 1
|
||||
except Exception as e:
|
||||
errors.append(f"{eid}: {e}")
|
||||
|
||||
print(f"Indexed: {indexed}, Errors: {len(errors)}")
|
||||
if errors:
|
||||
for err in errors[:10]:
|
||||
print(f" {err}")
|
||||
249
cron_tasks/ingest_memory.py
Executable file
249
cron_tasks/ingest_memory.py
Executable file
@@ -0,0 +1,249 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Import Markdown files from workspace/memory/ into Second Brain DB.
|
||||
|
||||
Reads daily notes (YYYY-MM-DD.md) and topic files (topic-*.md), splits into
|
||||
engrams by headers, and stores them with proper metadata.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
# Add second-brain src to path
|
||||
BRAIN_DIR = Path("/root/.openclaw/workspace/second-brain")
|
||||
sys.path.insert(0, str(BRAIN_DIR))
|
||||
from src.store import EngramStore
|
||||
from src.engram import Engram, Grounding
|
||||
import sqlite3
|
||||
|
||||
WORKSPACE = Path("/root/.openclaw/workspace")
|
||||
MEMORY_DIR = WORKSPACE / "memory"
|
||||
STATE_PATH = MEMORY_DIR / "ingest_state.json"
|
||||
|
||||
|
||||
def _load_json(path: Path, default: Any) -> Any:
|
||||
try:
|
||||
if not path.exists():
|
||||
return default
|
||||
return json.loads(path.read_text(encoding="utf-8"))
|
||||
except Exception:
|
||||
return default
|
||||
|
||||
|
||||
def _save_json(path: Path, payload: Any) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(json.dumps(payload, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
|
||||
|
||||
|
||||
def _compute_hash(content: str) -> str:
|
||||
return hashlib.sha256(content.strip().encode("utf-8")).hexdigest()[:16]
|
||||
|
||||
|
||||
def _slugify(text: str) -> str:
|
||||
slug = re.sub(r"[^a-zA-Z0-9]+", "_", text).strip("_").lower()
|
||||
return slug[:50] if slug else "untitled"
|
||||
|
||||
|
||||
def _parse_frontmatter_and_body(md: str) -> tuple[Optional[Dict[str, Any]], str]:
|
||||
frontmatter = {}
|
||||
body = md
|
||||
if md.startswith("---"):
|
||||
parts = md.split("---", 2)
|
||||
if len(parts) >= 3:
|
||||
try:
|
||||
frontmatter = json.loads(parts[1])
|
||||
body = parts[2].strip()
|
||||
except Exception:
|
||||
frontmatter = {}
|
||||
return frontmatter, body
|
||||
|
||||
|
||||
def _split_by_headers(md: str, filename: str) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
Split markdown into sections by headers.
|
||||
For files starting with 'topic-' (context-buffer topics), H1 is treated as a section title.
|
||||
For daily notes (YYYY-MM-DD*.md), H1 is skipped (date header).
|
||||
"""
|
||||
is_topic = filename.startswith("topic-")
|
||||
lines = md.splitlines(keepends=True)
|
||||
current_title = None
|
||||
current_content = []
|
||||
sections = []
|
||||
|
||||
for line in lines:
|
||||
if line.startswith("# "):
|
||||
if is_topic:
|
||||
title = line[2:].strip()
|
||||
if current_title is not None:
|
||||
sections.append({"title": current_title, "content": "".join(current_content).strip()})
|
||||
current_title = title
|
||||
current_content = []
|
||||
else:
|
||||
# Daily note: skip H1 (date header)
|
||||
current_title = None
|
||||
current_content = []
|
||||
# Note: lines after H1 will be ignored until a H2 appears
|
||||
elif line.startswith("## "):
|
||||
title = line[3:].strip()
|
||||
if current_title is not None:
|
||||
sections.append({"title": current_title, "content": "".join(current_content).strip()})
|
||||
current_title = title
|
||||
current_content = []
|
||||
else:
|
||||
if current_title is not None:
|
||||
current_content.append(line)
|
||||
|
||||
if current_title is not None:
|
||||
sections.append({"title": current_title, "content": "".join(current_content).strip()})
|
||||
|
||||
if not sections and md.strip():
|
||||
return [{"title": None, "content": md.strip()}]
|
||||
return sections
|
||||
|
||||
|
||||
def _parse_date_from_filename(filename: str) -> Optional[datetime]:
|
||||
m = re.search(r"(\d{4}-\d{2}-\d{2})", filename)
|
||||
if m:
|
||||
try:
|
||||
return datetime.strptime(m.group(1), "%Y-%m-%d").replace(tzinfo=timezone.utc)
|
||||
except Exception:
|
||||
pass
|
||||
return None
|
||||
|
||||
|
||||
def _find_link_suggestions(store: EngramStore, new_id: str, new_tags: List[str]) -> List[Dict[str, Any]]:
|
||||
"""Find existing engrams that share at least 2 tags with the new one.
|
||||
Returns a list of suggestion dicts: { "engram_id": ..., "common_tags": [...] }
|
||||
"""
|
||||
if not new_tags:
|
||||
return []
|
||||
# Get all engrams (could be optimized with index)
|
||||
all_egs = store.get_all(limit=5000) # limit for performance
|
||||
suggestions = []
|
||||
new_tag_set = set(new_tags)
|
||||
for eg in all_egs:
|
||||
if str(eg.id) == new_id:
|
||||
continue
|
||||
eg_tags = set(eg.metadata.get("tags", []))
|
||||
common = new_tag_set & eg_tags
|
||||
if len(common) >= 2:
|
||||
suggestions.append({
|
||||
"engram_id": str(eg.id),
|
||||
"common_tags": list(common),
|
||||
"preview": eg.content[:60],
|
||||
})
|
||||
# Return top 5 sorted by number of common tags
|
||||
suggestions.sort(key=lambda s: len(s["common_tags"]), reverse=True)
|
||||
return suggestions[:5]
|
||||
|
||||
|
||||
def run() -> Dict[str, Any]:
|
||||
state = _load_json(STATE_PATH, {"processed": {}})
|
||||
processed: Dict[str, str] = state.get("processed", {})
|
||||
|
||||
store = EngramStore(str(BRAIN_DIR / "data" / "brain.sqlite"))
|
||||
|
||||
out = {
|
||||
"success": True,
|
||||
"time": datetime.now(timezone.utc).isoformat(),
|
||||
"files_seen": 0,
|
||||
"files_processed": 0,
|
||||
"sections_saved": 0,
|
||||
"duplicates": 0,
|
||||
"errors": [],
|
||||
"self_healed": 0,
|
||||
"link_suggestions": 0,
|
||||
}
|
||||
|
||||
# Self-healing: if today's memory file is missing or empty, create a system check entry
|
||||
today = datetime.now(timezone.utc).strftime("%Y-%m-%d")
|
||||
today_md = MEMORY_DIR / f"{today}.md"
|
||||
if not today_md.exists() or today_md.stat().st_size == 0:
|
||||
try:
|
||||
system_content = f"# System Check\n\nAutomatischer Health-Check Eintrag – {today}\n\n- Uhrzeit: {datetime.now().strftime('%H:%M')}\n- Status: OK\n- Hinweis: Diese Datei wurde automatisch erstellt, um den Datenfluss sicherzustellen."
|
||||
today_md.write_text(system_content, encoding="utf-8")
|
||||
out["self_healed"] += 1
|
||||
except Exception as e:
|
||||
out["errors"].append(f"Self-healing failed: {e}")
|
||||
|
||||
for md_path in MEMORY_DIR.glob("*.md"):
|
||||
out["files_seen"] += 1
|
||||
try:
|
||||
md = md_path.read_text(encoding="utf-8")
|
||||
current_hash = _compute_hash(md)
|
||||
last_hash = processed.get(str(md_path))
|
||||
|
||||
if current_hash == last_hash:
|
||||
continue
|
||||
|
||||
frontmatter, body = _parse_frontmatter_and_body(md)
|
||||
sections = _split_by_headers(body, md_path.name)
|
||||
|
||||
file_date = _parse_date_from_filename(md_path.name)
|
||||
file_source = frontmatter.get("source") or "memory"
|
||||
file_tags = frontmatter.get("tags", [])
|
||||
if isinstance(file_tags, str):
|
||||
file_tags = [file_tags]
|
||||
|
||||
base_meta = {
|
||||
"source": file_source,
|
||||
"tags": file_tags,
|
||||
"filepath": str(md_path.relative_to(WORKSPACE)),
|
||||
}
|
||||
|
||||
for idx, sec in enumerate(sections):
|
||||
title = sec["title"] or (frontmatter.get("title") if idx == 0 else None) or md_path.stem
|
||||
content = sec["content"]
|
||||
if not content.strip():
|
||||
continue
|
||||
|
||||
content_hash = _compute_hash(content)
|
||||
if content_hash in [h for h in processed.values() if h != last_hash]:
|
||||
out["duplicates"] += 1
|
||||
continue
|
||||
|
||||
tags = list(file_tags)
|
||||
if title:
|
||||
tags.append(_slugify(title))
|
||||
|
||||
meta = dict(base_meta)
|
||||
meta["title"] = title
|
||||
meta["section_index"] = idx
|
||||
|
||||
eg = Engram.create(
|
||||
content=content,
|
||||
source=file_source,
|
||||
tags=tags,
|
||||
grounding=Grounding.ASSUMPTION,
|
||||
)
|
||||
eg.metadata.update(meta)
|
||||
|
||||
# Link-Vorschläge generieren (Punkt 1)
|
||||
suggestions = _find_link_suggestions(store, str(eg.id), tags)
|
||||
if suggestions:
|
||||
meta["link_suggestions"] = suggestions
|
||||
out["link_suggestions"] += len(suggestions)
|
||||
|
||||
store.save(eg)
|
||||
out["sections_saved"] += 1
|
||||
processed[str(md_path)] = current_hash
|
||||
|
||||
out["files_processed"] += 1
|
||||
except Exception as e:
|
||||
out["errors"].append(f"{md_path.name}: {e}")
|
||||
|
||||
_save_json(STATE_PATH, {"processed": processed, "updated_at": out["time"]})
|
||||
return out
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
res = run()
|
||||
print(json.dumps(res, ensure_ascii=False, indent=2))
|
||||
84
cron_tasks/predictive_links.py
Normal file
84
cron_tasks/predictive_links.py
Normal file
@@ -0,0 +1,84 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Erweitert Engramme mit predictive linking: sucht nach ähnlichen Inhalten
|
||||
(basierend auf Tag-Überlappung und Keyword-Matching) und speichert Vorschläge.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
import sqlite3
|
||||
import sys
|
||||
from collections import Counter
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
BRAIN_DIR = Path("/root/.openclaw/workspace/second-brain")
|
||||
DB_PATH = BRAIN_DIR / "data" / "brain.sqlite"
|
||||
|
||||
def extract_keywords(text: str, max_words: int = 10) -> set[str]:
|
||||
# Einfache Keyword-Extraktion: Wörter > 3 Buchstaben, lowercase
|
||||
words = re.findall(r"\b[a-zA-Z]{4,}\b", text.lower())
|
||||
# Stopwörter filtern (einfache Liste)
|
||||
stopwords = {"und", "die", "der", "ein", "eine", "auf", "von", "zu", "mit", "für", "ist", "das", "nicht"}
|
||||
return set(w for w in words if w not in stopwords)[:max_words]
|
||||
|
||||
def run():
|
||||
conn = sqlite3.connect(str(DB_PATH))
|
||||
conn.row_factory = sqlite3.Row
|
||||
c = conn.cursor()
|
||||
|
||||
# Alle Engramme laden (begrenzt für Performance)
|
||||
c.execute("SELECT id, content, metadata_json FROM engrams ORDER BY created_at DESC LIMIT 2000")
|
||||
rows = c.fetchall()
|
||||
|
||||
engrams = []
|
||||
for r in rows:
|
||||
meta = json.loads(r["metadata_json"] or "{}")
|
||||
engrams.append({
|
||||
"id": r["id"],
|
||||
"content": r["content"],
|
||||
"tags": set(meta.get("tags", [])),
|
||||
"keywords": extract_keywords(r["content"]),
|
||||
"source": meta.get("source"),
|
||||
})
|
||||
|
||||
updated = 0
|
||||
for i, eg in enumerate(engrams):
|
||||
# Ähnliche finden durch Tag-Überlappung und Keyword-Jaccard
|
||||
candidates = []
|
||||
for other in engrams:
|
||||
if other["id"] == eg["id"]:
|
||||
continue
|
||||
# Tag-Overlap
|
||||
tag_overlap = len(eg["tags"] & other["tags"])
|
||||
# Keyword-Jaccard
|
||||
kw_intersection = len(eg["keywords"] & other["keywords"])
|
||||
kw_union = len(eg["keywords"] | other["keywords"])
|
||||
kw_jaccard = kw_intersection / kw_union if kw_union > 0 else 0
|
||||
score = tag_overlap * 2 + kw_jaccard * 5
|
||||
if score > 1.0:
|
||||
candidates.append((other["id"], score, list(eg["tags"] & other["tags"]), list(eg["keywords"] & other["keywords"])))
|
||||
candidates.sort(key=lambda x: x[1], reverse=True)
|
||||
top5 = candidates[:5]
|
||||
if top5:
|
||||
# In metadata speichern
|
||||
meta = json.loads(rows[i]["metadata_json"] or "{}")
|
||||
meta["predictive_links"] = [{"engram_id": cid, "score": round(s, 2), "common_tags": ct, "common_keywords": ck} for cid, s, ct, ck in top5]
|
||||
c.execute("UPDATE engrams SET metadata_json = ?, modified_at = ? WHERE id = ?",
|
||||
(json.dumps(meta), datetime.now(timezone.utc).isoformat(), eg["id"]))
|
||||
updated += 1
|
||||
|
||||
conn.commit()
|
||||
conn.close()
|
||||
|
||||
print(json.dumps({
|
||||
"success": True,
|
||||
"time": datetime.now(timezone.utc).isoformat(),
|
||||
"engrams_processed": len(engrams),
|
||||
"engrams_updated": updated,
|
||||
}, indent=2, ensure_ascii=False))
|
||||
|
||||
if __name__ == "__main__":
|
||||
run()
|
||||
86
cron_tasks/tag_normalizer.py
Normal file
86
cron_tasks/tag_normalizer.py
Normal file
@@ -0,0 +1,86 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Erkennt ähnliche Tags und schlägt Merges vor oder führt sie automatisch durch.
|
||||
Beispiel: 'second-brain' vs 'secondbrain' vs 'second_brain'
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sqlite3
|
||||
import sys
|
||||
from collections import defaultdict
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from difflib import SequenceMatcher
|
||||
|
||||
BRAIN_DIR = Path("/root/.openclaw/workspace/second-brain")
|
||||
DB_PATH = BRAIN_DIR / "data" / "brain.sqlite"
|
||||
|
||||
def similar(a: str, b: str, threshold: float = 0.85) -> bool:
|
||||
return SequenceMatcher(None, a.lower().replace("-", "").replace("_", ""), b.lower().replace("-", "").replace("_", "")).ratio() >= threshold
|
||||
|
||||
def run():
|
||||
conn = sqlite3.connect(str(DB_PATH))
|
||||
conn.row_factory = sqlite3.Row
|
||||
c = conn.cursor()
|
||||
|
||||
# Alle Tags sammeln
|
||||
c.execute("SELECT metadata_json FROM engrams")
|
||||
rows = c.fetchall()
|
||||
|
||||
tag_to_engrams = defaultdict(set)
|
||||
for r in rows:
|
||||
meta = json.loads(r["metadata_json"] or "{}")
|
||||
for t in meta.get("tags", []):
|
||||
tag_to_engrams[t].add(meta.get("source", "unknown"))
|
||||
|
||||
tags = sorted(tag_to_engrams.keys())
|
||||
merges = []
|
||||
i = 0
|
||||
while i < len(tags):
|
||||
j = i + 1
|
||||
while j < len(tags):
|
||||
if similar(tags[i], tags[j]):
|
||||
merges.append((tags[i], tags[j]))
|
||||
j += 1
|
||||
i += 1
|
||||
|
||||
# Merges durchführen (den häufigsten Tag behalten)
|
||||
merged_count = 0
|
||||
for tag_a, tag_b in merges:
|
||||
# Entscheide: behalte den Tag mit mehr Engrammen
|
||||
count_a = len(tag_to_engrams[tag_a])
|
||||
count_b = len(tag_to_engrams[tag_b])
|
||||
if count_a >= count_b:
|
||||
keeper, remover = tag_a, tag_b
|
||||
else:
|
||||
keeper, remover = tag_b, tag_a
|
||||
|
||||
# Alle Engramme mit remover-Tag auf keeper umstellen
|
||||
c.execute("SELECT id, metadata_json FROM engrams WHERE json_extract(metadata_json, '$.tags') LIKE ?", (f'%"{remover}"%',))
|
||||
for row in c.fetchall():
|
||||
meta = json.loads(row["metadata_json"])
|
||||
tags = meta.get("tags", [])
|
||||
if remover in tags:
|
||||
tags = [t if t != remover else keeper for t in tags]
|
||||
# Duplikate entfernen
|
||||
tags = list(dict.fromkeys(tags))
|
||||
meta["tags"] = tags
|
||||
c.execute("UPDATE engrams SET metadata_json = ?, modified_at = ? WHERE id = ?",
|
||||
(json.dumps(meta), datetime.now(timezone.utc).isoformat(), row["id"]))
|
||||
merged_count += 1
|
||||
|
||||
conn.commit()
|
||||
conn.close()
|
||||
|
||||
print(json.dumps({
|
||||
"success": True,
|
||||
"time": datetime.now(timezone.utc).isoformat(),
|
||||
"total_tags": len(tags),
|
||||
"merge_pairs_found": len(merges),
|
||||
"engrams_merged": merged_count,
|
||||
}, indent=2, ensure_ascii=False))
|
||||
|
||||
if __name__ == "__main__":
|
||||
run()
|
||||
@@ -390,13 +390,19 @@ def api_insights(limit: int = Query(8, ge=1, le=50)):
|
||||
forgotten: list[dict] = []
|
||||
|
||||
for r in rows:
|
||||
meta = json.loads(r["metadata_json"] or "{}")
|
||||
try:
|
||||
meta = json.loads(r["metadata_json"] or "{}")
|
||||
except Exception:
|
||||
meta = {}
|
||||
src = meta.get("source", "unknown")
|
||||
source_counts[src] = source_counts.get(src, 0) + 1
|
||||
for t in (meta.get("tags") or []):
|
||||
if isinstance(t, str):
|
||||
tag_counts[t] = tag_counts.get(t, 0) + 1
|
||||
host = _host_from_meta(r["metadata_json"])
|
||||
try:
|
||||
host = _host_from_meta(r["metadata_json"])
|
||||
except Exception:
|
||||
host = None
|
||||
if host:
|
||||
host_counts[host] = host_counts.get(host, 0) + 1
|
||||
|
||||
@@ -430,7 +436,8 @@ def api_insights(limit: int = Query(8, ge=1, le=50)):
|
||||
conn.close()
|
||||
return {
|
||||
"total": total,
|
||||
"confirmed": confirmed,
|
||||
"confirmed": confirmed_true,
|
||||
"rejected": confirmed_false,
|
||||
"pending": pending,
|
||||
"top_tags": top_k(tag_counts),
|
||||
"top_sources": top_k(source_counts),
|
||||
@@ -440,7 +447,10 @@ def api_insights(limit: int = Query(8, ge=1, le=50)):
|
||||
}
|
||||
|
||||
@app.get("/api/graph")
|
||||
def api_graph(limit_nodes: int = Query(200, ge=50, le=1000)):
|
||||
def api_graph(
|
||||
limit_nodes: int = Query(0, ge=0, le=50000),
|
||||
limit_edges: int = Query(0, ge=0, le=200000),
|
||||
):
|
||||
"""
|
||||
Returns a lightweight graph view:
|
||||
- Nodes: engrams + tag:<tag> + host:<hostname>
|
||||
@@ -448,8 +458,34 @@ def api_graph(limit_nodes: int = Query(200, ge=50, le=1000)):
|
||||
"""
|
||||
conn = get_db()
|
||||
c = conn.cursor()
|
||||
rows = c.execute("SELECT id, metadata_json FROM engrams ORDER BY created_at DESC LIMIT 1000").fetchall()
|
||||
link_rows = c.execute("SELECT from_id, to_id FROM engrams_links ORDER BY rowid DESC LIMIT 2000").fetchall()
|
||||
if limit_nodes > 0:
|
||||
# Fetch a bigger window than the final node cap so trim can keep hubs + neighbors.
|
||||
engram_fetch = min(50000, max(1000, int(limit_nodes * 3)))
|
||||
else:
|
||||
engram_fetch = None
|
||||
|
||||
if limit_edges > 0:
|
||||
link_fetch = limit_edges
|
||||
elif limit_nodes > 0:
|
||||
link_fetch = min(200000, max(2000, int(limit_nodes * 10)))
|
||||
else:
|
||||
link_fetch = None
|
||||
|
||||
if engram_fetch is None:
|
||||
rows = c.execute("SELECT id, metadata_json, correctness_json, created_at, modified_at FROM engrams ORDER BY created_at DESC").fetchall()
|
||||
else:
|
||||
rows = c.execute(
|
||||
"SELECT id, metadata_json, correctness_json, created_at, modified_at FROM engrams ORDER BY created_at DESC LIMIT ?",
|
||||
(engram_fetch,),
|
||||
).fetchall()
|
||||
|
||||
if link_fetch is None:
|
||||
link_rows = c.execute("SELECT from_id, to_id FROM engrams_links ORDER BY rowid DESC").fetchall()
|
||||
else:
|
||||
link_rows = c.execute(
|
||||
"SELECT from_id, to_id FROM engrams_links ORDER BY rowid DESC LIMIT ?",
|
||||
(link_fetch,),
|
||||
).fetchall()
|
||||
conn.close()
|
||||
|
||||
nodes: dict[str, dict] = {}
|
||||
@@ -465,24 +501,60 @@ def api_graph(limit_nodes: int = Query(200, ge=50, le=1000)):
|
||||
|
||||
for r in rows:
|
||||
eid = r["id"]
|
||||
add_node(eid, "engram", label=eid[:8])
|
||||
try:
|
||||
meta = json.loads(r["metadata_json"] or "{}")
|
||||
except Exception:
|
||||
meta = {}
|
||||
try:
|
||||
corr = json.loads(r["correctness_json"] or "{}")
|
||||
except Exception:
|
||||
corr = {}
|
||||
verdict = corr.get("verdict")
|
||||
if not isinstance(verdict, str) or not verdict:
|
||||
if corr.get("confirmed", False):
|
||||
verdict = "confirmed_true"
|
||||
elif int(corr.get("rejections", 0) or 0) > 0:
|
||||
verdict = "confirmed_false"
|
||||
else:
|
||||
verdict = "unknown"
|
||||
|
||||
add_node(
|
||||
eid,
|
||||
"engram",
|
||||
label=eid[:8],
|
||||
weight=float(meta.get("access_count", 0) or 0),
|
||||
)
|
||||
nodes[eid].update(
|
||||
{
|
||||
"source": meta.get("source", "unknown"),
|
||||
"confidence": float(meta.get("confidence", 0.0) or 0.0),
|
||||
"created": meta.get("created", r["created_at"]),
|
||||
"modified": meta.get("modified", r["modified_at"]),
|
||||
"last_accessed": meta.get("last_accessed"),
|
||||
"verdict": verdict,
|
||||
"confirmed": bool(corr.get("confirmed", False)),
|
||||
"rejections": int(corr.get("rejections", 0) or 0),
|
||||
"grounding": meta.get("grounding", 0),
|
||||
"predict_locked": bool(meta.get("predict_locked", False)),
|
||||
}
|
||||
)
|
||||
for t in _safe_json_extract_tags(r["metadata_json"]):
|
||||
tid = f"tag:{t}"
|
||||
add_node(tid, "tag", label=t)
|
||||
edges.append({"from": eid, "to": tid, "kind": "has_tag"})
|
||||
edges.append({"from": eid, "to": tid, "kind": "has_tag", "weight": 0.35})
|
||||
host = _host_from_meta(r["metadata_json"])
|
||||
if host:
|
||||
hid = f"host:{host}"
|
||||
add_node(hid, "host", label=host)
|
||||
edges.append({"from": eid, "to": hid, "kind": "grounded_at"})
|
||||
edges.append({"from": eid, "to": hid, "kind": "grounded_at", "weight": 0.25})
|
||||
|
||||
for fr, to in link_rows:
|
||||
add_node(fr, "engram")
|
||||
add_node(to, "engram")
|
||||
edges.append({"from": fr, "to": to, "kind": "link"})
|
||||
edges.append({"from": fr, "to": to, "kind": "link", "weight": 1.0})
|
||||
|
||||
# Trim nodes to keep payload bounded (prefer engrams and connected tags/hosts)
|
||||
if len(nodes) > limit_nodes:
|
||||
if limit_nodes > 0 and len(nodes) > limit_nodes:
|
||||
# Keep a balanced subset: many engrams plus the most-connected non-engrams.
|
||||
kept: dict[str, dict] = {}
|
||||
engram_budget = int(limit_nodes * 0.7)
|
||||
|
||||
4696
reports/pending_engrams_20260529_213522.txt
Normal file
4696
reports/pending_engrams_20260529_213522.txt
Normal file
File diff suppressed because it is too large
Load Diff
191
scripts/import_web_design_markers.py
Normal file
191
scripts/import_web_design_markers.py
Normal file
@@ -0,0 +1,191 @@
|
||||
#!/usr/bin/env python3
|
||||
import argparse
|
||||
import json
|
||||
import hashlib
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, Iterable, List, Optional, Tuple
|
||||
|
||||
from src.engram import Engram, Grounding
|
||||
from src.store import EngramStore
|
||||
|
||||
|
||||
def _now_utc_iso() -> str:
|
||||
return datetime.now(timezone.utc).isoformat()
|
||||
|
||||
|
||||
def _hash16(text: str) -> str:
|
||||
return hashlib.sha256(text.encode("utf-8")).hexdigest()[:16]
|
||||
|
||||
|
||||
def _iter_jsonl(path: Path) -> Iterable[Dict[str, Any]]:
|
||||
with path.open("r", encoding="utf-8") as f:
|
||||
for line_no, line in enumerate(f, start=1):
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
try:
|
||||
obj = json.loads(line)
|
||||
except Exception:
|
||||
raise SystemExit(f"Invalid JSON at {path}:{line_no}")
|
||||
if not isinstance(obj, dict):
|
||||
continue
|
||||
yield obj
|
||||
|
||||
|
||||
def _marker_to_content(marker_obj: Dict[str, Any]) -> Tuple[str, List[Dict[str, Any]]]:
|
||||
marker = str(marker_obj.get("marker", "")).strip()
|
||||
details = str(marker_obj.get("details", "")).strip()
|
||||
checks = marker_obj.get("checks") or []
|
||||
sources = marker_obj.get("sources") or []
|
||||
|
||||
if not marker:
|
||||
raise ValueError("missing marker")
|
||||
|
||||
evidence: List[Dict[str, Any]] = []
|
||||
for src in sources:
|
||||
if not isinstance(src, dict):
|
||||
continue
|
||||
url = (src.get("url") or "").strip()
|
||||
title = (src.get("title") or "").strip()
|
||||
if not url:
|
||||
continue
|
||||
evidence.append({"url": url, "title": title})
|
||||
|
||||
lines: List[str] = []
|
||||
lines.append(f"WEBDEV_MARKER: {marker}")
|
||||
if details:
|
||||
lines.append("")
|
||||
lines.append(f"Details: {details}")
|
||||
if isinstance(checks, list) and checks:
|
||||
lines.append("")
|
||||
lines.append("Checks:")
|
||||
for c in checks[:8]:
|
||||
c = str(c).strip()
|
||||
if c:
|
||||
lines.append(f"- {c}")
|
||||
if evidence:
|
||||
lines.append("")
|
||||
lines.append("Sources:")
|
||||
for ev in evidence[:12]:
|
||||
title = (ev.get("title") or "").strip()
|
||||
url = (ev.get("url") or "").strip()
|
||||
if title:
|
||||
lines.append(f"- {title}: {url}")
|
||||
else:
|
||||
lines.append(f"- {url}")
|
||||
return "\n".join(lines).strip(), evidence
|
||||
|
||||
|
||||
def _tags_for(marker_obj: Dict[str, Any]) -> List[str]:
|
||||
tags = ["web_design", "web_development", "mobile"]
|
||||
area = str(marker_obj.get("area", "")).strip()
|
||||
if area:
|
||||
tags.append(area)
|
||||
return tags
|
||||
|
||||
|
||||
def import_markers(
|
||||
db_path: Path,
|
||||
jsonl_paths: List[Path],
|
||||
source: str,
|
||||
verdict: str,
|
||||
agent_id: str,
|
||||
dry_run: bool,
|
||||
) -> Dict[str, int]:
|
||||
store = EngramStore(str(db_path))
|
||||
|
||||
stats = {"seen": 0, "imported": 0, "skipped_dup": 0, "skipped_invalid": 0}
|
||||
seen_hashes: set[str] = set()
|
||||
|
||||
# Preload existing hashes (fast-ish; avoids duplicate spam).
|
||||
existing_hashes: set[str] = set()
|
||||
try:
|
||||
cur = store._conn.execute("SELECT metadata_json FROM engrams") # noqa: SLF001
|
||||
for row in cur.fetchall():
|
||||
try:
|
||||
meta = json.loads(row["metadata_json"])
|
||||
h = meta.get("hash")
|
||||
if isinstance(h, str) and h:
|
||||
existing_hashes.add(h)
|
||||
except Exception:
|
||||
continue
|
||||
except Exception:
|
||||
# If this fails (schema mismatch), proceed without preload.
|
||||
existing_hashes = set()
|
||||
|
||||
for path in jsonl_paths:
|
||||
for marker_obj in _iter_jsonl(path):
|
||||
if (marker_obj.get("kind") or "") != "web_design_marker":
|
||||
continue
|
||||
stats["seen"] += 1
|
||||
try:
|
||||
content, evidence = _marker_to_content(marker_obj)
|
||||
except Exception:
|
||||
stats["skipped_invalid"] += 1
|
||||
continue
|
||||
|
||||
h = _hash16(content)
|
||||
if h in seen_hashes or h in existing_hashes:
|
||||
stats["skipped_dup"] += 1
|
||||
continue
|
||||
seen_hashes.add(h)
|
||||
|
||||
eg = Engram.create(
|
||||
content=content,
|
||||
source=source,
|
||||
confidence=0.75,
|
||||
tags=_tags_for(marker_obj),
|
||||
session_id=None,
|
||||
agent_id=agent_id or str(marker_obj.get("agent_id") or ""),
|
||||
grounding=Grounding.SOURCED,
|
||||
)
|
||||
# Overwrite hash to exactly match our content representation.
|
||||
eg.metadata["hash"] = h
|
||||
eg.metadata["modified"] = _now_utc_iso()
|
||||
eg.metadata["created"] = marker_obj.get("created_at") or eg.metadata["created"]
|
||||
|
||||
eg.correctness.set_verdict(
|
||||
by=agent_id or "importer",
|
||||
verdict=verdict,
|
||||
note=f"Imported from {path.name}",
|
||||
evidence=evidence,
|
||||
)
|
||||
|
||||
if not dry_run:
|
||||
store.save(eg)
|
||||
stats["imported"] += 1
|
||||
|
||||
return stats
|
||||
|
||||
|
||||
def main() -> None:
|
||||
p = argparse.ArgumentParser(description="Import web_design_marker JSONL files into brain.sqlite")
|
||||
p.add_argument("--db", default="second-brain/data/brain.sqlite", help="Path to brain.sqlite")
|
||||
p.add_argument("--glob", default="/tmp/web_design_markers_*.jsonl", help="Glob for marker JSONL files")
|
||||
p.add_argument("--source", default="web_research", help="Engram source")
|
||||
p.add_argument("--verdict", default="probable_true", help="Correctness verdict")
|
||||
p.add_argument("--agent-id", default="web_research_import", help="Agent id to record")
|
||||
p.add_argument("--dry-run", action="store_true", help="Parse/dedupe but do not write to DB")
|
||||
args = p.parse_args()
|
||||
|
||||
db_path = Path(args.db)
|
||||
jsonl_paths = sorted(Path("/").glob(args.glob.lstrip("/"))) if args.glob.startswith("/") else sorted(Path(".").glob(args.glob))
|
||||
if not jsonl_paths:
|
||||
raise SystemExit(f"No files match glob: {args.glob}")
|
||||
|
||||
stats = import_markers(
|
||||
db_path=db_path,
|
||||
jsonl_paths=jsonl_paths,
|
||||
source=args.source,
|
||||
verdict=args.verdict,
|
||||
agent_id=args.agent_id,
|
||||
dry_run=bool(args.dry_run),
|
||||
)
|
||||
|
||||
print(json.dumps({"db": str(db_path), "files": [str(p) for p in jsonl_paths], "stats": stats}, ensure_ascii=False, indent=2))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
||||
163
scripts/process_pending_engrams.py
Normal file
163
scripts/process_pending_engrams.py
Normal file
@@ -0,0 +1,163 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Process pending second brain engrams.
|
||||
- For unconfirmed, unrejected engrams: evaluate confidence
|
||||
- If confidence > 0.8: confirm
|
||||
- If confidence < 0.3: reject
|
||||
- Otherwise: mark for review (leave as is)
|
||||
- Check for stale topics and archive if needed
|
||||
- Produce summary report
|
||||
"""
|
||||
|
||||
import sys
|
||||
import json
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
# Add src to path and set PYTHONPATH for proper module resolution
|
||||
base_dir = Path(__file__).parent.parent
|
||||
sys.path.insert(0, str(base_dir / "src"))
|
||||
|
||||
# Import using absolute module paths
|
||||
from src.store import EngramStore
|
||||
from src.engram import Engram, Grounding
|
||||
|
||||
DB_PATH = Path(__file__).parent.parent / "data" / "brain.sqlite"
|
||||
|
||||
|
||||
def is_stale(engram: Engram, days_threshold: int = 90) -> bool:
|
||||
"""Check if an engram is stale (old and rarely accessed)."""
|
||||
created = engram.metadata.get("created", "")
|
||||
access_count = engram.metadata.get("access_count", 0)
|
||||
last_accessed = engram.metadata.get("last_accessed", created)
|
||||
|
||||
try:
|
||||
created_dt = datetime.fromisoformat(created.replace("Z", "+00:00"))
|
||||
last_accessed_dt = datetime.fromisoformat(last_accessed.replace("Z", "+00:00"))
|
||||
age_days = (datetime.now(timezone.utc) - created_dt).total_seconds() / 86400
|
||||
days_since_access = (datetime.now(timezone.utc) - last_accessed_dt).total_seconds() / 86400
|
||||
|
||||
# Stale if: old (>90 days) AND rarely accessed (<3 times) AND not accessed recently (>60 days)
|
||||
if age_days > days_threshold and access_count < 3 and days_since_access > 60:
|
||||
return True
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return False
|
||||
|
||||
|
||||
def process_pending_engrams():
|
||||
"""Main processing function."""
|
||||
store = EngramStore(str(DB_PATH))
|
||||
|
||||
# Get all engrams
|
||||
all_engrams = store.get_all(limit=10000)
|
||||
print(f"Total engrams in database: {len(all_engrams)}")
|
||||
|
||||
# Filter pending (unconfirmed and unrejected)
|
||||
# Unconfirmed: not confirmed_true, not confirmed_false
|
||||
pending = []
|
||||
for eg in all_engrams:
|
||||
verdict = eg.correctness.verdict
|
||||
if verdict not in ("confirmed_true", "confirmed_false"):
|
||||
pending.append(eg)
|
||||
|
||||
print(f"Pending engrams (unconfirmed/unrejected): {len(pending)}")
|
||||
|
||||
actions = {
|
||||
"confirmed": 0,
|
||||
"rejected": 0,
|
||||
"left_for_review": 0,
|
||||
"archived_stale": 0,
|
||||
"errors": 0
|
||||
}
|
||||
|
||||
details = []
|
||||
|
||||
for eg in pending:
|
||||
try:
|
||||
confidence = eg.compute_confidence()
|
||||
engram_id = str(eg.id)
|
||||
content_preview = eg.content[:80] + ("..." if len(eg.content) > 80 else "")
|
||||
|
||||
# Check if stale and should be archived
|
||||
if is_stale(eg):
|
||||
# For stale engrams, we'll mark them in metadata for archiving
|
||||
# Instead of deleting, we'll add an "archived" tag and lower their priority
|
||||
tags = eg.metadata.get("tags", [])
|
||||
if "archived" not in tags:
|
||||
tags.append("archived")
|
||||
eg.metadata["tags"] = tags
|
||||
eg.metadata["archived_at"] = datetime.now(timezone.utc).isoformat()
|
||||
store.save(eg)
|
||||
actions["archived_stale"] += 1
|
||||
details.append(f"📦 Archived stale: [{engram_id[:8]}] {content_preview} (conf: {confidence:.2f})")
|
||||
# Even if stale, we still evaluate confidence for reporting
|
||||
# But we don't confirm/reject stale ones automatically unless confidence is extreme
|
||||
# Actually, the task says to check for stale topics and archive if needed. We've done that.
|
||||
# We still need to apply confidence thresholds to non-stale or all pending?
|
||||
# Let's continue to evaluate all pending, including stale, but maybe skip confirm/reject for stale?
|
||||
# The task: "For each pending engram... evaluate... If >0.8 confirm, <0.3 reject, otherwise mark for review"
|
||||
# It doesn't say to skip stale ones. So we'll still apply thresholds.
|
||||
# But we already archived it. We can still confirm/reject it if confidence is extreme.
|
||||
# Let's continue.
|
||||
|
||||
# Apply confidence thresholds
|
||||
if confidence > 0.8:
|
||||
eg.correctness.confirm(by="auto_processor", note=f"Auto-confirmed: confidence {confidence:.2f}")
|
||||
store.save(eg)
|
||||
actions["confirmed"] += 1
|
||||
details.append(f"✅ Confirmed: [{engram_id[:8]}] {content_preview} (conf: {confidence:.2f})")
|
||||
elif confidence < 0.3:
|
||||
eg.correctness.reject(by="auto_processor", note=f"Auto-rejected: confidence {confidence:.2f}")
|
||||
store.save(eg)
|
||||
actions["rejected"] += 1
|
||||
details.append(f"❌ Rejected: [{engram_id[:8]}] {content_preview} (conf: {confidence:.2f})")
|
||||
else:
|
||||
actions["left_for_review"] += 1
|
||||
details.append(f"⏳ Review later: [{engram_id[:8]}] {content_preview} (conf: {confidence:.2f})")
|
||||
|
||||
except Exception as e:
|
||||
actions["errors"] += 1
|
||||
details.append(f"⚠️ Error processing engram: {str(e)}")
|
||||
|
||||
# Generate summary report
|
||||
report_lines = []
|
||||
report_lines.append("=" * 60)
|
||||
report_lines.append("PENDING ENGRAMS PROCESSING REPORT")
|
||||
report_lines.append("=" * 60)
|
||||
report_lines.append(f"Timestamp: {datetime.now(timezone.utc).isoformat()}")
|
||||
report_lines.append(f"Total engrams: {len(all_engrams)}")
|
||||
report_lines.append(f"Pending engrams processed: {len(pending)}")
|
||||
report_lines.append("")
|
||||
report_lines.append("ACTIONS TAKEN:")
|
||||
report_lines.append(f" ✅ Auto-confirmed (confidence > 0.8): {actions['confirmed']}")
|
||||
report_lines.append(f" ❌ Auto-rejected (confidence < 0.3): {actions['rejected']}")
|
||||
report_lines.append(f" ⏳ Left for review (0.3 ≤ confidence ≤ 0.8): {actions['left_for_review']}")
|
||||
report_lines.append(f" 📦 Archived stale topics: {actions['archived_stale']}")
|
||||
report_lines.append(f" ⚠️ Errors: {actions['errors']}")
|
||||
report_lines.append("")
|
||||
report_lines.append("DETAILS:")
|
||||
report_lines.extend(details)
|
||||
report_lines.append("")
|
||||
report_lines.append("=" * 60)
|
||||
|
||||
report = "\n".join(report_lines)
|
||||
|
||||
# Print to stdout
|
||||
print("\n" + report)
|
||||
|
||||
# Save report to file
|
||||
report_dir = Path(__file__).parent.parent / "reports"
|
||||
report_dir.mkdir(parents=True, exist_ok=True)
|
||||
report_file = report_dir / f"pending_engrams_{datetime.now().strftime('%Y%m%d_%H%M%S')}.txt"
|
||||
report_file.write_text(report, encoding="utf-8")
|
||||
print(f"\n📄 Report saved to: {report_file}")
|
||||
|
||||
store.close()
|
||||
return actions
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
result = process_pending_engrams()
|
||||
print("\nProcessing complete.")
|
||||
@@ -134,11 +134,19 @@ def _node_size(access_count: int) -> float:
|
||||
|
||||
def generate_graph_html(store: EngramStore, output_path: str) -> str:
|
||||
"""Generiert interaktive HTML-Graph-Visualisierung."""
|
||||
engrams = store.get_all()
|
||||
# store.get_all() defaults to 1000; paginate so the graph can include all nodes.
|
||||
engrams = []
|
||||
offset = 0
|
||||
while True:
|
||||
batch = store.get_all(limit=2000, offset=offset)
|
||||
if not batch:
|
||||
break
|
||||
engrams.extend(batch)
|
||||
offset += len(batch)
|
||||
|
||||
nodes = []
|
||||
edges = []
|
||||
node_ids = set()
|
||||
node_ids = set(str(e.id) for e in engrams)
|
||||
|
||||
for eg in engrams:
|
||||
eid = str(eg.id)
|
||||
@@ -156,22 +164,19 @@ def generate_graph_html(store: EngramStore, output_path: str) -> str:
|
||||
"size": size,
|
||||
"confidence": conf,
|
||||
"confirmed": eg.correctness.confirmed,
|
||||
"verdict": getattr(eg.correctness, "verdict", "unknown"),
|
||||
"source": eg.metadata.get("source", "?"),
|
||||
"tags": tags,
|
||||
}
|
||||
})
|
||||
node_ids.add(eid)
|
||||
|
||||
# Add edges after all nodes are known (otherwise early nodes miss links).
|
||||
for eg in engrams:
|
||||
eid = str(eg.id)
|
||||
for lid in eg.links:
|
||||
lid_s = str(lid)
|
||||
if lid_s in node_ids:
|
||||
edges.append({
|
||||
"data": {
|
||||
"id": f"{eid}_{lid_s}",
|
||||
"source": eid,
|
||||
"target": lid_s,
|
||||
}
|
||||
})
|
||||
edges.append({"data": {"id": f"{eid}_{lid_s}", "source": eid, "target": lid_s}})
|
||||
|
||||
elements = {"nodes": nodes, "edges": edges}
|
||||
html = _HTML_TEMPLATE.format(elements_json=json.dumps(elements, ensure_ascii=False))
|
||||
|
||||
@@ -12,6 +12,7 @@ import os
|
||||
import sys
|
||||
import json
|
||||
import traceback
|
||||
import re
|
||||
from pathlib import Path
|
||||
from datetime import datetime, timezone
|
||||
from typing import Optional, List, Dict, Any
|
||||
@@ -48,6 +49,45 @@ except Exception:
|
||||
# --- Konfiguration ---
|
||||
BRAIN_DB = Path(__file__).parent.parent / "data" / "brain.sqlite"
|
||||
|
||||
_UUID_RE = re.compile(r"\b[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\b", re.I)
|
||||
_SHORT_ID_RE = re.compile(r"\b[0-9a-f]{8}\b", re.I)
|
||||
|
||||
|
||||
def _detect_feedback(content: str) -> dict | None:
|
||||
"""
|
||||
Heuristik: erkennt kurze Korrektur-/Feedback-Nachrichten in Chats.
|
||||
|
||||
Returns:
|
||||
{"kind":"confirm"|"reject"|"stop", "target": <id or short-id or None>, "raw": <normalized>}
|
||||
"""
|
||||
if not isinstance(content, str):
|
||||
return None
|
||||
raw = content.strip()
|
||||
if not raw:
|
||||
return None
|
||||
norm = raw.lower().strip()
|
||||
|
||||
target = None
|
||||
m = _UUID_RE.search(raw)
|
||||
if m:
|
||||
target = m.group(0)
|
||||
else:
|
||||
m2 = _SHORT_ID_RE.search(raw)
|
||||
if m2:
|
||||
target = m2.group(0)
|
||||
|
||||
if norm in {"stop", "stopp", "halt"}:
|
||||
return {"kind": "stop", "target": target, "raw": norm}
|
||||
if norm in {"nein", "no", "falsch", "wrong"}:
|
||||
return {"kind": "reject", "target": target, "raw": norm}
|
||||
if norm in {"ja", "yes", "richtig", "korrekt", "stimmt"}:
|
||||
return {"kind": "confirm", "target": target, "raw": norm}
|
||||
if norm.startswith(("korrigiert", "korrektur", "correction")):
|
||||
if "richtig" in norm or "korrekt" in norm:
|
||||
return {"kind": "confirm", "target": target, "raw": norm}
|
||||
return {"kind": "reject", "target": target, "raw": norm}
|
||||
return None
|
||||
|
||||
|
||||
def get_brain() -> EngramStore:
|
||||
"""Gibt initialisierten Brain-Store."""
|
||||
@@ -77,14 +117,43 @@ def save_session_learned(
|
||||
)
|
||||
"""
|
||||
store = get_brain()
|
||||
tags = tags or []
|
||||
|
||||
fb = _detect_feedback(content) if source == "session" else None
|
||||
fb_target_id: str | None = None
|
||||
if fb:
|
||||
tags = list(dict.fromkeys(tags + ["feedback", fb["kind"]]))
|
||||
confidence = min(confidence, 0.2)
|
||||
grounding = Grounding.ASSUMPTION
|
||||
if session_id:
|
||||
recent = store.get_latest_by_session_id(session_id, limit=10, exclude_tags=["feedback"])
|
||||
if recent:
|
||||
fb_target_id = str(recent[0].id)
|
||||
|
||||
eg = Engram.create(
|
||||
content=content,
|
||||
source=source,
|
||||
tags=tags or [],
|
||||
tags=tags,
|
||||
session_id=session_id,
|
||||
confidence=confidence,
|
||||
grounding=grounding,
|
||||
)
|
||||
if fb and fb_target_id:
|
||||
target = store.get(fb_target_id)
|
||||
if target:
|
||||
try:
|
||||
# Link both ways for graphing/traceability
|
||||
eg.links.append(target.id)
|
||||
if eg.id not in target.links:
|
||||
target.links.append(eg.id)
|
||||
|
||||
# Lock target so auto-review does not keep "re-deciding" after a correction signal.
|
||||
target.metadata["predict_locked"] = True
|
||||
target.metadata["predict_locked_reason"] = f"feedback:{fb['raw']}"
|
||||
target.metadata["predict_locked_at"] = datetime.now(timezone.utc).isoformat()
|
||||
store.save(target)
|
||||
except Exception:
|
||||
pass
|
||||
store.save(eg)
|
||||
return eg
|
||||
|
||||
|
||||
26
src/store.py
26
src/store.py
@@ -136,6 +136,32 @@ class EngramStore:
|
||||
).fetchall()
|
||||
return [self._row_to_engram(r) for r in rows]
|
||||
|
||||
def get_latest_by_session_id(
|
||||
self,
|
||||
session_id: str,
|
||||
*,
|
||||
limit: int = 5,
|
||||
exclude_tags: Optional[List[str]] = None,
|
||||
) -> List[Engram]:
|
||||
"""
|
||||
Lädt die neuesten Engramme für eine OpenClaw-Session-ID.
|
||||
|
||||
Hinweis: session_id liegt im `metadata_json`; wir nutzen eine robuste
|
||||
LIKE-Suche, damit auch Legacy-Records gefunden werden.
|
||||
"""
|
||||
if not session_id:
|
||||
return []
|
||||
rows = self._conn.execute(
|
||||
"SELECT * FROM engrams WHERE metadata_json LIKE ? ORDER BY created_at DESC LIMIT ?",
|
||||
(f'%"session_id": "{session_id}"%', limit),
|
||||
).fetchall()
|
||||
engrams = [self._row_to_engram(r) for r in rows]
|
||||
if exclude_tags:
|
||||
ex = set(t for t in exclude_tags if isinstance(t, str))
|
||||
if ex:
|
||||
engrams = [e for e in engrams if not (ex & set(e.metadata.get("tags", []) or []))]
|
||||
return engrams
|
||||
|
||||
def delete(self, engram_id: str) -> bool:
|
||||
"""Löscht ein Engramm und alle Verknüpfungen."""
|
||||
rowid = self._conn.execute(
|
||||
|
||||
@@ -156,7 +156,45 @@ body {
|
||||
background:#12121a;
|
||||
border:1px solid #252533;
|
||||
border-radius: 14px;
|
||||
touch-action: none;
|
||||
}
|
||||
|
||||
.graph-controls{
|
||||
display:flex;
|
||||
gap:8px;
|
||||
padding: 10px 12px 0;
|
||||
align-items:center;
|
||||
flex-wrap: wrap;
|
||||
}
|
||||
.graph-controls .btn{
|
||||
background:#1e1e28;
|
||||
border:1px solid #2a2a3a;
|
||||
border-radius: 10px;
|
||||
padding: 8px 10px;
|
||||
color:#cfd3ff;
|
||||
font-weight:700;
|
||||
font-size:0.82rem;
|
||||
}
|
||||
.graph-controls .btn.primary{
|
||||
border-color:#6c8af5;
|
||||
box-shadow:0 0 0 1px rgba(108,138,245,0.18) inset;
|
||||
}
|
||||
.graph-legend{
|
||||
margin: 8px 12px 0;
|
||||
padding: 10px 12px;
|
||||
background:#1a1a24;
|
||||
border:1px solid #252533;
|
||||
border-radius: 14px;
|
||||
color:#b9b9c9;
|
||||
font-size:0.8rem;
|
||||
line-height:1.4;
|
||||
}
|
||||
.legend-row{ display:flex; align-items:center; gap:8px; margin-top:6px; }
|
||||
.legend-dot{ width:10px; height:10px; border-radius:50%; display:inline-block; }
|
||||
.legend-dot.engram{ background:#6c8af5; }
|
||||
.legend-dot.tag{ background:#8a9aff; }
|
||||
.legend-dot.match{ background:#f7d154; }
|
||||
.graph-hint{ padding: 4px 12px 10px; }
|
||||
#searchInput {
|
||||
flex: 1;
|
||||
background: #1e1e28;
|
||||
|
||||
7
systemd/openclaw-secondbrain-archive-stale.service
Normal file
7
systemd/openclaw-secondbrain-archive-stale.service
Normal file
@@ -0,0 +1,7 @@
|
||||
[Unit]
|
||||
Description=Second Brain Archive Stale
|
||||
PartOf=openclaw-secondbrain.target
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
ExecStart=/usr/bin/python3 /root/.openclaw/workspace/second-brain/cron_tasks/archive_stale.py
|
||||
10
systemd/openclaw-secondbrain-archive-stale.timer
Normal file
10
systemd/openclaw-secondbrain-archive-stale.timer
Normal file
@@ -0,0 +1,10 @@
|
||||
[Unit]
|
||||
Description=Archive stale engrams weekly (Sunday 03:00)
|
||||
PartOf=openclaw-secondbrain.target
|
||||
|
||||
[Timer]
|
||||
OnCalendar=Sun *-*-* 03:00:00
|
||||
Persistent=true
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
7
systemd/openclaw-secondbrain-auto-review.service
Normal file
7
systemd/openclaw-secondbrain-auto-review.service
Normal file
@@ -0,0 +1,7 @@
|
||||
[Unit]
|
||||
Description=Second Brain Auto Assign Review
|
||||
PartOf=openclaw-secondbrain.target
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
ExecStart=/usr/bin/python3 /root/.openclaw/workspace/second-brain/cron_tasks/auto_assign_review.py
|
||||
10
systemd/openclaw-secondbrain-auto-review.timer
Normal file
10
systemd/openclaw-secondbrain-auto-review.timer
Normal file
@@ -0,0 +1,10 @@
|
||||
[Unit]
|
||||
Description=Run auto assign review daily at 14:30
|
||||
PartOf=openclaw-secondbrain.target
|
||||
|
||||
[Timer]
|
||||
OnCalendar=*-*-* 14:30:00
|
||||
Persistent=true
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
7
systemd/openclaw-secondbrain-daily-summary.service
Normal file
7
systemd/openclaw-secondbrain-daily-summary.service
Normal file
@@ -0,0 +1,7 @@
|
||||
[Unit]
|
||||
Description=Second Brain Daily Summary
|
||||
PartOf=openclaw-secondbrain.target
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
ExecStart=/usr/bin/python3 /root/.openclaw/workspace/second-brain/cron_tasks/daily_summary.py
|
||||
10
systemd/openclaw-secondbrain-daily-summary.timer
Normal file
10
systemd/openclaw-secondbrain-daily-summary.timer
Normal file
@@ -0,0 +1,10 @@
|
||||
[Unit]
|
||||
Description=Daily Summary at 14:00
|
||||
PartOf=openclaw-secondbrain.target
|
||||
|
||||
[Timer]
|
||||
OnCalendar=*-*-* 14:00:00
|
||||
Persistent=true
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
7
systemd/openclaw-secondbrain-evaluate-pendings.service
Normal file
7
systemd/openclaw-secondbrain-evaluate-pendings.service
Normal file
@@ -0,0 +1,7 @@
|
||||
[Unit]
|
||||
Description=Second Brain Evaluate Pending Engrams
|
||||
After=network.target
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
ExecStart=/root/.openclaw/workspace/second-brain/.venv/bin/python3 /root/.openclaw/workspace/second-brain/cron_tasks/evaluate_all_pendings.py
|
||||
9
systemd/openclaw-secondbrain-evaluate-pendings.timer
Normal file
9
systemd/openclaw-secondbrain-evaluate-pendings.timer
Normal file
@@ -0,0 +1,9 @@
|
||||
[Unit]
|
||||
Description=Run Second Brain Evaluate Pending every hour
|
||||
|
||||
[Timer]
|
||||
OnCalendar=hourly
|
||||
Persistent=true
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
7
systemd/openclaw-secondbrain-health-check.service
Normal file
7
systemd/openclaw-secondbrain-health-check.service
Normal file
@@ -0,0 +1,7 @@
|
||||
[Unit]
|
||||
Description=Second Brain Health Check
|
||||
PartOf=openclaw-secondbrain.target
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
ExecStart=/usr/bin/python3 /root/.openclaw/workspace/second-brain/cron_tasks/health_check.py
|
||||
10
systemd/openclaw-secondbrain-health-check.timer
Normal file
10
systemd/openclaw-secondbrain-health-check.timer
Normal file
@@ -0,0 +1,10 @@
|
||||
[Unit]
|
||||
Description=Run health check at 02:00 and 14:00 daily
|
||||
PartOf=openclaw-secondbrain.target
|
||||
|
||||
[Timer]
|
||||
OnCalendar=*-*-* 02,14:00:00
|
||||
Persistent=true
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
@@ -0,0 +1,7 @@
|
||||
[Unit]
|
||||
Description=Second Brain Import Context Buffer
|
||||
PartOf=openclaw-secondbrain.target
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
ExecStart=/usr/bin/python3 /root/.openclaw/workspace/second-brain/cron_tasks/import_context_buffer.py
|
||||
10
systemd/openclaw-secondbrain-import-context-buffer.timer
Normal file
10
systemd/openclaw-secondbrain-import-context-buffer.timer
Normal file
@@ -0,0 +1,10 @@
|
||||
[Unit]
|
||||
Description=Import Context Buffer every 4 hours
|
||||
PartOf=openclaw-secondbrain.target
|
||||
|
||||
[Timer]
|
||||
OnCalendar=*-*-* 02,06,10,14,18,22:00:00
|
||||
Persistent=true
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
7
systemd/openclaw-secondbrain-predictive-links.service
Normal file
7
systemd/openclaw-secondbrain-predictive-links.service
Normal file
@@ -0,0 +1,7 @@
|
||||
[Unit]
|
||||
Description=Second Brain Predictive Links
|
||||
PartOf=openclaw-secondbrain.target
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
ExecStart=/usr/bin/python3 /root/.openclaw/workspace/second-brain/cron_tasks/predictive_links.py
|
||||
10
systemd/openclaw-secondbrain-predictive-links.timer
Normal file
10
systemd/openclaw-secondbrain-predictive-links.timer
Normal file
@@ -0,0 +1,10 @@
|
||||
[Unit]
|
||||
Description=Run predictive links daily at 02:30
|
||||
PartOf=openclaw-secondbrain.target
|
||||
|
||||
[Timer]
|
||||
OnCalendar=*-*-* 02:30:00
|
||||
Persistent=true
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
7
systemd/openclaw-secondbrain-tag-normalizer.service
Normal file
7
systemd/openclaw-secondbrain-tag-normalizer.service
Normal file
@@ -0,0 +1,7 @@
|
||||
[Unit]
|
||||
Description=Second Brain Tag Normalizer
|
||||
PartOf=openclaw-secondbrain.target
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
ExecStart=/usr/bin/python3 /root/.openclaw/workspace/second-brain/cron_tasks/tag_normalizer.py
|
||||
10
systemd/openclaw-secondbrain-tag-normalizer.timer
Normal file
10
systemd/openclaw-secondbrain-tag-normalizer.timer
Normal file
@@ -0,0 +1,10 @@
|
||||
[Unit]
|
||||
Description=Tag Normalizer weekly (Sunday 03:15)
|
||||
PartOf=openclaw-secondbrain.target
|
||||
|
||||
[Timer]
|
||||
OnCalendar=Sun *-*-* 03:15:00
|
||||
Persistent=true
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
@@ -45,9 +45,32 @@
|
||||
<div class="cards" id="cards"></div>
|
||||
|
||||
<!-- Graph -->
|
||||
<div class="graph" id="graph" style="display:none;">
|
||||
<div class="graph" id="graph" style="display:none;">
|
||||
<div class="graph-controls">
|
||||
<button class="btn primary" id="btnGraphPhysics" onclick="toggleGraphPhysics()">Physics: off</button>
|
||||
<button class="btn" onclick="resetGraphView()">Reset view</button>
|
||||
<button class="btn" onclick="fitGraphView()">Fit</button>
|
||||
<label class="muted small" style="display:flex;align-items:center;gap:8px">
|
||||
<span>Physics</span>
|
||||
<input id="physicsStrength" type="range" min="0" max="100" value="60" oninput="setPhysicsStrength(this.value)" style="width:140px">
|
||||
<span id="physicsStrengthVal">60</span>
|
||||
</label>
|
||||
<select class="btn" id="graphLimit" onchange="reloadGraph()" title="Wie viele Knoten laden? 0=all">
|
||||
<option value="0">Nodes: all</option>
|
||||
<option value="200">Nodes: 200</option>
|
||||
<option value="1000">Nodes: 1000</option>
|
||||
<option value="5000">Nodes: 5000</option>
|
||||
</select>
|
||||
<button class="btn" onclick="reloadGraph()">Reload</button>
|
||||
</div>
|
||||
<canvas id="graphCanvas" width="440" height="520"></canvas>
|
||||
<div class="muted small" id="graphHint">Lade Graph…</div>
|
||||
<div class="graph-hint muted small" id="graphHint">Lade Graph…</div>
|
||||
<div class="graph-legend">
|
||||
<div><strong>Graph</strong>: Zoom per Pinch (2 Finger), Pan per Drag (1 Finger). Tap auf Engram öffnet Details, Tap auf Tag setzt Suche.</div>
|
||||
<div class="legend-row"><span class="legend-dot engram"></span> Engram</div>
|
||||
<div class="legend-row"><span class="legend-dot tag"></span> Tag</div>
|
||||
<div class="legend-row"><span class="legend-dot match"></span> Match (Suche)</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Status -->
|
||||
@@ -90,7 +113,18 @@ let state = {
|
||||
// ─── Fetch ──────────────────────────────────────────────────────────────────
|
||||
async function api(path, opts = {}) {
|
||||
const r = await fetch(path, opts);
|
||||
if (!r.ok) throw new Error((await r.json()).error || r.statusText);
|
||||
if (!r.ok) {
|
||||
let msg = r.statusText;
|
||||
try {
|
||||
const j = await r.json();
|
||||
msg = j.error || j.detail || msg;
|
||||
if (Array.isArray(msg)) msg = JSON.stringify(msg);
|
||||
if (typeof msg !== 'string') msg = JSON.stringify(msg);
|
||||
} catch (e) {
|
||||
// ignore JSON parse errors
|
||||
}
|
||||
throw new Error(msg);
|
||||
}
|
||||
return r.json();
|
||||
}
|
||||
|
||||
@@ -143,14 +177,23 @@ async function loadCards() {
|
||||
}
|
||||
|
||||
async function loadStatus() {
|
||||
const [cfg, db, jobs, ins, stor] = await Promise.all([
|
||||
const reqs = await Promise.allSettled([
|
||||
api('/api/config'),
|
||||
api('/api/db_info'),
|
||||
api('/api/jobs'),
|
||||
api('/api/insights?limit=8'),
|
||||
api('/api/storage_stats'),
|
||||
api('/api/pending?limit=20&offset=0'),
|
||||
]);
|
||||
const pend = await api('/api/pending?limit=20&offset=0');
|
||||
const pick = (i, fallback) => (reqs[i].status === 'fulfilled' ? reqs[i].value : fallback);
|
||||
const err = (i) => (reqs[i].status === 'rejected' ? (reqs[i].reason && reqs[i].reason.message ? reqs[i].reason.message : String(reqs[i].reason)) : null);
|
||||
|
||||
const cfg = pick(0, { workspace: '-', db_path: '-' });
|
||||
const db = pick(1, { db_path: '-', mtime: null });
|
||||
const jobs = pick(2, { items: [], error: err(2) });
|
||||
const ins = pick(3, { pending: '-', top_tags: [], top_hosts: [], error: err(3) });
|
||||
const stor = pick(4, { sql: { total_engrams: '-', confirmed: '-', pending: '-', by_source: {} }, vector: { chroma_size_bytes: 0, embedding_cache_files: 0 }, obsidian: { configured: false } });
|
||||
const pend = pick(5, { items: [], error: err(5) });
|
||||
|
||||
const el = document.getElementById('status');
|
||||
const jobsHtml = (jobs.items || []).map(j => `
|
||||
@@ -198,99 +241,559 @@ async function loadStatus() {
|
||||
</div>
|
||||
<div class="panel">
|
||||
<div class="panel-title">Jobs</div>
|
||||
${jobsHtml || '<div class="muted">Keine Daten</div>'}
|
||||
${jobs.error ? `<div class="muted">Fehler: ${escapeHtml(jobs.error)}</div>` : (jobsHtml || '<div class="muted">Keine Daten</div>')}
|
||||
</div>
|
||||
<div class="panel">
|
||||
<div class="panel-title">Insights</div>
|
||||
${ins.error ? `<div class="muted">Fehler: ${escapeHtml(ins.error)}</div>` : ''}
|
||||
<div class="kv-row"><div class="kv-key">pending</div><div class="kv-val">${ins.pending}</div></div>
|
||||
<div class="kv-row"><div class="kv-key">top tags</div><div class="kv-val">${topTags || '-'}</div></div>
|
||||
<div class="kv-row"><div class="kv-key">top hosts</div><div class="kv-val">${topHosts || '-'}</div></div>
|
||||
</div>
|
||||
<div class="panel">
|
||||
<div class="panel-title">Pending Queue (latest)</div>
|
||||
${pendHtml || '<div class="muted">Keine Pendings</div>'}
|
||||
${pend.error ? `<div class="muted">Fehler: ${escapeHtml(pend.error)}</div>` : (pendHtml || '<div class="muted">Keine Pendings</div>')}
|
||||
</div>
|
||||
`;
|
||||
}
|
||||
|
||||
async function loadGraph() {
|
||||
const g = await api('/api/graph?limit_nodes=200');
|
||||
renderGraph(g.nodes || [], g.edges || []);
|
||||
const sel = document.getElementById('graphLimit');
|
||||
const fromSel = sel ? parseInt(sel.value || '0', 10) : NaN;
|
||||
const fromStore = parseInt(localStorage.getItem('graphLimit') || '0', 10);
|
||||
const q = (!Number.isNaN(fromSel)) ? fromSel : (Number.isNaN(fromStore) ? 0 : fromStore);
|
||||
if (sel) sel.value = String(q);
|
||||
if (sel) localStorage.setItem('graphLimit', String(q));
|
||||
const hint = document.getElementById('graphHint');
|
||||
if (hint) hint.textContent = 'Lade Graph…';
|
||||
try {
|
||||
const g = await api(`/api/graph?limit_nodes=${q}`);
|
||||
renderGraph(g.nodes || [], g.edges || []);
|
||||
} catch (e) {
|
||||
if (hint) hint.textContent = `Graph-Fehler: ${e && e.message ? e.message : String(e)}`;
|
||||
const canvas = _graphCanvas();
|
||||
const ctx = _graphCtx();
|
||||
if (canvas && ctx) ctx.clearRect(0,0,canvas.width,canvas.height);
|
||||
}
|
||||
}
|
||||
|
||||
function reloadGraph() { loadGraph(); }
|
||||
|
||||
// ─── Graph Renderer (Canvas) ────────────────────────────────────────────────
|
||||
let graphState = {
|
||||
nodes: [],
|
||||
edges: [],
|
||||
sim: [],
|
||||
links: [],
|
||||
nodeById: new Map(),
|
||||
simById: new Map(),
|
||||
degree: new Map(),
|
||||
physicsOn: false,
|
||||
draggingId: null,
|
||||
selectedId: null,
|
||||
panning: false,
|
||||
lastX: 0,
|
||||
lastY: 0,
|
||||
panX: 0,
|
||||
panY: 0,
|
||||
zoom: 1,
|
||||
raf: null,
|
||||
search: '',
|
||||
pointers: new Map(),
|
||||
pinchStartDist: null,
|
||||
pinchStartZoom: null,
|
||||
pinchStartPan: null,
|
||||
down: null, // {pointerId, cx, cy, t}
|
||||
physicsStrength: parseInt(localStorage.getItem('physicsStrength') || '60', 10),
|
||||
};
|
||||
|
||||
function _graphCanvas() { return document.getElementById('graphCanvas'); }
|
||||
function _graphCtx() { return _graphCanvas().getContext('2d'); }
|
||||
|
||||
function _graphNodeRadius(n) {
|
||||
const d = graphState.degree.get(n.id) || 0;
|
||||
const base = n.kind === 'tag' ? 4 : (n.kind === 'host' ? 5 : 7);
|
||||
const w = (n.weight || 0);
|
||||
const bonus = Math.min(6, Math.sqrt(Math.max(0, w)) * 0.8);
|
||||
return Math.max(3, Math.min(18, base + Math.sqrt(d) + bonus));
|
||||
}
|
||||
|
||||
function _graphNodeFill(n) {
|
||||
const d = graphState.degree.get(n.id) || 0;
|
||||
const t = Math.max(0, Math.min(0.55, d / 18)); // higher degree -> brighter
|
||||
const mix = (rgb) => rgb.map(c => Math.round(c + (255 - c) * t));
|
||||
|
||||
if (n.kind === 'tag') {
|
||||
const [r,g,b] = mix([167, 139, 250]);
|
||||
return `rgb(${r},${g},${b})`;
|
||||
}
|
||||
if (n.kind === 'host') {
|
||||
const [r,g,b] = mix([245, 158, 11]);
|
||||
return `rgb(${r},${g},${b})`;
|
||||
}
|
||||
|
||||
const verdict = (n.verdict || '').toString();
|
||||
let base = [96, 165, 250]; // pending/unknown = blue
|
||||
if (verdict === 'confirmed_true') base = [34, 197, 94]; // green
|
||||
else if (verdict === 'confirmed_false') base = [239, 68, 68]; // red
|
||||
else if (verdict === 'probable_true') base = [52, 211, 153]; // teal
|
||||
else if (verdict === 'probable_false') base = [251, 191, 36]; // amber
|
||||
|
||||
// Recency brightening (brand new pops)
|
||||
const now = Date.now();
|
||||
const created = Date.parse(n.created || '') || 0;
|
||||
const ageMin = created ? (now - created) / 60000 : 999999;
|
||||
const rec = Math.max(0, Math.min(0.35, (30 - ageMin) / 30 * 0.35));
|
||||
const bump = (c) => Math.round(c + (255 - c) * rec);
|
||||
const [r,g,b] = mix(base).map(bump);
|
||||
return `rgb(${r},${g},${b})`;
|
||||
}
|
||||
|
||||
function _graphMatches(n, term) {
|
||||
const t = (term || '').trim().toLowerCase();
|
||||
if (!t) return false;
|
||||
const id = (n.id || '').toLowerCase();
|
||||
const label = (n.label || '').toLowerCase();
|
||||
return id.includes(t) || label.includes(t);
|
||||
}
|
||||
|
||||
function _graphWorldFromScreen(cx, cy) {
|
||||
return {
|
||||
x: (cx - graphState.panX) / graphState.zoom,
|
||||
y: (cy - graphState.panY) / graphState.zoom,
|
||||
};
|
||||
}
|
||||
|
||||
function _graphHitTest(wx, wy) {
|
||||
for (let i = graphState.sim.length - 1; i >= 0; i--) {
|
||||
const n = graphState.sim[i];
|
||||
const r = _graphNodeRadius(n) + 2;
|
||||
const dx = wx - n.x;
|
||||
const dy = wy - n.y;
|
||||
if ((dx*dx + dy*dy) <= r*r) return n;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function _graphInitInteractions() {
|
||||
const canvas = _graphCanvas();
|
||||
if (canvas._graphBound) return;
|
||||
canvas._graphBound = true;
|
||||
|
||||
const toCanvasXY = (ev) => {
|
||||
const rect = canvas.getBoundingClientRect();
|
||||
return { cx: ev.clientX - rect.left, cy: ev.clientY - rect.top };
|
||||
};
|
||||
|
||||
const pinchMidpoint = () => {
|
||||
const pts = Array.from(graphState.pointers.values());
|
||||
if (pts.length < 2) return null;
|
||||
return { cx: (pts[0].cx + pts[1].cx) / 2, cy: (pts[0].cy + pts[1].cy) / 2 };
|
||||
};
|
||||
|
||||
const pinchDistance = () => {
|
||||
const pts = Array.from(graphState.pointers.values());
|
||||
if (pts.length < 2) return null;
|
||||
const dx = pts[0].cx - pts[1].cx;
|
||||
const dy = pts[0].cy - pts[1].cy;
|
||||
return Math.sqrt(dx*dx + dy*dy);
|
||||
};
|
||||
|
||||
canvas.addEventListener('pointerdown', (ev) => {
|
||||
canvas.setPointerCapture(ev.pointerId);
|
||||
const pos = toCanvasXY(ev);
|
||||
graphState.pointers.set(ev.pointerId, pos);
|
||||
graphState.down = { pointerId: ev.pointerId, cx: pos.cx, cy: pos.cy, t: Date.now() };
|
||||
|
||||
// When 2 pointers: start pinch
|
||||
if (graphState.pointers.size === 2) {
|
||||
graphState.panning = false;
|
||||
graphState.draggingId = null;
|
||||
graphState.down = null;
|
||||
graphState.pinchStartDist = pinchDistance();
|
||||
graphState.pinchStartZoom = graphState.zoom;
|
||||
graphState.pinchStartPan = { panX: graphState.panX, panY: graphState.panY };
|
||||
_graphDraw();
|
||||
return;
|
||||
}
|
||||
|
||||
// Single pointer: pan or drag node
|
||||
const { cx, cy } = pos;
|
||||
const w = _graphWorldFromScreen(cx, cy);
|
||||
const hit = _graphHitTest(w.x, w.y);
|
||||
graphState.lastX = cx; graphState.lastY = cy;
|
||||
if (hit) {
|
||||
graphState.draggingId = hit.id;
|
||||
graphState.selectedId = hit.id;
|
||||
} else {
|
||||
graphState.panning = true;
|
||||
graphState.selectedId = null;
|
||||
}
|
||||
_graphDraw();
|
||||
});
|
||||
|
||||
canvas.addEventListener('pointermove', (ev) => {
|
||||
if (!graphState.pointers.has(ev.pointerId)) return;
|
||||
const pos = toCanvasXY(ev);
|
||||
graphState.pointers.set(ev.pointerId, pos);
|
||||
|
||||
// Pinch zoom (2 pointers)
|
||||
if (graphState.pointers.size >= 2 && graphState.pinchStartDist) {
|
||||
const dist = pinchDistance();
|
||||
const mid = pinchMidpoint();
|
||||
if (!dist || !mid) return;
|
||||
|
||||
const zoomOld = graphState.zoom;
|
||||
const wx = (mid.cx - graphState.panX) / zoomOld;
|
||||
const wy = (mid.cy - graphState.panY) / zoomOld;
|
||||
|
||||
const factor = dist / graphState.pinchStartDist;
|
||||
const next = Math.max(0.35, Math.min(3.0, graphState.pinchStartZoom * factor));
|
||||
graphState.zoom = next;
|
||||
graphState.panX = mid.cx - wx * graphState.zoom;
|
||||
graphState.panY = mid.cy - wy * graphState.zoom;
|
||||
_graphDraw();
|
||||
return;
|
||||
}
|
||||
|
||||
// Single pointer
|
||||
const { cx, cy } = pos;
|
||||
const dx = cx - graphState.lastX;
|
||||
const dy = cy - graphState.lastY;
|
||||
graphState.lastX = cx; graphState.lastY = cy;
|
||||
|
||||
if (graphState.draggingId) {
|
||||
const w = _graphWorldFromScreen(cx, cy);
|
||||
const n = graphState.simById.get(graphState.draggingId);
|
||||
if (n) { n.x = w.x; n.y = w.y; n.vx = 0; n.vy = 0; }
|
||||
_graphDraw();
|
||||
return;
|
||||
}
|
||||
if (graphState.panning) {
|
||||
graphState.panX += dx;
|
||||
graphState.panY += dy;
|
||||
_graphDraw();
|
||||
}
|
||||
});
|
||||
|
||||
const endPointer = (ev) => {
|
||||
if (graphState.pointers.has(ev.pointerId)) graphState.pointers.delete(ev.pointerId);
|
||||
try { canvas.releasePointerCapture(ev.pointerId); } catch {}
|
||||
|
||||
if (graphState.pointers.size < 2) {
|
||||
graphState.pinchStartDist = null;
|
||||
graphState.pinchStartZoom = null;
|
||||
graphState.pinchStartPan = null;
|
||||
}
|
||||
|
||||
const pos = toCanvasXY(ev);
|
||||
const down = graphState.down && graphState.down.pointerId === ev.pointerId ? graphState.down : null;
|
||||
graphState.down = null;
|
||||
|
||||
const moved = down ? Math.hypot(pos.cx - down.cx, pos.cy - down.cy) : 999;
|
||||
const isTap = !!down && moved <= 8 && (Date.now() - down.t) <= 500;
|
||||
|
||||
const w = _graphWorldFromScreen(pos.cx, pos.cy);
|
||||
const hit = _graphHitTest(w.x, w.y);
|
||||
|
||||
graphState.draggingId = null;
|
||||
graphState.panning = false;
|
||||
|
||||
if (isTap && hit) {
|
||||
graphState.selectedId = hit.id;
|
||||
_graphDraw();
|
||||
if (hit.kind === 'engram') {
|
||||
showDetail(hit.id);
|
||||
} else if (hit.kind === 'tag') {
|
||||
const label = (hit.label || '').replace(/^tag:/, '');
|
||||
const inp = document.getElementById('searchInput');
|
||||
inp.value = label;
|
||||
inp.dispatchEvent(new Event('input'));
|
||||
}
|
||||
return;
|
||||
}
|
||||
_graphDraw();
|
||||
};
|
||||
canvas.addEventListener('pointerup', endPointer);
|
||||
canvas.addEventListener('pointercancel', endPointer);
|
||||
}
|
||||
|
||||
function renderGraph(nodes, edges) {
|
||||
const canvas = document.getElementById('graphCanvas');
|
||||
const canvas = _graphCanvas();
|
||||
const hint = document.getElementById('graphHint');
|
||||
const ctx = canvas.getContext('2d');
|
||||
const ctx = _graphCtx();
|
||||
|
||||
// sync physics slider
|
||||
const slider = document.getElementById('physicsStrength');
|
||||
const sliderVal = document.getElementById('physicsStrengthVal');
|
||||
const s = Math.max(0, Math.min(100, parseInt(graphState.physicsStrength || 60, 10)));
|
||||
graphState.physicsStrength = s;
|
||||
if (slider) slider.value = String(s);
|
||||
if (sliderVal) sliderVal.textContent = String(s);
|
||||
|
||||
// Fit canvas to container width (mobile)
|
||||
const w = canvas.parentElement.clientWidth - 24;
|
||||
canvas.width = Math.max(320, Math.min(520, w));
|
||||
canvas.height = 520;
|
||||
canvas.width = Math.max(320, w);
|
||||
canvas.height = Math.max(520, Math.min(900, (window.innerHeight || 900) - 260));
|
||||
|
||||
if (!nodes.length || !edges.length) {
|
||||
hint.textContent = 'Graph: keine Kanten (noch keine Links/Tags/Hosts im Sample).';
|
||||
graphState.nodes = nodes || [];
|
||||
graphState.edges = edges || [];
|
||||
graphState.nodeById = new Map(graphState.nodes.map(n => [n.id, n]));
|
||||
graphState.degree = new Map();
|
||||
for (const e of graphState.edges) {
|
||||
graphState.degree.set(e.from, (graphState.degree.get(e.from) || 0) + 1);
|
||||
graphState.degree.set(e.to, (graphState.degree.get(e.to) || 0) + 1);
|
||||
}
|
||||
|
||||
if (!graphState.nodes.length) {
|
||||
hint.textContent = 'Graph: keine Nodes.';
|
||||
ctx.clearRect(0,0,canvas.width,canvas.height);
|
||||
return;
|
||||
}
|
||||
|
||||
hint.textContent = `nodes=${nodes.length} edges=${edges.length}`;
|
||||
hint.textContent = `nodes=${graphState.nodes.length} edges=${graphState.edges.length}`;
|
||||
|
||||
const nodeById = new Map(nodes.map(n => [n.id, n]));
|
||||
const sim = nodes.map(n => ({
|
||||
id: n.id,
|
||||
kind: n.kind,
|
||||
label: n.label || n.id,
|
||||
x: Math.random()*canvas.width,
|
||||
y: Math.random()*canvas.height,
|
||||
vx: 0, vy: 0,
|
||||
}));
|
||||
const simById = new Map(sim.map(n => [n.id, n]));
|
||||
// Build adjacency for deterministic layout (O(n+e))
|
||||
const adj = new Map();
|
||||
const addAdj = (a, b) => {
|
||||
if (!adj.has(a)) adj.set(a, []);
|
||||
adj.get(a).push(b);
|
||||
};
|
||||
for (const e of graphState.edges) {
|
||||
addAdj(e.from, e.to);
|
||||
addAdj(e.to, e.from);
|
||||
}
|
||||
|
||||
const links = edges
|
||||
.map(e => ({a: simById.get(e.from), b: simById.get(e.to), kind: e.kind}))
|
||||
const degree = graphState.degree;
|
||||
const visited = new Set();
|
||||
const comps = [];
|
||||
for (const n of graphState.nodes) {
|
||||
if (visited.has(n.id)) continue;
|
||||
const q = [n.id];
|
||||
visited.add(n.id);
|
||||
const comp = [];
|
||||
while (q.length) {
|
||||
const cur = q.pop();
|
||||
comp.push(cur);
|
||||
const neigh = adj.get(cur) || [];
|
||||
for (const nb of neigh) {
|
||||
if (!visited.has(nb)) {
|
||||
visited.add(nb);
|
||||
q.push(nb);
|
||||
}
|
||||
}
|
||||
}
|
||||
comps.push(comp);
|
||||
}
|
||||
comps.sort((a, b) => b.length - a.length);
|
||||
|
||||
// component centers in a loose grid/spiral
|
||||
const centers = [];
|
||||
const gap = 240;
|
||||
const cols = Math.max(1, Math.floor(canvas.width / gap));
|
||||
for (let i = 0; i < comps.length; i++) {
|
||||
const cx = (i % cols) * gap - ((cols - 1) * gap) / 2;
|
||||
const cy = Math.floor(i / cols) * gap - (Math.floor((comps.length - 1) / cols) * gap) / 2;
|
||||
centers.push({ cx, cy });
|
||||
}
|
||||
|
||||
const pos = new Map();
|
||||
const R = 46;
|
||||
for (let ci = 0; ci < comps.length; ci++) {
|
||||
const comp = comps[ci];
|
||||
const center = centers[ci] || { cx: 0, cy: 0 };
|
||||
|
||||
// hub: highest degree engram preferred
|
||||
let hub = comp[0];
|
||||
let hubScore = -1;
|
||||
for (const id of comp) {
|
||||
const n = graphState.nodeById.get(id) || {};
|
||||
const score = (degree.get(id) || 0) + (n.kind === 'engram' ? 3 : 0);
|
||||
if (score > hubScore) { hubScore = score; hub = id; }
|
||||
}
|
||||
|
||||
// BFS layers from hub
|
||||
const layer = new Map();
|
||||
layer.set(hub, 0);
|
||||
const qq = [hub];
|
||||
while (qq.length) {
|
||||
const cur = qq.shift();
|
||||
const l = layer.get(cur) || 0;
|
||||
const neigh = adj.get(cur) || [];
|
||||
for (const nb of neigh) {
|
||||
if (!layer.has(nb)) {
|
||||
layer.set(nb, l + 1);
|
||||
qq.push(nb);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const buckets = new Map();
|
||||
for (const id of comp) {
|
||||
const l = layer.get(id);
|
||||
const ll = (typeof l === 'number') ? l : 99;
|
||||
if (!buckets.has(ll)) buckets.set(ll, []);
|
||||
buckets.get(ll).push(id);
|
||||
}
|
||||
|
||||
// Place hubs first, then rings
|
||||
pos.set(hub, { x: center.cx, y: center.cy });
|
||||
const maxLayer = Math.max(...Array.from(buckets.keys()));
|
||||
for (let l = 1; l <= maxLayer; l++) {
|
||||
const ids = buckets.get(l) || [];
|
||||
if (!ids.length) continue;
|
||||
// Stable order: engrams first, then by degree desc
|
||||
ids.sort((a, b) => {
|
||||
const na = graphState.nodeById.get(a) || {};
|
||||
const nb = graphState.nodeById.get(b) || {};
|
||||
const ka = na.kind === 'engram' ? 0 : (na.kind === 'tag' ? 1 : 2);
|
||||
const kb = nb.kind === 'engram' ? 0 : (nb.kind === 'tag' ? 1 : 2);
|
||||
if (ka !== kb) return ka - kb;
|
||||
return (degree.get(b) || 0) - (degree.get(a) || 0);
|
||||
});
|
||||
const rad = l * R;
|
||||
for (let i = 0; i < ids.length; i++) {
|
||||
const ang = (i / ids.length) * Math.PI * 2;
|
||||
pos.set(ids[i], {
|
||||
x: center.cx + Math.cos(ang) * rad,
|
||||
y: center.cy + Math.sin(ang) * rad,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
graphState.sim = graphState.nodes.map(n => {
|
||||
const p = pos.get(n.id) || { x: (Math.random() - 0.5) * canvas.width, y: (Math.random() - 0.5) * canvas.height };
|
||||
return {
|
||||
id: n.id,
|
||||
kind: n.kind,
|
||||
label: n.label || n.id,
|
||||
weight: n.weight,
|
||||
verdict: n.verdict,
|
||||
confidence: n.confidence,
|
||||
created: n.created,
|
||||
modified: n.modified,
|
||||
last_accessed: n.last_accessed,
|
||||
predict_locked: n.predict_locked,
|
||||
x: p.x,
|
||||
y: p.y,
|
||||
vx: 0, vy: 0,
|
||||
};
|
||||
});
|
||||
graphState.simById = new Map(graphState.sim.map(n => [n.id, n]));
|
||||
graphState.links = graphState.edges
|
||||
.map(e => ({a: graphState.simById.get(e.from), b: graphState.simById.get(e.to), kind: e.kind, weight: e.weight || 1.0}))
|
||||
.filter(l => l.a && l.b);
|
||||
|
||||
// Simple force layout (few iterations)
|
||||
for (let iter=0; iter<180; iter++) {
|
||||
// repulsion
|
||||
for (let i=0; i<sim.length; i++) {
|
||||
for (let j=i+1; j<sim.length; j++) {
|
||||
graphState.panX = canvas.width / 2;
|
||||
graphState.panY = canvas.height / 2;
|
||||
graphState.zoom = 1;
|
||||
graphState.search = state.search || '';
|
||||
graphState.selectedId = null;
|
||||
|
||||
_graphInitInteractions();
|
||||
// Physics is expensive; pre-relax a little but keep it optional for big graphs.
|
||||
const relaxIters = graphState.sim.length <= 700 ? 70 : 12;
|
||||
for (let i = 0; i < relaxIters; i++) _graphStepPhysics(0.85);
|
||||
_graphDraw();
|
||||
}
|
||||
|
||||
function _graphStepPhysics(alpha = 1.0) {
|
||||
const canvas = _graphCanvas();
|
||||
const repulsion = (graphState.sim.length > 700) ? 120 : 180;
|
||||
const damping = 0.86;
|
||||
const target = 80;
|
||||
const springK = 0.018;
|
||||
|
||||
const sim = graphState.sim;
|
||||
if (sim.length > 700) {
|
||||
// Fast approximate repulsion via spatial hashing (checks only local neighbor cells).
|
||||
const cell = 120;
|
||||
const grid = new Map();
|
||||
const key = (cx, cy) => `${cx},${cy}`;
|
||||
for (let i = 0; i < sim.length; i++) {
|
||||
const n = sim[i];
|
||||
const cx = Math.floor(n.x / cell);
|
||||
const cy = Math.floor(n.y / cell);
|
||||
const k = key(cx, cy);
|
||||
if (!grid.has(k)) grid.set(k, []);
|
||||
grid.get(k).push(i);
|
||||
}
|
||||
for (let i = 0; i < sim.length; i++) {
|
||||
const a = sim[i];
|
||||
const cx = Math.floor(a.x / cell);
|
||||
const cy = Math.floor(a.y / cell);
|
||||
for (let ox = -1; ox <= 1; ox++) {
|
||||
for (let oy = -1; oy <= 1; oy++) {
|
||||
const idxs = grid.get(key(cx + ox, cy + oy));
|
||||
if (!idxs) continue;
|
||||
for (const j of idxs) {
|
||||
if (j === i) continue;
|
||||
const b = sim[j];
|
||||
const dx = a.x - b.x, dy = a.y - b.y;
|
||||
const d2 = dx*dx + dy*dy + 0.12;
|
||||
const f = (repulsion / d2) * alpha * 0.55;
|
||||
a.vx += dx*f; a.vy += dy*f;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
for (let i = 0; i < sim.length; i++) {
|
||||
for (let j = i + 1; j < sim.length; j++) {
|
||||
const a = sim[i], b = sim[j];
|
||||
const dx = a.x - b.x, dy = a.y - b.y;
|
||||
const d2 = dx*dx + dy*dy + 0.01;
|
||||
const f = 120 / d2;
|
||||
const d2 = dx*dx + dy*dy + 0.02;
|
||||
const f = (repulsion / d2) * alpha;
|
||||
a.vx += dx*f; a.vy += dy*f;
|
||||
b.vx -= dx*f; b.vy -= dy*f;
|
||||
}
|
||||
}
|
||||
// springs
|
||||
for (const l of links) {
|
||||
const a = l.a, b = l.b;
|
||||
const dx = b.x - a.x, dy = b.y - a.y;
|
||||
const dist = Math.sqrt(dx*dx + dy*dy) || 1;
|
||||
const target = 60;
|
||||
const k = 0.02;
|
||||
const f = (dist - target) * k;
|
||||
const fx = (dx/dist)*f, fy = (dy/dist)*f;
|
||||
a.vx += fx; a.vy += fy;
|
||||
b.vx -= fx; b.vy -= fy;
|
||||
}
|
||||
// integrate + bounds
|
||||
for (const n of sim) {
|
||||
n.vx *= 0.85; n.vy *= 0.85;
|
||||
n.x += n.vx; n.y += n.vy;
|
||||
n.x = Math.max(10, Math.min(canvas.width-10, n.x));
|
||||
n.y = Math.max(10, Math.min(canvas.height-10, n.y));
|
||||
}
|
||||
}
|
||||
for (const l of graphState.links) {
|
||||
const a = l.a, b = l.b;
|
||||
const dx = b.x - a.x, dy = b.y - a.y;
|
||||
const dist = Math.sqrt(dx*dx + dy*dy) || 1;
|
||||
const f = (dist - target) * springK * alpha;
|
||||
const fx = (dx/dist) * f, fy = (dy/dist) * f;
|
||||
a.vx += fx; a.vy += fy;
|
||||
b.vx -= fx; b.vy -= fy;
|
||||
}
|
||||
for (const n of sim) {
|
||||
n.vx *= damping; n.vy *= damping;
|
||||
n.x += n.vx; n.y += n.vy;
|
||||
const pad = 10;
|
||||
const bx = canvas.width / graphState.zoom;
|
||||
const by = canvas.height / graphState.zoom;
|
||||
n.x = Math.max(-bx/2 + pad, Math.min(bx/2 - pad, n.x));
|
||||
n.y = Math.max(-by/2 + pad, Math.min(by/2 - pad, n.y));
|
||||
}
|
||||
}
|
||||
|
||||
function _graphEdgeColor(kind) {
|
||||
const k = (kind || '').toString().toLowerCase();
|
||||
if (k.includes('tag')) return '#7c3aed';
|
||||
if (k.includes('host')) return '#f59e0b';
|
||||
if (k.includes('ref')) return '#10b981';
|
||||
return '#3a3a55';
|
||||
}
|
||||
|
||||
function _graphDraw() {
|
||||
const canvas = _graphCanvas();
|
||||
const ctx = _graphCtx();
|
||||
const hint = document.getElementById('graphHint');
|
||||
|
||||
ctx.clearRect(0,0,canvas.width,canvas.height);
|
||||
// edges
|
||||
ctx.globalAlpha = 0.5;
|
||||
ctx.strokeStyle = '#3a3a55';
|
||||
ctx.lineWidth = 1;
|
||||
for (const l of links) {
|
||||
ctx.save();
|
||||
ctx.translate(graphState.panX, graphState.panY);
|
||||
ctx.scale(graphState.zoom, graphState.zoom);
|
||||
|
||||
for (const l of graphState.links) {
|
||||
const term = (graphState.search || '').trim();
|
||||
const isMatchEdge = term && (_graphMatches(l.a, term) || _graphMatches(l.b, term));
|
||||
const w = Math.max(0.2, Math.min(3.0, (l.weight || 1.0)));
|
||||
ctx.lineWidth = (0.6 + w) / graphState.zoom;
|
||||
ctx.globalAlpha = isMatchEdge ? 0.85 : (0.25 + Math.min(0.35, w * 0.18));
|
||||
ctx.strokeStyle = isMatchEdge ? '#f7d154' : _graphEdgeColor(l.kind);
|
||||
ctx.beginPath();
|
||||
ctx.moveTo(l.a.x, l.a.y);
|
||||
ctx.lineTo(l.b.x, l.b.y);
|
||||
@@ -298,19 +801,124 @@ function renderGraph(nodes, edges) {
|
||||
}
|
||||
ctx.globalAlpha = 1.0;
|
||||
|
||||
// nodes
|
||||
for (const n of sim) {
|
||||
let r = 5;
|
||||
let fill = '#6c8af5';
|
||||
if (n.kind === 'tag') { fill = '#8a9aff'; r = 4; }
|
||||
if (n.kind === 'host') { fill = '#f5b46c'; r = 4; }
|
||||
if (n.kind === 'engram') { fill = '#6c8af5'; r = 5; }
|
||||
const term = (graphState.search || '').trim();
|
||||
let matches = 0;
|
||||
for (const n of graphState.sim) {
|
||||
const r = _graphNodeRadius(n);
|
||||
const isMatch = _graphMatches(n, term);
|
||||
if (isMatch) matches++;
|
||||
|
||||
if (isMatch) {
|
||||
ctx.beginPath();
|
||||
ctx.strokeStyle = '#f7d154';
|
||||
ctx.lineWidth = 3 / graphState.zoom;
|
||||
ctx.arc(n.x, n.y, r + 3, 0, Math.PI*2);
|
||||
ctx.stroke();
|
||||
}
|
||||
|
||||
if (graphState.selectedId === n.id) {
|
||||
ctx.beginPath();
|
||||
ctx.strokeStyle = '#ffffff';
|
||||
ctx.lineWidth = 2 / graphState.zoom;
|
||||
ctx.arc(n.x, n.y, r + 2, 0, Math.PI*2);
|
||||
ctx.stroke();
|
||||
}
|
||||
|
||||
ctx.beginPath();
|
||||
ctx.fillStyle = fill;
|
||||
ctx.fillStyle = _graphNodeFill(n);
|
||||
ctx.arc(n.x, n.y, r, 0, Math.PI*2);
|
||||
ctx.fill();
|
||||
|
||||
// status/recency/lock border
|
||||
let stroke = null;
|
||||
const v = (n.verdict || '').toString();
|
||||
if (v === 'confirmed_true') stroke = '#bbf7d0';
|
||||
else if (v === 'confirmed_false') stroke = '#fecaca';
|
||||
else if (v) stroke = '#c7d2fe';
|
||||
|
||||
const now = Date.now();
|
||||
const created = Date.parse(n.created || '') || 0;
|
||||
const modified = Date.parse(n.modified || '') || 0;
|
||||
const isNew = created && (now - created) < (30 * 60 * 1000);
|
||||
const isHot = modified && (now - modified) < (10 * 60 * 1000);
|
||||
if (isNew) stroke = '#f7d154';
|
||||
if (isHot) stroke = '#ffffff';
|
||||
if (n.predict_locked) stroke = '#a78bfa';
|
||||
|
||||
if (stroke) {
|
||||
ctx.beginPath();
|
||||
ctx.strokeStyle = stroke;
|
||||
ctx.lineWidth = (n.predict_locked ? 3 : 1.5) / graphState.zoom;
|
||||
ctx.arc(n.x, n.y, r + 0.8, 0, Math.PI*2);
|
||||
ctx.stroke();
|
||||
}
|
||||
}
|
||||
|
||||
ctx.restore();
|
||||
hint.textContent = `nodes=${graphState.nodes.length} edges=${graphState.edges.length}` + (term ? ` | match=${matches}` : '');
|
||||
}
|
||||
|
||||
function _graphLoop() {
|
||||
if (!graphState.physicsOn) return;
|
||||
const speed = 0.25 + (Math.max(0, Math.min(100, graphState.physicsStrength || 60)) / 100) * 0.95;
|
||||
_graphStepPhysics(speed);
|
||||
_graphDraw();
|
||||
graphState.raf = requestAnimationFrame(_graphLoop);
|
||||
}
|
||||
|
||||
function setPhysicsStrength(v) {
|
||||
const n = Math.max(0, Math.min(100, parseInt(v || '0', 10)));
|
||||
graphState.physicsStrength = n;
|
||||
localStorage.setItem('physicsStrength', String(n));
|
||||
const el = document.getElementById('physicsStrengthVal');
|
||||
if (el) el.textContent = String(n);
|
||||
}
|
||||
|
||||
function toggleGraphPhysics() {
|
||||
graphState.physicsOn = !graphState.physicsOn;
|
||||
const b = document.getElementById('btnGraphPhysics');
|
||||
const fast = (graphState.sim || []).length > 700;
|
||||
b.textContent = `Physics: ${graphState.physicsOn ? ('on' + (fast ? ' (fast)' : '')) : 'off'}`;
|
||||
b.classList.toggle('primary', graphState.physicsOn);
|
||||
if (graphState.physicsOn) {
|
||||
if (graphState.raf) cancelAnimationFrame(graphState.raf);
|
||||
graphState.raf = requestAnimationFrame(_graphLoop);
|
||||
} else {
|
||||
if (graphState.raf) cancelAnimationFrame(graphState.raf);
|
||||
graphState.raf = null;
|
||||
_graphDraw();
|
||||
}
|
||||
}
|
||||
|
||||
function resetGraphView() {
|
||||
const canvas = _graphCanvas();
|
||||
graphState.panX = canvas.width / 2;
|
||||
graphState.panY = canvas.height / 2;
|
||||
graphState.zoom = 1;
|
||||
_graphDraw();
|
||||
}
|
||||
|
||||
function fitGraphView() {
|
||||
const canvas = _graphCanvas();
|
||||
if (!graphState.sim.length) return;
|
||||
let minX = Infinity, minY = Infinity, maxX = -Infinity, maxY = -Infinity;
|
||||
for (const n of graphState.sim) {
|
||||
minX = Math.min(minX, n.x); minY = Math.min(minY, n.y);
|
||||
maxX = Math.max(maxX, n.x); maxY = Math.max(maxY, n.y);
|
||||
}
|
||||
const w = Math.max(1, maxX - minX);
|
||||
const h = Math.max(1, maxY - minY);
|
||||
const zx = (canvas.width - 40) / w;
|
||||
const zy = (canvas.height - 40) / h;
|
||||
graphState.zoom = Math.max(0.35, Math.min(2.5, Math.min(zx, zy)));
|
||||
graphState.panX = canvas.width / 2;
|
||||
graphState.panY = canvas.height / 2;
|
||||
_graphDraw();
|
||||
}
|
||||
|
||||
function graphApplySearch(term) {
|
||||
graphState.search = term || '';
|
||||
_graphDraw();
|
||||
}
|
||||
|
||||
// Real-time updates via SSE
|
||||
@@ -402,6 +1010,8 @@ async function confirm(id, ev) {
|
||||
body: new URLSearchParams({reason})
|
||||
});
|
||||
await loadCards(); await loadStats();
|
||||
if (state.view === 'graph') loadGraph();
|
||||
if (state.view === 'status') loadStatus();
|
||||
}
|
||||
|
||||
async function reject(id, ev) {
|
||||
@@ -413,6 +1023,8 @@ async function reject(id, ev) {
|
||||
body: new URLSearchParams({reason})
|
||||
});
|
||||
await loadCards(); await loadStats();
|
||||
if (state.view === 'graph') loadGraph();
|
||||
if (state.view === 'status') loadStatus();
|
||||
}
|
||||
|
||||
async function refresh(id, ev) {
|
||||
@@ -433,6 +1045,8 @@ async function createEngram() {
|
||||
document.getElementById('newContent').value = '';
|
||||
document.getElementById('newTags').value = '';
|
||||
await loadCards(); await loadStats();
|
||||
if (state.view === 'graph') loadGraph();
|
||||
if (state.view === 'status') loadStatus();
|
||||
}
|
||||
|
||||
async function showDetail(id) {
|
||||
@@ -478,6 +1092,7 @@ document.getElementById('searchInput').addEventListener('input', (e) => {
|
||||
state.search = e.target.value;
|
||||
state.offset = 0;
|
||||
loadCards();
|
||||
if (state.view === 'graph') graphApplySearch(state.search);
|
||||
});
|
||||
|
||||
document.getElementById('filterSelect').addEventListener('change', (e) => {
|
||||
|
||||
Reference in New Issue
Block a user