diff --git a/.gitignore b/.gitignore index 6a2547f..17d7702 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ __pycache__/ *.pyc .venv/ data/ +backups/ diff --git a/fastapi_app.py b/fastapi_app.py index 3e8005f..79563bd 100644 --- a/fastapi_app.py +++ b/fastapi_app.py @@ -42,6 +42,9 @@ def create_app() -> FastAPI: app = create_app() +_ACTIVITY_EVENTS: list[dict] = [] +_ACTIVITY_MAX = 250 + # ─── Helpers ───────────────────────────────────────────────────────────────── def get_db(): if not DB_PATH.exists(): @@ -93,6 +96,24 @@ def _now_iso() -> str: return datetime.now(timezone.utc).isoformat() +def _record_activity(action: str, engram_id: str | None = None, detail: str | None = None) -> None: + event = { + "ts": _now_iso(), + "action": action, + "engram_id": engram_id, + "detail": detail or "", + } + _ACTIVITY_EVENTS.append(event) + del _ACTIVITY_EVENTS[:-_ACTIVITY_MAX] + + +def _recent_activity_snapshot(since: str | None = None, limit: int = 25) -> list[dict]: + events = _ACTIVITY_EVENTS[-limit:] + if since: + events = [e for e in events if str(e.get("ts", "")) > since] + return events[-limit:] + + def _update_correctness(engram_id: str, *, action: str, reason: str | None = None) -> dict: """ Update correctness_json for an engram. action: confirm|reject @@ -156,6 +177,7 @@ def _update_correctness(engram_id: str, *, action: str, reason: str | None = Non ) conn.commit() conn.close() + _record_activity("review_confirm" if action == "confirm" else "review_reject", engram_id, reason) return {"ok": True} @@ -175,6 +197,7 @@ def _bump_access(engram_id: str) -> dict: ) conn.commit() conn.close() + _record_activity("read", engram_id, "detail/access") return {"ok": True} def _safe_json_extract_tags(meta_json: str) -> list[str]: @@ -783,16 +806,21 @@ def api_events(): import time def gen(): + tick = 0 while True: + tick += 1 payload = { "ts": datetime.now(timezone.utc).isoformat(), "stats": api_stats(), - "storage": api_storage_stats(), - "jobs": api_jobs(), - "insights": api_insights(limit=8), + "activity": _recent_activity_snapshot(limit=25), } + # Systemd and storage snapshots are useful but too expensive to run + # every live tick on large brains. Send them at a slower cadence. + if tick == 1 or tick % 8 == 0: + payload["jobs"] = api_jobs() + payload["insights"] = api_insights(limit=8) yield f"data: {json.dumps(payload, ensure_ascii=False)}\n\n" - time.sleep(5) + time.sleep(2) return StreamingResponse(gen(), media_type="text/event-stream") @@ -954,6 +982,7 @@ def api_engram_detail(engram_id: str): result = parse_engram(row) result["links"] = [r[0] for r in links] conn.close() + _record_activity("read", engram_id, "detail") return result @@ -1033,6 +1062,7 @@ def api_create_engram(content: str = Form(...), tags: str = Form("")): ) conn.commit() conn.close() + _record_activity("write", engram_id, content[:80]) return {"id": engram_id} @@ -1089,6 +1119,7 @@ def api_refresh(engram_id: str): ) conn.commit() conn.close() + _record_activity("score_refresh", engram_id, f"confidence {round(conf, 2)}") return {"success": True, "new_confidence": round(conf, 2)} @@ -1114,6 +1145,7 @@ def api_accept_link(from_id: str = Form(...), to_id: str = Form(...)): ) conn.commit() conn.close() + _record_activity("write_link", from_id, to_id) return {"ok": True} @@ -1154,6 +1186,7 @@ def api_create_engram(content: str = Form(...), tags: str = Form(""), source: st ) conn.commit() conn.close() + _record_activity("write", engram_id, content[:80]) return {"success": True, "engram_id": engram_id} diff --git a/openclaw_cron_wrapper.py b/openclaw_cron_wrapper.py index 8a25a84..ff5a18d 100644 --- a/openclaw_cron_wrapper.py +++ b/openclaw_cron_wrapper.py @@ -14,9 +14,9 @@ from datetime import datetime, timezone # --- Konfiguration (persistent) --- WORKSPACE = Path("/root/.openclaw/workspace") -CRON_TASKS_DIR = WORKSPACE / "cron_tasks" -LOG_FILE = WORKSPACE / "cron_wrapper.log" BRAIN_DIR = WORKSPACE / "second-brain" +CRON_TASKS_DIR = BRAIN_DIR / "cron_tasks" +LOG_FILE = WORKSPACE / "cron_wrapper.log" def log(msg: str): diff --git a/src/embedder.py b/src/embedder.py index 02e049e..272d9df 100644 --- a/src/embedder.py +++ b/src/embedder.py @@ -4,22 +4,22 @@ Offlined-fähig, cached auf Disk. """ import json import hashlib -import os +import math from pathlib import Path from typing import List, Optional -import numpy as np -from sentence_transformers import SentenceTransformer _MODEL_NAME = "all-MiniLM-L6-v2" _EMBED_DIM = 384 _CACHE_DIR = Path(__file__).resolve().parent.parent / "data" / "embedding_cache" -__model: Optional[SentenceTransformer] = None +__model = None +__fallback_warned = False -def _get_model() -> SentenceTransformer: +def _get_model(): global __model if __model is None: + from sentence_transformers import SentenceTransformer __model = SentenceTransformer(_MODEL_NAME) return __model @@ -33,6 +33,39 @@ def _cache_path(h: str) -> Path: return _CACHE_DIR / f"{h}.json" +def _normalize(vec: List[float]) -> List[float]: + norm = math.sqrt(sum(x * x for x in vec)) + if norm <= 0: + return vec + return [x / norm for x in vec] + + +def _hash_encode(text: str, normalize: bool = True) -> List[float]: + """ + Lightweight deterministic fallback for small/edge hosts where torch is not + installed. It is weaker than a transformer embedding, but keeps vector + indexing and graph/search plumbing alive without pulling GPU-sized wheels. + """ + vec = [0.0] * _EMBED_DIM + words = [w for w in text.lower().split() if w] + if not words: + words = [text] + for word in words: + digest = hashlib.sha256(word.encode("utf-8")).digest() + for i, b in enumerate(digest): + idx = (b + i * 131) % _EMBED_DIM + sign = 1.0 if (b & 1) else -1.0 + vec[idx] += sign + return _normalize(vec) if normalize else vec + + +def _warn_fallback(message: str) -> None: + global __fallback_warned + if not __fallback_warned: + print(message) + __fallback_warned = True + + def encode(text: str, cache: bool = True, normalize: bool = True) -> Optional[List[float]]: """Embeddiert einen Text. Gibt None zurück wenn Modell nicht verfügbar.""" try: @@ -43,13 +76,15 @@ def encode(text: str, cache: bool = True, normalize: bool = True) -> Optional[Li data = json.load(f) return data["embedding"] - model = _get_model() - vec = model.encode(text, convert_to_numpy=True) - if normalize: - norm = np.linalg.norm(vec) - if norm > 0: - vec = vec / norm - vec_list = vec.tolist() + try: + model = _get_model() + vec = model.encode(text, convert_to_numpy=True) + vec_list = vec.tolist() + if normalize: + vec_list = _normalize([float(x) for x in vec_list]) + except Exception as e: + _warn_fallback(f"[embedder] transformer unavailable, using hash fallback: {e}") + vec_list = _hash_encode(text, normalize=normalize) if cache: with open(cp, "w", encoding="utf-8") as f: @@ -81,14 +116,16 @@ def encode_batch(texts: List[str], cache: bool = True, normalize: bool = True) - idx_map.append(i) if to_encode: - model = _get_model() - vecs = model.encode(to_encode, convert_to_numpy=True) - if normalize: - norms = np.linalg.norm(vecs, axis=1, keepdims=True) - norms[norms == 0] = 1 - vecs = vecs / norms - for m, vec in zip(idx_map, vecs): - vec_list = vec.tolist() + try: + model = _get_model() + vecs = model.encode(to_encode, convert_to_numpy=True) + encoded = [vec.tolist() for vec in vecs] + if normalize: + encoded = [_normalize([float(x) for x in vec]) for vec in encoded] + except Exception as e: + _warn_fallback(f"[embedder] batch transformer unavailable, using hash fallback: {e}") + encoded = [_hash_encode(text, normalize=normalize) for text in to_encode] + for m, vec_list in zip(idx_map, encoded): results[m] = vec_list if cache: h = _text_hash(texts[m]) @@ -104,13 +141,12 @@ def encode_batch(texts: List[str], cache: bool = True, normalize: bool = True) - def similar(query: str, candidates: List[str], top_k: int = 5) -> List[tuple]: """Gibt die top-k besten Kandidaten für eine Query zurück.""" - q_vec = np.array(encode(query)) + q_vec = encode(query) or [] c_vecs = encode_batch(candidates) scores = [] for i, c_vec in enumerate(c_vecs): if c_vec is not None: - c_arr = np.array(c_vec) - score = float(np.dot(q_vec, c_arr)) + score = float(sum(a * b for a, b in zip(q_vec, c_vec))) scores.append((i, score)) scores.sort(key=lambda x: x[1], reverse=True) return [(candidates[i], s) for i, s in scores[:top_k]] diff --git a/src/store.py b/src/store.py index c800474..cd4a97e 100644 --- a/src/store.py +++ b/src/store.py @@ -161,6 +161,7 @@ class EngramStore: ).fetchone() if not row: return None + self._touch_access([engram_id]) return self._row_to_engram(row) def get_all(self, limit: int = 1000, offset: int = 0) -> List[Engram]: @@ -276,6 +277,7 @@ class EngramStore: LIMIT ? """ rows = self._conn.execute(sql, (safe_query, limit)).fetchall() + self._touch_access([r["id"] for r in rows]) return [self._row_to_engram(r) for r in rows] def search_tag(self, tag: str, limit: int = 50) -> List[Engram]: @@ -285,6 +287,7 @@ class EngramStore: "SELECT * FROM engrams WHERE metadata_json LIKE ? ORDER BY created_at DESC LIMIT ?", (f'%"{tag}"%', limit) ).fetchall() + self._touch_access([r["id"] for r in rows]) return [self._row_to_engram(r) for r in rows] def search_source(self, source: str, limit: int = 50) -> List[Engram]: @@ -293,6 +296,7 @@ class EngramStore: "SELECT * FROM engrams WHERE metadata_json LIKE ? ORDER BY created_at DESC LIMIT ?", (f'%"source": "{source}"%', limit) ).fetchall() + self._touch_access([r["id"] for r in rows]) return [self._row_to_engram(r) for r in rows] # ---- Stats ---- @@ -367,6 +371,32 @@ class EngramStore: d["embedding"] = json.loads(emb) return Engram.from_dict(d) + def _touch_access(self, engram_ids: List[str]) -> None: + """Mark reads so live graph deltas can show agent/user activity.""" + ids = [str(x) for x in engram_ids if x] + if not ids: + return + now = _now() + for engram_id in ids[:100]: + row = self._conn.execute( + "SELECT metadata_json FROM engrams WHERE id=?", + (engram_id,), + ).fetchone() + if not row: + continue + try: + meta = json.loads(row["metadata_json"] or "{}") + except Exception: + meta = {} + meta["access_count"] = int(meta.get("access_count", 0) or 0) + 1 + meta["last_accessed"] = now + meta["modified"] = now + self._conn.execute( + "UPDATE engrams SET metadata_json=?, modified_at=? WHERE id=?", + (json.dumps(meta, ensure_ascii=False), now, engram_id), + ) + self._conn.commit() + def close(self) -> None: self._conn.close() diff --git a/static/style.css b/static/style.css index 7e1ceef..c44c513 100644 --- a/static/style.css +++ b/static/style.css @@ -168,7 +168,11 @@ body { margin: 10px auto 0; background:#01030a; border:0; - border-radius: 50%; + border-radius: 6px; + width: min(100%, 1180px); + max-width: calc(100vw - 24px); + height: auto; + aspect-ratio: 16 / 10; box-shadow: 0 0 48px rgba(34,211,238,0.14), 0 0 120px rgba(236,72,153,0.08), @@ -221,6 +225,9 @@ body { .legend-dot.tag{ background:#8a9aff; } .legend-dot.source{ background:#14b8a6; } .legend-dot.match{ background:#f7d154; } +.legend-dot.read{ background:#38bdf8; } +.legend-dot.write{ background:#4ade80; } +.legend-dot.review{ background:#f7d154; } .graph-hint{ padding: 4px 12px 10px; } .graph-live{ margin: 8px 12px 0; @@ -451,7 +458,7 @@ body { .actions button { width: 34px; height: 34px; - border-radius: 50%; + border-radius: 10px; border: none; font-size: 1rem; cursor: pointer; @@ -502,7 +509,7 @@ body { .refresh-btn { background: #252535; border: none; - border-radius: 50%; + border-radius: 10px; width: 36px; height: 36px; color: #8888aa; diff --git a/systemd/openclaw-memory-archive.service b/systemd/openclaw-memory-archive.service index a2f8c44..c716910 100644 --- a/systemd/openclaw-memory-archive.service +++ b/systemd/openclaw-memory-archive.service @@ -5,5 +5,5 @@ OnFailure=openclaw-secondbrain-notify@%n.service [Service] Type=oneshot WorkingDirectory=/root/.openclaw/workspace -ExecStart=/bin/bash -lc 'flock -n /tmp/%n.lock /usr/bin/python3 /root/.openclaw/workspace/openclaw_cron_wrapper.py archive_memory_md' +ExecStart=/bin/bash -lc 'flock -n /tmp/%n.lock /usr/bin/python3 /root/.openclaw/workspace/second-brain/openclaw_cron_wrapper.py archive_memory_md' diff --git a/systemd/openclaw-secondbrain-backup.service b/systemd/openclaw-secondbrain-backup.service index 54e23bd..c265169 100644 --- a/systemd/openclaw-secondbrain-backup.service +++ b/systemd/openclaw-secondbrain-backup.service @@ -5,4 +5,4 @@ OnFailure=openclaw-secondbrain-notify@%n.service [Service] Type=oneshot WorkingDirectory=/root/.openclaw/workspace -ExecStart=/bin/bash -lc 'flock -n /tmp/%n.lock /usr/bin/python3 /root/.openclaw/workspace/openclaw_cron_wrapper.py backup_secondbrain' +ExecStart=/bin/bash -lc 'flock -n /tmp/%n.lock /usr/bin/python3 /root/.openclaw/workspace/second-brain/openclaw_cron_wrapper.py backup_secondbrain' diff --git a/systemd/openclaw-secondbrain-export-obsidian.service b/systemd/openclaw-secondbrain-export-obsidian.service index c3a8964..44b4b3e 100644 --- a/systemd/openclaw-secondbrain-export-obsidian.service +++ b/systemd/openclaw-secondbrain-export-obsidian.service @@ -5,4 +5,4 @@ OnFailure=openclaw-secondbrain-notify@%n.service [Service] Type=oneshot WorkingDirectory=/root/.openclaw/workspace -ExecStart=/bin/bash -lc 'flock -n /tmp/%n.lock /usr/bin/python3 /root/.openclaw/workspace/openclaw_cron_wrapper.py export_obsidian' +ExecStart=/bin/bash -lc 'flock -n /tmp/%n.lock /usr/bin/python3 /root/.openclaw/workspace/second-brain/openclaw_cron_wrapper.py export_obsidian' diff --git a/systemd/openclaw-secondbrain-heartbeat.service b/systemd/openclaw-secondbrain-heartbeat.service index 61b09ee..6f91b70 100644 --- a/systemd/openclaw-secondbrain-heartbeat.service +++ b/systemd/openclaw-secondbrain-heartbeat.service @@ -5,4 +5,4 @@ OnFailure=openclaw-secondbrain-notify@%n.service [Service] Type=oneshot WorkingDirectory=/root/.openclaw/workspace -ExecStart=/bin/bash -lc 'flock -n /tmp/%n.lock /usr/bin/python3 /root/.openclaw/workspace/openclaw_cron_wrapper.py heartbeat_secondbrain' +ExecStart=/bin/bash -lc 'flock -n /tmp/%n.lock /usr/bin/python3 /root/.openclaw/workspace/second-brain/openclaw_cron_wrapper.py heartbeat_secondbrain' diff --git a/systemd/openclaw-secondbrain-index-vectors.service b/systemd/openclaw-secondbrain-index-vectors.service index 03b569f..2ab4c93 100644 --- a/systemd/openclaw-secondbrain-index-vectors.service +++ b/systemd/openclaw-secondbrain-index-vectors.service @@ -7,4 +7,4 @@ Type=oneshot WorkingDirectory=/root/.openclaw/workspace Environment=HF_HOME=/root/.openclaw/workspace/second-brain/data/hf_cache Environment=SENTENCE_TRANSFORMERS_HOME=/root/.openclaw/workspace/second-brain/data/st_cache -ExecStart=/bin/bash -lc 'flock -n /tmp/%n.lock /usr/bin/python3 /root/.openclaw/workspace/openclaw_cron_wrapper.py index_vectors' +ExecStart=/bin/bash -lc 'flock -n /tmp/%n.lock /root/.openclaw/workspace/second-brain/.venv/bin/python /root/.openclaw/workspace/second-brain/openclaw_cron_wrapper.py index_vectors' diff --git a/systemd/openclaw-secondbrain-ingest-memory.service b/systemd/openclaw-secondbrain-ingest-memory.service index c7a7aa1..715a946 100644 --- a/systemd/openclaw-secondbrain-ingest-memory.service +++ b/systemd/openclaw-secondbrain-ingest-memory.service @@ -5,4 +5,4 @@ OnFailure=openclaw-secondbrain-notify@%n.service [Service] Type=oneshot WorkingDirectory=/root/.openclaw/workspace -ExecStart=/bin/bash -lc 'flock -n /tmp/%n.lock /usr/bin/python3 /root/.openclaw/workspace/openclaw_cron_wrapper.py ingest_memory; code=$?; [ "$code" -eq 1 ] && exit 0; exit "$code"' +ExecStart=/bin/bash -lc 'flock -n /tmp/%n.lock /usr/bin/python3 /root/.openclaw/workspace/second-brain/openclaw_cron_wrapper.py ingest_memory; code=$?; [ "$code" -eq 1 ] && exit 0; exit "$code"' diff --git a/systemd/openclaw-secondbrain-ingest-obsidian.service b/systemd/openclaw-secondbrain-ingest-obsidian.service index 643742b..44bc68b 100644 --- a/systemd/openclaw-secondbrain-ingest-obsidian.service +++ b/systemd/openclaw-secondbrain-ingest-obsidian.service @@ -5,4 +5,4 @@ OnFailure=openclaw-secondbrain-notify@%n.service [Service] Type=oneshot WorkingDirectory=/root/.openclaw/workspace -ExecStart=/bin/bash -lc 'flock -n /tmp/%n.lock /usr/bin/python3 /root/.openclaw/workspace/openclaw_cron_wrapper.py ingest_obsidian' +ExecStart=/bin/bash -lc 'flock -n /tmp/%n.lock /usr/bin/python3 /root/.openclaw/workspace/second-brain/openclaw_cron_wrapper.py ingest_obsidian' diff --git a/systemd/openclaw-secondbrain-ingest-transcript-to-db.service b/systemd/openclaw-secondbrain-ingest-transcript-to-db.service index d8c8019..7952cd5 100644 --- a/systemd/openclaw-secondbrain-ingest-transcript-to-db.service +++ b/systemd/openclaw-secondbrain-ingest-transcript-to-db.service @@ -5,5 +5,5 @@ OnFailure=openclaw-secondbrain-notify@%n.service [Service] Type=oneshot WorkingDirectory=/root/.openclaw/workspace -ExecStart=/bin/bash -lc 'flock -n /tmp/%n.lock /usr/bin/python3 /root/.openclaw/workspace/openclaw_cron_wrapper.py ingest_transcript_to_db' +ExecStart=/bin/bash -lc 'flock -n /tmp/%n.lock /usr/bin/python3 /root/.openclaw/workspace/second-brain/openclaw_cron_wrapper.py ingest_transcript_to_db' diff --git a/systemd/openclaw-secondbrain-proactive-search.service b/systemd/openclaw-secondbrain-proactive-search.service index f57be9d..8580eb9 100644 --- a/systemd/openclaw-secondbrain-proactive-search.service +++ b/systemd/openclaw-secondbrain-proactive-search.service @@ -8,4 +8,4 @@ OnFailure=openclaw-secondbrain-notify@%n.service Type=oneshot WorkingDirectory=/root/.openclaw/workspace Environment=LLM_PROXY_MODEL=stepfun-ai/Step-3.5-Flash -ExecStart=/bin/bash -lc 'flock -n /tmp/%n.lock /usr/bin/python3 /root/.openclaw/workspace/openclaw_cron_wrapper.py proactive_search_wrapper' +ExecStart=/bin/bash -lc 'flock -n /tmp/%n.lock /usr/bin/python3 /root/.openclaw/workspace/second-brain/openclaw_cron_wrapper.py proactive_search_wrapper' diff --git a/systemd/openclaw-secondbrain-review.service b/systemd/openclaw-secondbrain-review.service index 2d161b0..21100e3 100644 --- a/systemd/openclaw-secondbrain-review.service +++ b/systemd/openclaw-secondbrain-review.service @@ -5,4 +5,4 @@ OnFailure=openclaw-secondbrain-notify@%n.service [Service] Type=oneshot WorkingDirectory=/root/.openclaw/workspace -ExecStart=/bin/bash -lc 'flock -n /tmp/%n.lock /usr/bin/python3 /root/.openclaw/workspace/openclaw_cron_wrapper.py review_brain' +ExecStart=/bin/bash -lc 'flock -n /tmp/%n.lock /usr/bin/python3 /root/.openclaw/workspace/second-brain/openclaw_cron_wrapper.py auto_assign_review' diff --git a/systemd/openclaw-secondbrain-task@.service b/systemd/openclaw-secondbrain-task@.service index 37e99cd..a19f909 100644 --- a/systemd/openclaw-secondbrain-task@.service +++ b/systemd/openclaw-secondbrain-task@.service @@ -6,7 +6,7 @@ After=network-online.target [Service] Type=oneshot WorkingDirectory=/root/.openclaw/workspace -ExecStart=/usr/bin/python3 /root/.openclaw/workspace/openclaw_cron_wrapper.py %i +ExecStart=/usr/bin/python3 /root/.openclaw/workspace/second-brain/openclaw_cron_wrapper.py %i Nice=10 IOSchedulingClass=best-effort IOSchedulingPriority=6 diff --git a/systemd/openclaw-secondbrain-verify-pending.service b/systemd/openclaw-secondbrain-verify-pending.service index a3ac875..ebc3c6d 100644 --- a/systemd/openclaw-secondbrain-verify-pending.service +++ b/systemd/openclaw-secondbrain-verify-pending.service @@ -5,5 +5,5 @@ OnFailure=openclaw-secondbrain-notify@%n.service [Service] Type=oneshot WorkingDirectory=/root/.openclaw/workspace -ExecStart=/bin/bash -lc 'flock -n /tmp/%n.lock /usr/bin/python3 /root/.openclaw/workspace/openclaw_cron_wrapper.py verify_pending_external' +ExecStart=/bin/bash -lc 'flock -n /tmp/%n.lock /usr/bin/python3 /root/.openclaw/workspace/second-brain/openclaw_cron_wrapper.py verify_pending_external' diff --git a/systemd/openclaw-secondbrain.target b/systemd/openclaw-secondbrain.target new file mode 100644 index 0000000..39089fd --- /dev/null +++ b/systemd/openclaw-secondbrain.target @@ -0,0 +1,7 @@ +[Unit] +Description=OpenClaw Second-Brain services +Wants=openclaw-secondbrain-dashboard.service +After=network-online.target + +[Install] +WantedBy=multi-user.target diff --git a/templates/dashboard.html b/templates/dashboard.html index 5301098..0268bb3 100644 --- a/templates/dashboard.html +++ b/templates/dashboard.html @@ -4,7 +4,7 @@