Add SQLite dashboard cache

README.md

@@ -27,6 +27,10 @@ nach einer ausdrücklichen Freigabe ausführen.
   [`docs/V1_5_0_OPERATING_GUIDE.md`](docs/V1_5_0_OPERATING_GUIDE.md)
 - Version 1.5.1 Stabilisierung der Dashboard-Ladepfade:
   [`docs/V1_5_1_OPERATING_GUIDE.md`](docs/V1_5_1_OPERATING_GUIDE.md)
+- Version 1.5.2 Rollback-Speicher und HA-Timeouts:
+  [`docs/V1_5_2_OPERATING_GUIDE.md`](docs/V1_5_2_OPERATING_GUIDE.md)
+- Version 1.5.3 SQLite-Cache fuer Ingress-Dashboard:
+  [`docs/V1_5_3_OPERATING_GUIDE.md`](docs/V1_5_3_OPERATING_GUIDE.md)
 - Arbeitsregeln für Coding-Agenten: [`AGENTS.md`](AGENTS.md)
 
 ## Reifegrad

addon/config.yaml

@@ -1,5 +1,5 @@
 name: SillyHome Next
-version: "1.5.1"
+version: "1.5.3"
 slug: sillyhome_next
 description: Lernt automatisch aus deinem Verhalten und steuert freigegebene Aktoren
 url: http://192.168.6.31:3000/pino/sillyhome-next

app/actuators/cache_db.py

@@ -0,0 +1,98 @@
+from __future__ import annotations
+
+import json
+import sqlite3
+from datetime import datetime, timezone
+from pathlib import Path
+from threading import RLock
+
+from app.ha.models import HaEntitySummary
+
+
+class DashboardCache:
+    def __init__(self, path: str | Path) -> None:
+        self._path = Path(path).resolve()
+        self._path.parent.mkdir(parents=True, exist_ok=True)
+        self._lock = RLock()
+        self._init()
+
+    def load_entities_payload(self) -> dict[str, object]:
+        with self._lock, self._connect() as connection:
+            rows = connection.execute(
+                "select entity_id, payload from ha_entities order by entity_id"
+            ).fetchall()
+            updated_at = self._get_meta(connection, "ha_entities_updated_at")
+            groups_json = self._get_meta(connection, "discovery_groups") or "[]"
+        try:
+            groups = json.loads(groups_json)
+        except ValueError:
+            groups = []
+        return {
+            "updated_at": updated_at,
+            "discovery_groups": groups if isinstance(groups, list) else [],
+            "entities": [json.loads(row[1]) for row in rows],
+        }
+
+    def save_entities_payload(
+        self,
+        *,
+        entities: list[HaEntitySummary],
+        discovery_groups: list[dict[str, object]],
+    ) -> None:
+        now = datetime.now(timezone.utc).isoformat()
+        rows = [
+            (entity.entity_id, entity.model_dump_json())
+            for entity in entities
+        ]
+        with self._lock, self._connect() as connection:
+            connection.execute("delete from ha_entities")
+            connection.executemany(
+                "insert into ha_entities(entity_id, payload) values (?, ?)",
+                rows,
+            )
+            self._set_meta(connection, "ha_entities_updated_at", now)
+            self._set_meta(
+                connection,
+                "discovery_groups",
+                json.dumps(discovery_groups, ensure_ascii=True, sort_keys=True),
+            )
+
+    def _init(self) -> None:
+        with self._connect() as connection:
+            connection.execute(
+                """
+                create table if not exists ha_entities (
+                    entity_id text primary key,
+                    payload text not null
+                )
+                """
+            )
+            connection.execute(
+                """
+                create table if not exists cache_meta (
+                    key text primary key,
+                    value text
+                )
+                """
+            )
+
+    def _connect(self) -> sqlite3.Connection:
+        return sqlite3.connect(self._path, timeout=30)
+
+    @staticmethod
+    def _get_meta(connection: sqlite3.Connection, key: str) -> str | None:
+        row = connection.execute(
+            "select value from cache_meta where key = ?",
+            (key,),
+        ).fetchone()
+        return str(row[0]) if row is not None and row[0] is not None else None
+
+    @staticmethod
+    def _set_meta(connection: sqlite3.Connection, key: str, value: str) -> None:
+        connection.execute(
+            """
+            insert into cache_meta(key, value) values (?, ?)
+            on conflict(key) do update set value = excluded.value
+            """,
+            (key, value),
+        )

app/api/v1/actuators.py

@@ -8,6 +8,7 @@ from pathlib import Path
 from fastapi import APIRouter, Depends, HTTPException, Query, Request, status
 from pydantic import BaseModel, Field
 
+from app.actuators.cache_db import DashboardCache
 from app.actuators.lifecycle import ActuatorReconciliationService
 from app.actuators.models import ActuatorRecord, AnomalyEvent, ReconciliationState, SensorWeightGroup
 from app.actuators.models import JobQueueItem, JobQueueState, JobStatus, SafetyProfile
@@ -447,7 +448,7 @@ def get_actuator_detail(actuator_entity_id: str, request: Request) -> ActuatorRe
     }
     compact_snapshots = [
         snapshot.model_copy(update={"patterns": []})
-        for snapshot in record.behavior.model_snapshots[-5:]
+        for snapshot in record.behavior.model_snapshots[-3:]
     ]
     compact_behavior = record.behavior.model_copy(
         update={
@@ -864,6 +865,11 @@ def _load_cached_entity_map(
 
 
 def _load_entity_cache_payload(request: Request) -> dict[str, object]:
+    cache = getattr(request.app.state, "dashboard_cache", None)
+    if isinstance(cache, DashboardCache):
+        payload = cache.load_entities_payload()
+        if payload.get("entities"):
+            return payload
     path = _entity_cache_path(request)
     if not path.exists():
         return {}
@@ -875,18 +881,18 @@ def _load_entity_cache_payload(request: Request) -> dict[str, object]:
 
 
 def _save_cached_entities(request: Request, entities: list[HaEntitySummary]) -> None:
+    group_payload = _discovery_group_payload(entities)
+    cache = getattr(request.app.state, "dashboard_cache", None)
+    if isinstance(cache, DashboardCache):
+        cache.save_entities_payload(
+            entities=entities,
+            discovery_groups=group_payload,
+        )
     path = _entity_cache_path(request)
     path.parent.mkdir(parents=True, exist_ok=True)
-    group_counts: dict[tuple[str, str], int] = {}
-    for entity in discover_entities(entities):
-        key = (entity.category, entity.role.value)
-        group_counts[key] = group_counts.get(key, 0) + 1
     payload = {
         "updated_at": datetime.now(timezone.utc).isoformat(),
-        "discovery_groups": [
+        "discovery_groups": group_payload,
-            {"category": category, "role": role, "count": count}
-            for (category, role), count in sorted(group_counts.items())
-        ],
         "entities": [entity.model_dump(mode="json") for entity in entities],
     }
     temporary = path.with_suffix(".json.tmp")
@@ -897,6 +903,17 @@ def _save_cached_entities(request: Request, entities: list[HaEntitySummary]) ->
     os.replace(temporary, path)
 
 
+def _discovery_group_payload(entities: list[HaEntitySummary]) -> list[dict[str, object]]:
+    group_counts: dict[tuple[str, str], int] = {}
+    for entity in discover_entities(entities):
+        key = (entity.category, entity.role.value)
+        group_counts[key] = group_counts.get(key, 0) + 1
+    return [
+        {"category": category, "role": role, "count": count}
+        for (category, role), count in sorted(group_counts.items())
+    ]
+
+
 def _deduplicate_actuator_ids(
     discovered: list[tuple[str, str]],
     entities: dict[str, HaEntitySummary],

app/behavior/engine.py

@@ -32,6 +32,8 @@ from app.ha.models import HaEntitySummary
 from app.ha.reader import HaReader
 
 _MAX_PATTERNS = 500
+_MAX_MODEL_SNAPSHOTS = 3
+_MAX_SNAPSHOT_PATTERNS = 120
 _MAX_EXECUTION_EVENTS = 100
 _ACTION_LOGBOOK_TOLERANCE = timedelta(seconds=10)
 _CONTEXT_TRIGGER_TOLERANCE = timedelta(seconds=3)
@@ -937,6 +939,11 @@ class BehaviorEngine:
         record: ActuatorRecord,
         behavior: BehaviorState,
     ) -> ActuatorRecord:
+        behavior = behavior.model_copy(
+            update={
+                "model_snapshots": _compact_model_snapshots(behavior.model_snapshots),
+            }
+        )
         updated = record.model_copy(
             update={
                 "behavior": behavior,
@@ -1132,10 +1139,19 @@ def _next_model_snapshots(
         high_confidence_sample_count=trusted_actions,
         average_confidence=round(average_confidence, 4),
         incorrect_feedback_count=incorrect_feedback_count,
-        patterns=patterns,
+        patterns=patterns[-_MAX_SNAPSHOT_PATTERNS:],
         reason=reason,
     )
-    return [*existing, snapshot][-10:]
+    return _compact_model_snapshots([*existing, snapshot])
+
+
+def _compact_model_snapshots(existing: list[ModelSnapshot]) -> list[ModelSnapshot]:
+    return [
+        snapshot.model_copy(
+            update={"patterns": snapshot.patterns[-_MAX_SNAPSHOT_PATTERNS:]}
+        )
+        for snapshot in existing[-_MAX_MODEL_SNAPSHOTS:]
+    ]
 
 
 def _average(values: list[float]) -> float:

app/config.py

@@ -21,6 +21,8 @@ class Settings:
     prediction_interval_seconds: int = 60
     execution_cooldown_seconds: int = 900
     timezone: str = "Europe/Berlin"
+    ha_timeout_seconds: int = 25
+    dashboard_cache_refresh_seconds: int = 3600
 
     @property
     def ha_configured(self) -> bool:
@@ -55,4 +57,8 @@ def load_settings() -> Settings:
             60, int(os.getenv("SILLYHOME_EXECUTION_COOLDOWN_SECONDS", "900"))
         ),
         timezone=os.getenv("SILLYHOME_TIMEZONE", "Europe/Berlin"),
+        ha_timeout_seconds=max(5, int(os.getenv("SILLYHOME_HA_TIMEOUT_SECONDS", "25"))),
+        dashboard_cache_refresh_seconds=max(
+            300, int(os.getenv("SILLYHOME_DASHBOARD_CACHE_REFRESH_SECONDS", "3600"))
+        ),
     )

app/main.py

@@ -12,6 +12,7 @@ from fastapi import FastAPI
 from fastapi.responses import FileResponse
 from fastapi.staticfiles import StaticFiles
 
+from app.actuators.cache_db import DashboardCache
 from app.actuators.lifecycle import ActuatorReconciliationService
 from app.actuators.store import ActuatorStore
 from app.api.v1.actuators import router as actuators_router
@@ -20,6 +21,7 @@ from app.behavior.engine import BehaviorEngine
 from app.config import load_settings
 from app.core.exception_handlers import register_exception_handlers
 from app.ha.client import HaClient, HaClientSettings
+from app.ha.discovery import discover_entities
 from app.ha.models import HaEntitySummary
 from app.ha.reader import HaReader
 from app.ml.registry.model_registry import ModelRegistry
@@ -47,8 +49,12 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]:
     reconcile_task: asyncio.Task[None] | None = None
     event_listener_task: asyncio.Task[None] | None = None
     fallback_task: asyncio.Task[None] | None = None
+    cache_refresh_task: asyncio.Task[None] | None = None
     app.state.registry = ModelRegistry(settings.model_store)
     app.state.actuator_store = ActuatorStore(settings.actuator_store)
+    app.state.dashboard_cache = DashboardCache(
+        Path(settings.actuator_store).resolve() / "dashboard_cache.sqlite3"
+    )
     if hasattr(app.state, "ha_reader"):
         del app.state.ha_reader
     if hasattr(app.state, "actuator_service"):
@@ -60,6 +66,7 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]:
             settings=HaClientSettings(
                 url=cast(str, settings.ha_url),
                 token=cast(str, settings.ha_token),
+                timeout_seconds=settings.ha_timeout_seconds,
             )
         )
         app.state.ha_reader = HaReader(client=client)
@@ -79,6 +86,7 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]:
         reconcile_task = asyncio.create_task(_periodic_reconciliation(app))
         event_listener_task = asyncio.create_task(_ha_event_listener(app, client))
         fallback_task = asyncio.create_task(_fallback_prediction(app))
+        cache_refresh_task = asyncio.create_task(_periodic_dashboard_cache_refresh(app))
     try:
         yield
     finally:
@@ -98,6 +106,10 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]:
             fallback_task.cancel()
             with suppress(asyncio.CancelledError):
                 await fallback_task
+        if cache_refresh_task is not None:
+            cache_refresh_task.cancel()
+            with suppress(asyncio.CancelledError):
+                await cache_refresh_task
         if client is not None:
             client.close()
 
@@ -105,7 +117,7 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]:
 app = FastAPI(
     title="SillyHome Next API",
     description="Lokales Smart-Home-Intelligenzsystem für Home Assistant.",
-    version="1.5.1",
+    version="1.5.3",
     lifespan=lifespan,
 )
 app.state.settings = load_settings()
@@ -159,6 +171,42 @@ async def _periodic_reconciliation(app: FastAPI) -> None:
             logger.exception("Geplante Reconciliation fehlgeschlagen; nächster Lauf versucht es erneut.")
 
 
+async def _periodic_dashboard_cache_refresh(app: FastAPI) -> None:
+    await asyncio.sleep(2)
+    while True:
+        await _refresh_dashboard_cache(app, trigger="scheduled")
+        await asyncio.sleep(app.state.settings.dashboard_cache_refresh_seconds)
+
+
+async def _refresh_dashboard_cache(app: FastAPI, *, trigger: str) -> None:
+    ha_reader = getattr(app.state, "ha_reader", None)
+    cache = getattr(app.state, "dashboard_cache", None)
+    if not isinstance(ha_reader, HaReader) or not isinstance(cache, DashboardCache):
+        return
+    try:
+        entities = await asyncio.to_thread(ha_reader.read_entities)
+        groups = _discovery_group_payload(list(entities))
+        await asyncio.to_thread(
+            cache.save_entities_payload,
+            entities=list(entities),
+            discovery_groups=groups,
+        )
+        logger.info("Dashboard-Cache aktualisiert (%s): %d Entities", trigger, len(entities))
+    except Exception as exc:
+        logger.warning("Dashboard-Cache konnte nicht aktualisiert werden (%s): %s", trigger, exc)
+
+
+def _discovery_group_payload(entities: list[HaEntitySummary]) -> list[dict[str, object]]:
+    group_counts: dict[tuple[str, str], int] = {}
+    for entity in discover_entities(entities):
+        key = (entity.category, entity.role.value)
+        group_counts[key] = group_counts.get(key, 0) + 1
+    return [
+        {"category": category, "role": role, "count": count}
+        for (category, role), count in sorted(group_counts.items())
+    ]
+
+
 async def _startup_reconciliation(app: FastAPI) -> None:
     delay_seconds = 5
     while True:
@@ -213,8 +261,8 @@ async def _ha_event_listener(app: FastAPI, client: HaClient) -> None:
         try:
             async with websockets.connect(
                 ws_url,
-                ping_interval=20,
+                ping_interval=30,
-                ping_timeout=10,
+                ping_timeout=30,
             ) as websocket:
                 auth_required_msg = await websocket.recv()
                 auth_required_data = json.loads(auth_required_msg)

docs/V1_5_2_OPERATING_GUIDE.md

@@ -0,0 +1,32 @@
+# SillyHome Next v1.5.2 Operating Guide
+
+v1.5.2 begrenzt den Rollback-Speicher und entschärft Home-Assistant-Timeouts,
+die in den Add-on-Logs sichtbar wurden.
+
+## Rollback-Speicher
+
+- Pro Aktor bleiben maximal 3 Modell-Snapshots erhalten.
+- Pro Snapshot bleiben maximal 120 Muster erhalten.
+- Beim Speichern eines Aktors werden ältere oder zu große Snapshots automatisch
+  gekappt.
+- Der kompakte Detail-Endpunkt liefert ebenfalls maximal 3 Rollback-Snapshots
+  und keine Musterlisten.
+
+Damit bleibt Rollback nutzbar, ohne dass die JSON-Dateien mit alten Modellen
+stark wachsen.
+
+## Home-Assistant-Zugriffe
+
+- REST-Zugriffe auf Home Assistant haben jetzt standardmäßig 25 Sekunden
+  Timeout statt 10 Sekunden.
+- Der Wert ist über `SILLYHOME_HA_TIMEOUT_SECONDS` konfigurierbar.
+- WebSocket-Keepalive wurde auf 30 Sekunden Ping-Intervall und 30 Sekunden
+  Ping-Timeout entschärft.
+
+## Log-Einordnung
+
+- `GET ... HTTP/1.1` ist bei Uvicorn/HA-Ingress normal und kein Fehler.
+- `Zeitüberschreitung beim Zugriff auf Home Assistant` bedeutet, dass HA selbst
+  zu langsam geantwortet hat oder der Ingress/Netzpfad verzögert war.
+- `keepalive ping timeout` bedeutet, dass die HA-WebSocket-Verbindung nicht
+  rechtzeitig geantwortet hat. SillyHome reconnectet automatisch.

docs/V1_5_3_OPERATING_GUIDE.md

@@ -0,0 +1,37 @@
+# SillyHome Next v1.5.3 Operating Guide
+
+v1.5.3 führt eine SQLite-Cache-Schicht für Ingress-Dashboarddaten ein.
+
+## Ziel
+
+Die Ingress-Seite soll nicht bei jedem Aufruf live Home Assistant abfragen.
+Home-Assistant-Daten werden geplant aktualisiert und lokal gelesen.
+
+## SQLite-Cache
+
+- Cache-Datei: `<actuator_store>/dashboard_cache.sqlite3`
+- Tabelle `ha_entities`: aktuelle HA-Entity-Summaries als JSON
+- Tabelle `cache_meta`: Aktualisierungszeitpunkt und Discovery-Gruppen
+
+Dashboard-APIs lesen bevorzugt aus SQLite. Der alte JSON-Cache bleibt als
+Fallback erhalten.
+
+## Aktualisierung
+
+- Beim App-Start läuft ein Hintergrund-Refresh nach kurzer Verzögerung.
+- Danach läuft der Refresh stündlich.
+- Konfiguration: `SILLYHOME_DASHBOARD_CACHE_REFRESH_SECONDS`
+- Mindestwert: 300 Sekunden.
+- Explizite Discovery aktualisiert SQLite und JSON-Fallback.
+
+## Schaltpfad
+
+Das direkte Schalten bleibt unverändert: Safety prüft lokale Daten, danach geht
+der Home-Assistant-Service-Call direkt raus. Der Dashboard-Cache liegt nicht im
+Schaltpfad.
+
+## Noch offen
+
+Diese Version verschiebt Entity-/Discovery-Daten in SQLite. Die vollständige
+Migration aller Aktor-Konfigurationen und Workflows aus JSON in relationale
+Tabellen ist ein größerer Folgeschritt und muss mit Migrationsplan erfolgen.

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "sillyhome-next"
-version = "1.5.1"
+version = "1.5.3"
 description = "Lokales Smart-Home-Intelligenzsystem für Home Assistant"
 requires-python = ">=3.11"
 dependencies = [

tests/api/test_actuators.py

@@ -6,6 +6,7 @@ from pathlib import Path
 
 from fastapi.testclient import TestClient
 
+from app.actuators.cache_db import DashboardCache
 from app.actuators.lifecycle import ActuatorReconciliationService
 from app.actuators.models import JobStatus, ModelSnapshot
 from app.actuators.store import ActuatorStore
@@ -146,6 +147,7 @@ def _install_service(tmp_path: Path) -> None:
     )
     app.state.registry = ModelRegistry(tmp_path / "models")
     app.state.actuator_store = ActuatorStore(tmp_path / "actuators")
+    app.state.dashboard_cache = DashboardCache(tmp_path / "actuators" / "dashboard_cache.sqlite3")
     app.state.ha_reader = FakeHaReader(
         entities,
         {"sensor.abstellkammer_illuminance": [10, 11, 12, 13, 14, 15]},

tests/test_main.py

@@ -92,11 +92,11 @@ def test_ha_event_listener_processes_state_change(tmp_path: Path) -> None:
             except asyncio.CancelledError:
                 pass
 
-        connect.assert_called_once_with(
+            connect.assert_called_once_with(
-            "ws://homeassistant:8123/api/websocket",
+                "ws://homeassistant:8123/api/websocket",
-            ping_interval=20,
+                ping_interval=30,
-            ping_timeout=10,
+                ping_timeout=30,
-        )
+            )
         assert fake_ws.sent == [
             {"type": "auth", "access_token": "test-token"},
             {"id": 1, "type": "subscribe_events", "event_type": "state_changed"},