Add HA history and model evaluation

addon/Dockerfile

@@ -2,7 +2,7 @@ FROM python:3.13-slim
 
 WORKDIR /app
 
-ARG SILLYHOME_FUTURE_REF=v2.0.0-alpha.11
+ARG SILLYHOME_FUTURE_REF=v2.0.0-alpha.12
 RUN python -m pip install --no-cache-dir \
     "http://192.168.6.31:3000/Otto/sillyhome-future/archive/${SILLYHOME_FUTURE_REF}.tar.gz"
 

addon/config.yaml

@@ -1,5 +1,5 @@
 name: SillyHome Future
-version: "2.0.0-alpha.11"
+version: "2.0.0-alpha.12"
 slug: sillyhome_future
 description: Event-first SillyHome v2 test controller
 url: http://192.168.6.31:3000/Otto/sillyhome-future

app/core/ha_client.py

@@ -72,6 +72,46 @@ class FutureHaClient:
             )
             response.raise_for_status()
 
+    def read_history(
+        self,
+        entity_ids: list[str],
+        start_time: datetime,
+        end_time: datetime,
+    ) -> dict[str, list[StateEvent]]:
+        params = {
+            "filter_entity_id": ",".join(entity_ids),
+            "end_time": end_time.isoformat(),
+            "minimal_response": "1",
+        }
+        with httpx.Client(timeout=self._config.timeout_seconds) as client:
+            response = client.get(
+                f"{self._core_url}/api/history/period/{start_time.isoformat()}",
+                headers=self._headers,
+                params=params,
+            )
+            response.raise_for_status()
+            payload = response.json()
+        result: dict[str, list[StateEvent]] = {entity_id: [] for entity_id in entity_ids}
+        for series in payload if isinstance(payload, list) else []:
+            if not isinstance(series, list):
+                continue
+            for item in series:
+                if not isinstance(item, dict):
+                    continue
+                entity_id = item.get("entity_id")
+                if not isinstance(entity_id, str):
+                    continue
+                result.setdefault(entity_id, []).append(
+                    StateEvent(
+                        entity_id=entity_id,
+                        new_state=item.get("state") if isinstance(item.get("state"), str) else None,
+                        changed_at=_parse_datetime(
+                            item.get("last_changed") or item.get("last_updated")
+                        ),
+                    )
+                )
+        return result
+
     async def listen_state_events(self) -> AsyncIterator[StateEvent]:
         websocket_url = self._config.websocket_url or _default_websocket_url(self._core_url)
         async with websockets.connect(websocket_url, ping_interval=None) as websocket:
@@ -160,4 +200,3 @@ def _parse_datetime(value: object) -> datetime:
     if parsed.tzinfo is None:
         return parsed.replace(tzinfo=timezone.utc)
     return parsed
-

app/core/models.py

@@ -123,6 +123,19 @@ class ModelRecord(BaseModel):
     trained_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc))
 
 
+class ModelEvaluation(BaseModel):
+    evaluation_id: str
+    model_id: str
+    actuator_entity_id: str
+    score: float = Field(default=0.0, ge=0.0, le=1.0)
+    coverage: float = Field(default=0.0, ge=0.0, le=1.0)
+    dry_run_success_rate: float = Field(default=0.0, ge=0.0, le=1.0)
+    feedback_score: float = Field(default=0.0, ge=0.0, le=1.0)
+    verdict: str
+    reasons: list[str] = Field(default_factory=list)
+    evaluated_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc))
+
+
 class JobQueueItem(BaseModel):
     job_id: str
     kind: str
@@ -144,6 +157,8 @@ class HistoryAnalysis(BaseModel):
     samples: int
     last_state: str | None = None
     changed_at: datetime | None = None
+    unique_states: int = 0
+    transitions: int = 0
     recommendation: str
 
 
@@ -181,6 +196,7 @@ class LearningState(BaseModel):
     scenes: dict[str, SceneProfile] = Field(default_factory=dict)
     automation_proposals: dict[str, AutomationProposal] = Field(default_factory=dict)
     models: dict[str, ModelRecord] = Field(default_factory=dict)
+    model_evaluations: dict[str, ModelEvaluation] = Field(default_factory=dict)
     jobs: dict[str, JobQueueItem] = Field(default_factory=dict)
     weight_overrides: dict[str, SensorWeightOverride] = Field(default_factory=dict)
 

app/main.py

@@ -28,6 +28,7 @@ from app.core.models import (
     JobStatus,
     LearningProfile,
     LearningState,
+    ModelEvaluation,
     ModelRecord,
     ProposalStatus,
     RuntimeState,
@@ -104,12 +105,19 @@ class AutomationProposalRequest(BaseModel):
 
 class HistoryAnalysisRequest(BaseModel):
     entity_ids: list[str] = Field(default_factory=list)
+    start_time: datetime | None = None
+    end_time: datetime | None = None
 
 
 class TrainModelRequest(BaseModel):
     actuator_entity_id: str | None = None
 
 
+class EvaluateModelRequest(BaseModel):
+    model_id: str | None = None
+    actuator_entity_id: str | None = None
+
+
 class WeightOverrideRequest(BaseModel):
     sensor_weights: dict[str, float] = Field(default_factory=dict)
     note: str | None = Field(default=None, max_length=500)
@@ -135,7 +143,7 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]:
 app = FastAPI(
     title="SillyHome Future API",
     description="SillyHome v2 event-core side project.",
-    version="2.0.0-alpha.11",
+    version="2.0.0-alpha.12",
     lifespan=lifespan,
 )
 
@@ -224,7 +232,7 @@ def feature_parity() -> dict[str, object]:
             "Sensor-Gewichte",
             "Job-Queue",
         ],
-        "next_to_expand": ["echte Langzeit-History aus HA", "fortgeschrittene Modellbewertung"],
+        "next_to_expand": ["Dashboard-Flaechen fuer History/Modelle"],
     }
 
 
@@ -287,16 +295,23 @@ def automation_proposal_yaml(proposal_id: str) -> str:
 def analyze_history(request: HistoryAnalysisRequest) -> list[HistoryAnalysis]:
     runtime = stores.runtime()
     ids = request.entity_ids or list(runtime.entities)[:50]
+    history: dict[str, list[StateEvent]] = {}
+    if ha_client is not None and request.start_time is not None and request.end_time is not None:
+        history = ha_client.read_history(ids, request.start_time, request.end_time)
     result: list[HistoryAnalysis] = []
     for entity_id in ids:
         entity = runtime.entities.get(entity_id)
+        samples = history.get(entity_id, [])
         matching_audit = [item for item in runtime.audit if item.entity_id == entity_id]
+        states = [sample.new_state for sample in samples if sample.new_state is not None]
         result.append(
             HistoryAnalysis(
                 entity_id=entity_id,
-                samples=max(1, len(matching_audit)),
+                samples=max(1, len(samples) or len(matching_audit)),
-                last_state=entity.state if entity else None,
+                last_state=states[-1] if states else (entity.state if entity else None),
-                changed_at=entity.changed_at if entity else None,
+                changed_at=samples[-1].changed_at if samples else (entity.changed_at if entity else None),
+                unique_states=len(set(states)) if states else int(entity is not None),
+                transitions=_count_transitions(states),
                 recommendation=(
                     "Als Trigger geeignet."
                     if entity is not None and entity.domain in {"binary_sensor", "sensor"}
@@ -341,6 +356,31 @@ def list_models() -> list[ModelRecord]:
     return list(stores.learning().models.values())
 
 
+@app.post("/v2/models/evaluate", response_model=list[ModelEvaluation])
+def evaluate_models(request: EvaluateModelRequest) -> list[ModelEvaluation]:
+    state = stores.learning()
+    control = stores.control()
+    models = list(state.models.values())
+    if request.model_id:
+        models = [model for model in models if model.model_id == request.model_id]
+    if request.actuator_entity_id:
+        models = [model for model in models if model.actuator_entity_id == request.actuator_entity_id]
+    evaluations = [
+        _evaluate_model(model, state, control)
+        for model in models
+    ]
+    for evaluation in evaluations:
+        state.model_evaluations[evaluation.evaluation_id] = evaluation
+    stores.save_learning(state)
+    _append_job("model_evaluation", f"{len(evaluations)} Modelle bewertet.")
+    return evaluations
+
+
+@app.get("/v2/models/evaluations", response_model=list[ModelEvaluation])
+def list_model_evaluations() -> list[ModelEvaluation]:
+    return list(stores.learning().model_evaluations.values())[-100:]
+
+
 @app.get("/v2/jobs", response_model=list[JobQueueItem])
 def list_jobs() -> list[JobQueueItem]:
     return list(stores.learning().jobs.values())[-100:]
@@ -884,6 +924,64 @@ def _decide_automation_proposal(
     return updated
 
 
+def _count_transitions(states: list[str]) -> int:
+    if not states:
+        return 0
+    transitions = 0
+    previous = states[0]
+    for state in states[1:]:
+        transitions += int(state != previous)
+        previous = state
+    return transitions
+
+
+def _evaluate_model(
+    model: ModelRecord,
+    learning: LearningState,
+    control: ControlState,
+) -> ModelEvaluation:
+    profile = learning.profiles.get(
+        model.actuator_entity_id,
+        LearningProfile(actuator_entity_id=model.actuator_entity_id),
+    )
+    control_profile = control.profiles.get(
+        model.actuator_entity_id,
+        ControlProfile(actuator_entity_id=model.actuator_entity_id),
+    )
+    coverage = min(1.0, model.pattern_count / 5)
+    dry_run_events = max(1, control_profile.dry_run_events)
+    dry_run_success_rate = control_profile.dry_run_successes / dry_run_events
+    feedback_total = profile.feedback_positive + profile.feedback_negative
+    feedback_score = (
+        profile.feedback_positive / feedback_total if feedback_total else 0.5
+    )
+    score = round(
+        (model.confidence * 0.35)
+        + (coverage * 0.2)
+        + (dry_run_success_rate * 0.3)
+        + (feedback_score * 0.15),
+        4,
+    )
+    reasons = [
+        f"Modell-Confidence {model.confidence:.0%}",
+        f"Pattern-Abdeckung {coverage:.0%}",
+        f"Dry-run-Erfolg {dry_run_success_rate:.0%}",
+        f"Feedback-Score {feedback_score:.0%}",
+    ]
+    verdict = "bereit" if score >= 0.82 and control_profile.active_ready else "weiter testen"
+    return ModelEvaluation(
+        evaluation_id=f"eval-{model.model_id}-{len(learning.model_evaluations) + 1}",
+        model_id=model.model_id,
+        actuator_entity_id=model.actuator_entity_id,
+        score=score,
+        coverage=coverage,
+        dry_run_success_rate=dry_run_success_rate,
+        feedback_score=feedback_score,
+        verdict=verdict,
+        reasons=reasons,
+    )
+
+
 def _execute_ha_decision(decision: object) -> bool:
     if ha_client is None or not hasattr(decision, "actuator_entity_id"):
         return False

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "sillyhome-future"
-version = "2.0.0-alpha.11"
+version = "2.0.0-alpha.12"
 description = "SillyHome v2 event-core prototype"
 requires-python = ">=3.11"
 dependencies = [

tests/test_addon_config.py

@@ -9,7 +9,7 @@ def test_addon_config_declares_future_addon() -> None:
     config = yaml.safe_load(Path("addon/config.yaml").read_text(encoding="utf-8"))
 
     assert config["slug"] == "sillyhome_future"
-    assert config["version"] == "2.0.0-alpha.11"
+    assert config["version"] == "2.0.0-alpha.12"
     assert config["ingress"] is True
     assert config["ingress_port"] == 8099
     assert config["homeassistant_api"] is True

tests/test_future_api.py

@@ -19,7 +19,7 @@ def test_health_and_backup_roundtrip(tmp_path, monkeypatch) -> None:  # type: ig
     restore = client.post("/v2/backup/restore", json=backup.json())
 
     assert health.status_code == 200
-    assert health.json()["version"] == "2.0.0-alpha.11"
+    assert health.json()["version"] == "2.0.0-alpha.12"
     assert backup.status_code == 200
     assert restore.status_code == 200
     assert restore.json() == {"status": "restored"}
@@ -102,6 +102,8 @@ def test_dashboard_control_and_learning_endpoints(tmp_path, monkeypatch) -> None
         json={"entity_ids": ["binary_sensor.storage_door"]},
     )
     models = client.post("/v2/models/train", json={"actuator_entity_id": "light.storage"})
+    evaluation = client.post("/v2/models/evaluate", json={"actuator_entity_id": "light.storage"})
+    evaluations = client.get("/v2/models/evaluations")
     jobs = client.get("/v2/jobs")
     dashboard = client.get("/v2/dashboard")
 
@@ -141,6 +143,10 @@ def test_dashboard_control_and_learning_endpoints(tmp_path, monkeypatch) -> None
     assert history.json()[0]["entity_id"] == "binary_sensor.storage_door"
     assert models.status_code == 200
     assert models.json()[0]["actuator_entity_id"] == "light.storage"
+    assert evaluation.status_code == 200
+    assert evaluation.json()[0]["verdict"] in {"bereit", "weiter testen"}
+    assert evaluations.status_code == 200
+    assert evaluations.json()
     assert jobs.status_code == 200
     assert jobs.json()
     assert dashboard.status_code == 200