From 6e6031cfdaa0fdf13fc4a8830be7af5da663e3c1 Mon Sep 17 00:00:00 2001 From: Otto Date: Thu, 18 Jun 2026 17:08:23 +0200 Subject: [PATCH] Add HA history and model evaluation --- addon/Dockerfile | 2 +- addon/config.yaml | 2 +- app/core/ha_client.py | 41 +++++++++++++- app/core/models.py | 16 ++++++ app/main.py | 108 +++++++++++++++++++++++++++++++++++-- pyproject.toml | 2 +- tests/test_addon_config.py | 2 +- tests/test_future_api.py | 8 ++- 8 files changed, 170 insertions(+), 11 deletions(-) diff --git a/addon/Dockerfile b/addon/Dockerfile index b719650..738bcc7 100644 --- a/addon/Dockerfile +++ b/addon/Dockerfile @@ -2,7 +2,7 @@ FROM python:3.13-slim WORKDIR /app -ARG SILLYHOME_FUTURE_REF=v2.0.0-alpha.11 +ARG SILLYHOME_FUTURE_REF=v2.0.0-alpha.12 RUN python -m pip install --no-cache-dir \ "http://192.168.6.31:3000/Otto/sillyhome-future/archive/${SILLYHOME_FUTURE_REF}.tar.gz" diff --git a/addon/config.yaml b/addon/config.yaml index 7b76ca1..dea8988 100644 --- a/addon/config.yaml +++ b/addon/config.yaml @@ -1,5 +1,5 @@ name: SillyHome Future -version: "2.0.0-alpha.11" +version: "2.0.0-alpha.12" slug: sillyhome_future description: Event-first SillyHome v2 test controller url: http://192.168.6.31:3000/Otto/sillyhome-future diff --git a/app/core/ha_client.py b/app/core/ha_client.py index c3be304..ab7420e 100644 --- a/app/core/ha_client.py +++ b/app/core/ha_client.py @@ -72,6 +72,46 @@ class FutureHaClient: ) response.raise_for_status() + def read_history( + self, + entity_ids: list[str], + start_time: datetime, + end_time: datetime, + ) -> dict[str, list[StateEvent]]: + params = { + "filter_entity_id": ",".join(entity_ids), + "end_time": end_time.isoformat(), + "minimal_response": "1", + } + with httpx.Client(timeout=self._config.timeout_seconds) as client: + response = client.get( + f"{self._core_url}/api/history/period/{start_time.isoformat()}", + headers=self._headers, + params=params, + ) + response.raise_for_status() + payload = response.json() + result: dict[str, list[StateEvent]] = {entity_id: [] for entity_id in entity_ids} + for series in payload if isinstance(payload, list) else []: + if not isinstance(series, list): + continue + for item in series: + if not isinstance(item, dict): + continue + entity_id = item.get("entity_id") + if not isinstance(entity_id, str): + continue + result.setdefault(entity_id, []).append( + StateEvent( + entity_id=entity_id, + new_state=item.get("state") if isinstance(item.get("state"), str) else None, + changed_at=_parse_datetime( + item.get("last_changed") or item.get("last_updated") + ), + ) + ) + return result + async def listen_state_events(self) -> AsyncIterator[StateEvent]: websocket_url = self._config.websocket_url or _default_websocket_url(self._core_url) async with websockets.connect(websocket_url, ping_interval=None) as websocket: @@ -160,4 +200,3 @@ def _parse_datetime(value: object) -> datetime: if parsed.tzinfo is None: return parsed.replace(tzinfo=timezone.utc) return parsed - diff --git a/app/core/models.py b/app/core/models.py index 5759d31..afc7077 100644 --- a/app/core/models.py +++ b/app/core/models.py @@ -123,6 +123,19 @@ class ModelRecord(BaseModel): trained_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc)) +class ModelEvaluation(BaseModel): + evaluation_id: str + model_id: str + actuator_entity_id: str + score: float = Field(default=0.0, ge=0.0, le=1.0) + coverage: float = Field(default=0.0, ge=0.0, le=1.0) + dry_run_success_rate: float = Field(default=0.0, ge=0.0, le=1.0) + feedback_score: float = Field(default=0.0, ge=0.0, le=1.0) + verdict: str + reasons: list[str] = Field(default_factory=list) + evaluated_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc)) + + class JobQueueItem(BaseModel): job_id: str kind: str @@ -144,6 +157,8 @@ class HistoryAnalysis(BaseModel): samples: int last_state: str | None = None changed_at: datetime | None = None + unique_states: int = 0 + transitions: int = 0 recommendation: str @@ -181,6 +196,7 @@ class LearningState(BaseModel): scenes: dict[str, SceneProfile] = Field(default_factory=dict) automation_proposals: dict[str, AutomationProposal] = Field(default_factory=dict) models: dict[str, ModelRecord] = Field(default_factory=dict) + model_evaluations: dict[str, ModelEvaluation] = Field(default_factory=dict) jobs: dict[str, JobQueueItem] = Field(default_factory=dict) weight_overrides: dict[str, SensorWeightOverride] = Field(default_factory=dict) diff --git a/app/main.py b/app/main.py index c8964a9..2aa7377 100644 --- a/app/main.py +++ b/app/main.py @@ -28,6 +28,7 @@ from app.core.models import ( JobStatus, LearningProfile, LearningState, + ModelEvaluation, ModelRecord, ProposalStatus, RuntimeState, @@ -104,12 +105,19 @@ class AutomationProposalRequest(BaseModel): class HistoryAnalysisRequest(BaseModel): entity_ids: list[str] = Field(default_factory=list) + start_time: datetime | None = None + end_time: datetime | None = None class TrainModelRequest(BaseModel): actuator_entity_id: str | None = None +class EvaluateModelRequest(BaseModel): + model_id: str | None = None + actuator_entity_id: str | None = None + + class WeightOverrideRequest(BaseModel): sensor_weights: dict[str, float] = Field(default_factory=dict) note: str | None = Field(default=None, max_length=500) @@ -135,7 +143,7 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]: app = FastAPI( title="SillyHome Future API", description="SillyHome v2 event-core side project.", - version="2.0.0-alpha.11", + version="2.0.0-alpha.12", lifespan=lifespan, ) @@ -224,7 +232,7 @@ def feature_parity() -> dict[str, object]: "Sensor-Gewichte", "Job-Queue", ], - "next_to_expand": ["echte Langzeit-History aus HA", "fortgeschrittene Modellbewertung"], + "next_to_expand": ["Dashboard-Flaechen fuer History/Modelle"], } @@ -287,16 +295,23 @@ def automation_proposal_yaml(proposal_id: str) -> str: def analyze_history(request: HistoryAnalysisRequest) -> list[HistoryAnalysis]: runtime = stores.runtime() ids = request.entity_ids or list(runtime.entities)[:50] + history: dict[str, list[StateEvent]] = {} + if ha_client is not None and request.start_time is not None and request.end_time is not None: + history = ha_client.read_history(ids, request.start_time, request.end_time) result: list[HistoryAnalysis] = [] for entity_id in ids: entity = runtime.entities.get(entity_id) + samples = history.get(entity_id, []) matching_audit = [item for item in runtime.audit if item.entity_id == entity_id] + states = [sample.new_state for sample in samples if sample.new_state is not None] result.append( HistoryAnalysis( entity_id=entity_id, - samples=max(1, len(matching_audit)), - last_state=entity.state if entity else None, - changed_at=entity.changed_at if entity else None, + samples=max(1, len(samples) or len(matching_audit)), + last_state=states[-1] if states else (entity.state if entity else None), + changed_at=samples[-1].changed_at if samples else (entity.changed_at if entity else None), + unique_states=len(set(states)) if states else int(entity is not None), + transitions=_count_transitions(states), recommendation=( "Als Trigger geeignet." if entity is not None and entity.domain in {"binary_sensor", "sensor"} @@ -341,6 +356,31 @@ def list_models() -> list[ModelRecord]: return list(stores.learning().models.values()) +@app.post("/v2/models/evaluate", response_model=list[ModelEvaluation]) +def evaluate_models(request: EvaluateModelRequest) -> list[ModelEvaluation]: + state = stores.learning() + control = stores.control() + models = list(state.models.values()) + if request.model_id: + models = [model for model in models if model.model_id == request.model_id] + if request.actuator_entity_id: + models = [model for model in models if model.actuator_entity_id == request.actuator_entity_id] + evaluations = [ + _evaluate_model(model, state, control) + for model in models + ] + for evaluation in evaluations: + state.model_evaluations[evaluation.evaluation_id] = evaluation + stores.save_learning(state) + _append_job("model_evaluation", f"{len(evaluations)} Modelle bewertet.") + return evaluations + + +@app.get("/v2/models/evaluations", response_model=list[ModelEvaluation]) +def list_model_evaluations() -> list[ModelEvaluation]: + return list(stores.learning().model_evaluations.values())[-100:] + + @app.get("/v2/jobs", response_model=list[JobQueueItem]) def list_jobs() -> list[JobQueueItem]: return list(stores.learning().jobs.values())[-100:] @@ -884,6 +924,64 @@ def _decide_automation_proposal( return updated +def _count_transitions(states: list[str]) -> int: + if not states: + return 0 + transitions = 0 + previous = states[0] + for state in states[1:]: + transitions += int(state != previous) + previous = state + return transitions + + +def _evaluate_model( + model: ModelRecord, + learning: LearningState, + control: ControlState, +) -> ModelEvaluation: + profile = learning.profiles.get( + model.actuator_entity_id, + LearningProfile(actuator_entity_id=model.actuator_entity_id), + ) + control_profile = control.profiles.get( + model.actuator_entity_id, + ControlProfile(actuator_entity_id=model.actuator_entity_id), + ) + coverage = min(1.0, model.pattern_count / 5) + dry_run_events = max(1, control_profile.dry_run_events) + dry_run_success_rate = control_profile.dry_run_successes / dry_run_events + feedback_total = profile.feedback_positive + profile.feedback_negative + feedback_score = ( + profile.feedback_positive / feedback_total if feedback_total else 0.5 + ) + score = round( + (model.confidence * 0.35) + + (coverage * 0.2) + + (dry_run_success_rate * 0.3) + + (feedback_score * 0.15), + 4, + ) + reasons = [ + f"Modell-Confidence {model.confidence:.0%}", + f"Pattern-Abdeckung {coverage:.0%}", + f"Dry-run-Erfolg {dry_run_success_rate:.0%}", + f"Feedback-Score {feedback_score:.0%}", + ] + verdict = "bereit" if score >= 0.82 and control_profile.active_ready else "weiter testen" + return ModelEvaluation( + evaluation_id=f"eval-{model.model_id}-{len(learning.model_evaluations) + 1}", + model_id=model.model_id, + actuator_entity_id=model.actuator_entity_id, + score=score, + coverage=coverage, + dry_run_success_rate=dry_run_success_rate, + feedback_score=feedback_score, + verdict=verdict, + reasons=reasons, + ) + + def _execute_ha_decision(decision: object) -> bool: if ha_client is None or not hasattr(decision, "actuator_entity_id"): return False diff --git a/pyproject.toml b/pyproject.toml index 1098f9b..fc8e86f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "sillyhome-future" -version = "2.0.0-alpha.11" +version = "2.0.0-alpha.12" description = "SillyHome v2 event-core prototype" requires-python = ">=3.11" dependencies = [ diff --git a/tests/test_addon_config.py b/tests/test_addon_config.py index 4e3f7d8..06fba45 100644 --- a/tests/test_addon_config.py +++ b/tests/test_addon_config.py @@ -9,7 +9,7 @@ def test_addon_config_declares_future_addon() -> None: config = yaml.safe_load(Path("addon/config.yaml").read_text(encoding="utf-8")) assert config["slug"] == "sillyhome_future" - assert config["version"] == "2.0.0-alpha.11" + assert config["version"] == "2.0.0-alpha.12" assert config["ingress"] is True assert config["ingress_port"] == 8099 assert config["homeassistant_api"] is True diff --git a/tests/test_future_api.py b/tests/test_future_api.py index d9506a8..bca0203 100644 --- a/tests/test_future_api.py +++ b/tests/test_future_api.py @@ -19,7 +19,7 @@ def test_health_and_backup_roundtrip(tmp_path, monkeypatch) -> None: # type: ig restore = client.post("/v2/backup/restore", json=backup.json()) assert health.status_code == 200 - assert health.json()["version"] == "2.0.0-alpha.11" + assert health.json()["version"] == "2.0.0-alpha.12" assert backup.status_code == 200 assert restore.status_code == 200 assert restore.json() == {"status": "restored"} @@ -102,6 +102,8 @@ def test_dashboard_control_and_learning_endpoints(tmp_path, monkeypatch) -> None json={"entity_ids": ["binary_sensor.storage_door"]}, ) models = client.post("/v2/models/train", json={"actuator_entity_id": "light.storage"}) + evaluation = client.post("/v2/models/evaluate", json={"actuator_entity_id": "light.storage"}) + evaluations = client.get("/v2/models/evaluations") jobs = client.get("/v2/jobs") dashboard = client.get("/v2/dashboard") @@ -141,6 +143,10 @@ def test_dashboard_control_and_learning_endpoints(tmp_path, monkeypatch) -> None assert history.json()[0]["entity_id"] == "binary_sensor.storage_door" assert models.status_code == 200 assert models.json()[0]["actuator_entity_id"] == "light.storage" + assert evaluation.status_code == 200 + assert evaluation.json()[0]["verdict"] in {"bereit", "weiter testen"} + assert evaluations.status_code == 200 + assert evaluations.json() assert jobs.status_code == 200 assert jobs.json() assert dashboard.status_code == 200