Add HA history and model evaluation

app/main.py

@@ -28,6 +28,7 @@ from app.core.models import (
     JobStatus,
     LearningProfile,
     LearningState,
+    ModelEvaluation,
     ModelRecord,
     ProposalStatus,
     RuntimeState,
@@ -104,12 +105,19 @@ class AutomationProposalRequest(BaseModel):
 
 class HistoryAnalysisRequest(BaseModel):
     entity_ids: list[str] = Field(default_factory=list)
+    start_time: datetime | None = None
+    end_time: datetime | None = None
 
 
 class TrainModelRequest(BaseModel):
     actuator_entity_id: str | None = None
 
 
+class EvaluateModelRequest(BaseModel):
+    model_id: str | None = None
+    actuator_entity_id: str | None = None
+
+
 class WeightOverrideRequest(BaseModel):
     sensor_weights: dict[str, float] = Field(default_factory=dict)
     note: str | None = Field(default=None, max_length=500)
@@ -135,7 +143,7 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]:
 app = FastAPI(
     title="SillyHome Future API",
     description="SillyHome v2 event-core side project.",
-    version="2.0.0-alpha.11",
+    version="2.0.0-alpha.12",
     lifespan=lifespan,
 )
 
@@ -224,7 +232,7 @@ def feature_parity() -> dict[str, object]:
             "Sensor-Gewichte",
             "Job-Queue",
         ],
-        "next_to_expand": ["echte Langzeit-History aus HA", "fortgeschrittene Modellbewertung"],
+        "next_to_expand": ["Dashboard-Flaechen fuer History/Modelle"],
     }
 
 
@@ -287,16 +295,23 @@ def automation_proposal_yaml(proposal_id: str) -> str:
 def analyze_history(request: HistoryAnalysisRequest) -> list[HistoryAnalysis]:
     runtime = stores.runtime()
     ids = request.entity_ids or list(runtime.entities)[:50]
+    history: dict[str, list[StateEvent]] = {}
+    if ha_client is not None and request.start_time is not None and request.end_time is not None:
+        history = ha_client.read_history(ids, request.start_time, request.end_time)
     result: list[HistoryAnalysis] = []
     for entity_id in ids:
         entity = runtime.entities.get(entity_id)
+        samples = history.get(entity_id, [])
         matching_audit = [item for item in runtime.audit if item.entity_id == entity_id]
+        states = [sample.new_state for sample in samples if sample.new_state is not None]
         result.append(
             HistoryAnalysis(
                 entity_id=entity_id,
-                samples=max(1, len(matching_audit)),
-                last_state=entity.state if entity else None,
-                changed_at=entity.changed_at if entity else None,
+                samples=max(1, len(samples) or len(matching_audit)),
+                last_state=states[-1] if states else (entity.state if entity else None),
+                changed_at=samples[-1].changed_at if samples else (entity.changed_at if entity else None),
+                unique_states=len(set(states)) if states else int(entity is not None),
+                transitions=_count_transitions(states),
                 recommendation=(
                     "Als Trigger geeignet."
                     if entity is not None and entity.domain in {"binary_sensor", "sensor"}
@@ -341,6 +356,31 @@ def list_models() -> list[ModelRecord]:
     return list(stores.learning().models.values())
 
 
+@app.post("/v2/models/evaluate", response_model=list[ModelEvaluation])
+def evaluate_models(request: EvaluateModelRequest) -> list[ModelEvaluation]:
+    state = stores.learning()
+    control = stores.control()
+    models = list(state.models.values())
+    if request.model_id:
+        models = [model for model in models if model.model_id == request.model_id]
+    if request.actuator_entity_id:
+        models = [model for model in models if model.actuator_entity_id == request.actuator_entity_id]
+    evaluations = [
+        _evaluate_model(model, state, control)
+        for model in models
+    ]
+    for evaluation in evaluations:
+        state.model_evaluations[evaluation.evaluation_id] = evaluation
+    stores.save_learning(state)
+    _append_job("model_evaluation", f"{len(evaluations)} Modelle bewertet.")
+    return evaluations
+
+
+@app.get("/v2/models/evaluations", response_model=list[ModelEvaluation])
+def list_model_evaluations() -> list[ModelEvaluation]:
+    return list(stores.learning().model_evaluations.values())[-100:]
+
+
 @app.get("/v2/jobs", response_model=list[JobQueueItem])
 def list_jobs() -> list[JobQueueItem]:
     return list(stores.learning().jobs.values())[-100:]
@@ -884,6 +924,64 @@ def _decide_automation_proposal(
     return updated
 
 
+def _count_transitions(states: list[str]) -> int:
+    if not states:
+        return 0
+    transitions = 0
+    previous = states[0]
+    for state in states[1:]:
+        transitions += int(state != previous)
+        previous = state
+    return transitions
+
+
+def _evaluate_model(
+    model: ModelRecord,
+    learning: LearningState,
+    control: ControlState,
+) -> ModelEvaluation:
+    profile = learning.profiles.get(
+        model.actuator_entity_id,
+        LearningProfile(actuator_entity_id=model.actuator_entity_id),
+    )
+    control_profile = control.profiles.get(
+        model.actuator_entity_id,
+        ControlProfile(actuator_entity_id=model.actuator_entity_id),
+    )
+    coverage = min(1.0, model.pattern_count / 5)
+    dry_run_events = max(1, control_profile.dry_run_events)
+    dry_run_success_rate = control_profile.dry_run_successes / dry_run_events
+    feedback_total = profile.feedback_positive + profile.feedback_negative
+    feedback_score = (
+        profile.feedback_positive / feedback_total if feedback_total else 0.5
+    )
+    score = round(
+        (model.confidence * 0.35)
+        + (coverage * 0.2)
+        + (dry_run_success_rate * 0.3)
+        + (feedback_score * 0.15),
+        4,
+    )
+    reasons = [
+        f"Modell-Confidence {model.confidence:.0%}",
+        f"Pattern-Abdeckung {coverage:.0%}",
+        f"Dry-run-Erfolg {dry_run_success_rate:.0%}",
+        f"Feedback-Score {feedback_score:.0%}",
+    ]
+    verdict = "bereit" if score >= 0.82 and control_profile.active_ready else "weiter testen"
+    return ModelEvaluation(
+        evaluation_id=f"eval-{model.model_id}-{len(learning.model_evaluations) + 1}",
+        model_id=model.model_id,
+        actuator_entity_id=model.actuator_entity_id,
+        score=score,
+        coverage=coverage,
+        dry_run_success_rate=dry_run_success_rate,
+        feedback_score=feedback_score,
+        verdict=verdict,
+        reasons=reasons,
+    )
+
+
 def _execute_ha_decision(decision: object) -> bool:
     if ha_client is None or not hasattr(decision, "actuator_entity_id"):
         return False