Otto/sillyhome-future
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add HA history and model evaluation

Browse Source
This commit is contained in:
Otto
2026-06-18 17:08:23 +02:00
parent 07e3e96c30
commit 6e6031cfda
8 changed files with 170 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

41
app/core/ha_client.py
View File

@@ -72,6 +72,46 @@ class FutureHaClient:
)
response.raise_for_status()
def read_history(
self,
entity_ids: list[str],
start_time: datetime,
end_time: datetime,
) -> dict[str, list[StateEvent]]:
params = {
"filter_entity_id": ",".join(entity_ids),
"end_time": end_time.isoformat(),
"minimal_response": "1",
}
with httpx.Client(timeout=self._config.timeout_seconds) as client:
response = client.get(
f"{self._core_url}/api/history/period/{start_time.isoformat()}",
headers=self._headers,
params=params,
)
response.raise_for_status()
payload = response.json()
result: dict[str, list[StateEvent]] = {entity_id: [] for entity_id in entity_ids}
for series in payload if isinstance(payload, list) else []:
if not isinstance(series, list):
continue
for item in series:
if not isinstance(item, dict):
continue
entity_id = item.get("entity_id")
if not isinstance(entity_id, str):
continue
result.setdefault(entity_id, []).append(
StateEvent(
entity_id=entity_id,
new_state=item.get("state") if isinstance(item.get("state"), str) else None,
changed_at=_parse_datetime(
item.get("last_changed") or item.get("last_updated")
),
)
)
return result
async def listen_state_events(self) -> AsyncIterator[StateEvent]:
websocket_url = self._config.websocket_url or _default_websocket_url(self._core_url)
async with websockets.connect(websocket_url, ping_interval=None) as websocket:
@@ -160,4 +200,3 @@ def _parse_datetime(value: object) -> datetime:
if parsed.tzinfo is None:
return parsed.replace(tzinfo=timezone.utc)
return parsed

16
app/core/models.py
View File

@@ -123,6 +123,19 @@ class ModelRecord(BaseModel):
trained_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc))
class ModelEvaluation(BaseModel):
evaluation_id: str
model_id: str
actuator_entity_id: str
score: float = Field(default=0.0, ge=0.0, le=1.0)
coverage: float = Field(default=0.0, ge=0.0, le=1.0)
dry_run_success_rate: float = Field(default=0.0, ge=0.0, le=1.0)
feedback_score: float = Field(default=0.0, ge=0.0, le=1.0)
verdict: str
reasons: list[str] = Field(default_factory=list)
evaluated_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc))
class JobQueueItem(BaseModel):
job_id: str
kind: str
@@ -144,6 +157,8 @@ class HistoryAnalysis(BaseModel):
samples: int
last_state: str | None = None
changed_at: datetime | None = None
unique_states: int = 0
transitions: int = 0
recommendation: str
@@ -181,6 +196,7 @@ class LearningState(BaseModel):
scenes: dict[str, SceneProfile] = Field(default_factory=dict)
automation_proposals: dict[str, AutomationProposal] = Field(default_factory=dict)
models: dict[str, ModelRecord] = Field(default_factory=dict)
model_evaluations: dict[str, ModelEvaluation] = Field(default_factory=dict)
jobs: dict[str, JobQueueItem] = Field(default_factory=dict)
weight_overrides: dict[str, SensorWeightOverride] = Field(default_factory=dict)

108
app/main.py
View File

@@ -28,6 +28,7 @@ from app.core.models import (
JobStatus,
LearningProfile,
LearningState,
ModelEvaluation,
ModelRecord,
ProposalStatus,
RuntimeState,
@@ -104,12 +105,19 @@ class AutomationProposalRequest(BaseModel):
class HistoryAnalysisRequest(BaseModel):
entity_ids: list[str] = Field(default_factory=list)
start_time: datetime | None = None
end_time: datetime | None = None
class TrainModelRequest(BaseModel):
actuator_entity_id: str | None = None
class EvaluateModelRequest(BaseModel):
model_id: str | None = None
actuator_entity_id: str | None = None
class WeightOverrideRequest(BaseModel):
sensor_weights: dict[str, float] = Field(default_factory=dict)
note: str | None = Field(default=None, max_length=500)
@@ -135,7 +143,7 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]:
app = FastAPI(
title="SillyHome Future API",
description="SillyHome v2 event-core side project.",
version="2.0.0-alpha.11",
version="2.0.0-alpha.12",
lifespan=lifespan,
)
@@ -224,7 +232,7 @@ def feature_parity() -> dict[str, object]:
"Sensor-Gewichte",
"Job-Queue",
],
"next_to_expand": ["echte Langzeit-History aus HA", "fortgeschrittene Modellbewertung"],
"next_to_expand": ["Dashboard-Flaechen fuer History/Modelle"],
}
@@ -287,16 +295,23 @@ def automation_proposal_yaml(proposal_id: str) -> str:
def analyze_history(request: HistoryAnalysisRequest) -> list[HistoryAnalysis]:
runtime = stores.runtime()
ids = request.entity_ids or list(runtime.entities)[:50]
history: dict[str, list[StateEvent]] = {}
if ha_client is not None and request.start_time is not None and request.end_time is not None:
history = ha_client.read_history(ids, request.start_time, request.end_time)
result: list[HistoryAnalysis] = []
for entity_id in ids:
entity = runtime.entities.get(entity_id)
samples = history.get(entity_id, [])
matching_audit = [item for item in runtime.audit if item.entity_id == entity_id]
states = [sample.new_state for sample in samples if sample.new_state is not None]
result.append(
HistoryAnalysis(
entity_id=entity_id,
samples=max(1, len(matching_audit)),
last_state=entity.state if entity else None,
changed_at=entity.changed_at if entity else None,
samples=max(1, len(samples) or len(matching_audit)),
last_state=states[-1] if states else (entity.state if entity else None),
changed_at=samples[-1].changed_at if samples else (entity.changed_at if entity else None),
unique_states=len(set(states)) if states else int(entity is not None),
transitions=_count_transitions(states),
recommendation=(
"Als Trigger geeignet."
if entity is not None and entity.domain in {"binary_sensor", "sensor"}
@@ -341,6 +356,31 @@ def list_models() -> list[ModelRecord]:
return list(stores.learning().models.values())
@app.post("/v2/models/evaluate", response_model=list[ModelEvaluation])
def evaluate_models(request: EvaluateModelRequest) -> list[ModelEvaluation]:
state = stores.learning()
control = stores.control()
models = list(state.models.values())
if request.model_id:
models = [model for model in models if model.model_id == request.model_id]
if request.actuator_entity_id:
models = [model for model in models if model.actuator_entity_id == request.actuator_entity_id]
evaluations = [
_evaluate_model(model, state, control)
for model in models
]
for evaluation in evaluations:
state.model_evaluations[evaluation.evaluation_id] = evaluation
stores.save_learning(state)
_append_job("model_evaluation", f"{len(evaluations)} Modelle bewertet.")
return evaluations
@app.get("/v2/models/evaluations", response_model=list[ModelEvaluation])
def list_model_evaluations() -> list[ModelEvaluation]:
return list(stores.learning().model_evaluations.values())[-100:]
@app.get("/v2/jobs", response_model=list[JobQueueItem])
def list_jobs() -> list[JobQueueItem]:
return list(stores.learning().jobs.values())[-100:]
@@ -884,6 +924,64 @@ def _decide_automation_proposal(
return updated
def _count_transitions(states: list[str]) -> int:
if not states:
return 0
transitions = 0
previous = states[0]
for state in states[1:]:
transitions += int(state != previous)
previous = state
return transitions
def _evaluate_model(
model: ModelRecord,
learning: LearningState,
control: ControlState,
) -> ModelEvaluation:
profile = learning.profiles.get(
model.actuator_entity_id,
LearningProfile(actuator_entity_id=model.actuator_entity_id),
)
control_profile = control.profiles.get(
model.actuator_entity_id,
ControlProfile(actuator_entity_id=model.actuator_entity_id),
)
coverage = min(1.0, model.pattern_count / 5)
dry_run_events = max(1, control_profile.dry_run_events)
dry_run_success_rate = control_profile.dry_run_successes / dry_run_events
feedback_total = profile.feedback_positive + profile.feedback_negative
feedback_score = (
profile.feedback_positive / feedback_total if feedback_total else 0.5
)
score = round(
(model.confidence * 0.35)
+ (coverage * 0.2)
+ (dry_run_success_rate * 0.3)
+ (feedback_score * 0.15),
4,
)
reasons = [
f"Modell-Confidence {model.confidence:.0%}",
f"Pattern-Abdeckung {coverage:.0%}",
f"Dry-run-Erfolg {dry_run_success_rate:.0%}",
f"Feedback-Score {feedback_score:.0%}",
]
verdict = "bereit" if score >= 0.82 and control_profile.active_ready else "weiter testen"
return ModelEvaluation(
evaluation_id=f"eval-{model.model_id}-{len(learning.model_evaluations) + 1}",
model_id=model.model_id,
actuator_entity_id=model.actuator_entity_id,
score=score,
coverage=coverage,
dry_run_success_rate=dry_run_success_rate,
feedback_score=feedback_score,
verdict=verdict,
reasons=reasons,
)
def _execute_ha_decision(decision: object) -> bool:
if ha_client is None or not hasattr(decision, "actuator_entity_id"):
return False