Compare commits

..

9 Commits

Author SHA1 Message Date
0de537572d ML-009: add explainable predictions
Some checks failed
quality / test (3.11) (push) Has been cancelled
quality / test (3.13) (push) Has been cancelled
Closes #20
2026-06-13 20:16:26 +02:00
9d9e08cc0b Merge pull request 'ML-008: Statistical Baseline Model' (#26) from feature/ml-baseline-model into main 2026-06-13 20:13:29 +02:00
df2ddacfbf ML-008: add statistical baseline model
Some checks failed
quality / test (3.11) (push) Has been cancelled
quality / test (3.13) (push) Has been cancelled
Closes #19
2026-06-13 20:13:06 +02:00
ea5a206a86 Merge pull request 'HA data pipeline: Discovery und History' (#25) from feature/ha-discovery-history into main 2026-06-13 20:06:29 +02:00
816a516106 HA-008 HA-009: add discovery and history pipeline
Some checks failed
quality / test (3.11) (push) Has been cancelled
quality / test (3.13) (push) Has been cancelled
Closes #17

Closes #18
2026-06-13 20:06:05 +02:00
1fbed37126 Merge pull request 'Release v0.1.0' (#16) from release/v0.1.0 into main 2026-06-13 19:12:06 +02:00
dd496f9cc3 release: finalize v0.1.0 changelog 2026-06-13 19:11:42 +02:00
74b75de0fa Merge pull request 'ML-007: Retraining Pipeline und Model Updates' (#15) from feature/ml-007-retraining-pipeline into main 2026-06-13 19:10:54 +02:00
840c404c1c ML-007: add retraining pipeline and API
Some checks failed
quality / test (3.11) (push) Has been cancelled
quality / test (3.13) (push) Has been cancelled
Closes #13
2026-06-13 19:10:17 +02:00
34 changed files with 1714 additions and 95 deletions

1
.gitignore vendored
View File

@@ -4,6 +4,7 @@
/.vscode
__pycache__/
*.pyc
*.egg-info/
.mypy_cache/
.pytest_cache/
.ruff_cache/

View File

@@ -1,6 +1,16 @@
# Changelog
## Unreleased
- Deterministische, nutzerverständliche Erklärungen für jede Modellvorhersage
## 0.2.0 - 2026-06-13
- Klassifizierte Home-Assistant-Entity-Discovery mit Lernrelevanz und Filtern
- Validierter Zugriff auf die Home-Assistant-History-API
- Normalisierte, chronologisch sortierte numerische Zeitreihen über `/v1/history`
- Trainierbares statistisches Baseline-Modell mit persistierten Parametern
- Numerische Vorhersagen mit Confidence sowie MAE-/RMSE-Evaluation
## 0.1.0 - 2026-06-13
- Projektinitiierung
- Architektur, ADRs und Roadmap
- Einheitliche produktive FastAPI-App für HA- und ML-Routen
@@ -8,3 +18,4 @@
- Persistente, validierte und gegen Path Traversal gehärtete Model Registry
- Reproduzierbares Packaging, CI-Gates und gehärteter non-root Container
- Definierte API-Fehler und korrigierte Evaluationsmetriken
- Scheduler-tauglicher Retraining-Service mit API und atomischem Registry-Update

View File

@@ -4,10 +4,11 @@ Lokaler, datenschutzfreundlicher API-Prototyp für Home Assistant.
## Reifegrad
Version `0.1.0` stellt eine gehärtete technische Basis bereit: Home-Assistant-Entities
lesen, regelbasierte Bausteine und eine persistente Modell-Artefakt-Registry. Die
aktuelle Trainings- und Vorhersagelogik ist noch eine deterministische
Schnittstellen-Implementierung und **kein produktives Machine-Learning-Modell**.
Die aktuelle Entwicklungslinie stellt eine gehärtete technische Basis bereit:
Home-Assistant-Entities und Historie lesen, Sensoren klassifizieren,
regelbasierte Bausteine sowie ein lokal trainierbares statistisches
Baseline-Modell mit persistenter Registry, Confidence und echten
Evaluationsmetriken.
## Motivation
TheSillyHome zeigte die Idee: statt statischer Regeln das Zuhause aus Verhaltensmustern verstehen. Diese Architektur modernisiert den Ansatz in Richtung Explainable AI, hybride Intelligenzebenen und langlebige Wartbarkeit.
@@ -42,7 +43,11 @@ uvicorn app.main:app --reload
- `http://127.0.0.1:8000/health` - Health-Check
- `http://127.0.0.1:8000/docs/` - OpenAPI-Dokumentation
- `http://127.0.0.1:8000/v1/entities` - Home-Assistant-Entities
- `http://127.0.0.1:8000/v1/discovery` - klassifizierte, filterbare Entities
- `http://127.0.0.1:8000/v1/history` - normalisierte numerische Zeitreihen
- `http://127.0.0.1:8000/ml/health` - Registry-/Serving-Health
- `POST http://127.0.0.1:8000/ml/retrain` - Modell-Metadaten aktualisieren
- `POST http://127.0.0.1:8000/ml/evaluate` - MAE/RMSE/Coverage berechnen
Ohne vollständige HA-Konfiguration liefert `/v1/entities` bewusst `503`.

View File

@@ -1,10 +1,13 @@
from __future__ import annotations
from datetime import datetime
from typing import List
from fastapi import APIRouter, Depends
from fastapi import APIRouter, Depends, HTTPException, Query, status
from app.dependencies import get_ha_reader
from app.ha.discovery import DiscoveredEntity
from app.ha.history import EntityHistorySeries
from app.ha.models import HaEntitySummary
from app.ha.reader import HaReader
@@ -19,3 +22,43 @@ router = APIRouter(prefix="/v1", tags=["entities"])
)
def list_entities(ha_reader: HaReader = Depends(get_ha_reader)) -> List[HaEntitySummary]:
return list(ha_reader.read_entities())
@router.get(
"/discovery",
summary="Home-Assistant-Entities klassifizieren",
description="Klassifiziert Entities nach Lernrelevanz, Kontextquelle und Aktor-Rolle.",
response_model=List[DiscoveredEntity],
)
def discovery(
domain: List[str] | None = Query(default=None),
learnable: bool | None = None,
ha_reader: HaReader = Depends(get_ha_reader),
) -> List[DiscoveredEntity]:
return list(
ha_reader.discover(
domains=set(domain) if domain else None,
learnable=learnable,
)
)
@router.get(
"/history",
summary="Numerische Home-Assistant-Historie lesen",
description="Lädt und normalisiert numerische Zustände ausgewählter Entities.",
response_model=List[EntityHistorySeries],
)
def history(
entity_id: List[str] = Query(),
start_time: datetime = Query(),
end_time: datetime = Query(),
ha_reader: HaReader = Depends(get_ha_reader),
) -> List[EntityHistorySeries]:
try:
return list(ha_reader.read_history(entity_id, start_time, end_time))
except ValueError as exc:
raise HTTPException(
status_code=status.HTTP_422_UNPROCESSABLE_CONTENT,
detail=str(exc),
) from exc

View File

@@ -2,6 +2,9 @@ from __future__ import annotations
import logging
from dataclasses import dataclass
from datetime import datetime
import re
from urllib.parse import quote
import requests
@@ -14,6 +17,9 @@ from app.ha.exceptions import (
logger = logging.getLogger(__name__)
_ENTITY_ID_PATTERN = re.compile(r"^[a-z0-9_]+\.[a-z0-9_]+$")
_MAX_HISTORY_SECONDS = 31 * 24 * 60 * 60
@dataclass(frozen=True)
class HaClientSettings:
@@ -35,9 +41,58 @@ class HaClient:
self._session.close()
def list_entities(self) -> list[dict[str, object]]:
payload = self._get_json("/api/states")
if not isinstance(payload, list):
raise HaUnexpectedPayloadError(
"Antwort von Home Assistant hat unerwartetes Format."
)
return payload
def get_history(
self,
entity_ids: list[str],
start_time: datetime,
end_time: datetime,
) -> list[object]:
if not entity_ids:
raise ValueError("Mindestens eine entity_id ist erforderlich.")
if len(entity_ids) > 100:
raise ValueError("Es können höchstens 100 Entities abgefragt werden.")
if any(not _ENTITY_ID_PATTERN.fullmatch(entity_id) for entity_id in entity_ids):
raise ValueError("entity_id enthält ein ungültiges Format.")
if start_time.tzinfo is None or end_time.tzinfo is None:
raise ValueError("start_time und end_time müssen eine Zeitzone enthalten.")
if end_time <= start_time:
raise ValueError("end_time muss nach start_time liegen.")
if (end_time - start_time).total_seconds() > _MAX_HISTORY_SECONDS:
raise ValueError("History-Abfragen sind auf 31 Tage begrenzt.")
start = quote(start_time.isoformat(), safe=":+")
payload = self._get_json(
f"/api/history/period/{start}",
params={
"filter_entity_id": ",".join(entity_ids),
"end_time": end_time.isoformat(),
"minimal_response": "1",
"no_attributes": "1",
},
)
if not isinstance(payload, list):
raise HaUnexpectedPayloadError(
"History-Antwort von Home Assistant hat unerwartetes Format."
)
return payload
def _get_json(
self,
path: str,
*,
params: dict[str, str] | None = None,
) -> object:
try:
response = self._session.get(
f"{self._settings.url.rstrip('/')}/api/states",
f"{self._settings.url.rstrip('/')}{path}",
params=params,
timeout=self._settings.timeout_seconds,
)
except requests.Timeout as exc:
@@ -69,9 +124,4 @@ class HaClient:
"Antwort von Home Assistant ist kein gültiges JSON."
) from exc
if not isinstance(payload, list):
raise HaUnexpectedPayloadError(
"Antwort von Home Assistant hat unerwartetes Format."
)
return payload

184
app/ha/discovery.py Normal file
View File

@@ -0,0 +1,184 @@
from __future__ import annotations
from enum import StrEnum
from pydantic import BaseModel
from app.ha.models import HaEntitySummary
class EntityRole(StrEnum):
MEASUREMENT = "measurement"
BINARY_CONTEXT = "binary_context"
CONTEXT = "context"
ACTUATOR = "actuator"
UNSUPPORTED = "unsupported"
class DiscoveredEntity(BaseModel):
entity_id: str
domain: str
device_class: str | None = None
state_class: str | None = None
unit_of_measurement: str | None = None
role: EntityRole
learnable: bool
reason: str
_MEASUREMENT_CLASSES = frozenset({
"apparent_power",
"atmospheric_pressure",
"battery",
"carbon_dioxide",
"carbon_monoxide",
"current",
"distance",
"duration",
"energy",
"frequency",
"gas",
"humidity",
"illuminance",
"moisture",
"monetary",
"nitrogen_dioxide",
"nitrogen_monoxide",
"nitrous_oxide",
"ozone",
"pm1",
"pm10",
"pm25",
"power",
"precipitation",
"pressure",
"reactive_power",
"signal_strength",
"sound_pressure",
"speed",
"sulphur_dioxide",
"temperature",
"volatile_organic_compounds",
"voltage",
"volume",
"volume_flow_rate",
"water",
"weight",
"wind_speed",
})
_BINARY_CONTEXT_CLASSES = frozenset({
"door",
"garage_door",
"lock",
"motion",
"occupancy",
"opening",
"presence",
"problem",
"safety",
"smoke",
"sound",
"vibration",
"window",
})
_ACTUATOR_DOMAINS = frozenset({
"button",
"climate",
"cover",
"fan",
"humidifier",
"light",
"lock",
"scene",
"select",
"siren",
"switch",
"valve",
})
_CONTEXT_DOMAINS = frozenset({"device_tracker", "person", "sun", "weather", "zone"})
_LEARNABLE_CONTEXT_DOMAINS = frozenset({"device_tracker", "person", "weather"})
_NUMERIC_STATE_CLASSES = frozenset({"measurement", "total", "total_increasing"})
def classify_entity(entity: HaEntitySummary) -> DiscoveredEntity:
if entity.domain == "sensor" and (
entity.state_class in _NUMERIC_STATE_CLASSES
or entity.device_class in _MEASUREMENT_CLASSES
or entity.unit_of_measurement is not None
):
return _result(
entity,
EntityRole.MEASUREMENT,
learnable=True,
reason="Numerischer Messsensor für Zeitreihen und Training.",
)
if entity.domain == "binary_sensor" and entity.device_class in _BINARY_CONTEXT_CLASSES:
return _result(
entity,
EntityRole.BINARY_CONTEXT,
learnable=True,
reason="Binärer Kontextsensor für Zustands- und Anwesenheitsmuster.",
)
if entity.domain in _CONTEXT_DOMAINS:
learnable = entity.domain in _LEARNABLE_CONTEXT_DOMAINS
return _result(
entity,
EntityRole.CONTEXT,
learnable=learnable,
reason=(
"Kontextquelle für Training und Erklärungen."
if learnable
else "Kontextquelle ohne direkte Trainingsfreigabe."
),
)
if entity.domain in _ACTUATOR_DOMAINS:
return _result(
entity,
EntityRole.ACTUATOR,
learnable=False,
reason="Aktor ist ein mögliches Automationsziel, aber kein Trainingssensor.",
)
return _result(
entity,
EntityRole.UNSUPPORTED,
learnable=False,
reason="Entity-Typ ist noch nicht für Lernen oder Automationen klassifiziert.",
)
def discover_entities(
entities: list[HaEntitySummary],
domains: set[str] | None = None,
learnable: bool | None = None,
) -> list[DiscoveredEntity]:
normalized_domains = {domain.strip().lower() for domain in domains or set() if domain.strip()}
discovered = [classify_entity(entity) for entity in entities]
return [
entity
for entity in discovered
if (not normalized_domains or entity.domain in normalized_domains)
and (learnable is None or entity.learnable is learnable)
]
def _result(
entity: HaEntitySummary,
role: EntityRole,
*,
learnable: bool,
reason: str,
) -> DiscoveredEntity:
return DiscoveredEntity(
entity_id=entity.entity_id,
domain=entity.domain,
device_class=entity.device_class,
state_class=entity.state_class,
unit_of_measurement=entity.unit_of_measurement,
role=role,
learnable=learnable,
reason=reason,
)

91
app/ha/history.py Normal file
View File

@@ -0,0 +1,91 @@
from __future__ import annotations
import math
from datetime import datetime
from pydantic import BaseModel
from app.ha.exceptions import HaUnexpectedPayloadError
class NumericHistoryPoint(BaseModel):
timestamp: datetime
value: float
class EntityHistorySeries(BaseModel):
entity_id: str
points: list[NumericHistoryPoint]
def normalize_history_payload(payload: object) -> list[EntityHistorySeries]:
if not isinstance(payload, list):
raise HaUnexpectedPayloadError("History-Payload muss eine Liste sein.")
normalized: list[EntityHistorySeries] = []
for raw_series in payload:
if not isinstance(raw_series, list):
raise HaUnexpectedPayloadError("History-Serie muss eine Liste sein.")
series = _normalize_series(raw_series)
if series is not None:
normalized.append(series)
return sorted(normalized, key=lambda item: item.entity_id)
def _normalize_series(raw_series: list[object]) -> EntityHistorySeries | None:
entity_id: str | None = None
points: list[NumericHistoryPoint] = []
for raw_entry in raw_series:
if not isinstance(raw_entry, dict):
raise HaUnexpectedPayloadError("History-Eintrag muss ein Objekt sein.")
raw_entity_id = raw_entry.get("entity_id")
if raw_entity_id is not None:
if not isinstance(raw_entity_id, str) or "." not in raw_entity_id:
raise HaUnexpectedPayloadError("History-Eintrag enthält ungültige entity_id.")
if entity_id is not None and entity_id != raw_entity_id:
raise HaUnexpectedPayloadError("History-Serie enthält mehrere Entities.")
entity_id = raw_entity_id
raw_state = raw_entry.get("state")
value = _finite_float(raw_state)
if value is None:
continue
if entity_id is None:
raise HaUnexpectedPayloadError("History-Serie enthält keine entity_id.")
raw_timestamp = raw_entry.get("last_changed") or raw_entry.get("last_updated")
timestamp = _parse_timestamp(raw_timestamp)
points.append(NumericHistoryPoint(timestamp=timestamp, value=value))
if entity_id is None or not points:
return None
points.sort(key=lambda point: point.timestamp)
return EntityHistorySeries(entity_id=entity_id, points=points)
def _finite_float(value: object) -> float | None:
if isinstance(value, bool) or value is None:
return None
if not isinstance(value, (str, int, float)):
return None
try:
converted = float(value)
except (TypeError, ValueError):
return None
return converted if math.isfinite(converted) else None
def _parse_timestamp(value: object) -> datetime:
if not isinstance(value, str):
raise HaUnexpectedPayloadError("Numerischer History-Eintrag enthält keinen Zeitstempel.")
try:
parsed = datetime.fromisoformat(value.replace("Z", "+00:00"))
except ValueError as exc:
raise HaUnexpectedPayloadError("History-Eintrag enthält ungültigen Zeitstempel.") from exc
if parsed.tzinfo is None:
raise HaUnexpectedPayloadError("History-Zeitstempel muss eine Zeitzone enthalten.")
return parsed

View File

@@ -1,9 +1,12 @@
from __future__ import annotations
from collections.abc import Sequence
from datetime import datetime
from typing import Any
from app.ha.client import HaClient
from app.ha.discovery import DiscoveredEntity, discover_entities
from app.ha.history import EntityHistorySeries, normalize_history_payload
from app.ha.models import HaEntitySummary
@@ -33,6 +36,22 @@ class HaReader:
)
return summaries
def discover(
self,
domains: set[str] | None = None,
learnable: bool | None = None,
) -> Sequence[DiscoveredEntity]:
return discover_entities(list(self.read_entities()), domains=domains, learnable=learnable)
def read_history(
self,
entity_ids: list[str],
start_time: datetime,
end_time: datetime,
) -> Sequence[EntityHistorySeries]:
payload = self._client.get_history(entity_ids, start_time, end_time)
return normalize_history_payload(payload)
def _optional_str(value: object) -> str | None:
if value is None or value == "":

View File

@@ -38,7 +38,7 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]:
app = FastAPI(
title="SillyHome Next API",
description="Lokales Smart-Home-Intelligenzsystem für Home Assistant.",
version="0.1.0",
version="0.2.0",
lifespan=lifespan,
)
app.state.settings = load_settings()

View File

@@ -1,5 +1,20 @@
"""Machine-Learning-Grundbausteine für SillyHome Next."""
__all__ = ["FeatureStore", "FeatureVector", "TrainedArtifact", "TrainingPipeline"]
__all__ = [
"FeatureStore",
"FeatureVector",
"FeatureModel",
"FeatureExplanation",
"PredictionResult",
"Predictor",
"RetrainingResult",
"RetrainingService",
"TrainedArtifact",
"TrainingPipeline",
"retrain_model",
]
from app.ml.feature_store import FeatureStore, FeatureVector
from app.ml.training import TrainedArtifact, TrainingPipeline
from app.ml.explanation import FeatureExplanation
from app.ml.predictor import PredictionResult, Predictor
from app.ml.retraining import RetrainingResult, RetrainingService, retrain_model
from app.ml.training import FeatureModel, TrainedArtifact, TrainingPipeline

View File

@@ -1,9 +1,13 @@
from __future__ import annotations
import logging
import math
from collections.abc import Sequence
from dataclasses import dataclass
from app.ml.feature_store import FeatureVector
from app.ml.predictor import Predictor
from app.ml.registry.model_registry import ModelRegistry
from app.ml.training import TrainingPipeline
logger = logging.getLogger(__name__)
@@ -24,41 +28,62 @@ class EvalReport:
class Evaluator:
def __init__(self, pipeline: TrainingPipeline) -> None:
def __init__(
self,
pipeline: TrainingPipeline | None = None,
registry: ModelRegistry | None = None,
) -> None:
if isinstance(pipeline, ModelRegistry) and registry is None:
registry = pipeline
pipeline = None
if pipeline is None and registry is None:
raise ValueError("Evaluator erfordert TrainingPipeline oder ModelRegistry.")
self._pipeline = pipeline
self._registry = registry
self._predictor = Predictor(pipeline=pipeline, registry=registry)
def evaluate(self, artifact_id: str, predictions: Sequence[str]) -> EvalReport:
def evaluate(self, artifact_id: str, samples: Sequence[FeatureVector]) -> EvalReport:
try:
supported_sensors = set(self._pipeline.export(artifact_id).supported_sensors)
if self._registry is not None:
self._registry.load_artifact(artifact_id)
elif self._pipeline is not None:
self._pipeline.export(artifact_id)
except KeyError as exc:
raise ValueError("Kein trainiertes Modell für Evaluation vorhanden.") from exc
parsed_sensors = [_prediction_sensor(prediction) for prediction in predictions]
supported_hits = sum(sensor in supported_sensors for sensor in parsed_sensors)
unknown_hits = sum(sensor not in supported_sensors for sensor in parsed_sensors)
sample_size = len(predictions)
coverage = supported_hits / sample_size if sample_size else 0.0
unknown_rate = unknown_hits / sample_size if sample_size else 0.0
absolute_errors: list[float] = []
squared_errors: list[float] = []
for sample in samples:
try:
prediction = self._predictor.predict(artifact_id, sample)
except ValueError:
continue
for feature_name, predicted in prediction.predictions.items():
actual = float(sample.values[feature_name])
error = predicted - actual
absolute_errors.append(abs(error))
squared_errors.append(error**2)
coverage_metric = Metric(name="coverage", value=coverage, threshold=0.8)
unknown_metric = Metric(name="unknown_rate", value=unknown_rate, threshold=0.1)
sample_size = len(absolute_errors)
mae = sum(absolute_errors) / sample_size if sample_size else 0.0
rmse = math.sqrt(sum(squared_errors) / sample_size) if sample_size else 0.0
expected_values = sum(len(sample.values) for sample in samples)
coverage = sample_size / expected_values if expected_values else 0.0
report = EvalReport(
artifact_id=artifact_id,
sample_size=sample_size,
metrics=[coverage_metric, unknown_metric],
metrics=[
Metric(name="mae", value=mae),
Metric(name="rmse", value=rmse),
Metric(name="coverage", value=coverage, threshold=0.8),
],
)
logger.info(
"Evaluation %s -> coverage=%.2f, unknown_rate=%.2f",
"Evaluation %s -> mae=%.4f, rmse=%.4f, coverage=%.2f",
artifact_id,
mae,
rmse,
coverage,
unknown_rate,
)
return report
def _prediction_sensor(prediction: str) -> str | None:
parts = prediction.split(":", 2)
if len(parts) != 3 or not parts[0] or not parts[1]:
return None
return parts[1]

57
app/ml/explanation.py Normal file
View File

@@ -0,0 +1,57 @@
from __future__ import annotations
from dataclasses import dataclass
from app.ml.training import FeatureModel
@dataclass(frozen=True)
class FeatureExplanation:
feature: str
current_value: float
predicted_value: float
change: float
direction: str
sample_count: int
historical_mean: float
historical_range: tuple[float, float]
standard_deviation: float
trend_per_step: float
confidence: float
summary: str
def explain_feature(
feature_name: str,
current_value: float,
predicted_value: float,
model: FeatureModel,
) -> FeatureExplanation:
change = predicted_value - current_value
direction = _direction(change)
summary = (
f"{feature_name}: {direction}; Prognose {predicted_value:.3f} "
f"aus aktuellem Wert {current_value:.3f} und Trend {model.slope:+.3f}. "
f"Basis: {model.sample_count} Messwerte, Mittelwert {model.mean:.3f}, "
f"Confidence {model.confidence:.0%}."
)
return FeatureExplanation(
feature=feature_name,
current_value=current_value,
predicted_value=predicted_value,
change=change,
direction=direction,
sample_count=model.sample_count,
historical_mean=model.mean,
historical_range=(model.minimum, model.maximum),
standard_deviation=model.standard_deviation,
trend_per_step=model.slope,
confidence=model.confidence,
summary=summary,
)
def _direction(change: float) -> str:
if abs(change) < 1e-12:
return "stabil"
return "steigend" if change > 0 else "fallend"

View File

@@ -1,8 +1,11 @@
from __future__ import annotations
import logging
import math
from dataclasses import dataclass
from typing import Sequence
from app.ml.explanation import FeatureExplanation, explain_feature
from app.ml.feature_store import FeatureVector
from app.ml.registry.model_registry import ModelRegistry
from app.ml.training import TrainedArtifact, TrainingPipeline
@@ -10,6 +13,16 @@ from app.ml.training import TrainedArtifact, TrainingPipeline
logger = logging.getLogger(__name__)
@dataclass(frozen=True)
class PredictionResult:
artifact_id: str
sensor_id: str
predictions: dict[str, float]
confidence: float
model_type: str
explanations: dict[str, FeatureExplanation]
class Predictor:
def __init__(
self,
@@ -24,15 +37,54 @@ class Predictor:
self._pipeline = pipeline
self._registry = registry
def predict(self, artifact_id: str, entity: FeatureVector) -> str:
def predict(self, artifact_id: str, entity: FeatureVector) -> PredictionResult:
artifact = self._get_artifact(artifact_id)
if entity.sensor_id not in artifact.supported_sensors:
raise ValueError(
f"Sensor '{entity.sensor_id}' wird vom Modell '{artifact_id}' nicht unterstützt."
)
return f"{artifact_id}:{entity.sensor_id}:{entity.values}"
sensor_models = artifact.feature_models.get(entity.sensor_id, {})
if not sensor_models:
raise ValueError(f"Modell '{artifact_id}' enthält keine statistischen Parameter.")
def predict_batch(self, artifact_id: str, entities: Sequence[FeatureVector]) -> list[str]:
feature_names = sorted(set(sensor_models).intersection(entity.values))
if not feature_names:
raise ValueError(
f"Keine Eingabemerkmale werden vom Modell '{artifact_id}' unterstützt."
)
predictions: dict[str, float] = {}
explanations: dict[str, FeatureExplanation] = {}
confidences: list[float] = []
for feature_name in feature_names:
model = sensor_models[feature_name]
current_value = float(entity.values[feature_name])
if not math.isfinite(current_value):
raise ValueError("Vorhersagewerte müssen endlich sein.")
predicted_value = model.forecast(current_value)
predictions[feature_name] = predicted_value
explanations[feature_name] = explain_feature(
feature_name,
current_value,
predicted_value,
model,
)
confidences.append(model.confidence)
return PredictionResult(
artifact_id=artifact_id,
sensor_id=entity.sensor_id,
predictions=predictions,
confidence=sum(confidences) / len(confidences),
model_type=artifact.model_type,
explanations=explanations,
)
def predict_batch(
self,
artifact_id: str,
entities: Sequence[FeatureVector],
) -> list[PredictionResult]:
return [self.predict(artifact_id, entity) for entity in entities]
@staticmethod
@@ -47,4 +99,4 @@ class Predictor:
return self._registry.load_artifact(artifact_id)
if self._pipeline is not None:
return self._pipeline.export(artifact_id)
raise RuntimeError("Predictor nicht initialisiert.")
raise RuntimeError("Predictor nicht initialisiert.")

View File

@@ -2,12 +2,14 @@ from __future__ import annotations
import json
import logging
import math
import os
from pathlib import Path
import re
from threading import RLock
from collections.abc import Iterable
from app.ml.training import TrainedArtifact
from app.ml.training import FeatureModel, TrainedArtifact
logger = logging.getLogger(__name__)
@@ -19,22 +21,31 @@ class ModelRegistry:
self._root = Path(root).resolve()
self._root.mkdir(parents=True, exist_ok=True)
self._artifacts: dict[str, TrainedArtifact] = {}
self._lock = RLock()
self._load_existing()
def register(self, artifact: TrainedArtifact) -> TrainedArtifact:
registered, _ = self.register_with_status(artifact)
return registered
def register_with_status(self, artifact: TrainedArtifact) -> tuple[TrainedArtifact, bool]:
self._validate_artifact_id(artifact.artifact_id)
self._persist(artifact)
self._artifacts[artifact.artifact_id] = artifact
return artifact
with self._lock:
replaced = artifact.artifact_id in self._artifacts
self._persist(artifact)
self._artifacts[artifact.artifact_id] = artifact
return artifact, replaced
def load_artifact(self, artifact_id: str) -> TrainedArtifact:
self._validate_artifact_id(artifact_id)
if artifact_id not in self._artifacts:
raise KeyError(f"Artifact '{artifact_id}' nicht registriert.")
return self._artifacts[artifact_id]
with self._lock:
if artifact_id not in self._artifacts:
raise KeyError(f"Artifact '{artifact_id}' nicht registriert.")
return self._artifacts[artifact_id]
def list_models(self) -> Iterable[TrainedArtifact]:
return list(self._artifacts.values())
with self._lock:
return [self._artifacts[key] for key in sorted(self._artifacts)]
def _load_existing(self) -> None:
for source in sorted(self._root.glob("*.json")):
@@ -42,19 +53,26 @@ class ModelRegistry:
raw = json.loads(source.read_text(encoding="utf-8"))
artifact_id = raw["artifact_id"]
supported_sensors = raw["supported_sensors"]
model_type = raw.get("model_type", "metadata")
raw_feature_models = raw.get("feature_models", {})
if not isinstance(artifact_id, str) or not isinstance(supported_sensors, list):
raise ValueError("invalid artifact structure")
if not isinstance(model_type, str):
raise ValueError("model_type must be a string")
self._validate_artifact_id(artifact_id)
if source.name != f"{artifact_id}.json":
raise ValueError("artifact id does not match filename")
if not all(isinstance(sensor, str) for sensor in supported_sensors):
raise ValueError("supported_sensors must contain strings")
feature_models = _deserialize_feature_models(raw_feature_models)
except (KeyError, TypeError, ValueError, json.JSONDecodeError) as exc:
raise ValueError(f"Ungültiges Modell-Artefakt: {source.name}") from exc
self._artifacts[artifact_id] = TrainedArtifact(
artifact_id=artifact_id,
supported_sensors=tuple(supported_sensors),
feature_models=feature_models,
model_type=model_type,
)
def _persist(self, artifact: TrainedArtifact) -> None:
@@ -63,6 +81,22 @@ class ModelRegistry:
payload = {
"artifact_id": artifact.artifact_id,
"supported_sensors": list(artifact.supported_sensors),
"model_type": artifact.model_type,
"feature_models": {
sensor_id: {
feature_name: {
"sample_count": model.sample_count,
"mean": model.mean,
"standard_deviation": model.standard_deviation,
"minimum": model.minimum,
"maximum": model.maximum,
"slope": model.slope,
"intercept": model.intercept,
}
for feature_name, model in sorted(models.items())
}
for sensor_id, models in sorted(artifact.feature_models.items())
},
}
temporary.write_text(
json.dumps(payload, ensure_ascii=True, sort_keys=True) + "\n",
@@ -78,3 +112,51 @@ class ModelRegistry:
"artifact_id darf nur Buchstaben, Ziffern, Punkt, Unterstrich "
"und Bindestrich enthalten."
)
def _deserialize_feature_models(raw: object) -> dict[str, dict[str, FeatureModel]]:
if not isinstance(raw, dict):
raise ValueError("feature_models must be an object")
result: dict[str, dict[str, FeatureModel]] = {}
for sensor_id, raw_features in raw.items():
if not isinstance(sensor_id, str) or not isinstance(raw_features, dict):
raise ValueError("invalid sensor feature models")
features: dict[str, FeatureModel] = {}
for feature_name, raw_model in raw_features.items():
if not isinstance(feature_name, str) or not isinstance(raw_model, dict):
raise ValueError("invalid feature model")
sample_count = raw_model.get("sample_count")
if not isinstance(sample_count, int) or isinstance(sample_count, bool) or sample_count < 1:
raise ValueError("sample_count must be a positive integer")
values = {
key: _finite_number(raw_model.get(key))
for key in (
"mean",
"standard_deviation",
"minimum",
"maximum",
"slope",
"intercept",
)
}
features[feature_name] = FeatureModel(
sample_count=sample_count,
mean=values["mean"],
standard_deviation=values["standard_deviation"],
minimum=values["minimum"],
maximum=values["maximum"],
slope=values["slope"],
intercept=values["intercept"],
)
result[sensor_id] = features
return result
def _finite_number(value: object) -> float:
if not isinstance(value, (int, float)) or isinstance(value, bool):
raise ValueError("feature model values must be finite numbers")
converted = float(value)
if not math.isfinite(converted):
raise ValueError("feature model values must be finite numbers")
return converted

43
app/ml/retraining.py Normal file
View File

@@ -0,0 +1,43 @@
from __future__ import annotations
from collections.abc import Iterable
from dataclasses import dataclass
from app.ml.feature_store import FeatureStore, FeatureVector
from app.ml.registry.model_registry import ModelRegistry
from app.ml.training import TrainedArtifact, TrainingPipeline
@dataclass(frozen=True)
class RetrainingResult:
artifact: TrainedArtifact
replaced: bool
class RetrainingService:
"""Runs one retraining cycle without owning scheduling or background threads."""
def __init__(self, registry: ModelRegistry) -> None:
self._registry = registry
def retrain(
self,
artifact_id: str,
vectors: Iterable[FeatureVector],
) -> RetrainingResult:
store = FeatureStore()
store.add_batch(vectors)
pipeline = TrainingPipeline(store)
artifact = pipeline.run(artifact_id)
_, replaced = self._registry.register_with_status(artifact)
return RetrainingResult(artifact=artifact, replaced=replaced)
def retrain_model(
registry: ModelRegistry,
artifact_id: str,
vectors: Iterable[FeatureVector],
) -> RetrainingResult:
"""Scheduler-compatible entry point for exactly one retraining run."""
return RetrainingService(registry).retrain(artifact_id, vectors)

View File

@@ -1,17 +1,44 @@
from __future__ import annotations
import logging
from dataclasses import dataclass
import math
from collections import defaultdict
from dataclasses import dataclass, field
from app.ml.feature_store import FeatureStore
logger = logging.getLogger(__name__)
@dataclass
@dataclass(frozen=True)
class FeatureModel:
sample_count: int
mean: float
standard_deviation: float
minimum: float
maximum: float
slope: float
intercept: float
def forecast(self, current_value: float | None = None) -> float:
if current_value is not None:
return current_value + self.slope
return self.intercept + self.slope * self.sample_count
@property
def confidence(self) -> float:
sample_score = self.sample_count / (self.sample_count + 2)
scale = abs(self.mean) if abs(self.mean) > 1e-9 else 1.0
stability_score = 1.0 / (1.0 + self.standard_deviation / scale)
return min(0.99, max(0.05, sample_score * stability_score))
@dataclass(frozen=True)
class TrainedArtifact:
artifact_id: str
supported_sensors: tuple[str, ...]
feature_models: dict[str, dict[str, FeatureModel]] = field(default_factory=dict)
model_type: str = "statistical_baseline"
class TrainingPipeline:
@@ -24,8 +51,33 @@ class TrainingPipeline:
if not vectors:
raise ValueError("FeatureStore enthält keine Trainingsdaten.")
sensors = tuple(sorted({vector.sensor_id for vector in vectors}))
artifact = TrainedArtifact(artifact_id=artifact_id, supported_sensors=sensors)
samples: dict[str, dict[str, list[float]]] = defaultdict(lambda: defaultdict(list))
for vector in vectors:
for feature_name, raw_value in vector.values.items():
value = float(raw_value)
if math.isfinite(value):
samples[vector.sensor_id][feature_name].append(value)
feature_models = {
sensor_id: {
feature_name: _fit_feature(values)
for feature_name, values in sorted(features.items())
if values
}
for sensor_id, features in sorted(samples.items())
}
feature_models = {
sensor_id: models for sensor_id, models in feature_models.items() if models
}
if not feature_models:
raise ValueError("Trainingsdaten enthalten keine endlichen numerischen Werte.")
sensors = tuple(feature_models)
artifact = TrainedArtifact(
artifact_id=artifact_id,
supported_sensors=sensors,
feature_models=feature_models,
)
self._artifacts[artifact_id] = artifact
logger.info("Training abgeschlossen für %s mit %d Sensoren", artifact_id, len(sensors))
return artifact
@@ -34,3 +86,32 @@ class TrainingPipeline:
if artifact_id not in self._artifacts:
raise KeyError(f"Artifact '{artifact_id}' nicht gefunden.")
return self._artifacts[artifact_id]
def _fit_feature(values: list[float]) -> FeatureModel:
sample_count = len(values)
mean = sum(values) / sample_count
variance = sum((value - mean) ** 2 for value in values) / sample_count
standard_deviation = math.sqrt(variance)
if sample_count == 1:
slope = 0.0
intercept = mean
else:
x_mean = (sample_count - 1) / 2
denominator = sum((index - x_mean) ** 2 for index in range(sample_count))
numerator = sum(
(index - x_mean) * (value - mean) for index, value in enumerate(values)
)
slope = numerator / denominator
intercept = mean - slope * x_mean
return FeatureModel(
sample_count=sample_count,
mean=mean,
standard_deviation=standard_deviation,
minimum=min(values),
maximum=max(values),
slope=slope,
intercept=intercept,
)

View File

@@ -7,9 +7,11 @@ from collections.abc import Sequence
from fastapi import APIRouter, FastAPI, HTTPException, Request, status
from pydantic import BaseModel, Field
from app.ml.evaluation import Evaluator
from app.ml.feature_store import FeatureVector
from app.ml.predictor import Predictor
from app.ml.registry.model_registry import ModelRegistry
from app.ml.retraining import retrain_model
logger = logging.getLogger(__name__)
@@ -30,7 +32,25 @@ class PredictRequest(BaseModel):
class PredictResponse(BaseModel):
model_id: str
sensor_id: str
prediction: str
predictions: dict[str, float]
confidence: float
model_type: str
explanations: dict[str, "FeatureExplanationResponse"]
class FeatureExplanationResponse(BaseModel):
feature: str
current_value: float
predicted_value: float
change: float
direction: str
sample_count: int
historical_mean: float
historical_range: tuple[float, float]
standard_deviation: float
trend_per_step: float
confidence: float
summary: str
class BatchRequest(BaseModel):
@@ -45,6 +65,42 @@ class ModelsResponse(BaseModel):
models: list[str]
class TrainingSample(BaseModel):
sensor_id: str = Field(min_length=1)
values: dict[str, float]
label: str | None = None
class RetrainRequest(BaseModel):
model_id: str = Field(..., alias="modelId", min_length=1, max_length=128)
samples: list[TrainingSample] = Field(min_length=1)
class RetrainResponse(BaseModel):
model_id: str
supported_sensors: list[str]
trained_features: int
model_type: str
replaced: bool
class EvaluateRequest(BaseModel):
model_id: str = Field(..., alias="modelId", min_length=1, max_length=128)
samples: list[TrainingSample] = Field(min_length=1)
class MetricResponse(BaseModel):
name: str
value: float
threshold: float | None = None
class EvaluateResponse(BaseModel):
model_id: str
sample_size: int
metrics: list[MetricResponse]
@router.get("/health", response_model=HealthResponse, status_code=200)
def health() -> HealthResponse:
return HealthResponse(status="ok")
@@ -57,6 +113,71 @@ def list_models(request: Request) -> ModelsResponse:
return ModelsResponse(models=models)
@router.post("/retrain", response_model=RetrainResponse, status_code=200)
def retrain(payload: RetrainRequest, request: Request) -> RetrainResponse:
registry = _require_registry(request)
vectors = [
FeatureVector(
sensor_id=sample.sensor_id,
values=sample.values,
label=sample.label,
)
for sample in payload.samples
]
try:
result = retrain_model(registry, payload.model_id, vectors)
except ValueError as exc:
raise HTTPException(
status_code=status.HTTP_422_UNPROCESSABLE_CONTENT,
detail=str(exc),
) from exc
return RetrainResponse(
model_id=result.artifact.artifact_id,
supported_sensors=list(result.artifact.supported_sensors),
trained_features=sum(
len(feature_models)
for feature_models in result.artifact.feature_models.values()
),
model_type=result.artifact.model_type,
replaced=result.replaced,
)
@router.post("/evaluate", response_model=EvaluateResponse, status_code=200)
def evaluate(payload: EvaluateRequest, request: Request) -> EvaluateResponse:
registry = _require_registry(request)
vectors = [
FeatureVector(
sensor_id=sample.sensor_id,
values=sample.values,
label=sample.label,
)
for sample in payload.samples
]
try:
report = Evaluator(registry=registry).evaluate(payload.model_id, vectors)
except ValueError as exc:
try:
registry.load_artifact(payload.model_id)
except KeyError:
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail=str(exc),
) from exc
raise HTTPException(
status_code=status.HTTP_422_UNPROCESSABLE_CONTENT,
detail=str(exc),
) from exc
return EvaluateResponse(
model_id=report.artifact_id,
sample_size=report.sample_size,
metrics=[
MetricResponse(name=metric.name, value=metric.value, threshold=metric.threshold)
for metric in report.metrics
],
)
@router.post("/predict", response_model=PredictResponse, status_code=200)
def predict(payload: PredictRequest, request: Request) -> PredictResponse:
registry = _require_registry(request)
@@ -74,7 +195,13 @@ def predict(payload: PredictRequest, request: Request) -> PredictResponse:
return PredictResponse(
model_id=payload.model_id,
sensor_id=payload.sensor_id,
prediction=prediction,
predictions=prediction.predictions,
confidence=prediction.confidence,
model_type=prediction.model_type,
explanations={
name: FeatureExplanationResponse(**explanation.__dict__)
for name, explanation in prediction.explanations.items()
},
)
@@ -95,7 +222,17 @@ def predict_batch(payload: BatchRequest, request: Request) -> BatchResponse:
detail=str(exc),
) from exc
responses.append(
PredictResponse(model_id=item.model_id, sensor_id=item.sensor_id, prediction=prediction)
PredictResponse(
model_id=item.model_id,
sensor_id=item.sensor_id,
predictions=prediction.predictions,
confidence=prediction.confidence,
model_type=prediction.model_type,
explanations={
name: FeatureExplanationResponse(**explanation.__dict__)
for name, explanation in prediction.explanations.items()
},
)
)
return BatchResponse(predictions=responses)

42
docs/ha_data.md Normal file
View File

@@ -0,0 +1,42 @@
# Home-Assistant-Datenpipeline
SillyHome Next trennt aktuelle Entity-Metadaten, Discovery und historische
Messwerte. Dadurch gelangen nur klassifizierte, geeignete Daten in spätere
Trainings- und Erklärungsprozesse.
## Entity Discovery
`GET /v1/discovery` klassifiziert Home-Assistant-Entities in:
- `measurement`: numerische Messsensoren, für Training geeignet
- `binary_context`: binäre Kontextsensoren wie Bewegung oder Anwesenheit
- `context`: Personen-, Wetter- und Standortkontext
- `actuator`: mögliche Automationsziele, nicht als Trainingssensor verwendet
- `unsupported`: noch nicht klassifizierte Entity-Typen
Optionale Query-Parameter:
- `domain=sensor` kann mehrfach angegeben werden
- `learnable=true|false` filtert nach Trainingsrelevanz
## Historische Daten
Historische Zustände werden über Home Assistants
`/api/history/period/<start>`-Schnittstelle geladen. Abfragen verlangen:
- mindestens eine Entity-ID, maximal 100
- zeitzonenbehaftete Start- und Endzeit
- ein Enddatum nach dem Startdatum
- maximal 31 Tage pro Abfrage
Die Normalisierung übernimmt nur endliche numerische Zustände. `unknown`,
`unavailable`, nichtnumerische Werte, `NaN` und unendliche Werte werden nicht
als Trainingsdaten verwendet. Ergebnisse werden je Entity chronologisch
sortiert.
## Datenschutz und Betrieb
Die Daten bleiben lokal. Home-Assistant-Tokens gehören ausschließlich in die
Umgebungskonfiguration und dürfen nicht protokolliert oder versioniert werden.
Die API sollte nur lokal oder hinter einem authentifizierenden Reverse Proxy
erreichbar sein.

View File

@@ -3,15 +3,15 @@
Diese Dokumentation beschreibt die REST-Endpunkte der aktuellen
Modell-Artefakt- und Vorhersage-Schnittstelle.
> Hinweis: Version 0.1.0 enthält noch kein statistisch trainiertes ML-Modell.
> Die Vorhersage ist eine deterministische Referenzimplementierung für den
> späteren Modellvertrag.
Das Serving verwendet ein lokal trainiertes statistisches Baseline-Modell.
## Basis-URL
- Standard: `http://127.0.0.1:8000/ml`
- Health: `/health`
- Modelle: `/models`
- Retraining: `/retrain`
- Evaluation: `/evaluate`
- Einzelvorhersage: `/predict`
- Batchvorhersage: `/batch`
@@ -61,10 +61,63 @@ Einzelne Vorhersage für einen Sensor.
{
"model_id": "default",
"sensor_id": "sensor.kitchen",
"prediction": "default:sensor.kitchen:{'temperature': 21.0}"
"predictions": {"temperature": 21.4},
"confidence": 0.78,
"model_type": "statistical_baseline",
"explanations": {
"temperature": {
"direction": "steigend",
"change": 0.4,
"sample_count": 24,
"historical_mean": 20.7,
"trend_per_step": 0.4,
"summary": "temperature: steigend; Prognose ..."
}
}
}
```
Die Erklärung nennt pro Merkmal den aktuellen und prognostizierten Wert,
Richtung, Veränderung, Datenbasis, historischen Bereich, Streuung, Trend und
Confidence. Sie wird deterministisch aus den gespeicherten Modellparametern
erzeugt.
### `POST /ml/retrain`
Trainiert die Artefakt-Metadaten aus neuen Sensordaten. Existiert `modelId`
bereits, wird das Artefakt atomisch ersetzt und beim nächsten Prozessstart aus
dem Modellverzeichnis geladen.
**Request**
```json
{
"modelId": "home-model",
"samples": [
{
"sensor_id": "sensor.kitchen",
"values": {"temperature": 21.0},
"label": "occupied"
}
]
}
```
**Antwort**
```json
{
"model_id": "home-model",
"supported_sensors": ["sensor.kitchen"],
"trained_features": 1,
"model_type": "statistical_baseline",
"replaced": false
}
```
### `POST /ml/evaluate`
Vergleicht Modellvorhersagen mit Validierungsdaten und liefert MAE, RMSE und
Coverage. Der Request verwendet dasselbe Sample-Format wie `/ml/retrain`.
### `POST /ml/batch`
Batch-Vorhersage für mehrere Sensorwerte.
@@ -94,12 +147,16 @@ Batch-Vorhersage für mehrere Sensorwerte.
{
"model_id": "default",
"sensor_id": "sensor.kitchen",
"prediction": "default:sensor.kitchen:{'temperature': 21.0}"
"predictions": {"temperature": 21.4},
"confidence": 0.78,
"model_type": "statistical_baseline"
},
{
"model_id": "default",
"sensor_id": "sensor.bedroom",
"prediction": "default:sensor.bedroom:{'temperature': 18.5}"
"predictions": {"temperature": 18.3},
"confidence": 0.74,
"model_type": "statistical_baseline"
}
]
}
@@ -114,11 +171,15 @@ Batch-Vorhersage für mehrere Sensorwerte.
## Betrieb
Die produktive App lädt Artefakte aus `SILLYHOME_MODEL_STORE`. Neue Artefakte
werden derzeit intern über `ModelRegistry.register(...)` registriert. Die
Registry speichert validiertes JSON atomisch und lädt es beim Neustart.
werden über `/ml/retrain`, `RetrainingService` oder direkt über
`ModelRegistry.register(...)` registriert. Die Registry speichert validiertes
JSON atomisch und lädt es beim Neustart. Die API sollte nur in einem
vertrauenswürdigen Netz oder hinter einem authentifizierenden Reverse Proxy
erreichbar sein.
## Verweise
- `app/ml/predictor.py`
- `app/ml/retraining.py`
- `app/ml/registry/model_registry.py`
- `backend/routes/ml.py`

View File

@@ -1,14 +1,13 @@
# ML Training- und Evaluations-Workflow
Dieser Workflow beschreibt den aktuellen Platzhalter für Modell-Metadaten,
Evaluation und Serving. Er trainiert in Version 0.1.0 noch kein statistisches
Modell.
SillyHome Next trainiert ein lokales statistisches Baseline-Modell pro Sensor
und Merkmal. Es benötigt keine Cloud und keine externe ML-Laufzeit.
## 1. Daten sammeln
Alle Trainingsvektoren werden über `FeatureStore.add(...)` oder `add_batch(...)` eingepflegt. Jeder Vektor enthält eine Sensor-ID sowie ein Dictionary mit Merkmalen.
## 2. Artefakt-Metadaten erzeugen
## 2. Statistisches Artefakt erzeugen
```python
store = FeatureStore()
@@ -18,27 +17,52 @@ artifact = pipeline.run("my_artifact")
pipeline.export("my_artifact")
```
`TrainingPipeline.run(...)` erzeugt ein `TrainedArtifact` mit den unterstützten
Sensor-IDs. Gewichte, Parameter oder ein echtes Modell werden noch nicht
berechnet.
`TrainingPipeline.run(...)` berechnet für jedes numerische Merkmal:
- Stichprobenzahl
- Mittelwert und Standardabweichung
- Minimum und Maximum
- linearen Trend mit Steigung und Achsenabschnitt
Die nächste Vorhersage kombiniert den letzten beobachteten Wert mit der
trainierten Trendsteigung. Die Confidence berücksichtigt Datenmenge und
Stabilität.
## 3. Modell evaluieren
```python
evaluator = Evaluator(pipeline)
report = evaluator.evaluate(artifact.artifact_id, predictions)
report = evaluator.evaluate(artifact.artifact_id, validation_samples)
```
Der Report enthält:
Der Report enthält echte numerische Vergleichsmetriken:
- `artifact_id`
- `sample_size`
- Metriken wie `coverage` und `unknown_rate` mit Default-Schwellenwerten
- `mae` (Mean Absolute Error)
- `rmse` (Root Mean Squared Error)
- `coverage` für den Anteil auswertbarer Merkmale
## 4. Modell registrieren
Das trainierte Artefakt kann anschließend über `ModelRegistry.register(artifact)` bereitgestellt werden. Die ML-Serving-API stellt es unter `/ml/predict` und `/ml/batch` zur Verfügung.
## 5. Retraining ausführen
`RetrainingService.retrain(...)` führt genau einen Trainingslauf aus und ersetzt
ein vorhandenes Artefakt mit derselben ID atomisch in der Registry:
```python
service = RetrainingService(registry)
result = service.retrain("home-model", vectors)
```
Scheduler, Cronjobs oder Home-Assistant-Automationen können alternativ die
zustandslose Funktion `retrain_model(registry, artifact_id, vectors)` aufrufen.
Der Service startet bewusst keinen eigenen Hintergrundprozess. Über
`POST /ml/retrain` kann derselbe Ablauf per API angestoßen werden.
## Hinweise
- Für reproduzierbare Sensor-Reihenfolgen wird in `TrainingPipeline.run(...)` eine sortierte Sensor-Liste verwendet.
- Fehlende Trainingsdaten lösen `ValueError` aus; nicht registrierte Artefakte lösen `KeyError` aus.
- `coverage` zählt nur exakte Sensor-Referenzen und bleibt im Bereich 0 bis 1.
- Nur endliche numerische Werte werden trainiert.
- `coverage` bleibt im Bereich 0 bis 1.

View File

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
[project]
name = "sillyhome-next"
version = "0.1.0"
version = "0.2.0"
description = "Lokales Smart-Home-Intelligenzsystem für Home Assistant"
requires-python = ">=3.11"
dependencies = [

View File

@@ -1,8 +1,11 @@
from collections.abc import Sequence
from datetime import datetime
from fastapi.testclient import TestClient
from app.ha.exceptions import HaTimeoutError
from app.ha.discovery import DiscoveredEntity, EntityRole
from app.ha.history import EntityHistorySeries, NumericHistoryPoint
from app.ha.models import HaEntitySummary
from app.ha.reader import HaReader
from app.main import app
@@ -15,6 +18,38 @@ class FakeHaReader(HaReader):
def read_entities(self) -> Sequence[HaEntitySummary]:
return [HaEntitySummary(entity_id="sensor.temperature", domain="sensor")]
def discover(
self,
domains: set[str] | None = None,
learnable: bool | None = None,
) -> Sequence[DiscoveredEntity]:
result = DiscoveredEntity(
entity_id="sensor.temperature",
domain="sensor",
device_class="temperature",
role=EntityRole.MEASUREMENT,
learnable=True,
reason="Numerischer Messsensor für Zeitreihen und Training.",
)
if domains and result.domain not in domains:
return []
if learnable is not None and result.learnable is not learnable:
return []
return [result]
def read_history(
self,
entity_ids: list[str],
start_time: datetime,
end_time: datetime,
) -> Sequence[EntityHistorySeries]:
return [
EntityHistorySeries(
entity_id=entity_ids[0],
points=[NumericHistoryPoint(timestamp=start_time, value=21.5)],
)
]
class TimeoutHaReader(HaReader):
def __init__(self) -> None:
@@ -59,3 +94,44 @@ def test_entities_maps_ha_errors_without_leaking_details() -> None:
response = client.get("/v1/entities")
assert response.status_code == 504
assert response.json() == {"detail": "Home Assistant request timed out."}
def test_discovery_filters_entities() -> None:
with TestClient(app) as client:
app.state.ha_reader = FakeHaReader()
response = client.get("/v1/discovery?domain=sensor&learnable=true")
assert response.status_code == 200
assert response.json() == [
{
"entity_id": "sensor.temperature",
"domain": "sensor",
"device_class": "temperature",
"state_class": None,
"unit_of_measurement": None,
"role": "measurement",
"learnable": True,
"reason": "Numerischer Messsensor für Zeitreihen und Training.",
}
]
def test_history_returns_normalized_series() -> None:
with TestClient(app) as client:
app.state.ha_reader = FakeHaReader()
response = client.get(
"/v1/history",
params=[
("entity_id", "sensor.temperature"),
("start_time", "2026-06-01T00:00:00Z"),
("end_time", "2026-06-02T00:00:00Z"),
],
)
assert response.status_code == 200
assert response.json() == [
{
"entity_id": "sensor.temperature",
"points": [{"timestamp": "2026-06-01T00:00:00Z", "value": 21.5}],
}
]

View File

@@ -50,3 +50,135 @@ def test_unsupported_sensor_returns_422(tmp_path: Path) -> None:
)
assert response.status_code == 422
def test_retrain_creates_and_replaces_persisted_model(tmp_path: Path) -> None:
from app.ml.registry.model_registry import ModelRegistry
registry = ModelRegistry(tmp_path)
with TestClient(app) as client:
app.state.registry = registry
created = client.post(
"/ml/retrain",
json={
"modelId": "home-model",
"samples": [
{
"sensor_id": "sensor.kitchen",
"values": {"temperature": 21.0},
}
],
},
)
replaced = client.post(
"/ml/retrain",
json={
"modelId": "home-model",
"samples": [
{
"sensor_id": "sensor.bedroom",
"values": {"temperature": 18.0},
}
],
},
)
assert created.status_code == 200
assert created.json() == {
"model_id": "home-model",
"supported_sensors": ["sensor.kitchen"],
"trained_features": 1,
"model_type": "statistical_baseline",
"replaced": False,
}
assert replaced.status_code == 200
assert replaced.json() == {
"model_id": "home-model",
"supported_sensors": ["sensor.bedroom"],
"trained_features": 1,
"model_type": "statistical_baseline",
"replaced": True,
}
restarted = ModelRegistry(tmp_path)
assert restarted.load_artifact("home-model").supported_sensors == ("sensor.bedroom",)
def test_retrain_rejects_empty_samples() -> None:
with TestClient(app) as client:
response = client.post(
"/ml/retrain",
json={"modelId": "home-model", "samples": []},
)
assert response.status_code == 422
def test_predict_returns_numeric_forecast_and_confidence(tmp_path: Path) -> None:
from app.ml.feature_store import FeatureStore, FeatureVector
from app.ml.registry.model_registry import ModelRegistry
from app.ml.training import TrainingPipeline
store = FeatureStore()
store.add_batch(
[
FeatureVector("sensor.kitchen", {"temperature": 19.0}),
FeatureVector("sensor.kitchen", {"temperature": 20.0}),
]
)
registry = ModelRegistry(tmp_path)
registry.register(TrainingPipeline(store).run("home-model"))
with TestClient(app) as client:
app.state.registry = registry
response = client.post(
"/ml/predict",
json={
"modelId": "home-model",
"sensor_id": "sensor.kitchen",
"values": {"temperature": 21.0},
},
)
assert response.status_code == 200
assert response.json()["predictions"] == {"temperature": 22.0}
assert 0.0 < response.json()["confidence"] <= 1.0
assert response.json()["model_type"] == "statistical_baseline"
explanation = response.json()["explanations"]["temperature"]
assert explanation["direction"] == "steigend"
assert explanation["change"] == 1.0
assert explanation["sample_count"] == 2
def test_evaluate_returns_real_error_metrics(tmp_path: Path) -> None:
from app.ml.feature_store import FeatureStore, FeatureVector
from app.ml.registry.model_registry import ModelRegistry
from app.ml.training import TrainingPipeline
store = FeatureStore()
store.add_batch(
[
FeatureVector("sensor.kitchen", {"temperature": 19.0}),
FeatureVector("sensor.kitchen", {"temperature": 20.0}),
]
)
registry = ModelRegistry(tmp_path)
registry.register(TrainingPipeline(store).run("home-model"))
with TestClient(app) as client:
app.state.registry = registry
response = client.post(
"/ml/evaluate",
json={
"modelId": "home-model",
"samples": [
{
"sensor_id": "sensor.kitchen",
"values": {"temperature": 21.0},
}
],
},
)
assert response.status_code == 200
metrics = {metric["name"]: metric["value"] for metric in response.json()["metrics"]}
assert metrics == {"mae": 1.0, "rmse": 1.0, "coverage": 1.0}

View File

@@ -0,0 +1,76 @@
from __future__ import annotations
import pytest
from app.ha.discovery import EntityRole, classify_entity, discover_entities
from app.ha.models import HaEntitySummary
@pytest.mark.parametrize(
("entity", "role", "learnable"),
[
(
HaEntitySummary(
entity_id="sensor.temperature",
domain="sensor",
device_class="temperature",
state_class="measurement",
unit_of_measurement="°C",
),
EntityRole.MEASUREMENT,
True,
),
(
HaEntitySummary(
entity_id="binary_sensor.motion",
domain="binary_sensor",
device_class="motion",
),
EntityRole.BINARY_CONTEXT,
True,
),
(
HaEntitySummary(entity_id="person.simon", domain="person"),
EntityRole.CONTEXT,
True,
),
(
HaEntitySummary(entity_id="light.living_room", domain="light"),
EntityRole.ACTUATOR,
False,
),
(
HaEntitySummary(entity_id="camera.driveway", domain="camera"),
EntityRole.UNSUPPORTED,
False,
),
],
)
def test_classify_entity(
entity: HaEntitySummary,
role: EntityRole,
learnable: bool,
) -> None:
result = classify_entity(entity)
assert result.role is role
assert result.learnable is learnable
def test_discovery_filters_domain_and_learnable() -> None:
entities = [
HaEntitySummary(
entity_id="sensor.temperature",
domain="sensor",
device_class="temperature",
),
HaEntitySummary(entity_id="sensor.status", domain="sensor"),
HaEntitySummary(
entity_id="binary_sensor.motion",
domain="binary_sensor",
device_class="motion",
),
]
result = discover_entities(entities, domains={" SENSOR "}, learnable=True)
assert [item.entity_id for item in result] == ["sensor.temperature"]

View File

@@ -1,5 +1,6 @@
from __future__ import annotations
from datetime import datetime, timezone
from unittest.mock import Mock
import pytest
@@ -68,4 +69,60 @@ def test_list_entities_rejects_invalid_json() -> None:
def test_list_entities_rejects_non_list_payload() -> None:
client = _client_with_response(_response(payload={"entity_id": "sensor.temperature"}))
with pytest.raises(HaUnexpectedPayloadError):
client.list_entities()
client.list_entities()
def test_get_history_calls_home_assistant_history_api() -> None:
response = _response(payload=[[{"entity_id": "sensor.temperature", "state": "21.0"}]])
client = _client_with_response(response)
start = datetime(2026, 6, 1, tzinfo=timezone.utc)
end = datetime(2026, 6, 2, tzinfo=timezone.utc)
payload = client.get_history(["sensor.temperature"], start, end)
assert payload == [[{"entity_id": "sensor.temperature", "state": "21.0"}]]
client._session.get.assert_called_once() # type: ignore[attr-defined]
call = client._session.get.call_args # type: ignore[attr-defined]
assert "/api/history/period/2026-06-01T00:00:00+00:00" in call.args[0]
assert call.kwargs["params"]["filter_entity_id"] == "sensor.temperature"
assert call.kwargs["params"]["end_time"] == "2026-06-02T00:00:00+00:00"
@pytest.mark.parametrize(
("entity_ids", "start", "end"),
[
(
[],
datetime(2026, 6, 1, tzinfo=timezone.utc),
datetime(2026, 6, 2, tzinfo=timezone.utc),
),
(
["sensor.temperature"],
datetime(2026, 6, 1),
datetime(2026, 6, 2, tzinfo=timezone.utc),
),
(
["sensor.temperature"],
datetime(2026, 6, 2, tzinfo=timezone.utc),
datetime(2026, 6, 1, tzinfo=timezone.utc),
),
(
["invalid entity"],
datetime(2026, 6, 1, tzinfo=timezone.utc),
datetime(2026, 6, 2, tzinfo=timezone.utc),
),
(
["sensor.temperature"],
datetime(2026, 5, 1, tzinfo=timezone.utc),
datetime(2026, 6, 2, tzinfo=timezone.utc),
),
],
)
def test_get_history_validates_request(
entity_ids: list[str],
start: datetime,
end: datetime,
) -> None:
client = HaClient(HaClientSettings(url="http://ha.local", token="test-token"))
with pytest.raises(ValueError):
client.get_history(entity_ids, start, end)

View File

@@ -1,5 +1,7 @@
from __future__ import annotations
from datetime import datetime, timezone
from app.ha.client import HaClient, HaClientSettings
from app.ha.reader import HaReader
@@ -26,6 +28,22 @@ class FakeHaClient(HaClient):
},
]
def get_history(
self,
entity_ids: list[str],
start_time: datetime,
end_time: datetime,
) -> list[object]:
return [
[
{
"entity_id": entity_ids[0],
"state": "21.5",
"last_changed": start_time.isoformat(),
}
]
]
def test_ha_reader_returns_summaries() -> None:
reader = HaReader(FakeHaClient())
@@ -35,3 +53,25 @@ def test_ha_reader_returns_summaries() -> None:
assert domains == {"sensor", "light"}
sensor = next(item for item in summaries if item.entity_id == "sensor.temperature")
assert sensor.unit_of_measurement == "°C"
def test_ha_reader_discovers_learnable_sensors() -> None:
reader = HaReader(FakeHaClient())
discovered = reader.discover(learnable=True)
assert [entity.entity_id for entity in discovered] == ["sensor.temperature"]
def test_ha_reader_normalizes_history() -> None:
reader = HaReader(FakeHaClient())
start = datetime(2026, 6, 1, tzinfo=timezone.utc)
history = reader.read_history(
["sensor.temperature"],
start,
datetime(2026, 6, 2, tzinfo=timezone.utc),
)
assert history[0].entity_id == "sensor.temperature"
assert history[0].points[0].value == 21.5

92
tests/ha/test_history.py Normal file
View File

@@ -0,0 +1,92 @@
from __future__ import annotations
from datetime import datetime, timezone
import pytest
from app.ha.exceptions import HaUnexpectedPayloadError
from app.ha.history import normalize_history_payload
def test_normalize_history_payload_groups_and_sorts_numeric_states() -> None:
payload = [
[
{
"entity_id": "sensor.temperature",
"state": "22.5",
"last_changed": "2026-06-01T12:15:00+00:00",
},
{
"state": "21.0",
"last_changed": "2026-06-01T12:00:00Z",
},
],
[
{
"entity_id": "sensor.humidity",
"state": 45,
"last_updated": "2026-06-01T12:00:00+00:00",
}
],
]
result = normalize_history_payload(payload)
assert [series.entity_id for series in result] == [
"sensor.humidity",
"sensor.temperature",
]
temperature = result[1]
assert [point.value for point in temperature.points] == [21.0, 22.5]
assert temperature.points[0].timestamp == datetime(
2026, 6, 1, 12, 0, tzinfo=timezone.utc
)
def test_normalize_history_payload_skips_non_numeric_and_non_finite_states() -> None:
payload = [
[
{
"entity_id": "sensor.temperature",
"state": state,
"last_changed": "2026-06-01T12:00:00+00:00",
}
for state in ("unknown", "unavailable", "nan", "inf", "-inf", True, None)
]
]
assert normalize_history_payload(payload) == []
@pytest.mark.parametrize(
"payload",
[
{},
[{}],
[["invalid"]],
[[{"entity_id": "invalid", "state": "21", "last_changed": "2026-06-01"}]],
[[{"entity_id": "sensor.a", "state": "21", "last_changed": "invalid"}]],
[[{"state": "21", "last_changed": "2026-06-01T12:00:00+00:00"}]],
[
[
{
"entity_id": "sensor.a",
"state": "21",
"last_changed": "2026-06-01T12:00:00+00:00",
},
{
"entity_id": "sensor.b",
"state": "22",
"last_changed": "2026-06-01T12:01:00+00:00",
},
]
],
],
)
def test_normalize_history_payload_rejects_malformed_structure(payload: object) -> None:
with pytest.raises(HaUnexpectedPayloadError):
normalize_history_payload(payload)
def test_normalize_history_payload_accepts_empty_series() -> None:
assert normalize_history_payload([[]]) == []

View File

@@ -24,14 +24,15 @@ def test_evaluate_returns_report_with_metrics() -> None:
report = evaluator.evaluate(
"artifact_v1",
[
"artifact_v1:sensor.kitchen:{'temperature': 21.0}",
"artifact_v1:sensor.bedroom:{'temperature': 18.5}",
_vector("sensor.kitchen", 21.0),
_vector("sensor.bedroom", 18.5),
],
)
assert report.artifact_id == "artifact_v1"
assert report.sample_size == 2
assert {metric.name for metric in report.metrics} == {"coverage", "unknown_rate"}
assert {metric.name for metric in report.metrics} == {"mae", "rmse", "coverage"}
assert next(metric.value for metric in report.metrics if metric.name == "coverage") == 1.0
assert next(metric.value for metric in report.metrics if metric.name == "mae") == 0.0
def test_evaluate_without_training_raises_value_error() -> None:
@@ -40,16 +41,16 @@ def test_evaluate_without_training_raises_value_error() -> None:
evaluator.evaluate("artifact_v1", [])
def test_coverage_is_bounded_and_requires_exact_sensor_match() -> None:
def test_coverage_counts_only_supported_sensor_features() -> None:
evaluator = evaluator_factory()
report = evaluator.evaluate(
"artifact_v1",
[
"artifact_v1:sensor.kitchen:{'note': 'sensor.bedroom'}",
"artifact_v1:sensor.kitchen_extra:{}",
"malformed",
_vector("sensor.kitchen", 21.0),
FeatureVector(sensor_id="sensor.kitchen", values={"humidity": 50.0}),
_vector("sensor.kitchen_extra", 20.0),
],
)
metrics = {metric.name: metric.value for metric in report.metrics}
assert metrics == {"coverage": pytest.approx(1 / 3), "unknown_rate": pytest.approx(2 / 3)}
assert metrics["coverage"] == pytest.approx(1 / 3)

View File

@@ -0,0 +1,34 @@
from __future__ import annotations
from app.ml.explanation import explain_feature
from app.ml.training import FeatureModel
def _model(slope: float) -> FeatureModel:
return FeatureModel(
sample_count=4,
mean=20.0,
standard_deviation=1.0,
minimum=18.0,
maximum=22.0,
slope=slope,
intercept=18.5,
)
def test_explain_feature_describes_rising_forecast() -> None:
explanation = explain_feature("temperature", 21.0, 21.5, _model(0.5))
assert explanation.direction == "steigend"
assert explanation.change == 0.5
assert explanation.historical_range == (18.0, 22.0)
assert "4 Messwerte" in explanation.summary
assert "Trend +0.500" in explanation.summary
def test_explain_feature_describes_stable_and_falling_forecasts() -> None:
stable = explain_feature("humidity", 50.0, 50.0, _model(0.0))
falling = explain_feature("temperature", 21.0, 20.5, _model(-0.5))
assert stable.direction == "stabil"
assert falling.direction == "fallend"

View File

@@ -19,6 +19,35 @@ def test_registry_loads_persisted_artifacts_after_restart(tmp_path: Path) -> Non
assert restarted.load_artifact("model-v1") == artifact
def test_registry_persists_statistical_parameters(tmp_path: Path) -> None:
from app.ml.feature_store import FeatureStore, FeatureVector
from app.ml.training import TrainingPipeline
store = FeatureStore()
store.add_batch(
[
FeatureVector("sensor.kitchen", {"temperature": 19.0}),
FeatureVector("sensor.kitchen", {"temperature": 20.0}),
]
)
artifact = TrainingPipeline(store).run("model-v1")
ModelRegistry(tmp_path).register(artifact)
assert ModelRegistry(tmp_path).load_artifact("model-v1") == artifact
def test_registry_replaces_persisted_artifact_after_restart(tmp_path: Path) -> None:
registry = ModelRegistry(tmp_path)
registry.register(TrainedArtifact("model-v1", ("sensor.kitchen",)))
replacement = TrainedArtifact("model-v1", ("sensor.bedroom",))
registry.register(replacement)
assert registry.load_artifact("model-v1") == replacement
assert ModelRegistry(tmp_path).load_artifact("model-v1") == replacement
@pytest.mark.parametrize("artifact_id", ["../escape", "nested/model", "..", ""])
def test_registry_rejects_unsafe_artifact_ids(tmp_path: Path, artifact_id: str) -> None:
registry = ModelRegistry(tmp_path)

View File

@@ -13,16 +13,31 @@ def _vector(sensor_id: str, temperature: float, label: str | None = None) -> Fea
def predictor() -> Predictor:
store = FeatureStore()
store.add_batch([_vector("sensor.kitchen", 19.0), _vector("sensor.bedroom", 18.5)])
store.add_batch(
[
_vector("sensor.kitchen", 19.0),
_vector("sensor.kitchen", 20.0),
_vector("sensor.bedroom", 18.5),
]
)
pipeline = TrainingPipeline(store)
pipeline.run("artifact_v1")
return Predictor(pipeline)
def test_predict_returns_expected_format() -> None:
def test_predict_returns_statistical_forecast() -> None:
p = predictor()
result = p.predict("artifact_v1", _vector("sensor.kitchen", 21.0))
assert result == "artifact_v1:sensor.kitchen:{'temperature': 21.0}"
assert result.artifact_id == "artifact_v1"
assert result.sensor_id == "sensor.kitchen"
assert result.predictions == {"temperature": 22.0}
assert 0.0 < result.confidence <= 1.0
assert result.model_type == "statistical_baseline"
explanation = result.explanations["temperature"]
assert explanation.direction == "steigend"
assert explanation.current_value == 21.0
assert explanation.predicted_value == 22.0
assert explanation.sample_count == 2
def test_predict_rejects_unknown_sensor() -> None:
@@ -45,4 +60,4 @@ def test_default_artifact_returns_last_registered() -> None:
pipeline = TrainingPipeline(store)
pipeline.run("first")
pipeline.run("second")
assert Predictor.default_artifact(pipeline).artifact_id == "second"
assert Predictor.default_artifact(pipeline).artifact_id == "second"

View File

@@ -0,0 +1,39 @@
from __future__ import annotations
from pathlib import Path
import pytest
from app.ml.feature_store import FeatureVector
from app.ml.registry.model_registry import ModelRegistry
from app.ml.retraining import RetrainingService, retrain_model
def _vector(sensor_id: str) -> FeatureVector:
return FeatureVector(sensor_id=sensor_id, values={"temperature": 21.0})
def test_retraining_registers_new_artifact(tmp_path: Path) -> None:
registry = ModelRegistry(tmp_path)
result = retrain_model(registry, "home-model", [_vector("sensor.kitchen")])
assert result.replaced is False
assert registry.load_artifact("home-model") == result.artifact
def test_retraining_replaces_existing_artifact(tmp_path: Path) -> None:
registry = ModelRegistry(tmp_path)
service = RetrainingService(registry)
service.retrain("home-model", [_vector("sensor.kitchen")])
result = service.retrain("home-model", [_vector("sensor.bedroom")])
assert result.replaced is True
assert result.artifact.supported_sensors == ("sensor.bedroom",)
assert ModelRegistry(tmp_path).load_artifact("home-model") == result.artifact
def test_retraining_rejects_empty_training_data(tmp_path: Path) -> None:
with pytest.raises(ValueError, match="keine Trainingsdaten"):
retrain_model(ModelRegistry(tmp_path), "home-model", [])

View File

@@ -27,6 +27,11 @@ def test_run_returns_trained_artifact() -> None:
artifact = pipeline.run("artifact_v1")
assert artifact.artifact_id == "artifact_v1"
assert artifact.supported_sensors == ("sensor.bedroom", "sensor.kitchen")
kitchen = artifact.feature_models["sensor.kitchen"]["temperature"]
assert kitchen.sample_count == 2
assert kitchen.mean == 19.5
assert kitchen.slope == 1.0
assert kitchen.forecast() == 21.0
def test_run_without_data_raises_value_error() -> None:

View File

@@ -16,18 +16,18 @@ def test_end_to_end_training_then_evaluation() -> None:
artifact = pipeline.run("artifact_v1")
evaluator = Evaluator(pipeline)
predictions = [
"artifact_v1:sensor.kitchen:{'temperature': 21.0}",
"artifact_v1:sensor.bedroom:{'temperature': 18.5}",
samples = [
_vector("sensor.kitchen", 21.0),
_vector("sensor.bedroom", 18.5),
]
report = evaluator.evaluate(artifact.artifact_id, predictions)
report = evaluator.evaluate(artifact.artifact_id, samples)
assert isinstance(report, EvalReport)
assert report.sample_size == len(predictions)
assert report.sample_size == len(samples)
assert any(metric.name == "coverage" for metric in report.metrics)
def test_metric_helpers_are_serializable() -> None:
metric = Metric(name="coverage", value=0.85, threshold=0.8)
assert metric.name == "coverage"
metric = Metric(name="mae", value=0.85, threshold=1.0)
assert metric.name == "mae"
assert metric.value == 0.85
assert metric.threshold == 0.8
assert metric.threshold == 1.0