From df2ddacfbf472d177039ca539ec75ba75d52f339 Mon Sep 17 00:00:00 2001 From: Otto Date: Sat, 13 Jun 2026 20:13:06 +0200 Subject: [PATCH] ML-008: add statistical baseline model Closes #19 --- CHANGELOG.md | 4 ++ README.md | 10 ++-- app/main.py | 2 +- app/ml/__init__.py | 6 +- app/ml/evaluation.py | 67 ++++++++++++++------- app/ml/predictor.py | 49 +++++++++++++-- app/ml/registry/model_registry.py | 74 ++++++++++++++++++++++- app/ml/training.py | 89 ++++++++++++++++++++++++++-- backend/routes/ml.py | 76 +++++++++++++++++++++++- docs/ml_api.md | 24 ++++++-- docs/ml_training.md | 31 ++++++---- pyproject.toml | 2 +- tests/api/test_ml_routes.py | 71 ++++++++++++++++++++++ tests/ml/test_evaluation.py | 17 +++--- tests/ml/test_model_registry.py | 18 ++++++ tests/ml/test_predictor.py | 18 ++++-- tests/ml/test_training.py | 5 ++ tests/ml/test_training_evaluation.py | 16 ++--- 18 files changed, 502 insertions(+), 77 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0716f3e..8d503a6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,9 +1,13 @@ # Changelog ## Unreleased + +## 0.2.0 - 2026-06-13 - Klassifizierte Home-Assistant-Entity-Discovery mit Lernrelevanz und Filtern - Validierter Zugriff auf die Home-Assistant-History-API - Normalisierte, chronologisch sortierte numerische Zeitreihen über `/v1/history` +- Trainierbares statistisches Baseline-Modell mit persistierten Parametern +- Numerische Vorhersagen mit Confidence sowie MAE-/RMSE-Evaluation ## 0.1.0 - 2026-06-13 - Projektinitiierung diff --git a/README.md b/README.md index 4af3963..0ae5590 100644 --- a/README.md +++ b/README.md @@ -4,10 +4,11 @@ Lokaler, datenschutzfreundlicher API-Prototyp für Home Assistant. ## Reifegrad -Version `0.1.0` stellt eine gehärtete technische Basis bereit: Home-Assistant-Entities -lesen, regelbasierte Bausteine und eine persistente Modell-Artefakt-Registry. Die -aktuelle Trainings- und Vorhersagelogik ist noch eine deterministische -Schnittstellen-Implementierung und **kein produktives Machine-Learning-Modell**. +Die aktuelle Entwicklungslinie stellt eine gehärtete technische Basis bereit: +Home-Assistant-Entities und Historie lesen, Sensoren klassifizieren, +regelbasierte Bausteine sowie ein lokal trainierbares statistisches +Baseline-Modell mit persistenter Registry, Confidence und echten +Evaluationsmetriken. ## Motivation TheSillyHome zeigte die Idee: statt statischer Regeln das Zuhause aus Verhaltensmustern verstehen. Diese Architektur modernisiert den Ansatz in Richtung Explainable AI, hybride Intelligenzebenen und langlebige Wartbarkeit. @@ -46,6 +47,7 @@ uvicorn app.main:app --reload - `http://127.0.0.1:8000/v1/history` - normalisierte numerische Zeitreihen - `http://127.0.0.1:8000/ml/health` - Registry-/Serving-Health - `POST http://127.0.0.1:8000/ml/retrain` - Modell-Metadaten aktualisieren +- `POST http://127.0.0.1:8000/ml/evaluate` - MAE/RMSE/Coverage berechnen Ohne vollständige HA-Konfiguration liefert `/v1/entities` bewusst `503`. diff --git a/app/main.py b/app/main.py index 49d55e8..0df65ed 100644 --- a/app/main.py +++ b/app/main.py @@ -38,7 +38,7 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]: app = FastAPI( title="SillyHome Next API", description="Lokales Smart-Home-Intelligenzsystem für Home Assistant.", - version="0.1.0", + version="0.2.0", lifespan=lifespan, ) app.state.settings = load_settings() diff --git a/app/ml/__init__.py b/app/ml/__init__.py index dabe1f6..63cf58a 100644 --- a/app/ml/__init__.py +++ b/app/ml/__init__.py @@ -3,6 +3,9 @@ __all__ = [ "FeatureStore", "FeatureVector", + "FeatureModel", + "PredictionResult", + "Predictor", "RetrainingResult", "RetrainingService", "TrainedArtifact", @@ -10,5 +13,6 @@ __all__ = [ "retrain_model", ] from app.ml.feature_store import FeatureStore, FeatureVector +from app.ml.predictor import PredictionResult, Predictor from app.ml.retraining import RetrainingResult, RetrainingService, retrain_model -from app.ml.training import TrainedArtifact, TrainingPipeline +from app.ml.training import FeatureModel, TrainedArtifact, TrainingPipeline diff --git a/app/ml/evaluation.py b/app/ml/evaluation.py index e163ebf..f3282b5 100644 --- a/app/ml/evaluation.py +++ b/app/ml/evaluation.py @@ -1,9 +1,13 @@ from __future__ import annotations import logging +import math from collections.abc import Sequence from dataclasses import dataclass +from app.ml.feature_store import FeatureVector +from app.ml.predictor import Predictor +from app.ml.registry.model_registry import ModelRegistry from app.ml.training import TrainingPipeline logger = logging.getLogger(__name__) @@ -24,41 +28,62 @@ class EvalReport: class Evaluator: - def __init__(self, pipeline: TrainingPipeline) -> None: + def __init__( + self, + pipeline: TrainingPipeline | None = None, + registry: ModelRegistry | None = None, + ) -> None: + if isinstance(pipeline, ModelRegistry) and registry is None: + registry = pipeline + pipeline = None + if pipeline is None and registry is None: + raise ValueError("Evaluator erfordert TrainingPipeline oder ModelRegistry.") self._pipeline = pipeline + self._registry = registry + self._predictor = Predictor(pipeline=pipeline, registry=registry) - def evaluate(self, artifact_id: str, predictions: Sequence[str]) -> EvalReport: + def evaluate(self, artifact_id: str, samples: Sequence[FeatureVector]) -> EvalReport: try: - supported_sensors = set(self._pipeline.export(artifact_id).supported_sensors) + if self._registry is not None: + self._registry.load_artifact(artifact_id) + elif self._pipeline is not None: + self._pipeline.export(artifact_id) except KeyError as exc: raise ValueError("Kein trainiertes Modell für Evaluation vorhanden.") from exc - parsed_sensors = [_prediction_sensor(prediction) for prediction in predictions] - supported_hits = sum(sensor in supported_sensors for sensor in parsed_sensors) - unknown_hits = sum(sensor not in supported_sensors for sensor in parsed_sensors) - sample_size = len(predictions) - coverage = supported_hits / sample_size if sample_size else 0.0 - unknown_rate = unknown_hits / sample_size if sample_size else 0.0 + absolute_errors: list[float] = [] + squared_errors: list[float] = [] + for sample in samples: + try: + prediction = self._predictor.predict(artifact_id, sample) + except ValueError: + continue + for feature_name, predicted in prediction.predictions.items(): + actual = float(sample.values[feature_name]) + error = predicted - actual + absolute_errors.append(abs(error)) + squared_errors.append(error**2) - coverage_metric = Metric(name="coverage", value=coverage, threshold=0.8) - unknown_metric = Metric(name="unknown_rate", value=unknown_rate, threshold=0.1) + sample_size = len(absolute_errors) + mae = sum(absolute_errors) / sample_size if sample_size else 0.0 + rmse = math.sqrt(sum(squared_errors) / sample_size) if sample_size else 0.0 + expected_values = sum(len(sample.values) for sample in samples) + coverage = sample_size / expected_values if expected_values else 0.0 report = EvalReport( artifact_id=artifact_id, sample_size=sample_size, - metrics=[coverage_metric, unknown_metric], + metrics=[ + Metric(name="mae", value=mae), + Metric(name="rmse", value=rmse), + Metric(name="coverage", value=coverage, threshold=0.8), + ], ) logger.info( - "Evaluation %s -> coverage=%.2f, unknown_rate=%.2f", + "Evaluation %s -> mae=%.4f, rmse=%.4f, coverage=%.2f", artifact_id, + mae, + rmse, coverage, - unknown_rate, ) return report - - -def _prediction_sensor(prediction: str) -> str | None: - parts = prediction.split(":", 2) - if len(parts) != 3 or not parts[0] or not parts[1]: - return None - return parts[1] diff --git a/app/ml/predictor.py b/app/ml/predictor.py index 578d822..d719e48 100644 --- a/app/ml/predictor.py +++ b/app/ml/predictor.py @@ -1,6 +1,8 @@ from __future__ import annotations import logging +import math +from dataclasses import dataclass from typing import Sequence from app.ml.feature_store import FeatureVector @@ -10,6 +12,15 @@ from app.ml.training import TrainedArtifact, TrainingPipeline logger = logging.getLogger(__name__) +@dataclass(frozen=True) +class PredictionResult: + artifact_id: str + sensor_id: str + predictions: dict[str, float] + confidence: float + model_type: str + + class Predictor: def __init__( self, @@ -24,15 +35,45 @@ class Predictor: self._pipeline = pipeline self._registry = registry - def predict(self, artifact_id: str, entity: FeatureVector) -> str: + def predict(self, artifact_id: str, entity: FeatureVector) -> PredictionResult: artifact = self._get_artifact(artifact_id) if entity.sensor_id not in artifact.supported_sensors: raise ValueError( f"Sensor '{entity.sensor_id}' wird vom Modell '{artifact_id}' nicht unterstützt." ) - return f"{artifact_id}:{entity.sensor_id}:{entity.values}" + sensor_models = artifact.feature_models.get(entity.sensor_id, {}) + if not sensor_models: + raise ValueError(f"Modell '{artifact_id}' enthält keine statistischen Parameter.") - def predict_batch(self, artifact_id: str, entities: Sequence[FeatureVector]) -> list[str]: + feature_names = sorted(set(sensor_models).intersection(entity.values)) + if not feature_names: + raise ValueError( + f"Keine Eingabemerkmale werden vom Modell '{artifact_id}' unterstützt." + ) + + predictions: dict[str, float] = {} + confidences: list[float] = [] + for feature_name in feature_names: + model = sensor_models[feature_name] + current_value = float(entity.values[feature_name]) + if not math.isfinite(current_value): + raise ValueError("Vorhersagewerte müssen endlich sein.") + predictions[feature_name] = model.forecast(current_value) + confidences.append(model.confidence) + + return PredictionResult( + artifact_id=artifact_id, + sensor_id=entity.sensor_id, + predictions=predictions, + confidence=sum(confidences) / len(confidences), + model_type=artifact.model_type, + ) + + def predict_batch( + self, + artifact_id: str, + entities: Sequence[FeatureVector], + ) -> list[PredictionResult]: return [self.predict(artifact_id, entity) for entity in entities] @staticmethod @@ -47,4 +88,4 @@ class Predictor: return self._registry.load_artifact(artifact_id) if self._pipeline is not None: return self._pipeline.export(artifact_id) - raise RuntimeError("Predictor nicht initialisiert.") \ No newline at end of file + raise RuntimeError("Predictor nicht initialisiert.") diff --git a/app/ml/registry/model_registry.py b/app/ml/registry/model_registry.py index e6a634b..cac7df2 100644 --- a/app/ml/registry/model_registry.py +++ b/app/ml/registry/model_registry.py @@ -2,13 +2,14 @@ from __future__ import annotations import json import logging +import math import os from pathlib import Path import re from threading import RLock from collections.abc import Iterable -from app.ml.training import TrainedArtifact +from app.ml.training import FeatureModel, TrainedArtifact logger = logging.getLogger(__name__) @@ -52,19 +53,26 @@ class ModelRegistry: raw = json.loads(source.read_text(encoding="utf-8")) artifact_id = raw["artifact_id"] supported_sensors = raw["supported_sensors"] + model_type = raw.get("model_type", "metadata") + raw_feature_models = raw.get("feature_models", {}) if not isinstance(artifact_id, str) or not isinstance(supported_sensors, list): raise ValueError("invalid artifact structure") + if not isinstance(model_type, str): + raise ValueError("model_type must be a string") self._validate_artifact_id(artifact_id) if source.name != f"{artifact_id}.json": raise ValueError("artifact id does not match filename") if not all(isinstance(sensor, str) for sensor in supported_sensors): raise ValueError("supported_sensors must contain strings") + feature_models = _deserialize_feature_models(raw_feature_models) except (KeyError, TypeError, ValueError, json.JSONDecodeError) as exc: raise ValueError(f"Ungültiges Modell-Artefakt: {source.name}") from exc self._artifacts[artifact_id] = TrainedArtifact( artifact_id=artifact_id, supported_sensors=tuple(supported_sensors), + feature_models=feature_models, + model_type=model_type, ) def _persist(self, artifact: TrainedArtifact) -> None: @@ -73,6 +81,22 @@ class ModelRegistry: payload = { "artifact_id": artifact.artifact_id, "supported_sensors": list(artifact.supported_sensors), + "model_type": artifact.model_type, + "feature_models": { + sensor_id: { + feature_name: { + "sample_count": model.sample_count, + "mean": model.mean, + "standard_deviation": model.standard_deviation, + "minimum": model.minimum, + "maximum": model.maximum, + "slope": model.slope, + "intercept": model.intercept, + } + for feature_name, model in sorted(models.items()) + } + for sensor_id, models in sorted(artifact.feature_models.items()) + }, } temporary.write_text( json.dumps(payload, ensure_ascii=True, sort_keys=True) + "\n", @@ -88,3 +112,51 @@ class ModelRegistry: "artifact_id darf nur Buchstaben, Ziffern, Punkt, Unterstrich " "und Bindestrich enthalten." ) + + +def _deserialize_feature_models(raw: object) -> dict[str, dict[str, FeatureModel]]: + if not isinstance(raw, dict): + raise ValueError("feature_models must be an object") + + result: dict[str, dict[str, FeatureModel]] = {} + for sensor_id, raw_features in raw.items(): + if not isinstance(sensor_id, str) or not isinstance(raw_features, dict): + raise ValueError("invalid sensor feature models") + features: dict[str, FeatureModel] = {} + for feature_name, raw_model in raw_features.items(): + if not isinstance(feature_name, str) or not isinstance(raw_model, dict): + raise ValueError("invalid feature model") + sample_count = raw_model.get("sample_count") + if not isinstance(sample_count, int) or isinstance(sample_count, bool) or sample_count < 1: + raise ValueError("sample_count must be a positive integer") + values = { + key: _finite_number(raw_model.get(key)) + for key in ( + "mean", + "standard_deviation", + "minimum", + "maximum", + "slope", + "intercept", + ) + } + features[feature_name] = FeatureModel( + sample_count=sample_count, + mean=values["mean"], + standard_deviation=values["standard_deviation"], + minimum=values["minimum"], + maximum=values["maximum"], + slope=values["slope"], + intercept=values["intercept"], + ) + result[sensor_id] = features + return result + + +def _finite_number(value: object) -> float: + if not isinstance(value, (int, float)) or isinstance(value, bool): + raise ValueError("feature model values must be finite numbers") + converted = float(value) + if not math.isfinite(converted): + raise ValueError("feature model values must be finite numbers") + return converted diff --git a/app/ml/training.py b/app/ml/training.py index c6d8bdf..97a4510 100644 --- a/app/ml/training.py +++ b/app/ml/training.py @@ -1,17 +1,44 @@ from __future__ import annotations import logging -from dataclasses import dataclass +import math +from collections import defaultdict +from dataclasses import dataclass, field from app.ml.feature_store import FeatureStore logger = logging.getLogger(__name__) -@dataclass +@dataclass(frozen=True) +class FeatureModel: + sample_count: int + mean: float + standard_deviation: float + minimum: float + maximum: float + slope: float + intercept: float + + def forecast(self, current_value: float | None = None) -> float: + if current_value is not None: + return current_value + self.slope + return self.intercept + self.slope * self.sample_count + + @property + def confidence(self) -> float: + sample_score = self.sample_count / (self.sample_count + 2) + scale = abs(self.mean) if abs(self.mean) > 1e-9 else 1.0 + stability_score = 1.0 / (1.0 + self.standard_deviation / scale) + return min(0.99, max(0.05, sample_score * stability_score)) + + +@dataclass(frozen=True) class TrainedArtifact: artifact_id: str supported_sensors: tuple[str, ...] + feature_models: dict[str, dict[str, FeatureModel]] = field(default_factory=dict) + model_type: str = "statistical_baseline" class TrainingPipeline: @@ -24,8 +51,33 @@ class TrainingPipeline: if not vectors: raise ValueError("FeatureStore enthält keine Trainingsdaten.") - sensors = tuple(sorted({vector.sensor_id for vector in vectors})) - artifact = TrainedArtifact(artifact_id=artifact_id, supported_sensors=sensors) + samples: dict[str, dict[str, list[float]]] = defaultdict(lambda: defaultdict(list)) + for vector in vectors: + for feature_name, raw_value in vector.values.items(): + value = float(raw_value) + if math.isfinite(value): + samples[vector.sensor_id][feature_name].append(value) + + feature_models = { + sensor_id: { + feature_name: _fit_feature(values) + for feature_name, values in sorted(features.items()) + if values + } + for sensor_id, features in sorted(samples.items()) + } + feature_models = { + sensor_id: models for sensor_id, models in feature_models.items() if models + } + if not feature_models: + raise ValueError("Trainingsdaten enthalten keine endlichen numerischen Werte.") + + sensors = tuple(feature_models) + artifact = TrainedArtifact( + artifact_id=artifact_id, + supported_sensors=sensors, + feature_models=feature_models, + ) self._artifacts[artifact_id] = artifact logger.info("Training abgeschlossen für %s mit %d Sensoren", artifact_id, len(sensors)) return artifact @@ -34,3 +86,32 @@ class TrainingPipeline: if artifact_id not in self._artifacts: raise KeyError(f"Artifact '{artifact_id}' nicht gefunden.") return self._artifacts[artifact_id] + + +def _fit_feature(values: list[float]) -> FeatureModel: + sample_count = len(values) + mean = sum(values) / sample_count + variance = sum((value - mean) ** 2 for value in values) / sample_count + standard_deviation = math.sqrt(variance) + + if sample_count == 1: + slope = 0.0 + intercept = mean + else: + x_mean = (sample_count - 1) / 2 + denominator = sum((index - x_mean) ** 2 for index in range(sample_count)) + numerator = sum( + (index - x_mean) * (value - mean) for index, value in enumerate(values) + ) + slope = numerator / denominator + intercept = mean - slope * x_mean + + return FeatureModel( + sample_count=sample_count, + mean=mean, + standard_deviation=standard_deviation, + minimum=min(values), + maximum=max(values), + slope=slope, + intercept=intercept, + ) diff --git a/backend/routes/ml.py b/backend/routes/ml.py index c5b42d4..244c023 100644 --- a/backend/routes/ml.py +++ b/backend/routes/ml.py @@ -7,6 +7,7 @@ from collections.abc import Sequence from fastapi import APIRouter, FastAPI, HTTPException, Request, status from pydantic import BaseModel, Field +from app.ml.evaluation import Evaluator from app.ml.feature_store import FeatureVector from app.ml.predictor import Predictor from app.ml.registry.model_registry import ModelRegistry @@ -31,7 +32,9 @@ class PredictRequest(BaseModel): class PredictResponse(BaseModel): model_id: str sensor_id: str - prediction: str + predictions: dict[str, float] + confidence: float + model_type: str class BatchRequest(BaseModel): @@ -60,9 +63,28 @@ class RetrainRequest(BaseModel): class RetrainResponse(BaseModel): model_id: str supported_sensors: list[str] + trained_features: int + model_type: str replaced: bool +class EvaluateRequest(BaseModel): + model_id: str = Field(..., alias="modelId", min_length=1, max_length=128) + samples: list[TrainingSample] = Field(min_length=1) + + +class MetricResponse(BaseModel): + name: str + value: float + threshold: float | None = None + + +class EvaluateResponse(BaseModel): + model_id: str + sample_size: int + metrics: list[MetricResponse] + + @router.get("/health", response_model=HealthResponse, status_code=200) def health() -> HealthResponse: return HealthResponse(status="ok") @@ -96,10 +118,50 @@ def retrain(payload: RetrainRequest, request: Request) -> RetrainResponse: return RetrainResponse( model_id=result.artifact.artifact_id, supported_sensors=list(result.artifact.supported_sensors), + trained_features=sum( + len(feature_models) + for feature_models in result.artifact.feature_models.values() + ), + model_type=result.artifact.model_type, replaced=result.replaced, ) +@router.post("/evaluate", response_model=EvaluateResponse, status_code=200) +def evaluate(payload: EvaluateRequest, request: Request) -> EvaluateResponse: + registry = _require_registry(request) + vectors = [ + FeatureVector( + sensor_id=sample.sensor_id, + values=sample.values, + label=sample.label, + ) + for sample in payload.samples + ] + try: + report = Evaluator(registry=registry).evaluate(payload.model_id, vectors) + except ValueError as exc: + try: + registry.load_artifact(payload.model_id) + except KeyError: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=str(exc), + ) from exc + raise HTTPException( + status_code=status.HTTP_422_UNPROCESSABLE_CONTENT, + detail=str(exc), + ) from exc + return EvaluateResponse( + model_id=report.artifact_id, + sample_size=report.sample_size, + metrics=[ + MetricResponse(name=metric.name, value=metric.value, threshold=metric.threshold) + for metric in report.metrics + ], + ) + + @router.post("/predict", response_model=PredictResponse, status_code=200) def predict(payload: PredictRequest, request: Request) -> PredictResponse: registry = _require_registry(request) @@ -117,7 +179,9 @@ def predict(payload: PredictRequest, request: Request) -> PredictResponse: return PredictResponse( model_id=payload.model_id, sensor_id=payload.sensor_id, - prediction=prediction, + predictions=prediction.predictions, + confidence=prediction.confidence, + model_type=prediction.model_type, ) @@ -138,7 +202,13 @@ def predict_batch(payload: BatchRequest, request: Request) -> BatchResponse: detail=str(exc), ) from exc responses.append( - PredictResponse(model_id=item.model_id, sensor_id=item.sensor_id, prediction=prediction) + PredictResponse( + model_id=item.model_id, + sensor_id=item.sensor_id, + predictions=prediction.predictions, + confidence=prediction.confidence, + model_type=prediction.model_type, + ) ) return BatchResponse(predictions=responses) diff --git a/docs/ml_api.md b/docs/ml_api.md index 05db5f4..30cdbca 100644 --- a/docs/ml_api.md +++ b/docs/ml_api.md @@ -3,9 +3,7 @@ Diese Dokumentation beschreibt die REST-Endpunkte der aktuellen Modell-Artefakt- und Vorhersage-Schnittstelle. -> Hinweis: Version 0.1.0 enthält noch kein statistisch trainiertes ML-Modell. -> Die Vorhersage ist eine deterministische Referenzimplementierung für den -> späteren Modellvertrag. +Das Serving verwendet ein lokal trainiertes statistisches Baseline-Modell. ## Basis-URL @@ -13,6 +11,7 @@ Modell-Artefakt- und Vorhersage-Schnittstelle. - Health: `/health` - Modelle: `/models` - Retraining: `/retrain` +- Evaluation: `/evaluate` - Einzelvorhersage: `/predict` - Batchvorhersage: `/batch` @@ -62,7 +61,9 @@ Einzelne Vorhersage für einen Sensor. { "model_id": "default", "sensor_id": "sensor.kitchen", - "prediction": "default:sensor.kitchen:{'temperature': 21.0}" + "predictions": {"temperature": 21.4}, + "confidence": 0.78, + "model_type": "statistical_baseline" } ``` @@ -91,10 +92,17 @@ dem Modellverzeichnis geladen. { "model_id": "home-model", "supported_sensors": ["sensor.kitchen"], + "trained_features": 1, + "model_type": "statistical_baseline", "replaced": false } ``` +### `POST /ml/evaluate` + +Vergleicht Modellvorhersagen mit Validierungsdaten und liefert MAE, RMSE und +Coverage. Der Request verwendet dasselbe Sample-Format wie `/ml/retrain`. + ### `POST /ml/batch` Batch-Vorhersage für mehrere Sensorwerte. @@ -124,12 +132,16 @@ Batch-Vorhersage für mehrere Sensorwerte. { "model_id": "default", "sensor_id": "sensor.kitchen", - "prediction": "default:sensor.kitchen:{'temperature': 21.0}" + "predictions": {"temperature": 21.4}, + "confidence": 0.78, + "model_type": "statistical_baseline" }, { "model_id": "default", "sensor_id": "sensor.bedroom", - "prediction": "default:sensor.bedroom:{'temperature': 18.5}" + "predictions": {"temperature": 18.3}, + "confidence": 0.74, + "model_type": "statistical_baseline" } ] } diff --git a/docs/ml_training.md b/docs/ml_training.md index bcc70a5..956e5f4 100644 --- a/docs/ml_training.md +++ b/docs/ml_training.md @@ -1,14 +1,13 @@ # ML Training- und Evaluations-Workflow -Dieser Workflow beschreibt den aktuellen Platzhalter für Modell-Metadaten, -Evaluation und Serving. Er trainiert in Version 0.1.0 noch kein statistisches -Modell. +SillyHome Next trainiert ein lokales statistisches Baseline-Modell pro Sensor +und Merkmal. Es benötigt keine Cloud und keine externe ML-Laufzeit. ## 1. Daten sammeln Alle Trainingsvektoren werden über `FeatureStore.add(...)` oder `add_batch(...)` eingepflegt. Jeder Vektor enthält eine Sensor-ID sowie ein Dictionary mit Merkmalen. -## 2. Artefakt-Metadaten erzeugen +## 2. Statistisches Artefakt erzeugen ```python store = FeatureStore() @@ -18,21 +17,30 @@ artifact = pipeline.run("my_artifact") pipeline.export("my_artifact") ``` -`TrainingPipeline.run(...)` erzeugt ein `TrainedArtifact` mit den unterstützten -Sensor-IDs. Gewichte, Parameter oder ein echtes Modell werden noch nicht -berechnet. +`TrainingPipeline.run(...)` berechnet für jedes numerische Merkmal: + +- Stichprobenzahl +- Mittelwert und Standardabweichung +- Minimum und Maximum +- linearen Trend mit Steigung und Achsenabschnitt + +Die nächste Vorhersage kombiniert den letzten beobachteten Wert mit der +trainierten Trendsteigung. Die Confidence berücksichtigt Datenmenge und +Stabilität. ## 3. Modell evaluieren ```python evaluator = Evaluator(pipeline) -report = evaluator.evaluate(artifact.artifact_id, predictions) +report = evaluator.evaluate(artifact.artifact_id, validation_samples) ``` -Der Report enthält: +Der Report enthält echte numerische Vergleichsmetriken: - `artifact_id` - `sample_size` -- Metriken wie `coverage` und `unknown_rate` mit Default-Schwellenwerten +- `mae` (Mean Absolute Error) +- `rmse` (Root Mean Squared Error) +- `coverage` für den Anteil auswertbarer Merkmale ## 4. Modell registrieren @@ -56,4 +64,5 @@ Der Service startet bewusst keinen eigenen Hintergrundprozess. Über ## Hinweise - Für reproduzierbare Sensor-Reihenfolgen wird in `TrainingPipeline.run(...)` eine sortierte Sensor-Liste verwendet. - Fehlende Trainingsdaten lösen `ValueError` aus; nicht registrierte Artefakte lösen `KeyError` aus. -- `coverage` zählt nur exakte Sensor-Referenzen und bleibt im Bereich 0 bis 1. +- Nur endliche numerische Werte werden trainiert. +- `coverage` bleibt im Bereich 0 bis 1. diff --git a/pyproject.toml b/pyproject.toml index 3e662ff..bc1f262 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "sillyhome-next" -version = "0.1.0" +version = "0.2.0" description = "Lokales Smart-Home-Intelligenzsystem für Home Assistant" requires-python = ">=3.11" dependencies = [ diff --git a/tests/api/test_ml_routes.py b/tests/api/test_ml_routes.py index 74fe38c..f061ea9 100644 --- a/tests/api/test_ml_routes.py +++ b/tests/api/test_ml_routes.py @@ -87,12 +87,16 @@ def test_retrain_creates_and_replaces_persisted_model(tmp_path: Path) -> None: assert created.json() == { "model_id": "home-model", "supported_sensors": ["sensor.kitchen"], + "trained_features": 1, + "model_type": "statistical_baseline", "replaced": False, } assert replaced.status_code == 200 assert replaced.json() == { "model_id": "home-model", "supported_sensors": ["sensor.bedroom"], + "trained_features": 1, + "model_type": "statistical_baseline", "replaced": True, } restarted = ModelRegistry(tmp_path) @@ -107,3 +111,70 @@ def test_retrain_rejects_empty_samples() -> None: ) assert response.status_code == 422 + + +def test_predict_returns_numeric_forecast_and_confidence(tmp_path: Path) -> None: + from app.ml.feature_store import FeatureStore, FeatureVector + from app.ml.registry.model_registry import ModelRegistry + from app.ml.training import TrainingPipeline + + store = FeatureStore() + store.add_batch( + [ + FeatureVector("sensor.kitchen", {"temperature": 19.0}), + FeatureVector("sensor.kitchen", {"temperature": 20.0}), + ] + ) + registry = ModelRegistry(tmp_path) + registry.register(TrainingPipeline(store).run("home-model")) + + with TestClient(app) as client: + app.state.registry = registry + response = client.post( + "/ml/predict", + json={ + "modelId": "home-model", + "sensor_id": "sensor.kitchen", + "values": {"temperature": 21.0}, + }, + ) + + assert response.status_code == 200 + assert response.json()["predictions"] == {"temperature": 22.0} + assert 0.0 < response.json()["confidence"] <= 1.0 + assert response.json()["model_type"] == "statistical_baseline" + + +def test_evaluate_returns_real_error_metrics(tmp_path: Path) -> None: + from app.ml.feature_store import FeatureStore, FeatureVector + from app.ml.registry.model_registry import ModelRegistry + from app.ml.training import TrainingPipeline + + store = FeatureStore() + store.add_batch( + [ + FeatureVector("sensor.kitchen", {"temperature": 19.0}), + FeatureVector("sensor.kitchen", {"temperature": 20.0}), + ] + ) + registry = ModelRegistry(tmp_path) + registry.register(TrainingPipeline(store).run("home-model")) + + with TestClient(app) as client: + app.state.registry = registry + response = client.post( + "/ml/evaluate", + json={ + "modelId": "home-model", + "samples": [ + { + "sensor_id": "sensor.kitchen", + "values": {"temperature": 21.0}, + } + ], + }, + ) + + assert response.status_code == 200 + metrics = {metric["name"]: metric["value"] for metric in response.json()["metrics"]} + assert metrics == {"mae": 1.0, "rmse": 1.0, "coverage": 1.0} diff --git a/tests/ml/test_evaluation.py b/tests/ml/test_evaluation.py index d54182a..2bc1c1f 100644 --- a/tests/ml/test_evaluation.py +++ b/tests/ml/test_evaluation.py @@ -24,14 +24,15 @@ def test_evaluate_returns_report_with_metrics() -> None: report = evaluator.evaluate( "artifact_v1", [ - "artifact_v1:sensor.kitchen:{'temperature': 21.0}", - "artifact_v1:sensor.bedroom:{'temperature': 18.5}", + _vector("sensor.kitchen", 21.0), + _vector("sensor.bedroom", 18.5), ], ) assert report.artifact_id == "artifact_v1" assert report.sample_size == 2 - assert {metric.name for metric in report.metrics} == {"coverage", "unknown_rate"} + assert {metric.name for metric in report.metrics} == {"mae", "rmse", "coverage"} assert next(metric.value for metric in report.metrics if metric.name == "coverage") == 1.0 + assert next(metric.value for metric in report.metrics if metric.name == "mae") == 0.0 def test_evaluate_without_training_raises_value_error() -> None: @@ -40,16 +41,16 @@ def test_evaluate_without_training_raises_value_error() -> None: evaluator.evaluate("artifact_v1", []) -def test_coverage_is_bounded_and_requires_exact_sensor_match() -> None: +def test_coverage_counts_only_supported_sensor_features() -> None: evaluator = evaluator_factory() report = evaluator.evaluate( "artifact_v1", [ - "artifact_v1:sensor.kitchen:{'note': 'sensor.bedroom'}", - "artifact_v1:sensor.kitchen_extra:{}", - "malformed", + _vector("sensor.kitchen", 21.0), + FeatureVector(sensor_id="sensor.kitchen", values={"humidity": 50.0}), + _vector("sensor.kitchen_extra", 20.0), ], ) metrics = {metric.name: metric.value for metric in report.metrics} - assert metrics == {"coverage": pytest.approx(1 / 3), "unknown_rate": pytest.approx(2 / 3)} + assert metrics["coverage"] == pytest.approx(1 / 3) diff --git a/tests/ml/test_model_registry.py b/tests/ml/test_model_registry.py index 77cde56..956b033 100644 --- a/tests/ml/test_model_registry.py +++ b/tests/ml/test_model_registry.py @@ -19,6 +19,24 @@ def test_registry_loads_persisted_artifacts_after_restart(tmp_path: Path) -> Non assert restarted.load_artifact("model-v1") == artifact +def test_registry_persists_statistical_parameters(tmp_path: Path) -> None: + from app.ml.feature_store import FeatureStore, FeatureVector + from app.ml.training import TrainingPipeline + + store = FeatureStore() + store.add_batch( + [ + FeatureVector("sensor.kitchen", {"temperature": 19.0}), + FeatureVector("sensor.kitchen", {"temperature": 20.0}), + ] + ) + artifact = TrainingPipeline(store).run("model-v1") + + ModelRegistry(tmp_path).register(artifact) + + assert ModelRegistry(tmp_path).load_artifact("model-v1") == artifact + + def test_registry_replaces_persisted_artifact_after_restart(tmp_path: Path) -> None: registry = ModelRegistry(tmp_path) registry.register(TrainedArtifact("model-v1", ("sensor.kitchen",))) diff --git a/tests/ml/test_predictor.py b/tests/ml/test_predictor.py index 3c09a53..f1ab616 100644 --- a/tests/ml/test_predictor.py +++ b/tests/ml/test_predictor.py @@ -13,16 +13,26 @@ def _vector(sensor_id: str, temperature: float, label: str | None = None) -> Fea def predictor() -> Predictor: store = FeatureStore() - store.add_batch([_vector("sensor.kitchen", 19.0), _vector("sensor.bedroom", 18.5)]) + store.add_batch( + [ + _vector("sensor.kitchen", 19.0), + _vector("sensor.kitchen", 20.0), + _vector("sensor.bedroom", 18.5), + ] + ) pipeline = TrainingPipeline(store) pipeline.run("artifact_v1") return Predictor(pipeline) -def test_predict_returns_expected_format() -> None: +def test_predict_returns_statistical_forecast() -> None: p = predictor() result = p.predict("artifact_v1", _vector("sensor.kitchen", 21.0)) - assert result == "artifact_v1:sensor.kitchen:{'temperature': 21.0}" + assert result.artifact_id == "artifact_v1" + assert result.sensor_id == "sensor.kitchen" + assert result.predictions == {"temperature": 22.0} + assert 0.0 < result.confidence <= 1.0 + assert result.model_type == "statistical_baseline" def test_predict_rejects_unknown_sensor() -> None: @@ -45,4 +55,4 @@ def test_default_artifact_returns_last_registered() -> None: pipeline = TrainingPipeline(store) pipeline.run("first") pipeline.run("second") - assert Predictor.default_artifact(pipeline).artifact_id == "second" \ No newline at end of file + assert Predictor.default_artifact(pipeline).artifact_id == "second" diff --git a/tests/ml/test_training.py b/tests/ml/test_training.py index df926da..0defa33 100644 --- a/tests/ml/test_training.py +++ b/tests/ml/test_training.py @@ -27,6 +27,11 @@ def test_run_returns_trained_artifact() -> None: artifact = pipeline.run("artifact_v1") assert artifact.artifact_id == "artifact_v1" assert artifact.supported_sensors == ("sensor.bedroom", "sensor.kitchen") + kitchen = artifact.feature_models["sensor.kitchen"]["temperature"] + assert kitchen.sample_count == 2 + assert kitchen.mean == 19.5 + assert kitchen.slope == 1.0 + assert kitchen.forecast() == 21.0 def test_run_without_data_raises_value_error() -> None: diff --git a/tests/ml/test_training_evaluation.py b/tests/ml/test_training_evaluation.py index 9442329..9c63157 100644 --- a/tests/ml/test_training_evaluation.py +++ b/tests/ml/test_training_evaluation.py @@ -16,18 +16,18 @@ def test_end_to_end_training_then_evaluation() -> None: artifact = pipeline.run("artifact_v1") evaluator = Evaluator(pipeline) - predictions = [ - "artifact_v1:sensor.kitchen:{'temperature': 21.0}", - "artifact_v1:sensor.bedroom:{'temperature': 18.5}", + samples = [ + _vector("sensor.kitchen", 21.0), + _vector("sensor.bedroom", 18.5), ] - report = evaluator.evaluate(artifact.artifact_id, predictions) + report = evaluator.evaluate(artifact.artifact_id, samples) assert isinstance(report, EvalReport) - assert report.sample_size == len(predictions) + assert report.sample_size == len(samples) assert any(metric.name == "coverage" for metric in report.metrics) def test_metric_helpers_are_serializable() -> None: - metric = Metric(name="coverage", value=0.85, threshold=0.8) - assert metric.name == "coverage" + metric = Metric(name="mae", value=0.85, threshold=1.0) + assert metric.name == "mae" assert metric.value == 0.85 - assert metric.threshold == 0.8 \ No newline at end of file + assert metric.threshold == 1.0