57 lines
1.9 KiB
Python
57 lines
1.9 KiB
Python
from __future__ import annotations
|
|
|
|
import pytest
|
|
|
|
from app.ml.evaluation import Evaluator
|
|
from app.ml.feature_store import FeatureStore, FeatureVector
|
|
from app.ml.training import TrainingPipeline
|
|
|
|
|
|
def _vector(sensor_id: str, temperature: float, label: str | None = None) -> FeatureVector:
|
|
return FeatureVector(sensor_id=sensor_id, values={"temperature": temperature}, label=label)
|
|
|
|
|
|
def evaluator_factory() -> Evaluator:
|
|
store = FeatureStore()
|
|
store.add_batch([_vector("sensor.kitchen", 19.0), _vector("sensor.bedroom", 18.5)])
|
|
pipeline = TrainingPipeline(store)
|
|
pipeline.run("artifact_v1")
|
|
return Evaluator(pipeline)
|
|
|
|
|
|
def test_evaluate_returns_report_with_metrics() -> None:
|
|
evaluator = evaluator_factory()
|
|
report = evaluator.evaluate(
|
|
"artifact_v1",
|
|
[
|
|
_vector("sensor.kitchen", 21.0),
|
|
_vector("sensor.bedroom", 18.5),
|
|
],
|
|
)
|
|
assert report.artifact_id == "artifact_v1"
|
|
assert report.sample_size == 2
|
|
assert {metric.name for metric in report.metrics} == {"mae", "rmse", "coverage"}
|
|
assert next(metric.value for metric in report.metrics if metric.name == "coverage") == 1.0
|
|
assert next(metric.value for metric in report.metrics if metric.name == "mae") == 0.0
|
|
|
|
|
|
def test_evaluate_without_training_raises_value_error() -> None:
|
|
evaluator = Evaluator(TrainingPipeline(FeatureStore()))
|
|
with pytest.raises(ValueError):
|
|
evaluator.evaluate("artifact_v1", [])
|
|
|
|
|
|
def test_coverage_counts_only_supported_sensor_features() -> None:
|
|
evaluator = evaluator_factory()
|
|
report = evaluator.evaluate(
|
|
"artifact_v1",
|
|
[
|
|
_vector("sensor.kitchen", 21.0),
|
|
FeatureVector(sensor_id="sensor.kitchen", values={"humidity": 50.0}),
|
|
_vector("sensor.kitchen_extra", 20.0),
|
|
],
|
|
)
|
|
|
|
metrics = {metric.name: metric.value for metric in report.metrics}
|
|
assert metrics["coverage"] == pytest.approx(1 / 3)
|