harden model registry persistence and evaluation

This commit is contained in:
2026-06-11 21:06:39 +02:00
parent 3bed5e790a
commit 471146761e
4 changed files with 129 additions and 19 deletions

View File

@@ -21,13 +21,35 @@ def evaluator_factory() -> Evaluator:
def test_evaluate_returns_report_with_metrics() -> None:
evaluator = evaluator_factory()
report = evaluator.evaluate("artifact_v1", ["artifact_v1:sensor.kitchen:{'temperature': 21.0}", "artifact_v1:sensor.bedroom:{'temperature': 18.5}"])
report = evaluator.evaluate(
"artifact_v1",
[
"artifact_v1:sensor.kitchen:{'temperature': 21.0}",
"artifact_v1:sensor.bedroom:{'temperature': 18.5}",
],
)
assert report.artifact_id == "artifact_v1"
assert report.sample_size == 2
assert {metric.name for metric in report.metrics} == {"coverage", "unknown_rate"}
assert next(metric.value for metric in report.metrics if metric.name == "coverage") == 1.0
def test_evaluate_without_training_raises_value_error() -> None:
evaluator = Evaluator(TrainingPipeline(FeatureStore()))
with pytest.raises(ValueError):
evaluator.evaluate("artifact_v1", [])
evaluator.evaluate("artifact_v1", [])
def test_coverage_is_bounded_and_requires_exact_sensor_match() -> None:
evaluator = evaluator_factory()
report = evaluator.evaluate(
"artifact_v1",
[
"artifact_v1:sensor.kitchen:{'note': 'sensor.bedroom'}",
"artifact_v1:sensor.kitchen_extra:{}",
"malformed",
],
)
metrics = {metric.name: metric.value for metric in report.metrics}
assert metrics == {"coverage": pytest.approx(1 / 3), "unknown_rate": pytest.approx(2 / 3)}

View File

@@ -0,0 +1,38 @@
from __future__ import annotations
import json
import pytest
from app.ml.registry.model_registry import ModelRegistry
from app.ml.training import TrainedArtifact
def test_registry_loads_persisted_artifacts_after_restart(tmp_path) -> None:
registry = ModelRegistry(tmp_path)
artifact = TrainedArtifact("model-v1", ("sensor.kitchen", "sensor.bedroom"))
registry.register(artifact)
restarted = ModelRegistry(tmp_path)
assert restarted.load_artifact("model-v1") == artifact
@pytest.mark.parametrize("artifact_id", ["../escape", "nested/model", "..", ""])
def test_registry_rejects_unsafe_artifact_ids(tmp_path, artifact_id: str) -> None:
registry = ModelRegistry(tmp_path)
with pytest.raises(ValueError):
registry.register(TrainedArtifact(artifact_id, ("sensor.kitchen",)))
assert list(tmp_path.parent.glob("escape.json")) == []
def test_registry_rejects_corrupt_persisted_artifact(tmp_path) -> None:
(tmp_path / "broken.json").write_text(
json.dumps({"artifact_id": "../broken", "supported_sensors": []}),
encoding="utf-8",
)
with pytest.raises(ValueError, match="broken.json"):
ModelRegistry(tmp_path)