harden model registry persistence and evaluation
This commit is contained in:
@@ -21,13 +21,35 @@ def evaluator_factory() -> Evaluator:
|
||||
|
||||
def test_evaluate_returns_report_with_metrics() -> None:
|
||||
evaluator = evaluator_factory()
|
||||
report = evaluator.evaluate("artifact_v1", ["artifact_v1:sensor.kitchen:{'temperature': 21.0}", "artifact_v1:sensor.bedroom:{'temperature': 18.5}"])
|
||||
report = evaluator.evaluate(
|
||||
"artifact_v1",
|
||||
[
|
||||
"artifact_v1:sensor.kitchen:{'temperature': 21.0}",
|
||||
"artifact_v1:sensor.bedroom:{'temperature': 18.5}",
|
||||
],
|
||||
)
|
||||
assert report.artifact_id == "artifact_v1"
|
||||
assert report.sample_size == 2
|
||||
assert {metric.name for metric in report.metrics} == {"coverage", "unknown_rate"}
|
||||
assert next(metric.value for metric in report.metrics if metric.name == "coverage") == 1.0
|
||||
|
||||
|
||||
def test_evaluate_without_training_raises_value_error() -> None:
|
||||
evaluator = Evaluator(TrainingPipeline(FeatureStore()))
|
||||
with pytest.raises(ValueError):
|
||||
evaluator.evaluate("artifact_v1", [])
|
||||
evaluator.evaluate("artifact_v1", [])
|
||||
|
||||
|
||||
def test_coverage_is_bounded_and_requires_exact_sensor_match() -> None:
|
||||
evaluator = evaluator_factory()
|
||||
report = evaluator.evaluate(
|
||||
"artifact_v1",
|
||||
[
|
||||
"artifact_v1:sensor.kitchen:{'note': 'sensor.bedroom'}",
|
||||
"artifact_v1:sensor.kitchen_extra:{}",
|
||||
"malformed",
|
||||
],
|
||||
)
|
||||
|
||||
metrics = {metric.name: metric.value for metric in report.metrics}
|
||||
assert metrics == {"coverage": pytest.approx(1 / 3), "unknown_rate": pytest.approx(2 / 3)}
|
||||
|
||||
38
tests/ml/test_model_registry.py
Normal file
38
tests/ml/test_model_registry.py
Normal file
@@ -0,0 +1,38 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
from app.ml.registry.model_registry import ModelRegistry
|
||||
from app.ml.training import TrainedArtifact
|
||||
|
||||
|
||||
def test_registry_loads_persisted_artifacts_after_restart(tmp_path) -> None:
|
||||
registry = ModelRegistry(tmp_path)
|
||||
artifact = TrainedArtifact("model-v1", ("sensor.kitchen", "sensor.bedroom"))
|
||||
registry.register(artifact)
|
||||
|
||||
restarted = ModelRegistry(tmp_path)
|
||||
|
||||
assert restarted.load_artifact("model-v1") == artifact
|
||||
|
||||
|
||||
@pytest.mark.parametrize("artifact_id", ["../escape", "nested/model", "..", ""])
|
||||
def test_registry_rejects_unsafe_artifact_ids(tmp_path, artifact_id: str) -> None:
|
||||
registry = ModelRegistry(tmp_path)
|
||||
|
||||
with pytest.raises(ValueError):
|
||||
registry.register(TrainedArtifact(artifact_id, ("sensor.kitchen",)))
|
||||
|
||||
assert list(tmp_path.parent.glob("escape.json")) == []
|
||||
|
||||
|
||||
def test_registry_rejects_corrupt_persisted_artifact(tmp_path) -> None:
|
||||
(tmp_path / "broken.json").write_text(
|
||||
json.dumps({"artifact_id": "../broken", "supported_sensors": []}),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="broken.json"):
|
||||
ModelRegistry(tmp_path)
|
||||
Reference in New Issue
Block a user