57 lines
1.8 KiB
Python
57 lines
1.8 KiB
Python
from __future__ import annotations
|
|
|
|
import logging
|
|
from dataclasses import dataclass, field
|
|
from typing import Sequence
|
|
|
|
from app.ml.feature_store import FeatureVector
|
|
from app.ml.training import TrainingPipeline, TrainedArtifact
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
@dataclass
|
|
class Metric:
|
|
name: str
|
|
value: float
|
|
threshold: float | None = None
|
|
|
|
|
|
@dataclass
|
|
class EvalReport:
|
|
artifact_id: str
|
|
sample_size: int
|
|
metrics: list[Metric]
|
|
|
|
|
|
class Evaluator:
|
|
def __init__(self, pipeline: TrainingPipeline) -> None:
|
|
self._pipeline = pipeline
|
|
|
|
def evaluate(self, artifact_id: str, predictions: Sequence[str]) -> EvalReport:
|
|
artifacts = list(self._pipeline._artifacts)
|
|
if not artifacts:
|
|
raise ValueError("Kein trainiertes Modell für Evaluation vorhanden.")
|
|
|
|
supported_sensors = self._pipeline.export(artifact_id).supported_sensors
|
|
unknown_hits = sum(1 for prediction in predictions if ":" not in prediction)
|
|
supported_references = sum(1 for sensor in supported_sensors for prediction in predictions if sensor in prediction)
|
|
sample_size = len(predictions)
|
|
coverage = supported_references / sample_size if sample_size else 0.0
|
|
unknown_rate = unknown_hits / sample_size if sample_size else 0.0
|
|
|
|
coverage_metric = Metric(name="coverage", value=coverage, threshold=0.8)
|
|
unknown_metric = Metric(name="unknown_rate", value=unknown_rate, threshold=0.1)
|
|
|
|
report = EvalReport(
|
|
artifact_id=artifact_id,
|
|
sample_size=sample_size,
|
|
metrics=[coverage_metric, unknown_metric],
|
|
)
|
|
logger.info(
|
|
"Evaluation %s -> coverage=%.2f, unknown_rate=%.2f",
|
|
artifact_id,
|
|
coverage,
|
|
unknown_rate,
|
|
)
|
|
return report |