90 lines
2.9 KiB
Python
90 lines
2.9 KiB
Python
from __future__ import annotations
|
|
|
|
import logging
|
|
import math
|
|
from collections.abc import Sequence
|
|
from dataclasses import dataclass
|
|
|
|
from app.ml.feature_store import FeatureVector
|
|
from app.ml.predictor import Predictor
|
|
from app.ml.registry.model_registry import ModelRegistry
|
|
from app.ml.training import TrainingPipeline
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
@dataclass
|
|
class Metric:
|
|
name: str
|
|
value: float
|
|
threshold: float | None = None
|
|
|
|
|
|
@dataclass
|
|
class EvalReport:
|
|
artifact_id: str
|
|
sample_size: int
|
|
metrics: list[Metric]
|
|
|
|
|
|
class Evaluator:
|
|
def __init__(
|
|
self,
|
|
pipeline: TrainingPipeline | None = None,
|
|
registry: ModelRegistry | None = None,
|
|
) -> None:
|
|
if isinstance(pipeline, ModelRegistry) and registry is None:
|
|
registry = pipeline
|
|
pipeline = None
|
|
if pipeline is None and registry is None:
|
|
raise ValueError("Evaluator erfordert TrainingPipeline oder ModelRegistry.")
|
|
self._pipeline = pipeline
|
|
self._registry = registry
|
|
self._predictor = Predictor(pipeline=pipeline, registry=registry)
|
|
|
|
def evaluate(self, artifact_id: str, samples: Sequence[FeatureVector]) -> EvalReport:
|
|
try:
|
|
if self._registry is not None:
|
|
self._registry.load_artifact(artifact_id)
|
|
elif self._pipeline is not None:
|
|
self._pipeline.export(artifact_id)
|
|
except KeyError as exc:
|
|
raise ValueError("Kein trainiertes Modell für Evaluation vorhanden.") from exc
|
|
|
|
absolute_errors: list[float] = []
|
|
squared_errors: list[float] = []
|
|
for sample in samples:
|
|
try:
|
|
prediction = self._predictor.predict(artifact_id, sample)
|
|
except ValueError:
|
|
continue
|
|
for feature_name, predicted in prediction.predictions.items():
|
|
actual = float(sample.values[feature_name])
|
|
error = predicted - actual
|
|
absolute_errors.append(abs(error))
|
|
squared_errors.append(error**2)
|
|
|
|
sample_size = len(absolute_errors)
|
|
mae = sum(absolute_errors) / sample_size if sample_size else 0.0
|
|
rmse = math.sqrt(sum(squared_errors) / sample_size) if sample_size else 0.0
|
|
expected_values = sum(len(sample.values) for sample in samples)
|
|
coverage = sample_size / expected_values if expected_values else 0.0
|
|
|
|
report = EvalReport(
|
|
artifact_id=artifact_id,
|
|
sample_size=sample_size,
|
|
metrics=[
|
|
Metric(name="mae", value=mae),
|
|
Metric(name="rmse", value=rmse),
|
|
Metric(name="coverage", value=coverage, threshold=0.8),
|
|
],
|
|
)
|
|
logger.info(
|
|
"Evaluation %s -> mae=%.4f, rmse=%.4f, coverage=%.2f",
|
|
artifact_id,
|
|
mae,
|
|
rmse,
|
|
coverage,
|
|
)
|
|
return report
|