Add anomaly and performance monitoring
Some checks failed
quality / test (3.11) (push) Has been cancelled
quality / test (3.13) (push) Has been cancelled

This commit is contained in:
2026-06-17 18:58:32 +02:00
parent 2ec2c64cba
commit 9419a9cd8c
10 changed files with 419 additions and 15 deletions

View File

@@ -223,6 +223,16 @@ class AutomationConflict(BaseModel):
updated_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc))
class AnomalyEvent(BaseModel):
anomaly_id: str = Field(pattern=r"^[a-z0-9_.-]{1,120}$")
severity: str = Field(default="info", max_length=20)
category: str = Field(max_length=40)
title: str = Field(min_length=1, max_length=160)
detail: str = Field(min_length=1, max_length=500)
detected_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc))
resolved: bool = False
class TimeProfile(BaseModel):
profile_id: str = Field(pattern=r"^[a-z0-9_-]{1,64}$")
label: str = Field(min_length=1, max_length=80)
@@ -270,6 +280,7 @@ class BehaviorState(BaseModel):
adaptive_weight_updates: list[AdaptiveWeightUpdate] = Field(default_factory=list)
automation_conflicts: list[AutomationConflict] = Field(default_factory=list)
time_profiles: list[TimeProfile] = Field(default_factory=list)
anomalies: list[AnomalyEvent] = Field(default_factory=list)
class ActuatorRecord(BaseModel):

View File

@@ -9,7 +9,7 @@ from fastapi import APIRouter, Depends, HTTPException, Query, Request, status
from pydantic import BaseModel, Field
from app.actuators.lifecycle import ActuatorReconciliationService
from app.actuators.models import ActuatorRecord, ReconciliationState, SensorWeightGroup
from app.actuators.models import ActuatorRecord, AnomalyEvent, ReconciliationState, SensorWeightGroup
from app.actuators.models import JobQueueItem, JobQueueState, JobStatus, SafetyProfile
from app.actuators.store import ActuatorStore
from app.behavior.engine import BehaviorEngine
@@ -89,6 +89,8 @@ class ActuatorSummary(BaseModel):
activation_ready: bool
activation_reason: str
sample_count: int
anomaly_count: int = 0
critical_anomaly_count: int = 0
prediction_target_state: str | None = None
prediction_confidence: float | None = None
updated_at: str
@@ -108,6 +110,12 @@ class DashboardSystemStatus(BaseModel):
configured_actuators: int = 0
trained_models: int = 0
review_required: int = 0
performance_budget_ms: int = 3000
job_p95_duration_ms: int | None = None
slow_job_count: int = 0
performance_status: str = "unknown"
anomaly_count: int = 0
critical_anomaly_count: int = 0
class DashboardDiscoveryGroup(BaseModel):
@@ -124,6 +132,12 @@ class DashboardOverview(BaseModel):
jobs: JobQueueState = Field(default_factory=JobQueueState)
class AnomalyOverview(BaseModel):
actuator_entity_id: str
friendly_name: str | None = None
anomalies: list[AnomalyEvent] = Field(default_factory=list)
@router.get("/discovery", response_model=list[HaEntitySummary])
def discover_actuators(
request: Request,
@@ -266,6 +280,14 @@ def list_configured_summary(request: Request) -> list[ActuatorSummary]:
activation_ready=record.behavior.activation_ready,
activation_reason=record.behavior.activation_reason,
sample_count=record.behavior.sample_count,
anomaly_count=len([item for item in record.behavior.anomalies if not item.resolved]),
critical_anomaly_count=len(
[
item
for item in record.behavior.anomalies
if not item.resolved and item.severity == "critical"
]
),
prediction_target_state=(
record.behavior.prediction.target_state
if record.behavior.prediction is not None
@@ -305,6 +327,9 @@ def dashboard_overview(request: Request) -> DashboardOverview:
if isinstance(store, ActuatorStore)
else JobQueueState()
)
job_p95_duration_ms, slow_job_count, performance_status = _performance_status(jobs)
anomaly_count = sum(record.anomaly_count for record in actuators)
critical_anomaly_count = sum(record.critical_anomaly_count for record in actuators)
return DashboardOverview(
system=DashboardSystemStatus(
websocket_status=getattr(ws_status, "status", "unavailable"),
@@ -317,6 +342,11 @@ def dashboard_overview(request: Request) -> DashboardOverview:
configured_actuators=len(actuators),
trained_models=reconciliation.trained_models,
review_required=reconciliation.review_required,
job_p95_duration_ms=job_p95_duration_ms,
slow_job_count=slow_job_count,
performance_status=performance_status,
anomaly_count=anomaly_count,
critical_anomaly_count=critical_anomaly_count,
),
cache=EntityCacheStatus(
available=bool(raw_entities),
@@ -329,6 +359,29 @@ def dashboard_overview(request: Request) -> DashboardOverview:
)
@router.get("/anomalies", response_model=list[AnomalyOverview])
def list_anomalies(request: Request) -> list[AnomalyOverview]:
records = _service(request).list_configured()
entity_map = _load_cached_entity_map(
request,
{record.actuator_entity_id for record in records},
)
overview: list[AnomalyOverview] = []
for record in records:
active = [item for item in record.behavior.anomalies if not item.resolved]
if not active:
continue
entity = entity_map.get(record.actuator_entity_id)
overview.append(
AnomalyOverview(
actuator_entity_id=record.actuator_entity_id,
friendly_name=entity.friendly_name if entity is not None else None,
anomalies=active,
)
)
return overview
@router.get("", response_model=list[ActuatorRecord])
def list_configured(request: Request) -> list[ActuatorRecord]:
return _service(request).list_configured()
@@ -635,6 +688,26 @@ def _finish_job(
store.finish_job(job.job_id, status=status, summary=summary, error=error)
def _performance_status(jobs: JobQueueState) -> tuple[int | None, int, str]:
budget_ms = 3000
durations = sorted(
job.duration_ms
for job in jobs.jobs
if job.status is JobStatus.COMPLETED and job.duration_ms is not None
)
slow_count = sum(1 for duration in durations if duration >= budget_ms)
if durations:
index = min(len(durations) - 1, int(round((len(durations) - 1) * 0.95)))
p95: int | None = durations[index]
status_value = "slow" if slow_count else "ok"
else:
p95 = None
status_value = "unknown"
if any(job.status is JobStatus.RUNNING for job in jobs.jobs):
status_value = "running" if status_value == "unknown" else status_value
return p95, slow_count, status_value
def _service(request: Request) -> ActuatorReconciliationService:
service = getattr(request.app.state, "actuator_service", None)
if not isinstance(service, ActuatorReconciliationService):

View File

@@ -8,6 +8,7 @@ from zoneinfo import ZoneInfo
from app.actuators.models import (
ActuatorRecord,
AdaptiveWeightUpdate,
AnomalyEvent,
AutomationConflict,
BehaviorMode,
BehaviorPattern,
@@ -88,6 +89,16 @@ class BehaviorEngine:
),
"last_trained_at": now,
"reason": "Noch kein geeigneter Kontext für Verhaltenslernen vorhanden.",
"anomalies": _detect_anomalies(
record,
now=now,
min_behavior_actions=self._settings.min_behavior_actions,
stale_hours=self._settings.retrain_stale_hours,
sample_count=0,
trusted_actions=0,
prediction=None,
safety_blockers=[],
),
}
),
)
@@ -128,6 +139,16 @@ class BehaviorEngine:
"patterns": [],
"last_trained_at": now,
"reason": "Noch keine historischen Aktorhandlungen gefunden.",
"anomalies": _detect_anomalies(
record,
now=now,
min_behavior_actions=self._settings.min_behavior_actions,
stale_hours=self._settings.retrain_stale_hours,
sample_count=0,
trusted_actions=0,
prediction=None,
safety_blockers=[],
),
}
),
)
@@ -200,6 +221,16 @@ class BehaviorEngine:
reason,
),
"active_model_version": model_version_id,
"anomalies": _detect_anomalies(
record,
now=now,
min_behavior_actions=self._settings.min_behavior_actions,
stale_hours=self._settings.retrain_stale_hours,
sample_count=len(patterns),
trusted_actions=trusted_actions,
prediction=record.behavior.prediction,
safety_blockers=record.behavior.safety_blockers,
),
}
)
return self._save_behavior(record, behavior)
@@ -326,6 +357,16 @@ class BehaviorEngine:
"assumptions": _assumption_lines(record),
"uncertainties": _uncertainty_lines(record, record.behavior.sample_count, record.behavior.high_confidence_sample_count),
"safety_blockers": safety_blockers if prediction is not None else [],
"anomalies": _detect_anomalies(
record,
now=now,
min_behavior_actions=self._settings.min_behavior_actions,
stale_hours=self._settings.retrain_stale_hours,
sample_count=record.behavior.sample_count,
trusted_actions=record.behavior.high_confidence_sample_count,
prediction=prediction,
safety_blockers=safety_blockers if prediction is not None else [],
),
"confidence_trend": (
[*record.behavior.confidence_trend, round(prediction.confidence, 4)][-30:]
if prediction is not None
@@ -493,6 +534,18 @@ class BehaviorEngine:
*record.behavior.adaptive_weight_updates,
*adaptive_updates,
][-50:],
"anomalies": _detect_anomalies(
record,
now=now,
min_behavior_actions=self._settings.min_behavior_actions,
stale_hours=self._settings.retrain_stale_hours,
sample_count=len(patterns),
trusted_actions=record.behavior.high_confidence_sample_count,
prediction=prediction,
safety_blockers=record.behavior.safety_blockers,
correct_feedback_count=correct_count,
incorrect_feedback_count=incorrect_count,
),
}
)
record_for_save = (
@@ -560,6 +613,20 @@ class BehaviorEngine:
"automation_conflicts": _automation_conflicts(record, related),
}
)
behavior = behavior.model_copy(
update={
"anomalies": _detect_anomalies(
record.model_copy(update={"behavior": behavior}),
now=datetime.now(timezone.utc),
min_behavior_actions=self._settings.min_behavior_actions,
stale_hours=self._settings.retrain_stale_hours,
sample_count=behavior.sample_count,
trusted_actions=behavior.high_confidence_sample_count,
prediction=behavior.prediction,
safety_blockers=behavior.safety_blockers,
)
}
)
return self._save_behavior(record, behavior)
def set_automation_enabled(
@@ -1208,6 +1275,115 @@ def _automation_conflicts(
return conflicts
def _detect_anomalies(
record: ActuatorRecord,
*,
now: datetime,
min_behavior_actions: int,
stale_hours: int,
sample_count: int,
trusted_actions: int,
prediction: BehaviorPrediction | None,
safety_blockers: list[str],
correct_feedback_count: int | None = None,
incorrect_feedback_count: int | None = None,
) -> list[AnomalyEvent]:
anomalies: list[AnomalyEvent] = []
def add(category: str, severity: str, title: str, detail: str) -> None:
anomalies.append(
AnomalyEvent(
anomaly_id=f"{record.actuator_entity_id}.{category}",
category=category,
severity=severity,
title=title,
detail=detail,
detected_at=now,
)
)
if not record.assignment.selected_context_entity_ids and not record.assignment.selected_numeric_entity_id:
add(
"missing_context",
"warning",
"Kein Kontext verbunden",
"Der Aktor hat keine Sensor-/Kontextbasis. Entscheidungen bleiben unsicher.",
)
if sample_count < min_behavior_actions:
add(
"low_samples",
"info",
"Zu wenig Lernbeispiele",
f"{sample_count} von {min_behavior_actions} benoetigten Handlungen gelernt.",
)
if trusted_actions < sample_count:
add(
"unclear_sources",
"info",
"Unklare Aktorhandlungen",
"Ein Teil der gelernten Handlungen stammt nicht eindeutig von Nutzer oder Automation.",
)
if record.behavior.last_trained_at is not None:
age = now - record.behavior.last_trained_at
if age > timedelta(hours=stale_hours):
add(
"stale_training",
"warning",
"Training ist veraltet",
f"Letztes Training liegt mehr als {stale_hours} Stunden zurueck.",
)
if prediction is not None and prediction.matching_patterns and prediction.confidence < record.behavior.safety.min_confidence:
add(
"low_confidence_prediction",
"warning",
"Vorhersage unter Sicherheitsgrenze",
(
f"Confidence {prediction.confidence:.0%} liegt unter "
f"{record.behavior.safety.min_confidence:.0%}."
),
)
if record.behavior.safety.manual_block:
add(
"manual_block",
"info",
"Manuelle Sicherheitssperre aktiv",
"Der Aktor ist bewusst gegen automatisches Schalten gesperrt.",
)
if safety_blockers:
add(
"safety_blockers",
"info",
"Safety blockiert aktuelle Aktion",
" ".join(safety_blockers)[:500],
)
if any(conflict.severity == "warning" for conflict in record.behavior.automation_conflicts):
add(
"automation_conflict",
"critical",
"Parallele Automation erkannt",
"SillyHome und mindestens eine passende HA-Automation koennen parallel schalten.",
)
correct = (
record.behavior.correct_feedback_count
if correct_feedback_count is None
else correct_feedback_count
)
incorrect = (
record.behavior.incorrect_feedback_count
if incorrect_feedback_count is None
else incorrect_feedback_count
)
total = correct + incorrect
if total >= 3 and incorrect / total >= 0.35:
add(
"feedback_error_rate",
"critical",
"Viele falsche Vorhersagen",
f"{incorrect} von {total} Feedbacks waren negativ. Modell pruefen oder Rollback nutzen.",
)
return anomalies[-30:]
def predict_behavior(
patterns: list[BehaviorPattern],
*,

View File

@@ -105,7 +105,7 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]:
app = FastAPI(
title="SillyHome Next API",
description="Lokales Smart-Home-Intelligenzsystem für Home Assistant.",
version="1.2.0",
version="1.3.0",
lifespan=lifespan,
)
app.state.settings = load_settings()

View File

@@ -99,6 +99,8 @@
.metric strong { display:block; margin-bottom:4px; color:#cfe0ec; font-size:.84rem; }
.decision-list { display:grid; gap:8px; margin:10px 0; }
.decision-row { background:#121922; border:1px solid var(--border); border-radius:8px; padding:9px; min-width:0; overflow-wrap:anywhere; }
.decision-row.slow,
.decision-row.critical { border-color:var(--warn); box-shadow:0 0 0 1px rgba(243,201,105,.25); }
.decision-row header { padding:0; border:0; background:transparent; display:flex; justify-content:space-between; gap:10px; flex-wrap:wrap; }
.actions { display:flex; flex-wrap:wrap; gap:8px; margin-top:10px; }
.actions button { flex:1 1 180px; margin-top:0; }
@@ -283,7 +285,7 @@ let discoveryLoadPromise = null;
let currentSensorWeightGroups = [];
const ACTUATOR_RESULT_LIMIT = 50;
const STATUS_TIMEOUT_MS = 2000;
const DASHBOARD_TIMEOUT_MS = 4500;
const DASHBOARD_TIMEOUT_MS = 3000;
function jumpToSection(target) {
if (!target) return;
@@ -439,11 +441,17 @@ async function loadOverview() {
document.getElementById("configured-actuators").innerHTML = "<p class='muted'>Beobachtete Geräte werden geladen ...</p>";
try {
const dashboard = await apiWithTimeout("v1/actuators/dashboard", DASHBOARD_TIMEOUT_MS);
dashboard._load_elapsed_ms = Math.round(performance.now() - startedAt);
cachedActuators = dashboard.actuators || [];
cachedEntities = [];
renderDashboardStatus(dashboard);
renderConfiguredActuators();
if (budget) budget.textContent = `Bereit in ${Math.round(performance.now() - startedAt)} ms`;
if (budget) {
const loadMs = dashboard._load_elapsed_ms;
budget.textContent = loadMs <= DASHBOARD_TIMEOUT_MS
? `Bereit in ${loadMs} ms`
: `Langsam: ${loadMs} ms`;
}
} catch (error) {
document.getElementById("configured-actuators").innerHTML = `<p class="bad">${escapeHtml(error.message)}</p>`;
if (budget) budget.textContent = "Startdaten verzögert";
@@ -504,15 +512,26 @@ function renderDashboardStatus(dashboard) {
).length;
const trainedCount = actuators.filter(record => record.behavior_status === "trained").length;
const sampleTotal = actuators.reduce((sum, record) => sum + Number(record.sample_count || 0), 0);
const anomalyTotal = Number(system.anomaly_count || 0);
const criticalAnomalyTotal = Number(system.critical_anomaly_count || 0);
const loadMs = Number(dashboard._load_elapsed_ms || 0);
const jobs = dashboard.jobs?.jobs || [];
const runningJobs = jobs.filter(job => job.status === "running").length;
const slowJobs = Number(system.slow_job_count || 0);
const p95 = system.job_p95_duration_ms == null ? "offen" : `${system.job_p95_duration_ms} ms`;
const performanceClass = (
loadMs > DASHBOARD_TIMEOUT_MS
|| slowJobs > 0
|| system.performance_status === "slow"
) ? "warn" : "ok";
const cacheLabel = cache.available
? `Cache aktuell mit ${cache.entity_count} Entities`
: "Cache wird nach Discovery aufgebaut";
status.innerHTML = `
<p class="${system.websocket_status === "connected" ? "ok" : "warn"}">
Dashboard bereit. WebSocket: ${escapeHtml(system.websocket_status || "unbekannt")}
<p class="${performanceClass}">
Dashboard bereit in ${escapeHtml(loadMs || "offen")} ms. Budget: ${escapeHtml(system.performance_budget_ms || DASHBOARD_TIMEOUT_MS)} ms.
</p>
<p class="${system.websocket_status === "connected" ? "ok" : "warn"}">WebSocket: ${escapeHtml(system.websocket_status || "unbekannt")}</p>
<p class="muted">Letzte automatische Prüfung: ${escapeHtml(system.reconciliation_last_completed_at || "noch nicht abgeschlossen")}</p>
`;
chips.innerHTML = [
@@ -522,6 +541,8 @@ function renderDashboardStatus(dashboard) {
`<span class="chip">Lernbereit: ${escapeHtml(system.trained_models ?? 0)}</span>`,
`<span class="chip">Prüfen: ${escapeHtml(system.review_required ?? 0)}</span>`,
`<span class="chip">Jobs aktiv: ${escapeHtml(runningJobs)}</span>`,
`<span class="chip">Anomalien: ${escapeHtml(anomalyTotal)}</span>`,
`<span class="chip">Kritisch: ${escapeHtml(criticalAnomalyTotal)}</span>`,
].join("");
stats.innerHTML = [
`<div class="metric"><strong>Geladene Startdaten</strong>${escapeHtml(actuators.length)} Geräte</div>`,
@@ -529,16 +550,20 @@ function renderDashboardStatus(dashboard) {
`<div class="metric"><strong>Aktiv / Shadow</strong>${escapeHtml(activeCount)} / ${escapeHtml(shadowCount)}</div>`,
`<div class="metric"><strong>Gelernt / Wartet</strong>${escapeHtml(trainedCount)} / ${escapeHtml(pendingCount)}</div>`,
`<div class="metric"><strong>Gelernte Handlungen</strong>${escapeHtml(sampleTotal)}</div>`,
`<div class="metric"><strong>Performance-Budget</strong>${escapeHtml(system.performance_budget_ms || 3000)} ms</div>`,
`<div class="metric"><strong>Job p95</strong>${escapeHtml(p95)}</div>`,
`<div class="metric"><strong>Langsame Jobs</strong>${escapeHtml(slowJobs)}</div>`,
`<div class="metric"><strong>Anomalien</strong>${escapeHtml(anomalyTotal)} offen</div>`,
`<div class="metric"><strong>Discovery-Gruppen</strong>${escapeHtml(discoveryGroups.length)} Kategorien</div>`,
`<div class="metric"><strong>Cache-Zeitpunkt</strong>${escapeHtml(cache.updated_at || "noch offen")}</div>`,
].join("");
jobsBox.innerHTML = jobs.length ? `
<h3>Job-Queue</h3>
${jobs.slice(-6).reverse().map(job => `
<div class="decision-row">
<div class="decision-row ${Number(job.duration_ms || 0) >= DASHBOARD_TIMEOUT_MS ? "slow" : ""}">
<header>
<strong>${escapeHtml(job.kind)}${job.target ? `: ${escapeHtml(job.target)}` : ""}</strong>
<span class="chip">${escapeHtml(job.status)}</span>
<span class="chip">${escapeHtml(job.status)}${Number(job.duration_ms || 0) >= DASHBOARD_TIMEOUT_MS ? " · langsam" : ""}</span>
</header>
<p class="muted">${escapeHtml(job.summary || "Keine Zusammenfassung")}</p>
<p class="muted">Start: ${escapeHtml(job.started_at || "offen")} · Dauer: ${escapeHtml(job.duration_ms == null ? "läuft/offen" : `${job.duration_ms} ms`)}</p>
@@ -908,6 +933,7 @@ async function showActuator(actuatorId, evaluationMessage = "") {
const adaptiveUpdates = record.behavior.adaptive_weight_updates || [];
const automationConflicts = record.behavior.automation_conflicts || [];
const timeProfiles = record.behavior.time_profiles || [];
const anomalies = (record.behavior.anomalies || []).filter(item => !item.resolved);
const safetyControls = `
<details class="manual-context" open>
<summary>Sicherheit und manuelles Gegensteuern</summary>
@@ -1004,6 +1030,23 @@ async function showActuator(actuatorId, evaluationMessage = "") {
`).join("") : "<li>Keine aktiven Automation-Konflikte erkannt.</li>"}</ul>
</details>
`;
const anomalyPanel = `
<details class="manual-context" ${anomalies.length ? "open" : ""}>
<summary>v1.3 Anomalie- und Performance-Hinweise</summary>
<div class="decision-list">
${anomalies.length ? anomalies.map(anomaly => `
<div class="decision-row ${anomaly.severity === "critical" ? "critical" : ""}">
<header>
<strong>${escapeHtml(anomaly.title)}</strong>
<span class="chip">${escapeHtml(anomaly.severity)} · ${escapeHtml(anomaly.category)}</span>
</header>
<p>${escapeHtml(anomaly.detail)}</p>
<p class="muted">Erkannt: ${escapeHtml(anomaly.detected_at || "offen")}</p>
</div>
`).join("") : "<p class='ok'>Keine offenen Anomalien fuer diesen Aktor.</p>"}
</div>
</details>
`;
const learnedAutomationActions = record.behavior.patterns.filter(
pattern => pattern.source === "automation",
).length;
@@ -1116,6 +1159,7 @@ async function showActuator(actuatorId, evaluationMessage = "") {
${safetyControls}
${decisionArchive}
${adaptivePanel}
${anomalyPanel}
<h3>Passende Home-Assistant-Automationen</h3>
<p class="muted">Bei einer Übernahme pausiert SillyHome diese Automationen. Beim Stoppen können sie gezielt fortgesetzt werden.</p>
<button class="secondary compact" onclick="refreshRelatedAutomations('${escapeHtml(record.actuator_entity_id)}')">Automationen neu suchen</button>