| @@ -271,6 +271,7 @@ func (b *txBridge) TXStats() map[string]any { | |||||
| "faultedTransitions": s.FaultedTransitions, | "faultedTransitions": s.FaultedTransitions, | ||||
| "faultCount": s.FaultCount, | "faultCount": s.FaultCount, | ||||
| "faultHistory": s.FaultHistory, | "faultHistory": s.FaultHistory, | ||||
| "transitionHistory": s.TransitionHistory, | |||||
| "lastFault": s.LastFault, | "lastFault": s.LastFault, | ||||
| } | } | ||||
| } | } | ||||
| @@ -78,6 +78,14 @@ Live engine and driver telemetry. Only populated when TX is active. | |||||
| "severity": "faulted", | "severity": "faulted", | ||||
| "message": "queue health critical for 5 checks" | "message": "queue health critical for 5 checks" | ||||
| } | } | ||||
| ], | |||||
| "transitionHistory": [ | |||||
| { | |||||
| "time": "2026-04-06T00:00:00Z", | |||||
| "from": "running", | |||||
| "to": "degraded", | |||||
| "severity": "warn" | |||||
| } | |||||
| ] | ] | ||||
| }, | }, | ||||
| "driver": { | "driver": { | ||||
| @@ -90,9 +98,10 @@ Live engine and driver telemetry. Only populated when TX is active. | |||||
| } | } | ||||
| } | } | ||||
| ``` | ``` | ||||
| `engine.state` spiegelt jetzt die Runtime-State-Maschine wider (idle, arming, prebuffering, running, degraded, muted, faulted, stopping) und bietet eine erste beobachtbare Basis für Fault-Transitions. | `engine.state` spiegelt jetzt die Runtime-State-Maschine wider (idle, arming, prebuffering, running, degraded, muted, faulted, stopping) und bietet eine erste beobachtbare Basis für Fault-Transitions. | ||||
| `transitionHistory` liefert die jüngsten Übergänge (from/to, severity, timestamp) damit API und UI die Runtime History synchronisieren können. | |||||
| --- | --- | ||||
| ### `POST /runtime/fault/reset` | ### `POST /runtime/fault/reset` | ||||
| @@ -69,26 +69,27 @@ func durationMs(ns uint64) float64 { | |||||
| } | } | ||||
| type EngineStats struct { | type EngineStats struct { | ||||
| State string `json:"state"` | |||||
| ChunksProduced uint64 `json:"chunksProduced"` | |||||
| TotalSamples uint64 `json:"totalSamples"` | |||||
| Underruns uint64 `json:"underruns"` | |||||
| LateBuffers uint64 `json:"lateBuffers,omitempty"` | |||||
| LastError string `json:"lastError,omitempty"` | |||||
| UptimeSeconds float64 `json:"uptimeSeconds"` | |||||
| MaxCycleMs float64 `json:"maxCycleMs,omitempty"` | |||||
| MaxGenerateMs float64 `json:"maxGenerateMs,omitempty"` | |||||
| MaxUpsampleMs float64 `json:"maxUpsampleMs,omitempty"` | |||||
| MaxWriteMs float64 `json:"maxWriteMs,omitempty"` | |||||
| Queue output.QueueStats `json:"queue"` | |||||
| RuntimeIndicator RuntimeIndicator `json:"runtimeIndicator"` | |||||
| RuntimeAlert string `json:"runtimeAlert,omitempty"` | |||||
| LastFault *FaultEvent `json:"lastFault,omitempty"` | |||||
| DegradedTransitions uint64 `json:"degradedTransitions"` | |||||
| MutedTransitions uint64 `json:"mutedTransitions"` | |||||
| FaultedTransitions uint64 `json:"faultedTransitions"` | |||||
| FaultCount uint64 `json:"faultCount"` | |||||
| FaultHistory []FaultEvent `json:"faultHistory,omitempty"` | |||||
| State string `json:"state"` | |||||
| ChunksProduced uint64 `json:"chunksProduced"` | |||||
| TotalSamples uint64 `json:"totalSamples"` | |||||
| Underruns uint64 `json:"underruns"` | |||||
| LateBuffers uint64 `json:"lateBuffers,omitempty"` | |||||
| LastError string `json:"lastError,omitempty"` | |||||
| UptimeSeconds float64 `json:"uptimeSeconds"` | |||||
| MaxCycleMs float64 `json:"maxCycleMs,omitempty"` | |||||
| MaxGenerateMs float64 `json:"maxGenerateMs,omitempty"` | |||||
| MaxUpsampleMs float64 `json:"maxUpsampleMs,omitempty"` | |||||
| MaxWriteMs float64 `json:"maxWriteMs,omitempty"` | |||||
| Queue output.QueueStats `json:"queue"` | |||||
| RuntimeIndicator RuntimeIndicator `json:"runtimeIndicator"` | |||||
| RuntimeAlert string `json:"runtimeAlert,omitempty"` | |||||
| LastFault *FaultEvent `json:"lastFault,omitempty"` | |||||
| DegradedTransitions uint64 `json:"degradedTransitions"` | |||||
| MutedTransitions uint64 `json:"mutedTransitions"` | |||||
| FaultedTransitions uint64 `json:"faultedTransitions"` | |||||
| FaultCount uint64 `json:"faultCount"` | |||||
| FaultHistory []FaultEvent `json:"faultHistory,omitempty"` | |||||
| TransitionHistory []RuntimeTransition `json:"transitionHistory,omitempty"` | |||||
| } | } | ||||
| type RuntimeIndicator string | type RuntimeIndicator string | ||||
| @@ -99,14 +100,22 @@ const ( | |||||
| RuntimeIndicatorQueueCritical RuntimeIndicator = "queueCritical" | RuntimeIndicatorQueueCritical RuntimeIndicator = "queueCritical" | ||||
| ) | ) | ||||
| type RuntimeTransition struct { | |||||
| Time time.Time `json:"time"` | |||||
| From RuntimeState `json:"from"` | |||||
| To RuntimeState `json:"to"` | |||||
| Severity string `json:"severity"` | |||||
| } | |||||
| const ( | const ( | ||||
| lateBufferIndicatorWindow = 5 * time.Second | |||||
| queueCriticalStreakThreshold = 3 | |||||
| queueMutedStreakThreshold = queueCriticalStreakThreshold * 2 | |||||
| queueMutedRecoveryThreshold = queueCriticalStreakThreshold | |||||
| queueFaultedStreakThreshold = queueCriticalStreakThreshold | |||||
| faultRepeatWindow = 1 * time.Second | |||||
| faultHistoryCapacity = 8 | |||||
| lateBufferIndicatorWindow = 5 * time.Second | |||||
| queueCriticalStreakThreshold = 3 | |||||
| queueMutedStreakThreshold = queueCriticalStreakThreshold * 2 | |||||
| queueMutedRecoveryThreshold = queueCriticalStreakThreshold | |||||
| queueFaultedStreakThreshold = queueCriticalStreakThreshold | |||||
| faultRepeatWindow = 1 * time.Second | |||||
| faultHistoryCapacity = 8 | |||||
| runtimeTransitionHistoryCapacity = 8 | |||||
| ) | ) | ||||
| // Engine is the continuous TX loop. It generates composite IQ in chunks, | // Engine is the continuous TX loop. It generates composite IQ in chunks, | ||||
| @@ -146,6 +155,8 @@ type Engine struct { | |||||
| lastFault atomic.Value // *FaultEvent | lastFault atomic.Value // *FaultEvent | ||||
| faultHistoryMu sync.Mutex | faultHistoryMu sync.Mutex | ||||
| faultHistory []FaultEvent | faultHistory []FaultEvent | ||||
| transitionHistoryMu sync.Mutex | |||||
| transitionHistory []RuntimeTransition | |||||
| degradedTransitions atomic.Uint64 | degradedTransitions atomic.Uint64 | ||||
| mutedTransitions atomic.Uint64 | mutedTransitions atomic.Uint64 | ||||
| @@ -217,15 +228,16 @@ func NewEngine(cfg cfgpkg.Config, driver platform.SoapyDriver) *Engine { | |||||
| } | } | ||||
| engine := &Engine{ | engine := &Engine{ | ||||
| cfg: cfg, | |||||
| driver: driver, | |||||
| generator: offpkg.NewGenerator(cfg), | |||||
| upsampler: upsampler, | |||||
| chunkDuration: 50 * time.Millisecond, | |||||
| deviceRate: deviceRate, | |||||
| state: EngineIdle, | |||||
| frameQueue: output.NewFrameQueue(cfg.Runtime.FrameQueueCapacity), | |||||
| faultHistory: make([]FaultEvent, 0, faultHistoryCapacity), | |||||
| cfg: cfg, | |||||
| driver: driver, | |||||
| generator: offpkg.NewGenerator(cfg), | |||||
| upsampler: upsampler, | |||||
| chunkDuration: 50 * time.Millisecond, | |||||
| deviceRate: deviceRate, | |||||
| state: EngineIdle, | |||||
| frameQueue: output.NewFrameQueue(cfg.Runtime.FrameQueueCapacity), | |||||
| faultHistory: make([]FaultEvent, 0, faultHistoryCapacity), | |||||
| transitionHistory: make([]RuntimeTransition, 0, runtimeTransitionHistoryCapacity), | |||||
| } | } | ||||
| engine.setRuntimeState(RuntimeStateIdle) | engine.setRuntimeState(RuntimeStateIdle) | ||||
| return engine | return engine | ||||
| @@ -423,6 +435,7 @@ func (e *Engine) Stats() EngineStats { | |||||
| FaultedTransitions: e.faultedTransitions.Load(), | FaultedTransitions: e.faultedTransitions.Load(), | ||||
| FaultCount: e.faultEvents.Load(), | FaultCount: e.faultEvents.Load(), | ||||
| FaultHistory: e.FaultHistory(), | FaultHistory: e.FaultHistory(), | ||||
| TransitionHistory: e.TransitionHistory(), | |||||
| } | } | ||||
| } | } | ||||
| @@ -450,6 +463,19 @@ func runtimeAlert(queueHealth output.QueueHealth, recentLateBuffers bool) string | |||||
| } | } | ||||
| } | } | ||||
| func runtimeStateSeverity(state RuntimeState) string { | |||||
| switch state { | |||||
| case RuntimeStateRunning: | |||||
| return "ok" | |||||
| case RuntimeStateDegraded, RuntimeStateMuted: | |||||
| return "warn" | |||||
| case RuntimeStateFaulted: | |||||
| return "err" | |||||
| default: | |||||
| return "info" | |||||
| } | |||||
| } | |||||
| func (e *Engine) run(ctx context.Context) { | func (e *Engine) run(ctx context.Context) { | ||||
| e.setRuntimeState(RuntimeStatePrebuffering) | e.setRuntimeState(RuntimeStatePrebuffering) | ||||
| e.wg.Add(1) | e.wg.Add(1) | ||||
| @@ -589,6 +615,7 @@ func cloneFrame(src *output.CompositeFrame) *output.CompositeFrame { | |||||
| func (e *Engine) setRuntimeState(state RuntimeState) { | func (e *Engine) setRuntimeState(state RuntimeState) { | ||||
| prev := e.currentRuntimeState() | prev := e.currentRuntimeState() | ||||
| if prev != state { | if prev != state { | ||||
| e.recordRuntimeTransition(prev, state) | |||||
| switch state { | switch state { | ||||
| case RuntimeStateDegraded: | case RuntimeStateDegraded: | ||||
| e.degradedTransitions.Add(1) | e.degradedTransitions.Add(1) | ||||
| @@ -635,6 +662,31 @@ func (e *Engine) FaultHistory() []FaultEvent { | |||||
| return history | return history | ||||
| } | } | ||||
| func (e *Engine) TransitionHistory() []RuntimeTransition { | |||||
| e.transitionHistoryMu.Lock() | |||||
| defer e.transitionHistoryMu.Unlock() | |||||
| history := make([]RuntimeTransition, len(e.transitionHistory)) | |||||
| copy(history, e.transitionHistory) | |||||
| return history | |||||
| } | |||||
| func (e *Engine) recordRuntimeTransition(from, to RuntimeState) { | |||||
| ev := RuntimeTransition{ | |||||
| Time: time.Now(), | |||||
| From: from, | |||||
| To: to, | |||||
| Severity: runtimeStateSeverity(to), | |||||
| } | |||||
| e.transitionHistoryMu.Lock() | |||||
| defer e.transitionHistoryMu.Unlock() | |||||
| if len(e.transitionHistory) >= runtimeTransitionHistoryCapacity { | |||||
| copy(e.transitionHistory, e.transitionHistory[1:]) | |||||
| e.transitionHistory[len(e.transitionHistory)-1] = ev | |||||
| return | |||||
| } | |||||
| e.transitionHistory = append(e.transitionHistory, ev) | |||||
| } | |||||
| func (e *Engine) recordFault(reason FaultReason, severity FaultSeverity, message string) { | func (e *Engine) recordFault(reason FaultReason, severity FaultSeverity, message string) { | ||||
| if reason == "" { | if reason == "" { | ||||
| reason = FaultReasonUnknown | reason = FaultReasonUnknown | ||||
| @@ -180,6 +180,29 @@ func TestRuntimeTransitionCounters(t *testing.T) { | |||||
| } | } | ||||
| } | } | ||||
| func TestEngineTransitionHistory(t *testing.T) { | |||||
| e := NewEngine(cfgpkg.Default(), platform.NewSimulatedDriver(nil)) | |||||
| e.setRuntimeState(RuntimeStateRunning) | |||||
| e.setRuntimeState(RuntimeStateDegraded) | |||||
| e.setRuntimeState(RuntimeStateMuted) | |||||
| history := e.Stats().TransitionHistory | |||||
| if len(history) != 3 { | |||||
| t.Fatalf("expected 3 transitions recorded, got %d", len(history)) | |||||
| } | |||||
| if history[0].From != RuntimeStateIdle || history[0].To != RuntimeStateRunning { | |||||
| t.Fatalf("unexpected first transition: %+v", history[0]) | |||||
| } | |||||
| if history[0].Severity != "ok" { | |||||
| t.Fatalf("expected ok severity for running transition, got %s", history[0].Severity) | |||||
| } | |||||
| if history[1].To != RuntimeStateDegraded || history[1].Severity != "warn" { | |||||
| t.Fatalf("expected degraded transition with warn severity, got %+v", history[1]) | |||||
| } | |||||
| if history[2].To != RuntimeStateMuted || history[2].Severity != "warn" { | |||||
| t.Fatalf("expected muted transition with warn severity, got %+v", history[2]) | |||||
| } | |||||
| } | |||||
| func TestEngineResetFaultRequiresFaultedState(t *testing.T) { | func TestEngineResetFaultRequiresFaultedState(t *testing.T) { | ||||
| e := NewEngine(cfgpkg.Default(), platform.NewSimulatedDriver(nil)) | e := NewEngine(cfgpkg.Default(), platform.NewSimulatedDriver(nil)) | ||||
| @@ -174,6 +174,36 @@ func TestRuntimeReportsFaultHistory(t *testing.T) { | |||||
| t.Fatalf("faultHistory length mismatch: want %d got %d", len(history), len(histRaw)) | t.Fatalf("faultHistory length mismatch: want %d got %d", len(history), len(histRaw)) | ||||
| } | } | ||||
| } | } | ||||
| func TestRuntimeReportsTransitionHistory(t *testing.T) { | |||||
| srv := NewServer(cfgpkg.Default()) | |||||
| history := []map[string]any{{ | |||||
| "time": "2026-04-06T00:00:00Z", | |||||
| "from": "running", | |||||
| "to": "degraded", | |||||
| "severity": "warn", | |||||
| }} | |||||
| srv.SetTXController(&fakeTXController{stats: map[string]any{"transitionHistory": history}}) | |||||
| rec := httptest.NewRecorder() | |||||
| srv.Handler().ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/runtime", nil)) | |||||
| if rec.Code != 200 { | |||||
| t.Fatalf("status: %d", rec.Code) | |||||
| } | |||||
| var body map[string]any | |||||
| if err := json.Unmarshal(rec.Body.Bytes(), &body); err != nil { | |||||
| t.Fatalf("unmarshal runtime: %v", err) | |||||
| } | |||||
| engineRaw, ok := body["engine"].(map[string]any) | |||||
| if !ok { | |||||
| t.Fatalf("runtime engine missing") | |||||
| } | |||||
| histRaw, ok := engineRaw["transitionHistory"].([]any) | |||||
| if !ok { | |||||
| t.Fatalf("transitionHistory missing or wrong type: %T", engineRaw["transitionHistory"]) | |||||
| } | |||||
| if len(histRaw) != len(history) { | |||||
| t.Fatalf("transitionHistory length mismatch: want %d got %d", len(history), len(histRaw)) | |||||
| } | |||||
| } | |||||
| func TestRuntimeFaultResetRejectsGet(t *testing.T) { | func TestRuntimeFaultResetRejectsGet(t *testing.T) { | ||||
| srv := NewServer(cfgpkg.Default()) | srv := NewServer(cfgpkg.Default()) | ||||
| @@ -1456,7 +1456,8 @@ async function loadRuntime({ silent = true } = {}) { | |||||
| state.server.runtime = runtime; | state.server.runtime = runtime; | ||||
| state.server.runtimeOk = true; | state.server.runtimeOk = true; | ||||
| state.server.lastRuntimeAt = nowTs(); | state.server.lastRuntimeAt = nowTs(); | ||||
| notifyRuntimeTransition(runtime.engine); | |||||
| const syncedTransitions = syncTransitionHistoryFromEngine(runtime.engine); | |||||
| notifyRuntimeTransition(runtime.engine, !syncedTransitions); | |||||
| pushHistory(runtime); | pushHistory(runtime); | ||||
| setConnection(true, state.pendingRequests > 0 ? 'busy' : 'connected'); | setConnection(true, state.pendingRequests > 0 ? 'busy' : 'connected'); | ||||
| render(); | render(); | ||||
| @@ -1495,6 +1496,30 @@ function pushTransitionHistory(from, to, severity) { | |||||
| updateTransitionHistory(); | updateTransitionHistory(); | ||||
| } | } | ||||
| function transitionEntryTime(value) { | |||||
| if (value == null) return nowTs(); | |||||
| if (typeof value === 'number') return value; | |||||
| const parsed = Date.parse(String(value)); | |||||
| return Number.isNaN(parsed) ? nowTs() : parsed; | |||||
| } | |||||
| function syncTransitionHistoryFromEngine(engine) { | |||||
| const entries = Array.isArray(engine?.transitionHistory) ? engine.transitionHistory : null; | |||||
| if (!entries) return false; | |||||
| const sliceStart = Math.max(0, entries.length - transitionHistoryLimit); | |||||
| const trimmed = entries.slice(sliceStart); | |||||
| const normalized = trimmed.map((entry) => ({ | |||||
| from: normalizeRuntimeState(entry?.from), | |||||
| to: normalizeRuntimeState(entry?.to), | |||||
| severity: String(entry?.severity || 'info').toLowerCase(), | |||||
| time: transitionEntryTime(entry?.time), | |||||
| })); | |||||
| normalized.reverse(); | |||||
| state.runtimeTransitions = normalized; | |||||
| updateTransitionHistory(); | |||||
| return true; | |||||
| } | |||||
| function transitionSeverityClass(severity) { | function transitionSeverityClass(severity) { | ||||
| switch (String(severity || '').toLowerCase()) { | switch (String(severity || '').toLowerCase()) { | ||||
| case 'err': | case 'err': | ||||
| @@ -1975,20 +2000,23 @@ function runtimeStateSeverity(stateName) { | |||||
| } | } | ||||
| } | } | ||||
| function notifyRuntimeTransition(engine) { | |||||
| function notifyRuntimeTransition(engine, pushHistory = true) { | |||||
| if (!engine) return; | if (!engine) return; | ||||
| const next = normalizeRuntimeState(engine.state); | const next = normalizeRuntimeState(engine.state); | ||||
| const prev = state.lastRuntimeState; | const prev = state.lastRuntimeState; | ||||
| state.lastRuntimeState = next; | state.lastRuntimeState = next; | ||||
| if (!prev || prev === next) return; | if (!prev || prev === next) return; | ||||
| const severity = runtimeStateSeverity(next); | const severity = runtimeStateSeverity(next); | ||||
| pushTransitionHistory(prev, next, severity); | |||||
| if (pushHistory) { | |||||
| pushTransitionHistory(prev, next, severity); | |||||
| } | |||||
| const message = `Runtime ${prev.toUpperCase()} → ${next.toUpperCase()}`; | const message = `Runtime ${prev.toUpperCase()} → ${next.toUpperCase()}`; | ||||
| const logLevel = severity === 'err' ? 'err' : (severity === 'warn' ? 'warn' : 'info'); | const logLevel = severity === 'err' ? 'err' : (severity === 'warn' ? 'warn' : 'info'); | ||||
| toast(message, severity); | toast(message, severity); | ||||
| log(message, logLevel); | log(message, logLevel); | ||||
| } | } | ||||
| function updateHealth(engine, audioStream) { | function updateHealth(engine, audioStream) { | ||||
| engine = engine || {}; | engine = engine || {}; | ||||
| updateText('health-http', state.server.configOk ? 'OK' : 'OFFLINE'); | updateText('health-http', state.server.configOk ? 'OK' : 'OFFLINE'); | ||||