ソースを参照

Expose runtime transition history

tags/v0.9.0
Jan Svabenik 1ヶ月前
コミット
a7549f4187
6個のファイルの変更183行の追加40行の削除
  1. +1
    -0
      cmd/fmrtx/main.go
  2. +10
    -1
      docs/API.md
  3. +88
    -36
      internal/app/engine.go
  4. +23
    -0
      internal/app/runtime_state_test.go
  5. +30
    -0
      internal/control/control_test.go
  6. +31
    -3
      internal/control/ui.html

+ 1
- 0
cmd/fmrtx/main.go ファイルの表示

@@ -271,6 +271,7 @@ func (b *txBridge) TXStats() map[string]any {
"faultedTransitions": s.FaultedTransitions, "faultedTransitions": s.FaultedTransitions,
"faultCount": s.FaultCount, "faultCount": s.FaultCount,
"faultHistory": s.FaultHistory, "faultHistory": s.FaultHistory,
"transitionHistory": s.TransitionHistory,
"lastFault": s.LastFault, "lastFault": s.LastFault,
} }
} }


+ 10
- 1
docs/API.md ファイルの表示

@@ -78,6 +78,14 @@ Live engine and driver telemetry. Only populated when TX is active.
"severity": "faulted", "severity": "faulted",
"message": "queue health critical for 5 checks" "message": "queue health critical for 5 checks"
} }
],
"transitionHistory": [
{
"time": "2026-04-06T00:00:00Z",
"from": "running",
"to": "degraded",
"severity": "warn"
}
] ]
}, },
"driver": { "driver": {
@@ -90,9 +98,10 @@ Live engine and driver telemetry. Only populated when TX is active.
} }
} }
``` ```

`engine.state` spiegelt jetzt die Runtime-State-Maschine wider (idle, arming, prebuffering, running, degraded, muted, faulted, stopping) und bietet eine erste beobachtbare Basis für Fault-Transitions. `engine.state` spiegelt jetzt die Runtime-State-Maschine wider (idle, arming, prebuffering, running, degraded, muted, faulted, stopping) und bietet eine erste beobachtbare Basis für Fault-Transitions.


`transitionHistory` liefert die jüngsten Übergänge (from/to, severity, timestamp) damit API und UI die Runtime History synchronisieren können.

--- ---


### `POST /runtime/fault/reset` ### `POST /runtime/fault/reset`


+ 88
- 36
internal/app/engine.go ファイルの表示

@@ -69,26 +69,27 @@ func durationMs(ns uint64) float64 {
} }


type EngineStats struct { type EngineStats struct {
State string `json:"state"`
ChunksProduced uint64 `json:"chunksProduced"`
TotalSamples uint64 `json:"totalSamples"`
Underruns uint64 `json:"underruns"`
LateBuffers uint64 `json:"lateBuffers,omitempty"`
LastError string `json:"lastError,omitempty"`
UptimeSeconds float64 `json:"uptimeSeconds"`
MaxCycleMs float64 `json:"maxCycleMs,omitempty"`
MaxGenerateMs float64 `json:"maxGenerateMs,omitempty"`
MaxUpsampleMs float64 `json:"maxUpsampleMs,omitempty"`
MaxWriteMs float64 `json:"maxWriteMs,omitempty"`
Queue output.QueueStats `json:"queue"`
RuntimeIndicator RuntimeIndicator `json:"runtimeIndicator"`
RuntimeAlert string `json:"runtimeAlert,omitempty"`
LastFault *FaultEvent `json:"lastFault,omitempty"`
DegradedTransitions uint64 `json:"degradedTransitions"`
MutedTransitions uint64 `json:"mutedTransitions"`
FaultedTransitions uint64 `json:"faultedTransitions"`
FaultCount uint64 `json:"faultCount"`
FaultHistory []FaultEvent `json:"faultHistory,omitempty"`
State string `json:"state"`
ChunksProduced uint64 `json:"chunksProduced"`
TotalSamples uint64 `json:"totalSamples"`
Underruns uint64 `json:"underruns"`
LateBuffers uint64 `json:"lateBuffers,omitempty"`
LastError string `json:"lastError,omitempty"`
UptimeSeconds float64 `json:"uptimeSeconds"`
MaxCycleMs float64 `json:"maxCycleMs,omitempty"`
MaxGenerateMs float64 `json:"maxGenerateMs,omitempty"`
MaxUpsampleMs float64 `json:"maxUpsampleMs,omitempty"`
MaxWriteMs float64 `json:"maxWriteMs,omitempty"`
Queue output.QueueStats `json:"queue"`
RuntimeIndicator RuntimeIndicator `json:"runtimeIndicator"`
RuntimeAlert string `json:"runtimeAlert,omitempty"`
LastFault *FaultEvent `json:"lastFault,omitempty"`
DegradedTransitions uint64 `json:"degradedTransitions"`
MutedTransitions uint64 `json:"mutedTransitions"`
FaultedTransitions uint64 `json:"faultedTransitions"`
FaultCount uint64 `json:"faultCount"`
FaultHistory []FaultEvent `json:"faultHistory,omitempty"`
TransitionHistory []RuntimeTransition `json:"transitionHistory,omitempty"`
} }


type RuntimeIndicator string type RuntimeIndicator string
@@ -99,14 +100,22 @@ const (
RuntimeIndicatorQueueCritical RuntimeIndicator = "queueCritical" RuntimeIndicatorQueueCritical RuntimeIndicator = "queueCritical"
) )


type RuntimeTransition struct {
Time time.Time `json:"time"`
From RuntimeState `json:"from"`
To RuntimeState `json:"to"`
Severity string `json:"severity"`
}

const ( const (
lateBufferIndicatorWindow = 5 * time.Second
queueCriticalStreakThreshold = 3
queueMutedStreakThreshold = queueCriticalStreakThreshold * 2
queueMutedRecoveryThreshold = queueCriticalStreakThreshold
queueFaultedStreakThreshold = queueCriticalStreakThreshold
faultRepeatWindow = 1 * time.Second
faultHistoryCapacity = 8
lateBufferIndicatorWindow = 5 * time.Second
queueCriticalStreakThreshold = 3
queueMutedStreakThreshold = queueCriticalStreakThreshold * 2
queueMutedRecoveryThreshold = queueCriticalStreakThreshold
queueFaultedStreakThreshold = queueCriticalStreakThreshold
faultRepeatWindow = 1 * time.Second
faultHistoryCapacity = 8
runtimeTransitionHistoryCapacity = 8
) )


// Engine is the continuous TX loop. It generates composite IQ in chunks, // Engine is the continuous TX loop. It generates composite IQ in chunks,
@@ -146,6 +155,8 @@ type Engine struct {
lastFault atomic.Value // *FaultEvent lastFault atomic.Value // *FaultEvent
faultHistoryMu sync.Mutex faultHistoryMu sync.Mutex
faultHistory []FaultEvent faultHistory []FaultEvent
transitionHistoryMu sync.Mutex
transitionHistory []RuntimeTransition


degradedTransitions atomic.Uint64 degradedTransitions atomic.Uint64
mutedTransitions atomic.Uint64 mutedTransitions atomic.Uint64
@@ -217,15 +228,16 @@ func NewEngine(cfg cfgpkg.Config, driver platform.SoapyDriver) *Engine {
} }


engine := &Engine{ engine := &Engine{
cfg: cfg,
driver: driver,
generator: offpkg.NewGenerator(cfg),
upsampler: upsampler,
chunkDuration: 50 * time.Millisecond,
deviceRate: deviceRate,
state: EngineIdle,
frameQueue: output.NewFrameQueue(cfg.Runtime.FrameQueueCapacity),
faultHistory: make([]FaultEvent, 0, faultHistoryCapacity),
cfg: cfg,
driver: driver,
generator: offpkg.NewGenerator(cfg),
upsampler: upsampler,
chunkDuration: 50 * time.Millisecond,
deviceRate: deviceRate,
state: EngineIdle,
frameQueue: output.NewFrameQueue(cfg.Runtime.FrameQueueCapacity),
faultHistory: make([]FaultEvent, 0, faultHistoryCapacity),
transitionHistory: make([]RuntimeTransition, 0, runtimeTransitionHistoryCapacity),
} }
engine.setRuntimeState(RuntimeStateIdle) engine.setRuntimeState(RuntimeStateIdle)
return engine return engine
@@ -423,6 +435,7 @@ func (e *Engine) Stats() EngineStats {
FaultedTransitions: e.faultedTransitions.Load(), FaultedTransitions: e.faultedTransitions.Load(),
FaultCount: e.faultEvents.Load(), FaultCount: e.faultEvents.Load(),
FaultHistory: e.FaultHistory(), FaultHistory: e.FaultHistory(),
TransitionHistory: e.TransitionHistory(),
} }
} }


@@ -450,6 +463,19 @@ func runtimeAlert(queueHealth output.QueueHealth, recentLateBuffers bool) string
} }
} }


func runtimeStateSeverity(state RuntimeState) string {
switch state {
case RuntimeStateRunning:
return "ok"
case RuntimeStateDegraded, RuntimeStateMuted:
return "warn"
case RuntimeStateFaulted:
return "err"
default:
return "info"
}
}

func (e *Engine) run(ctx context.Context) { func (e *Engine) run(ctx context.Context) {
e.setRuntimeState(RuntimeStatePrebuffering) e.setRuntimeState(RuntimeStatePrebuffering)
e.wg.Add(1) e.wg.Add(1)
@@ -589,6 +615,7 @@ func cloneFrame(src *output.CompositeFrame) *output.CompositeFrame {
func (e *Engine) setRuntimeState(state RuntimeState) { func (e *Engine) setRuntimeState(state RuntimeState) {
prev := e.currentRuntimeState() prev := e.currentRuntimeState()
if prev != state { if prev != state {
e.recordRuntimeTransition(prev, state)
switch state { switch state {
case RuntimeStateDegraded: case RuntimeStateDegraded:
e.degradedTransitions.Add(1) e.degradedTransitions.Add(1)
@@ -635,6 +662,31 @@ func (e *Engine) FaultHistory() []FaultEvent {
return history return history
} }


func (e *Engine) TransitionHistory() []RuntimeTransition {
e.transitionHistoryMu.Lock()
defer e.transitionHistoryMu.Unlock()
history := make([]RuntimeTransition, len(e.transitionHistory))
copy(history, e.transitionHistory)
return history
}

func (e *Engine) recordRuntimeTransition(from, to RuntimeState) {
ev := RuntimeTransition{
Time: time.Now(),
From: from,
To: to,
Severity: runtimeStateSeverity(to),
}
e.transitionHistoryMu.Lock()
defer e.transitionHistoryMu.Unlock()
if len(e.transitionHistory) >= runtimeTransitionHistoryCapacity {
copy(e.transitionHistory, e.transitionHistory[1:])
e.transitionHistory[len(e.transitionHistory)-1] = ev
return
}
e.transitionHistory = append(e.transitionHistory, ev)
}

func (e *Engine) recordFault(reason FaultReason, severity FaultSeverity, message string) { func (e *Engine) recordFault(reason FaultReason, severity FaultSeverity, message string) {
if reason == "" { if reason == "" {
reason = FaultReasonUnknown reason = FaultReasonUnknown


+ 23
- 0
internal/app/runtime_state_test.go ファイルの表示

@@ -180,6 +180,29 @@ func TestRuntimeTransitionCounters(t *testing.T) {
} }
} }


func TestEngineTransitionHistory(t *testing.T) {
e := NewEngine(cfgpkg.Default(), platform.NewSimulatedDriver(nil))
e.setRuntimeState(RuntimeStateRunning)
e.setRuntimeState(RuntimeStateDegraded)
e.setRuntimeState(RuntimeStateMuted)

history := e.Stats().TransitionHistory
if len(history) != 3 {
t.Fatalf("expected 3 transitions recorded, got %d", len(history))
}
if history[0].From != RuntimeStateIdle || history[0].To != RuntimeStateRunning {
t.Fatalf("unexpected first transition: %+v", history[0])
}
if history[0].Severity != "ok" {
t.Fatalf("expected ok severity for running transition, got %s", history[0].Severity)
}
if history[1].To != RuntimeStateDegraded || history[1].Severity != "warn" {
t.Fatalf("expected degraded transition with warn severity, got %+v", history[1])
}
if history[2].To != RuntimeStateMuted || history[2].Severity != "warn" {
t.Fatalf("expected muted transition with warn severity, got %+v", history[2])
}
}


func TestEngineResetFaultRequiresFaultedState(t *testing.T) { func TestEngineResetFaultRequiresFaultedState(t *testing.T) {
e := NewEngine(cfgpkg.Default(), platform.NewSimulatedDriver(nil)) e := NewEngine(cfgpkg.Default(), platform.NewSimulatedDriver(nil))


+ 30
- 0
internal/control/control_test.go ファイルの表示

@@ -174,6 +174,36 @@ func TestRuntimeReportsFaultHistory(t *testing.T) {
t.Fatalf("faultHistory length mismatch: want %d got %d", len(history), len(histRaw)) t.Fatalf("faultHistory length mismatch: want %d got %d", len(history), len(histRaw))
} }
} }
func TestRuntimeReportsTransitionHistory(t *testing.T) {
srv := NewServer(cfgpkg.Default())
history := []map[string]any{{
"time": "2026-04-06T00:00:00Z",
"from": "running",
"to": "degraded",
"severity": "warn",
}}
srv.SetTXController(&fakeTXController{stats: map[string]any{"transitionHistory": history}})
rec := httptest.NewRecorder()
srv.Handler().ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/runtime", nil))
if rec.Code != 200 {
t.Fatalf("status: %d", rec.Code)
}
var body map[string]any
if err := json.Unmarshal(rec.Body.Bytes(), &body); err != nil {
t.Fatalf("unmarshal runtime: %v", err)
}
engineRaw, ok := body["engine"].(map[string]any)
if !ok {
t.Fatalf("runtime engine missing")
}
histRaw, ok := engineRaw["transitionHistory"].([]any)
if !ok {
t.Fatalf("transitionHistory missing or wrong type: %T", engineRaw["transitionHistory"])
}
if len(histRaw) != len(history) {
t.Fatalf("transitionHistory length mismatch: want %d got %d", len(history), len(histRaw))
}
}


func TestRuntimeFaultResetRejectsGet(t *testing.T) { func TestRuntimeFaultResetRejectsGet(t *testing.T) {
srv := NewServer(cfgpkg.Default()) srv := NewServer(cfgpkg.Default())


+ 31
- 3
internal/control/ui.html ファイルの表示

@@ -1456,7 +1456,8 @@ async function loadRuntime({ silent = true } = {}) {
state.server.runtime = runtime; state.server.runtime = runtime;
state.server.runtimeOk = true; state.server.runtimeOk = true;
state.server.lastRuntimeAt = nowTs(); state.server.lastRuntimeAt = nowTs();
notifyRuntimeTransition(runtime.engine);
const syncedTransitions = syncTransitionHistoryFromEngine(runtime.engine);
notifyRuntimeTransition(runtime.engine, !syncedTransitions);
pushHistory(runtime); pushHistory(runtime);
setConnection(true, state.pendingRequests > 0 ? 'busy' : 'connected'); setConnection(true, state.pendingRequests > 0 ? 'busy' : 'connected');
render(); render();
@@ -1495,6 +1496,30 @@ function pushTransitionHistory(from, to, severity) {
updateTransitionHistory(); updateTransitionHistory();
} }


function transitionEntryTime(value) {
if (value == null) return nowTs();
if (typeof value === 'number') return value;
const parsed = Date.parse(String(value));
return Number.isNaN(parsed) ? nowTs() : parsed;
}

function syncTransitionHistoryFromEngine(engine) {
const entries = Array.isArray(engine?.transitionHistory) ? engine.transitionHistory : null;
if (!entries) return false;
const sliceStart = Math.max(0, entries.length - transitionHistoryLimit);
const trimmed = entries.slice(sliceStart);
const normalized = trimmed.map((entry) => ({
from: normalizeRuntimeState(entry?.from),
to: normalizeRuntimeState(entry?.to),
severity: String(entry?.severity || 'info').toLowerCase(),
time: transitionEntryTime(entry?.time),
}));
normalized.reverse();
state.runtimeTransitions = normalized;
updateTransitionHistory();
return true;
}

function transitionSeverityClass(severity) { function transitionSeverityClass(severity) {
switch (String(severity || '').toLowerCase()) { switch (String(severity || '').toLowerCase()) {
case 'err': case 'err':
@@ -1975,20 +2000,23 @@ function runtimeStateSeverity(stateName) {
} }
} }


function notifyRuntimeTransition(engine) {
function notifyRuntimeTransition(engine, pushHistory = true) {
if (!engine) return; if (!engine) return;
const next = normalizeRuntimeState(engine.state); const next = normalizeRuntimeState(engine.state);
const prev = state.lastRuntimeState; const prev = state.lastRuntimeState;
state.lastRuntimeState = next; state.lastRuntimeState = next;
if (!prev || prev === next) return; if (!prev || prev === next) return;
const severity = runtimeStateSeverity(next); const severity = runtimeStateSeverity(next);
pushTransitionHistory(prev, next, severity);
if (pushHistory) {
pushTransitionHistory(prev, next, severity);
}
const message = `Runtime ${prev.toUpperCase()} → ${next.toUpperCase()}`; const message = `Runtime ${prev.toUpperCase()} → ${next.toUpperCase()}`;
const logLevel = severity === 'err' ? 'err' : (severity === 'warn' ? 'warn' : 'info'); const logLevel = severity === 'err' ? 'err' : (severity === 'warn' ? 'warn' : 'info');
toast(message, severity); toast(message, severity);
log(message, logLevel); log(message, logLevel);
} }



function updateHealth(engine, audioStream) { function updateHealth(engine, audioStream) {
engine = engine || {}; engine = engine || {};
updateText('health-http', state.server.configOk ? 'OK' : 'OFFLINE'); updateText('health-http', state.server.configOk ? 'OK' : 'OFFLINE');


読み込み中…
キャンセル
保存