Selaa lähdekoodia

Expose runtime transition history

tags/v0.9.0
Jan Svabenik 1 kuukausi sitten
vanhempi
commit
a7549f4187
6 muutettua tiedostoa jossa 183 lisäystä ja 40 poistoa
  1. +1
    -0
      cmd/fmrtx/main.go
  2. +10
    -1
      docs/API.md
  3. +88
    -36
      internal/app/engine.go
  4. +23
    -0
      internal/app/runtime_state_test.go
  5. +30
    -0
      internal/control/control_test.go
  6. +31
    -3
      internal/control/ui.html

+ 1
- 0
cmd/fmrtx/main.go Näytä tiedosto

@@ -271,6 +271,7 @@ func (b *txBridge) TXStats() map[string]any {
"faultedTransitions": s.FaultedTransitions,
"faultCount": s.FaultCount,
"faultHistory": s.FaultHistory,
"transitionHistory": s.TransitionHistory,
"lastFault": s.LastFault,
}
}


+ 10
- 1
docs/API.md Näytä tiedosto

@@ -78,6 +78,14 @@ Live engine and driver telemetry. Only populated when TX is active.
"severity": "faulted",
"message": "queue health critical for 5 checks"
}
],
"transitionHistory": [
{
"time": "2026-04-06T00:00:00Z",
"from": "running",
"to": "degraded",
"severity": "warn"
}
]
},
"driver": {
@@ -90,9 +98,10 @@ Live engine and driver telemetry. Only populated when TX is active.
}
}
```

`engine.state` spiegelt jetzt die Runtime-State-Maschine wider (idle, arming, prebuffering, running, degraded, muted, faulted, stopping) und bietet eine erste beobachtbare Basis für Fault-Transitions.

`transitionHistory` liefert die jüngsten Übergänge (from/to, severity, timestamp) damit API und UI die Runtime History synchronisieren können.

---

### `POST /runtime/fault/reset`


+ 88
- 36
internal/app/engine.go Näytä tiedosto

@@ -69,26 +69,27 @@ func durationMs(ns uint64) float64 {
}

type EngineStats struct {
State string `json:"state"`
ChunksProduced uint64 `json:"chunksProduced"`
TotalSamples uint64 `json:"totalSamples"`
Underruns uint64 `json:"underruns"`
LateBuffers uint64 `json:"lateBuffers,omitempty"`
LastError string `json:"lastError,omitempty"`
UptimeSeconds float64 `json:"uptimeSeconds"`
MaxCycleMs float64 `json:"maxCycleMs,omitempty"`
MaxGenerateMs float64 `json:"maxGenerateMs,omitempty"`
MaxUpsampleMs float64 `json:"maxUpsampleMs,omitempty"`
MaxWriteMs float64 `json:"maxWriteMs,omitempty"`
Queue output.QueueStats `json:"queue"`
RuntimeIndicator RuntimeIndicator `json:"runtimeIndicator"`
RuntimeAlert string `json:"runtimeAlert,omitempty"`
LastFault *FaultEvent `json:"lastFault,omitempty"`
DegradedTransitions uint64 `json:"degradedTransitions"`
MutedTransitions uint64 `json:"mutedTransitions"`
FaultedTransitions uint64 `json:"faultedTransitions"`
FaultCount uint64 `json:"faultCount"`
FaultHistory []FaultEvent `json:"faultHistory,omitempty"`
State string `json:"state"`
ChunksProduced uint64 `json:"chunksProduced"`
TotalSamples uint64 `json:"totalSamples"`
Underruns uint64 `json:"underruns"`
LateBuffers uint64 `json:"lateBuffers,omitempty"`
LastError string `json:"lastError,omitempty"`
UptimeSeconds float64 `json:"uptimeSeconds"`
MaxCycleMs float64 `json:"maxCycleMs,omitempty"`
MaxGenerateMs float64 `json:"maxGenerateMs,omitempty"`
MaxUpsampleMs float64 `json:"maxUpsampleMs,omitempty"`
MaxWriteMs float64 `json:"maxWriteMs,omitempty"`
Queue output.QueueStats `json:"queue"`
RuntimeIndicator RuntimeIndicator `json:"runtimeIndicator"`
RuntimeAlert string `json:"runtimeAlert,omitempty"`
LastFault *FaultEvent `json:"lastFault,omitempty"`
DegradedTransitions uint64 `json:"degradedTransitions"`
MutedTransitions uint64 `json:"mutedTransitions"`
FaultedTransitions uint64 `json:"faultedTransitions"`
FaultCount uint64 `json:"faultCount"`
FaultHistory []FaultEvent `json:"faultHistory,omitempty"`
TransitionHistory []RuntimeTransition `json:"transitionHistory,omitempty"`
}

type RuntimeIndicator string
@@ -99,14 +100,22 @@ const (
RuntimeIndicatorQueueCritical RuntimeIndicator = "queueCritical"
)

type RuntimeTransition struct {
Time time.Time `json:"time"`
From RuntimeState `json:"from"`
To RuntimeState `json:"to"`
Severity string `json:"severity"`
}

const (
lateBufferIndicatorWindow = 5 * time.Second
queueCriticalStreakThreshold = 3
queueMutedStreakThreshold = queueCriticalStreakThreshold * 2
queueMutedRecoveryThreshold = queueCriticalStreakThreshold
queueFaultedStreakThreshold = queueCriticalStreakThreshold
faultRepeatWindow = 1 * time.Second
faultHistoryCapacity = 8
lateBufferIndicatorWindow = 5 * time.Second
queueCriticalStreakThreshold = 3
queueMutedStreakThreshold = queueCriticalStreakThreshold * 2
queueMutedRecoveryThreshold = queueCriticalStreakThreshold
queueFaultedStreakThreshold = queueCriticalStreakThreshold
faultRepeatWindow = 1 * time.Second
faultHistoryCapacity = 8
runtimeTransitionHistoryCapacity = 8
)

// Engine is the continuous TX loop. It generates composite IQ in chunks,
@@ -146,6 +155,8 @@ type Engine struct {
lastFault atomic.Value // *FaultEvent
faultHistoryMu sync.Mutex
faultHistory []FaultEvent
transitionHistoryMu sync.Mutex
transitionHistory []RuntimeTransition

degradedTransitions atomic.Uint64
mutedTransitions atomic.Uint64
@@ -217,15 +228,16 @@ func NewEngine(cfg cfgpkg.Config, driver platform.SoapyDriver) *Engine {
}

engine := &Engine{
cfg: cfg,
driver: driver,
generator: offpkg.NewGenerator(cfg),
upsampler: upsampler,
chunkDuration: 50 * time.Millisecond,
deviceRate: deviceRate,
state: EngineIdle,
frameQueue: output.NewFrameQueue(cfg.Runtime.FrameQueueCapacity),
faultHistory: make([]FaultEvent, 0, faultHistoryCapacity),
cfg: cfg,
driver: driver,
generator: offpkg.NewGenerator(cfg),
upsampler: upsampler,
chunkDuration: 50 * time.Millisecond,
deviceRate: deviceRate,
state: EngineIdle,
frameQueue: output.NewFrameQueue(cfg.Runtime.FrameQueueCapacity),
faultHistory: make([]FaultEvent, 0, faultHistoryCapacity),
transitionHistory: make([]RuntimeTransition, 0, runtimeTransitionHistoryCapacity),
}
engine.setRuntimeState(RuntimeStateIdle)
return engine
@@ -423,6 +435,7 @@ func (e *Engine) Stats() EngineStats {
FaultedTransitions: e.faultedTransitions.Load(),
FaultCount: e.faultEvents.Load(),
FaultHistory: e.FaultHistory(),
TransitionHistory: e.TransitionHistory(),
}
}

@@ -450,6 +463,19 @@ func runtimeAlert(queueHealth output.QueueHealth, recentLateBuffers bool) string
}
}

func runtimeStateSeverity(state RuntimeState) string {
switch state {
case RuntimeStateRunning:
return "ok"
case RuntimeStateDegraded, RuntimeStateMuted:
return "warn"
case RuntimeStateFaulted:
return "err"
default:
return "info"
}
}

func (e *Engine) run(ctx context.Context) {
e.setRuntimeState(RuntimeStatePrebuffering)
e.wg.Add(1)
@@ -589,6 +615,7 @@ func cloneFrame(src *output.CompositeFrame) *output.CompositeFrame {
func (e *Engine) setRuntimeState(state RuntimeState) {
prev := e.currentRuntimeState()
if prev != state {
e.recordRuntimeTransition(prev, state)
switch state {
case RuntimeStateDegraded:
e.degradedTransitions.Add(1)
@@ -635,6 +662,31 @@ func (e *Engine) FaultHistory() []FaultEvent {
return history
}

func (e *Engine) TransitionHistory() []RuntimeTransition {
e.transitionHistoryMu.Lock()
defer e.transitionHistoryMu.Unlock()
history := make([]RuntimeTransition, len(e.transitionHistory))
copy(history, e.transitionHistory)
return history
}

func (e *Engine) recordRuntimeTransition(from, to RuntimeState) {
ev := RuntimeTransition{
Time: time.Now(),
From: from,
To: to,
Severity: runtimeStateSeverity(to),
}
e.transitionHistoryMu.Lock()
defer e.transitionHistoryMu.Unlock()
if len(e.transitionHistory) >= runtimeTransitionHistoryCapacity {
copy(e.transitionHistory, e.transitionHistory[1:])
e.transitionHistory[len(e.transitionHistory)-1] = ev
return
}
e.transitionHistory = append(e.transitionHistory, ev)
}

func (e *Engine) recordFault(reason FaultReason, severity FaultSeverity, message string) {
if reason == "" {
reason = FaultReasonUnknown


+ 23
- 0
internal/app/runtime_state_test.go Näytä tiedosto

@@ -180,6 +180,29 @@ func TestRuntimeTransitionCounters(t *testing.T) {
}
}

func TestEngineTransitionHistory(t *testing.T) {
e := NewEngine(cfgpkg.Default(), platform.NewSimulatedDriver(nil))
e.setRuntimeState(RuntimeStateRunning)
e.setRuntimeState(RuntimeStateDegraded)
e.setRuntimeState(RuntimeStateMuted)

history := e.Stats().TransitionHistory
if len(history) != 3 {
t.Fatalf("expected 3 transitions recorded, got %d", len(history))
}
if history[0].From != RuntimeStateIdle || history[0].To != RuntimeStateRunning {
t.Fatalf("unexpected first transition: %+v", history[0])
}
if history[0].Severity != "ok" {
t.Fatalf("expected ok severity for running transition, got %s", history[0].Severity)
}
if history[1].To != RuntimeStateDegraded || history[1].Severity != "warn" {
t.Fatalf("expected degraded transition with warn severity, got %+v", history[1])
}
if history[2].To != RuntimeStateMuted || history[2].Severity != "warn" {
t.Fatalf("expected muted transition with warn severity, got %+v", history[2])
}
}

func TestEngineResetFaultRequiresFaultedState(t *testing.T) {
e := NewEngine(cfgpkg.Default(), platform.NewSimulatedDriver(nil))


+ 30
- 0
internal/control/control_test.go Näytä tiedosto

@@ -174,6 +174,36 @@ func TestRuntimeReportsFaultHistory(t *testing.T) {
t.Fatalf("faultHistory length mismatch: want %d got %d", len(history), len(histRaw))
}
}
func TestRuntimeReportsTransitionHistory(t *testing.T) {
srv := NewServer(cfgpkg.Default())
history := []map[string]any{{
"time": "2026-04-06T00:00:00Z",
"from": "running",
"to": "degraded",
"severity": "warn",
}}
srv.SetTXController(&fakeTXController{stats: map[string]any{"transitionHistory": history}})
rec := httptest.NewRecorder()
srv.Handler().ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/runtime", nil))
if rec.Code != 200 {
t.Fatalf("status: %d", rec.Code)
}
var body map[string]any
if err := json.Unmarshal(rec.Body.Bytes(), &body); err != nil {
t.Fatalf("unmarshal runtime: %v", err)
}
engineRaw, ok := body["engine"].(map[string]any)
if !ok {
t.Fatalf("runtime engine missing")
}
histRaw, ok := engineRaw["transitionHistory"].([]any)
if !ok {
t.Fatalf("transitionHistory missing or wrong type: %T", engineRaw["transitionHistory"])
}
if len(histRaw) != len(history) {
t.Fatalf("transitionHistory length mismatch: want %d got %d", len(history), len(histRaw))
}
}

func TestRuntimeFaultResetRejectsGet(t *testing.T) {
srv := NewServer(cfgpkg.Default())


+ 31
- 3
internal/control/ui.html Näytä tiedosto

@@ -1456,7 +1456,8 @@ async function loadRuntime({ silent = true } = {}) {
state.server.runtime = runtime;
state.server.runtimeOk = true;
state.server.lastRuntimeAt = nowTs();
notifyRuntimeTransition(runtime.engine);
const syncedTransitions = syncTransitionHistoryFromEngine(runtime.engine);
notifyRuntimeTransition(runtime.engine, !syncedTransitions);
pushHistory(runtime);
setConnection(true, state.pendingRequests > 0 ? 'busy' : 'connected');
render();
@@ -1495,6 +1496,30 @@ function pushTransitionHistory(from, to, severity) {
updateTransitionHistory();
}

function transitionEntryTime(value) {
if (value == null) return nowTs();
if (typeof value === 'number') return value;
const parsed = Date.parse(String(value));
return Number.isNaN(parsed) ? nowTs() : parsed;
}

function syncTransitionHistoryFromEngine(engine) {
const entries = Array.isArray(engine?.transitionHistory) ? engine.transitionHistory : null;
if (!entries) return false;
const sliceStart = Math.max(0, entries.length - transitionHistoryLimit);
const trimmed = entries.slice(sliceStart);
const normalized = trimmed.map((entry) => ({
from: normalizeRuntimeState(entry?.from),
to: normalizeRuntimeState(entry?.to),
severity: String(entry?.severity || 'info').toLowerCase(),
time: transitionEntryTime(entry?.time),
}));
normalized.reverse();
state.runtimeTransitions = normalized;
updateTransitionHistory();
return true;
}

function transitionSeverityClass(severity) {
switch (String(severity || '').toLowerCase()) {
case 'err':
@@ -1975,20 +2000,23 @@ function runtimeStateSeverity(stateName) {
}
}

function notifyRuntimeTransition(engine) {
function notifyRuntimeTransition(engine, pushHistory = true) {
if (!engine) return;
const next = normalizeRuntimeState(engine.state);
const prev = state.lastRuntimeState;
state.lastRuntimeState = next;
if (!prev || prev === next) return;
const severity = runtimeStateSeverity(next);
pushTransitionHistory(prev, next, severity);
if (pushHistory) {
pushTransitionHistory(prev, next, severity);
}
const message = `Runtime ${prev.toUpperCase()} → ${next.toUpperCase()}`;
const logLevel = severity === 'err' ? 'err' : (severity === 'warn' ? 'warn' : 'info');
toast(message, severity);
log(message, logLevel);
}


function updateHealth(engine, audioStream) {
engine = engine || {};
updateText('health-http', state.server.configOk ? 'OK' : 'OFFLINE');


Loading…
Peruuta
Tallenna