|
|
|
@@ -69,26 +69,27 @@ func durationMs(ns uint64) float64 { |
|
|
|
} |
|
|
|
|
|
|
|
type EngineStats struct { |
|
|
|
State string `json:"state"` |
|
|
|
ChunksProduced uint64 `json:"chunksProduced"` |
|
|
|
TotalSamples uint64 `json:"totalSamples"` |
|
|
|
Underruns uint64 `json:"underruns"` |
|
|
|
LateBuffers uint64 `json:"lateBuffers,omitempty"` |
|
|
|
LastError string `json:"lastError,omitempty"` |
|
|
|
UptimeSeconds float64 `json:"uptimeSeconds"` |
|
|
|
MaxCycleMs float64 `json:"maxCycleMs,omitempty"` |
|
|
|
MaxGenerateMs float64 `json:"maxGenerateMs,omitempty"` |
|
|
|
MaxUpsampleMs float64 `json:"maxUpsampleMs,omitempty"` |
|
|
|
MaxWriteMs float64 `json:"maxWriteMs,omitempty"` |
|
|
|
Queue output.QueueStats `json:"queue"` |
|
|
|
RuntimeIndicator RuntimeIndicator `json:"runtimeIndicator"` |
|
|
|
RuntimeAlert string `json:"runtimeAlert,omitempty"` |
|
|
|
LastFault *FaultEvent `json:"lastFault,omitempty"` |
|
|
|
DegradedTransitions uint64 `json:"degradedTransitions"` |
|
|
|
MutedTransitions uint64 `json:"mutedTransitions"` |
|
|
|
FaultedTransitions uint64 `json:"faultedTransitions"` |
|
|
|
FaultCount uint64 `json:"faultCount"` |
|
|
|
FaultHistory []FaultEvent `json:"faultHistory,omitempty"` |
|
|
|
State string `json:"state"` |
|
|
|
ChunksProduced uint64 `json:"chunksProduced"` |
|
|
|
TotalSamples uint64 `json:"totalSamples"` |
|
|
|
Underruns uint64 `json:"underruns"` |
|
|
|
LateBuffers uint64 `json:"lateBuffers,omitempty"` |
|
|
|
LastError string `json:"lastError,omitempty"` |
|
|
|
UptimeSeconds float64 `json:"uptimeSeconds"` |
|
|
|
MaxCycleMs float64 `json:"maxCycleMs,omitempty"` |
|
|
|
MaxGenerateMs float64 `json:"maxGenerateMs,omitempty"` |
|
|
|
MaxUpsampleMs float64 `json:"maxUpsampleMs,omitempty"` |
|
|
|
MaxWriteMs float64 `json:"maxWriteMs,omitempty"` |
|
|
|
Queue output.QueueStats `json:"queue"` |
|
|
|
RuntimeIndicator RuntimeIndicator `json:"runtimeIndicator"` |
|
|
|
RuntimeAlert string `json:"runtimeAlert,omitempty"` |
|
|
|
LastFault *FaultEvent `json:"lastFault,omitempty"` |
|
|
|
DegradedTransitions uint64 `json:"degradedTransitions"` |
|
|
|
MutedTransitions uint64 `json:"mutedTransitions"` |
|
|
|
FaultedTransitions uint64 `json:"faultedTransitions"` |
|
|
|
FaultCount uint64 `json:"faultCount"` |
|
|
|
FaultHistory []FaultEvent `json:"faultHistory,omitempty"` |
|
|
|
TransitionHistory []RuntimeTransition `json:"transitionHistory,omitempty"` |
|
|
|
} |
|
|
|
|
|
|
|
type RuntimeIndicator string |
|
|
|
@@ -99,14 +100,22 @@ const ( |
|
|
|
RuntimeIndicatorQueueCritical RuntimeIndicator = "queueCritical" |
|
|
|
) |
|
|
|
|
|
|
|
type RuntimeTransition struct { |
|
|
|
Time time.Time `json:"time"` |
|
|
|
From RuntimeState `json:"from"` |
|
|
|
To RuntimeState `json:"to"` |
|
|
|
Severity string `json:"severity"` |
|
|
|
} |
|
|
|
|
|
|
|
const ( |
|
|
|
lateBufferIndicatorWindow = 5 * time.Second |
|
|
|
queueCriticalStreakThreshold = 3 |
|
|
|
queueMutedStreakThreshold = queueCriticalStreakThreshold * 2 |
|
|
|
queueMutedRecoveryThreshold = queueCriticalStreakThreshold |
|
|
|
queueFaultedStreakThreshold = queueCriticalStreakThreshold |
|
|
|
faultRepeatWindow = 1 * time.Second |
|
|
|
faultHistoryCapacity = 8 |
|
|
|
lateBufferIndicatorWindow = 5 * time.Second |
|
|
|
queueCriticalStreakThreshold = 3 |
|
|
|
queueMutedStreakThreshold = queueCriticalStreakThreshold * 2 |
|
|
|
queueMutedRecoveryThreshold = queueCriticalStreakThreshold |
|
|
|
queueFaultedStreakThreshold = queueCriticalStreakThreshold |
|
|
|
faultRepeatWindow = 1 * time.Second |
|
|
|
faultHistoryCapacity = 8 |
|
|
|
runtimeTransitionHistoryCapacity = 8 |
|
|
|
) |
|
|
|
|
|
|
|
// Engine is the continuous TX loop. It generates composite IQ in chunks, |
|
|
|
@@ -146,6 +155,8 @@ type Engine struct { |
|
|
|
lastFault atomic.Value // *FaultEvent |
|
|
|
faultHistoryMu sync.Mutex |
|
|
|
faultHistory []FaultEvent |
|
|
|
transitionHistoryMu sync.Mutex |
|
|
|
transitionHistory []RuntimeTransition |
|
|
|
|
|
|
|
degradedTransitions atomic.Uint64 |
|
|
|
mutedTransitions atomic.Uint64 |
|
|
|
@@ -217,15 +228,16 @@ func NewEngine(cfg cfgpkg.Config, driver platform.SoapyDriver) *Engine { |
|
|
|
} |
|
|
|
|
|
|
|
engine := &Engine{ |
|
|
|
cfg: cfg, |
|
|
|
driver: driver, |
|
|
|
generator: offpkg.NewGenerator(cfg), |
|
|
|
upsampler: upsampler, |
|
|
|
chunkDuration: 50 * time.Millisecond, |
|
|
|
deviceRate: deviceRate, |
|
|
|
state: EngineIdle, |
|
|
|
frameQueue: output.NewFrameQueue(cfg.Runtime.FrameQueueCapacity), |
|
|
|
faultHistory: make([]FaultEvent, 0, faultHistoryCapacity), |
|
|
|
cfg: cfg, |
|
|
|
driver: driver, |
|
|
|
generator: offpkg.NewGenerator(cfg), |
|
|
|
upsampler: upsampler, |
|
|
|
chunkDuration: 50 * time.Millisecond, |
|
|
|
deviceRate: deviceRate, |
|
|
|
state: EngineIdle, |
|
|
|
frameQueue: output.NewFrameQueue(cfg.Runtime.FrameQueueCapacity), |
|
|
|
faultHistory: make([]FaultEvent, 0, faultHistoryCapacity), |
|
|
|
transitionHistory: make([]RuntimeTransition, 0, runtimeTransitionHistoryCapacity), |
|
|
|
} |
|
|
|
engine.setRuntimeState(RuntimeStateIdle) |
|
|
|
return engine |
|
|
|
@@ -423,6 +435,7 @@ func (e *Engine) Stats() EngineStats { |
|
|
|
FaultedTransitions: e.faultedTransitions.Load(), |
|
|
|
FaultCount: e.faultEvents.Load(), |
|
|
|
FaultHistory: e.FaultHistory(), |
|
|
|
TransitionHistory: e.TransitionHistory(), |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
@@ -450,6 +463,19 @@ func runtimeAlert(queueHealth output.QueueHealth, recentLateBuffers bool) string |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
func runtimeStateSeverity(state RuntimeState) string { |
|
|
|
switch state { |
|
|
|
case RuntimeStateRunning: |
|
|
|
return "ok" |
|
|
|
case RuntimeStateDegraded, RuntimeStateMuted: |
|
|
|
return "warn" |
|
|
|
case RuntimeStateFaulted: |
|
|
|
return "err" |
|
|
|
default: |
|
|
|
return "info" |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
func (e *Engine) run(ctx context.Context) { |
|
|
|
e.setRuntimeState(RuntimeStatePrebuffering) |
|
|
|
e.wg.Add(1) |
|
|
|
@@ -589,6 +615,7 @@ func cloneFrame(src *output.CompositeFrame) *output.CompositeFrame { |
|
|
|
func (e *Engine) setRuntimeState(state RuntimeState) { |
|
|
|
prev := e.currentRuntimeState() |
|
|
|
if prev != state { |
|
|
|
e.recordRuntimeTransition(prev, state) |
|
|
|
switch state { |
|
|
|
case RuntimeStateDegraded: |
|
|
|
e.degradedTransitions.Add(1) |
|
|
|
@@ -635,6 +662,31 @@ func (e *Engine) FaultHistory() []FaultEvent { |
|
|
|
return history |
|
|
|
} |
|
|
|
|
|
|
|
func (e *Engine) TransitionHistory() []RuntimeTransition { |
|
|
|
e.transitionHistoryMu.Lock() |
|
|
|
defer e.transitionHistoryMu.Unlock() |
|
|
|
history := make([]RuntimeTransition, len(e.transitionHistory)) |
|
|
|
copy(history, e.transitionHistory) |
|
|
|
return history |
|
|
|
} |
|
|
|
|
|
|
|
func (e *Engine) recordRuntimeTransition(from, to RuntimeState) { |
|
|
|
ev := RuntimeTransition{ |
|
|
|
Time: time.Now(), |
|
|
|
From: from, |
|
|
|
To: to, |
|
|
|
Severity: runtimeStateSeverity(to), |
|
|
|
} |
|
|
|
e.transitionHistoryMu.Lock() |
|
|
|
defer e.transitionHistoryMu.Unlock() |
|
|
|
if len(e.transitionHistory) >= runtimeTransitionHistoryCapacity { |
|
|
|
copy(e.transitionHistory, e.transitionHistory[1:]) |
|
|
|
e.transitionHistory[len(e.transitionHistory)-1] = ev |
|
|
|
return |
|
|
|
} |
|
|
|
e.transitionHistory = append(e.transitionHistory, ev) |
|
|
|
} |
|
|
|
|
|
|
|
func (e *Engine) recordFault(reason FaultReason, severity FaultSeverity, message string) { |
|
|
|
if reason == "" { |
|
|
|
reason = FaultReasonUnknown |
|
|
|
|