|
|
|
@@ -94,6 +94,7 @@ const ( |
|
|
|
) |
|
|
|
|
|
|
|
const lateBufferIndicatorWindow = 5 * time.Second |
|
|
|
const queueCriticalStreakThreshold = 3 |
|
|
|
|
|
|
|
// Engine is the continuous TX loop. It generates composite IQ in chunks, |
|
|
|
// resamples to device rate, and pushes to hardware in a tight loop. |
|
|
|
@@ -121,6 +122,7 @@ type Engine struct { |
|
|
|
underruns atomic.Uint64 |
|
|
|
lateBuffers atomic.Uint64 |
|
|
|
lateBufferAlertAt atomic.Uint64 |
|
|
|
criticalStreak atomic.Uint64 |
|
|
|
maxCycleNs atomic.Uint64 |
|
|
|
maxGenerateNs atomic.Uint64 |
|
|
|
maxUpsampleNs atomic.Uint64 |
|
|
|
@@ -373,9 +375,7 @@ func (e *Engine) Stats() EngineStats { |
|
|
|
|
|
|
|
queue := e.frameQueue.Stats() |
|
|
|
lateBuffers := e.lateBuffers.Load() |
|
|
|
now := time.Now() |
|
|
|
lateAlertAt := e.lateBufferAlertAt.Load() |
|
|
|
hasRecentLateBuffers := lateAlertAt > 0 && now.Sub(time.Unix(0, int64(lateAlertAt))) <= lateBufferIndicatorWindow |
|
|
|
hasRecentLateBuffers := e.hasRecentLateBuffers() |
|
|
|
ri := runtimeIndicator(queue.Health, hasRecentLateBuffers) |
|
|
|
return EngineStats{ |
|
|
|
State: string(e.currentRuntimeState()), |
|
|
|
@@ -420,7 +420,7 @@ func runtimeAlert(queueHealth output.QueueHealth, recentLateBuffers bool) string |
|
|
|
} |
|
|
|
|
|
|
|
func (e *Engine) run(ctx context.Context) { |
|
|
|
e.setRuntimeState(RuntimeStateRunning) |
|
|
|
e.setRuntimeState(RuntimeStatePrebuffering) |
|
|
|
e.wg.Add(1) |
|
|
|
go e.writerLoop(ctx) |
|
|
|
defer e.wg.Done() |
|
|
|
@@ -477,6 +477,8 @@ func (e *Engine) run(ctx context.Context) { |
|
|
|
} |
|
|
|
continue |
|
|
|
} |
|
|
|
queueStats := e.frameQueue.Stats() |
|
|
|
e.evaluateRuntimeState(queueStats, e.hasRecentLateBuffers()) |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
@@ -507,6 +509,8 @@ func (e *Engine) writerLoop(ctx context.Context) { |
|
|
|
|
|
|
|
updateMaxDuration(&e.maxWriteNs, writeDur) |
|
|
|
updateMaxDuration(&e.maxCycleNs, cycleDur) |
|
|
|
queueStats := e.frameQueue.Stats() |
|
|
|
e.evaluateRuntimeState(queueStats, e.hasRecentLateBuffers()) |
|
|
|
|
|
|
|
if cycleDur > e.chunkDuration { |
|
|
|
late := e.lateBuffers.Add(1) |
|
|
|
@@ -563,3 +567,39 @@ func (e *Engine) currentRuntimeState() RuntimeState { |
|
|
|
} |
|
|
|
return RuntimeStateIdle |
|
|
|
} |
|
|
|
|
|
|
|
func (e *Engine) hasRecentLateBuffers() bool { |
|
|
|
lateAlertAt := e.lateBufferAlertAt.Load() |
|
|
|
if lateAlertAt == 0 { |
|
|
|
return false |
|
|
|
} |
|
|
|
return time.Since(time.Unix(0, int64(lateAlertAt))) <= lateBufferIndicatorWindow |
|
|
|
} |
|
|
|
|
|
|
|
func (e *Engine) evaluateRuntimeState(queue output.QueueStats, hasLateBuffers bool) { |
|
|
|
state := e.currentRuntimeState() |
|
|
|
switch state { |
|
|
|
case RuntimeStateStopping, RuntimeStateFaulted: |
|
|
|
return |
|
|
|
} |
|
|
|
if state == RuntimeStatePrebuffering { |
|
|
|
if queue.Depth >= 1 { |
|
|
|
e.setRuntimeState(RuntimeStateRunning) |
|
|
|
} |
|
|
|
return |
|
|
|
} |
|
|
|
critical := queue.Health == output.QueueHealthCritical |
|
|
|
if critical { |
|
|
|
if e.criticalStreak.Add(1) >= queueCriticalStreakThreshold { |
|
|
|
e.setRuntimeState(RuntimeStateDegraded) |
|
|
|
return |
|
|
|
} |
|
|
|
} else { |
|
|
|
e.criticalStreak.Store(0) |
|
|
|
} |
|
|
|
if hasLateBuffers { |
|
|
|
e.setRuntimeState(RuntimeStateDegraded) |
|
|
|
return |
|
|
|
} |
|
|
|
e.setRuntimeState(RuntimeStateRunning) |
|
|
|
} |