diff --git a/docs/Kim_-_Parikh_-_Distribution_of_the_Analog_or_Digital_FM_Composite.pdf b/docs/Kim_-_Parikh_-_Distribution_of_the_Analog_or_Digital_FM_Composite.pdf new file mode 100644 index 0000000..310d16d Binary files /dev/null and b/docs/Kim_-_Parikh_-_Distribution_of_the_Analog_or_Digital_FM_Composite.pdf differ diff --git a/docs/composite-mpx-metering-concept.json b/docs/composite-mpx-metering-concept.json new file mode 100644 index 0000000..f87609a --- /dev/null +++ b/docs/composite-mpx-metering-concept.json @@ -0,0 +1,631 @@ +{ + "concept": "fm-rds-tx composite/mpx live metering", + "version": 3.1, + "status": "draft", + "summary": "Introduce true runtime broadcast-style metering for the FM stereo/RDS multiplex chain by measuring multiple semantically distinct DSP stages and exposing them through a dedicated measurements endpoint for Overview, Flow, and diagnostics UI surfaces.", + "problemStatement": { + "currentState": [ + "The current UI mostly exposes configured MPX-related values such as pilotLevel, rdsInjection, mpxGain, BS.412 state, and composite clipper state.", + "These values describe intended configuration but not the actual measured runtime behavior of the multiplex chain.", + "The current Flow Composite / MPX node is therefore closer to a config summary than to a true live broadcast metering surface.", + "Runtime telemetry currently focuses on transport and engine health such as queue health, underruns, runtime states, and fault transitions, but not on actual composite/MPX signal composition." + ], + "operationalGap": [ + "Operators cannot currently see how audio processing, stereo encoding, composite clipping, BS.412, pilot injection, and RDS injection interact on the live signal.", + "Without live stage-aware metering, it is difficult to identify whether signal behavior is caused by audio processing, compliance shaping, or final composite assembly.", + "A single config snapshot is not enough to answer whether the transmitted multiplex is nominal, compliance-limited, overdriven, missing pilot, or missing RDS." + ] + }, + "objective": { + "primary": "Turn the Composite / MPX area of fm-rds-tx into a true runtime metering surface that reflects measured DSP behavior rather than only saved configuration values.", + "secondary": [ + "Give operators a trustworthy view of what the DSP chain is actually producing at critical stages.", + "Separate audio-path behavior, BS.412 compliance behavior, and final composite/on-air behavior into distinct measured views.", + "Support a richer Overview and Flow UI without increasing full-runtime polling cost.", + "Create a stable foundation for future alerts, engineering diagnostics, and signal-quality workflows." + ] + }, + "whatWeAreTryingToAchieve": [ + { + "goal": "Show real measured multiplex behavior", + "description": "Measure actual runtime signal values at key DSP stages so the UI reflects what is really happening in the chain, not only what was configured." + }, + { + "goal": "Explain stage interactions", + "description": "Make visible how processing, stereo encoding, composite clipping, BS.412, pilot injection, and RDS injection each affect the final multiplex." + }, + { + "goal": "Separate compliance and on-air views", + "description": "Provide distinct views before BS.412, after BS.412, and at the final composite point directly before FM modulation." + }, + { + "goal": "Support operator decisions", + "description": "Allow operators to quickly assess whether the system is nominal, overdriven, compliance-limited, missing pilot/RDS, or behaving unexpectedly." + }, + { + "goal": "Stay aligned with the actual DSP architecture", + "description": "Add metering in a way that matches the current GenerateFrame() signal flow and preserves semantic clarity between stages." + } + ], + "expectedOutcome": { + "overviewUi": [ + "Overview can show measured L/R and final MPX/composite values instead of only static configuration values.", + "Pilot and RDS indicators become measurement-backed rather than purely declarative." + ], + "flowUi": [ + "The Composite / MPX node becomes a true live-status node.", + "The Flow popover can compare pre-BS.412, post-BS.412, and final pre-IQ behavior." + ], + "diagnostics": [ + "Engineers can determine whether a problem is introduced before BS.412, by BS.412, or only in the final composite stage." + ], + "futureWork": [ + "The same model can support alerts, trends, compliance heuristics, and richer engineering telemetry later." + ] + }, + "designPrinciples": [ + "Meter measured runtime signal states, not only configured target values.", + "Keep measurement taps aligned with actual DSP semantics already present in generator.GenerateFrame().", + "Separate audio-path compliance measurements from final composite/on-air measurements.", + "Expose a dedicated faster measurements endpoint instead of raising full UI polling frequency.", + "Provide a stable and explicit measurement model that can serve UI, diagnostics, and future alerting.", + "Use chunk-level aggregation, not per-sample external reporting.", + "Keep measurement logic low-overhead and safe for the hot path.", + "Prefer semantically conservative field names over catchy but ambiguous meter labels." + ], + "whyMultipleTaps": [ + "A single final composite reading cannot explain whether a change came from L/R processing, composite clipping, BS.412 gain reduction, or fixed pilot/RDS injection.", + "BS.412 acts on the audio MPX budget, not on pilot/RDS directly, so pre-BS.412 and post-BS.412 views are semantically different.", + "Pilot and RDS become meaningful only in the final composite stage, so they require final pre-IQ measurement.", + "Operator-facing UI needs both an audio/compliance view and a final on-air-adjacent composite view." + ], + "semanticAdjustmentsFromReview": [ + "The L/R tap must account for optional watermark insertion between first-pass audio processing and stereo encoding.", + "Pilot and RDS percent-style fields must not conflate RMS measurements with injection-equivalent display values.", + "Composite threshold counters must avoid overmodulation-suggestive names unless their semantics are explicitly regulatory, which they are not.", + "Composite clipper-related metrics should avoid claiming hard clip event semantics where the implementation is actually shaping/protection-oriented.", + "BS.412 gain must be described as a chunk-level running control state, not a sample-exact causal response.", + "Final-composite interpretation should expose feature flags such as watermark, RDS2, and license injection activity.", + "stereoMode should be exposed alongside stereoEnabled so non-DSB operation remains interpretable in diagnostics and UI.", + "licenseInjectionActive must mean chunk-local actual activity, not merely that the feature exists.", + "rdsInjectionEquivalentPercent must be mathematically defined as a derived operator-facing quantity before implementation begins." + ], + "measurementTaps": [ + { + "id": "lr_pre_encode_post_watermark", + "label": "L/R Pre-Encode Post-Watermark", + "stage": "after pre-emphasis, audio LPF, pilot notch, output drive, stereo limiter, cleanup LPF, final L/R hard clipping, and optional watermark injection; immediately before stereo encoding", + "locationInCode": { + "file": "internal/offline/generator.go", + "function": "GenerateFrame", + "region": "after optional watermark has been added back into lBuf/rBuf and before stereo encoding pass" + }, + "semanticPurpose": [ + "Represents the effective left/right program audio that actually enters the stereo encoder.", + "Avoids ambiguity when watermarking is enabled." + ], + "primaryUseCases": [ + "Overview L/R meters", + "Channel imbalance visibility", + "Audio processing inspection" + ], + "metrics": [ + "l_rms", + "r_rms", + "l_peak_abs", + "r_peak_abs", + "lr_balance_db", + "l_clip_events", + "r_clip_events" + ], + "notes": [ + "If a future pre-watermark tap is desired for engineering comparison, it should be added as a separate optional diagnostic tap rather than overloading this one." + ], + "required": true + }, + { + "id": "audio_mpx_pre_bs412", + "label": "Audio MPX Pre-BS.412", + "stage": "after stereo encode and composite clipper/protection, before BS.412 gain is applied", + "locationInCode": { + "file": "internal/offline/generator.go", + "function": "GenerateFrame", + "region": "after audioMPX is finalized from mono/stereo and after composite clipper or legacy clip/notch path, before bs412Gain multiplication" + }, + "semanticPurpose": [ + "Represents the raw audio-only multiplex signal before BS.412 compliance shaping.", + "Shows what the multiplex audio path is structurally producing." + ], + "primaryUseCases": [ + "Raw MPX level view", + "Comparison against post-BS.412 attenuation", + "Explaining why final loudness changed" + ], + "metrics": [ + "audio_mpx_rms", + "audio_mpx_peak_abs", + "mono_component_rms", + "stereo_component_rms", + "crest_factor", + "clipper_lookahead_gain", + "clipper_envelope", + "clipper_or_protection_active" + ], + "notes": [ + "Avoid promising exact hard clip event semantics here when the active implementation may be a true composite clipper/protection stage rather than simple hard clipping.", + "If clipper_or_protection_active is exposed, it must be derived from a documented condition. Prefer exposing raw clipper diagnostics such as lookahead gain and envelope so UI can derive simpler booleans later." + ], + "required": true + }, + { + "id": "audio_mpx_post_bs412", + "label": "Audio MPX Post-BS.412", + "stage": "after BS.412 gain reduction on audio MPX, before pilot and RDS injection", + "locationInCode": { + "file": "internal/offline/generator.go", + "function": "GenerateFrame", + "region": "immediately after bs412Gain is applied to audioMPX and before composite starts from audioMPX" + }, + "semanticPurpose": [ + "Represents the compliance-shaped audio multiplex signal.", + "Shows what survives BS.412 before fixed protected components are added." + ], + "primaryUseCases": [ + "BS.412 activity visibility", + "Compliance-focused diagnostics", + "Gain reduction explanation" + ], + "metrics": [ + "audio_mpx_rms", + "audio_mpx_peak_abs", + "bs412_gain_applied", + "bs412_attenuation_db", + "estimated_audio_power" + ], + "notes": [ + "bs412_gain_applied is a chunk-level running control value and must not be presented as a sample-instantaneous causal reaction." + ], + "required": true + }, + { + "id": "composite_final_pre_iq", + "label": "Composite Final Pre-IQ", + "stage": "after audio MPX plus pilot plus RDS plus optional RDS2 and optional license injection, immediately before fmMod.Modulate(composite) or direct composite output", + "locationInCode": { + "file": "internal/offline/generator.go", + "function": "GenerateFrame", + "region": "final composite variable just before FM modulation or direct composite write" + }, + "semanticPurpose": [ + "Represents the true final composite signal that is about to enter FM modulation or composite output.", + "This is the most operator-relevant on-air-adjacent metering point." + ], + "primaryUseCases": [ + "Final MPX meter", + "Pilot and RDS measurement", + "Peak/deviation proxy", + "On-air confidence" + ], + "metrics": [ + "composite_rms", + "composite_peak_abs", + "pilot_rms", + "pilot_peak_abs", + "pilot_injection_equivalent_percent", + "rds_rms", + "rds_peak_abs", + "rds_injection_equivalent_percent", + "composite_over_nominal_events", + "composite_over_headroom_events" + ], + "notes": [ + "Percent-style fields here are operator-facing derived values and must be clearly separated from raw RMS values.", + "Composite threshold counters are internal normalized envelope indicators, not regulatory overmodulation judgements.", + "rdsInjectionEquivalentPercent must be implemented only after its mathematical derivation has been explicitly documented." + ], + "required": true + } + ], + "preferredMetricSemantics": { + "rms": "Chunk-local RMS of the measured signal component.", + "peak_abs": "Maximum absolute sample magnitude within the chunk.", + "injection_equivalent_percent": "A derived operator-facing injection/deviation-style display value, not identical to RMS. This should be documented separately from raw measured signal energy.", + "clip_events": "Count of samples that hit or exceed a defined clipping threshold. Use only where the tapped stage really has hard-threshold clip semantics.", + "attenuation_db": "Derived gain-reduction view, typically 20*log10(post/pre) when meaningful.", + "estimated_audio_power": "Chunk-local power estimate suitable for BS.412-related operator display, but not legal certification.", + "over_nominal_events": "Count of samples above an internal nominal composite reference threshold in normalized composite units.", + "over_headroom_events": "Count of samples above a higher internal headroom reference threshold in normalized composite units.", + "crest_factor": "Derived peak-versus-RMS characteristic useful where clipper/protection stages do not map cleanly to simple clip event counters." + }, + "recommendedMeasurementFields": { + "topLevel": [ + "timestamp", + "sampleRateHz", + "chunkSamples", + "chunkDurationMs", + "sequence", + "stale", + "noData", + "flags" + ], + "flags": [ + "stereoEnabled", + "stereoMode", + "rdsEnabled", + "rds2Enabled", + "bs412Enabled", + "compositeClipperEnabled", + "watermarkEnabled", + "licenseInjectionActive" + ], + "groups": { + "lrPreEncodePostWatermark": [ + "lRms", + "rRms", + "lPeakAbs", + "rPeakAbs", + "lrBalanceDb", + "lClipEvents", + "rClipEvents" + ], + "audioMpxPreBs412": [ + "rms", + "peakAbs", + "monoRms", + "stereoRms", + "crestFactor", + "clipperLookaheadGain", + "clipperEnvelope", + "clipperOrProtectionActive" + ], + "audioMpxPostBs412": [ + "rms", + "peakAbs", + "bs412GainApplied", + "bs412AttenuationDb", + "estimatedAudioPower" + ], + "compositeFinalPreIq": [ + "rms", + "peakAbs", + "pilotRms", + "pilotPeakAbs", + "pilotInjectionEquivalentPercent", + "rdsRms", + "rdsPeakAbs", + "rdsInjectionEquivalentPercent", + "overNominalEvents", + "overHeadroomEvents" + ] + } + }, + "dataModelProposal": { + "goTypeName": "MeasurementSnapshot", + "shape": { + "timestamp": "time.Time", + "sampleRateHz": "float64", + "chunkSamples": "int", + "chunkDurationMs": "float64", + "sequence": "uint64", + "stale": "bool", + "noData": "bool", + "flags": { + "stereoEnabled": "bool", + "stereoMode": "string", + "rdsEnabled": "bool", + "rds2Enabled": "bool", + "bs412Enabled": "bool", + "compositeClipperEnabled": "bool", + "watermarkEnabled": "bool", + "licenseInjectionActive": "bool" + }, + "lrPreEncodePostWatermark": { + "lRms": "float64", + "rRms": "float64", + "lPeakAbs": "float64", + "rPeakAbs": "float64", + "lrBalanceDb": "float64", + "lClipEvents": "uint32", + "rClipEvents": "uint32" + }, + "audioMpxPreBs412": { + "rms": "float64", + "peakAbs": "float64", + "monoRms": "float64", + "stereoRms": "float64", + "crestFactor": "float64", + "clipperLookaheadGain": "float64", + "clipperEnvelope": "float64", + "clipperOrProtectionActive": "bool" + }, + "audioMpxPostBs412": { + "rms": "float64", + "peakAbs": "float64", + "bs412GainApplied": "float64", + "bs412AttenuationDb": "float64", + "estimatedAudioPower": "float64" + }, + "compositeFinalPreIq": { + "rms": "float64", + "peakAbs": "float64", + "pilotRms": "float64", + "pilotPeakAbs": "float64", + "pilotInjectionEquivalentPercent": "float64", + "rdsRms": "float64", + "rdsPeakAbs": "float64", + "rdsInjectionEquivalentPercent": "float64 // TODO: only expose after mathematical derivation is fixed; otherwise omit or null in MVP", + "overNominalEvents": "uint32", + "overHeadroomEvents": "uint32" + } + } + }, + "aggregationApproach": { + "method": "chunk-local accumulation", + "details": [ + "Accumulate sums, absolute peaks, and counters during one GenerateFrame() call.", + "Finalize RMS and derived values once per chunk into one immutable snapshot.", + "Publish only the latest completed snapshot.", + "Do not emit per-sample external telemetry or unbounded histories from the hot path." + ], + "why": [ + "Matches current engine/generator architecture.", + "Provides enough temporal resolution for 4-10 Hz UI polling.", + "Keeps CPU and allocation overhead predictable." + ] + }, + "placementRecommendation": { + "primaryOwner": "generator", + "reasoning": [ + "All semantically relevant intermediate signals already exist in GenerateFrame().", + "The generator has direct access to mono, stereo, pilot, RDS, audioMPX, and final composite values before they are flattened into output samples.", + "Later measurement would lose stage separation and component identity." + ], + "preferredPublicationModel": "Generator computes and stores the latest MeasurementSnapshot through an atomic pointer or equivalent low-contention accessor." + }, + "apiProposal": { + "newEndpoint": "/measurements", + "method": "GET", + "pollingHz": { + "min": 4, + "recommended": 5, + "max": 10 + }, + "whySeparateEndpoint": [ + "/runtime should remain focused on operational/runtime telemetry.", + "UI should not need to poll the full runtime payload at high frequency for metering.", + "Measurements have different freshness and payload-shape requirements than runtime state and config." + ], + "responseShapeExample": { + "timestamp": "2026-04-13T05:30:00Z", + "sampleRateHz": 228000, + "chunkSamples": 11400, + "chunkDurationMs": 50, + "sequence": 12345, + "stale": false, + "noData": false, + "flags": { + "stereoEnabled": true, + "stereoMode": "DSB", + "rdsEnabled": true, + "rds2Enabled": false, + "bs412Enabled": true, + "compositeClipperEnabled": true, + "watermarkEnabled": false, + "licenseInjectionActive": false + }, + "lrPreEncodePostWatermark": { + "lRms": 0.41, + "rRms": 0.39, + "lPeakAbs": 0.98, + "rPeakAbs": 0.96, + "lrBalanceDb": 0.42, + "lClipEvents": 12, + "rClipEvents": 8 + }, + "audioMpxPreBs412": { + "rms": 0.52, + "peakAbs": 1.0, + "monoRms": 0.34, + "stereoRms": 0.18, + "crestFactor": 1.92, + "clipperLookaheadGain": 0.94, + "clipperEnvelope": 1.03, + "clipperOrProtectionActive": true + }, + "audioMpxPostBs412": { + "rms": 0.46, + "peakAbs": 0.91, + "bs412GainApplied": 0.88, + "bs412AttenuationDb": -1.11, + "estimatedAudioPower": 0.21 + }, + "compositeFinalPreIq": { + "rms": 0.49, + "peakAbs": 1.08, + "pilotRms": 0.064, + "pilotPeakAbs": 0.09, + "pilotInjectionEquivalentPercent": 9.0, + "rdsRms": 0.028, + "rdsPeakAbs": 0.04, + "rdsInjectionEquivalentPercent": 4.0, + "overNominalEvents": 91, + "overHeadroomEvents": 0 + } + }, + "emptyStateBehavior": { + "whenTxInactive": "return either noData=true or the last snapshot with stale=true", + "preferred": "explicit noData=true when no current generator output exists" + } + }, + "uiIntegrationPlan": { + "overview": { + "changes": [ + "Add or extend a metering section for measured L/R and final composite/MPX values.", + "Show measured pilot and RDS values with explicit separation between raw measurements and operator-facing injection-equivalent display values.", + "Optionally add compact L/R bar meters from lrPreEncodePostWatermark." + ], + "avoid": [ + "Do not replace queue/runtime health indicators with signal metering.", + "Do not raise /runtime polling frequency just to animate meters.", + "Do not label derived percent-style values as RMS values." + ] + }, + "flowTab": { + "compositeMpxNode": [ + "Keep config-derived metadata as fallback.", + "Use measured values as the primary state/detail source where available.", + "Drive state and detail from final composite measurements plus compliance indicators.", + "Include flags-based context when watermark, RDS2, or license injection changes the final composite interpretation.", + "Include stereoMode context when non-DSB stereo operation changes signal expectations." + ], + "popover": [ + "Add pre-BS.412 vs post-BS.412 comparison rows.", + "Show final composite peak and measured pilot/RDS values.", + "If desired, show both raw measured values and operator-friendly injection-equivalent display values side by side." + ] + }, + "diagnostics": { + "future": [ + "Add a dedicated engineering metering panel.", + "Optionally add short client-side sparklines for composite RMS or BS.412 attenuation." + ] + } + }, + "statusLogicProposal": { + "compositeMpxNode": { + "green": [ + "Measurements available", + "Final composite is within expected internal normalized envelope", + "Pilot and RDS are near expected target windows", + "No severe compliance attention" + ], + "amber": [ + "Measurement is stale", + "Pilot or RDS materially deviates from expected target", + "High composite peak or visible compliance shaping", + "Restart-pending settings affect MPX semantics" + ], + "red": [ + "Measurement unavailable while TX is running", + "Final composite exceeds internal critical envelope threshold repeatedly", + "Pilot missing when stereo is enabled", + "RDS missing when RDS is enabled and expected" + ], + "gray": [ + "TX idle or no active signal path" + ] + } + }, + "performanceConstraints": { + "must": [ + "No contended locks in the per-sample inner loop.", + "No heap allocation per sample.", + "At most one snapshot publication per chunk.", + "Endpoint read must be cheap and non-blocking." + ], + "acceptableCosts": [ + "A few additional per-sample accumulations such as abs, squares, and counters.", + "A small per-chunk finalize step with sqrt and log calculations.", + "Sampling lightweight clipper diagnostics when available." + ] + }, + "implementationPhases": [ + { + "phase": 1, + "name": "measurement model and generator instrumentation", + "deliverables": [ + "Define MeasurementSnapshot and stage structs.", + "Add accumulator logic in GenerateFrame().", + "Publish latest completed snapshot from generator." + ] + }, + { + "phase": 2, + "name": "control-plane endpoint", + "deliverables": [ + "Add GET /measurements handler.", + "Wire generator/engine access into control server.", + "Document endpoint in docs/API.md.", + "Add tests for active, stale, and noData cases." + ] + }, + { + "phase": 3, + "name": "UI integration MVP", + "deliverables": [ + "Poll /measurements separately at 4-10 Hz.", + "Add measured MPX display in Overview.", + "Drive Flow Composite / MPX node from measurement data." + ] + }, + { + "phase": 4, + "name": "diagnostics refinement", + "deliverables": [ + "Add engineering-oriented meter details.", + "Add client-side trend memory for sparkline rendering.", + "Refine thresholds and warning logic." + ] + } + ], + "nonGoals": [ + "This is not a legal or certification-grade compliance measurement system.", + "This does not replace external tools such as MpxTool or RF instrumentation.", + "This does not implement full spectral analysis in the browser polling path.", + "This does not yet define Prometheus/exporter metrics." + ], + "openQuestions": [ + { + "id": "Q1", + "question": "Should pilot and RDS percentages be derived from configured injection targets, measured component values, or both?", + "currentPreference": "Expose measured component values and derived operator-facing injection-equivalent display fields separately, with the RDS derivation documented mathematically before implementation." + }, + { + "id": "Q2", + "question": "Should internal thresholds be stored as normalized signal values or broadcast-style deviation percentages?", + "currentPreference": "Store normalized internal values and format into broadcast-friendly display values in UI." + }, + { + "id": "Q3", + "question": "Should /measurements include short rolling history arrays?", + "currentPreference": "No for MVP; keep endpoint single-snapshot and let UI build short client-side history." + }, + { + "id": "Q4", + "question": "Should long-term ownership remain in generator or later move into engine stats?", + "currentPreference": "Generator ownership first, engine exposure second." + }, + { + "id": "Q5", + "question": "Should clipperOrProtectionActive exist as a first-class field or be derived from raw clipper diagnostics?", + "currentPreference": "Expose raw clipper diagnostics such as lookahead gain and envelope; only expose the boolean if its derivation rule is explicitly documented." + }, + { + "id": "Q6", + "question": "Should a second optional pre-watermark engineering tap be added later?", + "currentPreference": "Not for MVP; only add it if there is a real diagnostic need." + } + ], + "reviewerFollowup": { + "resolvedContradictions": [ + "audioMpxPreBs412.shape now includes clipperLookaheadGain and clipperEnvelope so the data-model shape matches the documented preference for raw clipper diagnostics first.", + "rdsInjectionEquivalentPercent is now explicitly marked as a mathematically pending field that must be omitted, null, or TODO-gated in MVP until its derivation is fixed." + ], + "implementationGuardrails": [ + "Do not let developers treat dataModelProposal.shape as permission to drop raw clipper diagnostics in favor of only a boolean.", + "Do not implement rdsInjectionEquivalentPercent with an ad-hoc plausible-looking formula; either define it rigorously first or keep it unexposed in MVP." + ] + }, + "recommendedNextStep": { + "action": "implement phase 1", + "firstConcreteWorkItems": [ + "Define MeasurementSnapshot and nested structs including flags and stereoMode.", + "Add latest-measurement storage/accessor in generator.", + "Instrument GenerateFrame() at lr_pre_encode_post_watermark, audio_mpx_pre_bs412, audio_mpx_post_bs412, and composite_final_pre_iq.", + "Keep derived injection-equivalent fields and raw RMS fields explicitly separate from day one.", + "Define rdsInjectionEquivalentPercent mathematically before exposing it.", + "Treat licenseInjectionActive as chunk-local actual activity, not feature presence.", + "Prefer raw clipper diagnostics over an underdefined active boolean unless the boolean rule is documented." + ] + } +} diff --git a/docs/composite-mpx-metering-concept.md b/docs/composite-mpx-metering-concept.md new file mode 100644 index 0000000..496ffc2 --- /dev/null +++ b/docs/composite-mpx-metering-concept.md @@ -0,0 +1,787 @@ +# Composite / MPX Live Metering Concept + +Status: Draft +Version: 3.1 +Scope: `fm-rds-tx` runtime, control API, Overview UI, Flow UI, diagnostics + +## 1. Summary + +`fm-rds-tx` already exposes a good operational control plane: config state, runtime state, queue health, underruns, transitions, fault history, and a flow-oriented UI concept. What is still missing is **true runtime broadcast-style signal metering** for the multiplex chain itself. + +Right now, the UI mostly shows: +- configured pilot level +- configured RDS injection +- configured MPX gain +- BS.412 enabled/disabled +- composite clipper enabled/disabled + +Those values are useful, but they describe **intent**, not **measured runtime signal behavior**. + +The goal of this concept is to turn the `Composite / MPX` area into a **real metering surface** by measuring several semantically distinct points in the DSP chain and exposing the results through a dedicated high-frequency measurements endpoint. + +--- + +## 2. Problem Statement + +### 2.1 Current situation + +The current UI and API are good at showing: +- control/config state +- runtime state +- queue health +- underruns / faults +- applied vs desired frequency +- flow-node semantics for source, ingest, audio, processing, stereo, RDS, MPX, TX + +But they are **not yet good at showing what the multiplex signal is actually doing**. + +The current `Composite / MPX` presentation is still mostly a **config summary**, not a **measured signal view**. + +### 2.2 Operational gap + +Without actual runtime metering, an operator cannot quickly answer questions like: +- Is the multiplex chain behaving nominally right now? +- Is BS.412 actually pulling audio MPX down? +- Is the final composite peaking too high? +- Is pilot actually present at the expected level? +- Is RDS actually present at the expected injection? +- Is the issue caused by audio processing, compliance shaping, or final composite assembly? + +A single config snapshot cannot answer those questions. + +### 2.3 Why this matters + +Broadcast-style operation needs more than transport/runtime health. The system should expose enough internal signal truth that the operator can understand: +- what the audio path is doing +- what the compliance path is doing +- what the final multiplex before FM modulation looks like + +--- + +## 3. Objective + +### Primary objective + +Turn the `Composite / MPX` area of `fm-rds-tx` into a **true runtime metering surface** that reflects measured DSP behavior instead of only saved configuration values. + +### Secondary objectives + +- Give operators a trustworthy view of what the DSP chain is actually producing. +- Separate audio-path behavior, BS.412 compliance behavior, and final composite/on-air behavior. +- Support a richer Overview and Flow UI without increasing full `/runtime` polling cost. +- Create a stable base for future alerts, trends, and engineering diagnostics. + +--- + +## 4. What We Are Trying to Achieve + +### 4.1 Show real measured multiplex behavior + +The UI should reflect **measured runtime signal values** at key DSP stages, not only configured values. + +### 4.2 Explain stage interactions + +The system should make visible how these stages interact: +- audio processing +- stereo encoding +- composite clipping / protection +- BS.412 shaping +- pilot injection +- RDS injection +- final composite assembly + +### 4.3 Separate compliance and on-air views + +The operator should be able to distinguish: +- raw audio MPX behavior +- BS.412-shaped audio MPX behavior +- final composite behavior right before FM modulation + +### 4.4 Support operator decisions + +The UI should help answer: +- nominal or suspicious? +- compliance-limited or not? +- pilot / RDS present or not? +- overdriven or not? +- where in the chain does the problem appear? + +### 4.5 Stay aligned with the actual DSP architecture + +Metering should map directly onto the existing `GenerateFrame()` signal flow so the system stays semantically clean. + +--- + +## 5. Design Principles + +- Meter **measured runtime signal states**, not only configured target values. +- Align taps with the real DSP chain already implemented in `internal/offline/generator.go`. +- Keep **audio/compliance measurements** separate from **final composite/on-air measurements**. +- Add a dedicated measurements endpoint instead of increasing full UI polling frequency. +- Keep the data model stable enough for UI, diagnostics, and future alerting. +- Use chunk-level aggregation. +- Keep the hot path cheap: no heavy locking, no per-sample allocations. +- Prefer conservative, precise field names over catchy but ambiguous meter terminology. + +--- + +## 6. Review-Driven Semantic Corrections + +Before implementation, the following semantics should be corrected or clarified: + +### 6.1 L/R tap and watermarking + +There is an optional STFT watermark step in the code path after the first-pass audio processing and before stereo encoding. + +That means a tap named simply `lr_post_processing` is ambiguous: +- it may describe the processed L/R before watermarking, or +- it may describe the effective L/R that actually enters the stereo encoder + +For MVP, the more useful semantic choice is: +- **measure after watermark insertion, immediately before stereo encoding** + +This makes the L/R meter describe the actual signal that goes into stereo encoding. + +### 6.2 Pilot and RDS percent-style fields + +This was the biggest conceptual risk in the first draft. + +A raw measured RMS of the pilot component is **not the same thing** as an operator-facing injection display value such as “pilot 9 %”. + +Similarly, an RDS energy measurement is not identical to a familiar “RDS 4 %” display concept. + +Therefore the API must separate: +- **raw measured fields** such as `pilotRms`, `pilotPeakAbs`, `rdsRms`, `rdsPeakAbs` +- **derived operator-facing fields** such as `pilotInjectionEquivalentPercent` and `rdsInjectionEquivalentPercent` + +The UI must never imply that RMS itself equals injection percent. + +### 6.3 Composite threshold counters + +Names like `over100Events` and `over110Events` are too suggestive and invite wrong interpretation as if they were legal overmodulation indicators. + +They are not. + +For MVP they should use semantically safer names such as: +- `overNominalEvents` +- `overHeadroomEvents` + +And the docs must state clearly that they are based on **internal normalized composite thresholds**, not legal overmodulation judgement. + +### 6.4 Composite clipper metrics + +A true composite clipper / protection stage is not always semantically equivalent to simple hard clipping. + +So the concept should avoid overpromising exact `clipEvents` semantics at the composite-clipping tap. + +Safer metrics are: +- `peakAbs` +- `crestFactor` +- `clipperOrProtectionActive` + +This avoids lying with names. + +### 6.5 BS.412 semantics + +The BS.412 path in the current code behaves as a chunk-based running control state. + +That is fine. + +But UI and docs must describe `bs412GainApplied` accordingly: +- it is a **chunk-level running gain state** +- not a sample-exact causal event marker + +### 6.6 Final-composite interpretation flags + +The final composite may include more than just audio MPX + pilot + RDS. + +It can also include: +- RDS2 +- watermark side-effects upstream +- license injection + +So the measurement snapshot should carry flags such as: +- `stereoEnabled` +- `stereoMode` +- `rdsEnabled` +- `rds2Enabled` +- `bs412Enabled` +- `compositeClipperEnabled` +- `watermarkEnabled` +- `licenseInjectionActive` + +That gives the UI enough context to interpret the meters correctly. + +### 6.7 Clipper diagnostics should be explicit + +`clipperOrProtectionActive` is acceptable only if its derivation is explicitly defined. + +Otherwise it becomes a soft, marketing-like boolean. + +Preferred approach: +- expose raw clipper diagnostics such as `clipperLookaheadGain` and `clipperEnvelope` +- optionally derive a simpler active boolean later in UI or in a well-documented API rule + +### 6.8 licenseInjectionActive must be chunk-local + +`licenseInjectionActive` should not mean merely that a license/jingle feature exists. + +It should mean: +- in the current chunk, a non-zero license/jingle contribution was actually mixed into the final composite + +That is the operator-meaningful interpretation. + +### 6.9 rdsInjectionEquivalentPercent needs a hard definition + +This field remains useful, but it must be mathematically defined before implementation. + +The concept should not leave open whether it means: +- a peak-derived equivalent +- a scaling-derived equivalent +- or a smoothed deviation proxy + +The exact derivation needs to be fixed before Phase 1 code lands. + +--- + +## 7. Why Multiple Taps Are Necessary + +One final composite measurement is not enough. + +A single final reading cannot tell the operator whether a change came from: +- L/R processing +- stereo encode balance +- composite clipping +- BS.412 gain reduction +- pilot/RDS injection +- final multiplex summation + +Also: +- **BS.412 acts on audio MPX**, not on pilot/RDS directly. +- **Pilot and RDS become meaningful in the final composite stage**. +- **L/R meters should be taken immediately before stereo encoding** if they are meant to reflect the effective encoded audio input. + +So the design must sample multiple semantically distinct points. + +--- + +## 8. Measurement Taps + +## 8.1 L/R Pre-Encode Post-Watermark + +### Meaning +Effective left/right program audio immediately before stereo encoding. + +### Code location +`internal/offline/generator.go` → `GenerateFrame()` +After optional watermark has been added back into `lBuf` / `rBuf`, before stereo encoding begins. + +### Why it exists +This is the best operator-facing L/R meter because it reflects the actual signal entering the stereo encoder, even when watermarking is active. + +### Suggested metrics +- `lRms` +- `rRms` +- `lPeakAbs` +- `rPeakAbs` +- `lrBalanceDb` +- `lClipEvents` +- `rClipEvents` + +### Primary UI use +- Overview L/R meters +- optional diagnostics / engineering details + +### Note +If later needed, an additional pre-watermark engineering tap can be added separately. It should not replace this main operator-facing tap. + +--- + +## 8.2 Audio MPX Pre-BS.412 + +### Meaning +Audio-only multiplex after stereo encode and composite clipper/protection, but before BS.412 gain reduction. + +### Code location +`internal/offline/generator.go` → `GenerateFrame()` +Right after `audioMPX` has been built and processed by: +- composite clipper, or +- legacy clip + notch path + +and before `bs412Gain` is applied. + +### Why it exists +This is the best place to see what the multiplex audio path is structurally producing **before compliance shaping**. + +### Suggested metrics +- `rms` +- `peakAbs` +- `monoRms` +- `stereoRms` +- `crestFactor` +- `clipperLookaheadGain` +- `clipperEnvelope` +- `clipperOrProtectionActive` + +### Primary UI use +- diagnostics +- Flow popover deep view +- future engineering view + +--- + +## 8.3 Audio MPX Post-BS.412 + +### Meaning +Audio-only multiplex after BS.412 gain reduction and before fixed protected components are added. + +### Code location +`internal/offline/generator.go` → `GenerateFrame()` +Immediately after: +- BS.412 gain is applied to `audioMPX` + +and before: +- pilot is added +- RDS is added +- final composite is formed + +### Why it exists +This is the operator’s compliance view of the audio multiplex path. + +It answers: +- is BS.412 active? +- how much attenuation is currently applied? +- what does the audio MPX budget look like after compliance shaping? + +### Suggested metrics +- `rms` +- `peakAbs` +- `bs412GainApplied` +- `bs412AttenuationDb` +- `estimatedAudioPower` + +### Primary UI use +- compliance details +- Flow popover deep view +- diagnostics + +### Note +`bs412GainApplied` is a running chunk-level control value, not a sample-instantaneous causal diagnosis. + +--- + +## 8.4 Composite Final Pre-IQ + +### Meaning +The actual final composite signal immediately before FM modulation or direct composite output. + +### Code location +`internal/offline/generator.go` → `GenerateFrame()` +At the final `composite` variable just before: +- `fmMod.Modulate(composite)` +- or direct composite write when FM modulation is disabled + +### Why it exists +This is the most operator-relevant metering point. + +It reflects the actual final multiplex including: +- audio MPX +- pilot +- RDS +- optional RDS2 +- optional license/jingle injection + +### Suggested metrics +- `rms` +- `peakAbs` +- `pilotRms` +- `pilotPeakAbs` +- `pilotInjectionEquivalentPercent` +- `rdsRms` +- `rdsPeakAbs` +- `rdsInjectionEquivalentPercent` *(TODO until mathematically defined; omit or expose as null in MVP if not yet specified)* +- `overNominalEvents` +- `overHeadroomEvents` + +### Primary UI use +- Overview MPX card +- Flow `Composite / MPX` node +- future sparklines and operator summary + +### Note +The percent-style fields are derived display values and must stay explicitly separate from the raw RMS fields. + +`rdsInjectionEquivalentPercent` must not be implemented until its mathematical derivation is explicitly documented. + +--- + +## 9. Metric Semantics + +### RMS +Chunk-local RMS for the signal component. + +### PeakAbs +Maximum absolute sample magnitude within the chunk. + +### InjectionEquivalentPercent +A derived operator-facing value that expresses the component in a deviation-/injection-style display form. It is not identical to RMS. + +### ClipEvents +Use only where the tapped stage really has meaningful hard-threshold clipping semantics. + +### BS.412AttenuationDb +A user-facing gain-reduction metric derived from pre/post behavior. + +### EstimatedAudioPower +A chunk-local operator metric for compliance visibility, not a certification-grade legal measurement. + +### OverNominal / OverHeadroom Events +Internal normalized composite-threshold counters. These are not legal overmodulation indicators. + +### CrestFactor +Useful where shape/protection stages do not map cleanly to simple clip-event semantics. + +--- + +## 10. Data Model Proposal + +Suggested top-level Go type: +- `MeasurementSnapshot` + +Suggested structure: +- top-level snapshot metadata +- `flags` +- `lrPreEncodePostWatermark` +- `audioMpxPreBs412` +- `audioMpxPostBs412` +- `compositeFinalPreIq` + +Important internal consistency rule: +- if the concept says raw clipper diagnostics are preferred, the actual data-model shape must include them +- the shape must therefore include `clipperLookaheadGain` and `clipperEnvelope`, not only `clipperOrProtectionActive` + +Suggested top-level metadata: +- `timestamp` +- `sampleRateHz` +- `chunkSamples` +- `chunkDurationMs` +- `sequence` +- `stale` +- `noData` + +Suggested flags: +- `stereoEnabled` +- `stereoMode` +- `rdsEnabled` +- `rds2Enabled` +- `bs412Enabled` +- `compositeClipperEnabled` +- `watermarkEnabled` +- `licenseInjectionActive` + +This should be stable enough that: +- UI can consume it directly +- API docs can describe it clearly +- tests can assert it deterministically +- future diagnostics can extend it without breaking MVP consumers + +--- + +## 11. Aggregation Approach + +### Approach +Use **chunk-local accumulation** during one `GenerateFrame()` call. + +### Mechanics +For each relevant tap, accumulate: +- sum of squares +- max absolute value +- event counters +- any required derived numerators + +At the end of the chunk: +- finalize RMS +- compute derived ratios and dB values +- publish one immutable snapshot + +### Why this approach +- aligns with current generator architecture +- avoids expensive runtime contention +- is enough for 4–10 Hz UI metering +- avoids unbounded telemetry history in the hot path + +--- + +## 12. Recommended Placement + +### Primary owner: generator + +The generator should own measurement production because that is where the semantically rich intermediate signals exist. + +The generator already sees: +- L / R +- mono component +- stereo component +- pilot component +- RDS component +- audio MPX +- final composite + +If measurement is delayed until engine or driver level, too much stage identity is lost. + +### Publication model +Preferred model: +- generator computes snapshot +- generator stores latest snapshot in an atomic pointer or similarly cheap accessor +- control/runtime layer reads latest completed snapshot through a getter + +--- + +## 13. API Proposal + +## 13.1 New endpoint +- `GET /measurements` + +## 13.2 Why separate endpoint +Do **not** overload `/runtime` for fast metering. + +Reason: +- `/runtime` is broad operational telemetry +- metering wants higher refresh +- metering payload shape is different +- UI should not poll the full runtime object at 5–10 Hz + +## 13.3 Polling recommendation +- minimum: `4 Hz` +- recommended: `5 Hz` +- maximum MVP target: `10 Hz` + +## 13.4 Empty state behavior +When TX is inactive, either: +- return `noData=true`, or +- return last snapshot with `stale=true` + +Preferred MVP behavior: +- explicit `noData=true` when there is no current meaningful signal snapshot + +--- + +## 14. UI Integration Plan + +## 14.1 Overview + +Add or extend a metering section for: +- measured L/R +- measured final MPX/composite +- measured pilot values +- measured RDS values +- derived injection-equivalent display values where useful + +Important: +- do not replace queue/runtime health with signal metering +- keep operational health and signal health as separate concepts +- do not label derived percent-style values as RMS values + +## 14.2 Flow Tab + +### Composite / MPX node +Today it is mostly config-derived. + +Target: +- use measurement data as primary live source +- keep config metadata only as fallback or context +- include flags-based interpretation context when watermark, RDS2, or license injection materially changes the final composite +- include `stereoMode` context when non-DSB stereo operation changes expectations + +### Popover +Show: +- pre-BS.412 vs post-BS.412 +- final composite peak +- measured pilot +- measured RDS +- compliance hints +- optionally both raw values and operator-friendly injection-equivalent values + +## 14.3 Diagnostics + +Later, add a dedicated engineering meter view that can show: +- all taps together +- optional short sparklines +- threshold explanations +- stale/no-data states + +--- + +## 15. Status Logic Proposal + +### Composite / MPX node states + +#### Green +- measurements available +- final composite within expected internal normalized envelope +- pilot and RDS near target windows +- no strong compliance concern + +#### Amber +- measurement stale +- pilot or RDS materially off target +- final composite high +- BS.412/compliance activity visible +- restart-pending settings affect MPX semantics + +#### Red +- measurement unavailable while TX is running +- final composite repeatedly exceeds an internal critical envelope threshold +- pilot missing while stereo is enabled +- RDS missing while RDS is enabled and expected + +#### Gray +- TX idle / no active signal path + +--- + +## 16. Performance Constraints + +### Must +- no contended locks in the per-sample loop +- no heap allocation per sample +- one snapshot publication per chunk at most +- endpoint reads must be cheap + +### Acceptable cost +- a few additional per-sample accumulations +- a small finalize step with `sqrt` and `log` at chunk end + +--- + +## 17. Implementation Phases + +## Phase 1 — Measurement model and generator instrumentation + +Deliverables: +- define `MeasurementSnapshot` +- define nested stage structs +- define `flags` +- add chunk accumulators in `GenerateFrame()` +- publish latest completed snapshot from generator + +## Phase 2 — Control-plane endpoint + +Deliverables: +- add `GET /measurements` +- wire snapshot access into control server +- document the endpoint in `docs/API.md` +- test active/stale/noData behavior + +## Phase 3 — UI integration MVP + +Deliverables: +- poll `/measurements` separately at 4–10 Hz +- add measured MPX display in Overview +- drive Flow `Composite / MPX` from measurement data + +## Phase 4 — Diagnostics refinement + +Deliverables: +- engineering-oriented metering details +- optional client-side trend buffers / sparklines +- refined thresholds and warnings + +--- + +## 18. Non-Goals + +This concept is **not**: +- a legal or certification-grade compliance measurement system +- a replacement for external tools such as MpxTool +- a replacement for RF instrumentation +- a browser-side spectral analysis project +- a Prometheus/export design yet + +--- + +## 19. Open Questions + +### Q1 +Should pilot and RDS display percentages be derived from: +- configured target values, +- measured runtime values, +- or both? + +Current preference: +- expose raw measured values and derived operator-facing injection-equivalent values separately + +### Q2 +Should internal thresholds be stored as: +- normalized signal values, +- or broadcast-style deviation percentages? + +Current preference: +- normalized internally, broadcast-friendly formatting in UI + +### Q3 +Should `/measurements` include history arrays? + +Current preference: +- no in MVP +- keep endpoint single-snapshot +- let UI build short client-side history + +### Q4 +Should long-term ownership remain in generator or later move into engine stats? + +Current preference: +- generator first +- broader engine exposure later if needed + +### Q5 +Should `clipperOrProtectionActive` exist as a first-class field or be derived from raw clipper diagnostics? + +Current preference: +- expose raw clipper diagnostics such as `clipperLookaheadGain` and `clipperEnvelope` +- only expose the boolean if the derivation rule is explicitly documented + +### Q6 +Should a separate pre-watermark engineering tap exist later? + +Current preference: +- not for MVP +- only add it if there is a real diagnostic need + +--- + +## 20. Recommended Next Step + +Implement **Phase 1** first. + +Concrete first steps: +1. Define `MeasurementSnapshot`, nested structs, flags, and `stereoMode`. +2. Add latest-measurement storage/accessor in generator. +3. Instrument `GenerateFrame()` at: + - `lr_pre_encode_post_watermark` + - `audio_mpx_pre_bs412` + - `audio_mpx_post_bs412` + - `composite_final_pre_iq` +4. Keep raw measured fields and derived injection-equivalent display fields explicitly separate from day one. +5. Define `rdsInjectionEquivalentPercent` mathematically before exposing it. +6. Treat `licenseInjectionActive` as chunk-local actual activity, not feature presence. +7. Prefer raw clipper diagnostics over an underdefined active boolean unless the boolean rule is documented. + +That gets the measurement semantics right first. API and UI can then be built on top of a stable internal model. + +--- + +## 21. Reviewer Follow-Up Incorporated + +Two implementation guardrails are worth stating explicitly: + +1. **Raw clipper diagnostics must not disappear from the actual shape.** + If the concept prefers `clipperLookaheadGain` and `clipperEnvelope`, then the data model must contain them. It is not sufficient to mention them in examples while leaving only `clipperOrProtectionActive` in the formal shape. + +2. **`rdsInjectionEquivalentPercent` must not be guessed into existence.** + Until its derivation is mathematically fixed, MVP should treat it as: + - omitted, + - `null`, or + - a clearly gated TODO field. + +Anything else risks permanently baking an arbitrary-looking formula into the API.