| @@ -0,0 +1,299 @@ | |||||
| # AGENTS.md | |||||
| This file is the repo-level working guide for humans, coding agents, and LLMs. | |||||
| Read it before making changes. | |||||
| --- | |||||
| ## 1. Purpose of this file | |||||
| Use this file as the canonical "how to work in this repo" guide. | |||||
| It is intentionally practical and operational. | |||||
| Use it to answer questions like: | |||||
| - Where should changes go? | |||||
| - What must not be committed? | |||||
| - How should builds/tests be run? | |||||
| - Which docs are canonical? | |||||
| - How should debugging work be documented? | |||||
| - How should agents behave when touching this repo? | |||||
| --- | |||||
| ## 2. Repo intent | |||||
| `sd r-wideband-suite` is a Go-based SDR analysis and streaming system with: | |||||
| - live spectrum/waterfall UI | |||||
| - signal detection/classification | |||||
| - extraction / demodulation / recording | |||||
| - GPU-assisted paths | |||||
| - streaming audio paths | |||||
| - extensive telemetry/debugging support | |||||
| This repo has gone through active streaming-path and audio-click debugging. | |||||
| Do not assume older comments, notes, or experimental code paths are still authoritative. | |||||
| Prefer current code, current docs in `docs/`, and current branch state over historical assumptions. | |||||
| --- | |||||
| ## 3. Canonical documentation | |||||
| ### Keep as primary references | |||||
| - `README.md` | |||||
| - high-level project overview | |||||
| - build/run basics | |||||
| - feature summary | |||||
| - `ROADMAP.md` | |||||
| - longer-lived architectural direction | |||||
| - `docs/known-issues.md` | |||||
| - curated open engineering issues | |||||
| - `docs/telemetry-api.md` | |||||
| - telemetry endpoint documentation | |||||
| - `docs/telemetry-debug-runbook.md` | |||||
| - telemetry/debug operating guide | |||||
| - `docs/audio-click-debug-notes-2026-03-24.md` | |||||
| - historical incident record and final resolution notes for the audio-click investigation | |||||
| ### Treat as historical / contextual docs | |||||
| Anything in `docs/` that reads like an incident log, deep debug note, or one-off investigation should be treated as supporting context, not automatic source of truth. | |||||
| ### Do not create multiple competing issue lists | |||||
| If new open problems are found: | |||||
| - update `docs/known-issues.md` | |||||
| - keep raw reviewer/ad-hoc reports out of the main repo flow unless they are converted into curated docs | |||||
| --- | |||||
| ## 4. Branching and workflow rules | |||||
| ### Current working model | |||||
| - Use focused branches for real feature/fix work. | |||||
| - Do not keep long-lived junk/debug branches alive once the useful work has been transferred. | |||||
| - Prefer short-lived cleanup branches for docs/config cleanup. | |||||
| ### Branch hygiene | |||||
| - Do not pile unrelated work onto one branch if it can be split cleanly. | |||||
| - Keep bugfixes, config cleanup, and large refactors logically separable when possible. | |||||
| - Before deleting an old branch, ensure all useful work is already present in the active branch or merged into the main line. | |||||
| ### Mainline policy | |||||
| - Do not merge to `master` blindly. | |||||
| - Before merge, prefer at least a short sanity pass on: | |||||
| - live playback | |||||
| - recording | |||||
| - WFM / WFM_STEREO / at least one non-WFM mode if relevant | |||||
| - restart behavior if the change affects runtime state | |||||
| --- | |||||
| ## 5. Commit policy | |||||
| ### Commit what matters | |||||
| Good commits are: | |||||
| - real code fixes | |||||
| - clear docs improvements | |||||
| - deliberate config-default changes | |||||
| - cleanup that reduces confusion | |||||
| ### Do not commit accidental noise | |||||
| Do **not** commit unless explicitly intended: | |||||
| - local debug dumps | |||||
| - ad-hoc telemetry exports | |||||
| - generated WAV debug windows | |||||
| - temporary patch files | |||||
| - throwaway reviewer JSON snapshots | |||||
| - local-only runtime artifacts | |||||
| ### Prefer small, readable commit scopes | |||||
| Examples of good separate commit scopes: | |||||
| - code fix | |||||
| - config default cleanup | |||||
| - doc cleanup | |||||
| - known-issues update | |||||
| --- | |||||
| ## 6. Files and paths that need extra care | |||||
| ### Config files | |||||
| - `config.yaml` | |||||
| - `config.autosave.yaml` | |||||
| Rules: | |||||
| - These can drift during debugging. | |||||
| - Do not commit config changes accidentally. | |||||
| - Only commit them when the intent is to change repo defaults. | |||||
| - Keep in mind that `config.autosave.yaml` can override expected runtime behavior after restart. | |||||
| ### Debug / dump artifacts | |||||
| Examples: | |||||
| - `debug/` | |||||
| - `tele-*.json` | |||||
| - ad-hoc patch/report scratch files | |||||
| - generated WAV capture windows | |||||
| Rules: | |||||
| - Treat these as local investigation material unless intentionally promoted into docs. | |||||
| - Do not leave them hanging around as tracked repo clutter. | |||||
| ### Root docs | |||||
| The repo root should stay relatively clean. | |||||
| Keep only genuinely canonical top-level docs there. | |||||
| One-off investigation output belongs in `docs/` or should be deleted. | |||||
| --- | |||||
| ## 7. Build and test rules | |||||
| ### General rule | |||||
| Prefer the repo's own scripts and established workflow over ad-hoc raw build commands. | |||||
| ### Important operational rule | |||||
| Before coding/build/test sessions on this repo: | |||||
| - stop the browser UI | |||||
| - stop `sdrd.exe` | |||||
| This avoids file locks, stale runtime state, and misleading live-test behavior. | |||||
| ### Build preference | |||||
| Use the project scripts where applicable, especially for the real app flows. | |||||
| Examples already used during this project include: | |||||
| - `build-sdrplay.ps1` | |||||
| - `start-sdr.ps1` | |||||
| Do **not** default to random raw `go build` commands for full workflow validation unless the goal is a narrow compile-only sanity check. | |||||
| ### GPU / native-path caution | |||||
| If working on GPU/native streaming code: | |||||
| - do not assume the CPU oracle path is currently trustworthy unless you have just validated it | |||||
| - do not assume old README notes inside subdirectories are current | |||||
| - check the current code and current docs first | |||||
| --- | |||||
| ## 8. Debugging rules | |||||
| ### Telemetry-first, but disciplined | |||||
| Telemetry is available and useful. | |||||
| However: | |||||
| - heavy telemetry can distort runtime behavior | |||||
| - debug config can accidentally persist via autosave | |||||
| - not every one-off probe belongs in permanent code | |||||
| ### When debugging | |||||
| Prefer this order: | |||||
| 1. existing telemetry and current docs | |||||
| 2. focused additional instrumentation | |||||
| 3. short-lived dumps / captures | |||||
| 4. cleanup afterward | |||||
| ### If you add debugging support | |||||
| Ask: | |||||
| - Is this reusable for future incidents? | |||||
| - Should it live in `docs/known-issues.md` or a runbook? | |||||
| - Is it temporary and should be removed after use? | |||||
| ### If a reviewer provides a raw report | |||||
| Do not blindly keep raw snapshots as canonical repo docs. | |||||
| Instead: | |||||
| - extract the durable findings | |||||
| - update `docs/known-issues.md` | |||||
| - keep only the cleaned/curated version in the main repo flow | |||||
| --- | |||||
| ## 9. Documentation rules | |||||
| ### Prefer curated docs over raw dumps | |||||
| Good: | |||||
| - `docs/known-issues.md` | |||||
| - runbooks | |||||
| - architectural notes | |||||
| - incident summaries with clear final status | |||||
| Bad: | |||||
| - random JSON reviewer dumps as primary docs | |||||
| - duplicate issue lists | |||||
| - stale TODO/STATE files that nobody maintains | |||||
| ### If a doc becomes stale | |||||
| Choose one: | |||||
| - update it | |||||
| - move it into `docs/` as historical context | |||||
| - delete it | |||||
| Do not keep stale docs in prominent locations if they compete with current truth. | |||||
| --- | |||||
| ## 10. Known lessons from recent work | |||||
| These are important enough to keep visible: | |||||
| ### Audio-click investigation lessons | |||||
| - The final click bug was not a single simple DSP bug. | |||||
| - Real causes included: | |||||
| - shared-buffer mutation / aliasing | |||||
| - extractor reset churn from unstable config hashing | |||||
| - streaming-path batch rejection / fallback behavior | |||||
| - Secondary contributing issues existed in discriminator bridging and WFM mono/plain-path filtering. | |||||
| ### Practical repo lessons | |||||
| - Silent fallback paths are dangerous; keep important fallthrough/fallback visibility. | |||||
| - Shared IQ buffers should be treated very carefully. | |||||
| - Debug artifacts should not become permanent repo clutter. | |||||
| - Curated issue tracking in Git is better than keeping raw review snapshots around. | |||||
| --- | |||||
| ## 11. Agent behavior expectations | |||||
| If you are an AI coding agent / LLM working in this repo: | |||||
| ### Do | |||||
| - read this file first | |||||
| - prefer current code and current docs over old assumptions | |||||
| - keep changes scoped and explainable | |||||
| - separate config cleanup from code fixes when possible | |||||
| - leave the repo cleaner than you found it | |||||
| - promote durable findings into curated docs | |||||
| ### Do not | |||||
| - commit local debug noise by default | |||||
| - create duplicate status/todo/issue files without a strong reason | |||||
| - assume experimental comments or old subdirectory READMEs are still correct | |||||
| - leave raw reviewer output as the only source of truth | |||||
| - hide fallback behavior or silently ignore critical path failures | |||||
| --- | |||||
| ## 12. Recommended doc update pattern after meaningful work | |||||
| When a meaningful fix or investigation lands: | |||||
| 1. update code | |||||
| 2. update any relevant canonical docs | |||||
| 3. update `docs/known-issues.md` if open issues changed | |||||
| 4. remove or archive temporary debug artifacts | |||||
| 5. keep the repo root and branch state clean | |||||
| --- | |||||
| ## 13. Minimal pre-commit checklist | |||||
| Before committing, quickly check: | |||||
| - Am I committing only intended files? | |||||
| - Are config changes intentional? | |||||
| - Am I accidentally committing dumps/logs/debug exports? | |||||
| - Should any reviewer findings be moved into `docs/known-issues.md`? | |||||
| - Did I leave stale temporary files behind? | |||||
| --- | |||||
| ## 14. If unsure | |||||
| If a file looks ambiguous: | |||||
| - canonical + actively maintained -> keep/update | |||||
| - historical but useful -> move or keep in `docs/` | |||||
| - stale and confusing -> delete | |||||
| Clarity beats nostalgia. | |||||
| @@ -192,6 +192,32 @@ go build -tags sdrplay ./cmd/sdrd | |||||
| - `GET /api/signals` -> current live signals | - `GET /api/signals` -> current live signals | ||||
| - `GET /api/events?limit=&since=` -> recent events | - `GET /api/events?limit=&since=` -> recent events | ||||
| ### Debug Telemetry | |||||
| - `GET /api/debug/telemetry/live` -> current telemetry snapshot (counters, gauges, distributions, recent events, collector status/config) | |||||
| - `GET /api/debug/telemetry/history` -> historical metric samples with filtering by time/name/prefix/tags | |||||
| - `GET /api/debug/telemetry/events` -> telemetry event/anomaly history with filtering by time/name/prefix/level/tags | |||||
| - `GET /api/debug/telemetry/config` -> current collector config plus `debug.telemetry` runtime config | |||||
| - `POST /api/debug/telemetry/config` -> update telemetry settings at runtime and persist them to autosave config | |||||
| Telemetry query params (`history` / `events`) include: | |||||
| - `since`, `until` -> unix seconds, unix milliseconds, or RFC3339 timestamps | |||||
| - `limit` | |||||
| - `name`, `prefix` | |||||
| - `signal_id`, `session_id`, `stage`, `trace_id`, `component` | |||||
| - `tag_<key>=<value>` for arbitrary tag filters | |||||
| - `include_persisted=true|false` (default `true`) | |||||
| - `level` on the events endpoint | |||||
| Telemetry config lives under `debug.telemetry`: | |||||
| - `enabled`, `heavy_enabled`, `heavy_sample_every` | |||||
| - `metric_sample_every`, `metric_history_max`, `event_history_max` | |||||
| - `retention_seconds` | |||||
| - `persist_enabled`, `persist_dir`, `rotate_mb`, `keep_files` | |||||
| See also: | |||||
| - `docs/telemetry-api.md` for the full telemetry API reference | |||||
| - `docs/telemetry-debug-runbook.md` for the short operational debug flow | |||||
| ### Recordings | ### Recordings | ||||
| - `GET /api/recordings` | - `GET /api/recordings` | ||||
| - `GET /api/recordings/:id` (meta.json) | - `GET /api/recordings/:id` (meta.json) | ||||
| @@ -1,184 +0,0 @@ | |||||
| # SDR Wideband Suite - Current State | |||||
| This file is the practical handoff / resume state for future work. | |||||
| Use it together with `ROADMAP.md`. | |||||
| - `ROADMAP.md` = long-term architecture and phase roadmap | |||||
| - `STATE.md` = current repo state, working conventions, and next recommended entry point | |||||
| ## Current Milestone State | |||||
| - **Phase 1 complete** | |||||
| - **Phase 2 complete** | |||||
| - **Phase 3 complete** | |||||
| - **Phase 4 complete** | |||||
| Current project state should be treated as: | |||||
| - Phase 1 = architecture foundation landed | |||||
| - Phase 2 = multi-resolution surveillance semantics landed | |||||
| - Phase 3 = conservative runtime prioritization/admission/rebalance landed | |||||
| - Phase 4 = monitor-window operating model landed | |||||
| Do not reopen these phases unless there is a concrete bug, mismatch, or regression. | |||||
| --- | |||||
| ## Most Recent Relevant Commits | |||||
| These are the most important recent milestone commits that define the current state: | |||||
| ### Phase 4 monitor-window operating model | |||||
| - `efe137b` Add monitor window goals for multi-span gating | |||||
| - `ac64d6b` Add monitor window matches and stats | |||||
| - `d7e457d` Expose monitor window summaries in runtime debug | |||||
| - `c520423` Add monitor window priority bias | |||||
| - `838c941` Add window-based record/decode actions | |||||
| - `962cf06` Add window zone biases for record/decode actions | |||||
| - `402a772` Consolidate monitor window summary in debug outputs | |||||
| - `8545b62` Add per-window outcome summaries for admission pressure | |||||
| - `65b9845` test: cover overlapping monitor windows | |||||
| - `efe3215` docs: capture Phase-4 monitor-window status | |||||
| ### Phase 3 runtime intelligence milestone | |||||
| - `4ebd51d` Add priority tiers and admission classes to pipeline | |||||
| - `18b179b` Expose admission metadata in debug output and tests | |||||
| - `ba9adca` Add budget preference and pressure modeling | |||||
| - `7a75367` Expose arbitration pressure summary | |||||
| - `592fa03` pipeline: deepen hold/displacement semantics | |||||
| - `30a5d11` pipeline: apply intent holds and family tier floors | |||||
| - `1f5d4ab` pipeline: add intent and family priority tests | |||||
| - `822829c` Add conservative budget rebalance layer | |||||
| - `da5fa22` Update Phase-3 Wave 3E status | |||||
| ### Documentation / stable defaults | |||||
| - `fd718d5` docs: finalize phase milestones and ukf test config | |||||
| If resuming after a long pause, inspect the current `git log` around these commits first. | |||||
| --- | |||||
| ## Current Important Files / Subsystems | |||||
| ### Long-term guidance | |||||
| - `ROADMAP.md` - durable roadmap across phases | |||||
| - `STATE.md` - practical resume/handoff state | |||||
| - `PLAN.md` - project plan / narrative (may be less pristine than ROADMAP.md) | |||||
| - `README.md` - user-facing/current feature status | |||||
| ### Config / runtime surface | |||||
| - `config.yaml` - current committed default config | |||||
| - `config.autosave.yaml` - local autosave; intentionally not tracked in git | |||||
| - `internal/config/config.go` | |||||
| - `internal/runtime/runtime.go` | |||||
| ### Phase 3 core runtime intelligence | |||||
| - `internal/pipeline/arbiter.go` | |||||
| - `internal/pipeline/arbitration.go` | |||||
| - `internal/pipeline/arbitration_state.go` | |||||
| - `internal/pipeline/priority.go` | |||||
| - `internal/pipeline/budget.go` | |||||
| - `internal/pipeline/pressure.go` | |||||
| - `internal/pipeline/rebalance.go` | |||||
| - `internal/pipeline/decision_queue.go` | |||||
| ### Phase 2 surveillance/evidence model | |||||
| - `internal/pipeline/types.go` | |||||
| - `internal/pipeline/evidence.go` | |||||
| - `internal/pipeline/candidate_fusion.go` | |||||
| - `internal/pipeline/scheduler.go` | |||||
| - `cmd/sdrd/pipeline_runtime.go` | |||||
| ### Phase 4 monitor-window model | |||||
| - `internal/pipeline/monitor_rules.go` | |||||
| - `cmd/sdrd/window_summary.go` | |||||
| - `cmd/sdrd/level_summary.go` | |||||
| - `cmd/sdrd/http_handlers.go` | |||||
| - `cmd/sdrd/decision_compact.go` | |||||
| - `cmd/sdrd/dsp_loop.go` | |||||
| --- | |||||
| ## Current Default Operator / Test Posture | |||||
| The repo was intentionally switched to an FM/UKW-friendly default test posture. | |||||
| ### Current committed config defaults | |||||
| - band: `87.5-108.0 MHz` | |||||
| - center: `99.5 MHz` | |||||
| - sample rate: `2.048 MHz` | |||||
| - FFT: `4096` | |||||
| - profile: `wideband-balanced` | |||||
| - intent: `broadcast-monitoring` | |||||
| - priorities include `wfm`, `rds`, `broadcast`, `digital` | |||||
| ### Important config note | |||||
| - `config.yaml` is committed and intended as the stable default reference | |||||
| - `config.autosave.yaml` is **not** git-tracked and may diverge locally | |||||
| - if behavior seems odd, compare the active runtime config against `config.yaml` | |||||
| --- | |||||
| ## Working Conventions That Matter | |||||
| ### Codex invocation on Windows | |||||
| Preferred stable flow: | |||||
| 1. write prompt to `codex_prompt.txt` | |||||
| 2. create/use `run_codex.ps1` containing: | |||||
| - read prompt file | |||||
| - pipe to `codex exec --yolo` | |||||
| 3. run with PTY/background from the repo root | |||||
| 4. remove `codex_prompt.txt` and `run_codex.ps1` after the run | |||||
| This was adopted specifically to avoid PowerShell quoting failures. | |||||
| ### Expectations for coding runs | |||||
| - before every commit: `go test ./...` and `go build ./cmd/sdrd` | |||||
| - commit in coherent blocks with clear messages | |||||
| - push after successful validation | |||||
| - avoid reopening already-closed phase work without a concrete reason | |||||
| --- | |||||
| ## Known Practical Caveats | |||||
| - `PLAN.md` has had encoding/character issues in some reads; treat `ROADMAP.md` + `STATE.md` as the cleaner authoritative continuity docs. | |||||
| - README is generally useful, but `ROADMAP.md`/`STATE.md` are better for architectural continuity. | |||||
| - `config.autosave.yaml` can become misleading because it is local/autosaved and not tracked. | |||||
| --- | |||||
| ## Recommended Next Entry Point | |||||
| If resuming technical work after this checkpoint: | |||||
| ### Start with **Phase 5** | |||||
| Do **not** reopen Phase 1-4 unless there is a concrete bug or regression. | |||||
| ### Recommended Phase 5 direction | |||||
| Move from monitor windows inside a single capture span toward richer span / operating orchestration: | |||||
| - span / zone groups | |||||
| - span-aware resource allocation | |||||
| - stronger profile-driven operating modes | |||||
| - retune / scan / dwell semantics where needed | |||||
| ### Avoid jumping ahead prematurely to | |||||
| - full adaptive QoS engine (Phase 6) | |||||
| - major GPU/performance re-architecture (Phase 7) | |||||
| - heavy UX/product polish (Phase 8) | |||||
| Those should build on Phase 5, not bypass it. | |||||
| --- | |||||
| ## Resume Checklist For A Future Agent | |||||
| 1. Read `ROADMAP.md` | |||||
| 2. Read `STATE.md` | |||||
| 3. Check current `git log` near the commits listed above | |||||
| 4. Inspect `config.yaml` | |||||
| 5. Confirm current repo state with: | |||||
| - `go test ./...` | |||||
| - `go build ./cmd/sdrd` | |||||
| 6. Then start Phase 5 planning from the actual repo state | |||||
| If these steps still match the repo, continuation should be seamless enough even after a hard context reset. | |||||
| @@ -1,23 +0,0 @@ | |||||
| # TODO — SDR Visual Suite | |||||
| ## UI | |||||
| - [ ] RDS RadioText (RT) Anzeige hinzufügen: | |||||
| - Overlay: 1 Zeile, sanfter Fade bei Updates, Ellipsis bei Überlänge, optional kleines „RT“-Badge. | |||||
| - Detail-Panel: 2 Zeilen Auto-Wrap; bei Überlänge Ellipsis + Expand (Modal/Zone) für Volltext. | |||||
| - Update-Logik: RT nur bei stabilem Text (z. B. 2–3 identische Blöcke), optional „RT · HH:MM“ Timestamp. | |||||
| ## Band Settings Profiles (v1.2) | |||||
| - [ ] Backend: built-in Profile-Struktur + embedded JSON (6 Profile) | |||||
| - [ ] Backend: Apply-Helper (shared mit /api/config) inkl. source/dsp/save | |||||
| - [ ] Backend: Merge-Patch mit Feld-Präsenz (nur explizite Felder anwenden) | |||||
| - [ ] Backend: DisallowUnknownFields + Config-Validierung → 400 | |||||
| - [ ] Backend: Endpoints GET /api/profiles, POST /api/profiles/apply, POST /api/profiles/undo, GET /api/profiles/suggest | |||||
| - [ ] Backend: Undo-Snapshot (1 Level) + Active Profile ID (Runtime-State) | |||||
| - [ ] Optional: Active Profile ID über Neustart persistieren (falls gewünscht) | |||||
| - [ ] UI: Dropdown + Split-Apply (full/dsp_only) + Undo + Active-Badge | |||||
| - [ ] UI: Suggest-Toast bei center_hz Wechsel, Dismiss-Schutz (>5 MHz) | |||||
| - [ ] UX: Loading-Indicator während Profilwechsel (1–3s Reset) | |||||
| - [ ] Tests: Patch-Semantik, dsp_only (center_hz/gain_db bleiben), Unknown Fields, Suggest-Match | |||||
| ## Notes | |||||
| - Ab jetzt hier die Todo-Liste führen. | |||||
| @@ -16,12 +16,25 @@ if (!(Test-Path $outDir)) { New-Item -ItemType Directory -Path $outDir | Out-Nul | |||||
| Remove-Item $dll,$lib,$exp -Force -ErrorAction SilentlyContinue | Remove-Item $dll,$lib,$exp -Force -ErrorAction SilentlyContinue | ||||
| $cmd = @" | |||||
| call "$vcvars" && "$nvcc" -shared "$src" -o "$dll" -cudart=hybrid -Xcompiler "/MD" -arch=sm_75 -gencode arch=compute_75,code=sm_75 -gencode arch=compute_80,code=sm_80 -gencode arch=compute_86,code=sm_86 -gencode arch=compute_89,code=sm_89 -gencode arch=compute_90,code=sm_90 | |||||
| $bat = Join-Path $env:TEMP 'build-gpudemod-dll.bat' | |||||
| $batContent = @" | |||||
| @echo off | |||||
| call "$vcvars" | |||||
| if errorlevel 1 exit /b %errorlevel% | |||||
| "$nvcc" -shared "$src" -o "$dll" -cudart=hybrid -Xcompiler "/MD" -arch=sm_75 ^ | |||||
| -gencode arch=compute_75,code=sm_75 ^ | |||||
| -gencode arch=compute_80,code=sm_80 ^ | |||||
| -gencode arch=compute_86,code=sm_86 ^ | |||||
| -gencode arch=compute_89,code=sm_89 ^ | |||||
| -gencode arch=compute_90,code=sm_90 | |||||
| exit /b %errorlevel% | |||||
| "@ | "@ | ||||
| Set-Content -Path $bat -Value $batContent -Encoding ASCII | |||||
| Write-Host 'Building gpudemod CUDA DLL...' -ForegroundColor Cyan | Write-Host 'Building gpudemod CUDA DLL...' -ForegroundColor Cyan | ||||
| cmd.exe /c $cmd | |||||
| if ($LASTEXITCODE -ne 0) { throw 'gpudemod DLL build failed' } | |||||
| cmd.exe /c ""$bat"" | |||||
| $exitCode = $LASTEXITCODE | |||||
| Remove-Item $bat -Force -ErrorAction SilentlyContinue | |||||
| if ($exitCode -ne 0) { throw 'gpudemod DLL build failed' } | |||||
| Write-Host "Built: $dll" -ForegroundColor Green | Write-Host "Built: $dll" -ForegroundColor Green | ||||
| @@ -21,10 +21,13 @@ if (Test-Path $sdrplayBin) { $env:PATH = "$sdrplayBin;" + $env:PATH } | |||||
| # CUDA runtime / cuFFT | # CUDA runtime / cuFFT | ||||
| $cudaInc = 'C:\CUDA\include' | $cudaInc = 'C:\CUDA\include' | ||||
| $cudaBin = 'C:\CUDA\bin' | $cudaBin = 'C:\CUDA\bin' | ||||
| $cudaBinX64 = 'C:\CUDA\bin\x64' | |||||
| if (-not (Test-Path $cudaInc)) { $cudaInc = 'C:\PROGRA~1\NVIDIA~2\CUDA\v13.2\include' } | if (-not (Test-Path $cudaInc)) { $cudaInc = 'C:\PROGRA~1\NVIDIA~2\CUDA\v13.2\include' } | ||||
| if (-not (Test-Path $cudaBin)) { $cudaBin = 'C:\PROGRA~1\NVIDIA~2\CUDA\v13.2\bin' } | if (-not (Test-Path $cudaBin)) { $cudaBin = 'C:\PROGRA~1\NVIDIA~2\CUDA\v13.2\bin' } | ||||
| if (-not (Test-Path $cudaBinX64)) { $cudaBinX64 = 'C:\PROGRA~1\NVIDIA~2\CUDA\v13.2\bin\x64' } | |||||
| $cudaMingw = Join-Path $PSScriptRoot 'cuda-mingw' | $cudaMingw = Join-Path $PSScriptRoot 'cuda-mingw' | ||||
| if (Test-Path $cudaInc) { $env:CGO_CFLAGS = "$env:CGO_CFLAGS -I$cudaInc" } | if (Test-Path $cudaInc) { $env:CGO_CFLAGS = "$env:CGO_CFLAGS -I$cudaInc" } | ||||
| if (Test-Path $cudaBinX64) { $env:PATH = "$cudaBinX64;" + $env:PATH } | |||||
| if (Test-Path $cudaBin) { $env:PATH = "$cudaBin;" + $env:PATH } | if (Test-Path $cudaBin) { $env:PATH = "$cudaBin;" + $env:PATH } | ||||
| if (Test-Path $cudaMingw) { $env:CGO_LDFLAGS = "$env:CGO_LDFLAGS -L$cudaMingw -lcudart64_13 -lcufft64_12 -lkernel32" } | if (Test-Path $cudaMingw) { $env:CGO_LDFLAGS = "$env:CGO_LDFLAGS -L$cudaMingw -lcudart64_13 -lcufft64_12 -lkernel32" } | ||||
| @@ -68,8 +71,11 @@ if ($dllSrc) { | |||||
| } | } | ||||
| $cudartCandidates = @( | $cudartCandidates = @( | ||||
| (Join-Path $cudaBinX64 'cudart64_13.dll'), | |||||
| (Join-Path $cudaBin 'cudart64_13.dll'), | (Join-Path $cudaBin 'cudart64_13.dll'), | ||||
| 'C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.2\bin\x64\cudart64_13.dll', | |||||
| 'C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.2\bin\cudart64_13.dll', | 'C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.2\bin\cudart64_13.dll', | ||||
| 'C:\CUDA\bin\x64\cudart64_13.dll', | |||||
| 'C:\CUDA\bin\cudart64_13.dll' | 'C:\CUDA\bin\cudart64_13.dll' | ||||
| ) | ) | ||||
| $cudartSrc = $cudartCandidates | Where-Object { $_ -and (Test-Path $_) } | Select-Object -First 1 | $cudartSrc = $cudartCandidates | Where-Object { $_ -and (Test-Path $_) } | Select-Object -First 1 | ||||
| @@ -3,6 +3,7 @@ package main | |||||
| import ( | import ( | ||||
| "context" | "context" | ||||
| "encoding/json" | "encoding/json" | ||||
| "fmt" | |||||
| "log" | "log" | ||||
| "os" | "os" | ||||
| "runtime/debug" | "runtime/debug" | ||||
| @@ -16,15 +17,16 @@ import ( | |||||
| "sdr-wideband-suite/internal/logging" | "sdr-wideband-suite/internal/logging" | ||||
| "sdr-wideband-suite/internal/pipeline" | "sdr-wideband-suite/internal/pipeline" | ||||
| "sdr-wideband-suite/internal/recorder" | "sdr-wideband-suite/internal/recorder" | ||||
| "sdr-wideband-suite/internal/telemetry" | |||||
| ) | ) | ||||
| func runDSP(ctx context.Context, srcMgr *sourceManager, cfg config.Config, det *detector.Detector, window []float64, h *hub, eventFile *os.File, eventMu *sync.RWMutex, updates <-chan dspUpdate, gpuState *gpuStatus, rec *recorder.Manager, sigSnap *signalSnapshot, extractMgr *extractionManager, phaseSnap *phaseSnapshot) { | |||||
| func runDSP(ctx context.Context, srcMgr *sourceManager, cfg config.Config, det *detector.Detector, window []float64, h *hub, eventFile *os.File, eventMu *sync.RWMutex, updates <-chan dspUpdate, gpuState *gpuStatus, rec *recorder.Manager, sigSnap *signalSnapshot, extractMgr *extractionManager, phaseSnap *phaseSnapshot, coll *telemetry.Collector) { | |||||
| defer func() { | defer func() { | ||||
| if r := recover(); r != nil { | if r := recover(); r != nil { | ||||
| log.Printf("FATAL: runDSP goroutine panic: %v\n%s", r, debug.Stack()) | log.Printf("FATAL: runDSP goroutine panic: %v\n%s", r, debug.Stack()) | ||||
| } | } | ||||
| }() | }() | ||||
| rt := newDSPRuntime(cfg, det, window, gpuState) | |||||
| rt := newDSPRuntime(cfg, det, window, gpuState, coll) | |||||
| ticker := time.NewTicker(cfg.FrameInterval()) | ticker := time.NewTicker(cfg.FrameInterval()) | ||||
| defer ticker.Stop() | defer ticker.Stop() | ||||
| logTicker := time.NewTicker(5 * time.Second) | logTicker := time.NewTicker(5 * time.Second) | ||||
| @@ -33,6 +35,9 @@ func runDSP(ctx context.Context, srcMgr *sourceManager, cfg config.Config, det * | |||||
| dcBlocker := dsp.NewDCBlocker(0.995) | dcBlocker := dsp.NewDCBlocker(0.995) | ||||
| state := &phaseState{} | state := &phaseState{} | ||||
| var frameID uint64 | var frameID uint64 | ||||
| prevDisplayed := map[int64]detector.Signal{} | |||||
| lastSourceDrops := uint64(0) | |||||
| lastSourceResets := uint64(0) | |||||
| for { | for { | ||||
| select { | select { | ||||
| case <-ctx.Done(): | case <-ctx.Done(): | ||||
| @@ -40,11 +45,28 @@ func runDSP(ctx context.Context, srcMgr *sourceManager, cfg config.Config, det * | |||||
| case <-logTicker.C: | case <-logTicker.C: | ||||
| st := srcMgr.Stats() | st := srcMgr.Stats() | ||||
| log.Printf("stats: buf=%d drop=%d reset=%d last=%dms", st.BufferSamples, st.Dropped, st.Resets, st.LastSampleAgoMs) | log.Printf("stats: buf=%d drop=%d reset=%d last=%dms", st.BufferSamples, st.Dropped, st.Resets, st.LastSampleAgoMs) | ||||
| if coll != nil { | |||||
| coll.SetGauge("source.buffer_samples", float64(st.BufferSamples), nil) | |||||
| coll.SetGauge("source.last_sample_ago_ms", float64(st.LastSampleAgoMs), nil) | |||||
| if st.Dropped > lastSourceDrops { | |||||
| coll.IncCounter("source.drop.count", float64(st.Dropped-lastSourceDrops), nil) | |||||
| } | |||||
| if st.Resets > lastSourceResets { | |||||
| coll.IncCounter("source.reset.count", float64(st.Resets-lastSourceResets), nil) | |||||
| coll.Event("source_reset", "warn", "source reset observed", nil, map[string]any{"resets": st.Resets}) | |||||
| } | |||||
| lastSourceDrops = st.Dropped | |||||
| lastSourceResets = st.Resets | |||||
| } | |||||
| case upd := <-updates: | case upd := <-updates: | ||||
| rt.applyUpdate(upd, srcMgr, rec, gpuState) | rt.applyUpdate(upd, srcMgr, rec, gpuState) | ||||
| dcBlocker.Reset() | dcBlocker.Reset() | ||||
| ticker.Reset(rt.cfg.FrameInterval()) | ticker.Reset(rt.cfg.FrameInterval()) | ||||
| if coll != nil { | |||||
| coll.IncCounter("dsp.update.apply", 1, nil) | |||||
| } | |||||
| case <-ticker.C: | case <-ticker.C: | ||||
| frameStart := time.Now() | |||||
| frameID++ | frameID++ | ||||
| art, err := rt.captureSpectrum(srcMgr, rec, dcBlocker, gpuState) | art, err := rt.captureSpectrum(srcMgr, rec, dcBlocker, gpuState) | ||||
| if err != nil { | if err != nil { | ||||
| @@ -61,8 +83,19 @@ func runDSP(ctx context.Context, srcMgr *sourceManager, cfg config.Config, det * | |||||
| rt.gotSamples = true | rt.gotSamples = true | ||||
| } | } | ||||
| logging.Debug("trace", "capture_done", "trace", frameID, "allIQ", len(art.allIQ), "detailIQ", len(art.detailIQ)) | logging.Debug("trace", "capture_done", "trace", frameID, "allIQ", len(art.allIQ), "detailIQ", len(art.detailIQ)) | ||||
| if coll != nil { | |||||
| coll.Observe("stage.capture.duration_ms", float64(time.Since(frameStart).Microseconds())/1000.0, telemetry.TagsFromPairs("frame_id", fmt.Sprintf("%d", frameID))) | |||||
| } | |||||
| survStart := time.Now() | |||||
| state.surveillance = rt.buildSurveillanceResult(art) | state.surveillance = rt.buildSurveillanceResult(art) | ||||
| if coll != nil { | |||||
| coll.Observe("stage.surveillance.duration_ms", float64(time.Since(survStart).Microseconds())/1000.0, telemetry.TagsFromPairs("frame_id", fmt.Sprintf("%d", frameID))) | |||||
| } | |||||
| refineStart := time.Now() | |||||
| state.refinement = rt.runRefinement(art, state.surveillance, extractMgr, rec) | state.refinement = rt.runRefinement(art, state.surveillance, extractMgr, rec) | ||||
| if coll != nil { | |||||
| coll.Observe("stage.refinement.duration_ms", float64(time.Since(refineStart).Microseconds())/1000.0, telemetry.TagsFromPairs("frame_id", fmt.Sprintf("%d", frameID))) | |||||
| } | |||||
| finished := state.surveillance.Finished | finished := state.surveillance.Finished | ||||
| thresholds := state.surveillance.Thresholds | thresholds := state.surveillance.Thresholds | ||||
| noiseFloor := state.surveillance.NoiseFloor | noiseFloor := state.surveillance.NoiseFloor | ||||
| @@ -75,11 +108,44 @@ func runDSP(ctx context.Context, srcMgr *sourceManager, cfg config.Config, det * | |||||
| streamSignals = stableSignals | streamSignals = stableSignals | ||||
| } | } | ||||
| if rec != nil && len(art.allIQ) > 0 { | if rec != nil && len(art.allIQ) > 0 { | ||||
| if art.streamDropped { | |||||
| rt.streamOverlap = &streamIQOverlap{} | |||||
| for k := range rt.streamPhaseState { | |||||
| rt.streamPhaseState[k].phase = 0 | |||||
| } | |||||
| resetStreamingOracleRunner() | |||||
| rec.ResetStreams() | |||||
| logging.Warn("gap", "iq_dropped", "msg", "buffer bloat caused extraction drop; overlap reset") | |||||
| if coll != nil { | |||||
| coll.IncCounter("capture.stream_reset", 1, nil) | |||||
| coll.Event("iq_dropped", "warn", "stream overlap reset after dropped IQ", nil, map[string]any{"frame_id": frameID}) | |||||
| } | |||||
| } | |||||
| if rt.cfg.Recorder.DebugLiveAudio { | if rt.cfg.Recorder.DebugLiveAudio { | ||||
| log.Printf("LIVEAUDIO DSP: detailIQ=%d displaySignals=%d streamSignals=%d stableSignals=%d allIQ=%d", len(art.detailIQ), len(displaySignals), len(streamSignals), len(stableSignals), len(art.allIQ)) | log.Printf("LIVEAUDIO DSP: detailIQ=%d displaySignals=%d streamSignals=%d stableSignals=%d allIQ=%d", len(art.detailIQ), len(displaySignals), len(streamSignals), len(stableSignals), len(art.allIQ)) | ||||
| } | } | ||||
| aqCfg := extractionConfig{firTaps: rt.cfg.Recorder.ExtractionTaps, bwMult: rt.cfg.Recorder.ExtractionBwMult} | aqCfg := extractionConfig{firTaps: rt.cfg.Recorder.ExtractionTaps, bwMult: rt.cfg.Recorder.ExtractionBwMult} | ||||
| streamSnips, streamRates := extractForStreaming(extractMgr, art.allIQ, rt.cfg.SampleRate, rt.cfg.CenterHz, streamSignals, rt.streamPhaseState, rt.streamOverlap, aqCfg) | |||||
| extractStart := time.Now() | |||||
| streamSnips, streamRates := extractForStreaming(extractMgr, art.allIQ, rt.cfg.SampleRate, rt.cfg.CenterHz, streamSignals, rt.streamPhaseState, rt.streamOverlap, aqCfg, rt.telemetry) | |||||
| if coll != nil { | |||||
| coll.Observe("stage.extract_stream.duration_ms", float64(time.Since(extractStart).Microseconds())/1000.0, telemetry.TagsFromPairs("frame_id", fmt.Sprintf("%d", frameID))) | |||||
| coll.SetGauge("stage.extract_stream.signals", float64(len(streamSignals)), nil) | |||||
| if coll.ShouldSampleHeavy() { | |||||
| for i := range streamSnips { | |||||
| if i >= len(streamSignals) { | |||||
| break | |||||
| } | |||||
| tags := telemetry.TagsFromPairs( | |||||
| "signal_id", fmt.Sprintf("%d", streamSignals[i].ID), | |||||
| "stage", "extract_stream", | |||||
| ) | |||||
| coll.SetGauge("iq.stage.extract.length", float64(len(streamSnips[i])), tags) | |||||
| if len(streamSnips[i]) > 0 { | |||||
| observeIQStats(coll, "extract_stream", streamSnips[i], tags) | |||||
| } | |||||
| } | |||||
| } | |||||
| } | |||||
| nonEmpty := 0 | nonEmpty := 0 | ||||
| minLen := 0 | minLen := 0 | ||||
| maxLen := 0 | maxLen := 0 | ||||
| @@ -127,10 +193,18 @@ func runDSP(ctx context.Context, srcMgr *sourceManager, cfg config.Config, det * | |||||
| log.Printf("LIVEAUDIO DSP: feedItems=%d", len(items)) | log.Printf("LIVEAUDIO DSP: feedItems=%d", len(items)) | ||||
| } | } | ||||
| if len(items) > 0 { | if len(items) > 0 { | ||||
| feedStart := time.Now() | |||||
| rec.FeedSnippets(items, frameID) | rec.FeedSnippets(items, frameID) | ||||
| if coll != nil { | |||||
| coll.Observe("stage.feed_enqueue.duration_ms", float64(time.Since(feedStart).Microseconds())/1000.0, telemetry.TagsFromPairs("frame_id", fmt.Sprintf("%d", frameID))) | |||||
| coll.SetGauge("stage.feed.items", float64(len(items)), nil) | |||||
| } | |||||
| logging.Debug("trace", "feed", "trace", frameID, "items", len(items), "signals", len(streamSignals), "allIQ", len(art.allIQ)) | logging.Debug("trace", "feed", "trace", frameID, "items", len(items), "signals", len(streamSignals), "allIQ", len(art.allIQ)) | ||||
| } else { | } else { | ||||
| logging.Warn("gap", "feed_empty", "signals", len(streamSignals), "trace", frameID) | logging.Warn("gap", "feed_empty", "signals", len(streamSignals), "trace", frameID) | ||||
| if coll != nil { | |||||
| coll.IncCounter("stage.feed.empty", 1, nil) | |||||
| } | |||||
| } | } | ||||
| } | } | ||||
| rt.maintenance(displaySignals, rec) | rt.maintenance(displaySignals, rec) | ||||
| @@ -156,6 +230,27 @@ func runDSP(ctx context.Context, srcMgr *sourceManager, cfg config.Config, det * | |||||
| if sigSnap != nil { | if sigSnap != nil { | ||||
| sigSnap.set(displaySignals) | sigSnap.set(displaySignals) | ||||
| } | } | ||||
| if coll != nil { | |||||
| coll.SetGauge("signals.display.count", float64(len(displaySignals)), nil) | |||||
| current := make(map[int64]detector.Signal, len(displaySignals)) | |||||
| for _, s := range displaySignals { | |||||
| current[s.ID] = s | |||||
| if _, ok := prevDisplayed[s.ID]; !ok { | |||||
| coll.Event("signal_create", "info", "signal entered display set", telemetry.TagsFromPairs("signal_id", fmt.Sprintf("%d", s.ID)), map[string]any{ | |||||
| "center_hz": s.CenterHz, | |||||
| "bw_hz": s.BWHz, | |||||
| }) | |||||
| } | |||||
| } | |||||
| for id, prev := range prevDisplayed { | |||||
| if _, ok := current[id]; !ok { | |||||
| coll.Event("signal_remove", "info", "signal left display set", telemetry.TagsFromPairs("signal_id", fmt.Sprintf("%d", id)), map[string]any{ | |||||
| "center_hz": prev.CenterHz, | |||||
| }) | |||||
| } | |||||
| } | |||||
| prevDisplayed = current | |||||
| } | |||||
| eventMu.Lock() | eventMu.Lock() | ||||
| for _, ev := range finished { | for _, ev := range finished { | ||||
| _ = enc.Encode(ev) | _ = enc.Encode(ev) | ||||
| @@ -244,6 +339,9 @@ func runDSP(ctx context.Context, srcMgr *sourceManager, cfg config.Config, det * | |||||
| debugInfo.Refinement = refinementDebug | debugInfo.Refinement = refinementDebug | ||||
| } | } | ||||
| h.broadcast(SpectrumFrame{Timestamp: art.now.UnixMilli(), CenterHz: rt.cfg.CenterHz, SampleHz: rt.cfg.SampleRate, FFTSize: rt.cfg.FFTSize, Spectrum: art.surveillanceSpectrum, Signals: displaySignals, Debug: debugInfo}) | h.broadcast(SpectrumFrame{Timestamp: art.now.UnixMilli(), CenterHz: rt.cfg.CenterHz, SampleHz: rt.cfg.SampleRate, FFTSize: rt.cfg.FFTSize, Spectrum: art.surveillanceSpectrum, Signals: displaySignals, Debug: debugInfo}) | ||||
| if coll != nil { | |||||
| coll.Observe("dsp.frame.duration_ms", float64(time.Since(frameStart).Microseconds())/1000.0, nil) | |||||
| } | |||||
| } | } | ||||
| } | } | ||||
| } | } | ||||
| @@ -1,10 +1,13 @@ | |||||
| package main | package main | ||||
| import ( | import ( | ||||
| "fmt" | |||||
| "log" | "log" | ||||
| "math" | "math" | ||||
| "os" | |||||
| "sort" | "sort" | ||||
| "strconv" | "strconv" | ||||
| "strings" | |||||
| "time" | "time" | ||||
| "sdr-wideband-suite/internal/config" | "sdr-wideband-suite/internal/config" | ||||
| @@ -12,6 +15,7 @@ import ( | |||||
| "sdr-wideband-suite/internal/detector" | "sdr-wideband-suite/internal/detector" | ||||
| "sdr-wideband-suite/internal/dsp" | "sdr-wideband-suite/internal/dsp" | ||||
| "sdr-wideband-suite/internal/logging" | "sdr-wideband-suite/internal/logging" | ||||
| "sdr-wideband-suite/internal/telemetry" | |||||
| ) | ) | ||||
| func mustParseDuration(raw string, fallback time.Duration) time.Duration { | func mustParseDuration(raw string, fallback time.Duration) time.Duration { | ||||
| @@ -227,15 +231,30 @@ type extractionConfig struct { | |||||
| const streamOverlapLen = 512 // must be >= FIR tap count with margin | const streamOverlapLen = 512 // must be >= FIR tap count with margin | ||||
| const ( | const ( | ||||
| wfmStreamOutRate = 500000 | |||||
| wfmStreamOutRate = 512000 | |||||
| wfmStreamMinBW = 250000 | wfmStreamMinBW = 250000 | ||||
| ) | ) | ||||
| var forceCPUStreamExtract = func() bool { | |||||
| raw := strings.TrimSpace(os.Getenv("SDR_FORCE_CPU_STREAM_EXTRACT")) | |||||
| if raw == "" { | |||||
| return false | |||||
| } | |||||
| v, err := strconv.ParseBool(raw) | |||||
| if err != nil { | |||||
| return false | |||||
| } | |||||
| return v | |||||
| }() | |||||
| // extractForStreaming performs GPU-accelerated extraction with: | // extractForStreaming performs GPU-accelerated extraction with: | ||||
| // - Per-signal phase-continuous FreqShift (via PhaseStart in ExtractJob) | // - Per-signal phase-continuous FreqShift (via PhaseStart in ExtractJob) | ||||
| // - IQ overlap prepended to allIQ so FIR kernel has real data in halo | // - IQ overlap prepended to allIQ so FIR kernel has real data in halo | ||||
| // | // | ||||
| // Returns extracted snippets with overlap trimmed, and updates phase state. | // Returns extracted snippets with overlap trimmed, and updates phase state. | ||||
| // extractForStreaming is the current legacy production path. | |||||
| // It still relies on overlap-prepend + trim semantics and is intentionally | |||||
| // kept separate from the new streaming refactor/oracle path under development. | |||||
| func extractForStreaming( | func extractForStreaming( | ||||
| extractMgr *extractionManager, | extractMgr *extractionManager, | ||||
| allIQ []complex64, | allIQ []complex64, | ||||
| @@ -245,7 +264,57 @@ func extractForStreaming( | |||||
| phaseState map[int64]*streamExtractState, | phaseState map[int64]*streamExtractState, | ||||
| overlap *streamIQOverlap, | overlap *streamIQOverlap, | ||||
| aqCfg extractionConfig, | aqCfg extractionConfig, | ||||
| coll *telemetry.Collector, | |||||
| ) ([][]complex64, []int) { | ) ([][]complex64, []int) { | ||||
| if useStreamingProductionPath { | |||||
| out, rates, err := extractForStreamingProduction(extractMgr, allIQ, sampleRate, centerHz, signals, aqCfg, coll) | |||||
| if err == nil { | |||||
| logging.Debug("extract", "path_active", "path", "streaming_production", "signals", len(signals), "allIQ", len(allIQ)) | |||||
| if coll != nil { | |||||
| coll.IncCounter("extract.path.streaming_production", 1, nil) | |||||
| } | |||||
| return out, rates | |||||
| } | |||||
| // CRITICAL: the streaming production path failed — log WHY before falling through | |||||
| log.Printf("EXTRACT PATH FALLTHROUGH: streaming production failed: %v — using legacy overlap+trim", err) | |||||
| logging.Warn("extract", "streaming_production_fallthrough", | |||||
| "err", err.Error(), | |||||
| "signals", len(signals), | |||||
| "allIQ", len(allIQ), | |||||
| "sampleRate", sampleRate, | |||||
| ) | |||||
| if coll != nil { | |||||
| coll.IncCounter("extract.path.streaming_production_failed", 1, nil) | |||||
| coll.Event("extraction_path_fallthrough", "warn", | |||||
| "streaming production path failed, using legacy overlap+trim", nil, | |||||
| map[string]any{ | |||||
| "error": err.Error(), | |||||
| "signals": len(signals), | |||||
| "allIQ_len": len(allIQ), | |||||
| "sampleRate": sampleRate, | |||||
| }) | |||||
| } | |||||
| } | |||||
| if useStreamingOraclePath { | |||||
| out, rates, err := extractForStreamingOracle(allIQ, sampleRate, centerHz, signals, aqCfg, coll) | |||||
| if err == nil { | |||||
| logging.Debug("extract", "path_active", "path", "streaming_oracle", "signals", len(signals)) | |||||
| if coll != nil { | |||||
| coll.IncCounter("extract.path.streaming_oracle", 1, nil) | |||||
| } | |||||
| return out, rates | |||||
| } | |||||
| log.Printf("EXTRACT PATH FALLTHROUGH: streaming oracle failed: %v", err) | |||||
| logging.Warn("extract", "streaming_oracle_fallthrough", "err", err.Error()) | |||||
| if coll != nil { | |||||
| coll.IncCounter("extract.path.streaming_oracle_failed", 1, nil) | |||||
| } | |||||
| } | |||||
| // If we reach here, the legacy overlap+trim path is running | |||||
| logging.Warn("extract", "path_active", "path", "legacy_overlap_trim", "signals", len(signals), "allIQ", len(allIQ)) | |||||
| if coll != nil { | |||||
| coll.IncCounter("extract.path.legacy_overlap_trim", 1, nil) | |||||
| } | |||||
| out := make([][]complex64, len(signals)) | out := make([][]complex64, len(signals)) | ||||
| rates := make([]int, len(signals)) | rates := make([]int, len(signals)) | ||||
| if len(allIQ) == 0 || sampleRate <= 0 || len(signals) == 0 { | if len(allIQ) == 0 || sampleRate <= 0 || len(signals) == 0 { | ||||
| @@ -286,6 +355,18 @@ func extractForStreaming( | |||||
| bwMult = 1.0 | bwMult = 1.0 | ||||
| } | } | ||||
| if coll != nil { | |||||
| coll.SetGauge("iq.extract.input.length", float64(len(allIQ)), nil) | |||||
| coll.SetGauge("iq.extract.input.overlap_length", float64(overlapLen), nil) | |||||
| headMean, tailMean, boundaryScore, _ := boundaryMetrics(overlap.tail, allIQ, 32) | |||||
| coll.SetGauge("iq.extract.input.head_mean_mag", headMean, nil) | |||||
| coll.SetGauge("iq.extract.input.prev_tail_mean_mag", tailMean, nil) | |||||
| coll.Observe("iq.extract.input.discontinuity_score", boundaryScore, nil) | |||||
| } | |||||
| rawBoundary := make(map[int64]boundaryProbeState, len(signals)) | |||||
| trimmedBoundary := make(map[int64]boundaryProbeState, len(signals)) | |||||
| // Build jobs with per-signal phase | // Build jobs with per-signal phase | ||||
| jobs := make([]gpudemod.ExtractJob, len(signals)) | jobs := make([]gpudemod.ExtractJob, len(signals)) | ||||
| for i, sig := range signals { | for i, sig := range signals { | ||||
| @@ -323,11 +404,45 @@ func extractForStreaming( | |||||
| OutRate: jobOutRate, | OutRate: jobOutRate, | ||||
| PhaseStart: gpuPhaseStart, | PhaseStart: gpuPhaseStart, | ||||
| } | } | ||||
| if coll != nil { | |||||
| tags := telemetry.TagsFromPairs("signal_id", fmt.Sprintf("%d", sig.ID), "path", "gpu") | |||||
| inputHead := probeHead(gpuIQ, 16, 1e-6) | |||||
| coll.SetGauge("iq.extract.input_head.zero_count", float64(inputHead.zeroCount), tags) | |||||
| coll.SetGauge("iq.extract.input_head.first_nonzero_index", float64(inputHead.firstNonZeroIndex), tags) | |||||
| coll.SetGauge("iq.extract.input_head.max_step", inputHead.maxStep, tags) | |||||
| coll.Event("extract_input_head_probe", "info", "extractor input head probe", tags, map[string]any{ | |||||
| "mags": inputHead.mags, | |||||
| "zero_count": inputHead.zeroCount, | |||||
| "first_nonzero_index": inputHead.firstNonZeroIndex, | |||||
| "head_max_step": inputHead.maxStep, | |||||
| "center_offset_hz": jobs[i].OffsetHz, | |||||
| "bandwidth_hz": bw, | |||||
| "out_rate": jobOutRate, | |||||
| "trim_samples": (overlapLen + int(math.Max(1, math.Round(float64(sampleRate)/float64(jobOutRate)))) - 1) / int(math.Max(1, math.Round(float64(sampleRate)/float64(jobOutRate)))), | |||||
| }) | |||||
| } | |||||
| } | } | ||||
| // Try GPU BatchRunner with phase | |||||
| runner := extractMgr.get(len(gpuIQ), sampleRate) | |||||
| // Try GPU BatchRunner with phase unless CPU-only debug is forced. | |||||
| var runner *gpudemod.BatchRunner | |||||
| if forceCPUStreamExtract { | |||||
| logging.Warn("boundary", "force_cpu_stream_extract", "allIQ_len", len(allIQ), "gpuIQ_len", len(gpuIQ), "signals", len(signals)) | |||||
| } else { | |||||
| runner = extractMgr.get(len(gpuIQ), sampleRate) | |||||
| } | |||||
| if runner != nil { | if runner != nil { | ||||
| if coll != nil && len(gpuIQ) > 0 { | |||||
| inputProbe := probeHead(gpuIQ, 16, 1e-6) | |||||
| coll.Event("gpu_kernel_input_head_probe", "info", "gpu kernel input head probe", nil, map[string]any{ | |||||
| "mags": inputProbe.mags, | |||||
| "zero_count": inputProbe.zeroCount, | |||||
| "first_nonzero_index": inputProbe.firstNonZeroIndex, | |||||
| "head_max_step": inputProbe.maxStep, | |||||
| "gpuIQ_len": len(gpuIQ), | |||||
| "sample_rate": sampleRate, | |||||
| "signals": len(signals), | |||||
| }) | |||||
| } | |||||
| results, err := runner.ShiftFilterDecimateBatchWithPhase(gpuIQ, jobs) | results, err := runner.ShiftFilterDecimateBatchWithPhase(gpuIQ, jobs) | ||||
| if err == nil && len(results) == len(signals) { | if err == nil && len(results) == len(signals) { | ||||
| for i, res := range results { | for i, res := range results { | ||||
| @@ -356,9 +471,95 @@ func extractForStreaming( | |||||
| // Trim overlap from output | // Trim overlap from output | ||||
| iq := res.IQ | iq := res.IQ | ||||
| rawLen := len(iq) | |||||
| if trimSamples > 0 && trimSamples < len(iq) { | if trimSamples > 0 && trimSamples < len(iq) { | ||||
| iq = iq[trimSamples:] | iq = iq[trimSamples:] | ||||
| } | } | ||||
| if i == 0 { | |||||
| logging.Debug("boundary", "extract_trim", "path", "gpu", "raw_len", rawLen, "trim", trimSamples, "out_len", len(iq), "overlap_len", overlapLen, "allIQ_len", len(allIQ), "gpuIQ_len", len(gpuIQ), "outRate", outRate, "signal", signals[i].ID) | |||||
| logExtractorHeadComparison(signals[i].ID, "gpu", overlapLen, res.IQ, trimSamples, iq) | |||||
| } | |||||
| if coll != nil { | |||||
| tags := telemetry.TagsFromPairs("signal_id", fmt.Sprintf("%d", signals[i].ID), "path", "gpu") | |||||
| kernelProbe := probeHead(res.IQ, 16, 1e-6) | |||||
| coll.Event("gpu_kernel_output_head_probe", "info", "gpu kernel output head probe", tags, map[string]any{ | |||||
| "mags": kernelProbe.mags, | |||||
| "zero_count": kernelProbe.zeroCount, | |||||
| "first_nonzero_index": kernelProbe.firstNonZeroIndex, | |||||
| "head_max_step": kernelProbe.maxStep, | |||||
| "raw_len": rawLen, | |||||
| "out_rate": outRate, | |||||
| "trim_samples": trimSamples, | |||||
| }) | |||||
| stats := computeIQHeadStats(iq, 64) | |||||
| coll.SetGauge("iq.extract.output.length", float64(len(iq)), tags) | |||||
| coll.Observe("iq.extract.output.head_mean_mag", stats.meanMag, tags) | |||||
| coll.Observe("iq.extract.output.head_min_mag", stats.minMag, tags) | |||||
| coll.Observe("iq.extract.output.head_max_step", stats.maxStep, tags) | |||||
| coll.Observe("iq.extract.output.head_p95_step", stats.p95Step, tags) | |||||
| coll.Observe("iq.extract.output.head_tail_ratio", stats.headTail, tags) | |||||
| coll.SetGauge("iq.extract.output.head_low_magnitude_count", float64(stats.lowMag), tags) | |||||
| coll.SetGauge("iq.extract.raw.length", float64(rawLen), tags) | |||||
| coll.SetGauge("iq.extract.trim.trim_samples", float64(trimSamples), tags) | |||||
| if rawLen > 0 { | |||||
| coll.SetGauge("iq.extract.raw.head_mag", math.Hypot(float64(real(res.IQ[0])), float64(imag(res.IQ[0]))), tags) | |||||
| coll.SetGauge("iq.extract.raw.tail_mag", math.Hypot(float64(real(res.IQ[rawLen-1])), float64(imag(res.IQ[rawLen-1]))), tags) | |||||
| rawHead := probeHead(res.IQ, 16, 1e-6) | |||||
| coll.SetGauge("iq.extract.raw.head_zero_count", float64(rawHead.zeroCount), tags) | |||||
| coll.SetGauge("iq.extract.raw.first_nonzero_index", float64(rawHead.firstNonZeroIndex), tags) | |||||
| coll.SetGauge("iq.extract.raw.head_max_step", rawHead.maxStep, tags) | |||||
| coll.Event("extract_raw_head_probe", "info", "raw extractor head probe", tags, map[string]any{ | |||||
| "mags": rawHead.mags, | |||||
| "zero_count": rawHead.zeroCount, | |||||
| "first_nonzero_index": rawHead.firstNonZeroIndex, | |||||
| "head_max_step": rawHead.maxStep, | |||||
| "trim_samples": trimSamples, | |||||
| }) | |||||
| } | |||||
| if len(iq) > 0 { | |||||
| coll.SetGauge("iq.extract.trimmed.head_mag", math.Hypot(float64(real(iq[0])), float64(imag(iq[0]))), tags) | |||||
| coll.SetGauge("iq.extract.trimmed.tail_mag", math.Hypot(float64(real(iq[len(iq)-1])), float64(imag(iq[len(iq)-1]))), tags) | |||||
| trimmedHead := probeHead(iq, 16, 1e-6) | |||||
| coll.SetGauge("iq.extract.trimmed.head_zero_count", float64(trimmedHead.zeroCount), tags) | |||||
| coll.SetGauge("iq.extract.trimmed.first_nonzero_index", float64(trimmedHead.firstNonZeroIndex), tags) | |||||
| coll.SetGauge("iq.extract.trimmed.head_max_step", trimmedHead.maxStep, tags) | |||||
| coll.Event("extract_trimmed_head_probe", "info", "trimmed extractor head probe", tags, map[string]any{ | |||||
| "mags": trimmedHead.mags, | |||||
| "zero_count": trimmedHead.zeroCount, | |||||
| "first_nonzero_index": trimmedHead.firstNonZeroIndex, | |||||
| "head_max_step": trimmedHead.maxStep, | |||||
| "trim_samples": trimSamples, | |||||
| }) | |||||
| } | |||||
| if rb := rawBoundary[signals[i].ID]; rb.set && rawLen > 0 { | |||||
| prevMag := math.Hypot(float64(real(rb.last)), float64(imag(rb.last))) | |||||
| currMag := math.Hypot(float64(real(res.IQ[0])), float64(imag(res.IQ[0]))) | |||||
| coll.SetGauge("iq.extract.raw.boundary.prev_tail_mag", prevMag, tags) | |||||
| coll.SetGauge("iq.extract.raw.boundary.curr_head_mag", currMag, tags) | |||||
| coll.Event("extract_raw_boundary", "info", "raw extractor boundary", tags, map[string]any{ | |||||
| "delta_mag": math.Abs(currMag - prevMag), | |||||
| "trim_samples": trimSamples, | |||||
| "raw_len": rawLen, | |||||
| }) | |||||
| } | |||||
| if tb := trimmedBoundary[signals[i].ID]; tb.set && len(iq) > 0 { | |||||
| prevMag := math.Hypot(float64(real(tb.last)), float64(imag(tb.last))) | |||||
| currMag := math.Hypot(float64(real(iq[0])), float64(imag(iq[0]))) | |||||
| coll.SetGauge("iq.extract.trimmed.boundary.prev_tail_mag", prevMag, tags) | |||||
| coll.SetGauge("iq.extract.trimmed.boundary.curr_head_mag", currMag, tags) | |||||
| coll.Event("extract_trimmed_boundary", "info", "trimmed extractor boundary", tags, map[string]any{ | |||||
| "delta_mag": math.Abs(currMag - prevMag), | |||||
| "trim_samples": trimSamples, | |||||
| "out_len": len(iq), | |||||
| }) | |||||
| } | |||||
| } | |||||
| if rawLen > 0 { | |||||
| rawBoundary[signals[i].ID] = boundaryProbeState{last: res.IQ[rawLen-1], set: true} | |||||
| } | |||||
| if len(iq) > 0 { | |||||
| trimmedBoundary[signals[i].ID] = boundaryProbeState{last: iq[len(iq)-1], set: true} | |||||
| } | |||||
| out[i] = iq | out[i] = iq | ||||
| rates[i] = res.Rate | rates[i] = res.Rate | ||||
| } | } | ||||
| @@ -424,10 +625,240 @@ func extractForStreaming( | |||||
| if i == 0 { | if i == 0 { | ||||
| logging.Debug("extract", "cpu_result", "outRate", outRate, "decim", decim, "trim", trimSamples) | logging.Debug("extract", "cpu_result", "outRate", outRate, "decim", decim, "trim", trimSamples) | ||||
| } | } | ||||
| rawIQ := decimated | |||||
| rawLen := len(rawIQ) | |||||
| if trimSamples > 0 && trimSamples < len(decimated) { | if trimSamples > 0 && trimSamples < len(decimated) { | ||||
| decimated = decimated[trimSamples:] | decimated = decimated[trimSamples:] | ||||
| } | } | ||||
| if i == 0 { | |||||
| logging.Debug("boundary", "extract_trim", "path", "cpu", "raw_len", rawLen, "trim", trimSamples, "out_len", len(decimated), "overlap_len", overlapLen, "allIQ_len", len(allIQ), "gpuIQ_len", len(gpuIQ), "outRate", outRate, "signal", signals[i].ID) | |||||
| logExtractorHeadComparison(signals[i].ID, "cpu", overlapLen, decimated, trimSamples, decimated) | |||||
| } | |||||
| if coll != nil { | |||||
| tags := telemetry.TagsFromPairs("signal_id", fmt.Sprintf("%d", signals[i].ID), "path", "cpu") | |||||
| stats := computeIQHeadStats(decimated, 64) | |||||
| coll.SetGauge("iq.extract.output.length", float64(len(decimated)), tags) | |||||
| coll.Observe("iq.extract.output.head_mean_mag", stats.meanMag, tags) | |||||
| coll.Observe("iq.extract.output.head_min_mag", stats.minMag, tags) | |||||
| coll.Observe("iq.extract.output.head_max_step", stats.maxStep, tags) | |||||
| coll.Observe("iq.extract.output.head_p95_step", stats.p95Step, tags) | |||||
| coll.Observe("iq.extract.output.head_tail_ratio", stats.headTail, tags) | |||||
| coll.SetGauge("iq.extract.output.head_low_magnitude_count", float64(stats.lowMag), tags) | |||||
| coll.SetGauge("iq.extract.raw.length", float64(rawLen), tags) | |||||
| coll.SetGauge("iq.extract.trim.trim_samples", float64(trimSamples), tags) | |||||
| if rb := rawBoundary[signals[i].ID]; rb.set && rawLen > 0 { | |||||
| observeBoundarySample(coll, "iq.extract.raw.boundary", tags, rb.last, rawIQ[0]) | |||||
| } | |||||
| if tb := trimmedBoundary[signals[i].ID]; tb.set && len(decimated) > 0 { | |||||
| observeBoundarySample(coll, "iq.extract.trimmed.boundary", tags, tb.last, decimated[0]) | |||||
| } | |||||
| } | |||||
| if rawLen > 0 { | |||||
| rawBoundary[signals[i].ID] = boundaryProbeState{last: rawIQ[rawLen-1], set: true} | |||||
| } | |||||
| if len(decimated) > 0 { | |||||
| trimmedBoundary[signals[i].ID] = boundaryProbeState{last: decimated[len(decimated)-1], set: true} | |||||
| } | |||||
| out[i] = decimated | out[i] = decimated | ||||
| } | } | ||||
| return out, rates | return out, rates | ||||
| } | } | ||||
| type iqHeadStats struct { | |||||
| length int | |||||
| minMag float64 | |||||
| maxMag float64 | |||||
| meanMag float64 | |||||
| lowMag int | |||||
| maxStep float64 | |||||
| maxStepIdx int | |||||
| p95Step float64 | |||||
| headTail float64 | |||||
| headMinIdx int | |||||
| stepSamples []float64 | |||||
| } | |||||
| type boundaryProbeState struct { | |||||
| last complex64 | |||||
| set bool | |||||
| } | |||||
| type headProbe struct { | |||||
| zeroCount int | |||||
| firstNonZeroIndex int | |||||
| maxStep float64 | |||||
| mags []float64 | |||||
| } | |||||
| func probeHead(samples []complex64, n int, zeroThreshold float64) headProbe { | |||||
| if n <= 0 || len(samples) == 0 { | |||||
| return headProbe{firstNonZeroIndex: -1} | |||||
| } | |||||
| if len(samples) < n { | |||||
| n = len(samples) | |||||
| } | |||||
| if zeroThreshold <= 0 { | |||||
| zeroThreshold = 1e-6 | |||||
| } | |||||
| out := headProbe{firstNonZeroIndex: -1, mags: make([]float64, 0, n)} | |||||
| for i := 0; i < n; i++ { | |||||
| v := samples[i] | |||||
| mag := math.Hypot(float64(real(v)), float64(imag(v))) | |||||
| out.mags = append(out.mags, mag) | |||||
| if mag <= zeroThreshold { | |||||
| out.zeroCount++ | |||||
| } else if out.firstNonZeroIndex < 0 { | |||||
| out.firstNonZeroIndex = i | |||||
| } | |||||
| if i > 0 { | |||||
| p := samples[i-1] | |||||
| num := float64(real(p))*float64(imag(v)) - float64(imag(p))*float64(real(v)) | |||||
| den := float64(real(p))*float64(real(v)) + float64(imag(p))*float64(imag(v)) | |||||
| step := math.Abs(math.Atan2(num, den)) | |||||
| if step > out.maxStep { | |||||
| out.maxStep = step | |||||
| } | |||||
| } | |||||
| } | |||||
| return out | |||||
| } | |||||
| func observeBoundarySample(coll *telemetry.Collector, metricPrefix string, tags map[string]string, prev complex64, curr complex64) { | |||||
| prevMag := math.Hypot(float64(real(prev)), float64(imag(prev))) | |||||
| currMag := math.Hypot(float64(real(curr)), float64(imag(curr))) | |||||
| deltaMag := math.Abs(currMag - prevMag) | |||||
| num := float64(real(prev))*float64(imag(curr)) - float64(imag(prev))*float64(real(curr)) | |||||
| den := float64(real(prev))*float64(real(curr)) + float64(imag(prev))*float64(imag(curr)) | |||||
| deltaPhase := math.Abs(math.Atan2(num, den)) | |||||
| d2 := float64(real(curr-prev))*float64(real(curr-prev)) + float64(imag(curr-prev))*float64(imag(curr-prev)) | |||||
| coll.Observe(metricPrefix+".delta_mag", deltaMag, tags) | |||||
| coll.Observe(metricPrefix+".delta_phase", deltaPhase, tags) | |||||
| coll.Observe(metricPrefix+".d2", d2, tags) | |||||
| coll.Observe(metricPrefix+".discontinuity_score", deltaMag+deltaPhase, tags) | |||||
| } | |||||
| func computeIQHeadStats(iq []complex64, headLen int) iqHeadStats { | |||||
| stats := iqHeadStats{minMag: math.MaxFloat64, headMinIdx: -1, maxStepIdx: -1} | |||||
| if len(iq) == 0 { | |||||
| stats.minMag = 0 | |||||
| return stats | |||||
| } | |||||
| n := len(iq) | |||||
| if headLen > 0 && headLen < n { | |||||
| n = headLen | |||||
| } | |||||
| stats.length = n | |||||
| stats.stepSamples = make([]float64, 0, max(0, n-1)) | |||||
| sumMag := 0.0 | |||||
| headSum := 0.0 | |||||
| tailSum := 0.0 | |||||
| tailCount := 0 | |||||
| for i := 0; i < n; i++ { | |||||
| v := iq[i] | |||||
| mag := math.Hypot(float64(real(v)), float64(imag(v))) | |||||
| if mag < stats.minMag { | |||||
| stats.minMag = mag | |||||
| stats.headMinIdx = i | |||||
| } | |||||
| if mag > stats.maxMag { | |||||
| stats.maxMag = mag | |||||
| } | |||||
| sumMag += mag | |||||
| if mag < 0.05 { | |||||
| stats.lowMag++ | |||||
| } | |||||
| if i < min(16, n) { | |||||
| headSum += mag | |||||
| } | |||||
| if i >= max(0, n-16) { | |||||
| tailSum += mag | |||||
| tailCount++ | |||||
| } | |||||
| if i > 0 { | |||||
| p := iq[i-1] | |||||
| num := float64(real(p))*float64(imag(v)) - float64(imag(p))*float64(real(v)) | |||||
| den := float64(real(p))*float64(real(v)) + float64(imag(p))*float64(imag(v)) | |||||
| step := math.Abs(math.Atan2(num, den)) | |||||
| if step > stats.maxStep { | |||||
| stats.maxStep = step | |||||
| stats.maxStepIdx = i - 1 | |||||
| } | |||||
| stats.stepSamples = append(stats.stepSamples, step) | |||||
| } | |||||
| } | |||||
| stats.meanMag = sumMag / float64(n) | |||||
| if len(stats.stepSamples) > 0 { | |||||
| sorted := append([]float64(nil), stats.stepSamples...) | |||||
| sort.Float64s(sorted) | |||||
| idx := int(float64(len(sorted)-1) * 0.95) | |||||
| stats.p95Step = sorted[idx] | |||||
| } else { | |||||
| stats.p95Step = stats.maxStep | |||||
| } | |||||
| if headSum > 0 && tailCount > 0 { | |||||
| headMean := headSum / float64(min(16, n)) | |||||
| tailMean := tailSum / float64(tailCount) | |||||
| if tailMean > 0 { | |||||
| stats.headTail = headMean / tailMean | |||||
| } | |||||
| } | |||||
| return stats | |||||
| } | |||||
| func observeIQStats(coll *telemetry.Collector, stage string, iq []complex64, tags telemetry.Tags) { | |||||
| if coll == nil || len(iq) == 0 { | |||||
| return | |||||
| } | |||||
| stats := computeIQHeadStats(iq, len(iq)) | |||||
| stageTags := telemetry.TagsWith(tags, "stage", stage) | |||||
| coll.Observe("iq.magnitude.min", stats.minMag, stageTags) | |||||
| coll.Observe("iq.magnitude.max", stats.maxMag, stageTags) | |||||
| coll.Observe("iq.magnitude.mean", stats.meanMag, stageTags) | |||||
| coll.Observe("iq.phase_step.max", stats.maxStep, stageTags) | |||||
| coll.Observe("iq.phase_step.p95", stats.p95Step, stageTags) | |||||
| coll.Observe("iq.low_magnitude.count", float64(stats.lowMag), stageTags) | |||||
| coll.SetGauge("iq.length", float64(stats.length), stageTags) | |||||
| } | |||||
| func logExtractorHeadComparison(signalID int64, path string, overlapLen int, raw []complex64, trimSamples int, out []complex64) { | |||||
| rawStats := computeIQHeadStats(raw, 96) | |||||
| trimmedStats := computeIQHeadStats(out, 96) | |||||
| logging.Debug("boundary", "extract_head_compare", | |||||
| "signal", signalID, | |||||
| "path", path, | |||||
| "raw_len", len(raw), | |||||
| "trim", trimSamples, | |||||
| "out_len", len(out), | |||||
| "overlap_len", overlapLen, | |||||
| "raw_min_mag", rawStats.minMag, | |||||
| "raw_min_idx", rawStats.headMinIdx, | |||||
| "raw_max_step", rawStats.maxStep, | |||||
| "raw_max_step_idx", rawStats.maxStepIdx, | |||||
| "raw_head_tail", rawStats.headTail, | |||||
| "trimmed_min_mag", trimmedStats.minMag, | |||||
| "trimmed_min_idx", trimmedStats.headMinIdx, | |||||
| "trimmed_max_step", trimmedStats.maxStep, | |||||
| "trimmed_max_step_idx", trimmedStats.maxStepIdx, | |||||
| "trimmed_head_tail", trimmedStats.headTail, | |||||
| ) | |||||
| for _, off := range []int{2, 4, 8, 16} { | |||||
| if len(out) <= off+8 { | |||||
| continue | |||||
| } | |||||
| offStats := computeIQHeadStats(out[off:], 96) | |||||
| logging.Debug("boundary", "extract_head_offset_compare", | |||||
| "signal", signalID, | |||||
| "path", path, | |||||
| "offset", off, | |||||
| "base_min_mag", trimmedStats.minMag, | |||||
| "base_min_idx", trimmedStats.headMinIdx, | |||||
| "base_max_step", trimmedStats.maxStep, | |||||
| "base_max_step_idx", trimmedStats.maxStepIdx, | |||||
| "offset_min_mag", offStats.minMag, | |||||
| "offset_min_idx", offStats.headMinIdx, | |||||
| "offset_max_step", offStats.maxStep, | |||||
| "offset_max_step_idx", offStats.maxStepIdx, | |||||
| "offset_head_tail", offStats.headTail, | |||||
| ) | |||||
| } | |||||
| } | |||||
| @@ -3,6 +3,7 @@ package main | |||||
| import ( | import ( | ||||
| "context" | "context" | ||||
| "encoding/json" | "encoding/json" | ||||
| "errors" | |||||
| "log" | "log" | ||||
| "net/http" | "net/http" | ||||
| "os" | "os" | ||||
| @@ -19,9 +20,10 @@ import ( | |||||
| "sdr-wideband-suite/internal/pipeline" | "sdr-wideband-suite/internal/pipeline" | ||||
| "sdr-wideband-suite/internal/recorder" | "sdr-wideband-suite/internal/recorder" | ||||
| "sdr-wideband-suite/internal/runtime" | "sdr-wideband-suite/internal/runtime" | ||||
| "sdr-wideband-suite/internal/telemetry" | |||||
| ) | ) | ||||
| func registerAPIHandlers(mux *http.ServeMux, cfgPath string, cfgManager *runtime.Manager, srcMgr *sourceManager, dspUpdates chan dspUpdate, gpuState *gpuStatus, recMgr *recorder.Manager, sigSnap *signalSnapshot, eventMu *sync.RWMutex, phaseSnap *phaseSnapshot) { | |||||
| func registerAPIHandlers(mux *http.ServeMux, cfgPath string, cfgManager *runtime.Manager, srcMgr *sourceManager, dspUpdates chan dspUpdate, gpuState *gpuStatus, recMgr *recorder.Manager, sigSnap *signalSnapshot, eventMu *sync.RWMutex, phaseSnap *phaseSnapshot, telem *telemetry.Collector) { | |||||
| mux.HandleFunc("/api/config", func(w http.ResponseWriter, r *http.Request) { | mux.HandleFunc("/api/config", func(w http.ResponseWriter, r *http.Request) { | ||||
| w.Header().Set("Content-Type", "application/json") | w.Header().Set("Content-Type", "application/json") | ||||
| switch r.Method { | switch r.Method { | ||||
| @@ -378,16 +380,196 @@ func registerAPIHandlers(mux *http.ServeMux, cfgPath string, cfgManager *runtime | |||||
| w.Header().Set("Content-Type", "audio/wav") | w.Header().Set("Content-Type", "audio/wav") | ||||
| _, _ = w.Write(data) | _, _ = w.Write(data) | ||||
| }) | }) | ||||
| mux.HandleFunc("/api/debug/telemetry/live", func(w http.ResponseWriter, r *http.Request) { | |||||
| w.Header().Set("Content-Type", "application/json") | |||||
| if telem == nil { | |||||
| _ = json.NewEncoder(w).Encode(map[string]any{"enabled": false, "error": "telemetry unavailable"}) | |||||
| return | |||||
| } | |||||
| _ = json.NewEncoder(w).Encode(telem.LiveSnapshot()) | |||||
| }) | |||||
| mux.HandleFunc("/api/debug/telemetry/history", func(w http.ResponseWriter, r *http.Request) { | |||||
| w.Header().Set("Content-Type", "application/json") | |||||
| if telem == nil { | |||||
| http.Error(w, "telemetry unavailable", http.StatusServiceUnavailable) | |||||
| return | |||||
| } | |||||
| query, err := telemetryQueryFromRequest(r) | |||||
| if err != nil { | |||||
| http.Error(w, err.Error(), http.StatusBadRequest) | |||||
| return | |||||
| } | |||||
| items, err := telem.QueryMetrics(query) | |||||
| if err != nil { | |||||
| http.Error(w, err.Error(), http.StatusInternalServerError) | |||||
| return | |||||
| } | |||||
| _ = json.NewEncoder(w).Encode(map[string]any{"items": items, "count": len(items)}) | |||||
| }) | |||||
| mux.HandleFunc("/api/debug/telemetry/events", func(w http.ResponseWriter, r *http.Request) { | |||||
| w.Header().Set("Content-Type", "application/json") | |||||
| if telem == nil { | |||||
| http.Error(w, "telemetry unavailable", http.StatusServiceUnavailable) | |||||
| return | |||||
| } | |||||
| query, err := telemetryQueryFromRequest(r) | |||||
| if err != nil { | |||||
| http.Error(w, err.Error(), http.StatusBadRequest) | |||||
| return | |||||
| } | |||||
| items, err := telem.QueryEvents(query) | |||||
| if err != nil { | |||||
| http.Error(w, err.Error(), http.StatusInternalServerError) | |||||
| return | |||||
| } | |||||
| _ = json.NewEncoder(w).Encode(map[string]any{"items": items, "count": len(items)}) | |||||
| }) | |||||
| mux.HandleFunc("/api/debug/telemetry/config", func(w http.ResponseWriter, r *http.Request) { | |||||
| w.Header().Set("Content-Type", "application/json") | |||||
| if telem == nil { | |||||
| http.Error(w, "telemetry unavailable", http.StatusServiceUnavailable) | |||||
| return | |||||
| } | |||||
| switch r.Method { | |||||
| case http.MethodGet: | |||||
| _ = json.NewEncoder(w).Encode(map[string]any{ | |||||
| "collector": telem.Config(), | |||||
| "config": cfgManager.Snapshot().Debug.Telemetry, | |||||
| }) | |||||
| case http.MethodPost: | |||||
| var update struct { | |||||
| Enabled *bool `json:"enabled"` | |||||
| HeavyEnabled *bool `json:"heavy_enabled"` | |||||
| HeavySampleEvery *int `json:"heavy_sample_every"` | |||||
| MetricSampleEvery *int `json:"metric_sample_every"` | |||||
| MetricHistoryMax *int `json:"metric_history_max"` | |||||
| EventHistoryMax *int `json:"event_history_max"` | |||||
| RetentionSeconds *int `json:"retention_seconds"` | |||||
| PersistEnabled *bool `json:"persist_enabled"` | |||||
| PersistDir *string `json:"persist_dir"` | |||||
| RotateMB *int `json:"rotate_mb"` | |||||
| KeepFiles *int `json:"keep_files"` | |||||
| } | |||||
| if err := json.NewDecoder(r.Body).Decode(&update); err != nil { | |||||
| http.Error(w, "invalid json", http.StatusBadRequest) | |||||
| return | |||||
| } | |||||
| next := cfgManager.Snapshot() | |||||
| cur := next.Debug.Telemetry | |||||
| if update.Enabled != nil { | |||||
| cur.Enabled = *update.Enabled | |||||
| } | |||||
| if update.HeavyEnabled != nil { | |||||
| cur.HeavyEnabled = *update.HeavyEnabled | |||||
| } | |||||
| if update.HeavySampleEvery != nil { | |||||
| cur.HeavySampleEvery = *update.HeavySampleEvery | |||||
| } | |||||
| if update.MetricSampleEvery != nil { | |||||
| cur.MetricSampleEvery = *update.MetricSampleEvery | |||||
| } | |||||
| if update.MetricHistoryMax != nil { | |||||
| cur.MetricHistoryMax = *update.MetricHistoryMax | |||||
| } | |||||
| if update.EventHistoryMax != nil { | |||||
| cur.EventHistoryMax = *update.EventHistoryMax | |||||
| } | |||||
| if update.RetentionSeconds != nil { | |||||
| cur.RetentionSeconds = *update.RetentionSeconds | |||||
| } | |||||
| if update.PersistEnabled != nil { | |||||
| cur.PersistEnabled = *update.PersistEnabled | |||||
| } | |||||
| if update.PersistDir != nil && *update.PersistDir != "" { | |||||
| cur.PersistDir = *update.PersistDir | |||||
| } | |||||
| if update.RotateMB != nil { | |||||
| cur.RotateMB = *update.RotateMB | |||||
| } | |||||
| if update.KeepFiles != nil { | |||||
| cur.KeepFiles = *update.KeepFiles | |||||
| } | |||||
| next.Debug.Telemetry = cur | |||||
| cfgManager.Replace(next) | |||||
| if err := config.Save(cfgPath, next); err != nil { | |||||
| log.Printf("telemetry config save failed: %v", err) | |||||
| } | |||||
| err := telem.Configure(telemetry.Config{ | |||||
| Enabled: cur.Enabled, | |||||
| HeavyEnabled: cur.HeavyEnabled, | |||||
| HeavySampleEvery: cur.HeavySampleEvery, | |||||
| MetricSampleEvery: cur.MetricSampleEvery, | |||||
| MetricHistoryMax: cur.MetricHistoryMax, | |||||
| EventHistoryMax: cur.EventHistoryMax, | |||||
| Retention: time.Duration(cur.RetentionSeconds) * time.Second, | |||||
| PersistEnabled: cur.PersistEnabled, | |||||
| PersistDir: cur.PersistDir, | |||||
| RotateMB: cur.RotateMB, | |||||
| KeepFiles: cur.KeepFiles, | |||||
| }) | |||||
| if err != nil { | |||||
| http.Error(w, err.Error(), http.StatusBadRequest) | |||||
| return | |||||
| } | |||||
| _ = json.NewEncoder(w).Encode(map[string]any{"ok": true, "collector": telem.Config(), "config": cur}) | |||||
| default: | |||||
| http.Error(w, "method not allowed", http.StatusMethodNotAllowed) | |||||
| } | |||||
| }) | |||||
| } | } | ||||
| func newHTTPServer(addr string, webRoot string, h *hub, cfgPath string, cfgManager *runtime.Manager, srcMgr *sourceManager, dspUpdates chan dspUpdate, gpuState *gpuStatus, recMgr *recorder.Manager, sigSnap *signalSnapshot, eventMu *sync.RWMutex, phaseSnap *phaseSnapshot) *http.Server { | |||||
| func newHTTPServer(addr string, webRoot string, h *hub, cfgPath string, cfgManager *runtime.Manager, srcMgr *sourceManager, dspUpdates chan dspUpdate, gpuState *gpuStatus, recMgr *recorder.Manager, sigSnap *signalSnapshot, eventMu *sync.RWMutex, phaseSnap *phaseSnapshot, telem *telemetry.Collector) *http.Server { | |||||
| mux := http.NewServeMux() | mux := http.NewServeMux() | ||||
| registerWSHandlers(mux, h, recMgr) | registerWSHandlers(mux, h, recMgr) | ||||
| registerAPIHandlers(mux, cfgPath, cfgManager, srcMgr, dspUpdates, gpuState, recMgr, sigSnap, eventMu, phaseSnap) | |||||
| registerAPIHandlers(mux, cfgPath, cfgManager, srcMgr, dspUpdates, gpuState, recMgr, sigSnap, eventMu, phaseSnap, telem) | |||||
| mux.Handle("/", http.FileServer(http.Dir(webRoot))) | mux.Handle("/", http.FileServer(http.Dir(webRoot))) | ||||
| return &http.Server{Addr: addr, Handler: mux} | return &http.Server{Addr: addr, Handler: mux} | ||||
| } | } | ||||
| func telemetryQueryFromRequest(r *http.Request) (telemetry.Query, error) { | |||||
| q := r.URL.Query() | |||||
| var out telemetry.Query | |||||
| var err error | |||||
| if out.From, err = telemetry.ParseTimeQuery(q.Get("since")); err != nil { | |||||
| return out, errors.New("invalid since") | |||||
| } | |||||
| if out.To, err = telemetry.ParseTimeQuery(q.Get("until")); err != nil { | |||||
| return out, errors.New("invalid until") | |||||
| } | |||||
| if v := q.Get("limit"); v != "" { | |||||
| if parsed, parseErr := strconv.Atoi(v); parseErr == nil { | |||||
| out.Limit = parsed | |||||
| } | |||||
| } | |||||
| out.Name = q.Get("name") | |||||
| out.NamePrefix = q.Get("prefix") | |||||
| out.Level = q.Get("level") | |||||
| out.IncludePersisted = true | |||||
| if v := q.Get("include_persisted"); v != "" { | |||||
| if b, parseErr := strconv.ParseBool(v); parseErr == nil { | |||||
| out.IncludePersisted = b | |||||
| } | |||||
| } | |||||
| tags := telemetry.Tags{} | |||||
| for key, vals := range q { | |||||
| if len(vals) == 0 { | |||||
| continue | |||||
| } | |||||
| if strings.HasPrefix(key, "tag_") { | |||||
| tags[strings.TrimPrefix(key, "tag_")] = vals[0] | |||||
| } | |||||
| } | |||||
| for _, key := range []string{"signal_id", "session_id", "stage", "trace_id", "component"} { | |||||
| if v := q.Get(key); v != "" { | |||||
| tags[key] = v | |||||
| } | |||||
| } | |||||
| if len(tags) > 0 { | |||||
| out.Tags = tags | |||||
| } | |||||
| return out, nil | |||||
| } | |||||
| func shutdownServer(server *http.Server) { | func shutdownServer(server *http.Server) { | ||||
| ctxTimeout, cancelTimeout := context.WithTimeout(context.Background(), 5*time.Second) | ctxTimeout, cancelTimeout := context.WithTimeout(context.Background(), 5*time.Second) | ||||
| defer cancelTimeout() | defer cancelTimeout() | ||||
| @@ -0,0 +1,6 @@ | |||||
| package main | |||||
| // NOTE: Legacy extractor logic still lives in helpers.go for now. | |||||
| // This file is intentionally reserved for the later explicit move once the | |||||
| // production-path rewrite is far enough along that the split can be done in one | |||||
| // safe pass instead of a risky mechanical half-step. | |||||
| @@ -23,6 +23,7 @@ import ( | |||||
| "sdr-wideband-suite/internal/runtime" | "sdr-wideband-suite/internal/runtime" | ||||
| "sdr-wideband-suite/internal/sdr" | "sdr-wideband-suite/internal/sdr" | ||||
| "sdr-wideband-suite/internal/sdrplay" | "sdr-wideband-suite/internal/sdrplay" | ||||
| "sdr-wideband-suite/internal/telemetry" | |||||
| ) | ) | ||||
| func main() { | func main() { | ||||
| @@ -51,6 +52,25 @@ func main() { | |||||
| cfgManager := runtime.New(cfg) | cfgManager := runtime.New(cfg) | ||||
| gpuState := &gpuStatus{Available: gpufft.Available()} | gpuState := &gpuStatus{Available: gpufft.Available()} | ||||
| telemetryCfg := telemetry.Config{ | |||||
| Enabled: cfg.Debug.Telemetry.Enabled, | |||||
| HeavyEnabled: cfg.Debug.Telemetry.HeavyEnabled, | |||||
| HeavySampleEvery: cfg.Debug.Telemetry.HeavySampleEvery, | |||||
| MetricSampleEvery: cfg.Debug.Telemetry.MetricSampleEvery, | |||||
| MetricHistoryMax: cfg.Debug.Telemetry.MetricHistoryMax, | |||||
| EventHistoryMax: cfg.Debug.Telemetry.EventHistoryMax, | |||||
| Retention: time.Duration(cfg.Debug.Telemetry.RetentionSeconds) * time.Second, | |||||
| PersistEnabled: cfg.Debug.Telemetry.PersistEnabled, | |||||
| PersistDir: cfg.Debug.Telemetry.PersistDir, | |||||
| RotateMB: cfg.Debug.Telemetry.RotateMB, | |||||
| KeepFiles: cfg.Debug.Telemetry.KeepFiles, | |||||
| } | |||||
| telemetryCollector, err := telemetry.New(telemetryCfg) | |||||
| if err != nil { | |||||
| log.Fatalf("telemetry init failed: %v", err) | |||||
| } | |||||
| defer telemetryCollector.Close() | |||||
| telemetryCollector.SetStatus("build", "sdrd") | |||||
| newSource := func(cfg config.Config) (sdr.Source, error) { | newSource := func(cfg config.Config) (sdr.Source, error) { | ||||
| if mockFlag { | if mockFlag { | ||||
| @@ -74,7 +94,7 @@ func main() { | |||||
| if err != nil { | if err != nil { | ||||
| log.Fatalf("sdrplay init failed: %v (try --mock or build with -tags sdrplay)", err) | log.Fatalf("sdrplay init failed: %v (try --mock or build with -tags sdrplay)", err) | ||||
| } | } | ||||
| srcMgr := newSourceManager(src, newSource) | |||||
| srcMgr := newSourceManagerWithTelemetry(src, newSource, telemetryCollector) | |||||
| if err := srcMgr.Start(); err != nil { | if err := srcMgr.Start(); err != nil { | ||||
| log.Fatalf("source start: %v", err) | log.Fatalf("source start: %v", err) | ||||
| } | } | ||||
| @@ -118,7 +138,7 @@ func main() { | |||||
| DeemphasisUs: cfg.Recorder.DeemphasisUs, | DeemphasisUs: cfg.Recorder.DeemphasisUs, | ||||
| ExtractionTaps: cfg.Recorder.ExtractionTaps, | ExtractionTaps: cfg.Recorder.ExtractionTaps, | ||||
| ExtractionBwMult: cfg.Recorder.ExtractionBwMult, | ExtractionBwMult: cfg.Recorder.ExtractionBwMult, | ||||
| }, cfg.CenterHz, decodeMap) | |||||
| }, cfg.CenterHz, decodeMap, telemetryCollector) | |||||
| defer recMgr.Close() | defer recMgr.Close() | ||||
| sigSnap := &signalSnapshot{} | sigSnap := &signalSnapshot{} | ||||
| @@ -126,9 +146,9 @@ func main() { | |||||
| defer extractMgr.reset() | defer extractMgr.reset() | ||||
| phaseSnap := &phaseSnapshot{} | phaseSnap := &phaseSnapshot{} | ||||
| go runDSP(ctx, srcMgr, cfg, det, window, h, eventFile, eventMu, dspUpdates, gpuState, recMgr, sigSnap, extractMgr, phaseSnap) | |||||
| go runDSP(ctx, srcMgr, cfg, det, window, h, eventFile, eventMu, dspUpdates, gpuState, recMgr, sigSnap, extractMgr, phaseSnap, telemetryCollector) | |||||
| server := newHTTPServer(cfg.WebAddr, cfg.WebRoot, h, cfgPath, cfgManager, srcMgr, dspUpdates, gpuState, recMgr, sigSnap, eventMu, phaseSnap) | |||||
| server := newHTTPServer(cfg.WebAddr, cfg.WebRoot, h, cfgPath, cfgManager, srcMgr, dspUpdates, gpuState, recMgr, sigSnap, eventMu, phaseSnap, telemetryCollector) | |||||
| go func() { | go func() { | ||||
| log.Printf("web listening on %s", cfg.WebAddr) | log.Printf("web listening on %s", cfg.WebAddr) | ||||
| if err := server.ListenAndServe(); err != nil && err != http.ErrServerClosed { | if err := server.ListenAndServe(); err != nil && err != http.ErrServerClosed { | ||||
| @@ -3,6 +3,8 @@ package main | |||||
| import ( | import ( | ||||
| "fmt" | "fmt" | ||||
| "math" | "math" | ||||
| "os" | |||||
| "strconv" | |||||
| "strings" | "strings" | ||||
| "sync" | "sync" | ||||
| "sync/atomic" | "sync/atomic" | ||||
| @@ -19,6 +21,7 @@ import ( | |||||
| "sdr-wideband-suite/internal/pipeline" | "sdr-wideband-suite/internal/pipeline" | ||||
| "sdr-wideband-suite/internal/rds" | "sdr-wideband-suite/internal/rds" | ||||
| "sdr-wideband-suite/internal/recorder" | "sdr-wideband-suite/internal/recorder" | ||||
| "sdr-wideband-suite/internal/telemetry" | |||||
| ) | ) | ||||
| type rdsState struct { | type rdsState struct { | ||||
| @@ -29,6 +32,18 @@ type rdsState struct { | |||||
| mu sync.Mutex | mu sync.Mutex | ||||
| } | } | ||||
| var forceFixedStreamReadSamples = func() int { | |||||
| raw := strings.TrimSpace(os.Getenv("SDR_FORCE_FIXED_STREAM_READ_SAMPLES")) | |||||
| if raw == "" { | |||||
| return 0 | |||||
| } | |||||
| v, err := strconv.Atoi(raw) | |||||
| if err != nil || v <= 0 { | |||||
| return 0 | |||||
| } | |||||
| return v | |||||
| }() | |||||
| type dspRuntime struct { | type dspRuntime struct { | ||||
| cfg config.Config | cfg config.Config | ||||
| det *detector.Detector | det *detector.Detector | ||||
| @@ -52,10 +67,13 @@ type dspRuntime struct { | |||||
| arbiter *pipeline.Arbiter | arbiter *pipeline.Arbiter | ||||
| arbitration pipeline.ArbitrationState | arbitration pipeline.ArbitrationState | ||||
| gotSamples bool | gotSamples bool | ||||
| telemetry *telemetry.Collector | |||||
| lastAllIQTail []complex64 | |||||
| } | } | ||||
| type spectrumArtifacts struct { | type spectrumArtifacts struct { | ||||
| allIQ []complex64 | allIQ []complex64 | ||||
| streamDropped bool | |||||
| surveillanceIQ []complex64 | surveillanceIQ []complex64 | ||||
| detailIQ []complex64 | detailIQ []complex64 | ||||
| surveillanceSpectrum []float64 | surveillanceSpectrum []float64 | ||||
| @@ -94,7 +112,7 @@ type surveillancePlan struct { | |||||
| const derivedIDBlock = int64(1_000_000_000) | const derivedIDBlock = int64(1_000_000_000) | ||||
| func newDSPRuntime(cfg config.Config, det *detector.Detector, window []float64, gpuState *gpuStatus) *dspRuntime { | |||||
| func newDSPRuntime(cfg config.Config, det *detector.Detector, window []float64, gpuState *gpuStatus, coll *telemetry.Collector) *dspRuntime { | |||||
| detailFFT := cfg.Refinement.DetailFFTSize | detailFFT := cfg.Refinement.DetailFFTSize | ||||
| if detailFFT <= 0 { | if detailFFT <= 0 { | ||||
| detailFFT = cfg.FFTSize | detailFFT = cfg.FFTSize | ||||
| @@ -119,6 +137,7 @@ func newDSPRuntime(cfg config.Config, det *detector.Detector, window []float64, | |||||
| streamPhaseState: map[int64]*streamExtractState{}, | streamPhaseState: map[int64]*streamExtractState{}, | ||||
| streamOverlap: &streamIQOverlap{}, | streamOverlap: &streamIQOverlap{}, | ||||
| arbiter: pipeline.NewArbiter(), | arbiter: pipeline.NewArbiter(), | ||||
| telemetry: coll, | |||||
| } | } | ||||
| if rt.useGPU && gpuState != nil { | if rt.useGPU && gpuState != nil { | ||||
| snap := gpuState.snapshot() | snap := gpuState.snapshot() | ||||
| @@ -216,6 +235,15 @@ func (rt *dspRuntime) applyUpdate(upd dspUpdate, srcMgr *sourceManager, rec *rec | |||||
| gpuState.set(false, nil) | gpuState.set(false, nil) | ||||
| } | } | ||||
| } | } | ||||
| if rt.telemetry != nil { | |||||
| rt.telemetry.Event("dsp_config_update", "info", "dsp runtime configuration updated", nil, map[string]any{ | |||||
| "fft_size": rt.cfg.FFTSize, | |||||
| "sample_rate": rt.cfg.SampleRate, | |||||
| "use_gpu_fft": rt.cfg.UseGPUFFT, | |||||
| "detail_fft": rt.detailFFT, | |||||
| "surv_strategy": rt.cfg.Surveillance.Strategy, | |||||
| }) | |||||
| } | |||||
| } | } | ||||
| func (rt *dspRuntime) spectrumFromIQ(iq []complex64, gpuState *gpuStatus) []float64 { | func (rt *dspRuntime) spectrumFromIQ(iq []complex64, gpuState *gpuStatus) []float64 { | ||||
| @@ -334,26 +362,112 @@ func (rt *dspRuntime) decimateSurveillanceIQ(iq []complex64, factor int) []compl | |||||
| return dsp.Decimate(filtered, factor) | return dsp.Decimate(filtered, factor) | ||||
| } | } | ||||
| func meanMagComplex(samples []complex64) float64 { | |||||
| if len(samples) == 0 { | |||||
| return 0 | |||||
| } | |||||
| var sum float64 | |||||
| for _, v := range samples { | |||||
| sum += math.Hypot(float64(real(v)), float64(imag(v))) | |||||
| } | |||||
| return sum / float64(len(samples)) | |||||
| } | |||||
| func phaseStepAbs(a, b complex64) float64 { | |||||
| num := float64(real(a))*float64(imag(b)) - float64(imag(a))*float64(real(b)) | |||||
| den := float64(real(a))*float64(real(b)) + float64(imag(a))*float64(imag(b)) | |||||
| return math.Abs(math.Atan2(num, den)) | |||||
| } | |||||
| func boundaryMetrics(prevTail []complex64, curr []complex64, window int) (float64, float64, float64, int) { | |||||
| if len(curr) == 0 { | |||||
| return 0, 0, 0, 0 | |||||
| } | |||||
| if window <= 0 { | |||||
| window = 16 | |||||
| } | |||||
| headN := window | |||||
| if len(curr) < headN { | |||||
| headN = len(curr) | |||||
| } | |||||
| headMean := meanMagComplex(curr[:headN]) | |||||
| if len(prevTail) == 0 { | |||||
| return headMean, 0, 0, headN | |||||
| } | |||||
| tailN := window | |||||
| if len(prevTail) < tailN { | |||||
| tailN = len(prevTail) | |||||
| } | |||||
| tailMean := meanMagComplex(prevTail[len(prevTail)-tailN:]) | |||||
| deltaMag := math.Abs(headMean - tailMean) | |||||
| phaseJump := phaseStepAbs(prevTail[len(prevTail)-1], curr[0]) | |||||
| score := deltaMag + phaseJump | |||||
| return headMean, tailMean, score, headN | |||||
| } | |||||
| func tailWindowComplex(src []complex64, n int) []complex64 { | |||||
| if n <= 0 || len(src) == 0 { | |||||
| return nil | |||||
| } | |||||
| if len(src) <= n { | |||||
| out := make([]complex64, len(src)) | |||||
| copy(out, src) | |||||
| return out | |||||
| } | |||||
| out := make([]complex64, n) | |||||
| copy(out, src[len(src)-n:]) | |||||
| return out | |||||
| } | |||||
| func (rt *dspRuntime) captureSpectrum(srcMgr *sourceManager, rec *recorder.Manager, dcBlocker *dsp.DCBlocker, gpuState *gpuStatus) (*spectrumArtifacts, error) { | func (rt *dspRuntime) captureSpectrum(srcMgr *sourceManager, rec *recorder.Manager, dcBlocker *dsp.DCBlocker, gpuState *gpuStatus) (*spectrumArtifacts, error) { | ||||
| start := time.Now() | |||||
| required := rt.cfg.FFTSize | required := rt.cfg.FFTSize | ||||
| if rt.detailFFT > required { | if rt.detailFFT > required { | ||||
| required = rt.detailFFT | required = rt.detailFFT | ||||
| } | } | ||||
| available := required | available := required | ||||
| st := srcMgr.Stats() | st := srcMgr.Stats() | ||||
| if st.BufferSamples > required { | |||||
| if rt.telemetry != nil { | |||||
| rt.telemetry.SetGauge("source.buffer_samples", float64(st.BufferSamples), nil) | |||||
| rt.telemetry.SetGauge("source.last_sample_ago_ms", float64(st.LastSampleAgoMs), nil) | |||||
| rt.telemetry.SetGauge("source.dropped", float64(st.Dropped), nil) | |||||
| rt.telemetry.SetGauge("source.resets", float64(st.Resets), nil) | |||||
| } | |||||
| if forceFixedStreamReadSamples > 0 { | |||||
| available = forceFixedStreamReadSamples | |||||
| if available < required { | |||||
| available = required | |||||
| } | |||||
| available = (available / required) * required | |||||
| if available < required { | |||||
| available = required | |||||
| } | |||||
| logging.Warn("boundary", "fixed_stream_read_samples", "configured", forceFixedStreamReadSamples, "effective", available, "required", required) | |||||
| } else if st.BufferSamples > required { | |||||
| available = (st.BufferSamples / required) * required | available = (st.BufferSamples / required) * required | ||||
| if available < required { | if available < required { | ||||
| available = required | available = required | ||||
| } | } | ||||
| } | } | ||||
| logging.Debug("capture", "read_iq", "required", required, "available", available, "buf", st.BufferSamples, "reset", st.Resets, "drop", st.Dropped) | logging.Debug("capture", "read_iq", "required", required, "available", available, "buf", st.BufferSamples, "reset", st.Resets, "drop", st.Dropped) | ||||
| readStart := time.Now() | |||||
| allIQ, err := srcMgr.ReadIQ(available) | allIQ, err := srcMgr.ReadIQ(available) | ||||
| if err != nil { | if err != nil { | ||||
| if rt.telemetry != nil { | |||||
| rt.telemetry.IncCounter("capture.read.error", 1, nil) | |||||
| } | |||||
| return nil, err | return nil, err | ||||
| } | } | ||||
| if rt.telemetry != nil { | |||||
| rt.telemetry.Observe("capture.read.duration_ms", float64(time.Since(readStart).Microseconds())/1000.0, nil) | |||||
| rt.telemetry.Observe("capture.read.samples", float64(len(allIQ)), nil) | |||||
| } | |||||
| if rec != nil { | if rec != nil { | ||||
| ingestStart := time.Now() | |||||
| rec.Ingest(time.Now(), allIQ) | rec.Ingest(time.Now(), allIQ) | ||||
| if rt.telemetry != nil { | |||||
| rt.telemetry.Observe("capture.ingest.duration_ms", float64(time.Since(ingestStart).Microseconds())/1000.0, nil) | |||||
| } | |||||
| } | } | ||||
| // Cap allIQ for downstream extraction to prevent buffer bloat. | // Cap allIQ for downstream extraction to prevent buffer bloat. | ||||
| // Without this cap, buffer accumulation during processing stalls causes | // Without this cap, buffer accumulation during processing stalls causes | ||||
| @@ -366,8 +480,17 @@ func (rt *dspRuntime) captureSpectrum(srcMgr *sourceManager, rec *recorder.Manag | |||||
| maxStreamSamples = required | maxStreamSamples = required | ||||
| } | } | ||||
| maxStreamSamples = (maxStreamSamples / required) * required | maxStreamSamples = (maxStreamSamples / required) * required | ||||
| streamDropped := false | |||||
| if len(allIQ) > maxStreamSamples { | if len(allIQ) > maxStreamSamples { | ||||
| allIQ = allIQ[len(allIQ)-maxStreamSamples:] | allIQ = allIQ[len(allIQ)-maxStreamSamples:] | ||||
| streamDropped = true | |||||
| if rt.telemetry != nil { | |||||
| rt.telemetry.IncCounter("capture.stream_drop.count", 1, nil) | |||||
| rt.telemetry.Event("iq_dropped", "warn", "capture IQ dropped before extraction", nil, map[string]any{ | |||||
| "max_stream_samples": maxStreamSamples, | |||||
| "required": required, | |||||
| }) | |||||
| } | |||||
| } | } | ||||
| logging.Debug("capture", "iq_len", "len", len(allIQ), "surv_fft", rt.cfg.FFTSize, "detail_fft", rt.detailFFT) | logging.Debug("capture", "iq_len", "len", len(allIQ), "surv_fft", rt.cfg.FFTSize, "detail_fft", rt.detailFFT) | ||||
| survIQ := allIQ | survIQ := allIQ | ||||
| @@ -380,14 +503,60 @@ func (rt *dspRuntime) captureSpectrum(srcMgr *sourceManager, rec *recorder.Manag | |||||
| } | } | ||||
| if rt.dcEnabled { | if rt.dcEnabled { | ||||
| dcBlocker.Apply(allIQ) | dcBlocker.Apply(allIQ) | ||||
| if rt.telemetry != nil { | |||||
| rt.telemetry.IncCounter("dsp.dc_block.apply", 1, nil) | |||||
| } | |||||
| } | } | ||||
| if rt.iqEnabled { | if rt.iqEnabled { | ||||
| // IQBalance must NOT modify allIQ in-place: allIQ goes to the extraction | |||||
| // pipeline and any in-place modification creates a phase/amplitude | |||||
| // discontinuity at the survIQ boundary (len-FFTSize) that the polyphase | |||||
| // extractor then sees as paired click artifacts in the FM discriminator. | |||||
| detailIsSurv := sameIQBuffer(detailIQ, survIQ) | |||||
| survIQ = append([]complex64(nil), survIQ...) | |||||
| dsp.IQBalance(survIQ) | dsp.IQBalance(survIQ) | ||||
| if !sameIQBuffer(detailIQ, survIQ) { | |||||
| if detailIsSurv { | |||||
| detailIQ = survIQ | |||||
| } else { | |||||
| detailIQ = append([]complex64(nil), detailIQ...) | detailIQ = append([]complex64(nil), detailIQ...) | ||||
| dsp.IQBalance(detailIQ) | dsp.IQBalance(detailIQ) | ||||
| } | } | ||||
| } | } | ||||
| if rt.telemetry != nil { | |||||
| rt.telemetry.SetGauge("iq.stage.all.length", float64(len(allIQ)), nil) | |||||
| rt.telemetry.SetGauge("iq.stage.surveillance.length", float64(len(survIQ)), nil) | |||||
| rt.telemetry.SetGauge("iq.stage.detail.length", float64(len(detailIQ)), nil) | |||||
| rt.telemetry.Observe("capture.total.duration_ms", float64(time.Since(start).Microseconds())/1000.0, nil) | |||||
| headMean, tailMean, boundaryScore, boundaryWindow := boundaryMetrics(rt.lastAllIQTail, allIQ, 32) | |||||
| rt.telemetry.SetGauge("iq.boundary.all.head_mean_mag", headMean, nil) | |||||
| rt.telemetry.SetGauge("iq.boundary.all.prev_tail_mean_mag", tailMean, nil) | |||||
| rt.telemetry.Observe("iq.boundary.all.discontinuity_score", boundaryScore, nil) | |||||
| if len(rt.lastAllIQTail) > 0 && len(allIQ) > 0 { | |||||
| deltaMag := math.Abs(math.Hypot(float64(real(allIQ[0])), float64(imag(allIQ[0]))) - math.Hypot(float64(real(rt.lastAllIQTail[len(rt.lastAllIQTail)-1])), float64(imag(rt.lastAllIQTail[len(rt.lastAllIQTail)-1])))) | |||||
| phaseJump := phaseStepAbs(rt.lastAllIQTail[len(rt.lastAllIQTail)-1], allIQ[0]) | |||||
| rt.telemetry.Observe("iq.boundary.all.delta_mag", deltaMag, nil) | |||||
| rt.telemetry.Observe("iq.boundary.all.delta_phase", phaseJump, nil) | |||||
| if rt.telemetry.ShouldSampleHeavy() { | |||||
| rt.telemetry.Event("alliq_boundary", "info", "allIQ boundary snapshot", nil, map[string]any{ | |||||
| "window": boundaryWindow, | |||||
| "head_mean_mag": headMean, | |||||
| "prev_tail_mean_mag": tailMean, | |||||
| "delta_mag": deltaMag, | |||||
| "delta_phase": phaseJump, | |||||
| "discontinuity_score": boundaryScore, | |||||
| "alliq_len": len(allIQ), | |||||
| "stream_dropped": streamDropped, | |||||
| }) | |||||
| } | |||||
| } | |||||
| if rt.telemetry.ShouldSampleHeavy() { | |||||
| observeIQStats(rt.telemetry, "capture_all", allIQ, nil) | |||||
| observeIQStats(rt.telemetry, "capture_surveillance", survIQ, nil) | |||||
| observeIQStats(rt.telemetry, "capture_detail", detailIQ, nil) | |||||
| } | |||||
| } | |||||
| rt.lastAllIQTail = tailWindowComplex(allIQ, 32) | |||||
| survSpectrum := rt.spectrumFromIQ(survIQ, gpuState) | survSpectrum := rt.spectrumFromIQ(survIQ, gpuState) | ||||
| sanitizeSpectrum(survSpectrum) | sanitizeSpectrum(survSpectrum) | ||||
| detailSpectrum := survSpectrum | detailSpectrum := survSpectrum | ||||
| @@ -430,8 +599,13 @@ func (rt *dspRuntime) captureSpectrum(srcMgr *sourceManager, rec *recorder.Manag | |||||
| } | } | ||||
| now := time.Now() | now := time.Now() | ||||
| finished, detected := rt.det.Process(now, survSpectrum, rt.cfg.CenterHz) | finished, detected := rt.det.Process(now, survSpectrum, rt.cfg.CenterHz) | ||||
| if rt.telemetry != nil { | |||||
| rt.telemetry.SetGauge("signals.detected.count", float64(len(detected)), nil) | |||||
| rt.telemetry.SetGauge("signals.finished.count", float64(len(finished)), nil) | |||||
| } | |||||
| return &spectrumArtifacts{ | return &spectrumArtifacts{ | ||||
| allIQ: allIQ, | allIQ: allIQ, | ||||
| streamDropped: streamDropped, | |||||
| surveillanceIQ: survIQ, | surveillanceIQ: survIQ, | ||||
| detailIQ: detailIQ, | detailIQ: detailIQ, | ||||
| surveillanceSpectrum: survSpectrum, | surveillanceSpectrum: survSpectrum, | ||||
| @@ -13,7 +13,7 @@ func TestNewDSPRuntime(t *testing.T) { | |||||
| cfg := config.Default() | cfg := config.Default() | ||||
| det := detector.New(cfg.Detector, cfg.SampleRate, cfg.FFTSize) | det := detector.New(cfg.Detector, cfg.SampleRate, cfg.FFTSize) | ||||
| window := fftutil.Hann(cfg.FFTSize) | window := fftutil.Hann(cfg.FFTSize) | ||||
| rt := newDSPRuntime(cfg, det, window, &gpuStatus{}) | |||||
| rt := newDSPRuntime(cfg, det, window, &gpuStatus{}, nil) | |||||
| if rt == nil { | if rt == nil { | ||||
| t.Fatalf("runtime is nil") | t.Fatalf("runtime is nil") | ||||
| } | } | ||||
| @@ -47,7 +47,7 @@ func TestSurveillanceLevelsRespectStrategy(t *testing.T) { | |||||
| cfg := config.Default() | cfg := config.Default() | ||||
| det := detector.New(cfg.Detector, cfg.SampleRate, cfg.FFTSize) | det := detector.New(cfg.Detector, cfg.SampleRate, cfg.FFTSize) | ||||
| window := fftutil.Hann(cfg.FFTSize) | window := fftutil.Hann(cfg.FFTSize) | ||||
| rt := newDSPRuntime(cfg, det, window, &gpuStatus{}) | |||||
| rt := newDSPRuntime(cfg, det, window, &gpuStatus{}, nil) | |||||
| policy := pipeline.Policy{SurveillanceStrategy: "single-resolution"} | policy := pipeline.Policy{SurveillanceStrategy: "single-resolution"} | ||||
| plan := rt.buildSurveillancePlan(policy) | plan := rt.buildSurveillancePlan(policy) | ||||
| if len(plan.Levels) != 1 { | if len(plan.Levels) != 1 { | ||||
| @@ -1,11 +1,16 @@ | |||||
| package main | package main | ||||
| import ( | import ( | ||||
| "fmt" | |||||
| "time" | |||||
| "sdr-wideband-suite/internal/config" | "sdr-wideband-suite/internal/config" | ||||
| "sdr-wideband-suite/internal/sdr" | "sdr-wideband-suite/internal/sdr" | ||||
| "sdr-wideband-suite/internal/telemetry" | |||||
| ) | ) | ||||
| func (m *sourceManager) Restart(cfg config.Config) error { | func (m *sourceManager) Restart(cfg config.Config) error { | ||||
| start := time.Now() | |||||
| m.mu.Lock() | m.mu.Lock() | ||||
| defer m.mu.Unlock() | defer m.mu.Unlock() | ||||
| old := m.src | old := m.src | ||||
| @@ -14,15 +19,27 @@ func (m *sourceManager) Restart(cfg config.Config) error { | |||||
| if err != nil { | if err != nil { | ||||
| _ = old.Start() | _ = old.Start() | ||||
| m.src = old | m.src = old | ||||
| if m.telemetry != nil { | |||||
| m.telemetry.IncCounter("source.restart.error", 1, nil) | |||||
| m.telemetry.Event("source_restart_failed", "warn", "source restart failed", nil, map[string]any{"error": err.Error()}) | |||||
| } | |||||
| return err | return err | ||||
| } | } | ||||
| if err := next.Start(); err != nil { | if err := next.Start(); err != nil { | ||||
| _ = next.Stop() | _ = next.Stop() | ||||
| _ = old.Start() | _ = old.Start() | ||||
| m.src = old | m.src = old | ||||
| if m.telemetry != nil { | |||||
| m.telemetry.IncCounter("source.restart.error", 1, nil) | |||||
| m.telemetry.Event("source_restart_failed", "warn", "source restart failed", nil, map[string]any{"error": err.Error()}) | |||||
| } | |||||
| return err | return err | ||||
| } | } | ||||
| m.src = next | m.src = next | ||||
| if m.telemetry != nil { | |||||
| m.telemetry.IncCounter("source.restart.count", 1, nil) | |||||
| m.telemetry.Observe("source.restart.duration_ms", float64(time.Since(start).Milliseconds()), nil) | |||||
| } | |||||
| return nil | return nil | ||||
| } | } | ||||
| @@ -44,7 +61,11 @@ func (m *sourceManager) Flush() { | |||||
| } | } | ||||
| func newSourceManager(src sdr.Source, newSource func(cfg config.Config) (sdr.Source, error)) *sourceManager { | func newSourceManager(src sdr.Source, newSource func(cfg config.Config) (sdr.Source, error)) *sourceManager { | ||||
| return &sourceManager{src: src, newSource: newSource} | |||||
| return newSourceManagerWithTelemetry(src, newSource, nil) | |||||
| } | |||||
| func newSourceManagerWithTelemetry(src sdr.Source, newSource func(cfg config.Config) (sdr.Source, error), coll *telemetry.Collector) *sourceManager { | |||||
| return &sourceManager{src: src, newSource: newSource, telemetry: coll} | |||||
| } | } | ||||
| func (m *sourceManager) Start() error { | func (m *sourceManager) Start() error { | ||||
| @@ -60,9 +81,27 @@ func (m *sourceManager) Stop() error { | |||||
| } | } | ||||
| func (m *sourceManager) ReadIQ(n int) ([]complex64, error) { | func (m *sourceManager) ReadIQ(n int) ([]complex64, error) { | ||||
| waitStart := time.Now() | |||||
| m.mu.RLock() | m.mu.RLock() | ||||
| wait := time.Since(waitStart) | |||||
| defer m.mu.RUnlock() | defer m.mu.RUnlock() | ||||
| return m.src.ReadIQ(n) | |||||
| if m.telemetry != nil { | |||||
| m.telemetry.Observe("source.lock_wait_ms", float64(wait.Microseconds())/1000.0, telemetry.TagsFromPairs("lock", "read")) | |||||
| if wait > 2*time.Millisecond { | |||||
| m.telemetry.IncCounter("source.lock_contention.count", 1, telemetry.TagsFromPairs("lock", "read")) | |||||
| } | |||||
| } | |||||
| readStart := time.Now() | |||||
| out, err := m.src.ReadIQ(n) | |||||
| if m.telemetry != nil { | |||||
| tags := telemetry.TagsFromPairs("requested", fmt.Sprintf("%d", n)) | |||||
| m.telemetry.Observe("source.read.duration_ms", float64(time.Since(readStart).Microseconds())/1000.0, tags) | |||||
| m.telemetry.SetGauge("source.read.samples", float64(len(out)), nil) | |||||
| if err != nil { | |||||
| m.telemetry.IncCounter("source.read.error", 1, nil) | |||||
| } | |||||
| } | |||||
| return out, err | |||||
| } | } | ||||
| func (m *sourceManager) ApplyConfig(cfg config.Config) error { | func (m *sourceManager) ApplyConfig(cfg config.Config) error { | ||||
| @@ -0,0 +1,45 @@ | |||||
| package main | |||||
| import ( | |||||
| "fmt" | |||||
| "sdr-wideband-suite/internal/demod/gpudemod" | |||||
| "sdr-wideband-suite/internal/telemetry" | |||||
| ) | |||||
| func observeStreamingComparison(coll *telemetry.Collector, oracle gpudemod.StreamingExtractResult, prod gpudemod.StreamingExtractResult) { | |||||
| if coll == nil { | |||||
| return | |||||
| } | |||||
| metrics, stats := gpudemod.CompareOracleAndGPUHostOracle(oracle, prod) | |||||
| tags := telemetry.TagsFromPairs("signal_id", fmt.Sprintf("%d", oracle.SignalID), "path", "streaming_compare") | |||||
| coll.SetGauge("streaming.compare.n_out", float64(metrics.NOut), tags) | |||||
| coll.SetGauge("streaming.compare.phase_count", float64(metrics.PhaseCount), tags) | |||||
| coll.SetGauge("streaming.compare.history_len", float64(metrics.HistoryLen), tags) | |||||
| coll.Observe("streaming.compare.ref_max_abs_err", metrics.RefMaxAbsErr, tags) | |||||
| coll.Observe("streaming.compare.ref_rms_err", metrics.RefRMSErr, tags) | |||||
| coll.SetGauge("streaming.compare.compare_count", float64(stats.Count), tags) | |||||
| coll.SetGauge("streaming.compare.oracle_rate", float64(oracle.Rate), tags) | |||||
| coll.SetGauge("streaming.compare.production_rate", float64(prod.Rate), tags) | |||||
| coll.SetGauge("streaming.compare.oracle_output_len", float64(len(oracle.IQ)), tags) | |||||
| coll.SetGauge("streaming.compare.production_output_len", float64(len(prod.IQ)), tags) | |||||
| if len(oracle.IQ) > 0 { | |||||
| oracleStats := computeIQHeadStats(oracle.IQ, 64) | |||||
| coll.Observe("streaming.compare.oracle_head_mean_mag", oracleStats.meanMag, tags) | |||||
| coll.Observe("streaming.compare.oracle_head_max_step", oracleStats.maxStep, tags) | |||||
| } | |||||
| if len(prod.IQ) > 0 { | |||||
| prodStats := computeIQHeadStats(prod.IQ, 64) | |||||
| coll.Observe("streaming.compare.production_head_mean_mag", prodStats.meanMag, tags) | |||||
| coll.Observe("streaming.compare.production_head_max_step", prodStats.maxStep, tags) | |||||
| } | |||||
| coll.Event("streaming_compare_snapshot", "info", "streaming comparison snapshot", tags, map[string]any{ | |||||
| "oracle_rate": oracle.Rate, | |||||
| "production_rate": prod.Rate, | |||||
| "oracle_output_len": len(oracle.IQ), | |||||
| "production_output_len": len(prod.IQ), | |||||
| "ref_max_abs_err": metrics.RefMaxAbsErr, | |||||
| "ref_rms_err": metrics.RefRMSErr, | |||||
| "compare_count": stats.Count, | |||||
| }) | |||||
| } | |||||
| @@ -0,0 +1,27 @@ | |||||
| package main | |||||
| import ( | |||||
| "fmt" | |||||
| "sdr-wideband-suite/internal/demod/gpudemod" | |||||
| "sdr-wideband-suite/internal/telemetry" | |||||
| ) | |||||
| func observeStreamingResult(coll *telemetry.Collector, prefix string, res gpudemod.StreamingExtractResult) { | |||||
| if coll == nil { | |||||
| return | |||||
| } | |||||
| tags := telemetry.TagsFromPairs("signal_id", fmt.Sprintf("%d", res.SignalID), "path", prefix) | |||||
| coll.SetGauge(prefix+".n_out", float64(res.NOut), tags) | |||||
| coll.SetGauge(prefix+".phase_count", float64(res.PhaseCount), tags) | |||||
| coll.SetGauge(prefix+".history_len", float64(res.HistoryLen), tags) | |||||
| coll.SetGauge(prefix+".rate", float64(res.Rate), tags) | |||||
| coll.SetGauge(prefix+".output_len", float64(len(res.IQ)), tags) | |||||
| if len(res.IQ) > 0 { | |||||
| stats := computeIQHeadStats(res.IQ, 64) | |||||
| coll.Observe(prefix+".head_mean_mag", stats.meanMag, tags) | |||||
| coll.Observe(prefix+".head_max_step", stats.maxStep, tags) | |||||
| coll.Observe(prefix+".head_p95_step", stats.p95Step, tags) | |||||
| coll.SetGauge(prefix+".head_low_magnitude_count", float64(stats.lowMag), tags) | |||||
| } | |||||
| } | |||||
| @@ -0,0 +1,50 @@ | |||||
| package main | |||||
| import ( | |||||
| "fmt" | |||||
| "sdr-wideband-suite/internal/demod/gpudemod" | |||||
| "sdr-wideband-suite/internal/detector" | |||||
| "sdr-wideband-suite/internal/telemetry" | |||||
| ) | |||||
| func extractForStreamingProduction( | |||||
| extractMgr *extractionManager, | |||||
| allIQ []complex64, | |||||
| sampleRate int, | |||||
| centerHz float64, | |||||
| signals []detector.Signal, | |||||
| aqCfg extractionConfig, | |||||
| coll *telemetry.Collector, | |||||
| ) ([][]complex64, []int, error) { | |||||
| out := make([][]complex64, len(signals)) | |||||
| rates := make([]int, len(signals)) | |||||
| jobs, err := buildStreamingJobs(sampleRate, centerHz, signals, aqCfg) | |||||
| if err != nil { | |||||
| return nil, nil, err | |||||
| } | |||||
| runner := extractMgr.get(len(allIQ), sampleRate) | |||||
| if runner == nil { | |||||
| return nil, nil, fmt.Errorf("streaming production path unavailable: no batch runner") | |||||
| } | |||||
| results, err := runner.StreamingExtractGPU(allIQ, jobs) | |||||
| if err != nil { | |||||
| return nil, nil, err | |||||
| } | |||||
| var oracleResults []gpudemod.StreamingExtractResult | |||||
| if useStreamingOraclePath { | |||||
| if streamingOracleRunner == nil || streamingOracleRunner.SampleRate != sampleRate { | |||||
| streamingOracleRunner = gpudemod.NewCPUOracleRunner(sampleRate) | |||||
| } | |||||
| oracleResults, _ = streamingOracleRunner.StreamingExtract(allIQ, jobs) | |||||
| } | |||||
| for i, res := range results { | |||||
| out[i] = res.IQ | |||||
| rates[i] = res.Rate | |||||
| observeStreamingResult(coll, "streaming.production", res) | |||||
| if i < len(oracleResults) { | |||||
| observeStreamingComparison(coll, oracleResults[i], res) | |||||
| } | |||||
| } | |||||
| return out, rates, nil | |||||
| } | |||||
| @@ -0,0 +1,137 @@ | |||||
| package main | |||||
| import ( | |||||
| "math" | |||||
| "sdr-wideband-suite/internal/demod/gpudemod" | |||||
| "sdr-wideband-suite/internal/detector" | |||||
| "sdr-wideband-suite/internal/telemetry" | |||||
| ) | |||||
| const useStreamingOraclePath = false // temporarily disable oracle during bring-up to isolate production-path runtime behavior | |||||
| const useStreamingProductionPath = true // route top-level extraction through the new production path during bring-up/validation | |||||
| var streamingOracleRunner *gpudemod.CPUOracleRunner | |||||
| func buildStreamingJobs(sampleRate int, centerHz float64, signals []detector.Signal, aqCfg extractionConfig) ([]gpudemod.StreamingExtractJob, error) { | |||||
| jobs := make([]gpudemod.StreamingExtractJob, len(signals)) | |||||
| bwMult := aqCfg.bwMult | |||||
| if bwMult <= 0 { | |||||
| bwMult = 1.0 | |||||
| } | |||||
| firTaps := aqCfg.firTaps | |||||
| if firTaps <= 0 { | |||||
| firTaps = 101 | |||||
| } | |||||
| for i, sig := range signals { | |||||
| bw := sig.BWHz * bwMult | |||||
| sigMHz := sig.CenterHz / 1e6 | |||||
| isWFM := (sigMHz >= 87.5 && sigMHz <= 108.0) || | |||||
| (sig.Class != nil && (sig.Class.ModType == "WFM" || sig.Class.ModType == "WFM_STEREO")) | |||||
| var outRate int | |||||
| if isWFM { | |||||
| outRate = wfmStreamOutRate | |||||
| if bw < wfmStreamMinBW { | |||||
| bw = wfmStreamMinBW | |||||
| } | |||||
| } else { | |||||
| // Non-WFM target: must be an exact integer divisor of sampleRate. | |||||
| // The old hardcoded 200000 fails for common SDR rates (e.g. 4096000/200000=20.48). | |||||
| // Find the nearest valid rate >= 128000 (enough for NFM/AM/SSB). | |||||
| outRate = nearestExactDecimationRate(sampleRate, 200000, 128000) | |||||
| if bw < 20000 { | |||||
| bw = 20000 | |||||
| } | |||||
| } | |||||
| if _, err := gpudemod.ExactIntegerDecimation(sampleRate, outRate); err != nil { | |||||
| return nil, err | |||||
| } | |||||
| offset := sig.CenterHz - centerHz | |||||
| jobs[i] = gpudemod.StreamingExtractJob{ | |||||
| SignalID: sig.ID, | |||||
| OffsetHz: offset, | |||||
| Bandwidth: bw, | |||||
| OutRate: outRate, | |||||
| NumTaps: firTaps, | |||||
| ConfigHash: gpudemod.StreamingConfigHash(sig.ID, offset, bw, outRate, firTaps, sampleRate), | |||||
| } | |||||
| } | |||||
| return jobs, nil | |||||
| } | |||||
| func resetStreamingOracleRunner() { | |||||
| if streamingOracleRunner != nil { | |||||
| streamingOracleRunner.ResetAllStates() | |||||
| } | |||||
| } | |||||
| func extractForStreamingOracle( | |||||
| allIQ []complex64, | |||||
| sampleRate int, | |||||
| centerHz float64, | |||||
| signals []detector.Signal, | |||||
| aqCfg extractionConfig, | |||||
| coll *telemetry.Collector, | |||||
| ) ([][]complex64, []int, error) { | |||||
| out := make([][]complex64, len(signals)) | |||||
| rates := make([]int, len(signals)) | |||||
| jobs, err := buildStreamingJobs(sampleRate, centerHz, signals, aqCfg) | |||||
| if err != nil { | |||||
| return nil, nil, err | |||||
| } | |||||
| if streamingOracleRunner == nil || streamingOracleRunner.SampleRate != sampleRate { | |||||
| streamingOracleRunner = gpudemod.NewCPUOracleRunner(sampleRate) | |||||
| } | |||||
| results, err := streamingOracleRunner.StreamingExtract(allIQ, jobs) | |||||
| if err != nil { | |||||
| return nil, nil, err | |||||
| } | |||||
| for i, res := range results { | |||||
| out[i] = res.IQ | |||||
| rates[i] = res.Rate | |||||
| observeStreamingResult(coll, "streaming.oracle", res) | |||||
| } | |||||
| return out, rates, nil | |||||
| } | |||||
| func phaseIncForOffset(sampleRate int, offsetHz float64) float64 { | |||||
| return -2.0 * math.Pi * offsetHz / float64(sampleRate) | |||||
| } | |||||
| // nearestExactDecimationRate finds the output rate closest to targetRate | |||||
| // (but not below minRate) that is an exact integer divisor of sampleRate. | |||||
| // This avoids the ExactIntegerDecimation check failing for rates like | |||||
| // 4096000/200000=20.48 which silently killed the entire streaming batch. | |||||
| func nearestExactDecimationRate(sampleRate int, targetRate int, minRate int) int { | |||||
| if sampleRate <= 0 || targetRate <= 0 { | |||||
| return targetRate | |||||
| } | |||||
| if sampleRate%targetRate == 0 { | |||||
| return targetRate // already exact | |||||
| } | |||||
| // Try decimation factors near the target | |||||
| targetDecim := sampleRate / targetRate // floor | |||||
| bestRate := 0 | |||||
| bestDist := sampleRate // impossibly large | |||||
| for d := max(1, targetDecim-2); d <= targetDecim+2; d++ { | |||||
| rate := sampleRate / d | |||||
| if rate < minRate { | |||||
| continue | |||||
| } | |||||
| if sampleRate%rate != 0 { | |||||
| continue // not exact (shouldn't happen since rate = sampleRate/d, but guard) | |||||
| } | |||||
| dist := targetRate - rate | |||||
| if dist < 0 { | |||||
| dist = -dist | |||||
| } | |||||
| if dist < bestDist { | |||||
| bestDist = dist | |||||
| bestRate = rate | |||||
| } | |||||
| } | |||||
| if bestRate > 0 { | |||||
| return bestRate | |||||
| } | |||||
| return targetRate // fallback — will fail ExactIntegerDecimation and surface the error | |||||
| } | |||||
| @@ -11,6 +11,7 @@ import ( | |||||
| "sdr-wideband-suite/internal/detector" | "sdr-wideband-suite/internal/detector" | ||||
| "sdr-wideband-suite/internal/pipeline" | "sdr-wideband-suite/internal/pipeline" | ||||
| "sdr-wideband-suite/internal/sdr" | "sdr-wideband-suite/internal/sdr" | ||||
| "sdr-wideband-suite/internal/telemetry" | |||||
| ) | ) | ||||
| type SpectrumDebug struct { | type SpectrumDebug struct { | ||||
| @@ -110,6 +111,7 @@ type sourceManager struct { | |||||
| mu sync.RWMutex | mu sync.RWMutex | ||||
| src sdr.Source | src sdr.Source | ||||
| newSource func(cfg config.Config) (sdr.Source, error) | newSource func(cfg config.Config) (sdr.Source, error) | ||||
| telemetry *telemetry.Collector | |||||
| } | } | ||||
| type extractionManager struct { | type extractionManager struct { | ||||
| @@ -0,0 +1,343 @@ | |||||
| bands: | |||||
| - name: uk-fm-broadcast | |||||
| start_hz: 8.75e+07 | |||||
| end_hz: 1.08e+08 | |||||
| center_hz: 1.02e+08 | |||||
| sample_rate: 4096000 | |||||
| fft_size: 512 | |||||
| gain_db: 32 | |||||
| tuner_bw_khz: 5000 | |||||
| use_gpu_fft: true | |||||
| classifier_mode: combined | |||||
| agc: true | |||||
| dc_block: true | |||||
| iq_balance: true | |||||
| pipeline: | |||||
| mode: wideband-balanced | |||||
| profile: wideband-balanced | |||||
| goals: | |||||
| intent: broadcast-monitoring | |||||
| monitor_start_hz: 8.8e+07 | |||||
| monitor_end_hz: 1.08e+08 | |||||
| monitor_span_hz: 2e+07 | |||||
| monitor_windows: | |||||
| - label: "" | |||||
| zone: focus | |||||
| start_hz: 8.75e+07 | |||||
| end_hz: 1.08e+08 | |||||
| center_hz: 0 | |||||
| span_hz: 0 | |||||
| priority: 1.25 | |||||
| auto_record: false | |||||
| auto_decode: false | |||||
| - label: "" | |||||
| zone: decode | |||||
| start_hz: 8.75e+07 | |||||
| end_hz: 1.08e+08 | |||||
| center_hz: 0 | |||||
| span_hz: 0 | |||||
| priority: 1.35 | |||||
| auto_record: false | |||||
| auto_decode: false | |||||
| signal_priorities: | |||||
| - wfm | |||||
| - rds | |||||
| - broadcast | |||||
| auto_record_classes: | |||||
| - WFM | |||||
| - WFM_STEREO | |||||
| auto_decode_classes: | |||||
| - WFM | |||||
| - WFM_STEREO | |||||
| - RDS | |||||
| surveillance: | |||||
| analysis_fft_size: 512 | |||||
| frame_rate: 12 | |||||
| strategy: multi-resolution | |||||
| display_bins: 2048 | |||||
| display_fps: 12 | |||||
| derived_detection: auto | |||||
| refinement: | |||||
| enabled: true | |||||
| max_concurrent: 24 | |||||
| detail_fft_size: 4096 | |||||
| min_candidate_snr_db: -3 | |||||
| min_span_hz: 60000 | |||||
| max_span_hz: 250000 | |||||
| auto_span: true | |||||
| resources: | |||||
| prefer_gpu: true | |||||
| max_refinement_jobs: 24 | |||||
| max_recording_streams: 32 | |||||
| max_decode_jobs: 16 | |||||
| decision_hold_ms: 2500 | |||||
| profiles: | |||||
| - name: legacy | |||||
| description: Current single-band pipeline behavior | |||||
| pipeline: | |||||
| mode: legacy | |||||
| profile: legacy | |||||
| goals: | |||||
| intent: general-monitoring | |||||
| monitor_start_hz: 0 | |||||
| monitor_end_hz: 0 | |||||
| monitor_span_hz: 0 | |||||
| monitor_windows: [] | |||||
| signal_priorities: [] | |||||
| auto_record_classes: [] | |||||
| auto_decode_classes: [] | |||||
| surveillance: | |||||
| analysis_fft_size: 2048 | |||||
| frame_rate: 15 | |||||
| strategy: single-resolution | |||||
| display_bins: 2048 | |||||
| display_fps: 15 | |||||
| derived_detection: auto | |||||
| refinement: | |||||
| enabled: true | |||||
| max_concurrent: 8 | |||||
| detail_fft_size: 2048 | |||||
| min_candidate_snr_db: 0 | |||||
| min_span_hz: 0 | |||||
| max_span_hz: 0 | |||||
| auto_span: true | |||||
| resources: | |||||
| prefer_gpu: false | |||||
| max_refinement_jobs: 8 | |||||
| max_recording_streams: 16 | |||||
| max_decode_jobs: 16 | |||||
| decision_hold_ms: 2000 | |||||
| - name: wideband-balanced | |||||
| description: Baseline multi-resolution wideband surveillance | |||||
| pipeline: | |||||
| mode: wideband-balanced | |||||
| profile: wideband-balanced | |||||
| goals: | |||||
| intent: broadcast-monitoring | |||||
| monitor_start_hz: 0 | |||||
| monitor_end_hz: 0 | |||||
| monitor_span_hz: 0 | |||||
| monitor_windows: [] | |||||
| signal_priorities: | |||||
| - wfm | |||||
| - rds | |||||
| - broadcast | |||||
| auto_record_classes: | |||||
| - WFM | |||||
| - WFM_STEREO | |||||
| auto_decode_classes: | |||||
| - WFM | |||||
| - WFM_STEREO | |||||
| - RDS | |||||
| surveillance: | |||||
| analysis_fft_size: 4096 | |||||
| frame_rate: 12 | |||||
| strategy: multi-resolution | |||||
| display_bins: 2048 | |||||
| display_fps: 12 | |||||
| derived_detection: auto | |||||
| refinement: | |||||
| enabled: true | |||||
| max_concurrent: 24 | |||||
| detail_fft_size: 4096 | |||||
| min_candidate_snr_db: -3 | |||||
| min_span_hz: 60000 | |||||
| max_span_hz: 250000 | |||||
| auto_span: true | |||||
| resources: | |||||
| prefer_gpu: true | |||||
| max_refinement_jobs: 24 | |||||
| max_recording_streams: 32 | |||||
| max_decode_jobs: 16 | |||||
| decision_hold_ms: 2500 | |||||
| - name: wideband-aggressive | |||||
| description: Higher surveillance/refinement budgets for dense wideband monitoring | |||||
| pipeline: | |||||
| mode: wideband-aggressive | |||||
| profile: wideband-aggressive | |||||
| goals: | |||||
| intent: high-density-wideband-surveillance | |||||
| monitor_start_hz: 0 | |||||
| monitor_end_hz: 0 | |||||
| monitor_span_hz: 0 | |||||
| monitor_windows: [] | |||||
| signal_priorities: | |||||
| - wfm | |||||
| - rds | |||||
| - broadcast | |||||
| - digital | |||||
| auto_record_classes: [] | |||||
| auto_decode_classes: [] | |||||
| surveillance: | |||||
| analysis_fft_size: 8192 | |||||
| frame_rate: 10 | |||||
| strategy: multi-resolution | |||||
| display_bins: 4096 | |||||
| display_fps: 10 | |||||
| derived_detection: auto | |||||
| refinement: | |||||
| enabled: true | |||||
| max_concurrent: 32 | |||||
| detail_fft_size: 8192 | |||||
| min_candidate_snr_db: -3 | |||||
| min_span_hz: 50000 | |||||
| max_span_hz: 280000 | |||||
| auto_span: true | |||||
| resources: | |||||
| prefer_gpu: true | |||||
| max_refinement_jobs: 32 | |||||
| max_recording_streams: 40 | |||||
| max_decode_jobs: 24 | |||||
| decision_hold_ms: 2500 | |||||
| - name: archive | |||||
| description: Record-first monitoring profile | |||||
| pipeline: | |||||
| mode: archive | |||||
| profile: archive | |||||
| goals: | |||||
| intent: archive-and-triage | |||||
| monitor_start_hz: 0 | |||||
| monitor_end_hz: 0 | |||||
| monitor_span_hz: 0 | |||||
| monitor_windows: [] | |||||
| signal_priorities: | |||||
| - wfm | |||||
| - broadcast | |||||
| - digital | |||||
| auto_record_classes: [] | |||||
| auto_decode_classes: [] | |||||
| surveillance: | |||||
| analysis_fft_size: 4096 | |||||
| frame_rate: 12 | |||||
| strategy: single-resolution | |||||
| display_bins: 2048 | |||||
| display_fps: 12 | |||||
| derived_detection: auto | |||||
| refinement: | |||||
| enabled: true | |||||
| max_concurrent: 16 | |||||
| detail_fft_size: 4096 | |||||
| min_candidate_snr_db: -2 | |||||
| min_span_hz: 50000 | |||||
| max_span_hz: 250000 | |||||
| auto_span: true | |||||
| resources: | |||||
| prefer_gpu: true | |||||
| max_refinement_jobs: 16 | |||||
| max_recording_streams: 40 | |||||
| max_decode_jobs: 16 | |||||
| decision_hold_ms: 3000 | |||||
| - name: digital-hunting | |||||
| description: Digital-first refinement and decode focus | |||||
| pipeline: | |||||
| mode: digital-hunting | |||||
| profile: digital-hunting | |||||
| goals: | |||||
| intent: digital-surveillance | |||||
| monitor_start_hz: 0 | |||||
| monitor_end_hz: 0 | |||||
| monitor_span_hz: 0 | |||||
| monitor_windows: [] | |||||
| signal_priorities: | |||||
| - rds | |||||
| - digital | |||||
| - wfm | |||||
| auto_record_classes: [] | |||||
| auto_decode_classes: [] | |||||
| surveillance: | |||||
| analysis_fft_size: 4096 | |||||
| frame_rate: 12 | |||||
| strategy: multi-resolution | |||||
| display_bins: 2048 | |||||
| display_fps: 12 | |||||
| derived_detection: auto | |||||
| refinement: | |||||
| enabled: true | |||||
| max_concurrent: 20 | |||||
| detail_fft_size: 4096 | |||||
| min_candidate_snr_db: -2 | |||||
| min_span_hz: 50000 | |||||
| max_span_hz: 200000 | |||||
| auto_span: true | |||||
| resources: | |||||
| prefer_gpu: true | |||||
| max_refinement_jobs: 20 | |||||
| max_recording_streams: 20 | |||||
| max_decode_jobs: 24 | |||||
| decision_hold_ms: 2500 | |||||
| detector: | |||||
| threshold_db: -60 | |||||
| min_duration_ms: 500 | |||||
| hold_ms: 1500 | |||||
| ema_alpha: 0.025 | |||||
| hysteresis_db: 10 | |||||
| min_stable_frames: 4 | |||||
| gap_tolerance_ms: 2000 | |||||
| cfar_mode: GOSCA | |||||
| cfar_guard_hz: 200000 | |||||
| cfar_train_hz: 100000 | |||||
| cfar_guard_cells: 3 | |||||
| cfar_train_cells: 24 | |||||
| cfar_rank: 36 | |||||
| cfar_scale_db: 23 | |||||
| cfar_wrap_around: true | |||||
| edge_margin_db: 6 | |||||
| max_signal_bw_hz: 260000 | |||||
| merge_gap_hz: 20000 | |||||
| class_history_size: 10 | |||||
| class_switch_ratio: 0.6 | |||||
| recorder: | |||||
| enabled: false | |||||
| min_snr_db: 0 | |||||
| min_duration: 500ms | |||||
| max_duration: 300s | |||||
| preroll_ms: 500 | |||||
| record_iq: false | |||||
| record_audio: true | |||||
| auto_demod: true | |||||
| auto_decode: false | |||||
| max_disk_mb: 0 | |||||
| output_dir: data/recordings | |||||
| class_filter: [] | |||||
| ring_seconds: 12 | |||||
| deemphasis_us: 50 | |||||
| extraction_fir_taps: 101 | |||||
| extraction_bw_mult: 1.35 | |||||
| debug_live_audio: false | |||||
| decoder: | |||||
| ft8_cmd: C:/WSJT/wsjtx-2.7.0-rc6/bin/jt9.exe -8 {audio} | |||||
| wspr_cmd: C:/WSJT/wsjtx-2.7.0-rc6/bin/wsprd.exe {audio} | |||||
| dmr_cmd: tools/dsd-neo/bin/dsd-neo.exe -fs -i {audio} -s {sr} -o null | |||||
| dstar_cmd: tools/dsd-neo/bin/dsd-neo.exe -fd -i {audio} -s {sr} -o null | |||||
| fsk_cmd: tools/fsk/fsk_decoder --iq {iq} --sample-rate {sr} | |||||
| psk_cmd: tools/psk/psk_decoder --iq {iq} --sample-rate {sr} | |||||
| debug: | |||||
| audio_dump_enabled: false | |||||
| cpu_monitoring: false | |||||
| telemetry: | |||||
| enabled: true | |||||
| heavy_enabled: false | |||||
| heavy_sample_every: 12 | |||||
| metric_sample_every: 8 | |||||
| metric_history_max: 6000 | |||||
| event_history_max: 1500 | |||||
| retention_seconds: 900 | |||||
| persist_enabled: false | |||||
| persist_dir: debug/telemetry | |||||
| rotate_mb: 16 | |||||
| keep_files: 8 | |||||
| logging: | |||||
| level: error | |||||
| categories: [] | |||||
| rate_limit_ms: 1000 | |||||
| stdout: true | |||||
| stdout_color: true | |||||
| file: logs/trace.log | |||||
| file_level: error | |||||
| time_format: "15:04:05" | |||||
| disable_time: false | |||||
| web_addr: :8080 | |||||
| event_path: data/events.jsonl | |||||
| frame_rate: 12 | |||||
| waterfall_lines: 200 | |||||
| web_root: web | |||||
| @@ -248,14 +248,29 @@ decoder: | |||||
| dstar_cmd: tools/dsd-neo/bin/dsd-neo.exe -fd -i {audio} -s {sr} -o null | dstar_cmd: tools/dsd-neo/bin/dsd-neo.exe -fd -i {audio} -s {sr} -o null | ||||
| fsk_cmd: tools/fsk/fsk_decoder --iq {iq} --sample-rate {sr} | fsk_cmd: tools/fsk/fsk_decoder --iq {iq} --sample-rate {sr} | ||||
| psk_cmd: tools/psk/psk_decoder --iq {iq} --sample-rate {sr} | psk_cmd: tools/psk/psk_decoder --iq {iq} --sample-rate {sr} | ||||
| debug: | |||||
| audio_dump_enabled: false | |||||
| cpu_monitoring: false | |||||
| telemetry: | |||||
| enabled: true | |||||
| heavy_enabled: false | |||||
| heavy_sample_every: 12 | |||||
| metric_sample_every: 8 | |||||
| metric_history_max: 6000 | |||||
| event_history_max: 1500 | |||||
| retention_seconds: 900 | |||||
| persist_enabled: true | |||||
| persist_dir: debug/telemetry | |||||
| rotate_mb: 16 | |||||
| keep_files: 8 | |||||
| logging: | logging: | ||||
| level: debug | |||||
| categories: [capture, extract, demod, resample, drop, ws, boundary] | |||||
| rate_limit_ms: 500 | |||||
| level: error | |||||
| categories: [] | |||||
| rate_limit_ms: 1000 | |||||
| stdout: true | stdout: true | ||||
| stdout_color: true | stdout_color: true | ||||
| file: logs/trace.log | |||||
| file_level: debug | |||||
| file: "" | |||||
| file_level: error | |||||
| time_format: "15:04:05" | time_format: "15:04:05" | ||||
| disable_time: false | disable_time: false | ||||
| web_addr: :8080 | web_addr: :8080 | ||||
| @@ -0,0 +1,48 @@ | |||||
| # GPU Streaming Refactor Plan (2026-03-25) | |||||
| ## Goal | |||||
| Replace the current overlap+trim GPU extractor model with a true stateful per-signal streaming architecture, and build a corrected CPU oracle/reference path for validation. | |||||
| ## Non-negotiables | |||||
| - No production start-index-only patch. | |||||
| - No production overlap-prepend + trim continuity model. | |||||
| - Exact integer decimation only in the new streaming production path. | |||||
| - Persistent per-signal state must include NCO phase, FIR history, and decimator phase/residue. | |||||
| - GPU validation must compare against a corrected CPU oracle, not the legacy CPU fallback. | |||||
| ## Work order | |||||
| 1. Introduce explicit stateful streaming types in `gpudemod`. | |||||
| 2. Add a clean CPU oracle implementation and monolithic-vs-chunked tests. | |||||
| 3. Add per-signal state ownership in batch runner. | |||||
| 4. Implement new streaming extractor semantics in Go using NEW IQ samples only. | |||||
| 5. Replace legacy GPU-path assumptions (rounding decimation, overlap-prepend, trim-defined validity) in the new path. | |||||
| 6. Add production telemetry that proves state continuity (`phase_count`, `history_len`, `n_out`, reference error). | |||||
| 7. Keep legacy path isolated only for temporary comparison if needed. | |||||
| ## Initial files in scope | |||||
| - `internal/demod/gpudemod/batch.go` | |||||
| - `internal/demod/gpudemod/batch_runner.go` | |||||
| - `internal/demod/gpudemod/batch_runner_windows.go` | |||||
| - `internal/demod/gpudemod/kernels.cu` | |||||
| - `internal/demod/gpudemod/native/exports.cu` | |||||
| - `cmd/sdrd/helpers.go` | |||||
| ## Immediate implementation strategy | |||||
| ### Phase 1 | |||||
| - Create explicit streaming state structs in Go. | |||||
| - Add CPU oracle/reference path with exact semantics and tests. | |||||
| - Introduce exact integer-decimation checks. | |||||
| ### Phase 2 | |||||
| - Rework batch runner to own persistent per-signal state. | |||||
| - Add config-hash-based resets. | |||||
| - Stop modeling continuity via overlap tail in the new path. | |||||
| ### Phase 3 | |||||
| - Introduce a real streaming GPU entry path that consumes NEW shifted samples plus carried state. | |||||
| - Move to a stateful polyphase decimator model. | |||||
| ## Validation expectations | |||||
| - CPU oracle monolithic == CPU oracle chunked within tolerance. | |||||
| - GPU streaming output == CPU oracle chunked within tolerance. | |||||
| - Former periodic block-boundary clicks gone in real-world testing. | |||||
| @@ -0,0 +1,196 @@ | |||||
| # Known Issues | |||||
| This file tracks durable open engineering issues that remain after the 2026-03-25 audio-click fix. | |||||
| Primary source: | |||||
| - `docs/open-issues-report-2026-03-25.json` | |||||
| Status values used here: | |||||
| - `open` | |||||
| - `deferred` | |||||
| - `info` | |||||
| --- | |||||
| ## High Priority | |||||
| ### OI-02 — `lastDiscrimIQ` missing from `dspStateSnapshot` | |||||
| - Status: `open` | |||||
| - Severity: High | |||||
| - Category: state-continuity | |||||
| - File: `internal/recorder/streamer.go` | |||||
| - Summary: FM discriminator bridging state is not preserved across `captureDSPState()` / `restoreDSPState()`, so recording segment splits can lose the final IQ sample and create a micro-click at the segment boundary. | |||||
| - Recommended fix: add `lastDiscrimIQ` and `lastDiscrimIQSet` to `dspStateSnapshot`. | |||||
| - Source: `docs/open-issues-report-2026-03-25.json` (OI-02) | |||||
| ### OI-03 — CPU oracle path not yet usable as validation baseline | |||||
| - Status: `open` | |||||
| - Severity: High | |||||
| - Category: architecture | |||||
| - File: `cmd/sdrd/streaming_refactor.go`, `internal/demod/gpudemod/cpu_oracle.go` | |||||
| - Summary: the CPU oracle exists, but the production comparison/integration path is not trusted yet. That means GPU-path regressions still cannot be checked automatically with confidence. | |||||
| - Recommended fix: repair oracle integration and restore GPU-vs-CPU validation flow. | |||||
| - Source: `docs/open-issues-report-2026-03-25.json` (OI-03) | |||||
| ### OI-18 — planned C2-C validation gate never completed | |||||
| - Status: `open` | |||||
| - Severity: Info | |||||
| - Category: architecture | |||||
| - File: `docs/audio-click-debug-notes-2026-03-24.md` | |||||
| - Summary: the final native streaming path works in practice, but the planned formal GPU-vs-oracle validation gate was never completed. | |||||
| - Recommended fix: complete this together with OI-03. | |||||
| - Source: `docs/open-issues-report-2026-03-25.json` (OI-18) | |||||
| --- | |||||
| ## Medium Priority | |||||
| ### OI-14 — no regression test for `allIQ` immutability through spectrum/detection pipeline | |||||
| - Status: `open` | |||||
| - Severity: Low | |||||
| - Category: test-coverage | |||||
| - File: `cmd/sdrd/pipeline_runtime.go` | |||||
| - Summary: the `IQBalance` aliasing bug showed that shared-buffer mutation can slip in undetected. There is still no test asserting that `allIQ` remains unchanged after capture/detection-side processing. | |||||
| - Recommended fix: add an integration test that compares `allIQ` before and after the relevant pipeline stage. | |||||
| - Source: `docs/open-issues-report-2026-03-25.json` (OI-14) | |||||
| ### OI-15 — very low test coverage for `processSnippet` audio pipeline | |||||
| - Status: `open` | |||||
| - Severity: Low | |||||
| - Category: test-coverage | |||||
| - File: `internal/recorder/streamer.go` | |||||
| - Summary: the main live audio pipeline still lacks focused tests for boundary continuity, WFM mono/stereo behavior, resampling, and demod-path regressions. | |||||
| - Recommended fix: add synthetic fixtures and continuity-oriented tests around repeated `processSnippet` calls. | |||||
| - Source: `docs/open-issues-report-2026-03-25.json` (OI-15) | |||||
| ### OI-07 — taps are recalculated every frame | |||||
| - Status: `open` | |||||
| - Severity: Medium | |||||
| - Category: correctness | |||||
| - File: `internal/demod/gpudemod/stream_state.go` | |||||
| - Summary: FIR/polyphase taps are recomputed every frame even when parameters do not change, which is unnecessary work and makes it easier for host/GPU tap state to drift apart. | |||||
| - Recommended fix: only rebuild taps when tap-relevant inputs actually change. | |||||
| - Source: `docs/open-issues-report-2026-03-25.json` (OI-07) | |||||
| ### OI-17 — bandwidth changes can change Go-side taps without GPU tap re-upload | |||||
| - Status: `open` | |||||
| - Severity: Low-Medium | |||||
| - Category: correctness | |||||
| - File: `internal/demod/gpudemod/streaming_gpu_native_prepare.go`, `internal/demod/gpudemod/stream_state.go` | |||||
| - Summary: after the config-hash fix, a bandwidth change may rebuild taps on the Go side while the GPU still keeps older uploaded taps unless a reset happens. | |||||
| - Recommended fix: add a separate tap-change detection/re-upload path without forcing full extractor reset. | |||||
| - Source: `docs/open-issues-report-2026-03-25.json` (OI-17) | |||||
| ### OI-09 — streaming feature flags are compile-time constants | |||||
| - Status: `open` | |||||
| - Severity: Medium | |||||
| - Category: architecture | |||||
| - File: `cmd/sdrd/streaming_refactor.go`, `internal/demod/gpudemod/streaming_gpu_modes.go` | |||||
| - Summary: switching between production/oracle/native-host modes still requires code changes and rebuilds, which makes field debugging and A/B validation harder than necessary. | |||||
| - Recommended fix: expose these as config or environment-driven switches. | |||||
| - Source: `docs/open-issues-report-2026-03-25.json` (OI-09) | |||||
| ### OI-05 — feed channel is shallow and can drop frames under pressure | |||||
| - Status: `open` | |||||
| - Severity: Medium | |||||
| - Category: reliability | |||||
| - File: `internal/recorder/streamer.go` | |||||
| - Summary: `feedCh` has a buffer of only 2. Under heavier processing or debug load, dropped feed messages can create audible gaps. | |||||
| - Recommended fix: increase channel depth or redesign backpressure behavior. | |||||
| - Source: `docs/open-issues-report-2026-03-25.json` (OI-05) | |||||
| ### OI-06 — legacy overlap/trim extractor path is now mostly legacy baggage | |||||
| - Status: `deferred` | |||||
| - Severity: Medium | |||||
| - Category: dead-code | |||||
| - File: `cmd/sdrd/helpers.go` | |||||
| - Summary: the old overlap/trim path is now mainly fallback/legacy code and adds complexity plus old instrumentation noise. | |||||
| - Recommended fix: isolate, simplify, or remove it once the production path and fallback strategy are formally settled. | |||||
| - Source: `docs/open-issues-report-2026-03-25.json` (OI-06) | |||||
| ### OI-04 — telemetry history storage still uses append+copy trim | |||||
| - Status: `deferred` | |||||
| - Severity: Medium | |||||
| - Category: telemetry | |||||
| - File: `internal/telemetry/telemetry.go` | |||||
| - Summary: heavy telemetry can still create avoidable allocation/copy pressure because history trimming is O(n) and happens under lock. | |||||
| - Recommended fix: replace with a ring-buffer design. | |||||
| - Source: `docs/open-issues-report-2026-03-25.json` (OI-04) | |||||
| --- | |||||
| ## Lower Priority / Nice-to-Have | |||||
| ### OI-01 — `DCBlocker.Apply(allIQ)` still mutates extraction input in-place | |||||
| - Status: `deferred` | |||||
| - Severity: High | |||||
| - Category: data-integrity | |||||
| - File: `cmd/sdrd/pipeline_runtime.go` | |||||
| - Summary: unlike the old `IQBalance` bug this does not create a boundary artifact, but it does mean live extraction and recorded/replayed data are not semantically identical. | |||||
| - Recommended fix: clarify the contract or move to immutable/copy-based handling. | |||||
| - Source: `docs/open-issues-report-2026-03-25.json` (OI-01) | |||||
| ### OI-08 — WFM audio LPF could reject pilot more strongly | |||||
| - Status: `deferred` | |||||
| - Severity: Medium | |||||
| - Category: audio-quality | |||||
| - File: `internal/recorder/streamer.go` | |||||
| - Summary: the current 15 kHz LPF is good enough functionally, but a steeper filter could further improve pilot suppression. | |||||
| - Recommended fix: more taps or a dedicated pilot notch. | |||||
| - Source: `docs/open-issues-report-2026-03-25.json` (OI-08) | |||||
| ### OI-10 — `demod.wav` debug dumps can clip and mislead analysis | |||||
| - Status: `deferred` | |||||
| - Severity: Medium | |||||
| - Category: correctness | |||||
| - File: `internal/recorder/streamer.go`, `internal/recorder/wavwriter.go` | |||||
| - Summary: raw discriminator output can exceed the WAV writer's `[-1,+1]` clip range, so debug dumps can show artifacts that are not part of the real downstream audio path. | |||||
| - Recommended fix: scale by `1/pi` before dumping or use float WAV output. | |||||
| - Source: `docs/open-issues-report-2026-03-25.json` (OI-10) | |||||
| ### OI-11 — browser AudioContext resync still causes audible micro-gaps | |||||
| - Status: `deferred` | |||||
| - Severity: Low | |||||
| - Category: reliability | |||||
| - File: `web/app.js` | |||||
| - Summary: underrun recovery is softened with a fade-in, but repeated resyncs still create audible stutter on the browser side. | |||||
| - Recommended fix: prefer the AudioWorklet/ring-player path wherever possible. | |||||
| - Source: `docs/open-issues-report-2026-03-25.json` (OI-11) | |||||
| ### OI-12 — tiny per-frame tail copy for boundary telemetry | |||||
| - Status: `info` | |||||
| - Severity: Low | |||||
| - Category: performance | |||||
| - File: `cmd/sdrd/pipeline_runtime.go` | |||||
| - Summary: the last-32-sample copy is trivial and not urgent, but it is one more small allocation in a path that already has several. | |||||
| - Recommended fix: none needed unless a broader allocation cleanup happens. | |||||
| - Source: `docs/open-issues-report-2026-03-25.json` (OI-12) | |||||
| ### OI-13 — temporary patch artifacts should not live in the repo long-term | |||||
| - Status: `deferred` | |||||
| - Severity: Low | |||||
| - Category: dead-code | |||||
| - File: `patches/*` | |||||
| - Summary: reviewer/debug patch artifacts were useful during the investigation, but they should either be removed or archived under docs rather than kept as loose patch files. | |||||
| - Recommended fix: delete or archive them once no longer needed. | |||||
| - Source: `docs/open-issues-report-2026-03-25.json` (OI-13) | |||||
| ### OI-16 — `config.autosave.yaml` can re-enable unwanted debug telemetry after restart | |||||
| - Status: `deferred` | |||||
| - Severity: Low | |||||
| - Category: config | |||||
| - File: `config.autosave.yaml` | |||||
| - Summary: autosave can silently restore debug-heavy telemetry settings after restart and distort future runs. | |||||
| - Recommended fix: stop persisting debug telemetry knobs to autosave or explicitly ignore them. | |||||
| - Source: `docs/open-issues-report-2026-03-25.json` (OI-16) | |||||
| --- | |||||
| ## Suggested next execution order | |||||
| 1. Fix OI-02 (`lastDiscrimIQ` snapshot/restore) | |||||
| 2. Repair OI-03 and close OI-18 (oracle + formal validation path) | |||||
| 3. Add OI-14 and OI-15 regression tests | |||||
| 4. Consolidate OI-07 and OI-17 (tap rebuild / tap upload logic) | |||||
| 5. Expose OI-09 feature flags via config or env | |||||
| 6. Revisit OI-05 / OI-06 / OI-04 when doing reliability/cleanup work | |||||
| @@ -0,0 +1,711 @@ | |||||
| # Telemetry API Reference | |||||
| This document describes the server-side telemetry collector, its runtime configuration, and the HTTP API exposed by `sdrd`. | |||||
| The telemetry system is intended for debugging and performance analysis of the SDR pipeline, especially around source cadence, extraction, DSP timing, boundary artifacts, queue pressure, and other runtime anomalies. | |||||
| ## Goals | |||||
| The telemetry layer gives you three different views of runtime state: | |||||
| 1. **Live snapshot** | |||||
| - Current counters, gauges, distributions, recent events, and collector status. | |||||
| 2. **Historical metrics** | |||||
| - Timestamped metric samples that can be filtered by name, prefix, or tags. | |||||
| 3. **Historical events** | |||||
| - Structured anomalies / warnings / debug events with optional fields. | |||||
| It is designed to be lightweight in normal operation and more detailed when `heavy_enabled` is turned on. | |||||
| --- | |||||
| ## Base URLs | |||||
| All telemetry endpoints live under: | |||||
| - `/api/debug/telemetry/live` | |||||
| - `/api/debug/telemetry/history` | |||||
| - `/api/debug/telemetry/events` | |||||
| - `/api/debug/telemetry/config` | |||||
| Responses are JSON. | |||||
| --- | |||||
| ## Data model | |||||
| ### Metric types | |||||
| Telemetry metrics are stored in three logical groups: | |||||
| - **counter** | |||||
| - Accumulating values, usually incremented over time. | |||||
| - **gauge** | |||||
| - Latest current value. | |||||
| - **distribution** | |||||
| - Observed numeric samples with summary stats. | |||||
| A historical metric sample is returned as: | |||||
| ```json | |||||
| { | |||||
| "ts": "2026-03-25T12:00:00Z", | |||||
| "name": "stage.extract_stream.duration_ms", | |||||
| "type": "distribution", | |||||
| "value": 4.83, | |||||
| "tags": { | |||||
| "stage": "extract_stream", | |||||
| "signal_id": "1" | |||||
| } | |||||
| } | |||||
| ``` | |||||
| ### Events | |||||
| Telemetry events are structured anomaly/debug records: | |||||
| ```json | |||||
| { | |||||
| "id": 123, | |||||
| "ts": "2026-03-25T12:00:02Z", | |||||
| "name": "demod_boundary", | |||||
| "level": "warn", | |||||
| "message": "boundary discontinuity detected", | |||||
| "tags": { | |||||
| "signal_id": "1", | |||||
| "stage": "demod" | |||||
| }, | |||||
| "fields": { | |||||
| "d2": 0.3358, | |||||
| "index": 25 | |||||
| } | |||||
| } | |||||
| ``` | |||||
| ### Tags | |||||
| Tags are string key/value metadata used for filtering and correlation. | |||||
| Common tag keys already supported by the HTTP layer: | |||||
| - `signal_id` | |||||
| - `session_id` | |||||
| - `stage` | |||||
| - `trace_id` | |||||
| - `component` | |||||
| You can also filter on arbitrary tags via `tag_<key>=<value>` query parameters. | |||||
| --- | |||||
| ## Endpoint: `GET /api/debug/telemetry/live` | |||||
| Returns a live snapshot of the in-memory collector state. | |||||
| ### Response shape | |||||
| ```json | |||||
| { | |||||
| "now": "2026-03-25T12:00:05Z", | |||||
| "started_at": "2026-03-25T11:52:10Z", | |||||
| "uptime_ms": 472500, | |||||
| "config": { | |||||
| "enabled": true, | |||||
| "heavy_enabled": false, | |||||
| "heavy_sample_every": 12, | |||||
| "metric_sample_every": 2, | |||||
| "metric_history_max": 12000, | |||||
| "event_history_max": 4000, | |||||
| "retention": 900000000000, | |||||
| "persist_enabled": false, | |||||
| "persist_dir": "debug/telemetry", | |||||
| "rotate_mb": 16, | |||||
| "keep_files": 8 | |||||
| }, | |||||
| "counters": [ | |||||
| { | |||||
| "name": "source.resets", | |||||
| "value": 1, | |||||
| "tags": { | |||||
| "component": "source" | |||||
| } | |||||
| } | |||||
| ], | |||||
| "gauges": [ | |||||
| { | |||||
| "name": "source.buffer_samples", | |||||
| "value": 304128, | |||||
| "tags": { | |||||
| "component": "source" | |||||
| } | |||||
| } | |||||
| ], | |||||
| "distributions": [ | |||||
| { | |||||
| "name": "dsp.frame.duration_ms", | |||||
| "count": 96, | |||||
| "min": 82.5, | |||||
| "max": 212.4, | |||||
| "mean": 104.8, | |||||
| "last": 98.3, | |||||
| "p95": 149.2, | |||||
| "tags": { | |||||
| "stage": "dsp" | |||||
| } | |||||
| } | |||||
| ], | |||||
| "recent_events": [], | |||||
| "status": { | |||||
| "source_state": "running" | |||||
| } | |||||
| } | |||||
| ``` | |||||
| ### Notes | |||||
| - `counters`, `gauges`, and `distributions` are sorted by metric name. | |||||
| - `recent_events` contains the most recent in-memory event slice. | |||||
| - `status` is optional and contains arbitrary runtime status published by code using `SetStatus(...)`. | |||||
| - If telemetry is unavailable, the server returns a small JSON object instead of a full snapshot. | |||||
| ### Typical uses | |||||
| - Check whether telemetry is enabled. | |||||
| - Look for timing hotspots in `*.duration_ms` distributions. | |||||
| - Inspect current queue or source gauges. | |||||
| - See recent anomaly events without querying history. | |||||
| --- | |||||
| ## Endpoint: `GET /api/debug/telemetry/history` | |||||
| Returns historical metric samples from in-memory history and, optionally, persisted JSONL files. | |||||
| ### Response shape | |||||
| ```json | |||||
| { | |||||
| "items": [ | |||||
| { | |||||
| "ts": "2026-03-25T12:00:01Z", | |||||
| "name": "stage.extract_stream.duration_ms", | |||||
| "type": "distribution", | |||||
| "value": 5.2, | |||||
| "tags": { | |||||
| "stage": "extract_stream", | |||||
| "signal_id": "2" | |||||
| } | |||||
| } | |||||
| ], | |||||
| "count": 1 | |||||
| } | |||||
| ``` | |||||
| ### Supported query parameters | |||||
| #### Time filters | |||||
| - `since` | |||||
| - `until` | |||||
| Accepted formats: | |||||
| - Unix seconds | |||||
| - Unix milliseconds | |||||
| - RFC3339 | |||||
| - RFC3339Nano | |||||
| Examples: | |||||
| - `?since=1711368000` | |||||
| - `?since=1711368000123` | |||||
| - `?since=2026-03-25T12:00:00Z` | |||||
| #### Result shaping | |||||
| - `limit` | |||||
| - Default normalization is 500. | |||||
| - Values above 5000 are clamped down by the collector query layer. | |||||
| #### Name filters | |||||
| - `name=<exact_metric_name>` | |||||
| - `prefix=<metric_name_prefix>` | |||||
| Examples: | |||||
| - `?name=source.read.duration_ms` | |||||
| - `?prefix=stage.` | |||||
| - `?prefix=iq.extract.` | |||||
| #### Tag filters | |||||
| Special convenience query params map directly to tag filters: | |||||
| - `signal_id` | |||||
| - `session_id` | |||||
| - `stage` | |||||
| - `trace_id` | |||||
| - `component` | |||||
| Arbitrary tag filters: | |||||
| - `tag_<key>=<value>` | |||||
| Examples: | |||||
| - `?signal_id=1` | |||||
| - `?stage=extract_stream` | |||||
| - `?tag_path=gpu` | |||||
| - `?tag_zone=broadcast` | |||||
| #### Persistence control | |||||
| - `include_persisted=true|false` | |||||
| - Default: `true` | |||||
| When enabled and persistence is active, the server reads matching data from rotated JSONL telemetry files in addition to in-memory history. | |||||
| ### Notes | |||||
| - Results are sorted by timestamp ascending. | |||||
| - If `limit` is hit, the most recent matching items are retained. | |||||
| - Exact retention depends on both in-memory retention and persisted file availability. | |||||
| - A small set of boundary-related IQ metrics is force-stored regardless of the normal metric sample cadence. | |||||
| ### Typical queries | |||||
| Get all stage timing since a specific start: | |||||
| ```text | |||||
| /api/debug/telemetry/history?since=2026-03-25T12:00:00Z&prefix=stage. | |||||
| ``` | |||||
| Get extraction metrics for a single signal: | |||||
| ```text | |||||
| /api/debug/telemetry/history?since=2026-03-25T12:00:00Z&prefix=extract.&signal_id=2 | |||||
| ``` | |||||
| Get source cadence metrics only from in-memory history: | |||||
| ```text | |||||
| /api/debug/telemetry/history?prefix=source.&include_persisted=false | |||||
| ``` | |||||
| --- | |||||
| ## Endpoint: `GET /api/debug/telemetry/events` | |||||
| Returns historical telemetry events from memory and, optionally, persisted storage. | |||||
| ### Response shape | |||||
| ```json | |||||
| { | |||||
| "items": [ | |||||
| { | |||||
| "id": 991, | |||||
| "ts": "2026-03-25T12:00:03Z", | |||||
| "name": "source_reset", | |||||
| "level": "warn", | |||||
| "message": "source reader reset observed", | |||||
| "tags": { | |||||
| "component": "source" | |||||
| }, | |||||
| "fields": { | |||||
| "reason": "short_read" | |||||
| } | |||||
| } | |||||
| ], | |||||
| "count": 1 | |||||
| } | |||||
| ``` | |||||
| ### Supported query parameters | |||||
| All `history` filters are also supported here, plus: | |||||
| - `level=<debug|info|warn|error|...>` | |||||
| Examples: | |||||
| - `?since=2026-03-25T12:00:00Z&level=warn` | |||||
| - `?prefix=audio.&signal_id=1` | |||||
| - `?name=demod_boundary&signal_id=1` | |||||
| ### Notes | |||||
| - Event matching supports `name`, `prefix`, `level`, time range, and tags. | |||||
| - Event `level` matching is case-insensitive. | |||||
| - Results are timestamp-sorted ascending. | |||||
| ### Typical queries | |||||
| Get warnings during a reproduction run: | |||||
| ```text | |||||
| /api/debug/telemetry/events?since=2026-03-25T12:00:00Z&level=warn | |||||
| ``` | |||||
| Get boundary-related events for one signal: | |||||
| ```text | |||||
| /api/debug/telemetry/events?since=2026-03-25T12:00:00Z&signal_id=1&prefix=demod_ | |||||
| ``` | |||||
| --- | |||||
| ## Endpoint: `GET /api/debug/telemetry/config` | |||||
| Returns both: | |||||
| 1. the active collector configuration, and | |||||
| 2. the current runtime config under `debug.telemetry` | |||||
| ### Response shape | |||||
| ```json | |||||
| { | |||||
| "collector": { | |||||
| "enabled": true, | |||||
| "heavy_enabled": false, | |||||
| "heavy_sample_every": 12, | |||||
| "metric_sample_every": 2, | |||||
| "metric_history_max": 12000, | |||||
| "event_history_max": 4000, | |||||
| "retention": 900000000000, | |||||
| "persist_enabled": false, | |||||
| "persist_dir": "debug/telemetry", | |||||
| "rotate_mb": 16, | |||||
| "keep_files": 8 | |||||
| }, | |||||
| "config": { | |||||
| "enabled": true, | |||||
| "heavy_enabled": false, | |||||
| "heavy_sample_every": 12, | |||||
| "metric_sample_every": 2, | |||||
| "metric_history_max": 12000, | |||||
| "event_history_max": 4000, | |||||
| "retention_seconds": 900, | |||||
| "persist_enabled": false, | |||||
| "persist_dir": "debug/telemetry", | |||||
| "rotate_mb": 16, | |||||
| "keep_files": 8 | |||||
| } | |||||
| } | |||||
| ``` | |||||
| ### Important distinction | |||||
| - `collector.retention` is a Go duration serialized in nanoseconds. | |||||
| - `config.retention_seconds` is the config-facing field used by YAML and the POST update API. | |||||
| If you are writing tooling, prefer `config.retention_seconds` for human-facing config edits. | |||||
| --- | |||||
| ## Endpoint: `POST /api/debug/telemetry/config` | |||||
| Updates telemetry settings at runtime and writes them back via the autosave config path. | |||||
| ### Request body | |||||
| All fields are optional. Only provided fields are changed. | |||||
| ```json | |||||
| { | |||||
| "enabled": true, | |||||
| "heavy_enabled": true, | |||||
| "heavy_sample_every": 8, | |||||
| "metric_sample_every": 1, | |||||
| "metric_history_max": 20000, | |||||
| "event_history_max": 6000, | |||||
| "retention_seconds": 1800, | |||||
| "persist_enabled": true, | |||||
| "persist_dir": "debug/telemetry", | |||||
| "rotate_mb": 32, | |||||
| "keep_files": 12 | |||||
| } | |||||
| ``` | |||||
| ### Response shape | |||||
| ```json | |||||
| { | |||||
| "ok": true, | |||||
| "collector": { | |||||
| "enabled": true, | |||||
| "heavy_enabled": true, | |||||
| "heavy_sample_every": 8, | |||||
| "metric_sample_every": 1, | |||||
| "metric_history_max": 20000, | |||||
| "event_history_max": 6000, | |||||
| "retention": 1800000000000, | |||||
| "persist_enabled": true, | |||||
| "persist_dir": "debug/telemetry", | |||||
| "rotate_mb": 32, | |||||
| "keep_files": 12 | |||||
| }, | |||||
| "config": { | |||||
| "enabled": true, | |||||
| "heavy_enabled": true, | |||||
| "heavy_sample_every": 8, | |||||
| "metric_sample_every": 1, | |||||
| "metric_history_max": 20000, | |||||
| "event_history_max": 6000, | |||||
| "retention_seconds": 1800, | |||||
| "persist_enabled": true, | |||||
| "persist_dir": "debug/telemetry", | |||||
| "rotate_mb": 32, | |||||
| "keep_files": 12 | |||||
| } | |||||
| } | |||||
| ``` | |||||
| ### Persistence behavior | |||||
| A POST updates: | |||||
| - the runtime manager snapshot/config | |||||
| - the in-process collector config | |||||
| - the autosave config file via `config.Save(...)` | |||||
| That means these updates are runtime-effective immediately and also survive restarts through autosave, unless manually reverted. | |||||
| ### Error cases | |||||
| - Invalid JSON -> `400 Bad Request` | |||||
| - Invalid collector reconfiguration -> `400 Bad Request` | |||||
| - Telemetry unavailable -> `503 Service Unavailable` | |||||
| --- | |||||
| ## Configuration fields (`debug.telemetry`) | |||||
| Telemetry config lives under: | |||||
| ```yaml | |||||
| debug: | |||||
| telemetry: | |||||
| enabled: true | |||||
| heavy_enabled: false | |||||
| heavy_sample_every: 12 | |||||
| metric_sample_every: 2 | |||||
| metric_history_max: 12000 | |||||
| event_history_max: 4000 | |||||
| retention_seconds: 900 | |||||
| persist_enabled: false | |||||
| persist_dir: debug/telemetry | |||||
| rotate_mb: 16 | |||||
| keep_files: 8 | |||||
| ``` | |||||
| ### Field reference | |||||
| #### `enabled` | |||||
| Master on/off switch for telemetry collection. | |||||
| If false: | |||||
| - metrics are not recorded | |||||
| - events are not recorded | |||||
| - live snapshot remains effectively empty/minimal | |||||
| #### `heavy_enabled` | |||||
| Enables more expensive / more detailed telemetry paths that should not be left on permanently unless needed. | |||||
| Use this for deep extractor/IQ/boundary debugging. | |||||
| #### `heavy_sample_every` | |||||
| Sampling cadence for heavy telemetry. | |||||
| - `1` means every eligible heavy sample | |||||
| - higher numbers reduce cost by sampling less often | |||||
| #### `metric_sample_every` | |||||
| Sampling cadence for normal historical metric point storage. | |||||
| Collector summaries still update live, but historical storage becomes less dense when this value is greater than 1. | |||||
| #### `metric_history_max` | |||||
| Maximum number of in-memory historical metric samples retained. | |||||
| #### `event_history_max` | |||||
| Maximum number of in-memory telemetry events retained. | |||||
| #### `retention_seconds` | |||||
| Time-based in-memory retention window. | |||||
| Older in-memory metrics/events are trimmed once they fall outside this retention period. | |||||
| #### `persist_enabled` | |||||
| When enabled, telemetry metrics/events are also appended to rotated JSONL files. | |||||
| #### `persist_dir` | |||||
| Directory where rotated telemetry JSONL files are written. | |||||
| Default: | |||||
| - `debug/telemetry` | |||||
| #### `rotate_mb` | |||||
| Approximate JSONL file rotation threshold in megabytes. | |||||
| #### `keep_files` | |||||
| How many rotated telemetry files to retain in `persist_dir`. | |||||
| Older files beyond this count are pruned. | |||||
| --- | |||||
| ## Collector behavior and caveats | |||||
| ### In-memory vs persisted data | |||||
| The query endpoints can read from both: | |||||
| - current in-memory collector state/history | |||||
| - persisted JSONL files | |||||
| This means a request may return data older than current in-memory retention if: | |||||
| - `persist_enabled=true`, and | |||||
| - `include_persisted=true` | |||||
| ### Sampling behavior | |||||
| Not every observation necessarily becomes a historical metric point. | |||||
| The collector: | |||||
| - always updates live counters/gauges/distributions while enabled | |||||
| - stores historical points according to `metric_sample_every` | |||||
| - force-stores selected boundary IQ metrics even when sampling would normally skip them | |||||
| So the live snapshot and historical series density are intentionally different. | |||||
| ### Distribution summaries | |||||
| Distribution values in the live snapshot include: | |||||
| - `count` | |||||
| - `min` | |||||
| - `max` | |||||
| - `mean` | |||||
| - `last` | |||||
| - `p95` | |||||
| The p95 estimate is based on the collector's bounded rolling sample buffer, not an unbounded full-history quantile computation. | |||||
| ### Config serialization detail | |||||
| The collector's `retention` field is a Go duration. In JSON this appears as an integer nanosecond count. | |||||
| This is expected. | |||||
| --- | |||||
| ## Recommended workflows | |||||
| ### Fast low-overhead runtime watch | |||||
| Use: | |||||
| - `enabled=true` | |||||
| - `heavy_enabled=false` | |||||
| - `persist_enabled=false` or `true` if you want an archive | |||||
| Then query: | |||||
| - `/api/debug/telemetry/live` | |||||
| - `/api/debug/telemetry/history?prefix=stage.` | |||||
| - `/api/debug/telemetry/events?level=warn` | |||||
| ### 5-10 minute anomaly capture | |||||
| Suggested settings: | |||||
| - `enabled=true` | |||||
| - `heavy_enabled=false` | |||||
| - `persist_enabled=true` | |||||
| - moderate `metric_sample_every` | |||||
| Then: | |||||
| 1. note start time | |||||
| 2. reproduce workload | |||||
| 3. fetch live snapshot | |||||
| 4. inspect warning events | |||||
| 5. inspect `stage.*`, `streamer.*`, and `source.*` history | |||||
| ### Deep extractor / boundary investigation | |||||
| Temporarily enable: | |||||
| - `heavy_enabled=true` | |||||
| - `heavy_sample_every` > 1 unless you really need every sample | |||||
| - `persist_enabled=true` | |||||
| Then inspect: | |||||
| - `iq.*` | |||||
| - `extract.*` | |||||
| - `audio.*` | |||||
| - boundary/anomaly events for specific `signal_id` or `session_id` | |||||
| Turn heavy telemetry back off once done. | |||||
| --- | |||||
| ## Example requests | |||||
| ### Fetch live snapshot | |||||
| ```bash | |||||
| curl http://localhost:8080/api/debug/telemetry/live | |||||
| ``` | |||||
| ### Fetch stage timings from the last 10 minutes | |||||
| ```bash | |||||
| curl "http://localhost:8080/api/debug/telemetry/history?since=2026-03-25T12:00:00Z&prefix=stage." | |||||
| ``` | |||||
| ### Fetch source metrics for one signal | |||||
| ```bash | |||||
| curl "http://localhost:8080/api/debug/telemetry/history?prefix=source.&signal_id=1" | |||||
| ``` | |||||
| ### Fetch warning events only | |||||
| ```bash | |||||
| curl "http://localhost:8080/api/debug/telemetry/events?since=2026-03-25T12:00:00Z&level=warn" | |||||
| ``` | |||||
| ### Fetch events with a custom tag filter | |||||
| ```bash | |||||
| curl "http://localhost:8080/api/debug/telemetry/events?tag_zone=broadcast" | |||||
| ``` | |||||
| ### Enable persistence and heavy telemetry temporarily | |||||
| ```bash | |||||
| curl -X POST http://localhost:8080/api/debug/telemetry/config \ | |||||
| -H "Content-Type: application/json" \ | |||||
| -d '{ | |||||
| "heavy_enabled": true, | |||||
| "heavy_sample_every": 8, | |||||
| "persist_enabled": true | |||||
| }' | |||||
| ``` | |||||
| --- | |||||
| ## Related docs | |||||
| - `README.md` - high-level project overview and endpoint summary | |||||
| - `docs/telemetry-debug-runbook.md` - quick operational runbook for short debug sessions | |||||
| - `internal/telemetry/telemetry.go` - collector implementation details | |||||
| - `cmd/sdrd/http_handlers.go` - HTTP wiring for telemetry endpoints | |||||
| @@ -0,0 +1,100 @@ | |||||
| # Debug Telemetry Runbook | |||||
| This project now includes structured server-side telemetry for the audio/DSP pipeline. | |||||
| ## Endpoints | |||||
| - `GET /api/debug/telemetry/live` | |||||
| - Current counters/gauges/distributions and recent events. | |||||
| - `GET /api/debug/telemetry/history` | |||||
| - Historical metric samples. | |||||
| - Query params: | |||||
| - `since`, `until`: unix seconds/ms or RFC3339 | |||||
| - `limit` | |||||
| - `name`, `prefix` | |||||
| - `signal_id`, `session_id`, `stage`, `trace_id`, `component` | |||||
| - `tag_<key>=<value>` for arbitrary tag filters | |||||
| - `include_persisted=true|false` | |||||
| - `GET /api/debug/telemetry/events` | |||||
| - Historical events/anomalies. | |||||
| - Same filters as history plus `level`. | |||||
| - `GET /api/debug/telemetry/config` | |||||
| - Active telemetry config from runtime + collector. | |||||
| - `POST /api/debug/telemetry/config` | |||||
| - Runtime config update (also saved to autosave config). | |||||
| ## Config knobs | |||||
| `debug.telemetry` in config: | |||||
| - `enabled` | |||||
| - `heavy_enabled` | |||||
| - `heavy_sample_every` | |||||
| - `metric_sample_every` | |||||
| - `metric_history_max` | |||||
| - `event_history_max` | |||||
| - `retention_seconds` | |||||
| - `persist_enabled` | |||||
| - `persist_dir` | |||||
| - `rotate_mb` | |||||
| - `keep_files` | |||||
| Persisted JSONL files rotate in `persist_dir` (default: `debug/telemetry`). | |||||
| ## 5-10 minute debug flow | |||||
| 1. Keep `enabled=true`, `heavy_enabled=false`, `persist_enabled=true`. | |||||
| 2. Run workload for 5-10 minutes. | |||||
| 3. Pull live state: | |||||
| - `GET /api/debug/telemetry/live` | |||||
| 4. Pull anomalies: | |||||
| - `GET /api/debug/telemetry/events?since=<start>&level=warn` | |||||
| 5. Pull pipeline timing and queue/backpressure: | |||||
| - `GET /api/debug/telemetry/history?since=<start>&prefix=stage.` | |||||
| - `GET /api/debug/telemetry/history?since=<start>&prefix=streamer.` | |||||
| 6. If IQ boundary issues persist, temporarily set `heavy_enabled=true` (keep sampling coarse with `heavy_sample_every` > 1), rerun, then inspect `iq.*` metrics and `audio.*` anomalies by `signal_id`/`session_id`. | |||||
| ## 2026-03-25 audio click incident — final resolved summary | |||||
| Status: **SOLVED** | |||||
| The March 2026 live-audio click investigation ultimately converged on a combination of three real root causes plus two secondary fixes: | |||||
| ### Root causes | |||||
| 1. **Shared `allIQ` corruption by `IQBalance` aliasing** | |||||
| - `cmd/sdrd/pipeline_runtime.go` | |||||
| - `survIQ` aliased the tail of `allIQ` | |||||
| - `dsp.IQBalance(survIQ)` modified `allIQ` in-place | |||||
| - extractor then saw a corrupted boundary inside the shared buffer | |||||
| - final fix: copy `survIQ` before `IQBalance` | |||||
| 2. **Per-frame extractor reset due to `StreamingConfigHash` jitter** | |||||
| - `internal/demod/gpudemod/streaming_types.go` | |||||
| - smoothed tuning values changed slightly every frame | |||||
| - offset/bandwidth in the hash caused repeated state resets | |||||
| - final fix: hash only structural parameters | |||||
| 3. **Streaming path batch rejection for non-WFM exact-decimation mismatch** | |||||
| - `cmd/sdrd/streaming_refactor.go` | |||||
| - one non-WFM signal could reject the whole batch and silently force fallback to the legacy path | |||||
| - final fix: choose nearest exact integer-divisor output rate and keep fallback logging visible | |||||
| ### Secondary fixes | |||||
| - FM discriminator cross-block carry in `internal/recorder/streamer.go` | |||||
| - WFM mono/plain-path 15 kHz audio lowpass in `internal/recorder/streamer.go` | |||||
| ### Verification notes | |||||
| - major discontinuities dropped sharply after the config-hash fix | |||||
| - remaining fine clicks were eliminated only after the `IQBalance` aliasing fix in `pipeline_runtime.go` | |||||
| - final confirmation was by operator listening test, backed by prior telemetry and WAV analysis | |||||
| ### Practical lesson | |||||
| When the same captured `allIQ` buffer feeds both: | |||||
| - surveillance/detail analysis | |||||
| - and extraction/streaming | |||||
| then surveillance-side DSP helpers must not mutate a shared sub-slice in-place unless that mutation is intentionally part of the extraction contract. | |||||
| @@ -96,6 +96,26 @@ type DecoderConfig struct { | |||||
| PSKCmd string `yaml:"psk_cmd" json:"psk_cmd"` | PSKCmd string `yaml:"psk_cmd" json:"psk_cmd"` | ||||
| } | } | ||||
| type DebugConfig struct { | |||||
| AudioDumpEnabled bool `yaml:"audio_dump_enabled" json:"audio_dump_enabled"` | |||||
| CPUMonitoring bool `yaml:"cpu_monitoring" json:"cpu_monitoring"` | |||||
| Telemetry TelemetryConfig `yaml:"telemetry" json:"telemetry"` | |||||
| } | |||||
| type TelemetryConfig struct { | |||||
| Enabled bool `yaml:"enabled" json:"enabled"` | |||||
| HeavyEnabled bool `yaml:"heavy_enabled" json:"heavy_enabled"` | |||||
| HeavySampleEvery int `yaml:"heavy_sample_every" json:"heavy_sample_every"` | |||||
| MetricSampleEvery int `yaml:"metric_sample_every" json:"metric_sample_every"` | |||||
| MetricHistoryMax int `yaml:"metric_history_max" json:"metric_history_max"` | |||||
| EventHistoryMax int `yaml:"event_history_max" json:"event_history_max"` | |||||
| RetentionSeconds int `yaml:"retention_seconds" json:"retention_seconds"` | |||||
| PersistEnabled bool `yaml:"persist_enabled" json:"persist_enabled"` | |||||
| PersistDir string `yaml:"persist_dir" json:"persist_dir"` | |||||
| RotateMB int `yaml:"rotate_mb" json:"rotate_mb"` | |||||
| KeepFiles int `yaml:"keep_files" json:"keep_files"` | |||||
| } | |||||
| type PipelineGoalConfig struct { | type PipelineGoalConfig struct { | ||||
| Intent string `yaml:"intent" json:"intent"` | Intent string `yaml:"intent" json:"intent"` | ||||
| MonitorStartHz float64 `yaml:"monitor_start_hz" json:"monitor_start_hz"` | MonitorStartHz float64 `yaml:"monitor_start_hz" json:"monitor_start_hz"` | ||||
| @@ -169,6 +189,7 @@ type Config struct { | |||||
| Detector DetectorConfig `yaml:"detector" json:"detector"` | Detector DetectorConfig `yaml:"detector" json:"detector"` | ||||
| Recorder RecorderConfig `yaml:"recorder" json:"recorder"` | Recorder RecorderConfig `yaml:"recorder" json:"recorder"` | ||||
| Decoder DecoderConfig `yaml:"decoder" json:"decoder"` | Decoder DecoderConfig `yaml:"decoder" json:"decoder"` | ||||
| Debug DebugConfig `yaml:"debug" json:"debug"` | |||||
| Logging LogConfig `yaml:"logging" json:"logging"` | Logging LogConfig `yaml:"logging" json:"logging"` | ||||
| WebAddr string `yaml:"web_addr" json:"web_addr"` | WebAddr string `yaml:"web_addr" json:"web_addr"` | ||||
| EventPath string `yaml:"event_path" json:"event_path"` | EventPath string `yaml:"event_path" json:"event_path"` | ||||
| @@ -421,6 +442,23 @@ func Default() Config { | |||||
| ExtractionBwMult: 1.2, | ExtractionBwMult: 1.2, | ||||
| }, | }, | ||||
| Decoder: DecoderConfig{}, | Decoder: DecoderConfig{}, | ||||
| Debug: DebugConfig{ | |||||
| AudioDumpEnabled: false, | |||||
| CPUMonitoring: false, | |||||
| Telemetry: TelemetryConfig{ | |||||
| Enabled: true, | |||||
| HeavyEnabled: false, | |||||
| HeavySampleEvery: 12, | |||||
| MetricSampleEvery: 2, | |||||
| MetricHistoryMax: 12000, | |||||
| EventHistoryMax: 4000, | |||||
| RetentionSeconds: 900, | |||||
| PersistEnabled: false, | |||||
| PersistDir: "debug/telemetry", | |||||
| RotateMB: 16, | |||||
| KeepFiles: 8, | |||||
| }, | |||||
| }, | |||||
| Logging: LogConfig{ | Logging: LogConfig{ | ||||
| Level: "informal", | Level: "informal", | ||||
| Categories: []string{}, | Categories: []string{}, | ||||
| @@ -664,6 +702,30 @@ func applyDefaults(cfg Config) Config { | |||||
| if cfg.Recorder.ExtractionBwMult <= 0 { | if cfg.Recorder.ExtractionBwMult <= 0 { | ||||
| cfg.Recorder.ExtractionBwMult = 1.2 | cfg.Recorder.ExtractionBwMult = 1.2 | ||||
| } | } | ||||
| if cfg.Debug.Telemetry.HeavySampleEvery <= 0 { | |||||
| cfg.Debug.Telemetry.HeavySampleEvery = 12 | |||||
| } | |||||
| if cfg.Debug.Telemetry.MetricSampleEvery <= 0 { | |||||
| cfg.Debug.Telemetry.MetricSampleEvery = 2 | |||||
| } | |||||
| if cfg.Debug.Telemetry.MetricHistoryMax <= 0 { | |||||
| cfg.Debug.Telemetry.MetricHistoryMax = 12000 | |||||
| } | |||||
| if cfg.Debug.Telemetry.EventHistoryMax <= 0 { | |||||
| cfg.Debug.Telemetry.EventHistoryMax = 4000 | |||||
| } | |||||
| if cfg.Debug.Telemetry.RetentionSeconds <= 0 { | |||||
| cfg.Debug.Telemetry.RetentionSeconds = 900 | |||||
| } | |||||
| if cfg.Debug.Telemetry.PersistDir == "" { | |||||
| cfg.Debug.Telemetry.PersistDir = "debug/telemetry" | |||||
| } | |||||
| if cfg.Debug.Telemetry.RotateMB <= 0 { | |||||
| cfg.Debug.Telemetry.RotateMB = 16 | |||||
| } | |||||
| if cfg.Debug.Telemetry.KeepFiles <= 0 { | |||||
| cfg.Debug.Telemetry.KeepFiles = 8 | |||||
| } | |||||
| return cfg | return cfg | ||||
| } | } | ||||
| @@ -4,6 +4,7 @@ import ( | |||||
| "math" | "math" | ||||
| "sdr-wideband-suite/internal/dsp" | "sdr-wideband-suite/internal/dsp" | ||||
| "sdr-wideband-suite/internal/logging" | |||||
| ) | ) | ||||
| type NFM struct{} | type NFM struct{} | ||||
| @@ -45,12 +46,45 @@ func fmDiscrim(iq []complex64) []float32 { | |||||
| return nil | return nil | ||||
| } | } | ||||
| out := make([]float32, len(iq)-1) | out := make([]float32, len(iq)-1) | ||||
| maxAbs := 0.0 | |||||
| maxIdx := 0 | |||||
| largeSteps := 0 | |||||
| minMag := math.MaxFloat64 | |||||
| maxMag := 0.0 | |||||
| for i := 1; i < len(iq); i++ { | for i := 1; i < len(iq); i++ { | ||||
| p := iq[i-1] | p := iq[i-1] | ||||
| c := iq[i] | c := iq[i] | ||||
| pmag := math.Hypot(float64(real(p)), float64(imag(p))) | |||||
| cmag := math.Hypot(float64(real(c)), float64(imag(c))) | |||||
| if pmag < minMag { | |||||
| minMag = pmag | |||||
| } | |||||
| if cmag < minMag { | |||||
| minMag = cmag | |||||
| } | |||||
| if pmag > maxMag { | |||||
| maxMag = pmag | |||||
| } | |||||
| if cmag > maxMag { | |||||
| maxMag = cmag | |||||
| } | |||||
| num := float64(real(p))*float64(imag(c)) - float64(imag(p))*float64(real(c)) | num := float64(real(p))*float64(imag(c)) - float64(imag(p))*float64(real(c)) | ||||
| den := float64(real(p))*float64(real(c)) + float64(imag(p))*float64(imag(c)) | den := float64(real(p))*float64(real(c)) + float64(imag(p))*float64(imag(c)) | ||||
| out[i-1] = float32(math.Atan2(num, den)) | |||||
| step := math.Atan2(num, den) | |||||
| if a := math.Abs(step); a > maxAbs { | |||||
| maxAbs = a | |||||
| maxIdx = i - 1 | |||||
| } | |||||
| if math.Abs(step) > 1.5 { | |||||
| largeSteps++ | |||||
| } | |||||
| out[i-1] = float32(step) | |||||
| } | |||||
| if logging.EnabledCategory("discrim") { | |||||
| logging.Debug("discrim", "fm_meter", "iq_len", len(iq), "audio_len", len(out), "min_mag", minMag, "max_mag", maxMag, "max_abs_step", maxAbs, "max_idx", maxIdx, "large_steps", largeSteps) | |||||
| if largeSteps > 0 { | |||||
| logging.Warn("discrim", "fm_large_steps", "iq_len", len(iq), "large_steps", largeSteps, "max_abs_step", maxAbs, "max_idx", maxIdx, "min_mag", minMag, "max_mag", maxMag) | |||||
| } | |||||
| } | } | ||||
| return out | return out | ||||
| } | } | ||||
| @@ -1,34 +0,0 @@ | |||||
| # gpudemod | |||||
| Phase 1 CUDA demod scaffolding. | |||||
| ## Current state | |||||
| - Standard Go builds use `gpudemod_stub.go` (`!cufft`). | |||||
| - `cufft` builds allocate GPU buffers and cross the CGO/CUDA launch boundary. | |||||
| - If CUDA launch wrappers are not backed by compiled kernels yet, the code falls back to CPU DSP. | |||||
| - The shifted IQ path is already wired so a successful GPU freq-shift result can be copied back and reused immediately. | |||||
| - Build orchestration should now be considered OS-specific; see `docs/build-cuda.md`. | |||||
| ## First real kernel | |||||
| `kernels.cu` contains the first candidate implementation: | |||||
| - `gpud_freq_shift_kernel` | |||||
| This is **not compiled automatically yet** in the current environment because the machine currently lacks a CUDA compiler toolchain in PATH (`nvcc` not found). | |||||
| ## Next machine-side step | |||||
| On a CUDA-capable dev machine with toolchain installed: | |||||
| 1. Compile `kernels.cu` into an object file and archive it into a linkable library | |||||
| - helper script: `tools/build-gpudemod-kernel.ps1` | |||||
| 2. On Jan's Windows machine, the working kernel-build path currently relies on `nvcc` + MSVC `cl.exe` in PATH | |||||
| 3. Link `gpudemod_kernels.lib` into the `cufft` build | |||||
| 3. Replace `gpud_launch_freq_shift(...)` stub body with the real kernel launch | |||||
| 4. Validate copied-back shifted IQ against `dsp.FreqShift` | |||||
| 5. Only then move the next stage (FM discriminator) onto the GPU | |||||
| ## Why this is still useful | |||||
| The runtime/buffer/recorder/fallback structure is already in place, so once kernel compilation is available, real acceleration can be inserted without another architecture rewrite. | |||||
| @@ -6,7 +6,7 @@ type ExtractJob struct { | |||||
| OffsetHz float64 | OffsetHz float64 | ||||
| BW float64 | BW float64 | ||||
| OutRate int | OutRate int | ||||
| PhaseStart float64 // FreqShift starting phase (0 for stateless, carry over for streaming) | |||||
| PhaseStart float64 // legacy batch phase field; retained only while migrating to streaming extractor semantics | |||||
| } | } | ||||
| // ExtractResult holds the output of a batch extraction including the ending | // ExtractResult holds the output of a batch extraction including the ending | ||||
| @@ -10,10 +10,12 @@ type batchSlot struct { | |||||
| } | } | ||||
| type BatchRunner struct { | type BatchRunner struct { | ||||
| eng *Engine | |||||
| slots []batchSlot | |||||
| slotBufs []slotBuffers | |||||
| eng *Engine | |||||
| slots []batchSlot | |||||
| slotBufs []slotBuffers | |||||
| slotBufSize int // number of IQ samples the slot buffers were allocated for | slotBufSize int // number of IQ samples the slot buffers were allocated for | ||||
| streamState map[int64]*ExtractStreamState | |||||
| nativeState map[int64]*nativeStreamingSignalState | |||||
| } | } | ||||
| func NewBatchRunner(maxSamples int, sampleRate int) (*BatchRunner, error) { | func NewBatchRunner(maxSamples int, sampleRate int) (*BatchRunner, error) { | ||||
| @@ -21,7 +23,11 @@ func NewBatchRunner(maxSamples int, sampleRate int) (*BatchRunner, error) { | |||||
| if err != nil { | if err != nil { | ||||
| return nil, err | return nil, err | ||||
| } | } | ||||
| return &BatchRunner{eng: eng}, nil | |||||
| return &BatchRunner{ | |||||
| eng: eng, | |||||
| streamState: make(map[int64]*ExtractStreamState), | |||||
| nativeState: make(map[int64]*nativeStreamingSignalState), | |||||
| }, nil | |||||
| } | } | ||||
| func (r *BatchRunner) Close() { | func (r *BatchRunner) Close() { | ||||
| @@ -29,9 +35,12 @@ func (r *BatchRunner) Close() { | |||||
| return | return | ||||
| } | } | ||||
| r.freeSlotBuffers() | r.freeSlotBuffers() | ||||
| r.freeAllNativeStreamingStates() | |||||
| r.eng.Close() | r.eng.Close() | ||||
| r.eng = nil | r.eng = nil | ||||
| r.slots = nil | r.slots = nil | ||||
| r.streamState = nil | |||||
| r.nativeState = nil | |||||
| } | } | ||||
| func (r *BatchRunner) prepare(jobs []ExtractJob) { | func (r *BatchRunner) prepare(jobs []ExtractJob) { | ||||
| @@ -160,9 +160,9 @@ func (r *BatchRunner) shiftFilterDecimateSlotParallel(iq []complex64, job Extrac | |||||
| if bridgeMemcpyH2D(buf.dTaps, unsafe.Pointer(&taps[0]), tapsBytes) != 0 { | if bridgeMemcpyH2D(buf.dTaps, unsafe.Pointer(&taps[0]), tapsBytes) != 0 { | ||||
| return 0, 0, errors.New("taps H2D failed") | return 0, 0, errors.New("taps H2D failed") | ||||
| } | } | ||||
| decim := int(math.Round(float64(e.sampleRate) / float64(job.OutRate))) | |||||
| if decim < 1 { | |||||
| decim = 1 | |||||
| decim, err := ExactIntegerDecimation(e.sampleRate, job.OutRate) | |||||
| if err != nil { | |||||
| return 0, 0, err | |||||
| } | } | ||||
| nOut := n / decim | nOut := n / decim | ||||
| if nOut <= 0 { | if nOut <= 0 { | ||||
| @@ -0,0 +1,47 @@ | |||||
| package gpudemod | |||||
| import "math/cmplx" | |||||
| type CompareStats struct { | |||||
| MaxAbsErr float64 | |||||
| RMSErr float64 | |||||
| Count int | |||||
| } | |||||
| func CompareComplexSlices(a []complex64, b []complex64) CompareStats { | |||||
| n := len(a) | |||||
| if len(b) < n { | |||||
| n = len(b) | |||||
| } | |||||
| if n == 0 { | |||||
| return CompareStats{} | |||||
| } | |||||
| var sumSq float64 | |||||
| var maxAbs float64 | |||||
| for i := 0; i < n; i++ { | |||||
| err := cmplx.Abs(complex128(a[i] - b[i])) | |||||
| if err > maxAbs { | |||||
| maxAbs = err | |||||
| } | |||||
| sumSq += err * err | |||||
| } | |||||
| return CompareStats{ | |||||
| MaxAbsErr: maxAbs, | |||||
| RMSErr: mathSqrt(sumSq / float64(n)), | |||||
| Count: n, | |||||
| } | |||||
| } | |||||
| func mathSqrt(v float64) float64 { | |||||
| // tiny shim to keep the compare helper self-contained and easy to move | |||||
| // without importing additional logic elsewhere | |||||
| z := v | |||||
| if z <= 0 { | |||||
| return 0 | |||||
| } | |||||
| x := z | |||||
| for i := 0; i < 12; i++ { | |||||
| x = 0.5 * (x + z/x) | |||||
| } | |||||
| return x | |||||
| } | |||||
| @@ -0,0 +1,19 @@ | |||||
| package gpudemod | |||||
| func BuildGPUStubDebugMetrics(res StreamingExtractResult) ExtractDebugMetrics { | |||||
| return ExtractDebugMetrics{ | |||||
| SignalID: res.SignalID, | |||||
| PhaseCount: res.PhaseCount, | |||||
| HistoryLen: res.HistoryLen, | |||||
| NOut: res.NOut, | |||||
| } | |||||
| } | |||||
| func BuildGPUHostOracleDebugMetrics(res StreamingExtractResult) ExtractDebugMetrics { | |||||
| return ExtractDebugMetrics{ | |||||
| SignalID: res.SignalID, | |||||
| PhaseCount: res.PhaseCount, | |||||
| HistoryLen: res.HistoryLen, | |||||
| NOut: res.NOut, | |||||
| } | |||||
| } | |||||
| @@ -0,0 +1,10 @@ | |||||
| package gpudemod | |||||
| func BuildOracleDebugMetrics(res StreamingExtractResult) ExtractDebugMetrics { | |||||
| return ExtractDebugMetrics{ | |||||
| SignalID: res.SignalID, | |||||
| PhaseCount: res.PhaseCount, | |||||
| HistoryLen: res.HistoryLen, | |||||
| NOut: res.NOut, | |||||
| } | |||||
| } | |||||
| @@ -0,0 +1,27 @@ | |||||
| package gpudemod | |||||
| func CompareOracleAndGPUStub(oracle StreamingExtractResult, gpu StreamingExtractResult) (ExtractDebugMetrics, CompareStats) { | |||||
| stats := CompareComplexSlices(oracle.IQ, gpu.IQ) | |||||
| metrics := ExtractDebugMetrics{ | |||||
| SignalID: oracle.SignalID, | |||||
| PhaseCount: gpu.PhaseCount, | |||||
| HistoryLen: gpu.HistoryLen, | |||||
| NOut: gpu.NOut, | |||||
| RefMaxAbsErr: stats.MaxAbsErr, | |||||
| RefRMSErr: stats.RMSErr, | |||||
| } | |||||
| return metrics, stats | |||||
| } | |||||
| func CompareOracleAndGPUHostOracle(oracle StreamingExtractResult, gpu StreamingExtractResult) (ExtractDebugMetrics, CompareStats) { | |||||
| stats := CompareComplexSlices(oracle.IQ, gpu.IQ) | |||||
| metrics := ExtractDebugMetrics{ | |||||
| SignalID: oracle.SignalID, | |||||
| PhaseCount: gpu.PhaseCount, | |||||
| HistoryLen: gpu.HistoryLen, | |||||
| NOut: gpu.NOut, | |||||
| RefMaxAbsErr: stats.MaxAbsErr, | |||||
| RefRMSErr: stats.RMSErr, | |||||
| } | |||||
| return metrics, stats | |||||
| } | |||||
| @@ -0,0 +1,32 @@ | |||||
| package gpudemod | |||||
| import "testing" | |||||
| func TestCompareOracleAndGPUStub(t *testing.T) { | |||||
| oracle := StreamingExtractResult{ | |||||
| SignalID: 1, | |||||
| IQ: []complex64{1 + 1i, 2 + 2i}, | |||||
| Rate: 200000, | |||||
| NOut: 2, | |||||
| PhaseCount: 0, | |||||
| HistoryLen: 64, | |||||
| } | |||||
| gpu := StreamingExtractResult{ | |||||
| SignalID: 1, | |||||
| IQ: []complex64{1 + 1i, 2.1 + 2i}, | |||||
| Rate: 200000, | |||||
| NOut: 2, | |||||
| PhaseCount: 3, | |||||
| HistoryLen: 64, | |||||
| } | |||||
| metrics, stats := CompareOracleAndGPUStub(oracle, gpu) | |||||
| if metrics.SignalID != 1 { | |||||
| t.Fatalf("unexpected signal id: %d", metrics.SignalID) | |||||
| } | |||||
| if stats.Count != 2 { | |||||
| t.Fatalf("unexpected compare count: %d", stats.Count) | |||||
| } | |||||
| if metrics.RefMaxAbsErr <= 0 { | |||||
| t.Fatalf("expected positive max abs error") | |||||
| } | |||||
| } | |||||
| @@ -0,0 +1,12 @@ | |||||
| package gpudemod | |||||
| type ExtractDebugMetrics struct { | |||||
| SignalID int64 | |||||
| PhaseCount int | |||||
| HistoryLen int | |||||
| NOut int | |||||
| RefMaxAbsErr float64 | |||||
| RefRMSErr float64 | |||||
| BoundaryDelta float64 | |||||
| BoundaryD2 float64 | |||||
| } | |||||
| @@ -0,0 +1,18 @@ | |||||
| package gpudemod | |||||
| import "testing" | |||||
| func TestCompareComplexSlices(t *testing.T) { | |||||
| a := []complex64{1 + 1i, 2 + 2i, 3 + 3i} | |||||
| b := []complex64{1 + 1i, 2.1 + 2i, 2.9 + 3.2i} | |||||
| stats := CompareComplexSlices(a, b) | |||||
| if stats.Count != 3 { | |||||
| t.Fatalf("unexpected count: %d", stats.Count) | |||||
| } | |||||
| if stats.MaxAbsErr <= 0 { | |||||
| t.Fatalf("expected positive max abs error") | |||||
| } | |||||
| if stats.RMSErr <= 0 { | |||||
| t.Fatalf("expected positive rms error") | |||||
| } | |||||
| } | |||||
| @@ -0,0 +1,170 @@ | |||||
| package gpudemod | |||||
| import ( | |||||
| "fmt" | |||||
| "math" | |||||
| ) | |||||
| type CPUOracleState struct { | |||||
| SignalID int64 | |||||
| ConfigHash uint64 | |||||
| NCOPhase float64 | |||||
| Decim int | |||||
| PhaseCount int | |||||
| NumTaps int | |||||
| ShiftedHistory []complex64 | |||||
| BaseTaps []float32 | |||||
| PolyphaseTaps []float32 | |||||
| } | |||||
| func ResetCPUOracleStateIfConfigChanged(state *CPUOracleState, newHash uint64) { | |||||
| if state == nil { | |||||
| return | |||||
| } | |||||
| if state.ConfigHash != newHash { | |||||
| state.ConfigHash = newHash | |||||
| state.NCOPhase = 0 | |||||
| state.PhaseCount = 0 | |||||
| state.ShiftedHistory = state.ShiftedHistory[:0] | |||||
| } | |||||
| } | |||||
| func CPUOracleExtract(iqNew []complex64, state *CPUOracleState, phaseInc float64) []complex64 { | |||||
| if state == nil || state.NumTaps <= 0 || state.Decim <= 0 || len(state.BaseTaps) < state.NumTaps { | |||||
| return nil | |||||
| } | |||||
| out := make([]complex64, 0, len(iqNew)/maxInt(1, state.Decim)+2) | |||||
| phase := state.NCOPhase | |||||
| hist := append([]complex64(nil), state.ShiftedHistory...) | |||||
| for _, x := range iqNew { | |||||
| rot := complex64(complex(math.Cos(phase), math.Sin(phase))) | |||||
| s := x * rot | |||||
| hist = append(hist, s) | |||||
| state.PhaseCount++ | |||||
| if state.PhaseCount == state.Decim { | |||||
| var y complex64 | |||||
| for k := 0; k < state.NumTaps; k++ { | |||||
| idx := len(hist) - 1 - k | |||||
| var sample complex64 | |||||
| if idx >= 0 { | |||||
| sample = hist[idx] | |||||
| } | |||||
| y += complex(state.BaseTaps[k], 0) * sample | |||||
| } | |||||
| out = append(out, y) | |||||
| state.PhaseCount = 0 | |||||
| } | |||||
| if len(hist) > state.NumTaps-1 { | |||||
| hist = hist[len(hist)-(state.NumTaps-1):] | |||||
| } | |||||
| phase += phaseInc | |||||
| if phase >= math.Pi { | |||||
| phase -= 2 * math.Pi | |||||
| } else if phase < -math.Pi { | |||||
| phase += 2 * math.Pi | |||||
| } | |||||
| } | |||||
| state.NCOPhase = phase | |||||
| state.ShiftedHistory = append(state.ShiftedHistory[:0], hist...) | |||||
| return out | |||||
| } | |||||
| // CPUOracleExtractPolyphase keeps the same streaming state semantics as CPUOracleExtract, | |||||
| // but computes outputs using the explicit phase-major polyphase tap layout. | |||||
| func CPUOracleExtractPolyphase(iqNew []complex64, state *CPUOracleState, phaseInc float64) []complex64 { | |||||
| if state == nil || state.NumTaps <= 0 || state.Decim <= 0 || len(state.BaseTaps) < state.NumTaps { | |||||
| return nil | |||||
| } | |||||
| if len(state.PolyphaseTaps) == 0 { | |||||
| state.PolyphaseTaps = BuildPolyphaseTapsPhaseMajor(state.BaseTaps, state.Decim) | |||||
| } | |||||
| phaseLen := PolyphasePhaseLen(len(state.BaseTaps), state.Decim) | |||||
| out := make([]complex64, 0, len(iqNew)/maxInt(1, state.Decim)+2) | |||||
| phase := state.NCOPhase | |||||
| hist := append([]complex64(nil), state.ShiftedHistory...) | |||||
| for _, x := range iqNew { | |||||
| rot := complex64(complex(math.Cos(phase), math.Sin(phase))) | |||||
| s := x * rot | |||||
| hist = append(hist, s) | |||||
| state.PhaseCount++ | |||||
| if state.PhaseCount == state.Decim { | |||||
| var y complex64 | |||||
| for p := 0; p < state.Decim; p++ { | |||||
| for k := 0; k < phaseLen; k++ { | |||||
| tap := state.PolyphaseTaps[p*phaseLen+k] | |||||
| if tap == 0 { | |||||
| continue | |||||
| } | |||||
| srcBack := p + k*state.Decim | |||||
| idx := len(hist) - 1 - srcBack | |||||
| if idx < 0 { | |||||
| continue | |||||
| } | |||||
| y += complex(tap, 0) * hist[idx] | |||||
| } | |||||
| } | |||||
| out = append(out, y) | |||||
| state.PhaseCount = 0 | |||||
| } | |||||
| if len(hist) > state.NumTaps-1 { | |||||
| hist = hist[len(hist)-(state.NumTaps-1):] | |||||
| } | |||||
| phase += phaseInc | |||||
| if phase >= math.Pi { | |||||
| phase -= 2 * math.Pi | |||||
| } else if phase < -math.Pi { | |||||
| phase += 2 * math.Pi | |||||
| } | |||||
| } | |||||
| state.NCOPhase = phase | |||||
| state.ShiftedHistory = append(state.ShiftedHistory[:0], hist...) | |||||
| return out | |||||
| } | |||||
| func RunChunkedCPUOracle(all []complex64, chunkSizes []int, mkState func() *CPUOracleState, phaseInc float64) []complex64 { | |||||
| state := mkState() | |||||
| out := make([]complex64, 0) | |||||
| pos := 0 | |||||
| for _, n := range chunkSizes { | |||||
| if pos >= len(all) { | |||||
| break | |||||
| } | |||||
| end := pos + n | |||||
| if end > len(all) { | |||||
| end = len(all) | |||||
| } | |||||
| out = append(out, CPUOracleExtract(all[pos:end], state, phaseInc)...) | |||||
| pos = end | |||||
| } | |||||
| if pos < len(all) { | |||||
| out = append(out, CPUOracleExtract(all[pos:], state, phaseInc)...) | |||||
| } | |||||
| return out | |||||
| } | |||||
| func ExactIntegerDecimation(sampleRate int, outRate int) (int, error) { | |||||
| if sampleRate <= 0 || outRate <= 0 { | |||||
| return 0, fmt.Errorf("invalid sampleRate/outRate: %d/%d", sampleRate, outRate) | |||||
| } | |||||
| if sampleRate%outRate != 0 { | |||||
| return 0, fmt.Errorf("streaming polyphase extractor requires integer decimation: sampleRate=%d outRate=%d", sampleRate, outRate) | |||||
| } | |||||
| return sampleRate / outRate, nil | |||||
| } | |||||
| func maxInt(a int, b int) int { | |||||
| if a > b { | |||||
| return a | |||||
| } | |||||
| return b | |||||
| } | |||||
| @@ -0,0 +1,89 @@ | |||||
| package gpudemod | |||||
| import ( | |||||
| "math" | |||||
| "math/cmplx" | |||||
| "testing" | |||||
| ) | |||||
| func makeDeterministicIQ(n int) []complex64 { | |||||
| out := make([]complex64, n) | |||||
| for i := 0; i < n; i++ { | |||||
| a := 0.017 * float64(i) | |||||
| b := 0.031 * float64(i) | |||||
| out[i] = complex64(complex(math.Cos(a)+0.2*math.Cos(b), math.Sin(a)+0.15*math.Sin(b))) | |||||
| } | |||||
| return out | |||||
| } | |||||
| func makeLowpassTaps(n int) []float32 { | |||||
| out := make([]float32, n) | |||||
| for i := range out { | |||||
| out[i] = 1.0 / float32(n) | |||||
| } | |||||
| return out | |||||
| } | |||||
| func requireComplexSlicesClose(t *testing.T, a []complex64, b []complex64, tol float64) { | |||||
| t.Helper() | |||||
| if len(a) != len(b) { | |||||
| t.Fatalf("length mismatch: %d vs %d", len(a), len(b)) | |||||
| } | |||||
| for i := range a { | |||||
| if cmplx.Abs(complex128(a[i]-b[i])) > tol { | |||||
| t.Fatalf("slice mismatch at %d: %v vs %v (tol=%f)", i, a[i], b[i], tol) | |||||
| } | |||||
| } | |||||
| } | |||||
| func TestCPUOracleMonolithicVsChunked(t *testing.T) { | |||||
| iq := makeDeterministicIQ(200000) | |||||
| mk := func() *CPUOracleState { | |||||
| return &CPUOracleState{ | |||||
| SignalID: 1, | |||||
| ConfigHash: 123, | |||||
| NCOPhase: 0, | |||||
| Decim: 20, | |||||
| PhaseCount: 0, | |||||
| NumTaps: 65, | |||||
| ShiftedHistory: make([]complex64, 0, 64), | |||||
| BaseTaps: makeLowpassTaps(65), | |||||
| } | |||||
| } | |||||
| phaseInc := 0.017 | |||||
| monoState := mk() | |||||
| mono := CPUOracleExtract(iq, monoState, phaseInc) | |||||
| chunked := RunChunkedCPUOracle(iq, []int{4096, 5000, 8192, 27307}, mk, phaseInc) | |||||
| requireComplexSlicesClose(t, mono, chunked, 1e-5) | |||||
| } | |||||
| func TestExactIntegerDecimation(t *testing.T) { | |||||
| if d, err := ExactIntegerDecimation(4000000, 200000); err != nil || d != 20 { | |||||
| t.Fatalf("unexpected exact decim result: d=%d err=%v", d, err) | |||||
| } | |||||
| if _, err := ExactIntegerDecimation(4000000, 192000); err == nil { | |||||
| t.Fatalf("expected non-integer decimation error") | |||||
| } | |||||
| } | |||||
| func TestCPUOracleDirectVsPolyphase(t *testing.T) { | |||||
| iq := makeDeterministicIQ(50000) | |||||
| mk := func() *CPUOracleState { | |||||
| taps := makeLowpassTaps(65) | |||||
| return &CPUOracleState{ | |||||
| SignalID: 1, | |||||
| ConfigHash: 123, | |||||
| NCOPhase: 0, | |||||
| Decim: 20, | |||||
| PhaseCount: 0, | |||||
| NumTaps: 65, | |||||
| ShiftedHistory: make([]complex64, 0, 64), | |||||
| BaseTaps: taps, | |||||
| PolyphaseTaps: BuildPolyphaseTapsPhaseMajor(taps, 20), | |||||
| } | |||||
| } | |||||
| phaseInc := 0.017 | |||||
| direct := CPUOracleExtract(iq, mk(), phaseInc) | |||||
| poly := CPUOracleExtractPolyphase(iq, mk(), phaseInc) | |||||
| requireComplexSlicesClose(t, direct, poly, 1e-5) | |||||
| } | |||||
| @@ -11,6 +11,10 @@ | |||||
| typedef void* gpud_stream_handle; | typedef void* gpud_stream_handle; | ||||
| static __forceinline__ int gpud_max_i(int a, int b) { | |||||
| return a > b ? a : b; | |||||
| } | |||||
| GPUD_API int GPUD_CALL gpud_stream_create(gpud_stream_handle* out) { | GPUD_API int GPUD_CALL gpud_stream_create(gpud_stream_handle* out) { | ||||
| if (!out) return -1; | if (!out) return -1; | ||||
| cudaStream_t stream; | cudaStream_t stream; | ||||
| @@ -320,3 +324,308 @@ GPUD_API int GPUD_CALL gpud_launch_ssb_product_cuda( | |||||
| gpud_ssb_product_kernel<<<grid, block>>>(in, out, n, phase_inc, phase_start); | gpud_ssb_product_kernel<<<grid, block>>>(in, out, n, phase_inc, phase_start); | ||||
| return (int)cudaGetLastError(); | return (int)cudaGetLastError(); | ||||
| } | } | ||||
| __global__ void gpud_streaming_polyphase_accum_kernel( | |||||
| const float2* __restrict__ history_state, | |||||
| int history_len, | |||||
| const float2* __restrict__ shifted_new, | |||||
| int n_new, | |||||
| const float* __restrict__ polyphase_taps, | |||||
| int polyphase_len, | |||||
| int decim, | |||||
| int phase_len, | |||||
| int start_idx, | |||||
| int n_out, | |||||
| float2* __restrict__ out | |||||
| ); | |||||
| __global__ void gpud_streaming_history_tail_kernel( | |||||
| const float2* __restrict__ history_state, | |||||
| int history_len, | |||||
| const float2* __restrict__ shifted_new, | |||||
| int n_new, | |||||
| int keep, | |||||
| float2* __restrict__ history_out | |||||
| ); | |||||
| static __forceinline__ double gpud_reduce_phase(double phase); | |||||
| // Transitional legacy entrypoint retained for bring-up and comparison. | |||||
| // The production-native streaming path is gpud_launch_streaming_polyphase_stateful_cuda, | |||||
| // which preserves per-signal carry state across NEW-samples-only chunks. | |||||
| GPUD_API int GPUD_CALL gpud_launch_streaming_polyphase_prepare_cuda( | |||||
| const float2* in_new, | |||||
| int n_new, | |||||
| const float2* history_in, | |||||
| int history_len, | |||||
| const float* polyphase_taps, | |||||
| int polyphase_len, | |||||
| int decim, | |||||
| int num_taps, | |||||
| int phase_count_in, | |||||
| double phase_start, | |||||
| double phase_inc, | |||||
| float2* out, | |||||
| int* n_out, | |||||
| int* phase_count_out, | |||||
| double* phase_end_out, | |||||
| float2* history_out | |||||
| ) { | |||||
| if (n_new < 0 || !polyphase_taps || polyphase_len <= 0 || decim <= 0 || num_taps <= 0) return -1; | |||||
| const int phase_len = (num_taps + decim - 1) / decim; | |||||
| if (polyphase_len < decim * phase_len) return -2; | |||||
| const int keep = num_taps > 1 ? num_taps - 1 : 0; | |||||
| int clamped_history_len = history_len; | |||||
| if (clamped_history_len < 0) clamped_history_len = 0; | |||||
| if (clamped_history_len > keep) clamped_history_len = keep; | |||||
| if (clamped_history_len > 0 && !history_in) return -5; | |||||
| float2* shifted = NULL; | |||||
| cudaError_t err = cudaSuccess; | |||||
| if (n_new > 0) { | |||||
| if (!in_new) return -3; | |||||
| err = cudaMalloc((void**)&shifted, (size_t)gpud_max_i(1, n_new) * sizeof(float2)); | |||||
| if (err != cudaSuccess) return (int)err; | |||||
| const int block = 256; | |||||
| const int grid_shift = (n_new + block - 1) / block; | |||||
| gpud_freq_shift_kernel<<<grid_shift, block>>>(in_new, shifted, n_new, phase_inc, phase_start); | |||||
| err = cudaGetLastError(); | |||||
| if (err != cudaSuccess) { | |||||
| cudaFree(shifted); | |||||
| return (int)err; | |||||
| } | |||||
| } | |||||
| int phase_count = phase_count_in; | |||||
| if (phase_count < 0) phase_count = 0; | |||||
| if (phase_count >= decim) phase_count %= decim; | |||||
| const int total_phase = phase_count + n_new; | |||||
| const int out_count = total_phase / decim; | |||||
| if (out_count > 0) { | |||||
| if (!out) { | |||||
| cudaFree(shifted); | |||||
| return -4; | |||||
| } | |||||
| const int block = 256; | |||||
| const int grid = (out_count + block - 1) / block; | |||||
| const int start_idx = decim - phase_count - 1; | |||||
| gpud_streaming_polyphase_accum_kernel<<<grid, block>>>( | |||||
| history_in, | |||||
| clamped_history_len, | |||||
| shifted, | |||||
| n_new, | |||||
| polyphase_taps, | |||||
| polyphase_len, | |||||
| decim, | |||||
| phase_len, | |||||
| start_idx, | |||||
| out_count, | |||||
| out | |||||
| ); | |||||
| err = cudaGetLastError(); | |||||
| if (err != cudaSuccess) { | |||||
| cudaFree(shifted); | |||||
| return (int)err; | |||||
| } | |||||
| } | |||||
| if (history_out && keep > 0) { | |||||
| const int new_history_len = clamped_history_len + n_new < keep ? clamped_history_len + n_new : keep; | |||||
| if (new_history_len > 0) { | |||||
| const int block = 256; | |||||
| const int grid = (new_history_len + block - 1) / block; | |||||
| gpud_streaming_history_tail_kernel<<<grid, block>>>( | |||||
| history_in, | |||||
| clamped_history_len, | |||||
| shifted, | |||||
| n_new, | |||||
| new_history_len, | |||||
| history_out | |||||
| ); | |||||
| err = cudaGetLastError(); | |||||
| if (err != cudaSuccess) { | |||||
| cudaFree(shifted); | |||||
| return (int)err; | |||||
| } | |||||
| } | |||||
| } | |||||
| if (n_out) *n_out = out_count; | |||||
| if (phase_count_out) *phase_count_out = total_phase % decim; | |||||
| if (phase_end_out) *phase_end_out = gpud_reduce_phase(phase_start + phase_inc * (double)n_new); | |||||
| if (shifted) cudaFree(shifted); | |||||
| return 0; | |||||
| } | |||||
| static __device__ __forceinline__ float2 gpud_stream_sample_at( | |||||
| const float2* __restrict__ history_state, | |||||
| int history_len, | |||||
| const float2* __restrict__ shifted_new, | |||||
| int n_new, | |||||
| int idx | |||||
| ) { | |||||
| if (idx < 0) return make_float2(0.0f, 0.0f); | |||||
| if (idx < history_len) return history_state[idx]; | |||||
| int shifted_idx = idx - history_len; | |||||
| if (shifted_idx < 0 || shifted_idx >= n_new) return make_float2(0.0f, 0.0f); | |||||
| return shifted_new[shifted_idx]; | |||||
| } | |||||
| __global__ void gpud_streaming_polyphase_accum_kernel( | |||||
| const float2* __restrict__ history_state, | |||||
| int history_len, | |||||
| const float2* __restrict__ shifted_new, | |||||
| int n_new, | |||||
| const float* __restrict__ polyphase_taps, | |||||
| int polyphase_len, | |||||
| int decim, | |||||
| int phase_len, | |||||
| int start_idx, | |||||
| int n_out, | |||||
| float2* __restrict__ out | |||||
| ) { | |||||
| int out_idx = blockIdx.x * blockDim.x + threadIdx.x; | |||||
| if (out_idx >= n_out) return; | |||||
| int newest = history_len + start_idx + out_idx * decim; | |||||
| float acc_r = 0.0f; | |||||
| float acc_i = 0.0f; | |||||
| for (int p = 0; p < decim; ++p) { | |||||
| for (int k = 0; k < phase_len; ++k) { | |||||
| int tap_idx = p * phase_len + k; | |||||
| if (tap_idx >= polyphase_len) continue; | |||||
| float tap = polyphase_taps[tap_idx]; | |||||
| if (tap == 0.0f) continue; | |||||
| int src_back = p + k * decim; | |||||
| int src_idx = newest - src_back; | |||||
| float2 sample = gpud_stream_sample_at(history_state, history_len, shifted_new, n_new, src_idx); | |||||
| acc_r += sample.x * tap; | |||||
| acc_i += sample.y * tap; | |||||
| } | |||||
| } | |||||
| out[out_idx] = make_float2(acc_r, acc_i); | |||||
| } | |||||
| __global__ void gpud_streaming_history_tail_kernel( | |||||
| const float2* __restrict__ history_state, | |||||
| int history_len, | |||||
| const float2* __restrict__ shifted_new, | |||||
| int n_new, | |||||
| int keep, | |||||
| float2* __restrict__ history_out | |||||
| ) { | |||||
| int idx = blockIdx.x * blockDim.x + threadIdx.x; | |||||
| if (idx >= keep) return; | |||||
| int combined_len = history_len + n_new; | |||||
| int src_idx = combined_len - keep + idx; | |||||
| history_out[idx] = gpud_stream_sample_at(history_state, history_len, shifted_new, n_new, src_idx); | |||||
| } | |||||
| static __forceinline__ double gpud_reduce_phase(double phase) { | |||||
| const double TWO_PI = 6.283185307179586; | |||||
| return phase - rint(phase / TWO_PI) * TWO_PI; | |||||
| } | |||||
| // Production-native candidate entrypoint for the stateful streaming extractor. | |||||
| // Callers provide only NEW samples; overlap+trim is intentionally not part of this path. | |||||
| GPUD_API int GPUD_CALL gpud_launch_streaming_polyphase_stateful_cuda( | |||||
| const float2* in_new, | |||||
| int n_new, | |||||
| float2* shifted_new_tmp, | |||||
| const float* polyphase_taps, | |||||
| int polyphase_len, | |||||
| int decim, | |||||
| int num_taps, | |||||
| float2* history_state, | |||||
| float2* history_scratch, | |||||
| int history_cap, | |||||
| int* history_len_io, | |||||
| int* phase_count_state, | |||||
| double* phase_state, | |||||
| double phase_inc, | |||||
| float2* out, | |||||
| int out_cap, | |||||
| int* n_out | |||||
| ) { | |||||
| if (!polyphase_taps || decim <= 0 || num_taps <= 0 || !history_len_io || !phase_count_state || !phase_state || !n_out) return -10; | |||||
| if (n_new < 0 || out_cap < 0 || history_cap < 0) return -11; | |||||
| const int phase_len = (num_taps + decim - 1) / decim; | |||||
| if (polyphase_len < decim * phase_len) return -12; | |||||
| int history_len = *history_len_io; | |||||
| if (history_len < 0) history_len = 0; | |||||
| if (history_len > history_cap) history_len = history_cap; | |||||
| int phase_count = *phase_count_state; | |||||
| if (phase_count < 0) phase_count = 0; | |||||
| if (phase_count >= decim) phase_count %= decim; | |||||
| double phase_start = *phase_state; | |||||
| if (n_new > 0) { | |||||
| if (!in_new || !shifted_new_tmp) return -13; | |||||
| const int block = 256; | |||||
| const int grid = (n_new + block - 1) / block; | |||||
| gpud_freq_shift_kernel<<<grid, block>>>(in_new, shifted_new_tmp, n_new, phase_inc, phase_start); | |||||
| cudaError_t err = cudaGetLastError(); | |||||
| if (err != cudaSuccess) return (int)err; | |||||
| } | |||||
| const int total_phase = phase_count + n_new; | |||||
| const int out_count = total_phase / decim; | |||||
| if (out_count > out_cap) return -14; | |||||
| if (out_count > 0) { | |||||
| if (!out) return -15; | |||||
| const int block = 256; | |||||
| const int grid = (out_count + block - 1) / block; | |||||
| const int start_idx = decim - phase_count - 1; | |||||
| gpud_streaming_polyphase_accum_kernel<<<grid, block>>>( | |||||
| history_state, | |||||
| history_len, | |||||
| shifted_new_tmp, | |||||
| n_new, | |||||
| polyphase_taps, | |||||
| polyphase_len, | |||||
| decim, | |||||
| phase_len, | |||||
| start_idx, | |||||
| out_count, | |||||
| out | |||||
| ); | |||||
| cudaError_t err = cudaGetLastError(); | |||||
| if (err != cudaSuccess) return (int)err; | |||||
| } | |||||
| int new_history_len = history_len; | |||||
| if (history_cap > 0) { | |||||
| new_history_len = history_len + n_new; | |||||
| if (new_history_len > history_cap) new_history_len = history_cap; | |||||
| if (new_history_len > 0) { | |||||
| if (!history_state || !history_scratch) return -16; | |||||
| const int block = 256; | |||||
| const int grid = (new_history_len + block - 1) / block; | |||||
| gpud_streaming_history_tail_kernel<<<grid, block>>>( | |||||
| history_state, | |||||
| history_len, | |||||
| shifted_new_tmp, | |||||
| n_new, | |||||
| new_history_len, | |||||
| history_scratch | |||||
| ); | |||||
| cudaError_t err = cudaGetLastError(); | |||||
| if (err != cudaSuccess) return (int)err; | |||||
| err = cudaMemcpy(history_state, history_scratch, (size_t)new_history_len * sizeof(float2), cudaMemcpyDeviceToDevice); | |||||
| if (err != cudaSuccess) return (int)err; | |||||
| } | |||||
| } else { | |||||
| new_history_len = 0; | |||||
| } | |||||
| *history_len_io = new_history_len; | |||||
| *phase_count_state = total_phase % decim; | |||||
| *phase_state = gpud_reduce_phase(phase_start + phase_inc * (double)n_new); | |||||
| *n_out = out_count; | |||||
| return 0; | |||||
| } | |||||
| @@ -0,0 +1,31 @@ | |||||
| package gpudemod | |||||
| import "testing" | |||||
| func TestCPUOracleRunnerCleansUpDisappearedSignals(t *testing.T) { | |||||
| r := NewCPUOracleRunner(4000000) | |||||
| jobs1 := []StreamingExtractJob{ | |||||
| {SignalID: 1, OffsetHz: 1000, Bandwidth: 20000, OutRate: 200000, NumTaps: 65, ConfigHash: 101}, | |||||
| {SignalID: 2, OffsetHz: 2000, Bandwidth: 20000, OutRate: 200000, NumTaps: 65, ConfigHash: 102}, | |||||
| } | |||||
| _, err := r.StreamingExtract(makeDeterministicIQ(4096), jobs1) | |||||
| if err != nil { | |||||
| t.Fatalf("unexpected error on first extract: %v", err) | |||||
| } | |||||
| if len(r.States) != 2 { | |||||
| t.Fatalf("expected 2 states, got %d", len(r.States)) | |||||
| } | |||||
| jobs2 := []StreamingExtractJob{ | |||||
| {SignalID: 2, OffsetHz: 2000, Bandwidth: 20000, OutRate: 200000, NumTaps: 65, ConfigHash: 102}, | |||||
| } | |||||
| _, err = r.StreamingExtract(makeDeterministicIQ(2048), jobs2) | |||||
| if err != nil { | |||||
| t.Fatalf("unexpected error on second extract: %v", err) | |||||
| } | |||||
| if len(r.States) != 1 { | |||||
| t.Fatalf("expected 1 state after cleanup, got %d", len(r.States)) | |||||
| } | |||||
| if _, ok := r.States[1]; ok { | |||||
| t.Fatalf("expected signal 1 state to be cleaned up") | |||||
| } | |||||
| } | |||||
| @@ -0,0 +1,45 @@ | |||||
| package gpudemod | |||||
| import "testing" | |||||
| func TestCPUOracleMonolithicVsChunkedPolyphase(t *testing.T) { | |||||
| iq := makeDeterministicIQ(120000) | |||||
| mk := func() *CPUOracleState { | |||||
| taps := makeLowpassTaps(65) | |||||
| return &CPUOracleState{ | |||||
| SignalID: 1, | |||||
| ConfigHash: 999, | |||||
| NCOPhase: 0, | |||||
| Decim: 20, | |||||
| PhaseCount: 0, | |||||
| NumTaps: 65, | |||||
| ShiftedHistory: make([]complex64, 0, 64), | |||||
| BaseTaps: taps, | |||||
| PolyphaseTaps: BuildPolyphaseTapsPhaseMajor(taps, 20), | |||||
| } | |||||
| } | |||||
| phaseInc := 0.013 | |||||
| mono := CPUOracleExtractPolyphase(iq, mk(), phaseInc) | |||||
| chunked := func() []complex64 { | |||||
| state := mk() | |||||
| out := make([]complex64, 0) | |||||
| chunks := []int{4096, 3000, 8192, 7777, 12000} | |||||
| pos := 0 | |||||
| for _, n := range chunks { | |||||
| if pos >= len(iq) { | |||||
| break | |||||
| } | |||||
| end := pos + n | |||||
| if end > len(iq) { | |||||
| end = len(iq) | |||||
| } | |||||
| out = append(out, CPUOracleExtractPolyphase(iq[pos:end], state, phaseInc)...) | |||||
| pos = end | |||||
| } | |||||
| if pos < len(iq) { | |||||
| out = append(out, CPUOracleExtractPolyphase(iq[pos:], state, phaseInc)...) | |||||
| } | |||||
| return out | |||||
| }() | |||||
| requireComplexSlicesClose(t, mono, chunked, 1e-5) | |||||
| } | |||||
| @@ -0,0 +1,28 @@ | |||||
| package gpudemod | |||||
| // BuildPolyphaseTapsPhaseMajor builds a phase-major polyphase tap layout: | |||||
| // tapsByPhase[p][k] = h[p + k*D] | |||||
| // Flattened as: [phase0 taps..., phase1 taps..., ...] | |||||
| func BuildPolyphaseTapsPhaseMajor(base []float32, decim int) []float32 { | |||||
| if decim <= 0 || len(base) == 0 { | |||||
| return nil | |||||
| } | |||||
| maxPhaseLen := (len(base) + decim - 1) / decim | |||||
| out := make([]float32, decim*maxPhaseLen) | |||||
| for p := 0; p < decim; p++ { | |||||
| for k := 0; k < maxPhaseLen; k++ { | |||||
| src := p + k*decim | |||||
| if src < len(base) { | |||||
| out[p*maxPhaseLen+k] = base[src] | |||||
| } | |||||
| } | |||||
| } | |||||
| return out | |||||
| } | |||||
| func PolyphasePhaseLen(baseLen int, decim int) int { | |||||
| if decim <= 0 || baseLen <= 0 { | |||||
| return 0 | |||||
| } | |||||
| return (baseLen + decim - 1) / decim | |||||
| } | |||||
| @@ -0,0 +1,22 @@ | |||||
| package gpudemod | |||||
| import "testing" | |||||
| func TestBuildPolyphaseTapsPhaseMajor(t *testing.T) { | |||||
| base := []float32{1, 2, 3, 4, 5, 6, 7} | |||||
| got := BuildPolyphaseTapsPhaseMajor(base, 3) | |||||
| // phase-major with phase len ceil(7/3)=3 | |||||
| want := []float32{ | |||||
| 1, 4, 7, | |||||
| 2, 5, 0, | |||||
| 3, 6, 0, | |||||
| } | |||||
| if len(got) != len(want) { | |||||
| t.Fatalf("len mismatch: got %d want %d", len(got), len(want)) | |||||
| } | |||||
| for i := range want { | |||||
| if got[i] != want[i] { | |||||
| t.Fatalf("mismatch at %d: got %v want %v", i, got[i], want[i]) | |||||
| } | |||||
| } | |||||
| } | |||||
| @@ -0,0 +1,57 @@ | |||||
| package gpudemod | |||||
| import "testing" | |||||
| func TestResetCPUOracleStateIfConfigChanged(t *testing.T) { | |||||
| state := &CPUOracleState{ | |||||
| SignalID: 1, | |||||
| ConfigHash: 111, | |||||
| NCOPhase: 1.23, | |||||
| Decim: 20, | |||||
| PhaseCount: 7, | |||||
| NumTaps: 65, | |||||
| ShiftedHistory: []complex64{1 + 1i, 2 + 2i}, | |||||
| } | |||||
| ResetCPUOracleStateIfConfigChanged(state, 222) | |||||
| if state.ConfigHash != 222 { | |||||
| t.Fatalf("config hash not updated") | |||||
| } | |||||
| if state.NCOPhase != 0 { | |||||
| t.Fatalf("expected phase reset") | |||||
| } | |||||
| if state.PhaseCount != 0 { | |||||
| t.Fatalf("expected phase count reset") | |||||
| } | |||||
| if len(state.ShiftedHistory) != 0 { | |||||
| t.Fatalf("expected shifted history reset") | |||||
| } | |||||
| } | |||||
| func TestResetExtractStreamState(t *testing.T) { | |||||
| state := &ExtractStreamState{ | |||||
| SignalID: 1, | |||||
| ConfigHash: 111, | |||||
| NCOPhase: 2.34, | |||||
| Decim: 20, | |||||
| PhaseCount: 9, | |||||
| NumTaps: 65, | |||||
| ShiftedHistory: []complex64{3 + 3i, 4 + 4i}, | |||||
| Initialized: true, | |||||
| } | |||||
| ResetExtractStreamState(state, 333) | |||||
| if state.ConfigHash != 333 { | |||||
| t.Fatalf("config hash not updated") | |||||
| } | |||||
| if state.NCOPhase != 0 { | |||||
| t.Fatalf("expected phase reset") | |||||
| } | |||||
| if state.PhaseCount != 0 { | |||||
| t.Fatalf("expected phase count reset") | |||||
| } | |||||
| if len(state.ShiftedHistory) != 0 { | |||||
| t.Fatalf("expected shifted history reset") | |||||
| } | |||||
| if state.Initialized { | |||||
| t.Fatalf("expected initialized=false after reset") | |||||
| } | |||||
| } | |||||
| @@ -0,0 +1,70 @@ | |||||
| package gpudemod | |||||
| import ( | |||||
| "log" | |||||
| "sdr-wideband-suite/internal/dsp" | |||||
| ) | |||||
| func (r *BatchRunner) ResetSignalState(signalID int64) { | |||||
| if r == nil || r.streamState == nil { | |||||
| return | |||||
| } | |||||
| delete(r.streamState, signalID) | |||||
| r.resetNativeStreamingState(signalID) | |||||
| } | |||||
| func (r *BatchRunner) ResetAllSignalStates() { | |||||
| if r == nil { | |||||
| return | |||||
| } | |||||
| r.streamState = make(map[int64]*ExtractStreamState) | |||||
| r.resetAllNativeStreamingStates() | |||||
| } | |||||
| func (r *BatchRunner) getOrInitExtractState(job StreamingExtractJob, sampleRate int) (*ExtractStreamState, error) { | |||||
| if r == nil { | |||||
| return nil, ErrUnavailable | |||||
| } | |||||
| if r.streamState == nil { | |||||
| r.streamState = make(map[int64]*ExtractStreamState) | |||||
| } | |||||
| decim, err := ExactIntegerDecimation(sampleRate, job.OutRate) | |||||
| if err != nil { | |||||
| return nil, err | |||||
| } | |||||
| state := r.streamState[job.SignalID] | |||||
| if state == nil { | |||||
| state = &ExtractStreamState{SignalID: job.SignalID} | |||||
| r.streamState[job.SignalID] = state | |||||
| } | |||||
| if state.ConfigHash != job.ConfigHash { | |||||
| if state.Initialized { | |||||
| log.Printf("STREAMING STATE RESET: signal=%d oldHash=%d newHash=%d historyLen=%d", | |||||
| job.SignalID, state.ConfigHash, job.ConfigHash, len(state.ShiftedHistory)) | |||||
| } | |||||
| ResetExtractStreamState(state, job.ConfigHash) | |||||
| } | |||||
| state.Decim = decim | |||||
| state.NumTaps = job.NumTaps | |||||
| if state.NumTaps <= 0 { | |||||
| state.NumTaps = 101 | |||||
| } | |||||
| cutoff := job.Bandwidth / 2 | |||||
| if cutoff < 200 { | |||||
| cutoff = 200 | |||||
| } | |||||
| base := dsp.LowpassFIR(cutoff, sampleRate, state.NumTaps) | |||||
| state.BaseTaps = make([]float32, len(base)) | |||||
| for i, v := range base { | |||||
| state.BaseTaps[i] = float32(v) | |||||
| } | |||||
| state.PolyphaseTaps = BuildPolyphaseTapsPhaseMajor(state.BaseTaps, state.Decim) | |||||
| if cap(state.ShiftedHistory) < maxInt(0, state.NumTaps-1) { | |||||
| state.ShiftedHistory = make([]complex64, 0, maxInt(0, state.NumTaps-1)) | |||||
| } else if state.ShiftedHistory == nil { | |||||
| state.ShiftedHistory = make([]complex64, 0, maxInt(0, state.NumTaps-1)) | |||||
| } | |||||
| state.Initialized = true | |||||
| return state, nil | |||||
| } | |||||
| @@ -0,0 +1,31 @@ | |||||
| package gpudemod | |||||
| import "testing" | |||||
| func TestGetOrInitExtractStateInitializesPolyphaseAndHistory(t *testing.T) { | |||||
| r := &BatchRunner{streamState: make(map[int64]*ExtractStreamState)} | |||||
| job := StreamingExtractJob{ | |||||
| SignalID: 7, | |||||
| OffsetHz: 12500, | |||||
| Bandwidth: 20000, | |||||
| OutRate: 200000, | |||||
| NumTaps: 65, | |||||
| ConfigHash: 555, | |||||
| } | |||||
| state, err := r.getOrInitExtractState(job, 4000000) | |||||
| if err != nil { | |||||
| t.Fatalf("unexpected error: %v", err) | |||||
| } | |||||
| if state.Decim != 20 { | |||||
| t.Fatalf("unexpected decim: %d", state.Decim) | |||||
| } | |||||
| if len(state.BaseTaps) != 65 { | |||||
| t.Fatalf("unexpected base taps len: %d", len(state.BaseTaps)) | |||||
| } | |||||
| if len(state.PolyphaseTaps) == 0 { | |||||
| t.Fatalf("expected polyphase taps") | |||||
| } | |||||
| if cap(state.ShiftedHistory) < 64 { | |||||
| t.Fatalf("expected shifted history capacity >= 64, got %d", cap(state.ShiftedHistory)) | |||||
| } | |||||
| } | |||||
| @@ -0,0 +1,39 @@ | |||||
| package gpudemod | |||||
| type StreamingGPUExecutionMode string | |||||
| const ( | |||||
| StreamingGPUExecUnavailable StreamingGPUExecutionMode = "unavailable" | |||||
| StreamingGPUExecHostOracle StreamingGPUExecutionMode = "host_oracle" | |||||
| StreamingGPUExecCUDA StreamingGPUExecutionMode = "cuda" | |||||
| ) | |||||
| type StreamingGPUInvocation struct { | |||||
| SignalID int64 | |||||
| ConfigHash uint64 | |||||
| OffsetHz float64 | |||||
| OutRate int | |||||
| Bandwidth float64 | |||||
| SampleRate int | |||||
| NumTaps int | |||||
| Decim int | |||||
| PhaseCountIn int | |||||
| NCOPhaseIn float64 | |||||
| HistoryLen int | |||||
| BaseTaps []float32 | |||||
| PolyphaseTaps []float32 | |||||
| ShiftedHistory []complex64 | |||||
| IQNew []complex64 | |||||
| } | |||||
| type StreamingGPUExecutionResult struct { | |||||
| SignalID int64 | |||||
| Mode StreamingGPUExecutionMode | |||||
| IQ []complex64 | |||||
| Rate int | |||||
| NOut int | |||||
| PhaseCountOut int | |||||
| NCOPhaseOut float64 | |||||
| HistoryOut []complex64 | |||||
| HistoryLenOut int | |||||
| } | |||||
| @@ -0,0 +1,29 @@ | |||||
| package gpudemod | |||||
| // StreamingExtractGPUExec is the internal execution selector for the new | |||||
| // production-path semantics. It intentionally keeps the public API stable while | |||||
| // allowing the implementation to evolve from host-side oracle execution toward | |||||
| // a real GPU polyphase path. | |||||
| func (r *BatchRunner) StreamingExtractGPUExec(iqNew []complex64, jobs []StreamingExtractJob) ([]StreamingExtractResult, error) { | |||||
| invocations, err := r.buildStreamingGPUInvocations(iqNew, jobs) | |||||
| if err != nil { | |||||
| return nil, err | |||||
| } | |||||
| if useGPUNativePreparedExecution { | |||||
| execResults, err := r.executeStreamingGPUNativePrepared(invocations) | |||||
| if err == nil { | |||||
| return r.applyStreamingGPUExecutionResults(execResults), nil | |||||
| } | |||||
| if !useGPUHostOracleExecution { | |||||
| return nil, err | |||||
| } | |||||
| } | |||||
| if useGPUHostOracleExecution { | |||||
| execResults, err := r.executeStreamingGPUHostOraclePrepared(invocations) | |||||
| if err != nil { | |||||
| return nil, err | |||||
| } | |||||
| return r.applyStreamingGPUExecutionResults(execResults), nil | |||||
| } | |||||
| return nil, ErrUnavailable | |||||
| } | |||||
| @@ -0,0 +1,112 @@ | |||||
| package gpudemod | |||||
| import "testing" | |||||
| func TestStreamingExtractGPUExecUsesSafeDefaultMode(t *testing.T) { | |||||
| r := &BatchRunner{eng: &Engine{sampleRate: 4000000}, streamState: make(map[int64]*ExtractStreamState)} | |||||
| job := StreamingExtractJob{ | |||||
| SignalID: 1, | |||||
| OffsetHz: 12500, | |||||
| Bandwidth: 20000, | |||||
| OutRate: 200000, | |||||
| NumTaps: 65, | |||||
| ConfigHash: 777, | |||||
| } | |||||
| res, err := r.StreamingExtractGPUExec(makeDeterministicIQ(2048), []StreamingExtractJob{job}) | |||||
| if err != nil { | |||||
| t.Fatalf("expected safe default execution path, got error: %v", err) | |||||
| } | |||||
| if len(res) != 1 { | |||||
| t.Fatalf("expected 1 result, got %d", len(res)) | |||||
| } | |||||
| if res[0].Rate != job.OutRate { | |||||
| t.Fatalf("expected output rate %d, got %d", job.OutRate, res[0].Rate) | |||||
| } | |||||
| if res[0].NOut <= 0 { | |||||
| t.Fatalf("expected streaming output samples") | |||||
| } | |||||
| } | |||||
| func TestStreamingGPUExecMatchesCPUOracleAcrossChunkPatterns(t *testing.T) { | |||||
| job := StreamingExtractJob{ | |||||
| SignalID: 1, | |||||
| OffsetHz: 12500, | |||||
| Bandwidth: 20000, | |||||
| OutRate: 200000, | |||||
| NumTaps: 65, | |||||
| ConfigHash: 777, | |||||
| } | |||||
| t.Run("DeterministicIQ", func(t *testing.T) { | |||||
| r := &BatchRunner{eng: &Engine{sampleRate: 4000000}, streamState: make(map[int64]*ExtractStreamState)} | |||||
| steps := makeStreamingValidationSteps( | |||||
| makeDeterministicIQ(1500), | |||||
| []int{0, 1, 2, 17, 63, 64, 65, 129, 511}, | |||||
| []StreamingExtractJob{job}, | |||||
| ) | |||||
| runStreamingExecSequenceAgainstOracle(t, r, steps, 1e-5, 1e-9) | |||||
| }) | |||||
| t.Run("ToneNoiseIQ", func(t *testing.T) { | |||||
| r := &BatchRunner{eng: &Engine{sampleRate: 4000000}, streamState: make(map[int64]*ExtractStreamState)} | |||||
| steps := makeStreamingValidationSteps( | |||||
| makeToneNoiseIQ(4096, 0.023), | |||||
| []int{7, 20, 3, 63, 64, 65, 777}, | |||||
| []StreamingExtractJob{job}, | |||||
| ) | |||||
| runStreamingExecSequenceAgainstOracle(t, r, steps, 1e-5, 1e-9) | |||||
| }) | |||||
| } | |||||
| func TestStreamingGPUExecLifecycleMatchesCPUOracle(t *testing.T) { | |||||
| r := &BatchRunner{ | |||||
| eng: &Engine{sampleRate: 4000000}, | |||||
| streamState: make(map[int64]*ExtractStreamState), | |||||
| nativeState: make(map[int64]*nativeStreamingSignalState), | |||||
| } | |||||
| baseA := StreamingExtractJob{ | |||||
| SignalID: 11, | |||||
| OffsetHz: 12500, | |||||
| Bandwidth: 20000, | |||||
| OutRate: 200000, | |||||
| NumTaps: 65, | |||||
| ConfigHash: 1001, | |||||
| } | |||||
| baseB := StreamingExtractJob{ | |||||
| SignalID: 22, | |||||
| OffsetHz: -18750, | |||||
| Bandwidth: 16000, | |||||
| OutRate: 100000, | |||||
| NumTaps: 33, | |||||
| ConfigHash: 2002, | |||||
| } | |||||
| steps := []streamingValidationStep{ | |||||
| { | |||||
| name: "prime_both_signals", | |||||
| iq: makeDeterministicIQ(512), | |||||
| jobs: []StreamingExtractJob{baseA, baseB}, | |||||
| }, | |||||
| { | |||||
| name: "config_reset_with_zero_new", | |||||
| iq: nil, | |||||
| jobs: []StreamingExtractJob{{SignalID: baseA.SignalID, OffsetHz: baseA.OffsetHz, Bandwidth: baseA.Bandwidth, OutRate: baseA.OutRate, NumTaps: baseA.NumTaps, ConfigHash: baseA.ConfigHash + 1}, baseB}, | |||||
| }, | |||||
| { | |||||
| name: "signal_b_disappears", | |||||
| iq: makeToneNoiseIQ(96, 0.041), | |||||
| jobs: []StreamingExtractJob{baseA}, | |||||
| }, | |||||
| { | |||||
| name: "signal_b_reappears_fresh", | |||||
| iq: makeDeterministicIQ(160), | |||||
| jobs: []StreamingExtractJob{baseA, baseB}, | |||||
| }, | |||||
| { | |||||
| name: "small_history_boundary_chunk", | |||||
| iq: makeToneNoiseIQ(65, 0.017), | |||||
| jobs: []StreamingExtractJob{baseA, baseB}, | |||||
| }, | |||||
| } | |||||
| runStreamingExecSequenceAgainstOracle(t, r, steps, 1e-5, 1e-9) | |||||
| if _, ok := r.nativeState[baseB.SignalID]; ok { | |||||
| t.Fatalf("expected safe host-oracle path to keep native state inactive while gate is off") | |||||
| } | |||||
| } | |||||
| @@ -0,0 +1,30 @@ | |||||
| package gpudemod | |||||
| func (r *BatchRunner) executeStreamingGPUHostOraclePrepared(invocations []StreamingGPUInvocation) ([]StreamingGPUExecutionResult, error) { | |||||
| results := make([]StreamingGPUExecutionResult, len(invocations)) | |||||
| for i, inv := range invocations { | |||||
| out, phase, phaseCount, hist := runStreamingPolyphaseHostCore( | |||||
| inv.IQNew, | |||||
| inv.SampleRate, | |||||
| inv.OffsetHz, | |||||
| inv.NCOPhaseIn, | |||||
| inv.PhaseCountIn, | |||||
| inv.NumTaps, | |||||
| inv.Decim, | |||||
| inv.ShiftedHistory, | |||||
| inv.PolyphaseTaps, | |||||
| ) | |||||
| results[i] = StreamingGPUExecutionResult{ | |||||
| SignalID: inv.SignalID, | |||||
| Mode: StreamingGPUExecHostOracle, | |||||
| IQ: out, | |||||
| Rate: inv.OutRate, | |||||
| NOut: len(out), | |||||
| PhaseCountOut: phaseCount, | |||||
| NCOPhaseOut: phase, | |||||
| HistoryOut: hist, | |||||
| HistoryLenOut: len(hist), | |||||
| } | |||||
| } | |||||
| return results, nil | |||||
| } | |||||
| @@ -0,0 +1,49 @@ | |||||
| package gpudemod | |||||
| // StreamingExtractGPUHostOracle is a temporary host-side execution of the intended | |||||
| // streaming semantics using GPU-owned stream state. It is not the final GPU | |||||
| // production implementation, but it allows the new production entrypoint to move | |||||
| // from pure stub semantics toward real NEW-samples-only streaming behavior | |||||
| // without reintroducing overlap+trim. | |||||
| func (r *BatchRunner) StreamingExtractGPUHostOracle(iqNew []complex64, jobs []StreamingExtractJob) ([]StreamingExtractResult, error) { | |||||
| if r == nil || r.eng == nil { | |||||
| return nil, ErrUnavailable | |||||
| } | |||||
| results := make([]StreamingExtractResult, len(jobs)) | |||||
| active := make(map[int64]struct{}, len(jobs)) | |||||
| for i, job := range jobs { | |||||
| active[job.SignalID] = struct{}{} | |||||
| state, err := r.getOrInitExtractState(job, r.eng.sampleRate) | |||||
| if err != nil { | |||||
| return nil, err | |||||
| } | |||||
| out, phase, phaseCount, hist := runStreamingPolyphaseHostCore( | |||||
| iqNew, | |||||
| r.eng.sampleRate, | |||||
| job.OffsetHz, | |||||
| state.NCOPhase, | |||||
| state.PhaseCount, | |||||
| state.NumTaps, | |||||
| state.Decim, | |||||
| state.ShiftedHistory, | |||||
| state.PolyphaseTaps, | |||||
| ) | |||||
| state.NCOPhase = phase | |||||
| state.PhaseCount = phaseCount | |||||
| state.ShiftedHistory = append(state.ShiftedHistory[:0], hist...) | |||||
| results[i] = StreamingExtractResult{ | |||||
| SignalID: job.SignalID, | |||||
| IQ: out, | |||||
| Rate: job.OutRate, | |||||
| NOut: len(out), | |||||
| PhaseCount: state.PhaseCount, | |||||
| HistoryLen: len(state.ShiftedHistory), | |||||
| } | |||||
| } | |||||
| for signalID := range r.streamState { | |||||
| if _, ok := active[signalID]; !ok { | |||||
| delete(r.streamState, signalID) | |||||
| } | |||||
| } | |||||
| return results, nil | |||||
| } | |||||
| @@ -0,0 +1,35 @@ | |||||
| package gpudemod | |||||
| import "testing" | |||||
| func TestStreamingGPUHostOracleComparableToCPUOracle(t *testing.T) { | |||||
| r := &BatchRunner{eng: &Engine{sampleRate: 4000000}, streamState: make(map[int64]*ExtractStreamState)} | |||||
| job := StreamingExtractJob{ | |||||
| SignalID: 1, | |||||
| OffsetHz: 12500, | |||||
| Bandwidth: 20000, | |||||
| OutRate: 200000, | |||||
| NumTaps: 65, | |||||
| ConfigHash: 777, | |||||
| } | |||||
| iq := makeDeterministicIQ(16000) | |||||
| gpuLike, err := r.StreamingExtractGPUHostOracle(iq, []StreamingExtractJob{job}) | |||||
| if err != nil { | |||||
| t.Fatalf("unexpected host-oracle error: %v", err) | |||||
| } | |||||
| oracleRunner := NewCPUOracleRunner(4000000) | |||||
| oracle, err := oracleRunner.StreamingExtract(iq, []StreamingExtractJob{job}) | |||||
| if err != nil { | |||||
| t.Fatalf("unexpected oracle error: %v", err) | |||||
| } | |||||
| if len(gpuLike) != 1 || len(oracle) != 1 { | |||||
| t.Fatalf("unexpected result lengths: gpuLike=%d oracle=%d", len(gpuLike), len(oracle)) | |||||
| } | |||||
| metrics, stats := CompareOracleAndGPUHostOracle(oracle[0], gpuLike[0]) | |||||
| if stats.Count == 0 { | |||||
| t.Fatalf("expected compare count > 0") | |||||
| } | |||||
| if metrics.RefMaxAbsErr > 1e-5 { | |||||
| t.Fatalf("expected host-oracle path to match cpu oracle closely, got max abs err %f", metrics.RefMaxAbsErr) | |||||
| } | |||||
| } | |||||
| @@ -0,0 +1,4 @@ | |||||
| package gpudemod | |||||
| const useGPUHostOracleExecution = false | |||||
| const useGPUNativePreparedExecution = true | |||||
| @@ -0,0 +1,284 @@ | |||||
| //go:build cufft && windows | |||||
| package gpudemod | |||||
| /* | |||||
| #cgo windows CFLAGS: -I"C:/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v13.2/include" | |||||
| #include <cuda_runtime.h> | |||||
| typedef struct { float x; float y; } gpud_float2; | |||||
| */ | |||||
| import "C" | |||||
| import ( | |||||
| "math" | |||||
| "unsafe" | |||||
| ) | |||||
| func (r *BatchRunner) executeStreamingGPUNativePrepared(invocations []StreamingGPUInvocation) ([]StreamingGPUExecutionResult, error) { | |||||
| if r == nil || r.eng == nil { | |||||
| return nil, ErrUnavailable | |||||
| } | |||||
| if r.nativeState == nil { | |||||
| r.nativeState = make(map[int64]*nativeStreamingSignalState) | |||||
| } | |||||
| results := make([]StreamingGPUExecutionResult, len(invocations)) | |||||
| for i, inv := range invocations { | |||||
| state, err := r.getOrInitNativeStreamingState(inv) | |||||
| if err != nil { | |||||
| return nil, err | |||||
| } | |||||
| if len(inv.IQNew) > 0 { | |||||
| if err := ensureNativeBuffer(&state.dInNew, &state.inNewCap, len(inv.IQNew), unsafe.Sizeof(C.gpud_float2{})); err != nil { | |||||
| return nil, err | |||||
| } | |||||
| if bridgeMemcpyH2D(state.dInNew, unsafe.Pointer(&inv.IQNew[0]), uintptr(len(inv.IQNew))*unsafe.Sizeof(complex64(0))) != 0 { | |||||
| return nil, ErrUnavailable | |||||
| } | |||||
| } | |||||
| outCap := len(inv.IQNew)/maxInt(1, inv.Decim) + 2 | |||||
| if outCap > 0 { | |||||
| if err := ensureNativeBuffer(&state.dOut, &state.outCap, outCap, unsafe.Sizeof(C.gpud_float2{})); err != nil { | |||||
| return nil, err | |||||
| } | |||||
| } | |||||
| phaseInc := -2.0 * math.Pi * inv.OffsetHz / float64(inv.SampleRate) | |||||
| // The native export consumes phase carry as host scalars while sample/history | |||||
| // buffers remain device-resident, so keep these counters in nativeState. | |||||
| var nOut C.int | |||||
| historyLen := C.int(state.historyLen) | |||||
| phaseCount := C.int(state.phaseCount) | |||||
| phaseNCO := C.double(state.phaseNCO) | |||||
| res := bridgeLaunchStreamingPolyphaseStateful( | |||||
| (*C.gpud_float2)(state.dInNew), | |||||
| len(inv.IQNew), | |||||
| (*C.gpud_float2)(state.dShifted), | |||||
| (*C.float)(state.dTaps), | |||||
| state.tapsLen, | |||||
| state.decim, | |||||
| state.numTaps, | |||||
| (*C.gpud_float2)(state.dHistory), | |||||
| (*C.gpud_float2)(state.dHistoryScratch), | |||||
| state.historyCap, | |||||
| &historyLen, | |||||
| &phaseCount, | |||||
| &phaseNCO, | |||||
| phaseInc, | |||||
| (*C.gpud_float2)(state.dOut), | |||||
| outCap, | |||||
| &nOut, | |||||
| ) | |||||
| if res != 0 { | |||||
| return nil, ErrUnavailable | |||||
| } | |||||
| state.historyLen = int(historyLen) | |||||
| state.phaseCount = int(phaseCount) | |||||
| state.phaseNCO = float64(phaseNCO) | |||||
| outHost := make([]complex64, int(nOut)) | |||||
| if len(outHost) > 0 { | |||||
| if bridgeMemcpyD2H(unsafe.Pointer(&outHost[0]), state.dOut, uintptr(len(outHost))*unsafe.Sizeof(complex64(0))) != 0 { | |||||
| return nil, ErrUnavailable | |||||
| } | |||||
| } | |||||
| histHost := make([]complex64, state.historyLen) | |||||
| if state.historyLen > 0 { | |||||
| if bridgeMemcpyD2H(unsafe.Pointer(&histHost[0]), state.dHistory, uintptr(state.historyLen)*unsafe.Sizeof(complex64(0))) != 0 { | |||||
| return nil, ErrUnavailable | |||||
| } | |||||
| } | |||||
| results[i] = StreamingGPUExecutionResult{ | |||||
| SignalID: inv.SignalID, | |||||
| Mode: StreamingGPUExecCUDA, | |||||
| IQ: outHost, | |||||
| Rate: inv.OutRate, | |||||
| NOut: len(outHost), | |||||
| PhaseCountOut: state.phaseCount, | |||||
| NCOPhaseOut: state.phaseNCO, | |||||
| HistoryOut: histHost, | |||||
| HistoryLenOut: len(histHost), | |||||
| } | |||||
| } | |||||
| return results, nil | |||||
| } | |||||
| func (r *BatchRunner) getOrInitNativeStreamingState(inv StreamingGPUInvocation) (*nativeStreamingSignalState, error) { | |||||
| state := r.nativeState[inv.SignalID] | |||||
| needReset := false | |||||
| historyCap := maxInt(0, inv.NumTaps-1) | |||||
| if state == nil { | |||||
| state = &nativeStreamingSignalState{signalID: inv.SignalID} | |||||
| r.nativeState[inv.SignalID] = state | |||||
| needReset = true | |||||
| } | |||||
| if state.configHash != inv.ConfigHash { | |||||
| needReset = true | |||||
| } | |||||
| if state.decim != inv.Decim || state.numTaps != inv.NumTaps || state.tapsLen != len(inv.PolyphaseTaps) { | |||||
| needReset = true | |||||
| } | |||||
| if state.historyCap != historyCap { | |||||
| needReset = true | |||||
| } | |||||
| if needReset { | |||||
| releaseNativeStreamingSignalState(state) | |||||
| } | |||||
| if len(inv.PolyphaseTaps) == 0 { | |||||
| return nil, ErrUnavailable | |||||
| } | |||||
| if state.dTaps == nil && len(inv.PolyphaseTaps) > 0 { | |||||
| if bridgeCudaMalloc(&state.dTaps, uintptr(len(inv.PolyphaseTaps))*unsafe.Sizeof(C.float(0))) != 0 { | |||||
| return nil, ErrUnavailable | |||||
| } | |||||
| if bridgeMemcpyH2D(state.dTaps, unsafe.Pointer(&inv.PolyphaseTaps[0]), uintptr(len(inv.PolyphaseTaps))*unsafe.Sizeof(float32(0))) != 0 { | |||||
| return nil, ErrUnavailable | |||||
| } | |||||
| state.tapsLen = len(inv.PolyphaseTaps) | |||||
| } | |||||
| if state.dShifted == nil { | |||||
| minCap := maxInt(1, len(inv.IQNew)) | |||||
| if bridgeCudaMalloc(&state.dShifted, uintptr(minCap)*unsafe.Sizeof(C.gpud_float2{})) != 0 { | |||||
| return nil, ErrUnavailable | |||||
| } | |||||
| state.shiftedCap = minCap | |||||
| } | |||||
| if state.shiftedCap < len(inv.IQNew) { | |||||
| if bridgeCudaFree(state.dShifted) != 0 { | |||||
| return nil, ErrUnavailable | |||||
| } | |||||
| state.dShifted = nil | |||||
| state.shiftedCap = 0 | |||||
| if bridgeCudaMalloc(&state.dShifted, uintptr(len(inv.IQNew))*unsafe.Sizeof(C.gpud_float2{})) != 0 { | |||||
| return nil, ErrUnavailable | |||||
| } | |||||
| state.shiftedCap = len(inv.IQNew) | |||||
| } | |||||
| if state.dHistory == nil && historyCap > 0 { | |||||
| if bridgeCudaMalloc(&state.dHistory, uintptr(historyCap)*unsafe.Sizeof(C.gpud_float2{})) != 0 { | |||||
| return nil, ErrUnavailable | |||||
| } | |||||
| } | |||||
| if state.dHistoryScratch == nil && historyCap > 0 { | |||||
| if bridgeCudaMalloc(&state.dHistoryScratch, uintptr(historyCap)*unsafe.Sizeof(C.gpud_float2{})) != 0 { | |||||
| return nil, ErrUnavailable | |||||
| } | |||||
| state.historyScratchCap = historyCap | |||||
| } | |||||
| if needReset { | |||||
| state.phaseCount = inv.PhaseCountIn | |||||
| state.phaseNCO = inv.NCOPhaseIn | |||||
| state.historyLen = minInt(len(inv.ShiftedHistory), historyCap) | |||||
| if state.historyLen > 0 { | |||||
| if bridgeMemcpyH2D(state.dHistory, unsafe.Pointer(&inv.ShiftedHistory[len(inv.ShiftedHistory)-state.historyLen]), uintptr(state.historyLen)*unsafe.Sizeof(complex64(0))) != 0 { | |||||
| return nil, ErrUnavailable | |||||
| } | |||||
| } | |||||
| } | |||||
| state.decim = inv.Decim | |||||
| state.numTaps = inv.NumTaps | |||||
| state.historyCap = historyCap | |||||
| state.historyScratchCap = historyCap | |||||
| state.configHash = inv.ConfigHash | |||||
| return state, nil | |||||
| } | |||||
| func ensureNativeBuffer(ptr *unsafe.Pointer, capRef *int, need int, elemSize uintptr) error { | |||||
| if need <= 0 { | |||||
| return nil | |||||
| } | |||||
| if *ptr != nil && *capRef >= need { | |||||
| return nil | |||||
| } | |||||
| if *ptr != nil { | |||||
| if bridgeCudaFree(*ptr) != 0 { | |||||
| return ErrUnavailable | |||||
| } | |||||
| *ptr = nil | |||||
| *capRef = 0 | |||||
| } | |||||
| if bridgeCudaMalloc(ptr, uintptr(need)*elemSize) != 0 { | |||||
| return ErrUnavailable | |||||
| } | |||||
| *capRef = need | |||||
| return nil | |||||
| } | |||||
| func (r *BatchRunner) syncNativeStreamingStates(active map[int64]struct{}) { | |||||
| if r == nil || r.nativeState == nil { | |||||
| return | |||||
| } | |||||
| for id, state := range r.nativeState { | |||||
| if _, ok := active[id]; ok { | |||||
| continue | |||||
| } | |||||
| releaseNativeStreamingSignalState(state) | |||||
| delete(r.nativeState, id) | |||||
| } | |||||
| } | |||||
| func (r *BatchRunner) resetNativeStreamingState(signalID int64) { | |||||
| if r == nil || r.nativeState == nil { | |||||
| return | |||||
| } | |||||
| if state := r.nativeState[signalID]; state != nil { | |||||
| releaseNativeStreamingSignalState(state) | |||||
| } | |||||
| delete(r.nativeState, signalID) | |||||
| } | |||||
| func (r *BatchRunner) resetAllNativeStreamingStates() { | |||||
| if r == nil { | |||||
| return | |||||
| } | |||||
| r.freeAllNativeStreamingStates() | |||||
| r.nativeState = make(map[int64]*nativeStreamingSignalState) | |||||
| } | |||||
| func (r *BatchRunner) freeAllNativeStreamingStates() { | |||||
| if r == nil || r.nativeState == nil { | |||||
| return | |||||
| } | |||||
| for id, state := range r.nativeState { | |||||
| releaseNativeStreamingSignalState(state) | |||||
| delete(r.nativeState, id) | |||||
| } | |||||
| } | |||||
| func releaseNativeStreamingSignalState(state *nativeStreamingSignalState) { | |||||
| if state == nil { | |||||
| return | |||||
| } | |||||
| for _, ptr := range []*unsafe.Pointer{ | |||||
| &state.dInNew, | |||||
| &state.dShifted, | |||||
| &state.dOut, | |||||
| &state.dTaps, | |||||
| &state.dHistory, | |||||
| &state.dHistoryScratch, | |||||
| } { | |||||
| if *ptr != nil { | |||||
| _ = bridgeCudaFree(*ptr) | |||||
| *ptr = nil | |||||
| } | |||||
| } | |||||
| state.inNewCap = 0 | |||||
| state.shiftedCap = 0 | |||||
| state.outCap = 0 | |||||
| state.tapsLen = 0 | |||||
| state.historyCap = 0 | |||||
| state.historyLen = 0 | |||||
| state.historyScratchCap = 0 | |||||
| state.phaseCount = 0 | |||||
| state.phaseNCO = 0 | |||||
| state.decim = 0 | |||||
| state.numTaps = 0 | |||||
| state.configHash = 0 | |||||
| } | |||||
| func minInt(a int, b int) int { | |||||
| if a < b { | |||||
| return a | |||||
| } | |||||
| return b | |||||
| } | |||||
| @@ -0,0 +1,44 @@ | |||||
| //go:build !cufft || !windows | |||||
| package gpudemod | |||||
| func (r *BatchRunner) executeStreamingGPUNativePrepared(invocations []StreamingGPUInvocation) ([]StreamingGPUExecutionResult, error) { | |||||
| _ = invocations | |||||
| return nil, ErrUnavailable | |||||
| } | |||||
| func (r *BatchRunner) syncNativeStreamingStates(active map[int64]struct{}) { | |||||
| _ = active | |||||
| if r == nil { | |||||
| return | |||||
| } | |||||
| if r.nativeState == nil { | |||||
| r.nativeState = make(map[int64]*nativeStreamingSignalState) | |||||
| } | |||||
| for id := range r.nativeState { | |||||
| if _, ok := active[id]; !ok { | |||||
| delete(r.nativeState, id) | |||||
| } | |||||
| } | |||||
| } | |||||
| func (r *BatchRunner) resetNativeStreamingState(signalID int64) { | |||||
| if r == nil || r.nativeState == nil { | |||||
| return | |||||
| } | |||||
| delete(r.nativeState, signalID) | |||||
| } | |||||
| func (r *BatchRunner) resetAllNativeStreamingStates() { | |||||
| if r == nil { | |||||
| return | |||||
| } | |||||
| r.nativeState = make(map[int64]*nativeStreamingSignalState) | |||||
| } | |||||
| func (r *BatchRunner) freeAllNativeStreamingStates() { | |||||
| if r == nil { | |||||
| return | |||||
| } | |||||
| r.nativeState = nil | |||||
| } | |||||
| @@ -0,0 +1,206 @@ | |||||
| //go:build cufft && windows | |||||
| package gpudemod | |||||
| import ( | |||||
| "os" | |||||
| "path/filepath" | |||||
| "testing" | |||||
| ) | |||||
| func configureNativePreparedDLLPath(t *testing.T) { | |||||
| t.Helper() | |||||
| candidates := []string{ | |||||
| filepath.Join("build", "gpudemod_kernels.dll"), | |||||
| filepath.Join("internal", "demod", "gpudemod", "build", "gpudemod_kernels.dll"), | |||||
| "gpudemod_kernels.dll", | |||||
| } | |||||
| for _, candidate := range candidates { | |||||
| if _, err := os.Stat(candidate); err == nil { | |||||
| abs, err := filepath.Abs(candidate) | |||||
| if err != nil { | |||||
| t.Fatalf("resolve native prepared DLL path: %v", err) | |||||
| } | |||||
| t.Setenv("GPUMOD_DLL", abs) | |||||
| return | |||||
| } | |||||
| } | |||||
| } | |||||
| func requireNativePreparedTestRunner(t *testing.T) *BatchRunner { | |||||
| t.Helper() | |||||
| configureNativePreparedDLLPath(t) | |||||
| if err := ensureDLLLoaded(); err != nil { | |||||
| t.Skipf("native prepared path unavailable: %v", err) | |||||
| } | |||||
| if !Available() { | |||||
| t.Skip("native prepared path unavailable: cuda device not available") | |||||
| } | |||||
| r, err := NewBatchRunner(32768, 4000000) | |||||
| if err != nil { | |||||
| t.Skipf("native prepared path unavailable: %v", err) | |||||
| } | |||||
| t.Cleanup(r.Close) | |||||
| return r | |||||
| } | |||||
| func TestStreamingGPUNativePreparedMatchesCPUOracleAcrossChunkPatterns(t *testing.T) { | |||||
| job := StreamingExtractJob{ | |||||
| SignalID: 1, | |||||
| OffsetHz: 12500, | |||||
| Bandwidth: 20000, | |||||
| OutRate: 200000, | |||||
| NumTaps: 65, | |||||
| ConfigHash: 777, | |||||
| } | |||||
| exec := func(r *BatchRunner, invocations []StreamingGPUInvocation) ([]StreamingGPUExecutionResult, error) { | |||||
| return r.executeStreamingGPUNativePrepared(invocations) | |||||
| } | |||||
| t.Run("DeterministicIQ", func(t *testing.T) { | |||||
| r := requireNativePreparedTestRunner(t) | |||||
| steps := makeStreamingValidationSteps( | |||||
| makeDeterministicIQ(8192), | |||||
| []int{0, 1, 2, 17, 63, 64, 65, 129, 511, 2048}, | |||||
| []StreamingExtractJob{job}, | |||||
| ) | |||||
| runPreparedSequenceAgainstOracle(t, r, exec, steps, 1e-4, 1e-8) | |||||
| }) | |||||
| t.Run("ToneNoiseIQ", func(t *testing.T) { | |||||
| r := requireNativePreparedTestRunner(t) | |||||
| steps := makeStreamingValidationSteps( | |||||
| makeToneNoiseIQ(12288, 0.023), | |||||
| []int{7, 20, 3, 63, 64, 65, 777, 2048, 4096}, | |||||
| []StreamingExtractJob{job}, | |||||
| ) | |||||
| runPreparedSequenceAgainstOracle(t, r, exec, steps, 1e-4, 1e-8) | |||||
| }) | |||||
| } | |||||
| func TestStreamingGPUNativePreparedLifecycleResetAndCapacity(t *testing.T) { | |||||
| r := requireNativePreparedTestRunner(t) | |||||
| exec := func(invocations []StreamingGPUInvocation) ([]StreamingGPUExecutionResult, error) { | |||||
| return r.executeStreamingGPUNativePrepared(invocations) | |||||
| } | |||||
| jobA := StreamingExtractJob{ | |||||
| SignalID: 11, | |||||
| OffsetHz: 12500, | |||||
| Bandwidth: 20000, | |||||
| OutRate: 200000, | |||||
| NumTaps: 65, | |||||
| ConfigHash: 3001, | |||||
| } | |||||
| jobB := StreamingExtractJob{ | |||||
| SignalID: 22, | |||||
| OffsetHz: -18750, | |||||
| Bandwidth: 16000, | |||||
| OutRate: 100000, | |||||
| NumTaps: 33, | |||||
| ConfigHash: 4002, | |||||
| } | |||||
| steps := []streamingValidationStep{ | |||||
| { | |||||
| name: "prime_both_signals", | |||||
| iq: makeDeterministicIQ(256), | |||||
| jobs: []StreamingExtractJob{jobA, jobB}, | |||||
| }, | |||||
| { | |||||
| name: "grow_capacity", | |||||
| iq: makeToneNoiseIQ(4096, 0.037), | |||||
| jobs: []StreamingExtractJob{jobA, jobB}, | |||||
| }, | |||||
| { | |||||
| name: "config_reset_zero_new", | |||||
| iq: nil, | |||||
| jobs: []StreamingExtractJob{{SignalID: jobA.SignalID, OffsetHz: jobA.OffsetHz, Bandwidth: jobA.Bandwidth, OutRate: jobA.OutRate, NumTaps: jobA.NumTaps, ConfigHash: jobA.ConfigHash + 1}, jobB}, | |||||
| }, | |||||
| { | |||||
| name: "signal_b_disappears", | |||||
| iq: makeDeterministicIQ(64), | |||||
| jobs: []StreamingExtractJob{jobA}, | |||||
| }, | |||||
| { | |||||
| name: "signal_b_reappears", | |||||
| iq: makeToneNoiseIQ(96, 0.017), | |||||
| jobs: []StreamingExtractJob{jobA, jobB}, | |||||
| }, | |||||
| { | |||||
| name: "history_boundary", | |||||
| iq: makeDeterministicIQ(65), | |||||
| jobs: []StreamingExtractJob{jobA, jobB}, | |||||
| }, | |||||
| } | |||||
| oracle := NewCPUOracleRunner(r.eng.sampleRate) | |||||
| var grownCap int | |||||
| for idx, step := range steps { | |||||
| invocations, err := r.buildStreamingGPUInvocations(step.iq, step.jobs) | |||||
| if err != nil { | |||||
| t.Fatalf("step %d (%s): build invocations failed: %v", idx, step.name, err) | |||||
| } | |||||
| got, err := exec(invocations) | |||||
| if err != nil { | |||||
| t.Fatalf("step %d (%s): native prepared exec failed: %v", idx, step.name, err) | |||||
| } | |||||
| want, err := oracle.StreamingExtract(step.iq, step.jobs) | |||||
| if err != nil { | |||||
| t.Fatalf("step %d (%s): oracle failed: %v", idx, step.name, err) | |||||
| } | |||||
| if len(got) != len(want) { | |||||
| t.Fatalf("step %d (%s): result count mismatch: got=%d want=%d", idx, step.name, len(got), len(want)) | |||||
| } | |||||
| applied := r.applyStreamingGPUExecutionResults(got) | |||||
| for i, job := range step.jobs { | |||||
| oracleState := oracle.States[job.SignalID] | |||||
| requirePreparedExecutionResultMatchesOracle(t, got[i], want[i], oracleState, 1e-4, 1e-8) | |||||
| requireStreamingExtractResultMatchesOracle(t, applied[i], want[i]) | |||||
| requireExtractStateMatchesOracle(t, r.streamState[job.SignalID], oracleState, 1e-8, 1e-4) | |||||
| state := r.nativeState[job.SignalID] | |||||
| if state == nil { | |||||
| t.Fatalf("step %d (%s): missing native state for signal %d", idx, step.name, job.SignalID) | |||||
| } | |||||
| if state.configHash != job.ConfigHash { | |||||
| t.Fatalf("step %d (%s): native config hash mismatch for signal %d: got=%d want=%d", idx, step.name, job.SignalID, state.configHash, job.ConfigHash) | |||||
| } | |||||
| if state.decim != oracleState.Decim { | |||||
| t.Fatalf("step %d (%s): native decim mismatch for signal %d: got=%d want=%d", idx, step.name, job.SignalID, state.decim, oracleState.Decim) | |||||
| } | |||||
| if state.numTaps != oracleState.NumTaps { | |||||
| t.Fatalf("step %d (%s): native num taps mismatch for signal %d: got=%d want=%d", idx, step.name, job.SignalID, state.numTaps, oracleState.NumTaps) | |||||
| } | |||||
| if state.historyCap != maxInt(0, oracleState.NumTaps-1) { | |||||
| t.Fatalf("step %d (%s): native history cap mismatch for signal %d: got=%d want=%d", idx, step.name, job.SignalID, state.historyCap, maxInt(0, oracleState.NumTaps-1)) | |||||
| } | |||||
| if state.historyLen != len(oracleState.ShiftedHistory) { | |||||
| t.Fatalf("step %d (%s): native history len mismatch for signal %d: got=%d want=%d", idx, step.name, job.SignalID, state.historyLen, len(oracleState.ShiftedHistory)) | |||||
| } | |||||
| if len(step.iq) > 0 && state.shiftedCap < len(step.iq) { | |||||
| t.Fatalf("step %d (%s): native shifted capacity too small for signal %d: got=%d need>=%d", idx, step.name, job.SignalID, state.shiftedCap, len(step.iq)) | |||||
| } | |||||
| if state.outCap < got[i].NOut { | |||||
| t.Fatalf("step %d (%s): native out capacity too small for signal %d: got=%d need>=%d", idx, step.name, job.SignalID, state.outCap, got[i].NOut) | |||||
| } | |||||
| if job.SignalID == jobA.SignalID && state.shiftedCap > grownCap { | |||||
| grownCap = state.shiftedCap | |||||
| } | |||||
| } | |||||
| if step.name == "grow_capacity" && grownCap < len(step.iq) { | |||||
| t.Fatalf("expected capacity growth for signal %d, got=%d want>=%d", jobA.SignalID, grownCap, len(step.iq)) | |||||
| } | |||||
| if step.name == "config_reset_zero_new" { | |||||
| state := r.nativeState[jobA.SignalID] | |||||
| if state == nil { | |||||
| t.Fatalf("missing native state for signal %d after config reset", jobA.SignalID) | |||||
| } | |||||
| if state.historyLen != 0 { | |||||
| t.Fatalf("expected cleared native history after config reset, got=%d", state.historyLen) | |||||
| } | |||||
| } | |||||
| if step.name == "signal_b_disappears" { | |||||
| if _, ok := r.nativeState[jobB.SignalID]; ok { | |||||
| t.Fatalf("expected native state for signal %d to be removed on disappearance", jobB.SignalID) | |||||
| } | |||||
| } | |||||
| } | |||||
| } | |||||
| @@ -0,0 +1,28 @@ | |||||
| package gpudemod | |||||
| import "unsafe" | |||||
| type nativeStreamingSignalState struct { | |||||
| signalID int64 | |||||
| configHash uint64 | |||||
| decim int | |||||
| numTaps int | |||||
| dInNew unsafe.Pointer | |||||
| dShifted unsafe.Pointer | |||||
| dOut unsafe.Pointer | |||||
| dTaps unsafe.Pointer | |||||
| dHistory unsafe.Pointer | |||||
| dHistoryScratch unsafe.Pointer | |||||
| inNewCap int | |||||
| shiftedCap int | |||||
| outCap int | |||||
| tapsLen int | |||||
| historyCap int | |||||
| historyLen int | |||||
| historyScratchCap int | |||||
| phaseCount int | |||||
| phaseNCO float64 | |||||
| } | |||||
| @@ -0,0 +1,61 @@ | |||||
| package gpudemod | |||||
| func (r *BatchRunner) buildStreamingGPUInvocations(iqNew []complex64, jobs []StreamingExtractJob) ([]StreamingGPUInvocation, error) { | |||||
| if r == nil || r.eng == nil { | |||||
| return nil, ErrUnavailable | |||||
| } | |||||
| invocations := make([]StreamingGPUInvocation, len(jobs)) | |||||
| active := make(map[int64]struct{}, len(jobs)) | |||||
| for i, job := range jobs { | |||||
| active[job.SignalID] = struct{}{} | |||||
| state, err := r.getOrInitExtractState(job, r.eng.sampleRate) | |||||
| if err != nil { | |||||
| return nil, err | |||||
| } | |||||
| invocations[i] = StreamingGPUInvocation{ | |||||
| SignalID: job.SignalID, | |||||
| ConfigHash: state.ConfigHash, | |||||
| OffsetHz: job.OffsetHz, | |||||
| OutRate: job.OutRate, | |||||
| Bandwidth: job.Bandwidth, | |||||
| SampleRate: r.eng.sampleRate, | |||||
| NumTaps: state.NumTaps, | |||||
| Decim: state.Decim, | |||||
| PhaseCountIn: state.PhaseCount, | |||||
| NCOPhaseIn: state.NCOPhase, | |||||
| HistoryLen: len(state.ShiftedHistory), | |||||
| BaseTaps: append([]float32(nil), state.BaseTaps...), | |||||
| PolyphaseTaps: append([]float32(nil), state.PolyphaseTaps...), | |||||
| ShiftedHistory: append([]complex64(nil), state.ShiftedHistory...), | |||||
| IQNew: iqNew, | |||||
| } | |||||
| } | |||||
| for signalID := range r.streamState { | |||||
| if _, ok := active[signalID]; !ok { | |||||
| delete(r.streamState, signalID) | |||||
| } | |||||
| } | |||||
| r.syncNativeStreamingStates(active) | |||||
| return invocations, nil | |||||
| } | |||||
| func (r *BatchRunner) applyStreamingGPUExecutionResults(results []StreamingGPUExecutionResult) []StreamingExtractResult { | |||||
| out := make([]StreamingExtractResult, len(results)) | |||||
| for i, res := range results { | |||||
| state := r.streamState[res.SignalID] | |||||
| if state != nil { | |||||
| state.NCOPhase = res.NCOPhaseOut | |||||
| state.PhaseCount = res.PhaseCountOut | |||||
| state.ShiftedHistory = append(state.ShiftedHistory[:0], res.HistoryOut...) | |||||
| } | |||||
| out[i] = StreamingExtractResult{ | |||||
| SignalID: res.SignalID, | |||||
| IQ: res.IQ, | |||||
| Rate: res.Rate, | |||||
| NOut: res.NOut, | |||||
| PhaseCount: res.PhaseCountOut, | |||||
| HistoryLen: res.HistoryLenOut, | |||||
| } | |||||
| } | |||||
| return out | |||||
| } | |||||
| @@ -0,0 +1,26 @@ | |||||
| package gpudemod | |||||
| func updateShiftedHistory(prev []complex64, shiftedNew []complex64, numTaps int) []complex64 { | |||||
| need := numTaps - 1 | |||||
| if need <= 0 { | |||||
| return nil | |||||
| } | |||||
| combined := append(append(make([]complex64, 0, len(prev)+len(shiftedNew)), prev...), shiftedNew...) | |||||
| if len(combined) <= need { | |||||
| out := make([]complex64, len(combined)) | |||||
| copy(out, combined) | |||||
| return out | |||||
| } | |||||
| out := make([]complex64, need) | |||||
| copy(out, combined[len(combined)-need:]) | |||||
| return out | |||||
| } | |||||
| // StreamingExtractGPU is the production entry point for the stateful streaming | |||||
| // extractor path. Execution strategy is selected by StreamingExtractGPUExec. | |||||
| func (r *BatchRunner) StreamingExtractGPU(iqNew []complex64, jobs []StreamingExtractJob) ([]StreamingExtractResult, error) { | |||||
| if r == nil || r.eng == nil { | |||||
| return nil, ErrUnavailable | |||||
| } | |||||
| return r.StreamingExtractGPUExec(iqNew, jobs) | |||||
| } | |||||
| @@ -0,0 +1,59 @@ | |||||
| package gpudemod | |||||
| import "testing" | |||||
| func TestStreamingGPUUsesSafeProductionDefault(t *testing.T) { | |||||
| r := &BatchRunner{eng: &Engine{sampleRate: 4000000}, streamState: make(map[int64]*ExtractStreamState)} | |||||
| job := StreamingExtractJob{ | |||||
| SignalID: 1, | |||||
| OffsetHz: 12500, | |||||
| Bandwidth: 20000, | |||||
| OutRate: 200000, | |||||
| NumTaps: 65, | |||||
| ConfigHash: 777, | |||||
| } | |||||
| iq := makeDeterministicIQ(1000) | |||||
| results, err := r.StreamingExtractGPU(iq, []StreamingExtractJob{job}) | |||||
| if err != nil { | |||||
| t.Fatalf("expected safe production default path, got error: %v", err) | |||||
| } | |||||
| if len(results) != 1 { | |||||
| t.Fatalf("expected 1 result, got %d", len(results)) | |||||
| } | |||||
| if results[0].NOut == 0 { | |||||
| t.Fatalf("expected non-zero output count from safe production path") | |||||
| } | |||||
| } | |||||
| func TestStreamingGPUHostOracleAdvancesState(t *testing.T) { | |||||
| r := &BatchRunner{eng: &Engine{sampleRate: 4000000}, streamState: make(map[int64]*ExtractStreamState)} | |||||
| job := StreamingExtractJob{ | |||||
| SignalID: 1, | |||||
| OffsetHz: 12500, | |||||
| Bandwidth: 20000, | |||||
| OutRate: 200000, | |||||
| NumTaps: 65, | |||||
| ConfigHash: 777, | |||||
| } | |||||
| iq := makeDeterministicIQ(1000) | |||||
| results, err := r.StreamingExtractGPUHostOracle(iq, []StreamingExtractJob{job}) | |||||
| if err != nil { | |||||
| t.Fatalf("unexpected host-oracle error: %v", err) | |||||
| } | |||||
| if len(results) != 1 { | |||||
| t.Fatalf("expected 1 result, got %d", len(results)) | |||||
| } | |||||
| state := r.streamState[1] | |||||
| if state == nil { | |||||
| t.Fatalf("expected state to be initialized") | |||||
| } | |||||
| if state.NCOPhase == 0 { | |||||
| t.Fatalf("expected phase to advance") | |||||
| } | |||||
| if len(state.ShiftedHistory) == 0 { | |||||
| t.Fatalf("expected shifted history to be updated") | |||||
| } | |||||
| if results[0].NOut == 0 { | |||||
| t.Fatalf("expected non-zero output count from host oracle path") | |||||
| } | |||||
| } | |||||
| @@ -0,0 +1,213 @@ | |||||
| package gpudemod | |||||
| import ( | |||||
| "math" | |||||
| "testing" | |||||
| ) | |||||
| type streamingValidationStep struct { | |||||
| name string | |||||
| iq []complex64 | |||||
| jobs []StreamingExtractJob | |||||
| } | |||||
| type streamingPreparedExecutor func(*BatchRunner, []StreamingGPUInvocation) ([]StreamingGPUExecutionResult, error) | |||||
| func makeToneNoiseIQ(n int, phaseInc float64) []complex64 { | |||||
| out := make([]complex64, n) | |||||
| phase := 0.0 | |||||
| for i := 0; i < n; i++ { | |||||
| tone := complex(math.Cos(phase), math.Sin(phase)) | |||||
| noiseI := 0.17*math.Cos(0.113*float64(i)+0.31) + 0.07*math.Sin(0.071*float64(i)) | |||||
| noiseQ := 0.13*math.Sin(0.097*float64(i)+0.11) - 0.05*math.Cos(0.043*float64(i)) | |||||
| out[i] = complex64(0.85*tone + 0.15*complex(noiseI, noiseQ)) | |||||
| phase += phaseInc | |||||
| } | |||||
| return out | |||||
| } | |||||
| func makeStreamingValidationSteps(iq []complex64, chunkSizes []int, jobs []StreamingExtractJob) []streamingValidationStep { | |||||
| steps := make([]streamingValidationStep, 0, len(chunkSizes)+1) | |||||
| pos := 0 | |||||
| for idx, n := range chunkSizes { | |||||
| if n < 0 { | |||||
| n = 0 | |||||
| } | |||||
| end := pos + n | |||||
| if end > len(iq) { | |||||
| end = len(iq) | |||||
| } | |||||
| steps = append(steps, streamingValidationStep{ | |||||
| name: "chunk", | |||||
| iq: append([]complex64(nil), iq[pos:end]...), | |||||
| jobs: append([]StreamingExtractJob(nil), jobs...), | |||||
| }) | |||||
| _ = idx | |||||
| pos = end | |||||
| } | |||||
| if pos < len(iq) { | |||||
| steps = append(steps, streamingValidationStep{ | |||||
| name: "remainder", | |||||
| iq: append([]complex64(nil), iq[pos:]...), | |||||
| jobs: append([]StreamingExtractJob(nil), jobs...), | |||||
| }) | |||||
| } | |||||
| return steps | |||||
| } | |||||
| func requirePhaseClose(t *testing.T, got float64, want float64, tol float64) { | |||||
| t.Helper() | |||||
| diff := got - want | |||||
| for diff > math.Pi { | |||||
| diff -= 2 * math.Pi | |||||
| } | |||||
| for diff < -math.Pi { | |||||
| diff += 2 * math.Pi | |||||
| } | |||||
| if math.Abs(diff) > tol { | |||||
| t.Fatalf("phase mismatch: got=%0.12f want=%0.12f diff=%0.12f tol=%0.12f", got, want, diff, tol) | |||||
| } | |||||
| } | |||||
| func requireStreamingExtractResultMatchesOracle(t *testing.T, got StreamingExtractResult, want StreamingExtractResult) { | |||||
| t.Helper() | |||||
| if got.SignalID != want.SignalID { | |||||
| t.Fatalf("signal id mismatch: got=%d want=%d", got.SignalID, want.SignalID) | |||||
| } | |||||
| if got.Rate != want.Rate { | |||||
| t.Fatalf("rate mismatch for signal %d: got=%d want=%d", got.SignalID, got.Rate, want.Rate) | |||||
| } | |||||
| if got.NOut != want.NOut { | |||||
| t.Fatalf("n_out mismatch for signal %d: got=%d want=%d", got.SignalID, got.NOut, want.NOut) | |||||
| } | |||||
| if got.PhaseCount != want.PhaseCount { | |||||
| t.Fatalf("phase count mismatch for signal %d: got=%d want=%d", got.SignalID, got.PhaseCount, want.PhaseCount) | |||||
| } | |||||
| if got.HistoryLen != want.HistoryLen { | |||||
| t.Fatalf("history len mismatch for signal %d: got=%d want=%d", got.SignalID, got.HistoryLen, want.HistoryLen) | |||||
| } | |||||
| } | |||||
| func requirePreparedExecutionResultMatchesOracle(t *testing.T, got StreamingGPUExecutionResult, want StreamingExtractResult, oracleState *CPUOracleState, sampleTol float64, phaseTol float64) { | |||||
| t.Helper() | |||||
| if oracleState == nil { | |||||
| t.Fatalf("missing oracle state for signal %d", got.SignalID) | |||||
| } | |||||
| if got.SignalID != want.SignalID { | |||||
| t.Fatalf("signal id mismatch: got=%d want=%d", got.SignalID, want.SignalID) | |||||
| } | |||||
| if got.Rate != want.Rate { | |||||
| t.Fatalf("rate mismatch for signal %d: got=%d want=%d", got.SignalID, got.Rate, want.Rate) | |||||
| } | |||||
| if got.NOut != want.NOut { | |||||
| t.Fatalf("n_out mismatch for signal %d: got=%d want=%d", got.SignalID, got.NOut, want.NOut) | |||||
| } | |||||
| if got.PhaseCountOut != oracleState.PhaseCount { | |||||
| t.Fatalf("phase count mismatch for signal %d: got=%d want=%d", got.SignalID, got.PhaseCountOut, oracleState.PhaseCount) | |||||
| } | |||||
| requirePhaseClose(t, got.NCOPhaseOut, oracleState.NCOPhase, phaseTol) | |||||
| if got.HistoryLenOut != len(oracleState.ShiftedHistory) { | |||||
| t.Fatalf("history len mismatch for signal %d: got=%d want=%d", got.SignalID, got.HistoryLenOut, len(oracleState.ShiftedHistory)) | |||||
| } | |||||
| requireComplexSlicesClose(t, got.IQ, want.IQ, sampleTol) | |||||
| requireComplexSlicesClose(t, got.HistoryOut, oracleState.ShiftedHistory, sampleTol) | |||||
| } | |||||
| func requireExtractStateMatchesOracle(t *testing.T, got *ExtractStreamState, want *CPUOracleState, phaseTol float64, sampleTol float64) { | |||||
| t.Helper() | |||||
| if got == nil || want == nil { | |||||
| t.Fatalf("state mismatch: got nil=%t want nil=%t", got == nil, want == nil) | |||||
| } | |||||
| if got.SignalID != want.SignalID { | |||||
| t.Fatalf("signal id mismatch: got=%d want=%d", got.SignalID, want.SignalID) | |||||
| } | |||||
| if got.ConfigHash != want.ConfigHash { | |||||
| t.Fatalf("config hash mismatch for signal %d: got=%d want=%d", got.SignalID, got.ConfigHash, want.ConfigHash) | |||||
| } | |||||
| if got.Decim != want.Decim { | |||||
| t.Fatalf("decim mismatch for signal %d: got=%d want=%d", got.SignalID, got.Decim, want.Decim) | |||||
| } | |||||
| if got.NumTaps != want.NumTaps { | |||||
| t.Fatalf("num taps mismatch for signal %d: got=%d want=%d", got.SignalID, got.NumTaps, want.NumTaps) | |||||
| } | |||||
| if got.PhaseCount != want.PhaseCount { | |||||
| t.Fatalf("phase count mismatch for signal %d: got=%d want=%d", got.SignalID, got.PhaseCount, want.PhaseCount) | |||||
| } | |||||
| requirePhaseClose(t, got.NCOPhase, want.NCOPhase, phaseTol) | |||||
| requireComplexSlicesClose(t, got.ShiftedHistory, want.ShiftedHistory, sampleTol) | |||||
| } | |||||
| func requireStateKeysMatchOracle(t *testing.T, got map[int64]*ExtractStreamState, want map[int64]*CPUOracleState) { | |||||
| t.Helper() | |||||
| if len(got) != len(want) { | |||||
| t.Fatalf("active state count mismatch: got=%d want=%d", len(got), len(want)) | |||||
| } | |||||
| for signalID := range want { | |||||
| if got[signalID] == nil { | |||||
| t.Fatalf("missing active state for signal %d", signalID) | |||||
| } | |||||
| } | |||||
| for signalID := range got { | |||||
| if want[signalID] == nil { | |||||
| t.Fatalf("unexpected active state for signal %d", signalID) | |||||
| } | |||||
| } | |||||
| } | |||||
| func runStreamingExecSequenceAgainstOracle(t *testing.T, runner *BatchRunner, steps []streamingValidationStep, sampleTol float64, phaseTol float64) { | |||||
| t.Helper() | |||||
| oracle := NewCPUOracleRunner(runner.eng.sampleRate) | |||||
| for idx, step := range steps { | |||||
| got, err := runner.StreamingExtractGPUExec(step.iq, step.jobs) | |||||
| if err != nil { | |||||
| t.Fatalf("step %d (%s): exec failed: %v", idx, step.name, err) | |||||
| } | |||||
| want, err := oracle.StreamingExtract(step.iq, step.jobs) | |||||
| if err != nil { | |||||
| t.Fatalf("step %d (%s): oracle failed: %v", idx, step.name, err) | |||||
| } | |||||
| if len(got) != len(want) { | |||||
| t.Fatalf("step %d (%s): result count mismatch: got=%d want=%d", idx, step.name, len(got), len(want)) | |||||
| } | |||||
| for i, job := range step.jobs { | |||||
| requireStreamingExtractResultMatchesOracle(t, got[i], want[i]) | |||||
| requireComplexSlicesClose(t, got[i].IQ, want[i].IQ, sampleTol) | |||||
| requireExtractStateMatchesOracle(t, runner.streamState[job.SignalID], oracle.States[job.SignalID], phaseTol, sampleTol) | |||||
| } | |||||
| requireStateKeysMatchOracle(t, runner.streamState, oracle.States) | |||||
| } | |||||
| } | |||||
| func runPreparedSequenceAgainstOracle(t *testing.T, runner *BatchRunner, exec streamingPreparedExecutor, steps []streamingValidationStep, sampleTol float64, phaseTol float64) { | |||||
| t.Helper() | |||||
| oracle := NewCPUOracleRunner(runner.eng.sampleRate) | |||||
| for idx, step := range steps { | |||||
| invocations, err := runner.buildStreamingGPUInvocations(step.iq, step.jobs) | |||||
| if err != nil { | |||||
| t.Fatalf("step %d (%s): build invocations failed: %v", idx, step.name, err) | |||||
| } | |||||
| got, err := exec(runner, invocations) | |||||
| if err != nil { | |||||
| t.Fatalf("step %d (%s): prepared exec failed: %v", idx, step.name, err) | |||||
| } | |||||
| want, err := oracle.StreamingExtract(step.iq, step.jobs) | |||||
| if err != nil { | |||||
| t.Fatalf("step %d (%s): oracle failed: %v", idx, step.name, err) | |||||
| } | |||||
| if len(got) != len(want) { | |||||
| t.Fatalf("step %d (%s): result count mismatch: got=%d want=%d", idx, step.name, len(got), len(want)) | |||||
| } | |||||
| applied := runner.applyStreamingGPUExecutionResults(got) | |||||
| if len(applied) != len(want) { | |||||
| t.Fatalf("step %d (%s): applied result count mismatch: got=%d want=%d", idx, step.name, len(applied), len(want)) | |||||
| } | |||||
| for i, job := range step.jobs { | |||||
| oracleState := oracle.States[job.SignalID] | |||||
| requirePreparedExecutionResultMatchesOracle(t, got[i], want[i], oracleState, sampleTol, phaseTol) | |||||
| requireStreamingExtractResultMatchesOracle(t, applied[i], want[i]) | |||||
| requireComplexSlicesClose(t, applied[i].IQ, want[i].IQ, sampleTol) | |||||
| requireExtractStateMatchesOracle(t, runner.streamState[job.SignalID], oracleState, phaseTol, sampleTol) | |||||
| } | |||||
| requireStateKeysMatchOracle(t, runner.streamState, oracle.States) | |||||
| } | |||||
| } | |||||
| @@ -0,0 +1,64 @@ | |||||
| package gpudemod | |||||
| import "math" | |||||
| func runStreamingPolyphaseHostCore( | |||||
| iqNew []complex64, | |||||
| sampleRate int, | |||||
| offsetHz float64, | |||||
| stateNCOPhase float64, | |||||
| statePhaseCount int, | |||||
| stateNumTaps int, | |||||
| stateDecim int, | |||||
| stateHistory []complex64, | |||||
| polyphaseTaps []float32, | |||||
| ) ([]complex64, float64, int, []complex64) { | |||||
| out := make([]complex64, 0, len(iqNew)/maxInt(1, stateDecim)+2) | |||||
| phase := stateNCOPhase | |||||
| phaseCount := statePhaseCount | |||||
| hist := append([]complex64(nil), stateHistory...) | |||||
| phaseLen := PolyphasePhaseLen(len(polyphaseTaps)/maxInt(1, stateDecim)*maxInt(1, stateDecim), stateDecim) | |||||
| if phaseLen == 0 { | |||||
| phaseLen = PolyphasePhaseLen(len(polyphaseTaps), stateDecim) | |||||
| } | |||||
| phaseInc := -2.0 * math.Pi * offsetHz / float64(sampleRate) | |||||
| for _, x := range iqNew { | |||||
| rot := complex64(complex(math.Cos(phase), math.Sin(phase))) | |||||
| s := x * rot | |||||
| hist = append(hist, s) | |||||
| phaseCount++ | |||||
| if phaseCount == stateDecim { | |||||
| var y complex64 | |||||
| for p := 0; p < stateDecim; p++ { | |||||
| for k := 0; k < phaseLen; k++ { | |||||
| idxTap := p*phaseLen + k | |||||
| if idxTap >= len(polyphaseTaps) { | |||||
| continue | |||||
| } | |||||
| tap := polyphaseTaps[idxTap] | |||||
| if tap == 0 { | |||||
| continue | |||||
| } | |||||
| srcBack := p + k*stateDecim | |||||
| idx := len(hist) - 1 - srcBack | |||||
| if idx < 0 { | |||||
| continue | |||||
| } | |||||
| y += complex(tap, 0) * hist[idx] | |||||
| } | |||||
| } | |||||
| out = append(out, y) | |||||
| phaseCount = 0 | |||||
| } | |||||
| if len(hist) > stateNumTaps-1 { | |||||
| hist = hist[len(hist)-(stateNumTaps-1):] | |||||
| } | |||||
| phase += phaseInc | |||||
| if phase >= math.Pi { | |||||
| phase -= 2 * math.Pi | |||||
| } else if phase < -math.Pi { | |||||
| phase += 2 * math.Pi | |||||
| } | |||||
| } | |||||
| return out, phase, phaseCount, append([]complex64(nil), hist...) | |||||
| } | |||||
| @@ -0,0 +1,40 @@ | |||||
| package gpudemod | |||||
| import "testing" | |||||
| func TestRunStreamingPolyphaseHostCoreMatchesCPUOraclePolyphase(t *testing.T) { | |||||
| cfg := OracleHarnessConfig{ | |||||
| SignalID: 1, | |||||
| ConfigHash: 123, | |||||
| NCOPhase: 0, | |||||
| Decim: 20, | |||||
| NumTaps: 65, | |||||
| PhaseInc: 0.017, | |||||
| } | |||||
| state := MakeCPUOracleState(cfg) | |||||
| iq := MakeDeterministicIQ(12000) | |||||
| oracle := CPUOracleExtractPolyphase(iq, state, cfg.PhaseInc) | |||||
| state2 := MakeCPUOracleState(cfg) | |||||
| out, phase, phaseCount, hist := runStreamingPolyphaseHostCore( | |||||
| iq, | |||||
| 4000000, | |||||
| -cfg.PhaseInc*4000000/(2*3.141592653589793), | |||||
| state2.NCOPhase, | |||||
| state2.PhaseCount, | |||||
| state2.NumTaps, | |||||
| state2.Decim, | |||||
| state2.ShiftedHistory, | |||||
| state2.PolyphaseTaps, | |||||
| ) | |||||
| requireComplexSlicesClose(t, oracle, out, 1e-5) | |||||
| if phase == 0 && len(iq) > 0 { | |||||
| t.Fatalf("expected phase to advance") | |||||
| } | |||||
| if phaseCount < 0 || phaseCount >= state2.Decim { | |||||
| t.Fatalf("unexpected phaseCount: %d", phaseCount) | |||||
| } | |||||
| if len(hist) == 0 { | |||||
| t.Fatalf("expected history to be retained") | |||||
| } | |||||
| } | |||||
| @@ -0,0 +1,111 @@ | |||||
| package gpudemod | |||||
| import ( | |||||
| "fmt" | |||||
| "sdr-wideband-suite/internal/dsp" | |||||
| ) | |||||
| type CPUOracleRunner struct { | |||||
| SampleRate int | |||||
| States map[int64]*CPUOracleState | |||||
| } | |||||
| func (r *CPUOracleRunner) ResetAllStates() { | |||||
| if r == nil { | |||||
| return | |||||
| } | |||||
| r.States = make(map[int64]*CPUOracleState) | |||||
| } | |||||
| func NewCPUOracleRunner(sampleRate int) *CPUOracleRunner { | |||||
| return &CPUOracleRunner{ | |||||
| SampleRate: sampleRate, | |||||
| States: make(map[int64]*CPUOracleState), | |||||
| } | |||||
| } | |||||
| func (r *CPUOracleRunner) ResetSignalState(signalID int64) { | |||||
| if r == nil || r.States == nil { | |||||
| return | |||||
| } | |||||
| delete(r.States, signalID) | |||||
| } | |||||
| func (r *CPUOracleRunner) getOrInitState(job StreamingExtractJob) (*CPUOracleState, error) { | |||||
| if r == nil { | |||||
| return nil, fmt.Errorf("nil CPUOracleRunner") | |||||
| } | |||||
| if r.States == nil { | |||||
| r.States = make(map[int64]*CPUOracleState) | |||||
| } | |||||
| decim, err := ExactIntegerDecimation(r.SampleRate, job.OutRate) | |||||
| if err != nil { | |||||
| return nil, err | |||||
| } | |||||
| state := r.States[job.SignalID] | |||||
| if state == nil { | |||||
| state = &CPUOracleState{SignalID: job.SignalID} | |||||
| r.States[job.SignalID] = state | |||||
| } | |||||
| ResetCPUOracleStateIfConfigChanged(state, job.ConfigHash) | |||||
| state.Decim = decim | |||||
| state.NumTaps = job.NumTaps | |||||
| if state.NumTaps <= 0 { | |||||
| state.NumTaps = 101 | |||||
| } | |||||
| cutoff := job.Bandwidth / 2 | |||||
| if cutoff < 200 { | |||||
| cutoff = 200 | |||||
| } | |||||
| base := dsp.LowpassFIR(cutoff, r.SampleRate, state.NumTaps) | |||||
| state.BaseTaps = make([]float32, len(base)) | |||||
| for i, v := range base { | |||||
| state.BaseTaps[i] = float32(v) | |||||
| } | |||||
| state.PolyphaseTaps = BuildPolyphaseTapsPhaseMajor(state.BaseTaps, state.Decim) | |||||
| if state.ShiftedHistory == nil { | |||||
| state.ShiftedHistory = make([]complex64, 0, maxInt(0, state.NumTaps-1)) | |||||
| } | |||||
| return state, nil | |||||
| } | |||||
| func (r *CPUOracleRunner) StreamingExtract(iqNew []complex64, jobs []StreamingExtractJob) ([]StreamingExtractResult, error) { | |||||
| results := make([]StreamingExtractResult, len(jobs)) | |||||
| active := make(map[int64]struct{}, len(jobs)) | |||||
| for i, job := range jobs { | |||||
| active[job.SignalID] = struct{}{} | |||||
| state, err := r.getOrInitState(job) | |||||
| if err != nil { | |||||
| return nil, err | |||||
| } | |||||
| out, phase, phaseCount, hist := runStreamingPolyphaseHostCore( | |||||
| iqNew, | |||||
| r.SampleRate, | |||||
| job.OffsetHz, | |||||
| state.NCOPhase, | |||||
| state.PhaseCount, | |||||
| state.NumTaps, | |||||
| state.Decim, | |||||
| state.ShiftedHistory, | |||||
| state.PolyphaseTaps, | |||||
| ) | |||||
| state.NCOPhase = phase | |||||
| state.PhaseCount = phaseCount | |||||
| state.ShiftedHistory = append(state.ShiftedHistory[:0], hist...) | |||||
| results[i] = StreamingExtractResult{ | |||||
| SignalID: job.SignalID, | |||||
| IQ: out, | |||||
| Rate: job.OutRate, | |||||
| NOut: len(out), | |||||
| PhaseCount: state.PhaseCount, | |||||
| HistoryLen: len(state.ShiftedHistory), | |||||
| } | |||||
| } | |||||
| for signalID := range r.States { | |||||
| if _, ok := active[signalID]; !ok { | |||||
| delete(r.States, signalID) | |||||
| } | |||||
| } | |||||
| return results, nil | |||||
| } | |||||
| @@ -0,0 +1,64 @@ | |||||
| package gpudemod | |||||
| import ( | |||||
| "fmt" | |||||
| "hash/fnv" | |||||
| ) | |||||
| type StreamingExtractJob struct { | |||||
| SignalID int64 | |||||
| OffsetHz float64 | |||||
| Bandwidth float64 | |||||
| OutRate int | |||||
| NumTaps int | |||||
| ConfigHash uint64 | |||||
| } | |||||
| type StreamingExtractResult struct { | |||||
| SignalID int64 | |||||
| IQ []complex64 | |||||
| Rate int | |||||
| NOut int | |||||
| PhaseCount int | |||||
| HistoryLen int | |||||
| } | |||||
| type ExtractStreamState struct { | |||||
| SignalID int64 | |||||
| ConfigHash uint64 | |||||
| NCOPhase float64 | |||||
| Decim int | |||||
| PhaseCount int | |||||
| NumTaps int | |||||
| ShiftedHistory []complex64 | |||||
| BaseTaps []float32 | |||||
| PolyphaseTaps []float32 | |||||
| Initialized bool | |||||
| } | |||||
| func ResetExtractStreamState(state *ExtractStreamState, cfgHash uint64) { | |||||
| if state == nil { | |||||
| return | |||||
| } | |||||
| state.ConfigHash = cfgHash | |||||
| state.NCOPhase = 0 | |||||
| state.PhaseCount = 0 | |||||
| state.ShiftedHistory = state.ShiftedHistory[:0] | |||||
| state.Initialized = false | |||||
| } | |||||
| func StreamingConfigHash(signalID int64, offsetHz float64, bandwidth float64, outRate int, numTaps int, sampleRate int) uint64 { | |||||
| // Hash only structural parameters that change the FIR/decimation geometry. | |||||
| // Offset is NOT included because the NCO phase_inc tracks it smoothly each frame. | |||||
| // Bandwidth is NOT included because taps are rebuilt every frame in getOrInitExtractState. | |||||
| // A state reset (zeroing NCO phase, history, phase count) is only needed when | |||||
| // decimation factor, tap count, or sample rate changes — all of which affect | |||||
| // buffer sizes and polyphase structure. | |||||
| // | |||||
| // Previous bug: offset and bandwidth were formatted at %.9f precision, causing | |||||
| // a new hash (and full state reset) every single frame because the detector's | |||||
| // exponential smoothing changes CenterHz by sub-Hz fractions each frame. | |||||
| h := fnv.New64a() | |||||
| _, _ = h.Write([]byte(fmt.Sprintf("sig=%d|out=%d|taps=%d|sr=%d", signalID, outRate, numTaps, sampleRate))) | |||||
| return h.Sum64() | |||||
| } | |||||
| @@ -0,0 +1,78 @@ | |||||
| package gpudemod | |||||
| import ( | |||||
| "math" | |||||
| ) | |||||
| type OracleHarnessConfig struct { | |||||
| SignalID int64 | |||||
| ConfigHash uint64 | |||||
| NCOPhase float64 | |||||
| Decim int | |||||
| NumTaps int | |||||
| PhaseInc float64 | |||||
| } | |||||
| func MakeDeterministicIQ(n int) []complex64 { | |||||
| out := make([]complex64, n) | |||||
| for i := 0; i < n; i++ { | |||||
| a := 0.017 * float64(i) | |||||
| b := 0.031 * float64(i) | |||||
| out[i] = complex64(complex(math.Cos(a)+0.2*math.Cos(b), math.Sin(a)+0.15*math.Sin(b))) | |||||
| } | |||||
| return out | |||||
| } | |||||
| func MakeToneIQ(n int, phaseInc float64) []complex64 { | |||||
| out := make([]complex64, n) | |||||
| phase := 0.0 | |||||
| for i := 0; i < n; i++ { | |||||
| out[i] = complex64(complex(math.Cos(phase), math.Sin(phase))) | |||||
| phase += phaseInc | |||||
| } | |||||
| return out | |||||
| } | |||||
| func MakeLowpassTaps(n int) []float32 { | |||||
| out := make([]float32, n) | |||||
| for i := range out { | |||||
| out[i] = 1.0 / float32(n) | |||||
| } | |||||
| return out | |||||
| } | |||||
| func MakeCPUOracleState(cfg OracleHarnessConfig) *CPUOracleState { | |||||
| taps := MakeLowpassTaps(cfg.NumTaps) | |||||
| return &CPUOracleState{ | |||||
| SignalID: cfg.SignalID, | |||||
| ConfigHash: cfg.ConfigHash, | |||||
| NCOPhase: cfg.NCOPhase, | |||||
| Decim: cfg.Decim, | |||||
| PhaseCount: 0, | |||||
| NumTaps: cfg.NumTaps, | |||||
| ShiftedHistory: make([]complex64, 0, maxInt(0, cfg.NumTaps-1)), | |||||
| BaseTaps: taps, | |||||
| PolyphaseTaps: BuildPolyphaseTapsPhaseMajor(taps, cfg.Decim), | |||||
| } | |||||
| } | |||||
| func RunChunkedCPUOraclePolyphase(all []complex64, chunkSizes []int, mkState func() *CPUOracleState, phaseInc float64) []complex64 { | |||||
| state := mkState() | |||||
| out := make([]complex64, 0) | |||||
| pos := 0 | |||||
| for _, n := range chunkSizes { | |||||
| if pos >= len(all) { | |||||
| break | |||||
| } | |||||
| end := pos + n | |||||
| if end > len(all) { | |||||
| end = len(all) | |||||
| } | |||||
| out = append(out, CPUOracleExtractPolyphase(all[pos:end], state, phaseInc)...) | |||||
| pos = end | |||||
| } | |||||
| if pos < len(all) { | |||||
| out = append(out, CPUOracleExtractPolyphase(all[pos:], state, phaseInc)...) | |||||
| } | |||||
| return out | |||||
| } | |||||
| @@ -0,0 +1,39 @@ | |||||
| package gpudemod | |||||
| import "testing" | |||||
| func requireComplexSlicesCloseHarness(t *testing.T, a []complex64, b []complex64, tol float64) { | |||||
| t.Helper() | |||||
| if len(a) != len(b) { | |||||
| t.Fatalf("length mismatch: %d vs %d", len(a), len(b)) | |||||
| } | |||||
| for i := range a { | |||||
| d := CompareComplexSlices([]complex64{a[i]}, []complex64{b[i]}) | |||||
| if d.MaxAbsErr > tol { | |||||
| t.Fatalf("slice mismatch at %d: %v vs %v (tol=%f)", i, a[i], b[i], tol) | |||||
| } | |||||
| } | |||||
| } | |||||
| func TestHarnessChunkedCPUOraclePolyphase(t *testing.T) { | |||||
| cfg := OracleHarnessConfig{ | |||||
| SignalID: 1, | |||||
| ConfigHash: 123, | |||||
| NCOPhase: 0, | |||||
| Decim: 20, | |||||
| NumTaps: 65, | |||||
| PhaseInc: 0.017, | |||||
| } | |||||
| iq := MakeDeterministicIQ(150000) | |||||
| mk := func() *CPUOracleState { return MakeCPUOracleState(cfg) } | |||||
| mono := CPUOracleExtractPolyphase(iq, mk(), cfg.PhaseInc) | |||||
| chunked := RunChunkedCPUOraclePolyphase(iq, []int{4096, 5000, 8192, 27307}, mk, cfg.PhaseInc) | |||||
| requireComplexSlicesCloseHarness(t, mono, chunked, 1e-5) | |||||
| } | |||||
| func TestHarnessToneIQ(t *testing.T) { | |||||
| iq := MakeToneIQ(1024, 0.05) | |||||
| if len(iq) != 1024 { | |||||
| t.Fatalf("unexpected tone iq length: %d", len(iq)) | |||||
| } | |||||
| } | |||||
| @@ -4,7 +4,7 @@ package gpudemod | |||||
| /* | /* | ||||
| #cgo windows CFLAGS: -I"C:/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v13.2/include" | #cgo windows CFLAGS: -I"C:/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v13.2/include" | ||||
| #cgo windows LDFLAGS: -lcudart64_13 -lkernel32 | |||||
| #cgo windows LDFLAGS: -L"C:/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v13.2/bin/x64" -l:cudart64_13.dll -lkernel32 | |||||
| #include <windows.h> | #include <windows.h> | ||||
| #include <stdlib.h> | #include <stdlib.h> | ||||
| #include <cuda_runtime.h> | #include <cuda_runtime.h> | ||||
| @@ -26,6 +26,8 @@ typedef int (__stdcall *gpud_launch_decimate_stream_fn)(const gpud_float2* in, g | |||||
| typedef int (__stdcall *gpud_launch_decimate_fn)(const gpud_float2* in, gpud_float2* out, int n_out, int factor); | typedef int (__stdcall *gpud_launch_decimate_fn)(const gpud_float2* in, gpud_float2* out, int n_out, int factor); | ||||
| typedef int (__stdcall *gpud_launch_am_envelope_fn)(const gpud_float2* in, float* out, int n); | typedef int (__stdcall *gpud_launch_am_envelope_fn)(const gpud_float2* in, float* out, int n); | ||||
| typedef int (__stdcall *gpud_launch_ssb_product_fn)(const gpud_float2* in, float* out, int n, double phase_inc, double phase_start); | typedef int (__stdcall *gpud_launch_ssb_product_fn)(const gpud_float2* in, float* out, int n, double phase_inc, double phase_start); | ||||
| typedef int (__stdcall *gpud_launch_streaming_polyphase_prepare_fn)(const gpud_float2* in_new, int n_new, const gpud_float2* history_in, int history_len, const float* polyphase_taps, int polyphase_len, int decim, int num_taps, int phase_count_in, double phase_start, double phase_inc, gpud_float2* out, int* n_out, int* phase_count_out, double* phase_end_out, gpud_float2* history_out); | |||||
| typedef int (__stdcall *gpud_launch_streaming_polyphase_stateful_fn)(const gpud_float2* in_new, int n_new, gpud_float2* shifted_new_tmp, const float* polyphase_taps, int polyphase_len, int decim, int num_taps, gpud_float2* history_state, gpud_float2* history_scratch, int history_cap, int* history_len_io, int* phase_count_state, double* phase_state, double phase_inc, gpud_float2* out, int out_cap, int* n_out); | |||||
| static HMODULE gpud_mod = NULL; | static HMODULE gpud_mod = NULL; | ||||
| static gpud_stream_create_fn gpud_p_stream_create = NULL; | static gpud_stream_create_fn gpud_p_stream_create = NULL; | ||||
| @@ -42,6 +44,8 @@ static gpud_launch_decimate_stream_fn gpud_p_launch_decimate_stream = NULL; | |||||
| static gpud_launch_decimate_fn gpud_p_launch_decimate = NULL; | static gpud_launch_decimate_fn gpud_p_launch_decimate = NULL; | ||||
| static gpud_launch_am_envelope_fn gpud_p_launch_am_envelope = NULL; | static gpud_launch_am_envelope_fn gpud_p_launch_am_envelope = NULL; | ||||
| static gpud_launch_ssb_product_fn gpud_p_launch_ssb_product = NULL; | static gpud_launch_ssb_product_fn gpud_p_launch_ssb_product = NULL; | ||||
| static gpud_launch_streaming_polyphase_prepare_fn gpud_p_launch_streaming_polyphase_prepare = NULL; | |||||
| static gpud_launch_streaming_polyphase_stateful_fn gpud_p_launch_streaming_polyphase_stateful = NULL; | |||||
| static int gpud_cuda_malloc(void **ptr, size_t bytes) { return (int)cudaMalloc(ptr, bytes); } | static int gpud_cuda_malloc(void **ptr, size_t bytes) { return (int)cudaMalloc(ptr, bytes); } | ||||
| static int gpud_cuda_free(void *ptr) { return (int)cudaFree(ptr); } | static int gpud_cuda_free(void *ptr) { return (int)cudaFree(ptr); } | ||||
| @@ -67,6 +71,8 @@ static int gpud_load_library(const char* path) { | |||||
| gpud_p_launch_decimate = (gpud_launch_decimate_fn)GetProcAddress(gpud_mod, "gpud_launch_decimate_cuda"); | gpud_p_launch_decimate = (gpud_launch_decimate_fn)GetProcAddress(gpud_mod, "gpud_launch_decimate_cuda"); | ||||
| gpud_p_launch_am_envelope = (gpud_launch_am_envelope_fn)GetProcAddress(gpud_mod, "gpud_launch_am_envelope_cuda"); | gpud_p_launch_am_envelope = (gpud_launch_am_envelope_fn)GetProcAddress(gpud_mod, "gpud_launch_am_envelope_cuda"); | ||||
| gpud_p_launch_ssb_product = (gpud_launch_ssb_product_fn)GetProcAddress(gpud_mod, "gpud_launch_ssb_product_cuda"); | gpud_p_launch_ssb_product = (gpud_launch_ssb_product_fn)GetProcAddress(gpud_mod, "gpud_launch_ssb_product_cuda"); | ||||
| gpud_p_launch_streaming_polyphase_prepare = (gpud_launch_streaming_polyphase_prepare_fn)GetProcAddress(gpud_mod, "gpud_launch_streaming_polyphase_prepare_cuda"); | |||||
| gpud_p_launch_streaming_polyphase_stateful = (gpud_launch_streaming_polyphase_stateful_fn)GetProcAddress(gpud_mod, "gpud_launch_streaming_polyphase_stateful_cuda"); | |||||
| if (!gpud_p_stream_create || !gpud_p_stream_destroy || !gpud_p_stream_sync || !gpud_p_upload_fir_taps || !gpud_p_launch_freq_shift_stream || !gpud_p_launch_freq_shift || !gpud_p_launch_fm_discrim || !gpud_p_launch_fir_stream || !gpud_p_launch_fir || !gpud_p_launch_decimate_stream || !gpud_p_launch_decimate || !gpud_p_launch_am_envelope || !gpud_p_launch_ssb_product) { | if (!gpud_p_stream_create || !gpud_p_stream_destroy || !gpud_p_stream_sync || !gpud_p_upload_fir_taps || !gpud_p_launch_freq_shift_stream || !gpud_p_launch_freq_shift || !gpud_p_launch_fm_discrim || !gpud_p_launch_fir_stream || !gpud_p_launch_fir || !gpud_p_launch_decimate_stream || !gpud_p_launch_decimate || !gpud_p_launch_am_envelope || !gpud_p_launch_ssb_product) { | ||||
| FreeLibrary(gpud_mod); | FreeLibrary(gpud_mod); | ||||
| gpud_mod = NULL; | gpud_mod = NULL; | ||||
| @@ -89,6 +95,8 @@ static int gpud_launch_decimate_stream(gpud_float2 *in, gpud_float2 *out, int n_ | |||||
| static int gpud_launch_decimate(gpud_float2 *in, gpud_float2 *out, int n_out, int factor) { if (!gpud_p_launch_decimate) return -1; return gpud_p_launch_decimate(in, out, n_out, factor); } | static int gpud_launch_decimate(gpud_float2 *in, gpud_float2 *out, int n_out, int factor) { if (!gpud_p_launch_decimate) return -1; return gpud_p_launch_decimate(in, out, n_out, factor); } | ||||
| static int gpud_launch_am_envelope(gpud_float2 *in, float *out, int n) { if (!gpud_p_launch_am_envelope) return -1; return gpud_p_launch_am_envelope(in, out, n); } | static int gpud_launch_am_envelope(gpud_float2 *in, float *out, int n) { if (!gpud_p_launch_am_envelope) return -1; return gpud_p_launch_am_envelope(in, out, n); } | ||||
| static int gpud_launch_ssb_product(gpud_float2 *in, float *out, int n, double phase_inc, double phase_start) { if (!gpud_p_launch_ssb_product) return -1; return gpud_p_launch_ssb_product(in, out, n, phase_inc, phase_start); } | static int gpud_launch_ssb_product(gpud_float2 *in, float *out, int n, double phase_inc, double phase_start) { if (!gpud_p_launch_ssb_product) return -1; return gpud_p_launch_ssb_product(in, out, n, phase_inc, phase_start); } | ||||
| static int gpud_launch_streaming_polyphase_prepare(gpud_float2 *in_new, int n_new, gpud_float2 *history_in, int history_len, float *polyphase_taps, int polyphase_len, int decim, int num_taps, int phase_count_in, double phase_start, double phase_inc, gpud_float2 *out, int *n_out, int *phase_count_out, double *phase_end_out, gpud_float2 *history_out) { if (!gpud_p_launch_streaming_polyphase_prepare) return -1; return gpud_p_launch_streaming_polyphase_prepare(in_new, n_new, history_in, history_len, polyphase_taps, polyphase_len, decim, num_taps, phase_count_in, phase_start, phase_inc, out, n_out, phase_count_out, phase_end_out, history_out); } | |||||
| static int gpud_launch_streaming_polyphase_stateful(gpud_float2 *in_new, int n_new, gpud_float2 *shifted_new_tmp, float *polyphase_taps, int polyphase_len, int decim, int num_taps, gpud_float2 *history_state, gpud_float2 *history_scratch, int history_cap, int *history_len_io, int *phase_count_state, double *phase_state, double phase_inc, gpud_float2 *out, int out_cap, int *n_out) { if (!gpud_p_launch_streaming_polyphase_stateful) return -1; return gpud_p_launch_streaming_polyphase_stateful(in_new, n_new, shifted_new_tmp, polyphase_taps, polyphase_len, decim, num_taps, history_state, history_scratch, history_cap, history_len_io, phase_count_state, phase_state, phase_inc, out, out_cap, n_out); } | |||||
| */ | */ | ||||
| import "C" | import "C" | ||||
| @@ -103,38 +111,68 @@ func bridgeLoadLibrary(path string) int { | |||||
| defer C.free(unsafe.Pointer(cp)) | defer C.free(unsafe.Pointer(cp)) | ||||
| return int(C.gpud_load_library(cp)) | return int(C.gpud_load_library(cp)) | ||||
| } | } | ||||
| func bridgeCudaMalloc(ptr *unsafe.Pointer, bytes uintptr) int { return int(C.gpud_cuda_malloc(ptr, C.size_t(bytes))) } | |||||
| func bridgeCudaMalloc(ptr *unsafe.Pointer, bytes uintptr) int { | |||||
| return int(C.gpud_cuda_malloc(ptr, C.size_t(bytes))) | |||||
| } | |||||
| func bridgeCudaFree(ptr unsafe.Pointer) int { return int(C.gpud_cuda_free(ptr)) } | func bridgeCudaFree(ptr unsafe.Pointer) int { return int(C.gpud_cuda_free(ptr)) } | ||||
| func bridgeMemcpyH2D(dst unsafe.Pointer, src unsafe.Pointer, bytes uintptr) int { return int(C.gpud_memcpy_h2d(dst, src, C.size_t(bytes))) } | |||||
| func bridgeMemcpyD2H(dst unsafe.Pointer, src unsafe.Pointer, bytes uintptr) int { return int(C.gpud_memcpy_d2h(dst, src, C.size_t(bytes))) } | |||||
| func bridgeMemcpyH2D(dst unsafe.Pointer, src unsafe.Pointer, bytes uintptr) int { | |||||
| return int(C.gpud_memcpy_h2d(dst, src, C.size_t(bytes))) | |||||
| } | |||||
| func bridgeMemcpyD2H(dst unsafe.Pointer, src unsafe.Pointer, bytes uintptr) int { | |||||
| return int(C.gpud_memcpy_d2h(dst, src, C.size_t(bytes))) | |||||
| } | |||||
| func bridgeDeviceSync() int { return int(C.gpud_device_sync()) } | func bridgeDeviceSync() int { return int(C.gpud_device_sync()) } | ||||
| func bridgeUploadFIRTaps(taps *C.float, n int) int { return int(C.gpud_upload_fir_taps(taps, C.int(n))) } | |||||
| func bridgeUploadFIRTaps(taps *C.float, n int) int { | |||||
| return int(C.gpud_upload_fir_taps(taps, C.int(n))) | |||||
| } | |||||
| func bridgeLaunchFreqShift(in *C.gpud_float2, out *C.gpud_float2, n int, phaseInc float64, phaseStart float64) int { | func bridgeLaunchFreqShift(in *C.gpud_float2, out *C.gpud_float2, n int, phaseInc float64, phaseStart float64) int { | ||||
| return int(C.gpud_launch_freq_shift(in, out, C.int(n), C.double(phaseInc), C.double(phaseStart))) | return int(C.gpud_launch_freq_shift(in, out, C.int(n), C.double(phaseInc), C.double(phaseStart))) | ||||
| } | } | ||||
| func bridgeLaunchFreqShiftStream(in *C.gpud_float2, out *C.gpud_float2, n int, phaseInc float64, phaseStart float64, stream streamHandle) int { | func bridgeLaunchFreqShiftStream(in *C.gpud_float2, out *C.gpud_float2, n int, phaseInc float64, phaseStart float64, stream streamHandle) int { | ||||
| return int(C.gpud_launch_freq_shift_stream(in, out, C.int(n), C.double(phaseInc), C.double(phaseStart), C.gpud_stream_handle(stream))) | return int(C.gpud_launch_freq_shift_stream(in, out, C.int(n), C.double(phaseInc), C.double(phaseStart), C.gpud_stream_handle(stream))) | ||||
| } | } | ||||
| func bridgeLaunchFIR(in *C.gpud_float2, out *C.gpud_float2, n int, numTaps int) int { return int(C.gpud_launch_fir(in, out, C.int(n), C.int(numTaps))) } | |||||
| func bridgeLaunchFIR(in *C.gpud_float2, out *C.gpud_float2, n int, numTaps int) int { | |||||
| return int(C.gpud_launch_fir(in, out, C.int(n), C.int(numTaps))) | |||||
| } | |||||
| func bridgeLaunchFIRStream(in *C.gpud_float2, out *C.gpud_float2, n int, numTaps int, stream streamHandle) int { | func bridgeLaunchFIRStream(in *C.gpud_float2, out *C.gpud_float2, n int, numTaps int, stream streamHandle) int { | ||||
| return int(C.gpud_launch_fir_stream(in, out, C.int(n), C.int(numTaps), C.gpud_stream_handle(stream))) | return int(C.gpud_launch_fir_stream(in, out, C.int(n), C.int(numTaps), C.gpud_stream_handle(stream))) | ||||
| } | } | ||||
| func bridgeLaunchFIRv2Stream(in *C.gpud_float2, out *C.gpud_float2, taps *C.float, n int, numTaps int, stream streamHandle) int { | func bridgeLaunchFIRv2Stream(in *C.gpud_float2, out *C.gpud_float2, taps *C.float, n int, numTaps int, stream streamHandle) int { | ||||
| return int(C.gpud_launch_fir_v2_stream(in, out, taps, C.int(n), C.int(numTaps), C.gpud_stream_handle(stream))) | return int(C.gpud_launch_fir_v2_stream(in, out, taps, C.int(n), C.int(numTaps), C.gpud_stream_handle(stream))) | ||||
| } | } | ||||
| func bridgeLaunchDecimate(in *C.gpud_float2, out *C.gpud_float2, nOut int, factor int) int { return int(C.gpud_launch_decimate(in, out, C.int(nOut), C.int(factor))) } | |||||
| func bridgeLaunchDecimate(in *C.gpud_float2, out *C.gpud_float2, nOut int, factor int) int { | |||||
| return int(C.gpud_launch_decimate(in, out, C.int(nOut), C.int(factor))) | |||||
| } | |||||
| func bridgeLaunchDecimateStream(in *C.gpud_float2, out *C.gpud_float2, nOut int, factor int, stream streamHandle) int { | func bridgeLaunchDecimateStream(in *C.gpud_float2, out *C.gpud_float2, nOut int, factor int, stream streamHandle) int { | ||||
| return int(C.gpud_launch_decimate_stream(in, out, C.int(nOut), C.int(factor), C.gpud_stream_handle(stream))) | return int(C.gpud_launch_decimate_stream(in, out, C.int(nOut), C.int(factor), C.gpud_stream_handle(stream))) | ||||
| } | } | ||||
| func bridgeLaunchFMDiscrim(in *C.gpud_float2, out *C.float, n int) int { return int(C.gpud_launch_fm_discrim(in, out, C.int(n))) } | |||||
| func bridgeLaunchAMEnvelope(in *C.gpud_float2, out *C.float, n int) int { return int(C.gpud_launch_am_envelope(in, out, C.int(n))) } | |||||
| func bridgeLaunchFMDiscrim(in *C.gpud_float2, out *C.float, n int) int { | |||||
| return int(C.gpud_launch_fm_discrim(in, out, C.int(n))) | |||||
| } | |||||
| func bridgeLaunchAMEnvelope(in *C.gpud_float2, out *C.float, n int) int { | |||||
| return int(C.gpud_launch_am_envelope(in, out, C.int(n))) | |||||
| } | |||||
| func bridgeLaunchSSBProduct(in *C.gpud_float2, out *C.float, n int, phaseInc float64, phaseStart float64) int { | func bridgeLaunchSSBProduct(in *C.gpud_float2, out *C.float, n int, phaseInc float64, phaseStart float64) int { | ||||
| return int(C.gpud_launch_ssb_product(in, out, C.int(n), C.double(phaseInc), C.double(phaseStart))) | return int(C.gpud_launch_ssb_product(in, out, C.int(n), C.double(phaseInc), C.double(phaseStart))) | ||||
| } | } | ||||
| // bridgeLaunchStreamingPolyphasePrepare is a transitional bridge for the | |||||
| // legacy single-call prepare path. The stateful native path uses | |||||
| // bridgeLaunchStreamingPolyphaseStateful. | |||||
| func bridgeLaunchStreamingPolyphasePrepare(inNew *C.gpud_float2, nNew int, historyIn *C.gpud_float2, historyLen int, polyphaseTaps *C.float, polyphaseLen int, decim int, numTaps int, phaseCountIn int, phaseStart float64, phaseInc float64, out *C.gpud_float2, nOut *C.int, phaseCountOut *C.int, phaseEndOut *C.double, historyOut *C.gpud_float2) int { | |||||
| return int(C.gpud_launch_streaming_polyphase_prepare(inNew, C.int(nNew), historyIn, C.int(historyLen), polyphaseTaps, C.int(polyphaseLen), C.int(decim), C.int(numTaps), C.int(phaseCountIn), C.double(phaseStart), C.double(phaseInc), out, nOut, phaseCountOut, phaseEndOut, historyOut)) | |||||
| } | |||||
| func bridgeLaunchStreamingPolyphaseStateful(inNew *C.gpud_float2, nNew int, shiftedNewTmp *C.gpud_float2, polyphaseTaps *C.float, polyphaseLen int, decim int, numTaps int, historyState *C.gpud_float2, historyScratch *C.gpud_float2, historyCap int, historyLenIO *C.int, phaseCountState *C.int, phaseState *C.double, phaseInc float64, out *C.gpud_float2, outCap int, nOut *C.int) int { | |||||
| return int(C.gpud_launch_streaming_polyphase_stateful(inNew, C.int(nNew), shiftedNewTmp, polyphaseTaps, C.int(polyphaseLen), C.int(decim), C.int(numTaps), historyState, historyScratch, C.int(historyCap), historyLenIO, phaseCountState, phaseState, C.double(phaseInc), out, C.int(outCap), nOut)) | |||||
| } | |||||
| func bridgeStreamCreate() (streamHandle, int) { | func bridgeStreamCreate() (streamHandle, int) { | ||||
| var s C.gpud_stream_handle | var s C.gpud_stream_handle | ||||
| res := int(C.gpud_stream_create(&s)) | res := int(C.gpud_stream_create(&s)) | ||||
| return streamHandle(s), res | return streamHandle(s), res | ||||
| } | } | ||||
| func bridgeStreamDestroy(stream streamHandle) int { return int(C.gpud_stream_destroy(C.gpud_stream_handle(stream))) } | |||||
| func bridgeStreamSync(stream streamHandle) int { return int(C.gpud_stream_sync(C.gpud_stream_handle(stream))) } | |||||
| func bridgeStreamDestroy(stream streamHandle) int { | |||||
| return int(C.gpud_stream_destroy(C.gpud_stream_handle(stream))) | |||||
| } | |||||
| func bridgeStreamSync(stream streamHandle) int { | |||||
| return int(C.gpud_stream_sync(C.gpud_stream_handle(stream))) | |||||
| } | |||||
| @@ -0,0 +1,95 @@ | |||||
| package dsp | |||||
| // StatefulDecimatingFIRComplex combines FIR filtering and decimation into a | |||||
| // single stateful stage. This avoids exposing FIR settling/transient output as | |||||
| // ordinary block-leading samples before decimation. | |||||
| type StatefulDecimatingFIRComplex struct { | |||||
| taps []float64 | |||||
| delayR []float64 | |||||
| delayI []float64 | |||||
| factor int | |||||
| phase int // number of input samples until next output sample (0 => emit now) | |||||
| } | |||||
| func (f *StatefulDecimatingFIRComplex) Phase() int { | |||||
| if f == nil { | |||||
| return 0 | |||||
| } | |||||
| return f.phase | |||||
| } | |||||
| func (f *StatefulDecimatingFIRComplex) TapsLen() int { | |||||
| if f == nil { | |||||
| return 0 | |||||
| } | |||||
| return len(f.taps) | |||||
| } | |||||
| func NewStatefulDecimatingFIRComplex(taps []float64, factor int) *StatefulDecimatingFIRComplex { | |||||
| if factor < 1 { | |||||
| factor = 1 | |||||
| } | |||||
| t := make([]float64, len(taps)) | |||||
| copy(t, taps) | |||||
| return &StatefulDecimatingFIRComplex{ | |||||
| taps: t, | |||||
| delayR: make([]float64, len(taps)), | |||||
| delayI: make([]float64, len(taps)), | |||||
| factor: factor, | |||||
| phase: 0, | |||||
| } | |||||
| } | |||||
| func (f *StatefulDecimatingFIRComplex) Reset() { | |||||
| for i := range f.delayR { | |||||
| f.delayR[i] = 0 | |||||
| f.delayI[i] = 0 | |||||
| } | |||||
| f.phase = 0 | |||||
| } | |||||
| func (f *StatefulDecimatingFIRComplex) Process(iq []complex64) []complex64 { | |||||
| if len(iq) == 0 || len(f.taps) == 0 { | |||||
| return nil | |||||
| } | |||||
| if f.factor <= 1 { | |||||
| out := make([]complex64, len(iq)) | |||||
| for i := 0; i < len(iq); i++ { | |||||
| copy(f.delayR[1:], f.delayR[:len(f.taps)-1]) | |||||
| copy(f.delayI[1:], f.delayI[:len(f.taps)-1]) | |||||
| f.delayR[0] = float64(real(iq[i])) | |||||
| f.delayI[0] = float64(imag(iq[i])) | |||||
| var accR, accI float64 | |||||
| for k := 0; k < len(f.taps); k++ { | |||||
| w := f.taps[k] | |||||
| accR += f.delayR[k] * w | |||||
| accI += f.delayI[k] * w | |||||
| } | |||||
| out[i] = complex(float32(accR), float32(accI)) | |||||
| } | |||||
| return out | |||||
| } | |||||
| out := make([]complex64, 0, len(iq)/f.factor+1) | |||||
| n := len(f.taps) | |||||
| for i := 0; i < len(iq); i++ { | |||||
| copy(f.delayR[1:], f.delayR[:n-1]) | |||||
| copy(f.delayI[1:], f.delayI[:n-1]) | |||||
| f.delayR[0] = float64(real(iq[i])) | |||||
| f.delayI[0] = float64(imag(iq[i])) | |||||
| if f.phase == 0 { | |||||
| var accR, accI float64 | |||||
| for k := 0; k < n; k++ { | |||||
| w := f.taps[k] | |||||
| accR += f.delayR[k] * w | |||||
| accI += f.delayI[k] * w | |||||
| } | |||||
| out = append(out, complex(float32(accR), float32(accI))) | |||||
| f.phase = f.factor - 1 | |||||
| } else { | |||||
| f.phase-- | |||||
| } | |||||
| } | |||||
| return out | |||||
| } | |||||
| @@ -0,0 +1,57 @@ | |||||
| package dsp | |||||
| import ( | |||||
| "math/cmplx" | |||||
| "testing" | |||||
| ) | |||||
| func TestStatefulDecimatingFIRComplexStreamContinuity(t *testing.T) { | |||||
| taps := LowpassFIR(90000, 512000, 101) | |||||
| factor := 2 | |||||
| input := make([]complex64, 8192) | |||||
| for i := range input { | |||||
| input[i] = complex(float32((i%17)-8)/8.0, float32((i%11)-5)/8.0) | |||||
| } | |||||
| one := NewStatefulDecimatingFIRComplex(taps, factor) | |||||
| whole := one.Process(input) | |||||
| chunkedProc := NewStatefulDecimatingFIRComplex(taps, factor) | |||||
| var chunked []complex64 | |||||
| for i := 0; i < len(input); i += 733 { | |||||
| end := i + 733 | |||||
| if end > len(input) { | |||||
| end = len(input) | |||||
| } | |||||
| chunked = append(chunked, chunkedProc.Process(input[i:end])...) | |||||
| } | |||||
| if len(whole) != len(chunked) { | |||||
| t.Fatalf("length mismatch whole=%d chunked=%d", len(whole), len(chunked)) | |||||
| } | |||||
| for i := range whole { | |||||
| if cmplx.Abs(complex128(whole[i]-chunked[i])) > 1e-5 { | |||||
| t.Fatalf("sample %d mismatch whole=%v chunked=%v", i, whole[i], chunked[i]) | |||||
| } | |||||
| } | |||||
| } | |||||
| func TestStatefulDecimatingFIRComplexMatchesBlockPipelineLength(t *testing.T) { | |||||
| taps := LowpassFIR(90000, 512000, 101) | |||||
| factor := 2 | |||||
| input := make([]complex64, 48640) | |||||
| for i := range input { | |||||
| input[i] = complex(float32((i%13)-6)/8.0, float32((i%7)-3)/8.0) | |||||
| } | |||||
| stateful := NewStatefulDecimatingFIRComplex(taps, factor) | |||||
| out := stateful.Process(input) | |||||
| filtered := ApplyFIR(input, taps) | |||||
| dec := Decimate(filtered, factor) | |||||
| if len(out) != len(dec) { | |||||
| t.Fatalf("unexpected output len got=%d want=%d", len(out), len(dec)) | |||||
| } | |||||
| } | |||||
| @@ -12,6 +12,7 @@ import ( | |||||
| "sdr-wideband-suite/internal/demod/gpudemod" | "sdr-wideband-suite/internal/demod/gpudemod" | ||||
| "sdr-wideband-suite/internal/detector" | "sdr-wideband-suite/internal/detector" | ||||
| "sdr-wideband-suite/internal/telemetry" | |||||
| ) | ) | ||||
| type Policy struct { | type Policy struct { | ||||
| @@ -54,9 +55,10 @@ type Manager struct { | |||||
| streamer *Streamer | streamer *Streamer | ||||
| streamedIDs map[int64]bool // signal IDs that were streamed (skip retroactive recording) | streamedIDs map[int64]bool // signal IDs that were streamed (skip retroactive recording) | ||||
| streamedMu sync.Mutex | streamedMu sync.Mutex | ||||
| telemetry *telemetry.Collector | |||||
| } | } | ||||
| func New(sampleRate int, blockSize int, policy Policy, centerHz float64, decodeCommands map[string]string) *Manager { | |||||
| func New(sampleRate int, blockSize int, policy Policy, centerHz float64, decodeCommands map[string]string, coll *telemetry.Collector) *Manager { | |||||
| if policy.OutputDir == "" { | if policy.OutputDir == "" { | ||||
| policy.OutputDir = "data/recordings" | policy.OutputDir = "data/recordings" | ||||
| } | } | ||||
| @@ -71,8 +73,9 @@ func New(sampleRate int, blockSize int, policy Policy, centerHz float64, decodeC | |||||
| centerHz: centerHz, | centerHz: centerHz, | ||||
| decodeCommands: decodeCommands, | decodeCommands: decodeCommands, | ||||
| queue: make(chan detector.Event, 64), | queue: make(chan detector.Event, 64), | ||||
| streamer: newStreamer(policy, centerHz), | |||||
| streamer: newStreamer(policy, centerHz, coll), | |||||
| streamedIDs: make(map[int64]bool), | streamedIDs: make(map[int64]bool), | ||||
| telemetry: coll, | |||||
| } | } | ||||
| m.initGPUDemod(sampleRate, blockSize) | m.initGPUDemod(sampleRate, blockSize) | ||||
| m.workerWG.Add(1) | m.workerWG.Add(1) | ||||
| @@ -103,6 +106,13 @@ func (m *Manager) Update(sampleRate int, blockSize int, policy Policy, centerHz | |||||
| if m.streamer != nil { | if m.streamer != nil { | ||||
| m.streamer.updatePolicy(policy, centerHz) | m.streamer.updatePolicy(policy, centerHz) | ||||
| } | } | ||||
| if m.telemetry != nil { | |||||
| m.telemetry.Event("recorder_update", "info", "recorder policy updated", nil, map[string]any{ | |||||
| "sample_rate": sampleRate, | |||||
| "block_size": blockSize, | |||||
| "enabled": policy.Enabled, | |||||
| }) | |||||
| } | |||||
| } | } | ||||
| func (m *Manager) Ingest(t0 time.Time, samples []complex64) { | func (m *Manager) Ingest(t0 time.Time, samples []complex64) { | ||||
| @@ -116,6 +126,9 @@ func (m *Manager) Ingest(t0 time.Time, samples []complex64) { | |||||
| return | return | ||||
| } | } | ||||
| ring.Push(t0, samples) | ring.Push(t0, samples) | ||||
| if m.telemetry != nil { | |||||
| m.telemetry.SetGauge("recorder.ring.push_samples", float64(len(samples)), nil) | |||||
| } | |||||
| } | } | ||||
| func (m *Manager) OnEvents(events []detector.Event) { | func (m *Manager) OnEvents(events []detector.Event) { | ||||
| @@ -134,8 +147,14 @@ func (m *Manager) OnEvents(events []detector.Event) { | |||||
| case m.queue <- ev: | case m.queue <- ev: | ||||
| default: | default: | ||||
| // drop if queue full | // drop if queue full | ||||
| if m.telemetry != nil { | |||||
| m.telemetry.IncCounter("recorder.event_queue.drop", 1, nil) | |||||
| } | |||||
| } | } | ||||
| } | } | ||||
| if m.telemetry != nil { | |||||
| m.telemetry.SetGauge("recorder.event_queue.len", float64(len(m.queue)), nil) | |||||
| } | |||||
| } | } | ||||
| func (m *Manager) worker() { | func (m *Manager) worker() { | ||||
| @@ -357,6 +376,13 @@ func (m *Manager) StreamerRef() *Streamer { | |||||
| return m.streamer | return m.streamer | ||||
| } | } | ||||
| func (m *Manager) ResetStreams() { | |||||
| if m == nil || m.streamer == nil { | |||||
| return | |||||
| } | |||||
| m.streamer.ResetStreams() | |||||
| } | |||||
| func (m *Manager) RuntimeInfoBySignalID() map[int64]RuntimeSignalInfo { | func (m *Manager) RuntimeInfoBySignalID() map[int64]RuntimeSignalInfo { | ||||
| if m == nil || m.streamer == nil { | if m == nil || m.streamer == nil { | ||||
| return nil | return nil | ||||
| @@ -0,0 +1,966 @@ | |||||
| package telemetry | |||||
| import ( | |||||
| "bufio" | |||||
| "encoding/json" | |||||
| "errors" | |||||
| "fmt" | |||||
| "os" | |||||
| "path/filepath" | |||||
| "sort" | |||||
| "strconv" | |||||
| "strings" | |||||
| "sync" | |||||
| "sync/atomic" | |||||
| "time" | |||||
| ) | |||||
| type Config struct { | |||||
| Enabled bool `json:"enabled"` | |||||
| HeavyEnabled bool `json:"heavy_enabled"` | |||||
| HeavySampleEvery int `json:"heavy_sample_every"` | |||||
| MetricSampleEvery int `json:"metric_sample_every"` | |||||
| MetricHistoryMax int `json:"metric_history_max"` | |||||
| EventHistoryMax int `json:"event_history_max"` | |||||
| Retention time.Duration `json:"retention"` | |||||
| PersistEnabled bool `json:"persist_enabled"` | |||||
| PersistDir string `json:"persist_dir"` | |||||
| RotateMB int `json:"rotate_mb"` | |||||
| KeepFiles int `json:"keep_files"` | |||||
| } | |||||
| func DefaultConfig() Config { | |||||
| return Config{ | |||||
| Enabled: true, | |||||
| HeavyEnabled: false, | |||||
| HeavySampleEvery: 12, | |||||
| MetricSampleEvery: 2, | |||||
| MetricHistoryMax: 12_000, | |||||
| EventHistoryMax: 4_000, | |||||
| Retention: 15 * time.Minute, | |||||
| PersistEnabled: false, | |||||
| PersistDir: "debug/telemetry", | |||||
| RotateMB: 16, | |||||
| KeepFiles: 8, | |||||
| } | |||||
| } | |||||
| type Tags map[string]string | |||||
| type MetricPoint struct { | |||||
| Timestamp time.Time `json:"ts"` | |||||
| Name string `json:"name"` | |||||
| Type string `json:"type"` | |||||
| Value float64 `json:"value"` | |||||
| Tags Tags `json:"tags,omitempty"` | |||||
| } | |||||
| type Event struct { | |||||
| ID uint64 `json:"id"` | |||||
| Timestamp time.Time `json:"ts"` | |||||
| Name string `json:"name"` | |||||
| Level string `json:"level"` | |||||
| Message string `json:"message,omitempty"` | |||||
| Tags Tags `json:"tags,omitempty"` | |||||
| Fields map[string]any `json:"fields,omitempty"` | |||||
| } | |||||
| type SeriesValue struct { | |||||
| Name string `json:"name"` | |||||
| Value float64 `json:"value"` | |||||
| Tags Tags `json:"tags,omitempty"` | |||||
| } | |||||
| type DistValue struct { | |||||
| Name string `json:"name"` | |||||
| Count int64 `json:"count"` | |||||
| Min float64 `json:"min"` | |||||
| Max float64 `json:"max"` | |||||
| Mean float64 `json:"mean"` | |||||
| Last float64 `json:"last"` | |||||
| P95 float64 `json:"p95"` | |||||
| Tags Tags `json:"tags,omitempty"` | |||||
| } | |||||
| type LiveSnapshot struct { | |||||
| Now time.Time `json:"now"` | |||||
| StartedAt time.Time `json:"started_at"` | |||||
| UptimeMs int64 `json:"uptime_ms"` | |||||
| Config Config `json:"config"` | |||||
| Counters []SeriesValue `json:"counters"` | |||||
| Gauges []SeriesValue `json:"gauges"` | |||||
| Distributions []DistValue `json:"distributions"` | |||||
| RecentEvents []Event `json:"recent_events"` | |||||
| Status map[string]any `json:"status,omitempty"` | |||||
| } | |||||
| type Query struct { | |||||
| From time.Time | |||||
| To time.Time | |||||
| Limit int | |||||
| Name string | |||||
| NamePrefix string | |||||
| Level string | |||||
| Tags Tags | |||||
| IncludePersisted bool | |||||
| } | |||||
| type collectorMetric struct { | |||||
| name string | |||||
| tags Tags | |||||
| value float64 | |||||
| } | |||||
| type distMetric struct { | |||||
| name string | |||||
| tags Tags | |||||
| count int64 | |||||
| sum float64 | |||||
| min float64 | |||||
| max float64 | |||||
| last float64 | |||||
| samples []float64 | |||||
| next int | |||||
| full bool | |||||
| } | |||||
| type persistedEnvelope struct { | |||||
| Kind string `json:"kind"` | |||||
| Metric *MetricPoint `json:"metric,omitempty"` | |||||
| Event *Event `json:"event,omitempty"` | |||||
| } | |||||
| type Collector struct { | |||||
| mu sync.RWMutex | |||||
| cfg Config | |||||
| startedAt time.Time | |||||
| counterSeq uint64 | |||||
| heavySeq uint64 | |||||
| eventSeq uint64 | |||||
| counters map[string]*collectorMetric | |||||
| gauges map[string]*collectorMetric | |||||
| dists map[string]*distMetric | |||||
| metricsHistory []MetricPoint | |||||
| events []Event | |||||
| status map[string]any | |||||
| writer *jsonlWriter | |||||
| } | |||||
| func New(cfg Config) (*Collector, error) { | |||||
| cfg = sanitizeConfig(cfg) | |||||
| c := &Collector{ | |||||
| cfg: cfg, | |||||
| startedAt: time.Now().UTC(), | |||||
| counters: map[string]*collectorMetric{}, | |||||
| gauges: map[string]*collectorMetric{}, | |||||
| dists: map[string]*distMetric{}, | |||||
| metricsHistory: make([]MetricPoint, 0, cfg.MetricHistoryMax), | |||||
| events: make([]Event, 0, cfg.EventHistoryMax), | |||||
| status: map[string]any{}, | |||||
| } | |||||
| if cfg.PersistEnabled { | |||||
| writer, err := newJSONLWriter(cfg) | |||||
| if err != nil { | |||||
| return nil, err | |||||
| } | |||||
| c.writer = writer | |||||
| } | |||||
| return c, nil | |||||
| } | |||||
| func (c *Collector) Close() error { | |||||
| if c == nil { | |||||
| return nil | |||||
| } | |||||
| c.mu.Lock() | |||||
| writer := c.writer | |||||
| c.writer = nil | |||||
| c.mu.Unlock() | |||||
| if writer != nil { | |||||
| return writer.Close() | |||||
| } | |||||
| return nil | |||||
| } | |||||
| func (c *Collector) Configure(cfg Config) error { | |||||
| if c == nil { | |||||
| return nil | |||||
| } | |||||
| cfg = sanitizeConfig(cfg) | |||||
| var writer *jsonlWriter | |||||
| var err error | |||||
| if cfg.PersistEnabled { | |||||
| writer, err = newJSONLWriter(cfg) | |||||
| if err != nil { | |||||
| return err | |||||
| } | |||||
| } | |||||
| c.mu.Lock() | |||||
| old := c.writer | |||||
| c.cfg = cfg | |||||
| c.writer = writer | |||||
| c.trimLocked(time.Now().UTC()) | |||||
| c.mu.Unlock() | |||||
| if old != nil { | |||||
| _ = old.Close() | |||||
| } | |||||
| return nil | |||||
| } | |||||
| func (c *Collector) Config() Config { | |||||
| c.mu.RLock() | |||||
| defer c.mu.RUnlock() | |||||
| return c.cfg | |||||
| } | |||||
| func (c *Collector) Enabled() bool { | |||||
| if c == nil { | |||||
| return false | |||||
| } | |||||
| c.mu.RLock() | |||||
| defer c.mu.RUnlock() | |||||
| return c.cfg.Enabled | |||||
| } | |||||
| func (c *Collector) ShouldSampleHeavy() bool { | |||||
| if c == nil { | |||||
| return false | |||||
| } | |||||
| c.mu.RLock() | |||||
| cfg := c.cfg | |||||
| c.mu.RUnlock() | |||||
| if !cfg.Enabled || !cfg.HeavyEnabled { | |||||
| return false | |||||
| } | |||||
| n := cfg.HeavySampleEvery | |||||
| if n <= 1 { | |||||
| return true | |||||
| } | |||||
| seq := atomic.AddUint64(&c.heavySeq, 1) | |||||
| return seq%uint64(n) == 0 | |||||
| } | |||||
| func (c *Collector) SetStatus(key string, value any) { | |||||
| if c == nil { | |||||
| return | |||||
| } | |||||
| c.mu.Lock() | |||||
| c.status[key] = value | |||||
| c.mu.Unlock() | |||||
| } | |||||
| func (c *Collector) IncCounter(name string, delta float64, tags Tags) { | |||||
| c.recordMetric("counter", name, delta, tags, true) | |||||
| } | |||||
| func (c *Collector) SetGauge(name string, value float64, tags Tags) { | |||||
| c.recordMetric("gauge", name, value, tags, false) | |||||
| } | |||||
| func (c *Collector) Observe(name string, value float64, tags Tags) { | |||||
| c.recordMetric("distribution", name, value, tags, false) | |||||
| } | |||||
| func (c *Collector) Event(name string, level string, message string, tags Tags, fields map[string]any) { | |||||
| if c == nil { | |||||
| return | |||||
| } | |||||
| now := time.Now().UTC() | |||||
| c.mu.Lock() | |||||
| if !c.cfg.Enabled { | |||||
| c.mu.Unlock() | |||||
| return | |||||
| } | |||||
| ev := Event{ | |||||
| ID: atomic.AddUint64(&c.eventSeq, 1), | |||||
| Timestamp: now, | |||||
| Name: name, | |||||
| Level: strings.TrimSpace(strings.ToLower(level)), | |||||
| Message: message, | |||||
| Tags: cloneTags(tags), | |||||
| Fields: cloneFields(fields), | |||||
| } | |||||
| if ev.Level == "" { | |||||
| ev.Level = "info" | |||||
| } | |||||
| c.events = append(c.events, ev) | |||||
| c.trimLocked(now) | |||||
| writer := c.writer | |||||
| c.mu.Unlock() | |||||
| if writer != nil { | |||||
| _ = writer.Write(persistedEnvelope{Kind: "event", Event: &ev}) | |||||
| } | |||||
| } | |||||
| func (c *Collector) recordMetric(kind string, name string, value float64, tags Tags, add bool) { | |||||
| if c == nil || strings.TrimSpace(name) == "" { | |||||
| return | |||||
| } | |||||
| now := time.Now().UTC() | |||||
| c.mu.Lock() | |||||
| if !c.cfg.Enabled { | |||||
| c.mu.Unlock() | |||||
| return | |||||
| } | |||||
| key := metricKey(name, tags) | |||||
| switch kind { | |||||
| case "counter": | |||||
| m := c.counters[key] | |||||
| if m == nil { | |||||
| m = &collectorMetric{name: name, tags: cloneTags(tags)} | |||||
| c.counters[key] = m | |||||
| } | |||||
| if add { | |||||
| m.value += value | |||||
| } else { | |||||
| m.value = value | |||||
| } | |||||
| case "gauge": | |||||
| m := c.gauges[key] | |||||
| if m == nil { | |||||
| m = &collectorMetric{name: name, tags: cloneTags(tags)} | |||||
| c.gauges[key] = m | |||||
| } | |||||
| m.value = value | |||||
| case "distribution": | |||||
| d := c.dists[key] | |||||
| if d == nil { | |||||
| d = &distMetric{ | |||||
| name: name, | |||||
| tags: cloneTags(tags), | |||||
| min: value, | |||||
| max: value, | |||||
| samples: make([]float64, 64), | |||||
| } | |||||
| c.dists[key] = d | |||||
| } | |||||
| d.count++ | |||||
| d.sum += value | |||||
| d.last = value | |||||
| if d.count == 1 || value < d.min { | |||||
| d.min = value | |||||
| } | |||||
| if d.count == 1 || value > d.max { | |||||
| d.max = value | |||||
| } | |||||
| if len(d.samples) > 0 { | |||||
| d.samples[d.next] = value | |||||
| d.next++ | |||||
| if d.next >= len(d.samples) { | |||||
| d.next = 0 | |||||
| d.full = true | |||||
| } | |||||
| } | |||||
| } | |||||
| sampleN := c.cfg.MetricSampleEvery | |||||
| seq := atomic.AddUint64(&c.counterSeq, 1) | |||||
| forceStore := strings.HasPrefix(name, "iq.extract.raw.boundary.") || strings.HasPrefix(name, "iq.extract.trimmed.boundary.") | |||||
| shouldStore := forceStore || sampleN <= 1 || seq%uint64(sampleN) == 0 || kind == "counter" | |||||
| var mp MetricPoint | |||||
| if shouldStore { | |||||
| mp = MetricPoint{ | |||||
| Timestamp: now, | |||||
| Name: name, | |||||
| Type: kind, | |||||
| Value: value, | |||||
| Tags: cloneTags(tags), | |||||
| } | |||||
| c.metricsHistory = append(c.metricsHistory, mp) | |||||
| } | |||||
| c.trimLocked(now) | |||||
| writer := c.writer | |||||
| c.mu.Unlock() | |||||
| if writer != nil && shouldStore { | |||||
| _ = writer.Write(persistedEnvelope{Kind: "metric", Metric: &mp}) | |||||
| } | |||||
| } | |||||
| func (c *Collector) LiveSnapshot() LiveSnapshot { | |||||
| now := time.Now().UTC() | |||||
| c.mu.RLock() | |||||
| cfg := c.cfg | |||||
| out := LiveSnapshot{ | |||||
| Now: now, | |||||
| StartedAt: c.startedAt, | |||||
| UptimeMs: now.Sub(c.startedAt).Milliseconds(), | |||||
| Config: cfg, | |||||
| Counters: make([]SeriesValue, 0, len(c.counters)), | |||||
| Gauges: make([]SeriesValue, 0, len(c.gauges)), | |||||
| Distributions: make([]DistValue, 0, len(c.dists)), | |||||
| RecentEvents: make([]Event, 0, min(40, len(c.events))), | |||||
| Status: cloneFields(c.status), | |||||
| } | |||||
| for _, m := range c.counters { | |||||
| out.Counters = append(out.Counters, SeriesValue{Name: m.name, Value: m.value, Tags: cloneTags(m.tags)}) | |||||
| } | |||||
| for _, m := range c.gauges { | |||||
| out.Gauges = append(out.Gauges, SeriesValue{Name: m.name, Value: m.value, Tags: cloneTags(m.tags)}) | |||||
| } | |||||
| for _, d := range c.dists { | |||||
| mean := 0.0 | |||||
| if d.count > 0 { | |||||
| mean = d.sum / float64(d.count) | |||||
| } | |||||
| out.Distributions = append(out.Distributions, DistValue{ | |||||
| Name: d.name, | |||||
| Count: d.count, | |||||
| Min: d.min, | |||||
| Max: d.max, | |||||
| Mean: mean, | |||||
| Last: d.last, | |||||
| P95: p95FromDist(d), | |||||
| Tags: cloneTags(d.tags), | |||||
| }) | |||||
| } | |||||
| start := len(c.events) - cap(out.RecentEvents) | |||||
| if start < 0 { | |||||
| start = 0 | |||||
| } | |||||
| for _, ev := range c.events[start:] { | |||||
| out.RecentEvents = append(out.RecentEvents, copyEvent(ev)) | |||||
| } | |||||
| c.mu.RUnlock() | |||||
| sort.Slice(out.Counters, func(i, j int) bool { return out.Counters[i].Name < out.Counters[j].Name }) | |||||
| sort.Slice(out.Gauges, func(i, j int) bool { return out.Gauges[i].Name < out.Gauges[j].Name }) | |||||
| sort.Slice(out.Distributions, func(i, j int) bool { return out.Distributions[i].Name < out.Distributions[j].Name }) | |||||
| return out | |||||
| } | |||||
| func (c *Collector) QueryMetrics(q Query) ([]MetricPoint, error) { | |||||
| if c == nil { | |||||
| return nil, nil | |||||
| } | |||||
| q = normalizeQuery(q) | |||||
| c.mu.RLock() | |||||
| items := make([]MetricPoint, 0, len(c.metricsHistory)) | |||||
| for _, m := range c.metricsHistory { | |||||
| if metricMatch(m, q) { | |||||
| items = append(items, copyMetric(m)) | |||||
| } | |||||
| } | |||||
| cfg := c.cfg | |||||
| c.mu.RUnlock() | |||||
| if q.IncludePersisted && cfg.PersistEnabled { | |||||
| persisted, err := readPersistedMetrics(cfg, q) | |||||
| if err != nil && !errors.Is(err, os.ErrNotExist) { | |||||
| return nil, err | |||||
| } | |||||
| items = append(items, persisted...) | |||||
| } | |||||
| sort.Slice(items, func(i, j int) bool { | |||||
| return items[i].Timestamp.Before(items[j].Timestamp) | |||||
| }) | |||||
| if q.Limit > 0 && len(items) > q.Limit { | |||||
| items = items[len(items)-q.Limit:] | |||||
| } | |||||
| return items, nil | |||||
| } | |||||
| func (c *Collector) QueryEvents(q Query) ([]Event, error) { | |||||
| if c == nil { | |||||
| return nil, nil | |||||
| } | |||||
| q = normalizeQuery(q) | |||||
| c.mu.RLock() | |||||
| items := make([]Event, 0, len(c.events)) | |||||
| for _, ev := range c.events { | |||||
| if eventMatch(ev, q) { | |||||
| items = append(items, copyEvent(ev)) | |||||
| } | |||||
| } | |||||
| cfg := c.cfg | |||||
| c.mu.RUnlock() | |||||
| if q.IncludePersisted && cfg.PersistEnabled { | |||||
| persisted, err := readPersistedEvents(cfg, q) | |||||
| if err != nil && !errors.Is(err, os.ErrNotExist) { | |||||
| return nil, err | |||||
| } | |||||
| items = append(items, persisted...) | |||||
| } | |||||
| sort.Slice(items, func(i, j int) bool { | |||||
| return items[i].Timestamp.Before(items[j].Timestamp) | |||||
| }) | |||||
| if q.Limit > 0 && len(items) > q.Limit { | |||||
| items = items[len(items)-q.Limit:] | |||||
| } | |||||
| return items, nil | |||||
| } | |||||
| func (c *Collector) trimLocked(now time.Time) { | |||||
| if c.cfg.MetricHistoryMax > 0 && len(c.metricsHistory) > c.cfg.MetricHistoryMax { | |||||
| c.metricsHistory = append([]MetricPoint(nil), c.metricsHistory[len(c.metricsHistory)-c.cfg.MetricHistoryMax:]...) | |||||
| } | |||||
| if c.cfg.EventHistoryMax > 0 && len(c.events) > c.cfg.EventHistoryMax { | |||||
| c.events = append([]Event(nil), c.events[len(c.events)-c.cfg.EventHistoryMax:]...) | |||||
| } | |||||
| ret := c.cfg.Retention | |||||
| if ret <= 0 { | |||||
| return | |||||
| } | |||||
| cut := now.Add(-ret) | |||||
| mStart := 0 | |||||
| for mStart < len(c.metricsHistory) && c.metricsHistory[mStart].Timestamp.Before(cut) { | |||||
| mStart++ | |||||
| } | |||||
| if mStart > 0 { | |||||
| c.metricsHistory = append([]MetricPoint(nil), c.metricsHistory[mStart:]...) | |||||
| } | |||||
| eStart := 0 | |||||
| for eStart < len(c.events) && c.events[eStart].Timestamp.Before(cut) { | |||||
| eStart++ | |||||
| } | |||||
| if eStart > 0 { | |||||
| c.events = append([]Event(nil), c.events[eStart:]...) | |||||
| } | |||||
| } | |||||
| func sanitizeConfig(cfg Config) Config { | |||||
| def := DefaultConfig() | |||||
| if cfg.HeavySampleEvery <= 0 { | |||||
| cfg.HeavySampleEvery = def.HeavySampleEvery | |||||
| } | |||||
| if cfg.MetricSampleEvery <= 0 { | |||||
| cfg.MetricSampleEvery = def.MetricSampleEvery | |||||
| } | |||||
| if cfg.MetricHistoryMax <= 0 { | |||||
| cfg.MetricHistoryMax = def.MetricHistoryMax | |||||
| } | |||||
| if cfg.EventHistoryMax <= 0 { | |||||
| cfg.EventHistoryMax = def.EventHistoryMax | |||||
| } | |||||
| if cfg.Retention <= 0 { | |||||
| cfg.Retention = def.Retention | |||||
| } | |||||
| if strings.TrimSpace(cfg.PersistDir) == "" { | |||||
| cfg.PersistDir = def.PersistDir | |||||
| } | |||||
| if cfg.RotateMB <= 0 { | |||||
| cfg.RotateMB = def.RotateMB | |||||
| } | |||||
| if cfg.KeepFiles <= 0 { | |||||
| cfg.KeepFiles = def.KeepFiles | |||||
| } | |||||
| return cfg | |||||
| } | |||||
| func normalizeQuery(q Query) Query { | |||||
| if q.Limit <= 0 || q.Limit > 5000 { | |||||
| q.Limit = 500 | |||||
| } | |||||
| if q.Tags == nil { | |||||
| q.Tags = Tags{} | |||||
| } | |||||
| return q | |||||
| } | |||||
| func metricMatch(m MetricPoint, q Query) bool { | |||||
| if !q.From.IsZero() && m.Timestamp.Before(q.From) { | |||||
| return false | |||||
| } | |||||
| if !q.To.IsZero() && m.Timestamp.After(q.To) { | |||||
| return false | |||||
| } | |||||
| if q.Name != "" && m.Name != q.Name { | |||||
| return false | |||||
| } | |||||
| if q.NamePrefix != "" && !strings.HasPrefix(m.Name, q.NamePrefix) { | |||||
| return false | |||||
| } | |||||
| for k, v := range q.Tags { | |||||
| if m.Tags[k] != v { | |||||
| return false | |||||
| } | |||||
| } | |||||
| return true | |||||
| } | |||||
| func eventMatch(ev Event, q Query) bool { | |||||
| if !q.From.IsZero() && ev.Timestamp.Before(q.From) { | |||||
| return false | |||||
| } | |||||
| if !q.To.IsZero() && ev.Timestamp.After(q.To) { | |||||
| return false | |||||
| } | |||||
| if q.Name != "" && ev.Name != q.Name { | |||||
| return false | |||||
| } | |||||
| if q.NamePrefix != "" && !strings.HasPrefix(ev.Name, q.NamePrefix) { | |||||
| return false | |||||
| } | |||||
| if q.Level != "" && !strings.EqualFold(q.Level, ev.Level) { | |||||
| return false | |||||
| } | |||||
| for k, v := range q.Tags { | |||||
| if ev.Tags[k] != v { | |||||
| return false | |||||
| } | |||||
| } | |||||
| return true | |||||
| } | |||||
| func metricKey(name string, tags Tags) string { | |||||
| if len(tags) == 0 { | |||||
| return name | |||||
| } | |||||
| keys := make([]string, 0, len(tags)) | |||||
| for k := range tags { | |||||
| keys = append(keys, k) | |||||
| } | |||||
| sort.Strings(keys) | |||||
| var b strings.Builder | |||||
| b.Grow(len(name) + len(keys)*16) | |||||
| b.WriteString(name) | |||||
| for _, k := range keys { | |||||
| b.WriteString("|") | |||||
| b.WriteString(k) | |||||
| b.WriteString("=") | |||||
| b.WriteString(tags[k]) | |||||
| } | |||||
| return b.String() | |||||
| } | |||||
| func cloneTags(tags Tags) Tags { | |||||
| if len(tags) == 0 { | |||||
| return nil | |||||
| } | |||||
| out := make(Tags, len(tags)) | |||||
| for k, v := range tags { | |||||
| out[k] = v | |||||
| } | |||||
| return out | |||||
| } | |||||
| func cloneFields(fields map[string]any) map[string]any { | |||||
| if len(fields) == 0 { | |||||
| return nil | |||||
| } | |||||
| out := make(map[string]any, len(fields)) | |||||
| for k, v := range fields { | |||||
| out[k] = v | |||||
| } | |||||
| return out | |||||
| } | |||||
| func copyMetric(m MetricPoint) MetricPoint { | |||||
| return MetricPoint{ | |||||
| Timestamp: m.Timestamp, | |||||
| Name: m.Name, | |||||
| Type: m.Type, | |||||
| Value: m.Value, | |||||
| Tags: cloneTags(m.Tags), | |||||
| } | |||||
| } | |||||
| func copyEvent(ev Event) Event { | |||||
| return Event{ | |||||
| ID: ev.ID, | |||||
| Timestamp: ev.Timestamp, | |||||
| Name: ev.Name, | |||||
| Level: ev.Level, | |||||
| Message: ev.Message, | |||||
| Tags: cloneTags(ev.Tags), | |||||
| Fields: cloneFields(ev.Fields), | |||||
| } | |||||
| } | |||||
| func p95FromDist(d *distMetric) float64 { | |||||
| if d == nil || d.count == 0 { | |||||
| return 0 | |||||
| } | |||||
| n := d.next | |||||
| if d.full { | |||||
| n = len(d.samples) | |||||
| } | |||||
| if n <= 0 { | |||||
| return d.last | |||||
| } | |||||
| buf := make([]float64, n) | |||||
| copy(buf, d.samples[:n]) | |||||
| sort.Float64s(buf) | |||||
| idx := int(float64(n-1) * 0.95) | |||||
| if idx < 0 { | |||||
| idx = 0 | |||||
| } | |||||
| if idx >= n { | |||||
| idx = n - 1 | |||||
| } | |||||
| return buf[idx] | |||||
| } | |||||
| type jsonlWriter struct { | |||||
| cfg Config | |||||
| mu sync.Mutex | |||||
| dir string | |||||
| f *os.File | |||||
| w *bufio.Writer | |||||
| currentPath string | |||||
| currentSize int64 | |||||
| seq int64 | |||||
| } | |||||
| func newJSONLWriter(cfg Config) (*jsonlWriter, error) { | |||||
| dir := filepath.Clean(cfg.PersistDir) | |||||
| if err := os.MkdirAll(dir, 0o755); err != nil { | |||||
| return nil, err | |||||
| } | |||||
| w := &jsonlWriter{cfg: cfg, dir: dir} | |||||
| if err := w.rotateLocked(); err != nil { | |||||
| return nil, err | |||||
| } | |||||
| return w, nil | |||||
| } | |||||
| func (w *jsonlWriter) Write(v persistedEnvelope) error { | |||||
| w.mu.Lock() | |||||
| defer w.mu.Unlock() | |||||
| if w.f == nil || w.w == nil { | |||||
| return nil | |||||
| } | |||||
| line, err := json.Marshal(v) | |||||
| if err != nil { | |||||
| return err | |||||
| } | |||||
| line = append(line, '\n') | |||||
| if w.currentSize+int64(len(line)) > int64(w.cfg.RotateMB)*1024*1024 { | |||||
| if err := w.rotateLocked(); err != nil { | |||||
| return err | |||||
| } | |||||
| } | |||||
| n, err := w.w.Write(line) | |||||
| w.currentSize += int64(n) | |||||
| if err != nil { | |||||
| return err | |||||
| } | |||||
| return w.w.Flush() | |||||
| } | |||||
| func (w *jsonlWriter) Close() error { | |||||
| w.mu.Lock() | |||||
| defer w.mu.Unlock() | |||||
| if w.w != nil { | |||||
| _ = w.w.Flush() | |||||
| } | |||||
| if w.f != nil { | |||||
| err := w.f.Close() | |||||
| w.f = nil | |||||
| w.w = nil | |||||
| return err | |||||
| } | |||||
| return nil | |||||
| } | |||||
| func (w *jsonlWriter) rotateLocked() error { | |||||
| if w.w != nil { | |||||
| _ = w.w.Flush() | |||||
| } | |||||
| if w.f != nil { | |||||
| _ = w.f.Close() | |||||
| } | |||||
| w.seq++ | |||||
| name := fmt.Sprintf("telemetry-%s-%04d.jsonl", time.Now().UTC().Format("20060102-150405"), w.seq) | |||||
| path := filepath.Join(w.dir, name) | |||||
| f, err := os.OpenFile(path, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0o644) | |||||
| if err != nil { | |||||
| return err | |||||
| } | |||||
| info, _ := f.Stat() | |||||
| size := int64(0) | |||||
| if info != nil { | |||||
| size = info.Size() | |||||
| } | |||||
| w.f = f | |||||
| w.w = bufio.NewWriterSize(f, 64*1024) | |||||
| w.currentPath = path | |||||
| w.currentSize = size | |||||
| _ = pruneFiles(w.dir, w.cfg.KeepFiles) | |||||
| return nil | |||||
| } | |||||
| func pruneFiles(dir string, keep int) error { | |||||
| if keep <= 0 { | |||||
| return nil | |||||
| } | |||||
| ents, err := os.ReadDir(dir) | |||||
| if err != nil { | |||||
| return err | |||||
| } | |||||
| files := make([]string, 0, len(ents)) | |||||
| for _, ent := range ents { | |||||
| if ent.IsDir() { | |||||
| continue | |||||
| } | |||||
| name := ent.Name() | |||||
| if !strings.HasPrefix(name, "telemetry-") || !strings.HasSuffix(name, ".jsonl") { | |||||
| continue | |||||
| } | |||||
| files = append(files, filepath.Join(dir, name)) | |||||
| } | |||||
| if len(files) <= keep { | |||||
| return nil | |||||
| } | |||||
| sort.Strings(files) | |||||
| for _, path := range files[:len(files)-keep] { | |||||
| _ = os.Remove(path) | |||||
| } | |||||
| return nil | |||||
| } | |||||
| func readPersistedMetrics(cfg Config, q Query) ([]MetricPoint, error) { | |||||
| files, err := listPersistedFiles(cfg.PersistDir) | |||||
| if err != nil { | |||||
| return nil, err | |||||
| } | |||||
| out := make([]MetricPoint, 0, 256) | |||||
| for _, path := range files { | |||||
| points, err := parsePersistedFile(path, q) | |||||
| if err != nil { | |||||
| continue | |||||
| } | |||||
| for _, p := range points.metrics { | |||||
| if metricMatch(p, q) { | |||||
| out = append(out, p) | |||||
| } | |||||
| } | |||||
| } | |||||
| return out, nil | |||||
| } | |||||
| func readPersistedEvents(cfg Config, q Query) ([]Event, error) { | |||||
| files, err := listPersistedFiles(cfg.PersistDir) | |||||
| if err != nil { | |||||
| return nil, err | |||||
| } | |||||
| out := make([]Event, 0, 128) | |||||
| for _, path := range files { | |||||
| points, err := parsePersistedFile(path, q) | |||||
| if err != nil { | |||||
| continue | |||||
| } | |||||
| for _, ev := range points.events { | |||||
| if eventMatch(ev, q) { | |||||
| out = append(out, ev) | |||||
| } | |||||
| } | |||||
| } | |||||
| return out, nil | |||||
| } | |||||
| type parsedFile struct { | |||||
| metrics []MetricPoint | |||||
| events []Event | |||||
| } | |||||
| func parsePersistedFile(path string, q Query) (parsedFile, error) { | |||||
| f, err := os.Open(path) | |||||
| if err != nil { | |||||
| return parsedFile{}, err | |||||
| } | |||||
| defer f.Close() | |||||
| out := parsedFile{ | |||||
| metrics: make([]MetricPoint, 0, 64), | |||||
| events: make([]Event, 0, 32), | |||||
| } | |||||
| s := bufio.NewScanner(f) | |||||
| s.Buffer(make([]byte, 0, 32*1024), 1024*1024) | |||||
| for s.Scan() { | |||||
| line := s.Bytes() | |||||
| if len(line) == 0 { | |||||
| continue | |||||
| } | |||||
| var env persistedEnvelope | |||||
| if err := json.Unmarshal(line, &env); err != nil { | |||||
| continue | |||||
| } | |||||
| if env.Metric != nil { | |||||
| out.metrics = append(out.metrics, *env.Metric) | |||||
| } else if env.Event != nil { | |||||
| out.events = append(out.events, *env.Event) | |||||
| } | |||||
| if q.Limit > 0 && len(out.metrics)+len(out.events) > q.Limit*2 { | |||||
| // keep bounded while scanning | |||||
| if len(out.metrics) > q.Limit { | |||||
| out.metrics = out.metrics[len(out.metrics)-q.Limit:] | |||||
| } | |||||
| if len(out.events) > q.Limit { | |||||
| out.events = out.events[len(out.events)-q.Limit:] | |||||
| } | |||||
| } | |||||
| } | |||||
| return out, s.Err() | |||||
| } | |||||
| func listPersistedFiles(dir string) ([]string, error) { | |||||
| ents, err := os.ReadDir(dir) | |||||
| if err != nil { | |||||
| return nil, err | |||||
| } | |||||
| files := make([]string, 0, len(ents)) | |||||
| for _, ent := range ents { | |||||
| if ent.IsDir() { | |||||
| continue | |||||
| } | |||||
| name := ent.Name() | |||||
| if strings.HasPrefix(name, "telemetry-") && strings.HasSuffix(name, ".jsonl") { | |||||
| files = append(files, filepath.Join(dir, name)) | |||||
| } | |||||
| } | |||||
| sort.Strings(files) | |||||
| return files, nil | |||||
| } | |||||
| func ParseTimeQuery(raw string) (time.Time, error) { | |||||
| raw = strings.TrimSpace(raw) | |||||
| if raw == "" { | |||||
| return time.Time{}, nil | |||||
| } | |||||
| if ms, err := strconv.ParseInt(raw, 10, 64); err == nil { | |||||
| if ms > 1e12 { | |||||
| return time.UnixMilli(ms).UTC(), nil | |||||
| } | |||||
| return time.Unix(ms, 0).UTC(), nil | |||||
| } | |||||
| if t, err := time.Parse(time.RFC3339Nano, raw); err == nil { | |||||
| return t.UTC(), nil | |||||
| } | |||||
| if t, err := time.Parse(time.RFC3339, raw); err == nil { | |||||
| return t.UTC(), nil | |||||
| } | |||||
| return time.Time{}, errors.New("invalid time query") | |||||
| } | |||||
| func TagsWith(base Tags, key string, value any) Tags { | |||||
| out := cloneTags(base) | |||||
| if out == nil { | |||||
| out = Tags{} | |||||
| } | |||||
| out[key] = fmt.Sprint(value) | |||||
| return out | |||||
| } | |||||
| func TagsFromPairs(kv ...string) Tags { | |||||
| if len(kv) < 2 { | |||||
| return nil | |||||
| } | |||||
| out := Tags{} | |||||
| for i := 0; i+1 < len(kv); i += 2 { | |||||
| k := strings.TrimSpace(kv[i]) | |||||
| if k == "" { | |||||
| continue | |||||
| } | |||||
| out[k] = kv[i+1] | |||||
| } | |||||
| if len(out) == 0 { | |||||
| return nil | |||||
| } | |||||
| return out | |||||
| } | |||||
| func min(a int, b int) int { | |||||
| if a < b { | |||||
| return a | |||||
| } | |||||
| return b | |||||
| } | |||||