diff --git a/internal/pipeline/candidate_fusion.go b/internal/pipeline/candidate_fusion.go index d754896..aa8aa55 100644 --- a/internal/pipeline/candidate_fusion.go +++ b/internal/pipeline/candidate_fusion.go @@ -16,10 +16,12 @@ func AddCandidateEvidence(candidate *Candidate, evidence LevelEvidence) { evLevel = "unknown" } if evLevel == levelName && ev.Provenance == evidence.Provenance { + RefreshCandidateEvidenceState(candidate) return } } candidate.Evidence = append(candidate.Evidence, evidence) + RefreshCandidateEvidenceState(candidate) } func MergeCandidateEvidence(dst *Candidate, src Candidate) { @@ -32,18 +34,8 @@ func MergeCandidateEvidence(dst *Candidate, src Candidate) { } func CandidateEvidenceLevelCount(candidate Candidate) int { - if len(candidate.Evidence) == 0 { - return 0 - } - levels := map[string]struct{}{} - for _, ev := range candidate.Evidence { - name := ev.Level.Name - if name == "" { - name = "unknown" - } - levels[name] = struct{}{} - } - return len(levels) + state := CandidateEvidenceStateFor(candidate) + return state.DetectionLevelCount } func FuseCandidates(primary []Candidate, derived []Candidate) []Candidate { @@ -74,6 +66,9 @@ func FuseCandidates(primary []Candidate, derived []Candidate) []Candidate { } out = append(out, cand) } + for i := range out { + RefreshCandidateEvidenceState(&out[i]) + } return out } @@ -113,6 +108,9 @@ func candidateSpanHz(candidate Candidate) float64 { func candidateBinHz(candidate Candidate) float64 { for _, ev := range candidate.Evidence { + if IsPresentationLevel(ev.Level) || !IsDetectionLevel(ev.Level) { + continue + } if ev.Level.BinHz > 0 { return ev.Level.BinHz } diff --git a/internal/pipeline/candidate_fusion_test.go b/internal/pipeline/candidate_fusion_test.go index ae33cfd..2ffd179 100644 --- a/internal/pipeline/candidate_fusion_test.go +++ b/internal/pipeline/candidate_fusion_test.go @@ -30,6 +30,9 @@ func TestFuseCandidatesDedup(t *testing.T) { if got := CandidateEvidenceLevelCount(fused[0]); got != 2 { t.Fatalf("expected 2 evidence levels after fuse, got %d", got) } + if fused[0].EvidenceState == nil || !fused[0].EvidenceState.Fused || !fused[0].EvidenceState.MultiLevelConfirmed { + t.Fatalf("expected fused multi-level evidence state, got %+v", fused[0].EvidenceState) + } } func TestFuseCandidatesSingleVsMultiResolution(t *testing.T) { diff --git a/internal/pipeline/evidence.go b/internal/pipeline/evidence.go new file mode 100644 index 0000000..f502002 --- /dev/null +++ b/internal/pipeline/evidence.go @@ -0,0 +1,173 @@ +package pipeline + +import ( + "fmt" + "sort" + "strings" +) + +// CandidateEvidenceState summarizes fused evidence semantics for a candidate. +type CandidateEvidenceState struct { + TotalLevelEntries int `json:"total_level_entries"` + LevelCount int `json:"level_count"` + DetectionLevelCount int `json:"detection_level_count"` + PrimaryLevelCount int `json:"primary_level_count,omitempty"` + DerivedLevelCount int `json:"derived_level_count,omitempty"` + PresentationLevelCount int `json:"presentation_level_count,omitempty"` + Levels []string `json:"levels,omitempty"` + Provenance []string `json:"provenance,omitempty"` + Fused bool `json:"fused,omitempty"` + DerivedOnly bool `json:"derived_only,omitempty"` + MultiLevelConfirmed bool `json:"multi_level_confirmed,omitempty"` + MultiLevelConfirmedHint string `json:"multi_level_confirmed_hint,omitempty"` +} + +// EvidenceScoreDetails explains how evidence influenced refinement scoring. +type EvidenceScoreDetails struct { + RawScore float64 `json:"raw_score"` + Weight float64 `json:"weight"` + WeightedScore float64 `json:"weighted_score"` + DetectionLevels int `json:"detection_levels"` + PrimaryLevels int `json:"primary_levels,omitempty"` + DerivedLevels int `json:"derived_levels,omitempty"` + ProvenanceCount int `json:"provenance_count,omitempty"` + DerivedOnly bool `json:"derived_only,omitempty"` + MultiLevelConfirmed bool `json:"multi_level_confirmed,omitempty"` + MultiLevelBonus float64 `json:"multi_level_bonus,omitempty"` + ProvenanceBonus float64 `json:"provenance_bonus,omitempty"` + DerivedPenalty float64 `json:"derived_penalty,omitempty"` + StrategyBias float64 `json:"strategy_bias,omitempty"` +} + +// IsPresentationLevel reports whether a level is intended only for presentation. +func IsPresentationLevel(level AnalysisLevel) bool { + role := strings.ToLower(strings.TrimSpace(level.Role)) + truth := strings.ToLower(strings.TrimSpace(level.Truth)) + name := strings.ToLower(strings.TrimSpace(level.Name)) + if strings.Contains(role, "presentation") || strings.Contains(truth, "presentation") { + return true + } + return strings.Contains(name, "presentation") || strings.Contains(name, "display") +} + +// IsDetectionLevel reports whether a level is intended for detection/analysis. +func IsDetectionLevel(level AnalysisLevel) bool { + if IsPresentationLevel(level) { + return false + } + role := strings.ToLower(strings.TrimSpace(level.Role)) + truth := strings.ToLower(strings.TrimSpace(level.Truth)) + name := strings.ToLower(strings.TrimSpace(level.Name)) + if strings.Contains(truth, "surveillance") { + return true + } + if role == "surveillance" || strings.HasPrefix(role, "surveillance-") { + return true + } + return strings.Contains(name, "surveillance") +} + +func isPrimarySurveillanceLevel(level AnalysisLevel) bool { + role := strings.ToLower(strings.TrimSpace(level.Role)) + name := strings.ToLower(strings.TrimSpace(level.Name)) + return role == "surveillance" || name == "surveillance" +} + +func isDerivedSurveillanceLevel(level AnalysisLevel) bool { + role := strings.ToLower(strings.TrimSpace(level.Role)) + name := strings.ToLower(strings.TrimSpace(level.Name)) + if strings.HasPrefix(role, "surveillance-") && role != "surveillance" { + return true + } + if strings.HasPrefix(name, "surveillance-") && name != "surveillance" { + return true + } + return strings.Contains(role, "lowres") || strings.Contains(name, "lowres") || strings.Contains(name, "derived") +} + +func evidenceLevelKey(level AnalysisLevel) string { + if level.Name != "" { + return level.Name + } + if level.SampleRate > 0 && level.FFTSize > 0 { + return fmt.Sprintf("sr%d-fft%d", level.SampleRate, level.FFTSize) + } + return "unknown" +} + +// CandidateEvidenceStateFor builds a fused evidence state from a candidate. +func CandidateEvidenceStateFor(candidate Candidate) CandidateEvidenceState { + state := CandidateEvidenceState{} + if len(candidate.Evidence) == 0 { + return state + } + levelSet := map[string]struct{}{} + provenanceSet := map[string]struct{}{} + detectionLevels := map[string]struct{}{} + primaryLevels := map[string]struct{}{} + derivedLevels := map[string]struct{}{} + presentationLevels := map[string]struct{}{} + for _, ev := range candidate.Evidence { + levelKey := evidenceLevelKey(ev.Level) + levelSet[levelKey] = struct{}{} + if ev.Provenance != "" { + provenanceSet[ev.Provenance] = struct{}{} + } + if IsPresentationLevel(ev.Level) { + presentationLevels[levelKey] = struct{}{} + continue + } + if IsDetectionLevel(ev.Level) { + detectionLevels[levelKey] = struct{}{} + if isPrimarySurveillanceLevel(ev.Level) { + primaryLevels[levelKey] = struct{}{} + } else if isDerivedSurveillanceLevel(ev.Level) { + derivedLevels[levelKey] = struct{}{} + } + } + } + state.TotalLevelEntries = len(candidate.Evidence) + state.LevelCount = len(levelSet) + state.DetectionLevelCount = len(detectionLevels) + state.PrimaryLevelCount = len(primaryLevels) + state.DerivedLevelCount = len(derivedLevels) + state.PresentationLevelCount = len(presentationLevels) + state.Levels = sortedKeys(levelSet) + state.Provenance = sortedKeys(provenanceSet) + state.Fused = state.LevelCount > 1 || len(state.Provenance) > 1 + state.DerivedOnly = state.DerivedLevelCount > 0 && state.PrimaryLevelCount == 0 && state.DetectionLevelCount == state.DerivedLevelCount + state.MultiLevelConfirmed = state.DetectionLevelCount >= 2 + if state.MultiLevelConfirmed { + if state.PrimaryLevelCount > 0 && state.DerivedLevelCount > 0 { + state.MultiLevelConfirmedHint = "primary+derived" + } else { + state.MultiLevelConfirmedHint = "multi-detection" + } + } + return state +} + +// RefreshCandidateEvidenceState updates the candidate's cached evidence summary. +func RefreshCandidateEvidenceState(candidate *Candidate) { + if candidate == nil { + return + } + state := CandidateEvidenceStateFor(*candidate) + if state.TotalLevelEntries == 0 { + candidate.EvidenceState = nil + return + } + candidate.EvidenceState = &state +} + +func sortedKeys(src map[string]struct{}) []string { + if len(src) == 0 { + return nil + } + out := make([]string, 0, len(src)) + for k := range src { + out = append(out, k) + } + sort.Strings(out) + return out +} diff --git a/internal/pipeline/scheduler.go b/internal/pipeline/scheduler.go index cb563f4..2824c6c 100644 --- a/internal/pipeline/scheduler.go +++ b/internal/pipeline/scheduler.go @@ -20,11 +20,12 @@ type RefinementScoreModel struct { } type RefinementScoreDetails struct { - SNRScore float64 `json:"snr_score"` - BandwidthScore float64 `json:"bandwidth_score"` - PeakScore float64 `json:"peak_score"` - PolicyBoost float64 `json:"policy_boost"` - EvidenceScore float64 `json:"evidence_score"` + SNRScore float64 `json:"snr_score"` + BandwidthScore float64 `json:"bandwidth_score"` + PeakScore float64 `json:"peak_score"` + PolicyBoost float64 `json:"policy_boost"` + EvidenceScore float64 `json:"evidence_score"` + EvidenceDetail *EvidenceScoreDetails `json:"evidence_detail,omitempty"` } type RefinementScore struct { @@ -114,35 +115,41 @@ func BuildRefinementPlan(candidates []Candidate, policy Policy) RefinementPlan { scored := make([]ScheduledCandidate, 0, len(candidates)) workItems := make([]RefinementWorkItem, 0, len(candidates)) for _, c := range candidates { - if !candidateInMonitor(policy, c) { + candidate := c + RefreshCandidateEvidenceState(&candidate) + if !candidateInMonitor(policy, candidate) { plan.DroppedByMonitor++ workItems = append(workItems, RefinementWorkItem{ - Candidate: c, + Candidate: candidate, Status: RefinementStatusDropped, Reason: RefinementReasonMonitorGate, }) continue } - if c.SNRDb < policy.MinCandidateSNRDb { + if candidate.SNRDb < policy.MinCandidateSNRDb { plan.DroppedBySNR++ workItems = append(workItems, RefinementWorkItem{ - Candidate: c, + Candidate: candidate, Status: RefinementStatusDropped, Reason: RefinementReasonBelowSNR, }) continue } - snrScore := c.SNRDb * scoreModel.SNRWeight + snrScore := candidate.SNRDb * scoreModel.SNRWeight bwScore := 0.0 peakScore := 0.0 - policyBoost := CandidatePriorityBoost(policy, c.Hint) - if c.BandwidthHz > 0 { - bwScore = minFloat64(c.BandwidthHz/25000.0, 6) * scoreModel.BandwidthWeight + policyBoost := CandidatePriorityBoost(policy, candidate.Hint) + if candidate.BandwidthHz > 0 { + bwScore = minFloat64(candidate.BandwidthHz/25000.0, 6) * scoreModel.BandwidthWeight } - if c.PeakDb > 0 { - peakScore = (c.PeakDb / 20.0) * scoreModel.PeakWeight + if candidate.PeakDb > 0 { + peakScore = (candidate.PeakDb / 20.0) * scoreModel.PeakWeight } - evidenceScore := candidateEvidenceScore(c) * scoreModel.EvidenceWeight + rawEvidenceScore, evidenceDetail := candidateEvidenceScore(candidate, strategy) + evidenceDetail.Weight = scoreModel.EvidenceWeight + evidenceDetail.RawScore = rawEvidenceScore + evidenceDetail.WeightedScore = rawEvidenceScore * scoreModel.EvidenceWeight + evidenceScore := evidenceDetail.WeightedScore priority := snrScore + bwScore + peakScore + policyBoost priority += evidenceScore score := &RefinementScore{ @@ -153,17 +160,18 @@ func BuildRefinementPlan(candidates []Candidate, policy Policy) RefinementPlan { PeakScore: peakScore, PolicyBoost: policyBoost, EvidenceScore: evidenceScore, + EvidenceDetail: &evidenceDetail, }, Weights: &scoreModel, } scored = append(scored, ScheduledCandidate{ - Candidate: c, + Candidate: candidate, Priority: priority, Score: score, Breakdown: &score.Breakdown, }) workItems = append(workItems, RefinementWorkItem{ - Candidate: c, + Candidate: candidate, Priority: priority, Score: score, Breakdown: &score.Breakdown, @@ -250,12 +258,67 @@ func applyStrategyWeights(strategy string, model RefinementScoreModel) Refinemen return model } -func candidateEvidenceScore(candidate Candidate) float64 { - levels := CandidateEvidenceLevelCount(candidate) - if levels <= 1 { - return 0 +func candidateEvidenceScore(candidate Candidate, strategy string) (float64, EvidenceScoreDetails) { + state := CandidateEvidenceStateFor(candidate) + details := EvidenceScoreDetails{ + DetectionLevels: state.DetectionLevelCount, + PrimaryLevels: state.PrimaryLevelCount, + DerivedLevels: state.DerivedLevelCount, + ProvenanceCount: len(state.Provenance), + DerivedOnly: state.DerivedOnly, + MultiLevelConfirmed: state.MultiLevelConfirmed, } - return float64(levels - 1) + score := 0.0 + if state.MultiLevelConfirmed && state.DetectionLevelCount > 1 { + bonus := 0.85 * float64(state.DetectionLevelCount-1) + score += bonus + details.MultiLevelBonus = bonus + } + if len(state.Provenance) > 1 { + bonus := 0.15 * float64(len(state.Provenance)-1) + score += bonus + details.ProvenanceBonus = bonus + } + if state.DerivedOnly { + penalty := 0.35 + score -= penalty + details.DerivedPenalty = -penalty + } + switch strings.ToLower(strings.TrimSpace(strategy)) { + case "multi-resolution", "multi", "multi-res", "multi_res": + if state.DerivedOnly { + bias := 0.2 + score += bias + details.StrategyBias = bias + } else if state.MultiLevelConfirmed { + bias := 0.1 + score += bias + details.StrategyBias = bias + } + case "digital-hunting": + if state.DerivedOnly { + bias := -0.15 + score += bias + details.StrategyBias = bias + } else if state.MultiLevelConfirmed { + bias := 0.05 + score += bias + details.StrategyBias = bias + } + case "archive-oriented": + if state.DerivedOnly { + bias := -0.1 + score += bias + details.StrategyBias = bias + } + case "single-resolution": + if state.MultiLevelConfirmed { + bias := 0.05 + score += bias + details.StrategyBias = bias + } + } + return score, details } func minFloat64(a, b float64) float64 { diff --git a/internal/pipeline/scheduler_test.go b/internal/pipeline/scheduler_test.go index e6fa4f4..38834f8 100644 --- a/internal/pipeline/scheduler_test.go +++ b/internal/pipeline/scheduler_test.go @@ -173,6 +173,60 @@ func TestScheduleCandidatesEvidenceBoost(t *testing.T) { if plan.Ranked[0].Breakdown == nil || plan.Ranked[0].Breakdown.EvidenceScore <= 0 { t.Fatalf("expected evidence score to be populated, got %+v", plan.Ranked[0].Breakdown) } + if plan.Ranked[0].Breakdown.EvidenceDetail == nil || !plan.Ranked[0].Breakdown.EvidenceDetail.MultiLevelConfirmed { + t.Fatalf("expected evidence detail for multi-level candidate, got %+v", plan.Ranked[0].Breakdown) + } +} + +func TestScheduleCandidatesDerivedOnlyPenalty(t *testing.T) { + policy := Policy{MaxRefinementJobs: 2, MinCandidateSNRDb: 0} + primary := Candidate{ + ID: 1, + SNRDb: 10, + BandwidthHz: 12000, + Evidence: []LevelEvidence{ + {Level: AnalysisLevel{Name: "surveillance", Role: "surveillance", Truth: "surveillance"}}, + }, + } + derived := Candidate{ + ID: 2, + SNRDb: 10, + BandwidthHz: 12000, + Evidence: []LevelEvidence{ + {Level: AnalysisLevel{Name: "surveillance-lowres", Role: "surveillance-lowres", Truth: "surveillance"}}, + }, + } + plan := BuildRefinementPlan([]Candidate{derived, primary}, policy) + if len(plan.Ranked) != 2 { + t.Fatalf("expected ranked candidates, got %d", len(plan.Ranked)) + } + if plan.Ranked[0].Candidate.ID != primary.ID { + t.Fatalf("expected primary evidence to outrank derived-only, got %+v", plan.Ranked[0]) + } +} + +func TestScheduleCandidatesDerivedOnlyStrategyBias(t *testing.T) { + cand := Candidate{ + ID: 1, + SNRDb: 9, + BandwidthHz: 12000, + Evidence: []LevelEvidence{ + {Level: AnalysisLevel{Name: "surveillance-lowres", Role: "surveillance-lowres", Truth: "surveillance"}}, + }, + } + singlePlan := BuildRefinementPlan([]Candidate{cand}, Policy{MinCandidateSNRDb: 0}) + multiPlan := BuildRefinementPlan([]Candidate{cand}, Policy{MinCandidateSNRDb: 0, SurveillanceStrategy: "multi-resolution"}) + if len(singlePlan.Ranked) == 0 || len(multiPlan.Ranked) == 0 { + t.Fatalf("expected ranked candidates in both plans") + } + singleScore := singlePlan.Ranked[0].Breakdown.EvidenceScore + multiScore := multiPlan.Ranked[0].Breakdown.EvidenceScore + if multiScore <= singleScore { + t.Fatalf("expected multi-resolution strategy to improve derived-only evidence score, got %.3f vs %.3f", multiScore, singleScore) + } + if multiPlan.Ranked[0].Breakdown.EvidenceDetail == nil || multiPlan.Ranked[0].Breakdown.EvidenceDetail.StrategyBias <= 0 { + t.Fatalf("expected strategy bias detail for multi-resolution, got %+v", multiPlan.Ranked[0].Breakdown.EvidenceDetail) + } } func TestBuildRefinementPlanPriorityStats(t *testing.T) { diff --git a/internal/pipeline/types.go b/internal/pipeline/types.go index f453edb..70889e6 100644 --- a/internal/pipeline/types.go +++ b/internal/pipeline/types.go @@ -8,17 +8,18 @@ import ( // Candidate is the coarse output of the surveillance detector. // It intentionally stays lightweight and cheap to produce. type Candidate struct { - ID int64 `json:"id"` - CenterHz float64 `json:"center_hz"` - BandwidthHz float64 `json:"bandwidth_hz"` - PeakDb float64 `json:"peak_db"` - SNRDb float64 `json:"snr_db"` - FirstBin int `json:"first_bin"` - LastBin int `json:"last_bin"` - NoiseDb float64 `json:"noise_db,omitempty"` - Source string `json:"source,omitempty"` - Hint string `json:"hint,omitempty"` - Evidence []LevelEvidence `json:"evidence,omitempty"` + ID int64 `json:"id"` + CenterHz float64 `json:"center_hz"` + BandwidthHz float64 `json:"bandwidth_hz"` + PeakDb float64 `json:"peak_db"` + SNRDb float64 `json:"snr_db"` + FirstBin int `json:"first_bin"` + LastBin int `json:"last_bin"` + NoiseDb float64 `json:"noise_db,omitempty"` + Source string `json:"source,omitempty"` + Hint string `json:"hint,omitempty"` + Evidence []LevelEvidence `json:"evidence,omitempty"` + EvidenceState *CandidateEvidenceState `json:"evidence_state,omitempty"` } // LevelEvidence captures which analysis level produced a candidate.