From 3058ffc2ad30f5cfcbc303dcb177ea1743acc2bd Mon Sep 17 00:00:00 2001 From: Alfred Date: Mon, 16 Mar 2026 09:47:22 +0100 Subject: [PATCH] feat: follow script assets --- cmd/radiostreamscan/main.go | 811 ++++++++++++++------------- internal/extractor/extractor.go | 342 +++++------ internal/extractor/extractor_test.go | 89 +-- 3 files changed, 662 insertions(+), 580 deletions(-) diff --git a/cmd/radiostreamscan/main.go b/cmd/radiostreamscan/main.go index 348f95c..0367649 100644 --- a/cmd/radiostreamscan/main.go +++ b/cmd/radiostreamscan/main.go @@ -1,114 +1,114 @@ package main import ( - "bufio" - "encoding/csv" - "encoding/json" - "flag" - "fmt" - "io" - "net/http" - "net/url" - "os" - "strings" - "sync" - "time" - - "radio-stream-extractor/internal/extractor" + "bufio" + "encoding/csv" + "encoding/json" + "flag" + "fmt" + "io" + "net/http" + "net/url" + "os" + "strings" + "sync" + "time" + + "radio-stream-extractor/internal/extractor" ) type scanResult struct { - URL string `json:"url"` - Streams []string `json:"streams"` - Playlists []string `json:"playlists,omitempty"` - Probes []probeResult `json:"probes,omitempty"` - Error string `json:"error,omitempty"` - FetchedAt time.Time `json:"fetchedAt"` - FromPlaylist bool `json:"fromPlaylist"` + URL string `json:"url"` + Streams []string `json:"streams"` + Playlists []string `json:"playlists,omitempty"` + Probes []probeResult `json:"probes,omitempty"` + Error string `json:"error,omitempty"` + FetchedAt time.Time `json:"fetchedAt"` + FromPlaylist bool `json:"fromPlaylist"` } type probeResult struct { - URL string `json:"url"` - Status string `json:"status"` - ContentType string `json:"contentType,omitempty"` + URL string `json:"url"` + Status string `json:"status"` + ContentType string `json:"contentType,omitempty"` } type config struct { - Format string - Probe bool - Headers headerList - Proxy string - HistoryPath string - Watch time.Duration - Concurrency int + Format string + Probe bool + Headers headerList + Proxy string + HistoryPath string + Watch time.Duration + Concurrency int } type headerList []string func (h *headerList) String() string { return strings.Join(*h, ", ") } func (h *headerList) Set(v string) error { - *h = append(*h, v) - return nil + *h = append(*h, v) + return nil } func main() { - port := flag.String("port", ":8080", "listen address for the web server (default :8080)") - web := flag.Bool("web", false, "force web-server mode even when URLs are provided") - - cfg := config{} - flag.StringVar(&cfg.Format, "format", "text", "output format: text|json|csv|pls") - flag.BoolVar(&cfg.Probe, "probe", true, "probe discovered stream URLs with HTTP HEAD") - flag.Var(&cfg.Headers, "header", "custom HTTP header (repeatable), e.g. -header 'Referer: https://example.com'") - flag.StringVar(&cfg.Proxy, "proxy", "", "HTTP proxy URL (optional)") - flag.StringVar(&cfg.HistoryPath, "history", "history.jsonl", "path to JSONL history log (empty to disable)") - flag.DurationVar(&cfg.Watch, "watch", 0, "repeat scan in CLI mode at interval (e.g. 30s, 2m)") - flag.IntVar(&cfg.Concurrency, "concurrency", 4, "number of concurrent fetch workers") - - flag.Usage = func() { - fmt.Fprintf(flag.CommandLine.Output(), "Usage: %s [flags] [url...]\n", os.Args[0]) - flag.PrintDefaults() - } - flag.Parse() - - urls := flag.Args() - client := newHTTPClient(cfg.Proxy) - history := newHistoryWriter(cfg.HistoryPath) - - if *web || len(urls) == 0 { - if err := runWebMode(*port, client, &cfg, history); err != nil { - fmt.Fprintf(os.Stderr, "web mode failed: %v\n", err) - os.Exit(1) - } - return - } - - runCLIMode(urls, client, &cfg, history) + port := flag.String("port", ":8080", "listen address for the web server (default :8080)") + web := flag.Bool("web", false, "force web-server mode even when URLs are provided") + + cfg := config{} + flag.StringVar(&cfg.Format, "format", "text", "output format: text|json|csv|pls") + flag.BoolVar(&cfg.Probe, "probe", true, "probe discovered stream URLs with HTTP HEAD") + flag.Var(&cfg.Headers, "header", "custom HTTP header (repeatable), e.g. -header 'Referer: https://example.com'") + flag.StringVar(&cfg.Proxy, "proxy", "", "HTTP proxy URL (optional)") + flag.StringVar(&cfg.HistoryPath, "history", "history.jsonl", "path to JSONL history log (empty to disable)") + flag.DurationVar(&cfg.Watch, "watch", 0, "repeat scan in CLI mode at interval (e.g. 30s, 2m)") + flag.IntVar(&cfg.Concurrency, "concurrency", 4, "number of concurrent fetch workers") + + flag.Usage = func() { + fmt.Fprintf(flag.CommandLine.Output(), "Usage: %s [flags] [url...]\n", os.Args[0]) + flag.PrintDefaults() + } + flag.Parse() + + urls := flag.Args() + client := newHTTPClient(cfg.Proxy) + history := newHistoryWriter(cfg.HistoryPath) + + if *web || len(urls) == 0 { + if err := runWebMode(*port, client, &cfg, history); err != nil { + fmt.Fprintf(os.Stderr, "web mode failed: %v\n", err) + os.Exit(1) + } + return + } + + runCLIMode(urls, client, &cfg, history) } func runCLIMode(urls []string, client *http.Client, cfg *config, history *historyWriter) { - for { - results := scanURLs(urls, client, cfg) - outputResults(results, cfg.Format, os.Stdout) - history.Write(results) - if cfg.Watch == 0 { - return - } - time.Sleep(cfg.Watch) - } + for { + results := scanURLs(urls, client, cfg) + outputResults(results, cfg.Format, os.Stdout) + history.Write(results) + if cfg.Watch == 0 { + return + } + time.Sleep(cfg.Watch) + } } func runWebMode(addr string, client *http.Client, cfg *config, history *historyWriter) error { - mux := http.NewServeMux() - mux.HandleFunc("/", indexHandler) - mux.HandleFunc("/scan", makeScanHandler(client, cfg, history)) - mux.HandleFunc("/watch", watchHandler) + mux := http.NewServeMux() + mux.HandleFunc("/", indexHandler) + mux.HandleFunc("/scan", makeScanHandler(client, cfg, history)) + mux.HandleFunc("/watch", watchHandler) - fmt.Printf("radiostreamscan listening on %s (GET /scan?url=... or POST url=...)\n", addr) - return http.ListenAndServe(addr, mux) + fmt.Printf("radiostreamscan listening on %s (GET /scan?url=... or POST url=...)\n", addr) + return http.ListenAndServe(addr, mux) } func indexHandler(w http.ResponseWriter, r *http.Request) { - fmt.Fprintf(w, ` + fmt.Fprintf(w, ` radiostreamscan @@ -136,10 +136,10 @@ func indexHandler(w http.ResponseWriter, r *http.Request) { } func watchHandler(w http.ResponseWriter, r *http.Request) { - urls := normalizeURLInputs(r.URL.Query()["url"]) - interval := r.URL.Query().Get("interval") - probe := r.URL.Query().Get("probe") - fmt.Fprintf(w, ` + urls := normalizeURLInputs(r.URL.Query()["url"]) + interval := r.URL.Query().Get("interval") + probe := r.URL.Query().Get("probe") + fmt.Fprintf(w, ` radiostreamscan results