jan
/
radio-stream-extractor
1
0
Форкнуть 0
Код Задачи 0 Pull Request'ы 0 Релизы 0 Вики Активность
Вы не можете выбрать более 25 тем Темы должны начинаться с буквы или цифры, могут содержать дефисы(-) и должны содержать не более 35 символов.
4 коммитов
1 ветка
58KB
Дерево: 76a7819dad
Ветки Теги
${ item.name }
Создать ветку ${ searchTerm }
из '76a7819dad'
${ noResults }
radio-stream-extractor/cmd/radiostreamscan/main.go

787 строки
20KB
Исходник Blame История 

  1. package main

  2. 

  3. import (

  4. 	"bufio"

  5. 	"context"

  6. 	"encoding/csv"

  7. 	"encoding/json"

  8. 	"flag"

  9. 	"fmt"

  10. 	"io"

  11. 	"net"

  12. 	"net/http"

  13. 	"net/url"

  14. 	"os"

  15. 	"strings"

  16. 	"sync"

  17. 	"time"

  18. 

  19. 	"radio-stream-extractor/internal/extractor"

  20. )

  21. 

  22. type scanResult struct {

  23. 	URL          string        `json:"url"`

  24. 	Streams      []string      `json:"streams"`

  25. 	Playlists    []string      `json:"playlists,omitempty"`

  26. 	Probes       []probeResult `json:"probes,omitempty"`

  27. 	Error        string        `json:"error,omitempty"`

  28. 	FetchedAt    time.Time     `json:"fetchedAt"`

  29. 	FromPlaylist bool          `json:"fromPlaylist"`

  30. }

  31. 

  32. type probeResult struct {

  33. 	URL         string `json:"url"`

  34. 	Status      string `json:"status"`

  35. 	ContentType string `json:"contentType,omitempty"`

  36. }

  37. 

  38. type config struct {

  39. 	Format         string

  40. 	Probe          bool

  41. 	Headers        headerList

  42. 	Proxy          string

  43. 	HistoryPath    string

  44. 	Watch          time.Duration

  45. 	Concurrency    int

  46. 	RequestTimeout time.Duration

  47. 	ProbeTimeout   time.Duration

  48. 	AllowPrivate   bool

  49. }

  50. 

  51. type headerList []string

  52. 

  53. func (h *headerList) String() string { return strings.Join(*h, ", ") }

  54. func (h *headerList) Set(v string) error {

  55. 	*h = append(*h, v)

  56. 	return nil

  57. }

  58. 

  59. func main() {

  60. 	port := flag.String("port", ":8080", "listen address for the web server (default :8080)")

  61. 	web := flag.Bool("web", false, "force web-server mode even when URLs are provided")

  62. 

  63. 	cfg := config{}

  64. 	flag.StringVar(&cfg.Format, "format", "text", "output format: text|json|csv|pls")

  65. 	flag.BoolVar(&cfg.Probe, "probe", true, "probe discovered stream URLs with HTTP HEAD")

  66. 	flag.Var(&cfg.Headers, "header", "custom HTTP header (repeatable), e.g. -header 'Referer: https://example.com'")

  67. 	flag.StringVar(&cfg.Proxy, "proxy", "", "HTTP proxy URL (optional)")

  68. 	flag.StringVar(&cfg.HistoryPath, "history", "history.jsonl", "path to JSONL history log (empty to disable)")

  69. 	flag.DurationVar(&cfg.Watch, "watch", 0, "repeat scan in CLI mode at interval (e.g. 30s, 2m)")

  70. 	flag.IntVar(&cfg.Concurrency, "concurrency", 4, "number of concurrent fetch workers")

  71. 	flag.DurationVar(&cfg.RequestTimeout, "timeout", 15*time.Second, "timeout per HTTP request (e.g. 10s, 2m)")

  72. 	flag.DurationVar(&cfg.ProbeTimeout, "probe-timeout", 8*time.Second, "timeout for probing stream URLs")

  73. 	flag.BoolVar(&cfg.AllowPrivate, "allow-private", false, "allow requests to private/localhost addresses")

  74. 

  75. 	flag.Usage = func() {

  76. 		fmt.Fprintf(flag.CommandLine.Output(), "Usage: %s [flags] <url> [url...]\n", os.Args[0])

  77. 		flag.PrintDefaults()

  78. 	}

  79. 	flag.Parse()

  80. 

  81. 	urls := flag.Args()

  82. 	client := newHTTPClient(cfg.Proxy, cfg.RequestTimeout)

  83. 	history := newHistoryWriter(cfg.HistoryPath)

  84. 

  85. 	if *web || len(urls) == 0 {

  86. 		if err := runWebMode(*port, client, &cfg, history); err != nil {

  87. 			fmt.Fprintf(os.Stderr, "web mode failed: %v\n", err)

  88. 			os.Exit(1)

  89. 		}

  90. 		return

  91. 	}

  92. 

  93. 	runCLIMode(urls, client, &cfg, history)

  94. }

  95. 

  96. func runCLIMode(urls []string, client *http.Client, cfg *config, history *historyWriter) {

  97. 	for {

  98. 		results := scanURLs(urls, client, cfg)

  99. 		if err := outputResults(results, cfg.Format, os.Stdout); err != nil {

  100. 			fmt.Fprintf(os.Stderr, "output failed: %v\n", err)

  101. 			return

  102. 		}

  103. 		history.Write(results)

  104. 		if cfg.Watch == 0 {

  105. 			return

  106. 		}

  107. 		time.Sleep(cfg.Watch)

  108. 	}

  109. }

  110. 

  111. func runWebMode(addr string, client *http.Client, cfg *config, history *historyWriter) error {

  112. 	mux := http.NewServeMux()

  113. 	mux.HandleFunc("/", indexHandler)

  114. 	mux.HandleFunc("/scan", makeScanHandler(client, cfg, history))

  115. 	mux.HandleFunc("/watch", watchHandler)

  116. 

  117. 	fmt.Printf("radiostreamscan listening on %s (GET /scan?url=... or POST url=...)\n", addr)

  118. 	return http.ListenAndServe(addr, mux)

  119. }

  120. 

  121. func indexHandler(w http.ResponseWriter, r *http.Request) {

  122. 	fmt.Fprintf(w, `<!doctype html>

  123. <html>

  124.   <head><meta charset="utf-8"><title>radiostreamscan</title></head>

  125.   <body>

  126.     <h1>radiostreamscan</h1>

  127.     <form method="get" action="/watch">

  128.       <label>Stream-URLs (eine pro Zeile)</label><br/>

  129.       <textarea name="url" rows="6" cols="80" required></textarea><br/>

  130.       <label>Format

  131.         <select name="format">

  132.           <option value="json">json</option>

  133.           <option value="text">text</option>

  134.           <option value="csv">csv</option>

  135.           <option value="pls">pls</option>

  136.         </select>

  137.       </label>

  138.       <label>Auto-Refresh (Sekunden)

  139.         <input type="number" name="interval" value="0" min="0" />

  140.       </label>

  141.       <label><input type="checkbox" name="probe" value="1" checked> Probing</label>

  142.       <button type="submit">Scan</button>

  143.     </form>

  144.     <p>Mehrere URLs: /scan?url=a&url=b&url=c</p>

  145.   </body>

  146. </html>`)

  147. }

  148. 

  149. func watchHandler(w http.ResponseWriter, r *http.Request) {

  150. 	urls := normalizeURLInputs(r.URL.Query()["url"])

  151. 	interval := r.URL.Query().Get("interval")

  152. 	probe := r.URL.Query().Get("probe")

  153. 	fmt.Fprintf(w, `<!doctype html>

  154. <html>

  155. <head><meta charset="utf-8"><title>radiostreamscan results</title>

  156. <style>

  157.   body { font-family: Arial, sans-serif; }

  158.   .url-block { margin: 10px 0; padding: 10px; border: 1px solid #ccc; }

  159.   .error { color: #b00020; }

  160.   button { margin: 8px 0; }

  161. </style>

  162. </head>

  163. <body>

  164. <h1>radiostreamscan results</h1>

  165. <button id="copy">Alle Streams kopieren</button>

  166. <div id="output">Loading...</div>

  167. <textarea id="clipboard" style="position:absolute; left:-9999px; top:-9999px;"></textarea>

  168. <script>

  169.   const urls = %q.split("\n").filter(Boolean);

  170.   const interval = %q;

  171.   const probe = %q;

  172. 

  173.   async function fetchData() {

  174.     const params = new URLSearchParams();

  175.     urls.forEach(u => params.append("url", u));

  176.     params.set("format", "json");

  177.     if (probe) params.set("probe", "1");

  178. 

  179.     const res = await fetch("/scan?" + params.toString());

  180.     const data = await res.json();

  181. 

  182.     const container = document.getElementById("output");

  183.     container.innerHTML = "";

  184. 

  185.     const allStreams = [];

  186. 

  187.     data.forEach(item => {

  188.       const block = document.createElement("div");

  189.       block.className = "url-block";

  190.       const title = document.createElement("h3");

  191.       title.textContent = item.url;

  192.       block.appendChild(title);

  193. 

  194.       if (item.error) {

  195.         const err = document.createElement("div");

  196.         err.className = "error";

  197.         err.textContent = item.error;

  198.         block.appendChild(err);

  199.         container.appendChild(block);

  200.         return;

  201.       }

  202. 

  203.       const list = document.createElement("ul");

  204.       (item.streams || []).forEach(s => {

  205.         const li = document.createElement("li");

  206.         li.textContent = s;

  207.         list.appendChild(li);

  208.         allStreams.push(s);

  209.       });

  210.       block.appendChild(list);

  211.       container.appendChild(block);

  212.     });

  213. 

  214.     document.getElementById("clipboard").value = allStreams.join("\n");

  215.   }

  216. 

  217.   document.getElementById("copy").addEventListener("click", () => {

  218.     const text = document.getElementById("clipboard").value;

  219.     if (navigator.clipboard && navigator.clipboard.writeText) {

  220.       navigator.clipboard.writeText(text);

  221.     } else {

  222.       const el = document.getElementById("clipboard");

  223.       el.select();

  224.       document.execCommand("copy");

  225.     }

  226.   });

  227. 

  228.   fetchData();

  229.   if (interval && Number(interval) > 0) {

  230.     setInterval(fetchData, Number(interval) * 1000);

  231.   }

  232. </script>

  233. </body>

  234. </html>`, strings.Join(urls, "\n"), interval, probe)

  235. }

  236. 

  237. func makeScanHandler(client *http.Client, cfg *config, history *historyWriter) http.HandlerFunc {

  238. 	return func(w http.ResponseWriter, r *http.Request) {

  239. 		var urls []string

  240. 		switch r.Method {

  241. 		case http.MethodGet:

  242. 			urls = r.URL.Query()["url"]

  243. 		case http.MethodPost:

  244. 			if err := r.ParseForm(); err != nil {

  245. 				http.Error(w, err.Error(), http.StatusBadRequest)

  246. 				return

  247. 			}

  248. 			urls = r.Form["url"]

  249. 		default:

  250. 			http.Error(w, "only GET and POST supported", http.StatusMethodNotAllowed)

  251. 			return

  252. 		}

  253. 

  254. 		urls = normalizeURLInputs(urls)

  255. 		if len(urls) == 0 {

  256. 			http.Error(w, "provide at least one url parameter", http.StatusBadRequest)

  257. 			return

  258. 		}

  259. 

  260. 		localCfg := *cfg

  261. 		if r.URL.Query().Get("probe") == "1" {

  262. 			localCfg.Probe = true

  263. 		} else if r.URL.Query().Get("probe") == "0" {

  264. 			localCfg.Probe = false

  265. 		}

  266. 		if f := r.URL.Query().Get("format"); f != "" {

  267. 			localCfg.Format = f

  268. 		}

  269. 

  270. 		results := scanURLs(urls, client, &localCfg)

  271. 		history.Write(results)

  272. 		if err := outputResults(results, localCfg.Format, w); err != nil {

  273. 			http.Error(w, err.Error(), http.StatusInternalServerError)

  274. 		}

  275. 	}

  276. }

  277. 

  278. func normalizeURLInputs(inputs []string) []string {

  279. 	var urls []string

  280. 	for _, item := range inputs {

  281. 		for _, line := range strings.Split(item, "\n") {

  282. 			line = strings.TrimSpace(line)

  283. 			if line == "" {

  284. 				continue

  285. 			}

  286. 			urls = append(urls, line)

  287. 		}

  288. 	}

  289. 	return urls

  290. }

  291. 

  292. func scanURLs(urls []string, client *http.Client, cfg *config) []scanResult {

  293. 	results := make([]scanResult, len(urls))

  294. 	type job struct {

  295. 		index int

  296. 		url   string

  297. 	}

  298. 	jobs := make(chan job)

  299. 	var wg sync.WaitGroup

  300. 

  301. 	workers := cfg.Concurrency

  302. 	if workers < 1 {

  303. 		workers = 1

  304. 	}

  305. 

  306. 	for i := 0; i < workers; i++ {

  307. 		wg.Add(1)

  308. 		go func() {

  309. 			defer wg.Done()

  310. 			for j := range jobs {

  311. 				res := scanOneURL(client, cfg, j.url)

  312. 				results[j.index] = res

  313. 			}

  314. 		}()

  315. 	}

  316. 

  317. 	for i, u := range urls {

  318. 		jobs <- job{index: i, url: u}

  319. 	}

  320. 	close(jobs)

  321. 	wg.Wait()

  322. 	return results

  323. }

  324. 

  325. func scanOneURL(client *http.Client, cfg *config, raw string) scanResult {

  326. 	res := scanResult{URL: raw, FetchedAt: time.Now()}

  327. 	html, contentType, err := fetchContent(client, cfg, raw)

  328. 	if err != nil {

  329. 		res.Error = err.Error()

  330. 		return res

  331. 	}

  332. 

  333. 	parsedBase, _ := url.Parse(raw)

  334. 	baseHost := ""

  335. 	if parsedBase != nil {

  336. 		baseHost = parsedBase.Hostname()

  337. 	}

  338. 

  339. 	streams := extractor.ExtractStreams(html)

  340. 	playlists := extractor.ExtractPlaylistLinks(html)

  341. 	resolvedPlaylists := make([]string, 0, len(playlists))

  342. 

  343. 	for _, pl := range playlists {

  344. 		plURL := resolveURL(raw, pl)

  345. 		if plURL == "" {

  346. 			continue

  347. 		}

  348. 		resolvedPlaylists = append(resolvedPlaylists, plURL)

  349. 		plContent, plType, err := fetchContent(client, cfg, plURL)

  350. 		if err != nil {

  351. 			continue

  352. 		}

  353. 		parsed := extractor.ParsePlaylist(plContent, plType, plURL)

  354. 		if len(parsed) > 0 {

  355. 			streams = append(streams, parsed...)

  356. 			res.FromPlaylist = true

  357. 		}

  358. 	}

  359. 

  360. 	embedURLs := extractor.ExtractEmbedURLs(html)

  361. 	seenEmbeds := make(map[string]struct{})

  362. 	for _, embed := range embedURLs {

  363. 		embedURL := resolveURL(raw, embed)

  364. 		if embedURL == "" || embedURL == raw {

  365. 			continue

  366. 		}

  367. 		if _, ok := seenEmbeds[embedURL]; ok {

  368. 			continue

  369. 		}

  370. 		seenEmbeds[embedURL] = struct{}{}

  371. 

  372. 		embedHTML, _, err := fetchContent(client, cfg, embedURL)

  373. 		if err != nil {

  374. 			continue

  375. 		}

  376. 

  377. 		streams = append(streams, extractor.ExtractStreams(embedHTML)...)

  378. 		embedPlaylists := extractor.ExtractPlaylistLinks(embedHTML)

  379. 		playlists = append(playlists, embedPlaylists...)

  380. 

  381. 		for _, pl := range embedPlaylists {

  382. 			plURL := resolveURL(embedURL, pl)

  383. 			if plURL == "" {

  384. 				continue

  385. 			}

  386. 			resolvedPlaylists = append(resolvedPlaylists, plURL)

  387. 			plContent, plType, err := fetchContent(client, cfg, plURL)

  388. 			if err != nil {

  389. 				continue

  390. 			}

  391. 			parsed := extractor.ParsePlaylist(plContent, plType, plURL)

  392. 			if len(parsed) > 0 {

  393. 				streams = append(streams, parsed...)

  394. 				res.FromPlaylist = true

  395. 			}

  396. 		}

  397. 	}

  398. 

  399. 	scriptURLs := extractor.ExtractScriptURLs(html)

  400. 	seenScripts := make(map[string]struct{})

  401. 	for _, script := range scriptURLs {

  402. 		scriptURL := resolveURL(raw, script)

  403. 		if scriptURL == "" || scriptURL == raw {

  404. 			continue

  405. 		}

  406. 		if baseHost != "" {

  407. 			parsedScript, err := url.Parse(scriptURL)

  408. 			if err != nil {

  409. 				continue

  410. 			}

  411. 			if parsedScript.Hostname() != "" && parsedScript.Hostname() != baseHost {

  412. 				continue

  413. 			}

  414. 		}

  415. 		if _, ok := seenScripts[scriptURL]; ok {

  416. 			continue

  417. 		}

  418. 		seenScripts[scriptURL] = struct{}{}

  419. 

  420. 		scriptHTML, _, err := fetchContent(client, cfg, scriptURL)

  421. 		if err != nil {

  422. 			continue

  423. 		}

  424. 

  425. 		streams = append(streams, extractor.ExtractStreams(scriptHTML)...)

  426. 		scriptPlaylists := extractor.ExtractPlaylistLinks(scriptHTML)

  427. 		playlists = append(playlists, scriptPlaylists...)

  428. 

  429. 		for _, pl := range scriptPlaylists {

  430. 			plURL := resolveURL(scriptURL, pl)

  431. 			if plURL == "" {

  432. 				continue

  433. 			}

  434. 			resolvedPlaylists = append(resolvedPlaylists, plURL)

  435. 			plContent, plType, err := fetchContent(client, cfg, plURL)

  436. 			if err != nil {

  437. 				continue

  438. 			}

  439. 			parsed := extractor.ParsePlaylist(plContent, plType, plURL)

  440. 			if len(parsed) > 0 {

  441. 				streams = append(streams, parsed...)

  442. 				res.FromPlaylist = true

  443. 			}

  444. 		}

  445. 	}

  446. 

  447. 	res.Playlists = uniqueStrings(resolvedPlaylists)

  448. 	res.Streams = uniqueStrings(streams)

  449. 

  450. 	if cfg.Probe {

  451. 		res.Probes = probeStreams(client, cfg, res.Streams)

  452. 	}

  453. 

  454. 	_ = contentType

  455. 	return res

  456. }

  457. 

  458. func fetchContent(client *http.Client, cfg *config, raw string) (string, string, error) {

  459. 	if !cfg.AllowPrivate {

  460. 		if blocked, reason := isPrivateURL(raw); blocked {

  461. 			return "", "", fmt.Errorf("blocked private address (%s)", reason)

  462. 		}

  463. 	}

  464. 

  465. 	timeout := cfg.RequestTimeout

  466. 	if timeout <= 0 {

  467. 		timeout = 15 * time.Second

  468. 	}

  469. 	ctx, cancel := context.WithTimeout(context.Background(), timeout)

  470. 	defer cancel()

  471. 

  472. 	req, err := http.NewRequestWithContext(ctx, http.MethodGet, raw, nil)

  473. 	if err != nil {

  474. 		return "", "", err

  475. 	}

  476. 	req.Header.Set("User-Agent", "radiostreamscan/0.2")

  477. 	for _, h := range cfg.Headers {

  478. 		parts := strings.SplitN(h, ":", 2)

  479. 		if len(parts) == 2 {

  480. 			req.Header.Set(strings.TrimSpace(parts[0]), strings.TrimSpace(parts[1]))

  481. 		}

  482. 	}

  483. 

  484. 	resp, err := client.Do(req)

  485. 	if err != nil {

  486. 		return "", "", err

  487. 	}

  488. 	defer resp.Body.Close()

  489. 

  490. 	if resp.StatusCode < 200 || resp.StatusCode >= 300 {

  491. 		snippet, _ := io.ReadAll(io.LimitReader(resp.Body, 1024))

  492. 		return "", "", fmt.Errorf("unexpected status %s: %s", resp.Status, strings.TrimSpace(string(snippet)))

  493. 	}

  494. 

  495. 	body, err := io.ReadAll(io.LimitReader(resp.Body, 2<<20))

  496. 	if err != nil {

  497. 		return "", "", err

  498. 	}

  499. 

  500. 	return string(body), resp.Header.Get("Content-Type"), nil

  501. }

  502. 

  503. func probeStreams(client *http.Client, cfg *config, streams []string) []probeResult {

  504. 	var results []probeResult

  505. 	for _, s := range streams {

  506. 		if !cfg.AllowPrivate {

  507. 			if blocked, reason := isPrivateURL(s); blocked {

  508. 				results = append(results, probeResult{URL: s, Status: "blocked: " + reason})

  509. 				continue

  510. 			}

  511. 		}

  512. 

  513. 		timeout := cfg.ProbeTimeout

  514. 		if timeout <= 0 {

  515. 			timeout = cfg.RequestTimeout

  516. 		}

  517. 		if timeout <= 0 {

  518. 			timeout = 8 * time.Second

  519. 		}

  520. 

  521. 		ctx, cancel := context.WithTimeout(context.Background(), timeout)

  522. 		req, err := http.NewRequestWithContext(ctx, http.MethodHead, s, nil)

  523. 		if err != nil {

  524. 			cancel()

  525. 			continue

  526. 		}

  527. 		for _, h := range cfg.Headers {

  528. 			parts := strings.SplitN(h, ":", 2)

  529. 			if len(parts) == 2 {

  530. 				req.Header.Set(strings.TrimSpace(parts[0]), strings.TrimSpace(parts[1]))

  531. 			}

  532. 		}

  533. 		resp, err := client.Do(req)

  534. 		if err != nil {

  535. 			cancel()

  536. 			// Fallback to GET with range.

  537. 			results = append(results, probeWithGet(client, cfg, s, timeout))

  538. 			continue

  539. 		}

  540. 		resp.Body.Close()

  541. 		cancel()

  542. 

  543. 		if resp.StatusCode == http.StatusMethodNotAllowed || resp.StatusCode == http.StatusNotImplemented {

  544. 			results = append(results, probeWithGet(client, cfg, s, timeout))

  545. 			continue

  546. 		}

  547. 		results = append(results, probeResult{URL: s, Status: resp.Status, ContentType: resp.Header.Get("Content-Type")})

  548. 	}

  549. 	return results

  550. }

  551. 

  552. func probeWithGet(client *http.Client, cfg *config, urlStr string, timeout time.Duration) probeResult {

  553. 	ctx, cancel := context.WithTimeout(context.Background(), timeout)

  554. 	defer cancel()

  555. 

  556. 	req, err := http.NewRequestWithContext(ctx, http.MethodGet, urlStr, nil)

  557. 	if err != nil {

  558. 		return probeResult{URL: urlStr, Status: err.Error()}

  559. 	}

  560. 	req.Header.Set("Range", "bytes=0-1023")

  561. 	for _, h := range cfg.Headers {

  562. 		parts := strings.SplitN(h, ":", 2)

  563. 		if len(parts) == 2 {

  564. 			req.Header.Set(strings.TrimSpace(parts[0]), strings.TrimSpace(parts[1]))

  565. 		}

  566. 	}

  567. 

  568. 	resp, err := client.Do(req)

  569. 	if err != nil {

  570. 		return probeResult{URL: urlStr, Status: err.Error()}

  571. 	}

  572. 	defer resp.Body.Close()

  573. 	_, _ = io.Copy(io.Discard, io.LimitReader(resp.Body, 1024))

  574. 	return probeResult{URL: urlStr, Status: resp.Status, ContentType: resp.Header.Get("Content-Type")}

  575. }

  576. 

  577. func outputResults(results []scanResult, format string, w io.Writer) error {

  578. 	if rw, ok := w.(http.ResponseWriter); ok {

  579. 		if strings.ToLower(format) == "json" {

  580. 			rw.Header().Set("Content-Type", "application/json")

  581. 		} else if strings.ToLower(format) == "csv" {

  582. 			rw.Header().Set("Content-Type", "text/csv")

  583. 		}

  584. 	}

  585. 

  586. 	switch strings.ToLower(format) {

  587. 	case "json":

  588. 		if err := json.NewEncoder(w).Encode(results); err != nil {

  589. 			return err

  590. 		}

  591. 	case "csv":

  592. 		cw := csv.NewWriter(w)

  593. 		if err := cw.Write([]string{"input_url", "stream_url"}); err != nil {

  594. 			return err

  595. 		}

  596. 		for _, res := range results {

  597. 			for _, s := range res.Streams {

  598. 				if err := cw.Write([]string{res.URL, s}); err != nil {

  599. 					return err

  600. 				}

  601. 			}

  602. 		}

  603. 		cw.Flush()

  604. 		if err := cw.Error(); err != nil {

  605. 			return err

  606. 		}

  607. 	case "pls":

  608. 		if _, err := fmt.Fprintln(w, "[playlist]"); err != nil {

  609. 			return err

  610. 		}

  611. 		i := 1

  612. 		for _, res := range results {

  613. 			for _, s := range res.Streams {

  614. 				if _, err := fmt.Fprintf(w, "File%d=%s\n", i, s); err != nil {

  615. 					return err

  616. 				}

  617. 				i++

  618. 			}

  619. 		}

  620. 		if _, err := fmt.Fprintf(w, "NumberOfEntries=%d\nVersion=2\n", i-1); err != nil {

  621. 			return err

  622. 		}

  623. 	default:

  624. 		for _, res := range results {

  625. 			if _, err := fmt.Fprintf(w, "URL: %s\n", res.URL); err != nil {

  626. 				return err

  627. 			}

  628. 			if res.Error != "" {

  629. 				if _, err := fmt.Fprintf(w, "  error: %s\n", res.Error); err != nil {

  630. 					return err

  631. 				}

  632. 				continue

  633. 			}

  634. 			if len(res.Streams) == 0 {

  635. 				if _, err := fmt.Fprintln(w, "  (no candidate streams found)"); err != nil {

  636. 					return err

  637. 				}

  638. 				continue

  639. 			}

  640. 			for _, s := range res.Streams {

  641. 				if _, err := fmt.Fprintf(w, "  - %s\n", s); err != nil {

  642. 					return err

  643. 				}

  644. 			}

  645. 		}

  646. 	}

  647. 	return nil

  648. }

  649. 

  650. func newHTTPClient(proxyURL string, timeout time.Duration) *http.Client {

  651. 	transport := &http.Transport{

  652. 		Proxy:                 http.ProxyFromEnvironment,

  653. 		ResponseHeaderTimeout: 8 * time.Second,

  654. 		TLSHandshakeTimeout:   6 * time.Second,

  655. 		IdleConnTimeout:       30 * time.Second,

  656. 		ExpectContinueTimeout: 1 * time.Second,

  657. 		MaxIdleConns:          100,

  658. 		MaxIdleConnsPerHost:   10,

  659. 	}

  660. 	if proxyURL != "" {

  661. 		if parsed, err := url.Parse(proxyURL); err == nil {

  662. 			transport.Proxy = http.ProxyURL(parsed)

  663. 		}

  664. 	}

  665. 	if timeout <= 0 {

  666. 		timeout = 15 * time.Second

  667. 	}

  668. 	return &http.Client{Timeout: timeout, Transport: transport}

  669. }

  670. 

  671. func uniqueStrings(values []string) []string {

  672. 	set := make(map[string]struct{}, len(values))

  673. 	out := make([]string, 0, len(values))

  674. 	for _, v := range values {

  675. 		if _, ok := set[v]; ok {

  676. 			continue

  677. 		}

  678. 		set[v] = struct{}{}

  679. 		out = append(out, v)

  680. 	}

  681. 	return out

  682. }

  683. 

  684. func resolveURL(base, href string) string {

  685. 	href = strings.TrimSpace(href)

  686. 	if href == "" {

  687. 		return ""

  688. 	}

  689. 	if strings.HasPrefix(href, "//") {

  690. 		return "https:" + href

  691. 	}

  692. 	parsed, err := url.Parse(href)

  693. 	if err != nil {

  694. 		return ""

  695. 	}

  696. 	if parsed.IsAbs() {

  697. 		return parsed.String()

  698. 	}

  699. 	baseURL, err := url.Parse(base)

  700. 	if err != nil {

  701. 		return parsed.String()

  702. 	}

  703. 	return baseURL.ResolveReference(parsed).String()

  704. }

  705. 

  706. func isPrivateURL(raw string) (bool, string) {

  707. 	parsed, err := url.Parse(raw)

  708. 	if err != nil {

  709. 		return false, ""

  710. 	}

  711. 	host := parsed.Hostname()

  712. 	if host == "" {

  713. 		return false, ""

  714. 	}

  715. 	lower := strings.ToLower(host)

  716. 	if lower == "localhost" || strings.HasSuffix(lower, ".local") || strings.HasSuffix(lower, ".internal") {

  717. 		return true, "hostname"

  718. 	}

  719. 	ip := net.ParseIP(host)

  720. 	if ip == nil {

  721. 		return false, ""

  722. 	}

  723. 	if ip.IsLoopback() || ip.IsLinkLocalUnicast() || ip.IsLinkLocalMulticast() {

  724. 		return true, "loopback/link-local"

  725. 	}

  726. 	if isPrivateIP(ip) {

  727. 		return true, "private range"

  728. 	}

  729. 	return false, ""

  730. }

  731. 

  732. func isPrivateIP(ip net.IP) bool {

  733. 	if ip4 := ip.To4(); ip4 != nil {

  734. 		switch {

  735. 		case ip4[0] == 10:

  736. 			return true

  737. 		case ip4[0] == 172 && ip4[1] >= 16 && ip4[1] <= 31:

  738. 			return true

  739. 		case ip4[0] == 192 && ip4[1] == 168:

  740. 			return true

  741. 		case ip4[0] == 169 && ip4[1] == 254:

  742. 			return true

  743. 		case ip4[0] == 127:

  744. 			return true

  745. 		}

  746. 	}

  747. 	// IPv6 unique local fc00::/7

  748. 	if ip.To16() != nil {

  749. 		return ip[0]&0xfe == 0xfc

  750. 	}

  751. 	return false

  752. }

  753. 

  754. type historyWriter struct {

  755. 	path string

  756. 	mu   sync.Mutex

  757. }

  758. 

  759. func newHistoryWriter(path string) *historyWriter {

  760. 	return &historyWriter{path: path}

  761. }

  762. 

  763. func (h *historyWriter) Write(results []scanResult) {

  764. 	if h == nil || h.path == "" {

  765. 		return

  766. 	}

  767. 	h.mu.Lock()

  768. 	defer h.mu.Unlock()

  769. 

  770. 	f, err := os.OpenFile(h.path, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0644)

  771. 	if err != nil {

  772. 		return

  773. 	}

  774. 	defer f.Close()

  775. 

  776. 	writer := bufio.NewWriter(f)

  777. 	for _, res := range results {

  778. 		data, err := json.Marshal(res)

  779. 		if err != nil {

  780. 			continue

  781. 		}

  782. 		writer.Write(data)

  783. 		writer.WriteString("\n")

  784. 	}

  785. 	writer.Flush()

  786. }