// SSRF-guarded HTTP fetching for the podcast fetcher (plan §7). // // This is the code path that turns USER-SUPPLIED URLs into server // fetches — treat it as a security boundary, not a convenience wrapper: // // - https-only schemes (feed URL, enclosure, artwork, EVERY redirect hop) // - private/loopback/link-local/multicast/unspecified IP ranges blocked // at DIAL time via Dialer.Control — the check sees the RESOLVED // address, so a DNS name that rebinding-resolves to 127.0.0.1 (or a // redirect into the internal network) is refused at the socket, not // merely at URL-parse time // - hard response size caps on every body (feed XML, artwork, enclosures) // // Go's redirect follower re-dials through the same dialer per hop, so // the IP guard re-applies automatically; the scheme guard re-applies // via CheckRedirect. package services import ( "context" "crypto/tls" "errors" "fmt" "io" "net" "net/http" "net/url" "syscall" "time" ) // Response size caps (bytes). Feeds are XML; artwork is an image; // enclosures are audio episodes — the enclosure cap is deliberately the // large one (audiobook chapters run long) but still bounded. const ( maxFeedSizeBytes = 25 << 20 // 25 MiB maxArtworkSizeBytes = 20 << 20 // 20 MiB maxEnclosureCapBytes = 1 << 30 // 1 GiB ) // ErrBlockedPrivateAddress marks SSRF-guard rejections (dial-time IP or // scheme checks). Handlers surface it as a validation failure, and the // download worker records it as the download_error text. var ErrBlockedPrivateAddress = errors.New("address blocked: private, loopback, or otherwise non-public") // podcastHTTPClient builds the shared guard client. Timeout is generous // (podcast servers are slow) but bounded; enclosures stream through a // LimitReader so a lying Content-Length cannot exhaust disk. func podcastHTTPClient() *http.Client { dialer := &net.Dialer{ Timeout: 15 * time.Second, Control: func(network, address string, _ syscall.RawConn) error { return guardDialAddress(address) }, } transport := &http.Transport{ DialContext: dialer.DialContext, TLSHandshakeTimeout: 15 * time.Second, ResponseHeaderTimeout: 60 * time.Second, // Enclosure downloads are long; idle conn reuse still applies. IdleConnTimeout: 90 * time.Second, TLSClientConfig: &tls.Config{MinVersion: tls.VersionTLS12}, Proxy: nil, // never route through a user-controllable proxy } return &http.Client{ Transport: transport, Timeout: 15 * time.Minute, // outer bound (biggest enclosures on slow lines) CheckRedirect: func(req *http.Request, via []*http.Request) error { if len(via) >= 10 { return fmt.Errorf("stopped after 10 redirects") } if err := guardURLScheme(req.URL); err != nil { return err } return nil }, } } // guardDialAddress validates the RESOLVED host:port at dial time. func guardDialAddress(address string) error { host, _, err := net.SplitHostPort(address) if err != nil { return fmt.Errorf("dial guard: %w", err) } ip := net.ParseIP(host) if ip == nil { return fmt.Errorf("dial guard: unparseable address %q", host) } if !isPublicIP(ip) { return ErrBlockedPrivateAddress } return nil } // guardURLScheme enforces https on any URL we are about to touch. func guardURLScheme(u *url.URL) error { if u == nil || u.Scheme != "https" { return fmt.Errorf("only https feed/enclosure URLs are supported") } return nil } // isPublicIP reports whether ip is a routable public unicast address. // Everything loopback/private/link-local/multicast/unspecified — // including IPv4-mapped IPv6 spellings of those — is rejected. func isPublicIP(ip net.IP) bool { if ip == nil { return false } // Normalize IPv4-mapped IPv6 (::ffff:10.0.0.1) to its v4 form so // the v4 range checks below actually fire. if ip4 := ip.To4(); ip4 != nil { ip = ip4 } if ip.IsLoopback() || ip.IsPrivate() || ip.IsLinkLocalUnicast() || ip.IsLinkLocalMulticast() || ip.IsInterfaceLocalMulticast() || ip.IsMulticast() || ip.IsUnspecified() { return false } return true } // guardFetch performs a GET with the scheme/IP guards, conditional-GET // validators, and a hard body cap. Returns the body bytes, the FINAL // URL after redirects (new-feed-url / migration detection), validators // to persist, and whether the server answered 304 Not Modified. type fetchResult struct { Body []byte FinalURL string ETag string LastModified string NotModified bool StatusCode int } // podcastHTTPClientFactory is the indirection TESTS use: the SSRF suite // exercises the REAL client; fetch-driven integration tests swap in a // client that trusts httptest's loopback TLS cert. Never reassign // outside tests (use SetPodcastHTTPClientForTests). var podcastHTTPClientFactory = podcastHTTPClient // SetPodcastHTTPClientForTests swaps the fetch client. nil restores the // production client (SSRF guards intact). func SetPodcastHTTPClientForTests(c *http.Client) { if c == nil { podcastHTTPClientFactory = podcastHTTPClient return } podcastHTTPClientFactory = func() *http.Client { return c } } func guardFetch(ctx context.Context, rawURL, etag, lastModified string, maxSize int64) (*fetchResult, error) { u, err := url.Parse(rawURL) if err != nil { return nil, fmt.Errorf("invalid URL: %w", err) } if err := guardURLScheme(u); err != nil { return nil, err } req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil) if err != nil { return nil, err } // A recognizable UA; some podcast hosts 403 empty/default agents. req.Header.Set("User-Agent", "Bookhoard/1.0 (podcast fetcher; +https://github.com/bookhoard)") req.Header.Set("Accept", "*/*") if etag != "" { req.Header.Set("If-None-Match", etag) } if lastModified != "" { req.Header.Set("If-Modified-Since", lastModified) } resp, err := podcastHTTPClientFactory().Do(req) if err != nil { // Unwrap redirect/scheme guard errors so callers can classify. return nil, err } defer func() { _ = resp.Body.Close() }() res := &fetchResult{ FinalURL: resp.Request.URL.String(), ETag: resp.Header.Get("ETag"), LastModified: resp.Header.Get("Last-Modified"), StatusCode: resp.StatusCode, } if resp.StatusCode == http.StatusNotModified { res.NotModified = true return res, nil } if resp.StatusCode != http.StatusOK { return res, fmt.Errorf("fetch %s: HTTP %d", rawURL, resp.StatusCode) } body, err := io.ReadAll(io.LimitReader(resp.Body, maxSize+1)) if err != nil { return res, fmt.Errorf("fetch %s: %w", rawURL, err) } if int64(len(body)) > maxSize { return res, fmt.Errorf("fetch %s: response exceeds %d byte cap", rawURL, maxSize) } res.Body = body return res, nil }