feat(podcasts): RSS fetcher — parse, SSRF guards, upsert-without-delete (task 1.4)
The fetcher is the SOLE writer of feed/episode metadata (D9). Three files, no new deps (stdlib encoding/xml + net/http): podcast_rss.go — RSS 2.0 + iTunes + Podcasting-2.0 structs. iTunes fields are namespace-qualified in tags (local names collide with core RSS: title/subtitle/author/summary/explicit/image/duration); podcast:* fields match on local name (collision-free, tolerant of both declared namespace URIs). Description/author fallback chains per §5; itunes:duration parses seconds/HH:MM:SS/MM:SS/prose; pubDate accepts the RFC-822/1123 spellings feeds actually emit. podcast_http.go — the security boundary for user-supplied URLs: https-only on every hop (CheckRedirect), private/loopback/link-local/ multicast IP ranges blocked at DIAL time (Dialer.Control sees the RESOLVED address — DNS rebinding into 127.0.0.1 is refused at the socket), 10-redirect cap, hard body caps (25MiB feeds / 20MiB artwork / 1GiB enclosures), no proxy. podcast_service.go — fetch → parse → upsert. Episodes upsert metadata-only keyed on (source_feed_id, episode_guid); rows for items that vanish from the XML are NEVER deleted (D14 search covers historical episodes). itunes:new-feed-url + permanent HTTP redirects rewrite feed_url in place (row/subscriptions/episodes stay put); podcast:guid dedupe reuses the canonical row instead of forking. Artwork downloads feed-level under <feed-slug>/artwork.<ext>. AddFeed authorizes ONLY on explicit arguments (library visible + allow_member_feeds for non-admins — the D20 scoping model); detached work runs under the first-admin identity via adminCtx (invariant 12). Feed layout: <download-root>/<feed-slug>/<pos:04d>-<slug>.<ext> so ServeFile/Range streaming resolve unchanged.
This commit is contained in:
1 parent
9a148c2344
commit
85a9873a75
3 files changed
+1209
No files matched your search
@@ -0,0 +1,190 @@
|
||||
// SSRF-guarded HTTP fetching for the podcast fetcher (plan §7).
|
||||
//
|
||||
// This is the code path that turns USER-SUPPLIED URLs into server
|
||||
// fetches — treat it as a security boundary, not a convenience wrapper:
|
||||
//
|
||||
// - https-only schemes (feed URL, enclosure, artwork, EVERY redirect hop)
|
||||
// - private/loopback/link-local/multicast/unspecified IP ranges blocked
|
||||
// at DIAL time via Dialer.Control — the check sees the RESOLVED
|
||||
// address, so a DNS name that rebinding-resolves to 127.0.0.1 (or a
|
||||
// redirect into the internal network) is refused at the socket, not
|
||||
// merely at URL-parse time
|
||||
// - hard response size caps on every body (feed XML, artwork, enclosures)
|
||||
//
|
||||
// Go's redirect follower re-dials through the same dialer per hop, so
|
||||
// the IP guard re-applies automatically; the scheme guard re-applies
|
||||
// via CheckRedirect.
|
||||
package services
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"syscall"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Response size caps (bytes). Feeds are XML; artwork is an image;
|
||||
// enclosures are audio episodes — the enclosure cap is deliberately the
|
||||
// large one (audiobook chapters run long) but still bounded.
|
||||
const (
|
||||
maxFeedSizeBytes = 25 << 20 // 25 MiB
|
||||
maxArtworkSizeBytes = 20 << 20 // 20 MiB
|
||||
maxEnclosureCapBytes = 1 << 30 // 1 GiB
|
||||
)
|
||||
|
||||
// ErrBlockedPrivateAddress marks SSRF-guard rejections (dial-time IP or
|
||||
// scheme checks). Handlers surface it as a validation failure, and the
|
||||
// download worker records it as the download_error text.
|
||||
var ErrBlockedPrivateAddress = errors.New("address blocked: private, loopback, or otherwise non-public")
|
||||
|
||||
// podcastHTTPClient builds the shared guard client. Timeout is generous
|
||||
// (podcast servers are slow) but bounded; enclosures stream through a
|
||||
// LimitReader so a lying Content-Length cannot exhaust disk.
|
||||
func podcastHTTPClient() *http.Client {
|
||||
dialer := &net.Dialer{
|
||||
Timeout: 15 * time.Second,
|
||||
Control: func(network, address string, _ syscall.RawConn) error {
|
||||
return guardDialAddress(address)
|
||||
},
|
||||
}
|
||||
transport := &http.Transport{
|
||||
DialContext: dialer.DialContext,
|
||||
TLSHandshakeTimeout: 15 * time.Second,
|
||||
ResponseHeaderTimeout: 60 * time.Second,
|
||||
// Enclosure downloads are long; idle conn reuse still applies.
|
||||
IdleConnTimeout: 90 * time.Second,
|
||||
TLSClientConfig: &tls.Config{MinVersion: tls.VersionTLS12},
|
||||
Proxy: nil, // never route through a user-controllable proxy
|
||||
}
|
||||
return &http.Client{
|
||||
Transport: transport,
|
||||
Timeout: 15 * time.Minute, // outer bound (biggest enclosures on slow lines)
|
||||
CheckRedirect: func(req *http.Request, via []*http.Request) error {
|
||||
if len(via) >= 10 {
|
||||
return fmt.Errorf("stopped after 10 redirects")
|
||||
}
|
||||
if err := guardURLScheme(req.URL); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// guardDialAddress validates the RESOLVED host:port at dial time.
|
||||
func guardDialAddress(address string) error {
|
||||
host, _, err := net.SplitHostPort(address)
|
||||
if err != nil {
|
||||
return fmt.Errorf("dial guard: %w", err)
|
||||
}
|
||||
ip := net.ParseIP(host)
|
||||
if ip == nil {
|
||||
return fmt.Errorf("dial guard: unparseable address %q", host)
|
||||
}
|
||||
if !isPublicIP(ip) {
|
||||
return ErrBlockedPrivateAddress
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// guardURLScheme enforces https on any URL we are about to touch.
|
||||
func guardURLScheme(u *url.URL) error {
|
||||
if u == nil || u.Scheme != "https" {
|
||||
return fmt.Errorf("only https feed/enclosure URLs are supported")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// isPublicIP reports whether ip is a routable public unicast address.
|
||||
// Everything loopback/private/link-local/multicast/unspecified —
|
||||
// including IPv4-mapped IPv6 spellings of those — is rejected.
|
||||
func isPublicIP(ip net.IP) bool {
|
||||
if ip == nil {
|
||||
return false
|
||||
}
|
||||
// Normalize IPv4-mapped IPv6 (::ffff:10.0.0.1) to its v4 form so
|
||||
// the v4 range checks below actually fire.
|
||||
if ip4 := ip.To4(); ip4 != nil {
|
||||
ip = ip4
|
||||
}
|
||||
if ip.IsLoopback() || ip.IsPrivate() || ip.IsLinkLocalUnicast() ||
|
||||
ip.IsLinkLocalMulticast() || ip.IsInterfaceLocalMulticast() ||
|
||||
ip.IsMulticast() || ip.IsUnspecified() {
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// guardFetch performs a GET with the scheme/IP guards, conditional-GET
|
||||
// validators, and a hard body cap. Returns the body bytes, the FINAL
|
||||
// URL after redirects (new-feed-url / migration detection), validators
|
||||
// to persist, and whether the server answered 304 Not Modified.
|
||||
type fetchResult struct {
|
||||
Body []byte
|
||||
FinalURL string
|
||||
ETag string
|
||||
LastModified string
|
||||
NotModified bool
|
||||
StatusCode int
|
||||
}
|
||||
|
||||
func guardFetch(ctx context.Context, rawURL, etag, lastModified string, maxSize int64) (*fetchResult, error) {
|
||||
u, err := url.Parse(rawURL)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("invalid URL: %w", err)
|
||||
}
|
||||
if err := guardURLScheme(u); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// A recognizable UA; some podcast hosts 403 empty/default agents.
|
||||
req.Header.Set("User-Agent", "Bookhoard/1.0 (podcast fetcher; +https://github.com/bookhoard)")
|
||||
req.Header.Set("Accept", "*/*")
|
||||
if etag != "" {
|
||||
req.Header.Set("If-None-Match", etag)
|
||||
}
|
||||
if lastModified != "" {
|
||||
req.Header.Set("If-Modified-Since", lastModified)
|
||||
}
|
||||
|
||||
resp, err := podcastHTTPClient().Do(req)
|
||||
if err != nil {
|
||||
// Unwrap redirect/scheme guard errors so callers can classify.
|
||||
return nil, err
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
|
||||
res := &fetchResult{
|
||||
FinalURL: resp.Request.URL.String(),
|
||||
ETag: resp.Header.Get("ETag"),
|
||||
LastModified: resp.Header.Get("Last-Modified"),
|
||||
StatusCode: resp.StatusCode,
|
||||
}
|
||||
if resp.StatusCode == http.StatusNotModified {
|
||||
res.NotModified = true
|
||||
return res, nil
|
||||
}
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return res, fmt.Errorf("fetch %s: HTTP %d", rawURL, resp.StatusCode)
|
||||
}
|
||||
|
||||
body, err := io.ReadAll(io.LimitReader(resp.Body, maxSize+1))
|
||||
if err != nil {
|
||||
return res, fmt.Errorf("fetch %s: %w", rawURL, err)
|
||||
}
|
||||
if int64(len(body)) > maxSize {
|
||||
return res, fmt.Errorf("fetch %s: response exceeds %d byte cap", rawURL, maxSize)
|
||||
}
|
||||
res.Body = body
|
||||
return res, nil
|
||||
}
|
||||
Reference in new issue
Block a user