feat(podcasts): RSS fetcher — parse, SSRF guards, upsert-without-delete (task 1.4)
The fetcher is the SOLE writer of feed/episode metadata (D9). Three files, no new deps (stdlib encoding/xml + net/http): podcast_rss.go — RSS 2.0 + iTunes + Podcasting-2.0 structs. iTunes fields are namespace-qualified in tags (local names collide with core RSS: title/subtitle/author/summary/explicit/image/duration); podcast:* fields match on local name (collision-free, tolerant of both declared namespace URIs). Description/author fallback chains per §5; itunes:duration parses seconds/HH:MM:SS/MM:SS/prose; pubDate accepts the RFC-822/1123 spellings feeds actually emit. podcast_http.go — the security boundary for user-supplied URLs: https-only on every hop (CheckRedirect), private/loopback/link-local/ multicast IP ranges blocked at DIAL time (Dialer.Control sees the RESOLVED address — DNS rebinding into 127.0.0.1 is refused at the socket), 10-redirect cap, hard body caps (25MiB feeds / 20MiB artwork / 1GiB enclosures), no proxy. podcast_service.go — fetch → parse → upsert. Episodes upsert metadata-only keyed on (source_feed_id, episode_guid); rows for items that vanish from the XML are NEVER deleted (D14 search covers historical episodes). itunes:new-feed-url + permanent HTTP redirects rewrite feed_url in place (row/subscriptions/episodes stay put); podcast:guid dedupe reuses the canonical row instead of forking. Artwork downloads feed-level under <feed-slug>/artwork.<ext>. AddFeed authorizes ONLY on explicit arguments (library visible + allow_member_feeds for non-admins — the D20 scoping model); detached work runs under the first-admin identity via adminCtx (invariant 12). Feed layout: <download-root>/<feed-slug>/<pos:04d>-<slug>.<ext> so ServeFile/Range streaming resolve unchanged.
This commit is contained in:
John O'Keefe committed
1 parent
9a148c2344
commit
85a9873a75
3 files changed
+1209
No files matched your search
@@ -0,0 +1,257 @@
|
||||
// RSS 2.0 + iTunes + Podcasting-2.0 feed parsing for the podcast
|
||||
// fetcher (plan podcast-integration.md §5/§7). Stdlib encoding/xml only
|
||||
// — no new deps.
|
||||
//
|
||||
// Namespace strategy: iTunes fields MUST be namespace-qualified in the
|
||||
// struct tags (the itunes URI) because their local names collide with
|
||||
// core RSS ("title", "subtitle", "author", "summary", "explicit",
|
||||
// "image", "duration"). Podcasting-2.0 fields (podcast:guid,
|
||||
// podcast:transcript, ...) have collision-free local names, so they
|
||||
// match on local name — encoding/xml matches Name.Local when the tag
|
||||
// carries no namespace, which also tolerates both declared namespace
|
||||
// URIs the ecosystem uses.
|
||||
//
|
||||
// Out of scope at parse time (plan §5): podcast:value, liveItem,
|
||||
// soundbite, alternateEnclosure, socialInteract, remoteItem/podroll,
|
||||
// locked/block, txt, chat. lastBuildDate/ttl are poll hints only.
|
||||
package services
|
||||
|
||||
import (
|
||||
"encoding/xml"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// itunesNS is the canonical iTunes podcast namespace URI.
|
||||
const itunesNS = "http://www.itunes.com/dtds/podcast-1.0.dtd"
|
||||
|
||||
// PodcastFeedXML is one parsed RSS document.
|
||||
type PodcastFeedXML struct {
|
||||
XMLName xml.Name `xml:"rss"`
|
||||
Channel PodcastChannelXML `xml:"channel"`
|
||||
}
|
||||
|
||||
type PodcastChannelXML struct {
|
||||
Title string `xml:"title"`
|
||||
Link string `xml:"link"`
|
||||
Language string `xml:"language"`
|
||||
Description string `xml:"description"`
|
||||
|
||||
// iTunes channel namespace.
|
||||
ITunesAuthor string `xml:"http://www.itunes.com/dtds/podcast-1.0.dtd author"`
|
||||
ITunesOwner struct {
|
||||
Name string `xml:"http://www.itunes.com/dtds/podcast-1.0.dtd name"`
|
||||
Email string `xml:"http://www.itunes.com/dtds/podcast-1.0.dtd email"`
|
||||
} `xml:"http://www.itunes.com/dtds/podcast-1.0.dtd owner"`
|
||||
ITunesImage struct {
|
||||
Href string `xml:"href,attr"`
|
||||
} `xml:"http://www.itunes.com/dtds/podcast-1.0.dtd image"`
|
||||
ITunesExplicit string `xml:"http://www.itunes.com/dtds/podcast-1.0.dtd explicit"`
|
||||
ITunesType string `xml:"http://www.itunes.com/dtds/podcast-1.0.dtd type"`
|
||||
ITunesCategory struct {
|
||||
Text string `xml:"text,attr"`
|
||||
Subcategory struct {
|
||||
Text string `xml:"text,attr"`
|
||||
} `xml:"http://www.itunes.com/dtds/podcast-1.0.dtd category"`
|
||||
} `xml:"http://www.itunes.com/dtds/podcast-1.0.dtd category"`
|
||||
ITunesNewFeedURL string `xml:"http://www.itunes.com/dtds/podcast-1.0.dtd new-feed-url"`
|
||||
ITunesComplete string `xml:"http://www.itunes.com/dtds/podcast-1.0.dtd complete"`
|
||||
|
||||
// Podcasting 2.0 (matched on local name; see file comment).
|
||||
PodcastGUID string `xml:"guid"`
|
||||
PodcastMedium string `xml:"medium"`
|
||||
|
||||
LastBuildDate string `xml:"lastBuildDate"`
|
||||
|
||||
Items []PodcastItemXML `xml:"item"`
|
||||
}
|
||||
|
||||
type PodcastItemXML struct {
|
||||
GUID string `xml:"guid"`
|
||||
Title string `xml:"title"`
|
||||
ITunesTitle string `xml:"http://www.itunes.com/dtds/podcast-1.0.dtd title"`
|
||||
Subtitle string `xml:"http://www.itunes.com/dtds/podcast-1.0.dtd subtitle"`
|
||||
// Description resolution order (plan §5): description >
|
||||
// itunes:summary > content:encoded.
|
||||
Description string `xml:"description"`
|
||||
ITunesSummary string `xml:"http://www.itunes.com/dtds/podcast-1.0.dtd summary"`
|
||||
ContentEncoded string `xml:"encoded"`
|
||||
|
||||
// Item author: itunes:author preferred, then RSS author, then dc:creator.
|
||||
ITunesAuthor string `xml:"http://www.itunes.com/dtds/podcast-1.0.dtd author"`
|
||||
Author string `xml:"author"`
|
||||
Creator string `xml:"http://purl.org/dc/elements/1.1/ creator"`
|
||||
|
||||
ITunesImage struct {
|
||||
Href string `xml:"href,attr"`
|
||||
} `xml:"http://www.itunes.com/dtds/podcast-1.0.dtd image"`
|
||||
ITunesExplicit string `xml:"http://www.itunes.com/dtds/podcast-1.0.dtd explicit"`
|
||||
ITunesDuration string `xml:"http://www.itunes.com/dtds/podcast-1.0.dtd duration"`
|
||||
ITunesEpisode string `xml:"http://www.itunes.com/dtds/podcast-1.0.dtd episode"`
|
||||
ITunesSeason string `xml:"http://www.itunes.com/dtds/podcast-1.0.dtd season"`
|
||||
|
||||
PubDate string `xml:"pubDate"`
|
||||
|
||||
Enclosure struct {
|
||||
URL string `xml:"url,attr"`
|
||||
Length int64 `xml:"length,attr"`
|
||||
Type string `xml:"type,attr"`
|
||||
} `xml:"enclosure"`
|
||||
|
||||
// Podcasting 2.0: store-now, feature-later URLs.
|
||||
Transcript struct {
|
||||
URL string `xml:"url,attr"`
|
||||
} `xml:"transcript"`
|
||||
Chapters struct {
|
||||
URL string `xml:"url,attr"`
|
||||
} `xml:"chapters"`
|
||||
Persons []struct {
|
||||
Href string `xml:"href,attr"`
|
||||
Role string `xml:"role,attr"`
|
||||
} `xml:"person"`
|
||||
}
|
||||
|
||||
// ParsePodcastFeed decodes a feed document. Returns an error when the
|
||||
// XML is unparseable or the document is not an RSS channel.
|
||||
func ParsePodcastFeed(data []byte) (*PodcastFeedXML, error) {
|
||||
var feed PodcastFeedXML
|
||||
if err := xml.Unmarshal(data, &feed); err != nil {
|
||||
return nil, fmt.Errorf("feed parse: %w", err)
|
||||
}
|
||||
if feed.Channel.Title == "" && len(feed.Channel.Items) == 0 {
|
||||
return nil, fmt.Errorf("feed parse: not an RSS channel (no title, no items)")
|
||||
}
|
||||
return &feed, nil
|
||||
}
|
||||
|
||||
// EpisodeTitle prefers itunes:title over the raw RSS title (plan §5).
|
||||
func (it *PodcastItemXML) EpisodeTitle() string {
|
||||
if t := strings.TrimSpace(it.ITunesTitle); t != "" {
|
||||
return t
|
||||
}
|
||||
return strings.TrimSpace(it.Title)
|
||||
}
|
||||
|
||||
// EpisodeDescription resolves the description fallback chain.
|
||||
func (it *PodcastItemXML) EpisodeDescription() string {
|
||||
for _, c := range []string{it.Description, it.ITunesSummary, it.ContentEncoded} {
|
||||
if t := strings.TrimSpace(c); t != "" {
|
||||
return t
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// EpisodeAuthor resolves the author fallback chain.
|
||||
func (it *PodcastItemXML) EpisodeAuthor() string {
|
||||
for _, c := range []string{it.ITunesAuthor, it.Author, it.Creator} {
|
||||
if t := strings.TrimSpace(c); t != "" {
|
||||
return t
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// EpisodeExplicit maps the itunes:explicit spelling variants to bool.
|
||||
func (it *PodcastItemXML) EpisodeExplicit() bool {
|
||||
return parseExplicit(it.ITunesExplicit)
|
||||
}
|
||||
|
||||
func parseExplicit(v string) bool {
|
||||
switch strings.ToLower(strings.TrimSpace(v)) {
|
||||
case "true", "yes", "explicit", "1":
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// HostsAndGuests splits podcast:person entries by role (plan §5).
|
||||
func (it *PodcastItemXML) HostsAndGuests() (hosts []string, guests []string) {
|
||||
for _, p := range it.Persons {
|
||||
name := p.Href
|
||||
if name == "" {
|
||||
continue
|
||||
}
|
||||
switch strings.ToLower(p.Role) {
|
||||
case "guest":
|
||||
guests = append(guests, name)
|
||||
default: // host and unlabeled roles surface as hosts
|
||||
hosts = append(hosts, name)
|
||||
}
|
||||
}
|
||||
return hosts, guests
|
||||
}
|
||||
|
||||
// parseITunesDuration accepts the three spellings the spec allows —
|
||||
// plain seconds, HH:MM:SS, MM:SS — plus a best-effort prose fallback
|
||||
// ("58 minutes"). Returns 0 for anything unparseable (the column stays
|
||||
// NULL; ffprobe reconciles after download anyway).
|
||||
func parseITunesDuration(v string) int {
|
||||
v = strings.TrimSpace(v)
|
||||
if v == "" {
|
||||
return 0
|
||||
}
|
||||
if secs, err := strconv.Atoi(v); err == nil {
|
||||
return secs
|
||||
}
|
||||
parts := strings.Split(v, ":")
|
||||
if len(parts) <= 3 {
|
||||
total := 0
|
||||
ok := true
|
||||
for _, p := range parts {
|
||||
n, err := strconv.Atoi(strings.TrimSpace(p))
|
||||
if err != nil {
|
||||
ok = false
|
||||
break
|
||||
}
|
||||
total = total*60 + n
|
||||
}
|
||||
if ok {
|
||||
return total
|
||||
}
|
||||
}
|
||||
// Prose best-effort: "<N> minutes/hours/seconds".
|
||||
lower := strings.ToLower(v)
|
||||
fields := strings.Fields(lower)
|
||||
for i, f := range fields {
|
||||
if n, err := strconv.Atoi(f); err == nil && i+1 < len(fields) {
|
||||
switch {
|
||||
case strings.HasPrefix(fields[i+1], "hour"):
|
||||
return n * 3600
|
||||
case strings.HasPrefix(fields[i+1], "minute"):
|
||||
return n * 60
|
||||
case strings.HasPrefix(fields[i+1], "second"):
|
||||
return n
|
||||
}
|
||||
}
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
// parseFeedTime parses the RFC-822/1123 variants feeds actually emit.
|
||||
func parseFeedTime(v string) (t time.Time, ok bool) {
|
||||
v = strings.TrimSpace(v)
|
||||
if v == "" {
|
||||
return time.Time{}, false
|
||||
}
|
||||
layouts := []string{
|
||||
time.RFC1123Z,
|
||||
time.RFC1123,
|
||||
time.RFC822Z,
|
||||
time.RFC822,
|
||||
"Mon, 2 Jan 2006 15:04:05 -0700",
|
||||
"Mon, 2 Jan 2006 15:04:05 MST",
|
||||
"2 Jan 2006 15:04:05 -0700",
|
||||
"2 Jan 2006 15:04:05 MST",
|
||||
time.ANSIC,
|
||||
time.UnixDate,
|
||||
}
|
||||
for _, layout := range layouts {
|
||||
if t, err := time.Parse(layout, v); err == nil {
|
||||
return t, true
|
||||
}
|
||||
}
|
||||
return time.Time{}, false
|
||||
}
|
||||
Reference in new issue
Block a user