feat(podcasts): RSS fetcher — parse, SSRF guards, upsert-without-delete (task 1.4)
The fetcher is the SOLE writer of feed/episode metadata (D9). Three files, no new deps (stdlib encoding/xml + net/http): podcast_rss.go — RSS 2.0 + iTunes + Podcasting-2.0 structs. iTunes fields are namespace-qualified in tags (local names collide with core RSS: title/subtitle/author/summary/explicit/image/duration); podcast:* fields match on local name (collision-free, tolerant of both declared namespace URIs). Description/author fallback chains per §5; itunes:duration parses seconds/HH:MM:SS/MM:SS/prose; pubDate accepts the RFC-822/1123 spellings feeds actually emit. podcast_http.go — the security boundary for user-supplied URLs: https-only on every hop (CheckRedirect), private/loopback/link-local/ multicast IP ranges blocked at DIAL time (Dialer.Control sees the RESOLVED address — DNS rebinding into 127.0.0.1 is refused at the socket), 10-redirect cap, hard body caps (25MiB feeds / 20MiB artwork / 1GiB enclosures), no proxy. podcast_service.go — fetch → parse → upsert. Episodes upsert metadata-only keyed on (source_feed_id, episode_guid); rows for items that vanish from the XML are NEVER deleted (D14 search covers historical episodes). itunes:new-feed-url + permanent HTTP redirects rewrite feed_url in place (row/subscriptions/episodes stay put); podcast:guid dedupe reuses the canonical row instead of forking. Artwork downloads feed-level under <feed-slug>/artwork.<ext>. AddFeed authorizes ONLY on explicit arguments (library visible + allow_member_feeds for non-admins — the D20 scoping model); detached work runs under the first-admin identity via adminCtx (invariant 12). Feed layout: <download-root>/<feed-slug>/<pos:04d>-<slug>.<ext> so ServeFile/Range streaming resolve unchanged.
This commit is contained in:
John O'Keefe committed
1 parent
9a148c2344
commit
85a9873a75
3 files changed
+1209
No files matched your search
@@ -0,0 +1,762 @@
|
||||
// Podcast fetcher (plan podcast-integration.md §7): fetch → parse →
|
||||
// upsert-without-delete. The fetcher is the SOLE writer of feed/episode
|
||||
// metadata (D9) — user-facing podcast writes are limited to
|
||||
// podcast_subscriptions rows and the reader_settings order override.
|
||||
//
|
||||
// RLS identity (invariant 12): fetches run detached from any request,
|
||||
// so every governed statement runs under the FIRST-ADMIN identity
|
||||
// (scanner fallback precedent). With the D20 admin escape that identity
|
||||
// sees and updates every feed/episode row. NEVER a bypass flag, never
|
||||
// the bootstrap superuser.
|
||||
//
|
||||
// Upsert semantics: episodes are created/updated METADATA-ONLY, keyed
|
||||
// on (source_feed_id, episode_guid); rows for items that vanish from
|
||||
// the XML are NEVER deleted (history accumulates — D14 search covers
|
||||
// gone episodes). Download state is owned by the download worker; the
|
||||
// upsert cannot clobber it (guarded in the query itself).
|
||||
package services
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"bookhoard/internal/database"
|
||||
"bookhoard/internal/rls"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgtype"
|
||||
)
|
||||
|
||||
// PodcastError is a status-carrying service error: handlers map it to
|
||||
// the HTTP response directly (services stay echo-free).
|
||||
type PodcastError struct {
|
||||
Status int
|
||||
Message string
|
||||
}
|
||||
|
||||
func (e *PodcastError) Error() string { return e.Message }
|
||||
|
||||
func errNotFound(msg string) error { return &PodcastError{Status: 404, Message: msg} }
|
||||
func errForbidden(msg string) error { return &PodcastError{Status: 403, Message: msg} }
|
||||
func errBadRequest(msg string) error { return &PodcastError{Status: 400, Message: msg} }
|
||||
|
||||
// PodcastService carries the fetcher/worker dependencies. One instance
|
||||
// per server (handlers + poller share it so the per-feed locks dedupe).
|
||||
type PodcastService struct {
|
||||
db *database.Queries
|
||||
|
||||
adminMu sync.Mutex
|
||||
adminID pgtype.UUID
|
||||
adminOnce bool
|
||||
|
||||
// Per-feed mutexes: a poll and a download for the same feed must
|
||||
// not race (e.g. both writing the episode row). Keyed by feed UUID.
|
||||
feedLocksMu sync.Mutex
|
||||
feedLocks map[string]*sync.Mutex
|
||||
}
|
||||
|
||||
func NewPodcastService(db *database.Queries) *PodcastService {
|
||||
return &PodcastService{
|
||||
db: db,
|
||||
feedLocks: make(map[string]*sync.Mutex),
|
||||
}
|
||||
}
|
||||
|
||||
// feedLock returns the per-feed mutex (creates on first use).
|
||||
func (s *PodcastService) feedLock(feedID string) *sync.Mutex {
|
||||
s.feedLocksMu.Lock()
|
||||
defer s.feedLocksMu.Unlock()
|
||||
if m, ok := s.feedLocks[feedID]; ok {
|
||||
return m
|
||||
}
|
||||
m := &sync.Mutex{}
|
||||
s.feedLocks[feedID] = m
|
||||
return m
|
||||
}
|
||||
|
||||
// adminIdentity resolves the acting identity for detached podcast work:
|
||||
// the first admin, resolved once per service lifetime. Errors surface
|
||||
// to the caller (no admin → no fetch — same contract as the scanner).
|
||||
func (s *PodcastService) adminIdentity() (pgtype.UUID, error) {
|
||||
s.adminMu.Lock()
|
||||
defer s.adminMu.Unlock()
|
||||
if !s.adminOnce && !s.adminID.Valid {
|
||||
id, err := s.db.GetFirstAdmin(context.Background())
|
||||
if err != nil {
|
||||
return pgtype.UUID{}, fmt.Errorf("podcasts: no admin identity available for detached work: %w", err)
|
||||
}
|
||||
s.adminID = id
|
||||
s.adminOnce = true
|
||||
}
|
||||
if !s.adminID.Valid {
|
||||
return pgtype.UUID{}, fmt.Errorf("podcasts: no admin identity available for detached work")
|
||||
}
|
||||
return s.adminID, nil
|
||||
}
|
||||
|
||||
// adminCtx wraps ctx with the first-admin identity (invariant 12).
|
||||
func (s *PodcastService) adminCtx(ctx context.Context) (context.Context, error) {
|
||||
id, err := s.adminIdentity()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return rls.WithUser(ctx, id), nil
|
||||
}
|
||||
|
||||
// AddFeedResult carries what the add-feed flow did (the handler turns
|
||||
// it into the response / serial feed hint).
|
||||
type AddFeedResult struct {
|
||||
Feed *database.PodcastFeeds
|
||||
// Created is true when this call created the feed row; false when
|
||||
// an existing row was reused (URL dedupe or guid migration match).
|
||||
Created bool
|
||||
// Subscribed is true when the requesting user's subscription was
|
||||
// created by this call (false = already subscribed).
|
||||
Subscribed bool
|
||||
// EpisodeCount is the number of episode rows the fetch upserted.
|
||||
EpisodeCount int
|
||||
// FetchError, when set, means the feed row exists but the fetch
|
||||
// failed — the subscription stands, the error surfaces to the user.
|
||||
FetchError string
|
||||
}
|
||||
|
||||
// AddFeed implements POST /api/podcasts/feeds (plan §8): fetch+parse the
|
||||
// user-supplied URL, create-or-reuse the feed row, upsert metadata-only
|
||||
// episodes, subscribe the creator. Authorization on explicit arguments
|
||||
// (D20 scoping model): the target library must be visible to the actor
|
||||
// and, for non-admins, allow member feeds.
|
||||
func (s *PodcastService) AddFeed(ctx context.Context, userID pgtype.UUID, isAdmin bool, libraryID pgtype.UUID, rawFeedURL string) (*AddFeedResult, error) {
|
||||
// ---- explicit-argument authorization (the ONLY Go-level gate) ----
|
||||
lib, err := s.db.GetLibrary(ctx, libraryID)
|
||||
if err != nil {
|
||||
// Invisible-or-nonexistent: 404 semantics (existence leak).
|
||||
return nil, errNotFound("library not found")
|
||||
}
|
||||
if !isAdmin {
|
||||
if lib.TypeName != LibraryTypePodcasts {
|
||||
return nil, errNotFound("library not found")
|
||||
}
|
||||
if !lib.AllowMemberFeeds {
|
||||
return nil, errForbidden("this library does not allow members to add feeds")
|
||||
}
|
||||
}
|
||||
|
||||
feedURL := strings.TrimSpace(rawFeedURL)
|
||||
if feedURL == "" {
|
||||
return nil, errBadRequest("feed URL is required")
|
||||
}
|
||||
|
||||
// The fetch/upsert runs under the first-admin identity (detached-
|
||||
// equivalent); the request context never carries special powers into
|
||||
// feed writes.
|
||||
actx, err := s.adminCtx(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
res := &AddFeedResult{}
|
||||
|
||||
// ---- fetch + parse (SSRF-guarded; plan §7) ----
|
||||
fetched, err := guardFetch(ctx, feedURL, "", "", maxFeedSizeBytes)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("could not fetch feed: %w", classifyFetchError(err))
|
||||
}
|
||||
parsed, err := ParsePodcastFeed(fetched.Body)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("could not parse feed: %w", err)
|
||||
}
|
||||
|
||||
// Follow itunes:new-feed-url once at add time so the row is born
|
||||
// pointing at the publisher's current home (migration semantics).
|
||||
finalURL := fetched.FinalURL
|
||||
if neu := strings.TrimSpace(parsed.Channel.ITunesNewFeedURL); neu != "" && neu != finalURL {
|
||||
if retried, err := guardFetch(ctx, neu, "", "", maxFeedSizeBytes); err == nil {
|
||||
if reparsed, err := ParsePodcastFeed(retried.Body); err == nil {
|
||||
parsed = reparsed
|
||||
finalURL = retried.FinalURL
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
title := parsed.Channel.Title
|
||||
if title == "" {
|
||||
title = "Untitled feed"
|
||||
}
|
||||
|
||||
// ---- create-or-reuse (UNIQUE(library_id, feed_url) + guid dedupe) ----
|
||||
feed, err := s.db.CreatePodcastFeed(actx, database.CreatePodcastFeedParams{
|
||||
LibraryID: libraryID,
|
||||
FeedUrl: finalURL,
|
||||
Title: truncate(title, 500),
|
||||
CreatedByAdminID: pgtype.UUID{}, // fetcher owns feed metadata; creator attribution is not user metadata
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("could not create feed: %w", err)
|
||||
}
|
||||
res.Feed = &feed
|
||||
res.Created = true
|
||||
|
||||
// Guid migration dedupe: a row with the same canonical podcast_guid
|
||||
// under a different URL reuses THAT row (do not fork the feed).
|
||||
if guid := strings.TrimSpace(parsed.Channel.PodcastGUID); guid != "" {
|
||||
if feed.PodcastGuid.String == "" || feed.PodcastGuid.String != guid {
|
||||
if existing, err := s.db.GetPodcastFeedByGuid(actx, database.GetPodcastFeedByGuidParams{
|
||||
LibraryID: feed.LibraryID,
|
||||
PodcastGuid: pgtypeText(guid),
|
||||
}); err == nil && existing.ID != feed.ID {
|
||||
// Attach the subscriber to the canonical row; drop the
|
||||
// freshly created stub (nothing subscribed to it yet).
|
||||
_, _ = s.db.DeletePodcastFeed(actx, feed.ID)
|
||||
feed = existing
|
||||
res.Feed = &existing
|
||||
// Re-point the canonical row at the fetched URL if it moved.
|
||||
if feedUrlForComparison(existing.FeedUrl) != feedUrlForComparison(finalURL) {
|
||||
_ = s.db.UpdatePodcastFeedRedirect(actx, database.UpdatePodcastFeedRedirectParams{
|
||||
ID: existing.ID,
|
||||
FeedUrl: finalURL,
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if _, err := s.db.SubscribePodcastFeed(actx, database.SubscribePodcastFeedParams{
|
||||
UserID: userID,
|
||||
FeedID: feed.ID,
|
||||
}); err == nil {
|
||||
res.Subscribed = true
|
||||
}
|
||||
|
||||
// ---- metadata + episodes ----
|
||||
if err := s.applyFetch(actx, &feed, parsed, fetched); err != nil {
|
||||
// Feed exists + subscribed, but the metadata pass failed: not
|
||||
// fatal for the request; the poller reconciles later.
|
||||
res.FetchError = err.Error()
|
||||
return res, nil
|
||||
}
|
||||
|
||||
updated, err := s.db.GetPodcastFeed(actx, feed.ID)
|
||||
if err == nil {
|
||||
res.Feed = &updated
|
||||
}
|
||||
return res, nil
|
||||
}
|
||||
|
||||
// RefreshFeed implements POST /api/podcasts/feeds/:id/refresh: a manual
|
||||
// poll. The per-feed lock serializes against poller ticks and downloads.
|
||||
func (s *PodcastService) RefreshFeed(ctx context.Context, feedID pgtype.UUID) error {
|
||||
actx, err := s.adminCtx(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
feed, err := s.db.GetPodcastFeed(actx, feedID)
|
||||
if err != nil {
|
||||
return errNotFound("feed not found")
|
||||
}
|
||||
return s.PollFeed(actx, &feed)
|
||||
}
|
||||
|
||||
// PollFeed fetches + reconciles ONE feed (caller supplies the acting
|
||||
// identity in ctx). Safe to call concurrently per feed — the per-feed
|
||||
// lock serializes poll/download.
|
||||
func (s *PodcastService) PollFeed(ctx context.Context, feed *database.PodcastFeeds) error {
|
||||
lock := s.feedLock(feed.ID.String())
|
||||
lock.Lock()
|
||||
defer lock.Unlock()
|
||||
|
||||
// Complete feeds back off: they are fetched only on explicit
|
||||
// refresh (the poller skips them entirely — this path is manual).
|
||||
fetched, err := guardFetch(ctx, feed.FeedUrl, feed.Etag.String, feed.LastModified.String, maxFeedSizeBytes)
|
||||
if err != nil {
|
||||
if feed.LastFetchedAt.Valid {
|
||||
// A previously-good feed with a transient failure: keep the
|
||||
// old state; the poller retries next tick.
|
||||
return fmt.Errorf("fetch %s: %w", feed.Title, classifyFetchError(err))
|
||||
}
|
||||
return fmt.Errorf("fetch %s: %w", feed.Title, classifyFetchError(err))
|
||||
}
|
||||
if fetched.NotModified {
|
||||
_, _ = s.db.UpdatePodcastFeedFetch(ctx, database.UpdatePodcastFeedFetchParams{
|
||||
ID: feed.ID,
|
||||
Etag: pgtypeText(feed.Etag.String),
|
||||
LastModified: pgtypeText(feed.LastModified.String),
|
||||
LastBuildDate: feed.LastBuildDate,
|
||||
})
|
||||
return nil
|
||||
}
|
||||
|
||||
parsed, err := ParsePodcastFeed(fetched.Body)
|
||||
if err != nil {
|
||||
return fmt.Errorf("parse %s: %w", feed.Title, err)
|
||||
}
|
||||
|
||||
// itunes:new-feed-url: the publisher MOVED the feed. Rewrite the
|
||||
// stored URL (row, subscriptions and episodes stay put) and refetch
|
||||
// the new location once so this poll applies its content.
|
||||
if neu := strings.TrimSpace(parsed.Channel.ITunesNewFeedURL); neu != "" && neu != feed.FeedUrl {
|
||||
_ = s.db.UpdatePodcastFeedRedirect(ctx, database.UpdatePodcastFeedRedirectParams{ID: feed.ID, FeedUrl: neu})
|
||||
if retried, err := guardFetch(ctx, neu, "", "", maxFeedSizeBytes); err == nil {
|
||||
if reparsed, err := ParsePodcastFeed(retried.Body); err == nil {
|
||||
parsed = reparsed
|
||||
fetched = retried
|
||||
}
|
||||
}
|
||||
} else if fetched.FinalURL != "" && feedUrlForComparison(fetched.FinalURL) != feedUrlForComparison(feed.FeedUrl) {
|
||||
// A permanent HTTP redirect chain moved the feed; persist the
|
||||
// final URL so conditional validators stop bouncing.
|
||||
_ = s.db.UpdatePodcastFeedRedirect(ctx, database.UpdatePodcastFeedRedirectParams{ID: feed.ID, FeedUrl: fetched.FinalURL})
|
||||
}
|
||||
|
||||
if err := s.applyFetch(ctx, feed, parsed, fetched); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// applyFetch writes channel metadata, upserts episodes metadata-only,
|
||||
// touches validators, and downloads artwork. Write-after-success rule
|
||||
// (invariant 7): failures here never clear existing rows.
|
||||
func (s *PodcastService) applyFetch(ctx context.Context, feed *database.PodcastFeeds, parsed *PodcastFeedXML, fetched *fetchResult) error {
|
||||
// --- channel metadata (fetcher-owned) ---
|
||||
title := parsed.Channel.Title
|
||||
if title == "" {
|
||||
title = feed.Title // never blank out a title on a broken doc
|
||||
}
|
||||
desc := stripHTML(parsed.Channel.Description)
|
||||
|
||||
host := parsed.Channel.ITunesAuthor
|
||||
publisher := parsed.Channel.ITunesOwner.Name
|
||||
|
||||
tagSet := map[string]bool{}
|
||||
if cat := strings.TrimSpace(parsed.Channel.ITunesCategory.Text); cat != "" {
|
||||
tagSet[cat] = true
|
||||
}
|
||||
if sub := strings.TrimSpace(parsed.Channel.ITunesCategory.Subcategory.Text); sub != "" {
|
||||
tagSet[sub] = true
|
||||
}
|
||||
tags := sortedStringSet(tagSet)
|
||||
|
||||
feedType := strings.TrimSpace(parsed.Channel.ITunesType)
|
||||
if feedType != "serial" && feedType != "episodic" {
|
||||
feedType = "" // unknown → column stays NULL
|
||||
}
|
||||
medium := strings.TrimSpace(parsed.Channel.PodcastMedium)
|
||||
|
||||
if _, err := s.db.UpdatePodcastFeedMetadata(ctx, database.UpdatePodcastFeedMetadataParams{
|
||||
ID: feed.ID,
|
||||
Title: truncate(title, 500),
|
||||
Description: pgtypeText(desc),
|
||||
Host: pgtypeText(truncate(host, 500)),
|
||||
Publisher: pgtypeText(truncate(publisher, 500)),
|
||||
OwnerName: pgtypeText(truncate(parsed.Channel.ITunesOwner.Name, 500)),
|
||||
OwnerEmail: pgtypeText(truncate(parsed.Channel.ITunesOwner.Email, 500)),
|
||||
Language: pgtypeText(truncate(parsed.Channel.Language, 50)),
|
||||
Tags: tags,
|
||||
Explicit: parseExplicit(parsed.Channel.ITunesExplicit),
|
||||
FeedType: pgtypeText(feedType),
|
||||
Medium: pgtypeText(truncate(medium, 20)),
|
||||
IsComplete: parseExplicit(parsed.Channel.ITunesComplete),
|
||||
PodcastGuid: pgtypeText(strings.TrimSpace(parsed.Channel.PodcastGUID)),
|
||||
}); err != nil {
|
||||
return fmt.Errorf("feed metadata update: %w", err)
|
||||
}
|
||||
// itunes:complete flips the poller's back-off flag (complete feeds
|
||||
// poll rarely); explicit refresh still works.
|
||||
|
||||
// --- episodes (metadata-only upsert, positions from THIS parse) ---
|
||||
root, err := s.downloadRootAbs(ctx, feed)
|
||||
if err != nil {
|
||||
return fmt.Errorf("download root: %w", err)
|
||||
}
|
||||
feedSlug := feedSlug(title, feed.ID)
|
||||
|
||||
count := 0
|
||||
for i := range parsed.Channel.Items {
|
||||
if err := s.upsertEpisode(ctx, feed, root, feedSlug, &parsed.Channel.Items[i], i+1); err != nil {
|
||||
// One bad item must not sink the batch.
|
||||
log.Printf("[PODCAST] episode upsert failed for feed %s item %d: %v", feed.ID, i+1, err)
|
||||
continue
|
||||
}
|
||||
count++
|
||||
}
|
||||
|
||||
// --- validators + fetch timestamp ---
|
||||
var lastBuildDate pgtype.Timestamptz
|
||||
if t, ok := parseFeedTime(parsed.Channel.LastBuildDate); ok {
|
||||
lastBuildDate = pgtypeTimestamptz(t)
|
||||
}
|
||||
if _, err := s.db.UpdatePodcastFeedFetch(ctx, database.UpdatePodcastFeedFetchParams{
|
||||
ID: feed.ID,
|
||||
Etag: pgtypeText(fetched.ETag),
|
||||
LastModified: pgtypeText(fetched.LastModified),
|
||||
LastBuildDate: lastBuildDate,
|
||||
}); err != nil {
|
||||
return fmt.Errorf("fetch touch: %w", err)
|
||||
}
|
||||
|
||||
// --- artwork (best-effort; feed renders fine without it) ---
|
||||
s.refreshArtwork(ctx, feed, root, parsed)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// upsertEpisode maps one XML item onto a media_items row (plan §5 field
|
||||
// inventory). Positions are 1-based, renumbered per fetch; file_path is
|
||||
// the PLANNED final layout — the upsert only applies it to rows that
|
||||
// are not yet downloaded (query-side guard).
|
||||
func (s *PodcastService) upsertEpisode(ctx context.Context, feed *database.PodcastFeeds, downloadRoot, feedSlug string, item *PodcastItemXML, position int) error {
|
||||
title := item.EpisodeTitle()
|
||||
if title == "" {
|
||||
title = fmt.Sprintf("Episode %d", position)
|
||||
}
|
||||
guid := strings.TrimSpace(item.GUID)
|
||||
if guid == "" {
|
||||
// GUID-less feeds exist; fall back to a stable identity from the
|
||||
// enclosure URL (the next-best immutable per-episode key).
|
||||
if item.Enclosure.URL != "" {
|
||||
guid = "enc:" + item.Enclosure.URL
|
||||
} else {
|
||||
return fmt.Errorf("item %q has neither guid nor enclosure; skipping", title)
|
||||
}
|
||||
}
|
||||
|
||||
ext := enclosureExtension(item.Enclosure.URL, item.Enclosure.Type)
|
||||
relPath := episodeRelPath(feedSlug, position, title, ext)
|
||||
|
||||
var published pgtype.Date
|
||||
if t, ok := parseFeedTime(item.PubDate); ok {
|
||||
published = pgtypeDate(t)
|
||||
}
|
||||
|
||||
authors := []string{}
|
||||
if a := item.EpisodeAuthor(); a != "" {
|
||||
authors = append(authors, truncate(a, 255))
|
||||
}
|
||||
hosts, guests := item.HostsAndGuests()
|
||||
|
||||
_, err := s.db.UpsertPodcastEpisode(ctx, database.UpsertPodcastEpisodeParams{
|
||||
LibraryID: feed.LibraryID,
|
||||
Title: truncate(title, 255),
|
||||
Subtitle: pgtypeText(truncate(item.Subtitle, 500)),
|
||||
Description: pgtypeText(stripHTML(item.EpisodeDescription())),
|
||||
Authors: authors,
|
||||
AuthorsSearch: lowerSearchSlice(authors),
|
||||
FilePath: relPath,
|
||||
FileSize: pgtypeInt8(item.Enclosure.Length),
|
||||
MimeType: pgtypeText(truncate(item.Enclosure.Type, 100)),
|
||||
CoverImagePath: pgtypeText(artworkRelPath(feedSlug, artworkExtFromURL(item.ITunesImage.Href))),
|
||||
Explicit: item.EpisodeExplicit(),
|
||||
DatePublished: published,
|
||||
Language: pgtypeText(truncate(feed.Language.String, 10)),
|
||||
Tags: feed.Tags,
|
||||
Genre: pgtypeText(strings.Join(feed.Tags, ", ")),
|
||||
SourceFeedID: feed.ID,
|
||||
EpisodeGuid: pgtypeText(truncate(guid, 500)),
|
||||
FeedPosition: pgtypeInt4(int32(position)),
|
||||
SeasonNumber: pgtypeInt4FromStr(item.ITunesSeason),
|
||||
EpisodeNumber: pgtypeInt4FromStr(item.ITunesEpisode),
|
||||
EnclosureUrl: pgtypeText(item.Enclosure.URL),
|
||||
DeclaredDurationSeconds: pgtypeInt4(int32(parseITunesDuration(item.ITunesDuration))),
|
||||
TranscriptUrl: pgtypeText(item.Transcript.URL),
|
||||
ChaptersUrl: pgtypeText(item.Chapters.URL),
|
||||
Hosts: truncateSlice(hosts, 255),
|
||||
Guests: truncateSlice(guests, 255),
|
||||
})
|
||||
_ = downloadRoot // layout is relative; abs root resolves at download time
|
||||
return err
|
||||
}
|
||||
|
||||
// refreshArtwork downloads the channel image once per changed URL.
|
||||
// Artwork is feed-level: episodes point at the same relative path.
|
||||
func (s *PodcastService) refreshArtwork(ctx context.Context, feed *database.PodcastFeeds, downloadRoot string, parsed *PodcastFeedXML) {
|
||||
imgURL := strings.TrimSpace(parsed.Channel.ITunesImage.Href)
|
||||
if imgURL == "" {
|
||||
return
|
||||
}
|
||||
target := filepath.Join(downloadRoot, artworkRelPath(feedSlug(feed.Title, feed.ID), artworkExtFromURL(imgURL)))
|
||||
if feed.ArtworkPath.Valid {
|
||||
if existing := filepath.Join(downloadRoot, feed.ArtworkPath.String); existing == target {
|
||||
if _, err := os.Stat(existing); err == nil {
|
||||
return // unchanged
|
||||
}
|
||||
}
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Dir(target), 0o755); err != nil {
|
||||
return
|
||||
}
|
||||
res, err := guardFetch(ctx, imgURL, "", "", maxArtworkSizeBytes)
|
||||
if err != nil || len(res.Body) == 0 {
|
||||
log.Printf("[PODCAST] artwork fetch failed for feed %s: %v", feed.ID, err)
|
||||
return
|
||||
}
|
||||
if err := os.WriteFile(target, res.Body, 0o644); err != nil {
|
||||
log.Printf("[PODCAST] artwork write failed for feed %s: %v", feed.ID, err)
|
||||
return
|
||||
}
|
||||
rel, err := filepath.Rel(downloadRoot, target)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
_ = s.db.UpdatePodcastFeedArtwork(ctx, database.UpdatePodcastFeedArtworkParams{
|
||||
ID: feed.ID,
|
||||
ArtworkPath: pgtypeText(rel),
|
||||
})
|
||||
}
|
||||
|
||||
// DeleteFeedFiles removes a feed's episode files + artwork from the
|
||||
// download root (admin feed-removal path; ?files=false skips it).
|
||||
// Best-effort per file — the row cascade proceeds regardless.
|
||||
func (s *PodcastService) DeleteFeedFiles(ctx context.Context, feed *database.PodcastFeeds, episodes []database.MediaItems) {
|
||||
root, err := s.downloadRootAbs(ctx, feed)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
for _, ep := range episodes {
|
||||
if ep.FilePath != "" {
|
||||
_ = os.Remove(filepath.Join(root, ep.FilePath))
|
||||
// Embedded-cover sibling (extractEmbeddedAudioCover convention).
|
||||
base := filepath.Join(root, strings.TrimSuffix(ep.FilePath, filepath.Ext(ep.FilePath)))
|
||||
_ = os.Remove(base + ".cover.jpg")
|
||||
}
|
||||
}
|
||||
if feed.ArtworkPath.Valid {
|
||||
_ = os.Remove(filepath.Join(root, feed.ArtworkPath.String))
|
||||
}
|
||||
// The feed directory itself: remove when empty (other feeds never
|
||||
// share a slug — the feed-id suffix guarantees that).
|
||||
_ = os.Remove(filepath.Join(root, feedSlug(feed.Title, feed.ID)))
|
||||
}
|
||||
|
||||
// downloadRootAbs resolves the podcasts library's single download root
|
||||
// (purpose='podcast') as an absolute filesystem path.
|
||||
func (s *PodcastService) downloadRootAbs(ctx context.Context, feed *database.PodcastFeeds) (string, error) {
|
||||
folders, err := s.db.GetPodcastLibraryFolder(ctx, feed.LibraryID)
|
||||
if err != nil || len(folders) == 0 {
|
||||
return "", fmt.Errorf("library %s has no download root registered", feed.LibraryID)
|
||||
}
|
||||
return folders[0].FolderPath, nil
|
||||
}
|
||||
|
||||
// episodeRelPath is the pinned episode layout (plan §6):
|
||||
// <feed-slug>/<feed_position:04d>-<slug>.<ext> — relative to the
|
||||
// download root. getFullFilePath/ServeFile/Range streaming resolve it
|
||||
// unchanged.
|
||||
func episodeRelPath(feedSlug string, position int, title, ext string) string {
|
||||
return fmt.Sprintf("%s/%04d-%s%s", feedSlug, position, slugify(title, 80), ext)
|
||||
}
|
||||
|
||||
// artworkRelPath is the feed-level artwork location (feed-level, so
|
||||
// every episode row can point at the same file).
|
||||
func artworkRelPath(feedSlug, ext string) string {
|
||||
return fmt.Sprintf("%s/artwork%s", feedSlug, ext)
|
||||
}
|
||||
|
||||
// feedSlug derives the stable per-feed directory name: slugged title +
|
||||
// short feed-id suffix so two identically-titled feeds never collide.
|
||||
func feedSlug(title string, feedID pgtype.UUID) string {
|
||||
base := slugify(title, 60)
|
||||
suffix := "feed"
|
||||
if feedID.Valid {
|
||||
s := feedID.String()
|
||||
if len(s) >= 8 {
|
||||
suffix = s[:8]
|
||||
}
|
||||
}
|
||||
if base == "" {
|
||||
base = "feed"
|
||||
}
|
||||
return base + "-" + suffix
|
||||
}
|
||||
|
||||
var slugStrip = regexp.MustCompile(`[^a-z0-9]+`)
|
||||
|
||||
// slugify lowercases, strips non-alphanumerics to dashes, and clamps
|
||||
// the length (file-name-safe; unicode letters drop — file layout is
|
||||
// an internal key, never display text).
|
||||
func slugify(s string, maxLen int) string {
|
||||
s = strings.ToLower(strings.TrimSpace(s))
|
||||
s = slugStrip.ReplaceAllString(s, "-")
|
||||
s = strings.Trim(s, "-")
|
||||
if len(s) > maxLen {
|
||||
s = strings.TrimRight(s[:maxLen], "-")
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// enclosureExtension picks a file extension from the URL, then the
|
||||
// MIME type, then a safe default.
|
||||
func enclosureExtension(rawURL, mimeType string) string {
|
||||
if u, err := urlParse(rawURL); err == nil {
|
||||
if ext := strings.ToLower(filepath.Ext(u.Path)); ext == ".mp3" || ext == ".m4a" || ext == ".m4b" ||
|
||||
ext == ".flac" || ext == ".ogg" || ext == ".oga" || ext == ".opus" || ext == ".wav" || ext == ".aac" {
|
||||
return ext
|
||||
}
|
||||
}
|
||||
switch {
|
||||
case strings.Contains(mimeType, "mp4") || strings.Contains(mimeType, "m4a"):
|
||||
return ".m4a"
|
||||
case strings.Contains(mimeType, "mpeg") || strings.Contains(mimeType, "mp3"):
|
||||
return ".mp3"
|
||||
case strings.Contains(mimeType, "flac"):
|
||||
return ".flac"
|
||||
case strings.Contains(mimeType, "ogg"):
|
||||
return ".ogg"
|
||||
case strings.Contains(mimeType, "wav"):
|
||||
return ".wav"
|
||||
}
|
||||
return ".mp3"
|
||||
}
|
||||
|
||||
// artworkExtFromURL keeps the source extension for artwork (jpg/png/webp).
|
||||
func artworkExtFromURL(rawURL string) string {
|
||||
if u, err := urlParse(rawURL); err == nil {
|
||||
switch strings.ToLower(filepath.Ext(u.Path)) {
|
||||
case ".png":
|
||||
return ".png"
|
||||
case ".webp":
|
||||
return ".webp"
|
||||
case ".gif":
|
||||
return ".gif"
|
||||
}
|
||||
}
|
||||
return ".jpg"
|
||||
}
|
||||
|
||||
// stripHTML removes tags and collapses whitespace — feed descriptions
|
||||
// are HTML; storage wants text (the TUI-ish surfaces render it plain).
|
||||
func stripHTML(s string) string {
|
||||
if s == "" {
|
||||
return ""
|
||||
}
|
||||
var b strings.Builder
|
||||
inTag := false
|
||||
for _, r := range s {
|
||||
switch {
|
||||
case r == '<':
|
||||
inTag = true
|
||||
case r == '>':
|
||||
inTag = false
|
||||
case !inTag:
|
||||
b.WriteRune(r)
|
||||
}
|
||||
}
|
||||
out := b.String()
|
||||
// Collapse runs of whitespace.
|
||||
out = strings.Join(strings.Fields(out), " ")
|
||||
return strings.TrimSpace(out)
|
||||
}
|
||||
|
||||
// truncate clamps a string to max runes (VARCHAR safety at the DB edge).
|
||||
func truncate(s string, max int) string {
|
||||
s = strings.TrimSpace(s)
|
||||
runes := []rune(s)
|
||||
if len(runes) <= max {
|
||||
return s
|
||||
}
|
||||
return string(runes[:max])
|
||||
}
|
||||
|
||||
// truncateSlice clamps each element and drops empties.
|
||||
func truncateSlice(in []string, max int) []string {
|
||||
out := make([]string, 0, len(in))
|
||||
for _, s := range in {
|
||||
if t := truncate(s, max); t != "" {
|
||||
out = append(out, t)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// lowerSearchSlice mirrors authors into authors_search (scanner convention).
|
||||
func lowerSearchSlice(in []string) []string {
|
||||
out := make([]string, 0, len(in))
|
||||
for _, s := range in {
|
||||
out = append(out, strings.ToLower(s))
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// sortedStringSet → deterministic TEXT[] writes.
|
||||
func sortedStringSet(set map[string]bool) []string {
|
||||
out := make([]string, 0, len(set))
|
||||
for k := range set {
|
||||
if strings.TrimSpace(k) == "" {
|
||||
continue
|
||||
}
|
||||
out = append(out, strings.TrimSpace(k))
|
||||
}
|
||||
// tiny insertion sort — tag sets are 1–5 entries
|
||||
for i := 1; i < len(out); i++ {
|
||||
for j := i; j > 0 && out[j] < out[j-1]; j-- {
|
||||
out[j], out[j-1] = out[j-1], out[j]
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// classifyFetchError maps guard/HTTP errors to user-facing text (the
|
||||
// add-feed dialog and the download_error column both render it).
|
||||
func classifyFetchError(err error) error {
|
||||
if errors.Is(err, ErrBlockedPrivateAddress) {
|
||||
return fmt.Errorf("blocked: private/loopback addresses are not fetchable")
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// urlParse local helper so podcast files don't each import net/url.
|
||||
func urlParse(raw string) (*url.URL, error) {
|
||||
return url.Parse(raw)
|
||||
}
|
||||
|
||||
// ---- pgtype helpers shared by the podcast files ----
|
||||
|
||||
func pgtypeText(s string) pgtype.Text {
|
||||
if strings.TrimSpace(s) == "" {
|
||||
return pgtype.Text{}
|
||||
}
|
||||
return pgtype.Text{String: s, Valid: true}
|
||||
}
|
||||
|
||||
func pgtypeInt4(v int32) pgtype.Int4 {
|
||||
if v == 0 {
|
||||
return pgtype.Int4{}
|
||||
}
|
||||
return pgtype.Int4{Int32: v, Valid: true}
|
||||
}
|
||||
|
||||
func pgtypeInt4FromStr(s string) pgtype.Int4 {
|
||||
n, err := strconv.Atoi(strings.TrimSpace(s))
|
||||
if err != nil || n <= 0 {
|
||||
return pgtype.Int4{}
|
||||
}
|
||||
return pgtype.Int4{Int32: int32(n), Valid: true}
|
||||
}
|
||||
|
||||
func pgtypeInt8(v int64) pgtype.Int8 {
|
||||
if v <= 0 {
|
||||
return pgtype.Int8{}
|
||||
}
|
||||
return pgtype.Int8{Int64: v, Valid: true}
|
||||
}
|
||||
|
||||
func pgtypeTimestamptz(t time.Time) pgtype.Timestamptz {
|
||||
return pgtype.Timestamptz{Time: t, Valid: true}
|
||||
}
|
||||
|
||||
func pgtypeDate(t time.Time) pgtype.Date {
|
||||
return pgtype.Date{Time: t.UTC().Truncate(24 * time.Hour), Valid: true}
|
||||
}
|
||||
|
||||
// feedUrlForComparison normalizes trailing slashes for the redirect check.
|
||||
func feedUrlForComparison(u string) string {
|
||||
return strings.TrimRight(strings.TrimSpace(u), "/")
|
||||
}
|
||||
Reference in new issue
Block a user