John O'Keefe
253d8d0e8e
Release / build-and-push (push) Successful in 3m29s
Apple scheme normalization (follow-up to the Scribl discussion): Add Feed accepts the launcher spellings podcast pages ship next to plain RSS — itpc://, podcasts://, feed:// — and rewrites them to the https feed URL they wrap (scheme match case-insensitive). http stays and the guard still rejects it. Unit-pinned. Build stamp (stale-container drift guard): the 2026-10-29 podcast session lost an hour to a container running a build that predated the feature. version.Commit rides ldflags from a GIT_COMMIT build arg; the dev compose passes git rev-parse --short HEAD via the Makefile; the app logs 'Bookhoard <ver> (commit <hash>)' at boot and /api/health reports it — the UI and the repo can no longer silently diverge.
304 lines
11 KiB
Go
304 lines
11 KiB
Go
// Unit tests for the podcast fetcher's pure logic (plan task 1.4/1.10):
|
|
// RSS/iTunes/Podcasting-2.0 parsing, duration/pubDate spellings, slug/
|
|
// path layout, and the SSRF guard (scheme + resolved-IP checks). No DB.
|
|
package services
|
|
|
|
import (
|
|
"errors"
|
|
"net"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"net/url"
|
|
"strings"
|
|
"testing"
|
|
|
|
"github.com/google/uuid"
|
|
"github.com/jackc/pgx/v5/pgtype"
|
|
"github.com/stretchr/testify/assert"
|
|
"github.com/stretchr/testify/require"
|
|
)
|
|
|
|
// pgUUID alias keeps the helper signatures short.
|
|
type pgUUID = pgtype.UUID
|
|
|
|
// Full-fixture feed covering the three namespaces and the fallback
|
|
// chains (itunes:title vs title; description vs itunes:summary vs
|
|
// content:encoded; itunes:author vs author vs dc:creator).
|
|
const podcastFixtureXML = `<?xml version="1.0" encoding="UTF-8"?>
|
|
<rss version="2.0"
|
|
xmlns:itunes="http://www.itunes.com/dtds/podcast-1.0.dtd"
|
|
xmlns:content="http://purl.org/rss/1.0/modules/content/"
|
|
xmlns:dc="http://purl.org/dc/elements/1.1/"
|
|
xmlns:podcast="https://podcastindex.org/namespace/1.0">
|
|
<channel>
|
|
<title>The Test Feed</title>
|
|
<link>https://example.com/show</link>
|
|
<language>en-us</language>
|
|
<description><p>A feed about <b>testing</b>.</p></description>
|
|
<itunes:author>Host Name</itunes:author>
|
|
<itunes:owner>
|
|
<itunes:name>Owner Name</itunes:name>
|
|
<itunes:email>owner@example.com</itunes:email>
|
|
</itunes:owner>
|
|
<itunes:image href="https://cdn.example.com/art.jpg"/>
|
|
<itunes:explicit>false</itunes:explicit>
|
|
<itunes:type>serial</itunes:type>
|
|
<itunes:category text="Fiction">
|
|
<itunes:category text="Drama"/>
|
|
</itunes:category>
|
|
<podcast:guid>9b0241c2-xxxx-4xxx-yxxx-xxxxxxxxxxxx</podcast:guid>
|
|
<podcast:medium>audiobook</podcast:medium>
|
|
<lastBuildDate>Mon, 26 Oct 2026 08:00:00 +0000</lastBuildDate>
|
|
<item>
|
|
<guid isPermaLink="false">episode-guid-1</guid>
|
|
<itunes:title>Episode One: The Beginning</itunes:title>
|
|
<title>Ep 1</title>
|
|
<itunes:subtitle>First chapter</itunes:subtitle>
|
|
<description><![CDATA[<p>Chapter one text.</p>]]></description>
|
|
<itunes:author>Narrator A</itunes:author>
|
|
<itunes:duration>1:02:03</itunes:duration>
|
|
<itunes:episode>1</itunes:episode>
|
|
<itunes:season>2</itunes:season>
|
|
<pubDate>Sun, 25 Oct 2026 12:00:00 +0000</pubDate>
|
|
<enclosure url="https://cdn.example.com/ep1.mp3" length="52428800" type="audio/mpeg"/>
|
|
<podcast:transcript url="https://cdn.example.com/ep1.json" type="application/json"/>
|
|
<podcast:chapters url="https://cdn.example.com/ep1-chapters.json" type="application/json+chapters"/>
|
|
<podcast:person href="Narrator A" role="host"/>
|
|
<podcast:person href="Guest B" role="guest"/>
|
|
</item>
|
|
<item>
|
|
<guid>episode-guid-2</guid>
|
|
<title>Episode Two</title>
|
|
<itunes:summary>Summary only description.</itunes:summary>
|
|
<dc:creator>Writer C</dc:creator>
|
|
<itunes:duration>5400</itunes:duration>
|
|
<pubDate>Mon, 26 Oct 2026 12:00:00 GMT</pubDate>
|
|
<enclosure url="https://cdn.example.com/ep2.m4a" length="1048576" type="audio/mp4"/>
|
|
</item>
|
|
</channel>
|
|
</rss>`
|
|
|
|
func TestParsePodcastFeed(t *testing.T) {
|
|
feed, err := ParsePodcastFeed([]byte(podcastFixtureXML))
|
|
require.NoError(t, err)
|
|
|
|
ch := feed.Channel
|
|
assert.Equal(t, "The Test Feed", ch.Title)
|
|
assert.Equal(t, "en-us", ch.Language)
|
|
assert.Equal(t, "Host Name", ch.ITunesAuthor)
|
|
assert.Equal(t, "Owner Name", ch.ITunesOwner.Name)
|
|
assert.Equal(t, "owner@example.com", ch.ITunesOwner.Email)
|
|
assert.Equal(t, "https://cdn.example.com/art.jpg", ch.ITunesImage.Href)
|
|
assert.Equal(t, "serial", ch.ITunesType)
|
|
assert.Equal(t, "Fiction", ch.ITunesCategory.Text)
|
|
assert.Equal(t, "Drama", ch.ITunesCategory.Subcategory.Text)
|
|
assert.Equal(t, "9b0241c2-xxxx-4xxx-yxxx-xxxxxxxxxxxx", ch.PodcastGUID)
|
|
assert.Equal(t, "audiobook", ch.PodcastMedium)
|
|
assert.False(t, parseExplicit(ch.ITunesExplicit))
|
|
|
|
buildAt, ok := parseFeedTime(ch.LastBuildDate)
|
|
require.True(t, ok)
|
|
assert.Equal(t, 2026, buildAt.Year())
|
|
|
|
require.Len(t, ch.Items, 2)
|
|
|
|
// Item 1: itunes:title preferred, description chain, hosts/guests.
|
|
it1 := &ch.Items[0]
|
|
assert.Equal(t, "Episode One: The Beginning", it1.EpisodeTitle())
|
|
assert.Equal(t, "First chapter", it1.Subtitle)
|
|
assert.Equal(t, "<p>Chapter one text.</p>", it1.EpisodeDescription()) // raw HTML; stripping happens at storage time
|
|
assert.Equal(t, "Narrator A", it1.EpisodeAuthor())
|
|
assert.Equal(t, 3723, parseITunesDuration(it1.ITunesDuration)) // 1:02:03
|
|
assert.Equal(t, 2, intMust(it1.ITunesSeason))
|
|
assert.Equal(t, 1, intMust(it1.ITunesEpisode))
|
|
assert.Equal(t, "https://cdn.example.com/ep1.mp3", it1.Enclosure.URL)
|
|
assert.Equal(t, int64(52428800), it1.Enclosure.Length)
|
|
assert.Equal(t, "audio/mpeg", it1.Enclosure.Type)
|
|
assert.Equal(t, "https://cdn.example.com/ep1.json", it1.Transcript.URL)
|
|
assert.Equal(t, "https://cdn.example.com/ep1-chapters.json", it1.Chapters.URL)
|
|
hosts, guests := it1.HostsAndGuests()
|
|
assert.Equal(t, []string{"Narrator A"}, hosts)
|
|
assert.Equal(t, []string{"Guest B"}, guests)
|
|
assert.Equal(t, "2026-10-25", mustDate(it1.PubDate))
|
|
|
|
// Item 2: fallback chains — plain title, itunes:summary, dc:creator.
|
|
it2 := &ch.Items[1]
|
|
assert.Equal(t, "Episode Two", it2.EpisodeTitle())
|
|
assert.Equal(t, "Summary only description.", it2.EpisodeDescription())
|
|
assert.Equal(t, "Writer C", it2.EpisodeAuthor())
|
|
assert.Equal(t, 5400, parseITunesDuration(it2.ITunesDuration)) // seconds
|
|
assert.Equal(t, "https://cdn.example.com/ep2.m4a", it2.Enclosure.URL)
|
|
}
|
|
|
|
func intMust(s string) int {
|
|
n, _ := parseSeasonOrEpisode(s)
|
|
return n
|
|
}
|
|
|
|
func parseSeasonOrEpisode(s string) (int, bool) {
|
|
if s == "" {
|
|
return 0, false
|
|
}
|
|
n := 0
|
|
for _, r := range s {
|
|
if r < '0' || r > '9' {
|
|
return 0, false
|
|
}
|
|
n = n*10 + int(r-'0')
|
|
}
|
|
return n, true
|
|
}
|
|
|
|
func mustDate(pubDate string) string {
|
|
if t, ok := parseFeedTime(pubDate); ok {
|
|
return t.UTC().Format("2006-01-02")
|
|
}
|
|
return ""
|
|
}
|
|
|
|
func TestParseITunesDuration(t *testing.T) {
|
|
assert.Equal(t, 0, parseITunesDuration(""))
|
|
assert.Equal(t, 90, parseITunesDuration("90"))
|
|
assert.Equal(t, 3723, parseITunesDuration("1:02:03"))
|
|
assert.Equal(t, 123, parseITunesDuration("2:03"))
|
|
assert.Equal(t, 3600, parseITunesDuration("1 hour"))
|
|
assert.Equal(t, 120, parseITunesDuration("2 minutes"))
|
|
assert.Equal(t, 0, parseITunesDuration("soonish"))
|
|
}
|
|
|
|
func TestParseFeedTimeVariants(t *testing.T) {
|
|
for _, tc := range []struct {
|
|
in string
|
|
year int
|
|
}{
|
|
{"Mon, 26 Oct 2026 08:00:00 +0000", 2026},
|
|
{"Mon, 26 Oct 2026 08:00:00 GMT", 2026},
|
|
{"26 Oct 2026 08:00:00 +0000", 2026},
|
|
} {
|
|
tm, ok := parseFeedTime(tc.in)
|
|
require.True(t, ok, "parse %q", tc.in)
|
|
assert.Equal(t, tc.year, tm.Year())
|
|
}
|
|
_, ok := parseFeedTime("not a date")
|
|
assert.False(t, ok)
|
|
}
|
|
|
|
func TestEpisodeExplicitSpellings(t *testing.T) {
|
|
assert.True(t, parseExplicit("true"))
|
|
assert.True(t, parseExplicit("yes"))
|
|
assert.True(t, parseExplicit("explicit"))
|
|
assert.False(t, parseExplicit("false"))
|
|
assert.False(t, parseExplicit("clean"))
|
|
assert.False(t, parseExplicit(""))
|
|
}
|
|
|
|
func TestSlugifyAndPaths(t *testing.T) {
|
|
assert.Equal(t, "the-test-feed", slugify("The Test Feed!", 60))
|
|
assert.Equal(t, "a-b-c", slugify(" A -- B // C ", 60))
|
|
assert.Equal(t, "abcdefghij", slugify("ABCDEFGHIJ", 10))
|
|
|
|
slug := feedSlug("My Feed", testUUID("11111111-1111-1111-1111-111111111111"))
|
|
assert.Equal(t, "my-feed-11111111", slug)
|
|
|
|
rel := episodeRelPath(slug, 42, "Episode: The Test", ".mp3")
|
|
assert.Equal(t, "my-feed-11111111/0042-episode-the-test.mp3", rel)
|
|
|
|
assert.Equal(t, "my-feed-11111111/artwork.jpg", artworkRelPath(slug, ".jpg"))
|
|
}
|
|
|
|
func TestEnclosureExtension(t *testing.T) {
|
|
assert.Equal(t, ".mp3", enclosureExtension("https://x.test/a/b/ep.mp3?token=1", ""))
|
|
assert.Equal(t, ".m4a", enclosureExtension("https://x.test/ep", "audio/mp4"))
|
|
assert.Equal(t, ".flac", enclosureExtension("https://x.test/ep", "audio/flac"))
|
|
assert.Equal(t, ".mp3", enclosureExtension("https://x.test/ep", "weird/type"))
|
|
}
|
|
|
|
func TestStripHTML(t *testing.T) {
|
|
assert.Equal(t, "hello world", stripHTML("<p>hello <b>world</b></p>"))
|
|
assert.Equal(t, "a b", stripHTML("a\n\n b"))
|
|
assert.Equal(t, "", stripHTML(""))
|
|
}
|
|
|
|
func TestIsPublicIP(t *testing.T) {
|
|
blocked := []string{
|
|
"127.0.0.1", "10.0.0.5", "172.16.0.9", "192.168.1.1",
|
|
"169.254.169.254", // cloud metadata endpoint
|
|
"0.0.0.0", "::1", "fe80::1", "fd00::5", "::ffff:127.0.0.1",
|
|
"::ffff:10.0.0.1", "224.0.0.1",
|
|
}
|
|
for _, s := range blocked {
|
|
assert.False(t, isPublicIP(net.ParseIP(s)), "%s must be blocked", s)
|
|
}
|
|
allowed := []string{
|
|
"93.184.216.34", "1.1.1.1", "2606:4700:4700::1111",
|
|
}
|
|
for _, s := range allowed {
|
|
assert.True(t, isPublicIP(net.ParseIP(s)), "%s must be allowed", s)
|
|
}
|
|
assert.False(t, isPublicIP(nil))
|
|
}
|
|
|
|
func TestGuardURLScheme(t *testing.T) {
|
|
u, _ := url.Parse("http://example.com/feed.xml")
|
|
assert.Error(t, guardURLScheme(u))
|
|
u, _ = url.Parse("ftp://example.com/feed.xml")
|
|
assert.Error(t, guardURLScheme(u))
|
|
u, _ = url.Parse("https://example.com/feed.xml")
|
|
assert.NoError(t, guardURLScheme(u))
|
|
assert.Error(t, guardURLScheme(nil))
|
|
}
|
|
|
|
// The REAL client (production factory) must refuse loopback targets —
|
|
// this is the SSRF boundary; httptest servers live on 127.0.0.1.
|
|
func TestGuardFetchBlocksLoopback(t *testing.T) {
|
|
srv := newInsecureFeedServer(t, podcastFixtureXML)
|
|
defer srv.Close()
|
|
_, err := guardFetch(t.Context(), srv.URL, "", "", maxFeedSizeBytes)
|
|
require.Error(t, err)
|
|
assert.True(t, strings.Contains(err.Error(), "blocked") ||
|
|
strings.Contains(err.Error(), "https"), "expected an SSRF/scheme rejection, got: %v", err)
|
|
}
|
|
|
|
func TestGuardFetchRejectsPlainHTTP(t *testing.T) {
|
|
// A public-looking http:// URL: scheme guard fires before any dial.
|
|
_, err := guardFetch(t.Context(), "http://example.com/feed.xml", "", "", maxFeedSizeBytes)
|
|
require.Error(t, err)
|
|
assert.Contains(t, err.Error(), "https")
|
|
}
|
|
|
|
func TestClassifyFetchError(t *testing.T) {
|
|
err := classifyFetchError(ErrBlockedPrivateAddress)
|
|
assert.Contains(t, err.Error(), "blocked")
|
|
someErr := &url.Error{Op: "Get", Err: errors.New("boom")}
|
|
assert.Equal(t, someErr.Error(), classifyFetchError(someErr).Error())
|
|
}
|
|
|
|
// testUUID builds a pgtype.UUID from a string (test helper).
|
|
func testUUID(s string) pgUUID {
|
|
u, _ := uuid.Parse(s)
|
|
return pgUUID{Bytes: u, Valid: true}
|
|
}
|
|
|
|
// newInsecureFeedServer serves body over httptest TLS (loopback) —
|
|
// used to prove the production client REFUSES it.
|
|
func newInsecureFeedServer(t *testing.T, body string) *httptest.Server {
|
|
t.Helper()
|
|
srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
|
w.Header().Set("Content-Type", "application/rss+xml")
|
|
_, _ = w.Write([]byte(body))
|
|
}))
|
|
t.Cleanup(srv.Close)
|
|
return srv
|
|
}
|
|
|
|
// Apple Podcast launcher schemes normalize to the https feed they wrap.
|
|
func TestNormalizeFeedURL(t *testing.T) {
|
|
assert.Equal(t, "https://x.test/feed.xml", normalizeFeedURL("itpc://x.test/feed.xml"))
|
|
assert.Equal(t, "https://x.test/feed.xml", normalizeFeedURL("podcasts://x.test/feed.xml"))
|
|
assert.Equal(t, "https://x.test/feed.xml", normalizeFeedURL("feed://x.test/feed.xml"))
|
|
assert.Equal(t, "https://x.test/feed.xml", normalizeFeedURL("ITPC://x.test/feed.xml"), "scheme match is case-insensitive")
|
|
assert.Equal(t, "https://x.test/feed.xml", normalizeFeedURL("https://x.test/feed.xml"))
|
|
assert.Equal(t, "http://x.test/feed.xml", normalizeFeedURL("http://x.test/feed.xml"), "http stays — the guard rejects it downstream")
|
|
assert.Equal(t, "", normalizeFeedURL(" "))
|
|
}
|