feat(podcasts): concurrent download pool — fair share, tickets, admin tunables
Replaces the sequential worker with a download-manager-style pool (§0 item 4). Model: the user's connection is the limit; semantics are fair-share ACROSS feeds, not goroutine-per-book (a global FIFO would make Book B wait behind Book A's 100 files). - Atomic claim: GetNextQueuedEpisode + SetPodcastEpisodeDownloading was a read-then-write race under concurrency; ClaimQueuedPodcastEpisode converts queued→downloading in ONE guarded statement (two claimants, one winner — the loser gets ErrNoRows). SetPodcastEpisodeDownloading retired. - Queue tickets: download_queue_pos BIGINT fed by a Postgres SEQUENCE stamped at enqueue — the enqueue loop's ordering IS the claim order (the foundation §0 item 5 builds on; previously all rows in a batch shared one updated_at, so within-feed order was arbitrary). - Pool: up to podcast_download_concurrency (default 4, live range 1–16) claim-loop goroutines per drain round; the setting re-reads every round so admin changes take effect without restart. - Fair share: claim prefers the feed least-recently-served (pickFeedToServe; never-served first), earliest ticket within a feed — books proceed in parallel while chapters land in order (start listening while the rest download). - Bandwidth: shared token bucket across all workers (podcast_download_speed_mbps, default 0 = unlimited); take() reserves bandwidth and callers sleep outside the lock. - Settings: new float setting type end to end (registry defaults + clamping, save validation, admin UI number-with-step input) + the two schema seeds; worker wires the registry via SetSettings (tests run on registered defaults). - Tests: unit coverage for the bucket reservation math + fair-share pick; integration TestPodcastDownloadPool pins ticket order = enqueue order, claim exclusivity, and a full 3-episode pool drain. Dev-DB note: the column/sequence/settings were applied live (idempotent DDL, same statements the schema runs at next boot) since the running container predates them.
This commit is contained in:
1 parent
e3675536aa
commit
0f3c4a8cec
14 files changed
+893
-264
No files matched your search
@@ -102,7 +102,9 @@ INSERT INTO system_settings (setting_key, setting_value, description, setting_ty
|
||||
('worker_pool_size', '3', 'Number of background worker goroutines', 'int', '1', '100', TRUE, 'performance'),
|
||||
('worker_queue_cap', '100', 'Background worker job queue capacity', 'int', '1', '10000', TRUE, 'performance'),
|
||||
-- podcasts
|
||||
('podcast_poll_interval_seconds', '3600', 'How often to poll podcast feeds for new episodes (seconds; complete feeds back off)', 'int', '300', '604800', FALSE, 'podcasts')
|
||||
('podcast_poll_interval_seconds', '3600', 'How often to poll podcast feeds for new episodes (seconds; complete feeds back off)', 'int', '300', '604800', FALSE, 'podcasts'),
|
||||
('podcast_download_concurrency', '4', 'Podcast episode downloads in flight server-wide (the connection is the limit; fair-share spreads them across feeds)', 'int', '1', '16', FALSE, 'podcasts'),
|
||||
('podcast_download_speed_mbps', '0', 'Shared bandwidth cap for podcast downloads in megabits/second (0 = unlimited)', 'float', '0', '10000', FALSE, 'podcasts')
|
||||
ON CONFLICT (setting_key) DO NOTHING;
|
||||
|
||||
-- Create refresh_tokens table
|
||||
@@ -1989,7 +1991,16 @@ ALTER TABLE media_items
|
||||
-- list renders this inline (D8/D19) — never Processing Issues.
|
||||
ADD COLUMN IF NOT EXISTS download_state VARCHAR(20) NOT NULL DEFAULT 'pending',
|
||||
ADD COLUMN IF NOT EXISTS download_error TEXT,
|
||||
ADD COLUMN IF NOT EXISTS download_attempts INT NOT NULL DEFAULT 0;
|
||||
ADD COLUMN IF NOT EXISTS download_attempts INT NOT NULL DEFAULT 0,
|
||||
-- Monotonic queue ticket (Postgres SEQUENCE nextval at enqueue
|
||||
-- time): the download pool claims in ticket order, so the enqueue
|
||||
-- loop's ordering IS the download order (effective listening order
|
||||
-- — §0 items 4/5). NULL for rows never queued.
|
||||
ADD COLUMN IF NOT EXISTS download_queue_pos BIGINT;
|
||||
|
||||
-- Queue ticket source: atomic across concurrent enqueues (per-feed
|
||||
-- locks serialize within a feed; the sequence covers across feeds).
|
||||
CREATE SEQUENCE IF NOT EXISTS podcast_queue_seq;
|
||||
|
||||
-- Identity + ordering + worker-worklist indexes (all partial: scan
|
||||
-- rows never touch these columns).
|
||||
@@ -1999,6 +2010,8 @@ CREATE INDEX IF NOT EXISTS idx_media_items_feed_pos
|
||||
ON media_items(source_feed_id, feed_position) WHERE source_feed_id IS NOT NULL;
|
||||
CREATE INDEX IF NOT EXISTS idx_media_items_feed_undownloaded
|
||||
ON media_items(source_feed_id) WHERE source_feed_id IS NOT NULL AND download_state <> 'downloaded';
|
||||
CREATE INDEX IF NOT EXISTS idx_media_items_queue
|
||||
ON media_items(download_queue_pos) WHERE source = 'podcast_episode' AND download_state = 'queued';
|
||||
|
||||
|
||||
-- ============================================================
|
||||
|
||||
Reference in new issue
Block a user