Bookhoard/bookhoard
0
0 Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases 65 Wiki Activity

feat(podcasts): concurrent download pool — fair share, tickets, admin tunables

Browse Source 
Replaces the sequential worker with a download-manager-style pool
(§0 item 4). Model: the user's connection is the limit; semantics are
fair-share ACROSS feeds, not goroutine-per-book (a global FIFO would
make Book B wait behind Book A's 100 files).

- Atomic claim: GetNextQueuedEpisode + SetPodcastEpisodeDownloading
  was a read-then-write race under concurrency; ClaimQueuedPodcastEpisode
  converts queued→downloading in ONE guarded statement (two claimants,
  one winner — the loser gets ErrNoRows). SetPodcastEpisodeDownloading
  retired.
- Queue tickets: download_queue_pos BIGINT fed by a Postgres SEQUENCE
  stamped at enqueue — the enqueue loop's ordering IS the claim order
  (the foundation §0 item 5 builds on; previously all rows in a batch
  shared one updated_at, so within-feed order was arbitrary).
- Pool: up to podcast_download_concurrency (default 4, live range 1–16)
  claim-loop goroutines per drain round; the setting re-reads every
  round so admin changes take effect without restart.
- Fair share: claim prefers the feed least-recently-served
  (pickFeedToServe; never-served first), earliest ticket within a feed —
  books proceed in parallel while chapters land in order (start
  listening while the rest download).
- Bandwidth: shared token bucket across all workers
  (podcast_download_speed_mbps, default 0 = unlimited); take() reserves
  bandwidth and callers sleep outside the lock.
- Settings: new float setting type end to end (registry defaults +
  clamping, save validation, admin UI number-with-step input) + the two
  schema seeds; worker wires the registry via SetSettings (tests run on
  registered defaults).
- Tests: unit coverage for the bucket reservation math + fair-share
  pick; integration TestPodcastDownloadPool pins ticket order = enqueue
  order, claim exclusivity, and a full 3-episode pool drain.

Dev-DB note: the column/sequence/settings were applied live (idempotent
DDL, same statements the schema runs at next boot) since the running
container predates them.
This commit is contained in:
John O'Keefe committed 2026-10-30 10:12:43 -04:00
1 parent e3675536aa
commit 0f3c4a8cec
14 files changed
+893 -264
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

internal/database/db.go
+1 -1
View File
@@ -1,6 +1,6 @@
// Code generated by sqlc. DO NOT EDIT.
// versions:
// sqlc v1.31.1
// sqlc v1.31.0
package database
internal/database/models.go
+2 -1
View File
@@ -1,6 +1,6 @@
// Code generated by sqlc. DO NOT EDIT.
// versions:
// sqlc v1.31.1
// sqlc v1.31.0
package database
@@ -355,6 +355,7 @@ type MediaItems struct {
DownloadState string `db:"download_state" json:"download_state"`
DownloadError pgtype.Text `db:"download_error" json:"download_error"`
DownloadAttempts int32 `db:"download_attempts" json:"download_attempts"`
DownloadQueuePos pgtype.Int8 `db:"download_queue_pos" json:"download_queue_pos"`
}
type MediaNotes struct {
internal/database/querier.go
+18 -6
View File
@@ -1,6 +1,6 @@
// Code generated by sqlc. DO NOT EDIT.
// versions:
// sqlc v1.31.1
// sqlc v1.31.0
package database
@@ -32,6 +32,11 @@ type Querier interface {
BulkUpdateFormatGroups(ctx context.Context) error
BulkUpdateProgressFromSync(ctx context.Context, arg BulkUpdateProgressFromSyncParams) ([]interface{}, error)
CheckForProgressConflicts(ctx context.Context, arg CheckForProgressConflictsParams) (int64, error)
// Atomic claim (§0 item 4): converts queued -> downloading in ONE
// statement. Two claimants racing on the same row: exactly one UPDATE
// matches download_state='queued'; the loser gets no rows (ErrNoRows).
// This is the SKIP-LOCKED guarantee without a second round-trip.
ClaimQueuedPodcastEpisode(ctx context.Context, id pgtype.UUID) (MediaItems, error)
// Cleanup expired OPDS tokens
CleanupExpiredOpdsTokens(ctx context.Context) error
CleanupExpiredRefreshTokens(ctx context.Context, dollar_1 float64) error
@@ -446,8 +451,6 @@ type Querier interface {
// GetMediaScanState reads one item's manifest (per-item rescan paths
// and the manifest integration tests).
GetMediaScanState(ctx context.Context, mediaItemID pgtype.UUID) (MediaScanState, error)
// Sequential download worker's claim source (single worker, day one).
GetNextQueuedEpisode(ctx context.Context) (MediaItems, error)
GetNextRetryTime(ctx context.Context) (interface{}, error)
GetNotStartedItems(ctx context.Context, arg GetNotStartedItemsParams) ([]MediaItems, error)
// Get OPDS token
@@ -656,6 +659,11 @@ type Querier interface {
// Item-side projections COALESCE to keep them non-nullable strings;
// the display file path falls back to the issue's own for row-less rows.
ListProcessingIssuesByLibrary(ctx context.Context, libraryID pgtype.UUID) ([]ListProcessingIssuesByLibraryRow, error)
// Fair-share candidate scan (§0 item 4): the download pool claims in
// TICKET order (download_queue_pos = SEQUENCE nextval at enqueue — the
// enqueue loop's ordering IS the download order, item 5). The worker
// groups these per feed and prefers the least-recently-served feed.
ListQueuedPodcastEpisodes(ctx context.Context) ([]ListQueuedPodcastEpisodesRow, error)
ListSyncConflictsByMediaItem(ctx context.Context, arg ListSyncConflictsByMediaItemParams) ([]SyncConflicts, error)
ListSyncConflictsByUser(ctx context.Context, userID pgtype.UUID) ([]ListSyncConflictsByUserRow, error)
// Download All + policy union worklist.
@@ -668,6 +676,9 @@ type Querier interface {
ListUsers(ctx context.Context) ([]ListUsersRow, error)
// First consecutive scan that cannot find the file on disk.
MarkMediaItemMissing(ctx context.Context, id pgtype.UUID) error
// SetPodcastEpisodeDownloading RETIRED (§0 item 4): the pair
// GetNextQueuedEpisode + SetPodcastEpisodeDownloading raced under
// concurrency; ClaimQueuedPodcastEpisode does both atomically.
// SUCCESS-only write (invariant 7): the file is on disk and probed.
// ffprobe facts (real duration via audio_metadata, precise size/mime)
// reconcile the feed's declared values.
@@ -753,10 +764,11 @@ type Querier interface {
SetDeviceSyncEnabled(ctx context.Context, arg SetDeviceSyncEnabledParams) (Devices, error)
// Library Visibility queries
SetLibraryVisibility(ctx context.Context, arg SetLibraryVisibilityParams) (LibraryVisibility, error)
// Worker claim: pending|failed -> queued. Attempts incremented at
// claim time so a crashed worker still counts the try.
// Enqueue: pending|failed -> queued, stamped with the NEXT QUEUE TICKET
// (§0 items 4/5) so claim order = enqueue order (the effective
// listening order the enqueue loop applied). Attempts increment at
// enqueue so a crashed worker still counts the try.
SetPodcastEpisodeDownloadQueued(ctx context.Context, id pgtype.UUID) (int64, error)
SetPodcastEpisodeDownloading(ctx context.Context, id pgtype.UUID) error
// Manual retry (same endpoint as download): reset to pending so the
// worker re-claims it; the reason text clears with the fresh attempt.
SetPodcastEpisodePending(ctx context.Context, id pgtype.UUID) error
internal/database/queries.sql.go
+249 -162
View File
File diff suppressed because it is too large. Load diff
internal/database/queries/queries.sql
+27 -11
View File
@@ -4862,10 +4862,13 @@ SELECT * FROM media_items
WHERE id = $1 AND source = 'podcast_episode';
-- name: SetPodcastEpisodeDownloadQueued :execrows
-- Worker claim: pending|failed -> queued. Attempts incremented at
-- claim time so a crashed worker still counts the try.
-- Enqueue: pending|failed -> queued, stamped with the NEXT QUEUE TICKET
-- (§0 items 4/5) so claim order = enqueue order (the effective
-- listening order the enqueue loop applied). Attempts increment at
-- enqueue so a crashed worker still counts the try.
UPDATE media_items
SET download_state = 'queued',
download_queue_pos = nextval('podcast_queue_seq'),
download_error = NULL,
download_attempts = download_attempts + 1,
updated_at = NOW()
@@ -4873,10 +4876,9 @@ WHERE id = $1
AND source = 'podcast_episode'
AND download_state IN ('pending', 'failed');
-- name: SetPodcastEpisodeDownloading :exec
UPDATE media_items
SET download_state = 'downloading', updated_at = NOW()
WHERE id = $1 AND source = 'podcast_episode';
-- SetPodcastEpisodeDownloading RETIRED (§0 item 4): the pair
-- GetNextQueuedEpisode + SetPodcastEpisodeDownloading raced under
-- concurrency; ClaimQueuedPodcastEpisode does both atomically.
-- name: MarkPodcastEpisodeDownloaded :one
-- SUCCESS-only write (invariant 7): the file is on disk and probed.
@@ -4964,12 +4966,26 @@ WHERE s.user_id = sqlc.arg('user_id')
OR COALESCE(f.publisher, '') ILIKE '%' || sqlc.arg('search_query')::text || '%')
ORDER BY COALESCE(s.display_name, f.title) ASC;
-- name: GetNextQueuedEpisode :one
-- Sequential download worker's claim source (single worker, day one).
SELECT * FROM media_items
-- name: ListQueuedPodcastEpisodes :many
-- Fair-share candidate scan (§0 item 4): the download pool claims in
-- TICKET order (download_queue_pos = SEQUENCE nextval at enqueue — the
-- enqueue loop's ordering IS the download order, item 5). The worker
-- groups these per feed and prefers the least-recently-served feed.
SELECT id, source_feed_id FROM media_items
WHERE source = 'podcast_episode' AND download_state = 'queued'
ORDER BY updated_at ASC
LIMIT 1;
ORDER BY download_queue_pos ASC NULLS LAST;
-- name: ClaimQueuedPodcastEpisode :one
-- Atomic claim (§0 item 4): converts queued -> downloading in ONE
-- statement. Two claimants racing on the same row: exactly one UPDATE
-- matches download_state='queued'; the loser gets no rows (ErrNoRows).
-- This is the SKIP-LOCKED guarantee without a second round-trip.
UPDATE media_items
SET download_state = 'downloading', updated_at = NOW()
WHERE id = $1
AND source = 'podcast_episode'
AND download_state = 'queued'
RETURNING media_items.*;
-- name: CountQueuedEpisodes :one
SELECT COUNT(*) FROM media_items
internal/database/settings_registry.go
+37 -3
View File
@@ -25,6 +25,7 @@ import (
// SettingType enumerates the value types stored in system_settings.setting_type.
const (
SettingTypeInt = "int"
SettingTypeFloat = "float"
SettingTypeBool = "bool"
SettingTypeString = "string"
SettingTypeStringList = "string_list"
@@ -83,6 +84,8 @@ var SettingDefaults = []SettingDefault{
{Key: "worker_pool_size", Value: "3", Type: SettingTypeInt, Min: "1", Max: "100", RequiresRestart: true, Category: "performance", Group: "Worker Pool", Description: "Number of background worker goroutines"},
{Key: "worker_queue_cap", Value: "100", Type: SettingTypeInt, Min: "1", Max: "10000", RequiresRestart: true, Category: "performance", Group: "Worker Pool", Description: "Background worker job queue capacity"},
{Key: "podcast_poll_interval_seconds", Value: "3600", Type: SettingTypeInt, Min: "300", Max: "604800", Category: "podcasts", Group: "Podcasts", Description: "How often to poll podcast feeds for new episodes (seconds; complete feeds back off)"},
{Key: "podcast_download_concurrency", Value: "4", Type: SettingTypeInt, Min: "1", Max: "16", Category: "podcasts", Group: "Podcasts", Description: "Podcast episode downloads in flight server-wide (the connection is the limit; fair-share spreads them across feeds)"},
{Key: "podcast_download_speed_mbps", Value: "0", Type: SettingTypeFloat, Min: "0", Max: "10000", Category: "podcasts", Group: "Podcasts", Description: "Shared bandwidth cap for podcast downloads in megabits/second (0 = unlimited)"},
}
// defaultBy indexes SettingDefaults by key for O(1) lookup.
@@ -173,6 +176,26 @@ func (r *SettingsRegistry) getInt(key string) int {
return n
}
func (r *SettingsRegistry) getFloat(key string) float64 {
d := defaultBy[key]
v := r.raw(key)
f, err := strconv.ParseFloat(v, 64)
if err != nil {
f, _ = strconv.ParseFloat(d.Value, 64)
}
if d.Min != "" {
if mn, err := strconv.ParseFloat(d.Min, 64); err == nil && f < mn {
f = mn
}
}
if d.Max != "" {
if mx, err := strconv.ParseFloat(d.Max, 64); err == nil && f > mx {
f = mx
}
}
return f
}
func (r *SettingsRegistry) getBool(key string) bool {
v := r.raw(key)
b, err := strconv.ParseBool(v)
@@ -189,13 +212,24 @@ func (r *SettingsRegistry) ScanPollInterval() time.Duration {
return time.Duration(r.getInt("scan_poll_interval_seconds")) * time.Second
}
// PodcastPollInterval is the feed-poller tick (live setting; the poller
// re-reads it every cycle — writes go through the settings API path,
// AGENTS.md gotcha #2).
// PodcastPollInterval is how often the poller checks feeds, as a
// duration (live setting; the poller re-reads it every cycle — writes
// go through the settings API path, AGENTS.md gotcha #2).
func (r *SettingsRegistry) PodcastPollInterval() time.Duration {
return time.Duration(r.getInt("podcast_poll_interval_seconds")) * time.Second
}
// PodcastDownloadConcurrency: episode downloads in flight server-wide
// (the connection is the limit; fair-share spreads them across feeds).
func (r *SettingsRegistry) PodcastDownloadConcurrency() int {
return r.getInt("podcast_download_concurrency")
}
// PodcastDownloadSpeedMbps: shared bandwidth cap (0 = unlimited).
func (r *SettingsRegistry) PodcastDownloadSpeedMbps() float64 {
return r.getFloat("podcast_download_speed_mbps")
}
// AutoScanEnabled reports whether auto-scanning is on.
func (r *SettingsRegistry) AutoScanEnabled() bool { return r.getBool("auto_scan_enabled") }