feat(podcasts): concurrent download pool — fair share, tickets, admin tunables
Replaces the sequential worker with a download-manager-style pool (§0 item 4). Model: the user's connection is the limit; semantics are fair-share ACROSS feeds, not goroutine-per-book (a global FIFO would make Book B wait behind Book A's 100 files). - Atomic claim: GetNextQueuedEpisode + SetPodcastEpisodeDownloading was a read-then-write race under concurrency; ClaimQueuedPodcastEpisode converts queued→downloading in ONE guarded statement (two claimants, one winner — the loser gets ErrNoRows). SetPodcastEpisodeDownloading retired. - Queue tickets: download_queue_pos BIGINT fed by a Postgres SEQUENCE stamped at enqueue — the enqueue loop's ordering IS the claim order (the foundation §0 item 5 builds on; previously all rows in a batch shared one updated_at, so within-feed order was arbitrary). - Pool: up to podcast_download_concurrency (default 4, live range 1–16) claim-loop goroutines per drain round; the setting re-reads every round so admin changes take effect without restart. - Fair share: claim prefers the feed least-recently-served (pickFeedToServe; never-served first), earliest ticket within a feed — books proceed in parallel while chapters land in order (start listening while the rest download). - Bandwidth: shared token bucket across all workers (podcast_download_speed_mbps, default 0 = unlimited); take() reserves bandwidth and callers sleep outside the lock. - Settings: new float setting type end to end (registry defaults + clamping, save validation, admin UI number-with-step input) + the two schema seeds; worker wires the registry via SetSettings (tests run on registered defaults). - Tests: unit coverage for the bucket reservation math + fair-share pick; integration TestPodcastDownloadPool pins ticket order = enqueue order, claim exclusivity, and a full 3-episode pool drain. Dev-DB note: the column/sequence/settings were applied live (idempotent DDL, same statements the schema runs at next boot) since the running container predates them.
This commit is contained in:
1 parent
e3675536aa
commit
0f3c4a8cec
14 files changed
+893
-264
No files matched your search
@@ -25,6 +25,7 @@ import (
|
||||
// SettingType enumerates the value types stored in system_settings.setting_type.
|
||||
const (
|
||||
SettingTypeInt = "int"
|
||||
SettingTypeFloat = "float"
|
||||
SettingTypeBool = "bool"
|
||||
SettingTypeString = "string"
|
||||
SettingTypeStringList = "string_list"
|
||||
@@ -83,6 +84,8 @@ var SettingDefaults = []SettingDefault{
|
||||
{Key: "worker_pool_size", Value: "3", Type: SettingTypeInt, Min: "1", Max: "100", RequiresRestart: true, Category: "performance", Group: "Worker Pool", Description: "Number of background worker goroutines"},
|
||||
{Key: "worker_queue_cap", Value: "100", Type: SettingTypeInt, Min: "1", Max: "10000", RequiresRestart: true, Category: "performance", Group: "Worker Pool", Description: "Background worker job queue capacity"},
|
||||
{Key: "podcast_poll_interval_seconds", Value: "3600", Type: SettingTypeInt, Min: "300", Max: "604800", Category: "podcasts", Group: "Podcasts", Description: "How often to poll podcast feeds for new episodes (seconds; complete feeds back off)"},
|
||||
{Key: "podcast_download_concurrency", Value: "4", Type: SettingTypeInt, Min: "1", Max: "16", Category: "podcasts", Group: "Podcasts", Description: "Podcast episode downloads in flight server-wide (the connection is the limit; fair-share spreads them across feeds)"},
|
||||
{Key: "podcast_download_speed_mbps", Value: "0", Type: SettingTypeFloat, Min: "0", Max: "10000", Category: "podcasts", Group: "Podcasts", Description: "Shared bandwidth cap for podcast downloads in megabits/second (0 = unlimited)"},
|
||||
}
|
||||
|
||||
// defaultBy indexes SettingDefaults by key for O(1) lookup.
|
||||
@@ -173,6 +176,26 @@ func (r *SettingsRegistry) getInt(key string) int {
|
||||
return n
|
||||
}
|
||||
|
||||
func (r *SettingsRegistry) getFloat(key string) float64 {
|
||||
d := defaultBy[key]
|
||||
v := r.raw(key)
|
||||
f, err := strconv.ParseFloat(v, 64)
|
||||
if err != nil {
|
||||
f, _ = strconv.ParseFloat(d.Value, 64)
|
||||
}
|
||||
if d.Min != "" {
|
||||
if mn, err := strconv.ParseFloat(d.Min, 64); err == nil && f < mn {
|
||||
f = mn
|
||||
}
|
||||
}
|
||||
if d.Max != "" {
|
||||
if mx, err := strconv.ParseFloat(d.Max, 64); err == nil && f > mx {
|
||||
f = mx
|
||||
}
|
||||
}
|
||||
return f
|
||||
}
|
||||
|
||||
func (r *SettingsRegistry) getBool(key string) bool {
|
||||
v := r.raw(key)
|
||||
b, err := strconv.ParseBool(v)
|
||||
@@ -189,13 +212,24 @@ func (r *SettingsRegistry) ScanPollInterval() time.Duration {
|
||||
return time.Duration(r.getInt("scan_poll_interval_seconds")) * time.Second
|
||||
}
|
||||
|
||||
// PodcastPollInterval is the feed-poller tick (live setting; the poller
|
||||
// re-reads it every cycle — writes go through the settings API path,
|
||||
// AGENTS.md gotcha #2).
|
||||
// PodcastPollInterval is how often the poller checks feeds, as a
|
||||
// duration (live setting; the poller re-reads it every cycle — writes
|
||||
// go through the settings API path, AGENTS.md gotcha #2).
|
||||
func (r *SettingsRegistry) PodcastPollInterval() time.Duration {
|
||||
return time.Duration(r.getInt("podcast_poll_interval_seconds")) * time.Second
|
||||
}
|
||||
|
||||
// PodcastDownloadConcurrency: episode downloads in flight server-wide
|
||||
// (the connection is the limit; fair-share spreads them across feeds).
|
||||
func (r *SettingsRegistry) PodcastDownloadConcurrency() int {
|
||||
return r.getInt("podcast_download_concurrency")
|
||||
}
|
||||
|
||||
// PodcastDownloadSpeedMbps: shared bandwidth cap (0 = unlimited).
|
||||
func (r *SettingsRegistry) PodcastDownloadSpeedMbps() float64 {
|
||||
return r.getFloat("podcast_download_speed_mbps")
|
||||
}
|
||||
|
||||
// AutoScanEnabled reports whether auto-scanning is on.
|
||||
func (r *SettingsRegistry) AutoScanEnabled() bool { return r.getBool("auto_scan_enabled") }
|
||||
|
||||
|
||||
Reference in new issue
Block a user