Replaces the sequential worker with a download-manager-style pool (§0 item 4). Model: the user's connection is the limit; semantics are fair-share ACROSS feeds, not goroutine-per-book (a global FIFO would make Book B wait behind Book A's 100 files). - Atomic claim: GetNextQueuedEpisode + SetPodcastEpisodeDownloading was a read-then-write race under concurrency; ClaimQueuedPodcastEpisode converts queued→downloading in ONE guarded statement (two claimants, one winner — the loser gets ErrNoRows). SetPodcastEpisodeDownloading retired. - Queue tickets: download_queue_pos BIGINT fed by a Postgres SEQUENCE stamped at enqueue — the enqueue loop's ordering IS the claim order (the foundation §0 item 5 builds on; previously all rows in a batch shared one updated_at, so within-feed order was arbitrary). - Pool: up to podcast_download_concurrency (default 4, live range 1–16) claim-loop goroutines per drain round; the setting re-reads every round so admin changes take effect without restart. - Fair share: claim prefers the feed least-recently-served (pickFeedToServe; never-served first), earliest ticket within a feed — books proceed in parallel while chapters land in order (start listening while the rest download). - Bandwidth: shared token bucket across all workers (podcast_download_speed_mbps, default 0 = unlimited); take() reserves bandwidth and callers sleep outside the lock. - Settings: new float setting type end to end (registry defaults + clamping, save validation, admin UI number-with-step input) + the two schema seeds; worker wires the registry via SetSettings (tests run on registered defaults). - Tests: unit coverage for the bucket reservation math + fair-share pick; integration TestPodcastDownloadPool pins ticket order = enqueue order, claim exclusivity, and a full 3-episode pool drain. Dev-DB note: the column/sequence/settings were applied live (idempotent DDL, same statements the schema runs at next boot) since the running container predates them.
407 lines
18 KiB
Go
407 lines
18 KiB
Go
package database
|
|
|
|
// SettingsRegistry provides a typed, cached view over the system_settings table.
|
|
// It is the single source of truth for tunable runtime values that used to be
|
|
// hardcoded as Go literals.
|
|
//
|
|
// Consumers call the domain-specific getters (SessionDuration, OpdsPageSize,
|
|
// etc.) which read from an in-memory cache. The cache is populated by Load at
|
|
// startup and refreshed by Reload whenever a setting is written. Getters always
|
|
// fall back to a compiled-in default if the DB value is missing or unparsable,
|
|
// so a corrupt or deleted row can never break the app.
|
|
//
|
|
// SettingsRegistry lives in the database package (rather than its own package)
|
|
// so that every consumer already imports database and does not need to take on
|
|
// a new package import.
|
|
|
|
import (
|
|
"context"
|
|
"log"
|
|
"strconv"
|
|
"sync"
|
|
"time"
|
|
)
|
|
|
|
// SettingType enumerates the value types stored in system_settings.setting_type.
|
|
const (
|
|
SettingTypeInt = "int"
|
|
SettingTypeFloat = "float"
|
|
SettingTypeBool = "bool"
|
|
SettingTypeString = "string"
|
|
SettingTypeStringList = "string_list"
|
|
)
|
|
|
|
// SecondsPerDay / SecondsPerHour are conversion helpers used by defaults.
|
|
const (
|
|
SecondsPerMinute = 60
|
|
SecondsPerHour = 3600
|
|
SecondsPerDay = 86400
|
|
)
|
|
|
|
// SettingDefault holds the fallback value for a key. These mirror the literals that
|
|
// were previously hardcoded in the source so an empty/corrupt DB row preserves
|
|
// prior behavior exactly.
|
|
type SettingDefault struct {
|
|
Key string
|
|
Value string
|
|
Type string
|
|
Min string
|
|
Max string
|
|
RequiresRestart bool
|
|
Category string
|
|
Group string
|
|
Description string
|
|
}
|
|
|
|
// SettingDefaults is the source of truth for fallback values and metadata. New keys
|
|
// must be added here AND seeded in database/schema/schema.sql. Entries are ordered
|
|
// by (RequiresRestart, Group) so the admin UI renders coherent sub-sections.
|
|
var SettingDefaults = []SettingDefault{
|
|
{Key: "scan_poll_interval_seconds", Value: "60", Type: SettingTypeInt, Min: "1", Max: "3600", Category: "scanner", Group: "Scanning", Description: "How often to scan all libraries (seconds)"},
|
|
{Key: "auto_scan_enabled", Value: "true", Type: SettingTypeBool, Category: "scanner", Group: "Scanning", Description: "Whether auto-scanning is enabled system-wide"},
|
|
{Key: "auto_playlist_collections", Value: "true", Type: SettingTypeBool, Category: "scanner", Group: "Scanning", Description: "Auto-create curated playlist collections from sidecar series marked as reading orders"},
|
|
{Key: "scan_extractor_version", Value: "1", Type: SettingTypeInt, Min: "1", Max: "1000000", Category: "scanner", Group: "Scanning", Description: "Parser version stamped on each book's scan manifest. Raising it re-extracts metadata from books scanned under an older parser, a bounded batch per scan until all are caught up"},
|
|
{Key: "default_timezone", Value: "UTC", Type: SettingTypeString, Category: "general", Group: "System Defaults", Description: "System default timezone"},
|
|
{Key: "session_duration_seconds", Value: "604800", Type: SettingTypeInt, Min: "300", Max: "31536000", Category: "security", Group: "Session", Description: "How long a login session stays valid"},
|
|
{Key: "password_min_length", Value: "8", Type: SettingTypeInt, Min: "1", Max: "128", Category: "security", Group: "Password Quality", Description: "Minimum password length"},
|
|
{Key: "password_require_upper", Value: "true", Type: SettingTypeBool, Category: "security", Group: "Password Quality", Description: "Require at least one uppercase letter (A-Z)"},
|
|
{Key: "password_require_lower", Value: "true", Type: SettingTypeBool, Category: "security", Group: "Password Quality", Description: "Require at least one lowercase letter (a-z)"},
|
|
{Key: "password_require_number", Value: "true", Type: SettingTypeBool, Category: "security", Group: "Password Quality", Description: "Require at least one number (0-9)"},
|
|
{Key: "password_require_special", Value: "true", Type: SettingTypeBool, Category: "security", Group: "Password Quality", Description: "Require at least one special character"},
|
|
{Key: "registration_enabled", Value: "true", Type: SettingTypeBool, Category: "security", Group: "Public Registration", Description: "Allow users to create their own accounts (admins can always create accounts)"},
|
|
{Key: "opds_default_page_size", Value: "50", Type: SettingTypeInt, Min: "1", Max: "500", Category: "api", Group: "OPDS Catalog", Description: "Default OPDS page size"},
|
|
{Key: "opds_max_page_size", Value: "200", Type: SettingTypeInt, Min: "1", Max: "1000", Category: "api", Group: "OPDS Catalog", Description: "Maximum OPDS page size"},
|
|
{Key: "device_rate_sync_per_min", Value: "60", Type: SettingTypeInt, Min: "1", Max: "10000", Category: "api", Group: "Device Rate Limits", Description: "Device sync requests per minute"},
|
|
{Key: "device_rate_progress_per_min", Value: "120", Type: SettingTypeInt, Min: "1", Max: "10000", Category: "api", Group: "Device Rate Limits", Description: "Device progress requests per minute"},
|
|
{Key: "device_rate_metadata_per_min", Value: "30", Type: SettingTypeInt, Min: "1", Max: "10000", Category: "api", Group: "Device Rate Limits", Description: "Device metadata requests per minute"},
|
|
{Key: "annotation_tombstone_ttl_days", Value: "30", Type: SettingTypeInt, Min: "1", Max: "3650", Category: "sync", Group: "Annotation Retention", Description: "How long deleted annotations are kept before purge"},
|
|
{Key: "conversion_cache_ttl_hours", Value: "24", Type: SettingTypeInt, Min: "1", Max: "720", Category: "performance", Group: "Conversion Cache", Description: "How long converted (kepub) files are cached"},
|
|
{Key: "auth_rate_limit_per_min", Value: "10", Type: SettingTypeInt, Min: "1", Max: "10000", RequiresRestart: true, Category: "security", Group: "Auth Rate Limiting", Description: "Global auth API rate limit (requests per minute)"},
|
|
{Key: "login_max_attempts", Value: "5", Type: SettingTypeInt, Min: "1", Max: "100", RequiresRestart: true, Category: "security", Group: "Login Lockout", Description: "Failed login attempts before lockout"},
|
|
{Key: "login_lockout_minutes", Value: "15", Type: SettingTypeInt, Min: "1", Max: "10080", RequiresRestart: true, Category: "security", Group: "Login Lockout", Description: "Lockout duration after too many failed logins"},
|
|
{Key: "sync_queue_interval_seconds", Value: "5", Type: SettingTypeInt, Min: "1", Max: "3600", RequiresRestart: true, Category: "sync", Group: "Sync Queue", Description: "How often the sync queue flushes"},
|
|
{Key: "sync_queue_batch_size", Value: "50", Type: SettingTypeInt, Min: "1", Max: "10000", RequiresRestart: true, Category: "sync", Group: "Sync Queue", Description: "Maximum items processed per sync queue flush"},
|
|
{Key: "worker_pool_size", Value: "3", Type: SettingTypeInt, Min: "1", Max: "100", RequiresRestart: true, Category: "performance", Group: "Worker Pool", Description: "Number of background worker goroutines"},
|
|
{Key: "worker_queue_cap", Value: "100", Type: SettingTypeInt, Min: "1", Max: "10000", RequiresRestart: true, Category: "performance", Group: "Worker Pool", Description: "Background worker job queue capacity"},
|
|
{Key: "podcast_poll_interval_seconds", Value: "3600", Type: SettingTypeInt, Min: "300", Max: "604800", Category: "podcasts", Group: "Podcasts", Description: "How often to poll podcast feeds for new episodes (seconds; complete feeds back off)"},
|
|
{Key: "podcast_download_concurrency", Value: "4", Type: SettingTypeInt, Min: "1", Max: "16", Category: "podcasts", Group: "Podcasts", Description: "Podcast episode downloads in flight server-wide (the connection is the limit; fair-share spreads them across feeds)"},
|
|
{Key: "podcast_download_speed_mbps", Value: "0", Type: SettingTypeFloat, Min: "0", Max: "10000", Category: "podcasts", Group: "Podcasts", Description: "Shared bandwidth cap for podcast downloads in megabits/second (0 = unlimited)"},
|
|
}
|
|
|
|
// defaultBy indexes SettingDefaults by key for O(1) lookup.
|
|
var defaultBy = func() map[string]SettingDefault {
|
|
m := make(map[string]SettingDefault, len(SettingDefaults))
|
|
for _, d := range SettingDefaults {
|
|
m[d.Key] = d
|
|
}
|
|
return m
|
|
}()
|
|
|
|
// Registry caches system_settings values in memory. The zero value is not
|
|
// usable; construct with New.
|
|
type SettingsRegistry struct {
|
|
q *Queries
|
|
|
|
mu sync.RWMutex
|
|
values map[string]string
|
|
loadedAt time.Time
|
|
}
|
|
|
|
// New returns a Registry backed by the given queries. The cache is empty
|
|
// until Load is called.
|
|
func NewSettingsRegistry(q *Queries) *SettingsRegistry {
|
|
return &SettingsRegistry{q: q, values: make(map[string]string)}
|
|
}
|
|
|
|
// Load populates the cache from the database. Missing rows fall back to the
|
|
// compiled defaults. Safe to call multiple times.
|
|
func (r *SettingsRegistry) Load(ctx context.Context) error {
|
|
rows, err := r.q.GetAllSystemSettings(ctx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
fresh := make(map[string]string, len(SettingDefaults))
|
|
for _, d := range SettingDefaults {
|
|
fresh[d.Key] = d.Value
|
|
}
|
|
for _, row := range rows {
|
|
if _, ok := fresh[row.SettingKey]; ok {
|
|
fresh[row.SettingKey] = row.SettingValue
|
|
}
|
|
}
|
|
r.mu.Lock()
|
|
r.values = fresh
|
|
r.loadedAt = time.Now()
|
|
r.mu.Unlock()
|
|
return nil
|
|
}
|
|
|
|
// Reload refreshes the cache from the database. Should be called after any
|
|
// setting write. On error the cache is left untouched and the error is logged.
|
|
func (r *SettingsRegistry) Reload(ctx context.Context) {
|
|
if err := r.Load(ctx); err != nil {
|
|
log.Printf("settings: reload failed: %v", err)
|
|
}
|
|
}
|
|
|
|
// raw returns the cached string value for a key (or the default), clamped to
|
|
// [min, max] for int-typed keys.
|
|
func (r *SettingsRegistry) raw(key string) string {
|
|
r.mu.RLock()
|
|
v, ok := r.values[key]
|
|
r.mu.RUnlock()
|
|
if !ok || v == "" {
|
|
v = defaultBy[key].Value
|
|
}
|
|
return v
|
|
}
|
|
|
|
func (r *SettingsRegistry) getInt(key string) int {
|
|
d := defaultBy[key]
|
|
v := r.raw(key)
|
|
n, err := strconv.Atoi(v)
|
|
if err != nil {
|
|
n, _ = strconv.Atoi(d.Value)
|
|
}
|
|
if d.Min != "" {
|
|
if mn, err := strconv.Atoi(d.Min); err == nil && n < mn {
|
|
n = mn
|
|
}
|
|
}
|
|
if d.Max != "" {
|
|
if mx, err := strconv.Atoi(d.Max); err == nil && n > mx {
|
|
n = mx
|
|
}
|
|
}
|
|
return n
|
|
}
|
|
|
|
func (r *SettingsRegistry) getFloat(key string) float64 {
|
|
d := defaultBy[key]
|
|
v := r.raw(key)
|
|
f, err := strconv.ParseFloat(v, 64)
|
|
if err != nil {
|
|
f, _ = strconv.ParseFloat(d.Value, 64)
|
|
}
|
|
if d.Min != "" {
|
|
if mn, err := strconv.ParseFloat(d.Min, 64); err == nil && f < mn {
|
|
f = mn
|
|
}
|
|
}
|
|
if d.Max != "" {
|
|
if mx, err := strconv.ParseFloat(d.Max, 64); err == nil && f > mx {
|
|
f = mx
|
|
}
|
|
}
|
|
return f
|
|
}
|
|
|
|
func (r *SettingsRegistry) getBool(key string) bool {
|
|
v := r.raw(key)
|
|
b, err := strconv.ParseBool(v)
|
|
if err != nil {
|
|
b, _ = strconv.ParseBool(defaultBy[key].Value)
|
|
}
|
|
return b
|
|
}
|
|
|
|
// ---- Domain-specific getters (call sites use these) ----
|
|
|
|
// ScanPollInterval is how often the scanner polls, as a duration.
|
|
func (r *SettingsRegistry) ScanPollInterval() time.Duration {
|
|
return time.Duration(r.getInt("scan_poll_interval_seconds")) * time.Second
|
|
}
|
|
|
|
// PodcastPollInterval is how often the poller checks feeds, as a
|
|
// duration (live setting; the poller re-reads it every cycle — writes
|
|
// go through the settings API path, AGENTS.md gotcha #2).
|
|
func (r *SettingsRegistry) PodcastPollInterval() time.Duration {
|
|
return time.Duration(r.getInt("podcast_poll_interval_seconds")) * time.Second
|
|
}
|
|
|
|
// PodcastDownloadConcurrency: episode downloads in flight server-wide
|
|
// (the connection is the limit; fair-share spreads them across feeds).
|
|
func (r *SettingsRegistry) PodcastDownloadConcurrency() int {
|
|
return r.getInt("podcast_download_concurrency")
|
|
}
|
|
|
|
// PodcastDownloadSpeedMbps: shared bandwidth cap (0 = unlimited).
|
|
func (r *SettingsRegistry) PodcastDownloadSpeedMbps() float64 {
|
|
return r.getFloat("podcast_download_speed_mbps")
|
|
}
|
|
|
|
// AutoScanEnabled reports whether auto-scanning is on.
|
|
func (r *SettingsRegistry) AutoScanEnabled() bool { return r.getBool("auto_scan_enabled") }
|
|
|
|
// AutoPlaylistCollections reports whether the scanner should turn
|
|
// obviously-playlist sidecar series ("The Witcher (Reading Order)") into
|
|
// curated static collections (A2c §2.4).
|
|
func (r *SettingsRegistry) AutoPlaylistCollections() bool {
|
|
return r.getBool("auto_playlist_collections")
|
|
}
|
|
|
|
// ScanExtractorVersion is the parser version stamped on each book's scan
|
|
// manifest. Books whose stored stamp differs re-extract their metadata on
|
|
// later scans (bounded per scan) — the mechanism that lets parser fixes
|
|
// reach rows ingested under an older parser without a force rescan.
|
|
func (r *SettingsRegistry) ScanExtractorVersion() int { return r.getInt("scan_extractor_version") }
|
|
|
|
// RegistrationEnabled reports whether public signup is allowed. Admin user
|
|
// creation is never affected — only self-service registration rides this
|
|
// flag; handlers additionally grant the first-user exception (see
|
|
// handlers/auth.go Register).
|
|
func (r *SettingsRegistry) RegistrationEnabled() bool { return r.getBool("registration_enabled") }
|
|
|
|
// DefaultTimezone returns the configured default timezone name.
|
|
func (r *SettingsRegistry) DefaultTimezone() string { return r.raw("default_timezone") }
|
|
|
|
// SessionDuration is how long a login session / refresh token stays valid.
|
|
func (r *SettingsRegistry) SessionDuration() time.Duration {
|
|
return time.Duration(r.getInt("session_duration_seconds")) * time.Second
|
|
}
|
|
|
|
// PasswordMinLength is the minimum password length.
|
|
func (r *SettingsRegistry) PasswordMinLength() int { return r.getInt("password_min_length") }
|
|
|
|
// PasswordRules bundles the active complexity requirements.
|
|
type PasswordRules struct {
|
|
MinLength int
|
|
Upper bool
|
|
Lower bool
|
|
Number bool
|
|
Special bool
|
|
}
|
|
|
|
// PasswordRules returns the active password complexity configuration.
|
|
func (r *SettingsRegistry) PasswordRules() PasswordRules {
|
|
return PasswordRules{
|
|
MinLength: r.PasswordMinLength(),
|
|
Upper: r.getBool("password_require_upper"),
|
|
Lower: r.getBool("password_require_lower"),
|
|
Number: r.getBool("password_require_number"),
|
|
Special: r.getBool("password_require_special"),
|
|
}
|
|
}
|
|
|
|
// AuthRateLimit is the global auth endpoint rate limit (requests/minute). Read
|
|
// once at startup.
|
|
func (r *SettingsRegistry) AuthRateLimit() int { return r.getInt("auth_rate_limit_per_min") }
|
|
|
|
// LoginLockout returns (max attempts, lockout duration). Read once at startup.
|
|
func (r *SettingsRegistry) LoginLockout() (int, time.Duration) {
|
|
return r.getInt("login_max_attempts"), time.Duration(r.getInt("login_lockout_minutes")) * time.Minute
|
|
}
|
|
|
|
// OpdsDefaultPageSize is the default OPDS items-per-page.
|
|
func (r *SettingsRegistry) OpdsDefaultPageSize() int { return r.getInt("opds_default_page_size") }
|
|
|
|
// OpdsMaxPageSize is the maximum items-per-page a client may request.
|
|
func (r *SettingsRegistry) OpdsMaxPageSize() int { return r.getInt("opds_max_page_size") }
|
|
|
|
// DeviceRateLimits bundles the per-route device rate limits (requests/minute).
|
|
type DeviceRateLimits struct {
|
|
Sync int
|
|
Progress int
|
|
Metadata int
|
|
}
|
|
|
|
// DeviceRateLimits returns the active device rate limits.
|
|
func (r *SettingsRegistry) DeviceRateLimits() DeviceRateLimits {
|
|
return DeviceRateLimits{
|
|
Sync: r.getInt("device_rate_sync_per_min"),
|
|
Progress: r.getInt("device_rate_progress_per_min"),
|
|
Metadata: r.getInt("device_rate_metadata_per_min"),
|
|
}
|
|
}
|
|
|
|
// TombstoneTTL is how long deleted annotations are retained before purge.
|
|
func (r *SettingsRegistry) TombstoneTTL() time.Duration {
|
|
return time.Duration(r.getInt("annotation_tombstone_ttl_days")) * 24 * time.Hour
|
|
}
|
|
|
|
// ConversionCacheTTL is how long converted (kepub) files are served from cache.
|
|
func (r *SettingsRegistry) ConversionCacheTTL() time.Duration {
|
|
return time.Duration(r.getInt("conversion_cache_ttl_hours")) * time.Hour
|
|
}
|
|
|
|
// SyncQueueConfig bundles the sync queue interval and batch size. Read at
|
|
// startup; changes require a restart.
|
|
type SyncQueueConfig struct {
|
|
Interval time.Duration
|
|
BatchSize int
|
|
}
|
|
|
|
// SyncQueueConfig returns the active sync queue configuration.
|
|
func (r *SettingsRegistry) SyncQueueConfig() SyncQueueConfig {
|
|
return SyncQueueConfig{
|
|
Interval: time.Duration(r.getInt("sync_queue_interval_seconds")) * time.Second,
|
|
BatchSize: r.getInt("sync_queue_batch_size"),
|
|
}
|
|
}
|
|
|
|
// WorkerPoolConfig bundles worker count and queue capacity. Read at startup;
|
|
// changes require a restart.
|
|
type WorkerPoolConfig struct {
|
|
Size int
|
|
QueueCap int
|
|
}
|
|
|
|
// WorkerPoolConfig returns the active worker pool configuration.
|
|
func (r *SettingsRegistry) WorkerPoolConfig() WorkerPoolConfig {
|
|
return WorkerPoolConfig{
|
|
Size: r.getInt("worker_pool_size"),
|
|
QueueCap: r.getInt("worker_queue_cap"),
|
|
}
|
|
}
|
|
|
|
// SettingEntry exposes one setting's metadata + current value, for the admin UI/API.
|
|
type SettingEntry struct {
|
|
Key string `json:"key"`
|
|
Value string `json:"value"`
|
|
Type string `json:"type"`
|
|
Min string `json:"min,omitempty"`
|
|
Max string `json:"max,omitempty"`
|
|
RequiresRestart bool `json:"requires_restart"`
|
|
Category string `json:"category"`
|
|
Group string `json:"group"`
|
|
Description string `json:"description"`
|
|
IsDefault bool `json:"is_default"`
|
|
}
|
|
|
|
// All returns metadata + current values for every known setting, grouped by
|
|
// the in-memory cache (which reflects the DB after Load/Reload).
|
|
func (r *SettingsRegistry) All() []SettingEntry {
|
|
r.mu.RLock()
|
|
vals := make(map[string]string, len(r.values))
|
|
for k, v := range r.values {
|
|
vals[k] = v
|
|
}
|
|
r.mu.RUnlock()
|
|
out := make([]SettingEntry, 0, len(SettingDefaults))
|
|
for _, d := range SettingDefaults {
|
|
v, ok := vals[d.Key]
|
|
if !ok {
|
|
v = d.Value
|
|
}
|
|
out = append(out, SettingEntry{
|
|
Key: d.Key,
|
|
Value: v,
|
|
Type: d.Type,
|
|
Min: d.Min,
|
|
Max: d.Max,
|
|
RequiresRestart: d.RequiresRestart,
|
|
Category: d.Category,
|
|
Group: d.Group,
|
|
Description: d.Description,
|
|
IsDefault: v == d.Value,
|
|
})
|
|
}
|
|
return out
|
|
}
|
|
|
|
// LookupDefault returns the compiled-in SettingDefault for a key (ok=false if unknown).
|
|
func LookupDefault(key string) (SettingDefault, bool) {
|
|
d, ok := defaultBy[key]
|
|
return d, ok
|
|
}
|