feat(scanner): extractor-version (ev) mechanism — parser fixes reach old rows without a force rescan (faceted plan Phase 10)
Parser fixes (7e4c97f6, genre model v2) never reached rows ingested
under an older parser: the sidecar never changed, so every fingerprint
said 'unchanged'. The manifest JSONB now carries the reserved 'ev'
stamp — the parser version that produced the item's row.
Semantics (all pinned by tests):
- ABSENT ev decodes as 1 ('v1-or-unknown'); stamping is truthful-only:
create paths + post-extraction stores stamp the setting's current
value; deploy-day backfill and the extract-failure guard stay UNSET
(a row never claims a version its data can't prove — a mid-backfill
bump self-heals next poll); repoint reconciliation preserves the
stored ev (stat-only refresh re-extracted nothing).
- NEW setting scan_extractor_version (seeded + registry + getter,
int default 1, category scanner): a manifest whose ev != the setting
re-extracts through the SAME change path (full extraction →
updateMediaItemWithMetadata → fresh stamped manifest). Sidecar-less
books get an entries-less manifest as their ev home (create: stamped;
first-sighting: unset).
- Catch-up bounded: evReextractBudgetPerScan=100, atomic (watcher pass
can overlap a poll scan), reset per ScanFolders — amortized +
resumable across polls; no force rescan, no flood.
- Multifile books included: the ev gate rides multifileSidecarChanged's
shared ladder; ev-driven manifestChanged drives the editorial apply
+ stamped re-store.
Settings API: the unified GET/PUT /api/system/settings now lists the
key (registry-driven, auto-surfaces in the admin UI). Test harness:
the registry is now Loaded + wired into the SystemSettingsHandler
(main.go parity) — settings writes through the API were 503s before.
Verified: unit (ev normalization, budget CAS, GetExtractorVersion
cache paths, registry clamping); integration state machine —
create-stamp ev=1 → equality skips (corruption probe persists) →
wiped scan-state backfills UNSET without re-extract → bump re-extracts
ONCE (sidecar value restored, ev=2) → stable (second corruption
persists); multifile twin green; adjacent suites (Audiobook, Calibre,
CrossLibraryMove, SchemaIdempotency) green; live app restart re-scans
1198 files with 0 changed (deploy-day rule), setting listed at
/api/system/settings.
This commit is contained in:
1 parent
43987262a6
commit
91b0c45dd4
10 files changed
+486
-10
No files matched your search
@@ -148,6 +148,13 @@ type MediaScanner struct {
|
||||
watching atomic.Bool
|
||||
settingsCache *SettingsCache
|
||||
|
||||
// evBudgetRemaining is the per-scan extractor-version catch-up
|
||||
// budget (faceted plan Phase 10): how many ev-mismatch books this
|
||||
// pass may still re-extract. Reset at the top of ScanFolders;
|
||||
// atomic because the watcher's scanDirectory pass can overlap a
|
||||
// poll scan. Zero → ev-mismatch books wait for a later poll.
|
||||
evBudgetRemaining atomic.Int64
|
||||
|
||||
totalFiles int
|
||||
newItems int
|
||||
errors int
|
||||
@@ -277,6 +284,40 @@ func (s *MediaScanner) GetPollInterval() time.Duration {
|
||||
return time.Duration(seconds) * time.Second
|
||||
}
|
||||
|
||||
// GetExtractorVersion is the parser version stamped on each book's
|
||||
// scan manifest (faceted plan Phase 10). Read through the same 30s-TTL
|
||||
// settings cache as GetPollInterval — a bump via the SETTINGS API (the
|
||||
// only sanctioned write path) applies on the next read within TTL.
|
||||
// Version 1 is the shipped baseline parser; absent/garbage rows ride it.
|
||||
func (s *MediaScanner) GetExtractorVersion() int {
|
||||
if cached, ok := s.settingsCache.Get("scan_extractor_version"); ok {
|
||||
if v, err := strconv.Atoi(cached); err == nil && v >= 1 {
|
||||
return v
|
||||
}
|
||||
return 1 // cached garbage: the registry-clamped default
|
||||
}
|
||||
|
||||
if s.db == nil {
|
||||
return 1
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
|
||||
setting, err := s.db.GetSystemSetting(ctx, "scan_extractor_version")
|
||||
if err != nil || setting == "" {
|
||||
return 1
|
||||
}
|
||||
|
||||
s.settingsCache.Set("scan_extractor_version", setting)
|
||||
|
||||
v, err := strconv.Atoi(setting)
|
||||
if err != nil || v < 1 {
|
||||
return 1
|
||||
}
|
||||
return v
|
||||
}
|
||||
|
||||
func (s *MediaScanner) GetAutoScanEnabled() bool {
|
||||
// Check cache first
|
||||
if cached, ok := s.settingsCache.Get("auto_scan_enabled"); ok {
|
||||
@@ -641,6 +682,11 @@ func (s *MediaScanner) ScanFolders(ctx context.Context) error {
|
||||
s.newItems = 0
|
||||
s.errors = 0
|
||||
s.changedItems = 0
|
||||
// Fresh parser-version catch-up budget every scan pass (faceted
|
||||
// plan Phase 10): each pass re-extracts at most
|
||||
// evReextractBudgetPerScan stale-parser books; the rest resume on
|
||||
// later polls.
|
||||
s.evBudgetRemaining.Store(evReextractBudgetPerScan)
|
||||
s.audioBookRoots = make(map[string]bool)
|
||||
// Bulk scan state: per-run reset; one load per distinct library.
|
||||
s.scanStateMu.Lock()
|
||||
@@ -1382,10 +1428,15 @@ func (s *MediaScanner) processMediaFile(ctx context.Context, path string) (bool,
|
||||
|
||||
// Seed the sidecar manifest at create time (Part 2 §3): the next
|
||||
// scan compares instead of backfilling, so an edit is detected on
|
||||
// its first post-seed scan.
|
||||
// its first post-seed scan. Sidecar-less books get an entries-less
|
||||
// manifest carrying the current ev (Phase 10) — the book was JUST
|
||||
// extracted by this binary's parser, so the stamp is truthful and
|
||||
// no re-extraction can be skipped by later bumps.
|
||||
dir := bookSidecarDir(path)
|
||||
if computed := statSidecarManifest(path, dir); computed != nil && len(computed.Entries) > 0 {
|
||||
s.storeManifest(ctx, libraryID, newEntry, hashSidecarManifest(path, dir, computed))
|
||||
} else {
|
||||
s.storeManifest(ctx, libraryID, newEntry, &sidecarManifest{Version: sidecarManifestVersion})
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
@@ -1513,7 +1564,12 @@ func (s *MediaScanner) processExistingViaScanState(ctx context.Context, entry *s
|
||||
// reachable today (mergeMetadata never fails) — this is the
|
||||
// regression net for the class of bug the guard exists for.
|
||||
fmt.Printf("Warning: sidecar re-extraction failed for %s: %v — row left unchanged\n", path, extractErr)
|
||||
s.storeManifest(ctx, libraryID, entry, hashSidecarManifest(path, dir, statSidecarManifest(path, dir)))
|
||||
// Baseline store, never a stamp: a stamp would record the
|
||||
// current parser version on a row the current parser never
|
||||
// successfully processed — for an ev-triggered change that
|
||||
// would PERMANENTLY skip the fix. Leaving ev stale retries
|
||||
// on the next poll (budget-bounded).
|
||||
s.storeManifestBaseline(ctx, libraryID, entry, hashSidecarManifest(path, dir, statSidecarManifest(path, dir)))
|
||||
return false, nil
|
||||
}
|
||||
// Playlist entries ride the FIRST extraction either way — capture
|
||||
|
||||
Reference in new issue
Block a user