feat(scanner): extractor-version (ev) mechanism — parser fixes reach old rows without a force rescan (faceted plan Phase 10)
Parser fixes (7e4c97f6, genre model v2) never reached rows ingested
under an older parser: the sidecar never changed, so every fingerprint
said 'unchanged'. The manifest JSONB now carries the reserved 'ev'
stamp — the parser version that produced the item's row.
Semantics (all pinned by tests):
- ABSENT ev decodes as 1 ('v1-or-unknown'); stamping is truthful-only:
create paths + post-extraction stores stamp the setting's current
value; deploy-day backfill and the extract-failure guard stay UNSET
(a row never claims a version its data can't prove — a mid-backfill
bump self-heals next poll); repoint reconciliation preserves the
stored ev (stat-only refresh re-extracted nothing).
- NEW setting scan_extractor_version (seeded + registry + getter,
int default 1, category scanner): a manifest whose ev != the setting
re-extracts through the SAME change path (full extraction →
updateMediaItemWithMetadata → fresh stamped manifest). Sidecar-less
books get an entries-less manifest as their ev home (create: stamped;
first-sighting: unset).
- Catch-up bounded: evReextractBudgetPerScan=100, atomic (watcher pass
can overlap a poll scan), reset per ScanFolders — amortized +
resumable across polls; no force rescan, no flood.
- Multifile books included: the ev gate rides multifileSidecarChanged's
shared ladder; ev-driven manifestChanged drives the editorial apply
+ stamped re-store.
Settings API: the unified GET/PUT /api/system/settings now lists the
key (registry-driven, auto-surfaces in the admin UI). Test harness:
the registry is now Loaded + wired into the SystemSettingsHandler
(main.go parity) — settings writes through the API were 503s before.
Verified: unit (ev normalization, budget CAS, GetExtractorVersion
cache paths, registry clamping); integration state machine —
create-stamp ev=1 → equality skips (corruption probe persists) →
wiped scan-state backfills UNSET without re-extract → bump re-extracts
ONCE (sidecar value restored, ev=2) → stable (second corruption
persists); multifile twin green; adjacent suites (Audiobook, Calibre,
CrossLibraryMove, SchemaIdempotency) green; live app restart re-scans
1198 files with 0 changed (deploy-day rule), setting listed at
/api/system/settings.
This commit is contained in:
1 parent
43987262a6
commit
91b0c45dd4
10 files changed
+486
-10
No files matched your search
@@ -52,6 +52,26 @@ type manifestSidecar struct {
|
||||
type sidecarManifest struct {
|
||||
Version int `json:"v"`
|
||||
Entries []manifestSidecar `json:"entries"`
|
||||
// Ev is the extractor (parser) version that produced the item's
|
||||
// CURRENT row. Absent (0) means "v1-or-unknown": backfill and
|
||||
// extract-failure stores leave it unset so a row never claims a
|
||||
// version its data can't prove; every store that FOLLOWS a
|
||||
// successful extraction stamps the setting's current value
|
||||
// (scan_extractor_version). A stored ev below the setting re-runs
|
||||
// extraction on a later scan — parser fixes reach old rows without
|
||||
// a force rescan (faceted plan Phase 10).
|
||||
Ev int `json:"ev,omitempty"`
|
||||
}
|
||||
|
||||
// manifestEv normalizes a stored manifest's extractor version: absent
|
||||
// manifest or unset ev means version 1 (the shipped baseline parser).
|
||||
// Current-parser versions only ever grow, so "unknown" safely rides the
|
||||
// oldest live version.
|
||||
func manifestEv(m *sidecarManifest) int {
|
||||
if m == nil || m.Ev < 1 {
|
||||
return 1
|
||||
}
|
||||
return m.Ev
|
||||
}
|
||||
|
||||
// coverSidecarNames mirrors findSidecarCover's priority list — the
|
||||
@@ -179,7 +199,7 @@ func reconcileSidecarManifest(dir string, stored, computed *sidecarManifest) (fr
|
||||
if !drifted {
|
||||
return nil, false
|
||||
}
|
||||
fresh = &sidecarManifest{Version: sidecarManifestVersion}
|
||||
fresh = &sidecarManifest{Version: sidecarManifestVersion, Ev: stored.Ev} // provenance preserved: stat-only reconciliation never re-extracted anything
|
||||
for _, cur := range computed.Entries {
|
||||
old, ok := storedByName[cur.Name]
|
||||
if !ok {
|
||||
@@ -202,8 +222,30 @@ func reconcileSidecarManifest(dir string, stored, computed *sidecarManifest) (fr
|
||||
return fresh, false
|
||||
}
|
||||
|
||||
// evReextractBudgetPerScan bounds the parser-version catch-up: at most
|
||||
// this many ev-triggered re-extractions per scan pass. Each one stamps
|
||||
// the fresh manifest with the new ev, so progress is resumable — later
|
||||
// polls pick up where this one stopped, and no single scan floods the
|
||||
// disk with a full-library re-extraction (faceted plan Phase 10).
|
||||
const evReextractBudgetPerScan = 100
|
||||
|
||||
// storeManifest persists the manifest and write-throughs the map entry.
|
||||
// The stored manifest is STAMPED with the setting's current extractor
|
||||
// version: every caller here follows (or is) a full extraction under
|
||||
// the current parser, so the stamp is truthful. Backfill and
|
||||
// extract-failure stores must use storeManifestBaseline instead.
|
||||
func (s *MediaScanner) storeManifest(ctx context.Context, libraryID pgtype.UUID, entry *scanEntry, manifest *sidecarManifest) {
|
||||
manifest.Ev = s.GetExtractorVersion()
|
||||
s.storeManifestBaseline(ctx, libraryID, entry, manifest)
|
||||
}
|
||||
|
||||
// storeManifestBaseline persists a manifest WITHOUT stamping the
|
||||
// extractor version — for stores that follow NO successful extraction:
|
||||
// the deploy-day backfill (the row's provenance is unknown; leaving ev
|
||||
// unset keeps it eligible for a later version bump) and the
|
||||
// extract-failure guard (stamping would mark parser work as done that
|
||||
// never ran; leaving it stale retries on the next poll).
|
||||
func (s *MediaScanner) storeManifestBaseline(ctx context.Context, libraryID pgtype.UUID, entry *scanEntry, manifest *sidecarManifest) {
|
||||
b, err := json.Marshal(manifest)
|
||||
if err != nil {
|
||||
fmt.Printf("Warning: failed to encode sidecar manifest for %s: %v\n", entry.FilePath, err)
|
||||
@@ -220,6 +262,22 @@ func (s *MediaScanner) storeManifest(ctx context.Context, libraryID pgtype.UUID,
|
||||
s.scanStateUpsert(libraryID, entry)
|
||||
}
|
||||
|
||||
// evBudgetTake consumes one slot of the per-scan ev catch-up budget.
|
||||
// Atomic: the poll scan walks sequentially, but the watcher's
|
||||
// scanDirectory pass can overlap a poll scan (only scanDirectory holds
|
||||
// scanMutex), so the budget is shared state.
|
||||
func (s *MediaScanner) evBudgetTake() bool {
|
||||
for {
|
||||
cur := s.evBudgetRemaining.Load()
|
||||
if cur <= 0 {
|
||||
return false
|
||||
}
|
||||
if s.evBudgetRemaining.CompareAndSwap(cur, cur-1) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// detectSidecarChange fingerprints the book's sidecar directory and
|
||||
// reconciles against the stored manifest. Returns true when the item's
|
||||
// metadata must be re-extracted (a sidecar's CONTENT changed, or
|
||||
@@ -242,16 +300,36 @@ func (s *MediaScanner) detectSidecarChangeWithRepresentative(ctx context.Context
|
||||
computed := statSidecarManifest(representative, dir)
|
||||
stored := decodeSidecarManifest(entry.Manifest)
|
||||
|
||||
// Extractor-version gate (faceted plan Phase 10): a manifest stamped
|
||||
// by an older parser re-extracts even when every sidecar byte is
|
||||
// unchanged — the fingerprint says "disk same", ev says "parser
|
||||
// newer". Budget-bounded: over budget → fall through, the stale ev
|
||||
// stays stored, a later poll resumes. Checked FIRST so books with
|
||||
// no sidecars (nothing to reconcile) are still caught up.
|
||||
if stored != nil && manifestEv(stored) != s.GetExtractorVersion() && s.evBudgetTake() {
|
||||
return true
|
||||
}
|
||||
|
||||
if len(computed.Entries) == 0 {
|
||||
if stored != nil && len(stored.Entries) > 0 {
|
||||
return true // every sidecar was removed — fall back to embedded
|
||||
}
|
||||
return false // never had sidecars: nothing to detect or store
|
||||
if stored == nil {
|
||||
// Never had sidecars: nothing to fingerprint, but the book
|
||||
// still needs an ev home so future parser bumps see it. A
|
||||
// first-sighting store follows no extraction → baseline
|
||||
// (ev unset → eligible for the next bump).
|
||||
s.storeManifestBaseline(ctx, libraryID, entry, &sidecarManifest{Version: sidecarManifestVersion})
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
if stored == nil {
|
||||
// Deploy-day backfill: hash + store WITHOUT re-extraction.
|
||||
s.storeManifest(ctx, libraryID, entry, hashSidecarManifest(representative, dir, computed))
|
||||
// Deploy-day backfill: hash + store WITHOUT re-extraction. The
|
||||
// row was extracted by an unknown parser version — the stamp is
|
||||
// deliberately OMITTED (ev unset → "v1-or-unknown") so a later
|
||||
// scan_extractor_version bump still catches it.
|
||||
s.storeManifestBaseline(ctx, libraryID, entry, hashSidecarManifest(representative, dir, computed))
|
||||
return false
|
||||
}
|
||||
|
||||
@@ -260,7 +338,11 @@ func (s *MediaScanner) detectSidecarChangeWithRepresentative(ctx context.Context
|
||||
return true
|
||||
}
|
||||
if fresh != nil {
|
||||
s.storeManifest(ctx, libraryID, entry, fresh)
|
||||
// Repoint reconciliation refreshed stat pairs only — the row was
|
||||
// NOT re-extracted, so the manifest must keep the row's stored
|
||||
// ev (a stamp here would claim parser work that never ran;
|
||||
// possible when the ev budget was exhausted this pass).
|
||||
s.storeManifestBaseline(ctx, libraryID, entry, fresh)
|
||||
}
|
||||
return false
|
||||
}
|
||||
Reference in new issue
Block a user