package services // Sidecar manifest fingerprints (faceted plan Part 2 §3): size-stable // books are changed by EDITING their metadata sidecars, which the old // size-only detection never noticed. One manifest per book (JSONB in // media_scan_state) lists every sidecar the extractors consume, so the // manifest and the extractors can never disagree — the manifest stats // exactly the names extractCalibreSidecar, extractAudiobookshelfSidecar, // and findSidecarCover look for. // // Change evaluation ladder, in cost order: // 1. stat pairs all equal → unchanged, ZERO bytes read (steady state); // 2. a stat pair differs → read + hash that one sidecar: // hash equal → refresh the stored stat pair (host repoint // reconciliation: bounded reads, zero re-extraction); // hash differs → the item changed → re-extract (caller); // 3. stored manifest NULL → backfill only: hash + store, NO // re-extraction (deploy-day rule — the first scan after release // must not look like an mtime flood). // // mtime is a CACHE field only — it is never a change signal. Docker // bind-mount repoints reset every mtime, which is exactly why the hash // reconciliation ladder exists. import ( "bookhoard/internal/database" "context" "crypto/sha256" "encoding/hex" "encoding/json" "fmt" "os" "path/filepath" "strings" "github.com/jackc/pgx/v5/pgtype" ) const sidecarManifestVersion = 1 type manifestSidecar struct { Name string `json:"n"` Size int64 `json:"s"` MTime int64 `json:"m"` // Sha is the content hash. Present for ALL entries — covers too: // the repoint reconciliation needs a baseline hash for any stat- // drifted sidecar, and a hashless cover would turn harmless mtime // drift into a re-extraction (the exact flood this design kills). Sha string `json:"h,omitempty"` } type sidecarManifest struct { Version int `json:"v"` Entries []manifestSidecar `json:"entries"` // Ev is the extractor (parser) version that produced the item's // CURRENT row. Absent (0) means "v1-or-unknown": backfill and // extract-failure stores leave it unset so a row never claims a // version its data can't prove; every store that FOLLOWS a // successful extraction stamps the setting's current value // (scan_extractor_version). A stored ev below the setting re-runs // extraction on a later scan — parser fixes reach old rows without // a force rescan (faceted plan Phase 10). Ev int `json:"ev,omitempty"` } // manifestEv normalizes a stored manifest's extractor version: absent // manifest or unset ev means version 1 (the shipped baseline parser). // Current-parser versions only ever grow, so "unknown" safely rides the // oldest live version. func manifestEv(m *sidecarManifest) int { if m == nil || m.Ev < 1 { return 1 } return m.Ev } // coverSidecarNames mirrors findSidecarCover's priority list — the // manifest must stat the same names the extractor would consume. var coverSidecarNames = []string{ "cover.jpg", "cover.jpeg", "cover.png", "cover.webp", "folder.jpg", "folder.jpeg", "folder.png", "folder.webp", ".folder.jpg", ".folder.jpeg", ".folder.png", } // bookSidecarDir is the directory whose sidecars feed a book: the book // file's directory (multifile book roots resolve to themselves because // file_path IS the directory). func bookSidecarDir(path string) string { info, err := os.Stat(path) if err == nil && info.IsDir() { return path } return filepath.Dir(path) } // sidecarNamesForBook lists every sidecar filename the extractors // consume for a book at path. The book basename covers the // findSidecarCover "." pattern. func sidecarNamesForBook(path string) []string { names := make([]string, 0, len(coverSidecarNames)+5) names = append(names, "metadata.opf", "metadata.json") names = append(names, coverSidecarNames...) base := strings.TrimSuffix(filepath.Base(path), filepath.Ext(path)) if base != "" && base != "." { for _, ext := range []string{".jpg", ".jpeg", ".png", ".webp"} { names = append(names, base+ext) } } return names } // statSidecarManifest stats every candidate sidecar (ReadDir-free: one // lstat per candidate name — the walk already visited the directory). // Returns nil when the directory itself is gone (deleted book mid-scan). func statSidecarManifest(path, dir string) *sidecarManifest { manifest := &sidecarManifest{Version: sidecarManifestVersion} for _, name := range sidecarNamesForBook(path) { info, err := os.Stat(filepath.Join(dir, name)) if err != nil || info.IsDir() { continue } manifest.Entries = append(manifest.Entries, manifestSidecar{ Name: name, Size: info.Size(), MTime: info.ModTime().UnixNano(), }) } return manifest } // hashSidecarManifest fills the content hashes of every entry — text // sidecars (KBs) and covers (a one-time bounded read at backfill; the // repoint ladder needs the baseline). Returns the same manifest for // chaining. func hashSidecarManifest(path, dir string, manifest *sidecarManifest) *sidecarManifest { for i := range manifest.Entries { entry := &manifest.Entries[i] sha, err := sha256File(filepath.Join(dir, entry.Name)) if err != nil { continue // unreadable: stays size-only this pass } entry.Sha = sha } return manifest } func sha256File(path string) (string, error) { data, err := os.ReadFile(path) if err != nil { return "", err } sum := sha256.Sum256(data) return hex.EncodeToString(sum[:]), nil } // decodeSidecarManifest parses a stored manifest; nil when absent or // corrupt (the caller then treats it as a backfill, self-healing any // shape breakage). func decodeSidecarManifest(raw []byte) *sidecarManifest { if len(raw) == 0 { return nil } var m sidecarManifest if err := json.Unmarshal(raw, &m); err != nil { return nil } return &m } // reconcileSidecarManifest runs the evaluation ladder against the // freshly-statted manifest. Returns the manifest to store and whether // the item's metadata must be re-extracted: // - stat pairs all equal → fresh=nil, changed=false (zero reads); // - stat drift + equal hash → fresh with the refreshed pair // (repoint reconciliation); // - hash differs, a sidecar appeared/vanished, or a differing text // sidecar has no stored hash → changed. func reconcileSidecarManifest(dir string, stored, computed *sidecarManifest) (fresh *sidecarManifest, changed bool) { if stored == nil { return nil, true } if len(stored.Entries) != len(computed.Entries) { return nil, true // sidecar appeared or vanished } storedByName := make(map[string]manifestSidecar, len(stored.Entries)) for _, e := range stored.Entries { storedByName[e.Name] = e } // Fast path: stat pairs all equal → unchanged, zero bytes read, // nothing to store. drifted := false for _, cur := range computed.Entries { old, ok := storedByName[cur.Name] if !ok || cur.Size != old.Size || cur.MTime != old.MTime { drifted = true break } } if !drifted { return nil, false } fresh = &sidecarManifest{Version: sidecarManifestVersion, Ev: stored.Ev} // provenance preserved: stat-only reconciliation never re-extracted anything for _, cur := range computed.Entries { old, ok := storedByName[cur.Name] if !ok { return nil, true } if cur.Size == old.Size && cur.MTime == old.MTime { fresh.Entries = append(fresh.Entries, old) // untouched: keep the stored hash continue } if old.Sha == "" { return nil, true // unhashable drift: fall back to re-extract } sha, err := sha256File(filepath.Join(dir, cur.Name)) if err != nil || sha != old.Sha { return nil, true } cur.Sha = old.Sha // repoint: content identical, stat pair refreshed fresh.Entries = append(fresh.Entries, cur) } return fresh, false } // evReextractBudgetPerScan bounds the parser-version catch-up: at most // this many ev-triggered re-extractions per scan pass. Each one stamps // the fresh manifest with the new ev, so progress is resumable — later // polls pick up where this one stopped, and no single scan floods the // disk with a full-library re-extraction (faceted plan Phase 10). const evReextractBudgetPerScan = 100 // storeManifest persists the manifest and write-throughs the map entry. // The stored manifest is STAMPED with the setting's current extractor // version: every caller here follows (or is) a full extraction under // the current parser, so the stamp is truthful. Backfill and // extract-failure stores must use storeManifestBaseline instead. func (s *MediaScanner) storeManifest(ctx context.Context, libraryID pgtype.UUID, entry *scanEntry, manifest *sidecarManifest) { manifest.Ev = s.GetExtractorVersion() s.storeManifestBaseline(ctx, libraryID, entry, manifest) } // storeManifestBaseline persists a manifest WITHOUT stamping the // extractor version — for stores that follow NO successful extraction: // the deploy-day backfill (the row's provenance is unknown; leaving ev // unset keeps it eligible for a later version bump) and the // extract-failure guard (stamping would mark parser work as done that // never ran; leaving it stale retries on the next poll). func (s *MediaScanner) storeManifestBaseline(ctx context.Context, libraryID pgtype.UUID, entry *scanEntry, manifest *sidecarManifest) { b, err := json.Marshal(manifest) if err != nil { fmt.Printf("Warning: failed to encode sidecar manifest for %s: %v\n", entry.FilePath, err) return } if err := s.db.UpsertMediaScanState(ctx, database.UpsertMediaScanStateParams{ MediaItemID: entry.ID, Manifest: b, }); err != nil { fmt.Printf("Warning: failed to store sidecar manifest for %s: %v\n", entry.FilePath, err) return } entry.Manifest = b s.scanStateUpsert(libraryID, entry) } // evBudgetTake consumes one slot of the per-scan ev catch-up budget. // Atomic: the poll scan walks sequentially, but the watcher's // scanDirectory pass can overlap a poll scan (only scanDirectory holds // scanMutex), so the budget is shared state. func (s *MediaScanner) evBudgetTake() bool { for { cur := s.evBudgetRemaining.Load() if cur <= 0 { return false } if s.evBudgetRemaining.CompareAndSwap(cur, cur-1) { return true } } } // detectSidecarChange fingerprints the book's sidecar directory and // reconciles against the stored manifest. Returns true when the item's // metadata must be re-extracted (a sidecar's CONTENT changed, or // sidecars appeared/vanished). Backfill and repoint reconciliation are // handled here without re-extraction; the caller stores a fresh full // manifest after a change-path update. func (s *MediaScanner) detectSidecarChange(ctx context.Context, entry *scanEntry, libraryID pgtype.UUID, path string) bool { dir := bookSidecarDir(path) return s.detectSidecarChangeWithRepresentative(ctx, entry, libraryID, path, dir) } // detectSidecarChangeWithRepresentative is detectSidecarChange with the // sidecar-name derivation decoupled from the map key. Multi-file books // (faceted plan Phase 6) are keyed by the item's file_path (the folder // root / zip), but the extractor reads a folder book's sidecars through // the sentinel INSIDE the root — whose basename yields no "." // cover candidates — so the manifest must stat the representative's name // list to never disagree with what extraction consumes. func (s *MediaScanner) detectSidecarChangeWithRepresentative(ctx context.Context, entry *scanEntry, libraryID pgtype.UUID, representative, dir string) bool { computed := statSidecarManifest(representative, dir) stored := decodeSidecarManifest(entry.Manifest) // Extractor-version gate (faceted plan Phase 10): a manifest stamped // by an older parser re-extracts even when every sidecar byte is // unchanged — the fingerprint says "disk same", ev says "parser // newer". Budget-bounded: over budget → fall through, the stale ev // stays stored, a later poll resumes. Checked FIRST so books with // no sidecars (nothing to reconcile) are still caught up. if stored != nil && manifestEv(stored) != s.GetExtractorVersion() && s.evBudgetTake() { return true } if len(computed.Entries) == 0 { if stored != nil && len(stored.Entries) > 0 { return true // every sidecar was removed — fall back to embedded } if stored == nil { // Never had sidecars: nothing to fingerprint, but the book // still needs an ev home so future parser bumps see it. A // first-sighting store follows no extraction → baseline // (ev unset → eligible for the next bump). s.storeManifestBaseline(ctx, libraryID, entry, &sidecarManifest{Version: sidecarManifestVersion}) } return false } if stored == nil { // Deploy-day backfill: hash + store WITHOUT re-extraction. The // row was extracted by an unknown parser version — the stamp is // deliberately OMITTED (ev unset → "v1-or-unknown") so a later // scan_extractor_version bump still catches it. s.storeManifestBaseline(ctx, libraryID, entry, hashSidecarManifest(representative, dir, computed)) return false } fresh, changed := reconcileSidecarManifest(dir, stored, computed) if changed { return true } if fresh != nil { // Repoint reconciliation refreshed stat pairs only — the row was // NOT re-extracted, so the manifest must keep the row's stored // ev (a stamp here would claim parser work that never ran; // possible when the ev budget was exhausted this pass). s.storeManifestBaseline(ctx, libraryID, entry, fresh) } return false }