bookhoard/internal/services/media_scanner_manifest.go

package services

// Sidecar manifest fingerprints (faceted plan Part 2 §3): size-stable
// books are changed by EDITING their metadata sidecars, which the old
// size-only detection never noticed. One manifest per book (JSONB in
// media_scan_state) lists every sidecar the extractors consume, so the
// manifest and the extractors can never disagree — the manifest stats
// exactly the names extractCalibreSidecar, extractAudiobookshelfSidecar,
// and findSidecarCover look for.
//
// Change evaluation ladder, in cost order:
//  1. stat pairs all equal → unchanged, ZERO bytes read (steady state);
//  2. a stat pair differs → read + hash that one sidecar:
//     hash equal → refresh the stored stat pair (host repoint
//     reconciliation: bounded reads, zero re-extraction);
//     hash differs → the item changed → re-extract (caller);
//  3. stored manifest NULL → backfill only: hash + store, NO
//     re-extraction (deploy-day rule — the first scan after release
//     must not look like an mtime flood).
//
// mtime is a CACHE field only — it is never a change signal. Docker
// bind-mount repoints reset every mtime, which is exactly why the hash
// reconciliation ladder exists.

import (
	"bookhoard/internal/database"
	"context"
	"crypto/sha256"
	"encoding/hex"
	"encoding/json"
	"fmt"
	"os"
	"path/filepath"
	"strings"

	"github.com/jackc/pgx/v5/pgtype"
)

const sidecarManifestVersion = 1

type manifestSidecar struct {
	Name  string `json:"n"`
	Size  int64  `json:"s"`
	MTime int64  `json:"m"`
	// Sha is the content hash. Present for ALL entries — covers too:
	// the repoint reconciliation needs a baseline hash for any stat-
	// drifted sidecar, and a hashless cover would turn harmless mtime
	// drift into a re-extraction (the exact flood this design kills).
	Sha string `json:"h,omitempty"`
}

type sidecarManifest struct {
	Version int               `json:"v"`
	Entries []manifestSidecar `json:"entries"`
	// Ev is the extractor (parser) version that produced the item's
	// CURRENT row. Absent (0) means "v1-or-unknown": backfill and
	// extract-failure stores leave it unset so a row never claims a
	// version its data can't prove; every store that FOLLOWS a
	// successful extraction stamps the setting's current value
	// (scan_extractor_version). A stored ev below the setting re-runs
	// extraction on a later scan — parser fixes reach old rows without
	// a force rescan (faceted plan Phase 10).
	Ev int `json:"ev,omitempty"`
}

// manifestEv normalizes a stored manifest's extractor version: absent
// manifest or unset ev means version 1 (the shipped baseline parser).
// Current-parser versions only ever grow, so "unknown" safely rides the
// oldest live version.
func manifestEv(m *sidecarManifest) int {
	if m == nil || m.Ev < 1 {
		return 1
	}
	return m.Ev
}

// coverSidecarNames mirrors findSidecarCover's priority list — the
// manifest must stat the same names the extractor would consume.
var coverSidecarNames = []string{
	"cover.jpg", "cover.jpeg", "cover.png", "cover.webp",
	"folder.jpg", "folder.jpeg", "folder.png", "folder.webp",
	".folder.jpg", ".folder.jpeg", ".folder.png",
}

// bookSidecarDir is the directory whose sidecars feed a book: the book
// file's directory (multifile book roots resolve to themselves because
// file_path IS the directory).
func bookSidecarDir(path string) string {
	info, err := os.Stat(path)
	if err == nil && info.IsDir() {
		return path
	}
	return filepath.Dir(path)
}

// sidecarNamesForBook lists every sidecar filename the extractors
// consume for a book at path. The book basename covers the
// findSidecarCover "<basename>.<ext>" pattern.
func sidecarNamesForBook(path string) []string {
	names := make([]string, 0, len(coverSidecarNames)+5)
	names = append(names, "metadata.opf", "metadata.json")
	names = append(names, coverSidecarNames...)
	base := strings.TrimSuffix(filepath.Base(path), filepath.Ext(path))
	if base != "" && base != "." {
		for _, ext := range []string{".jpg", ".jpeg", ".png", ".webp"} {
			names = append(names, base+ext)
		}
	}
	return names
}

// statSidecarManifest stats every candidate sidecar (ReadDir-free: one
// lstat per candidate name — the walk already visited the directory).
// Returns nil when the directory itself is gone (deleted book mid-scan).
func statSidecarManifest(path, dir string) *sidecarManifest {
	manifest := &sidecarManifest{Version: sidecarManifestVersion}
	for _, name := range sidecarNamesForBook(path) {
		info, err := os.Stat(filepath.Join(dir, name))
		if err != nil || info.IsDir() {
			continue
		}
		manifest.Entries = append(manifest.Entries, manifestSidecar{
			Name:  name,
			Size:  info.Size(),
			MTime: info.ModTime().UnixNano(),
		})
	}
	return manifest
}

// hashSidecarManifest fills the content hashes of every entry — text
// sidecars (KBs) and covers (a one-time bounded read at backfill; the
// repoint ladder needs the baseline). Returns the same manifest for
// chaining.
func hashSidecarManifest(path, dir string, manifest *sidecarManifest) *sidecarManifest {
	for i := range manifest.Entries {
		entry := &manifest.Entries[i]
		sha, err := sha256File(filepath.Join(dir, entry.Name))
		if err != nil {
			continue // unreadable: stays size-only this pass
		}
		entry.Sha = sha
	}
	return manifest
}

func sha256File(path string) (string, error) {
	data, err := os.ReadFile(path)
	if err != nil {
		return "", err
	}
	sum := sha256.Sum256(data)
	return hex.EncodeToString(sum[:]), nil
}

// decodeSidecarManifest parses a stored manifest; nil when absent or
// corrupt (the caller then treats it as a backfill, self-healing any
// shape breakage).
func decodeSidecarManifest(raw []byte) *sidecarManifest {
	if len(raw) == 0 {
		return nil
	}
	var m sidecarManifest
	if err := json.Unmarshal(raw, &m); err != nil {
		return nil
	}
	return &m
}

// reconcileSidecarManifest runs the evaluation ladder against the
// freshly-statted manifest. Returns the manifest to store and whether
// the item's metadata must be re-extracted:
//   - stat pairs all equal → fresh=nil, changed=false (zero reads);
//   - stat drift + equal hash → fresh with the refreshed pair
//     (repoint reconciliation);
//   - hash differs, a sidecar appeared/vanished, or a differing text
//     sidecar has no stored hash → changed.
func reconcileSidecarManifest(dir string, stored, computed *sidecarManifest) (fresh *sidecarManifest, changed bool) {
	if stored == nil {
		return nil, true
	}
	if len(stored.Entries) != len(computed.Entries) {
		return nil, true // sidecar appeared or vanished
	}
	storedByName := make(map[string]manifestSidecar, len(stored.Entries))
	for _, e := range stored.Entries {
		storedByName[e.Name] = e
	}
	// Fast path: stat pairs all equal → unchanged, zero bytes read,
	// nothing to store.
	drifted := false
	for _, cur := range computed.Entries {
		old, ok := storedByName[cur.Name]
		if !ok || cur.Size != old.Size || cur.MTime != old.MTime {
			drifted = true
			break
		}
	}
	if !drifted {
		return nil, false
	}
	fresh = &sidecarManifest{Version: sidecarManifestVersion, Ev: stored.Ev} // provenance preserved: stat-only reconciliation never re-extracted anything
	for _, cur := range computed.Entries {
		old, ok := storedByName[cur.Name]
		if !ok {
			return nil, true
		}
		if cur.Size == old.Size && cur.MTime == old.MTime {
			fresh.Entries = append(fresh.Entries, old) // untouched: keep the stored hash
			continue
		}
		if old.Sha == "" {
			return nil, true // unhashable drift: fall back to re-extract
		}
		sha, err := sha256File(filepath.Join(dir, cur.Name))
		if err != nil || sha != old.Sha {
			return nil, true
		}
		cur.Sha = old.Sha // repoint: content identical, stat pair refreshed
		fresh.Entries = append(fresh.Entries, cur)
	}
	return fresh, false
}

// evReextractBudgetPerScan bounds the parser-version catch-up: at most
// this many ev-triggered re-extractions per scan pass. Each one stamps
// the fresh manifest with the new ev, so progress is resumable — later
// polls pick up where this one stopped, and no single scan floods the
// disk with a full-library re-extraction (faceted plan Phase 10).
const evReextractBudgetPerScan = 100

// storeManifest persists the manifest and write-throughs the map entry.
// The stored manifest is STAMPED with the setting's current extractor
// version: every caller here follows (or is) a full extraction under
// the current parser, so the stamp is truthful. Backfill and
// extract-failure stores must use storeManifestBaseline instead.
func (s *MediaScanner) storeManifest(ctx context.Context, libraryID pgtype.UUID, entry *scanEntry, manifest *sidecarManifest) {
	manifest.Ev = s.GetExtractorVersion()
	s.storeManifestBaseline(ctx, libraryID, entry, manifest)
}

// storeManifestBaseline persists a manifest WITHOUT stamping the
// extractor version — for stores that follow NO successful extraction:
// the deploy-day backfill (the row's provenance is unknown; leaving ev
// unset keeps it eligible for a later version bump) and the
// extract-failure guard (stamping would mark parser work as done that
// never ran; leaving it stale retries on the next poll).
func (s *MediaScanner) storeManifestBaseline(ctx context.Context, libraryID pgtype.UUID, entry *scanEntry, manifest *sidecarManifest) {
	b, err := json.Marshal(manifest)
	if err != nil {
		fmt.Printf("Warning: failed to encode sidecar manifest for %s: %v\n", entry.FilePath, err)
		return
	}
	if err := s.db.UpsertMediaScanState(ctx, database.UpsertMediaScanStateParams{
		MediaItemID: entry.ID,
		Manifest:    b,
	}); err != nil {
		fmt.Printf("Warning: failed to store sidecar manifest for %s: %v\n", entry.FilePath, err)
		return
	}
	entry.Manifest = b
	s.scanStateUpsert(libraryID, entry)
}

// evBudgetTake consumes one slot of the per-scan ev catch-up budget.
// Atomic: the poll scan walks sequentially, but the watcher's
// scanDirectory pass can overlap a poll scan (only scanDirectory holds
// scanMutex), so the budget is shared state.
func (s *MediaScanner) evBudgetTake() bool {
	for {
		cur := s.evBudgetRemaining.Load()
		if cur <= 0 {
			return false
		}
		if s.evBudgetRemaining.CompareAndSwap(cur, cur-1) {
			return true
		}
	}
}

// detectSidecarChange fingerprints the book's sidecar directory and
// reconciles against the stored manifest. Returns true when the item's
// metadata must be re-extracted (a sidecar's CONTENT changed, or
// sidecars appeared/vanished). Backfill and repoint reconciliation are
// handled here without re-extraction; the caller stores a fresh full
// manifest after a change-path update.
func (s *MediaScanner) detectSidecarChange(ctx context.Context, entry *scanEntry, libraryID pgtype.UUID, path string) bool {
	dir := bookSidecarDir(path)
	return s.detectSidecarChangeWithRepresentative(ctx, entry, libraryID, path, dir)
}

// detectSidecarChangeWithRepresentative is detectSidecarChange with the
// sidecar-name derivation decoupled from the map key. Multi-file books
// (faceted plan Phase 6) are keyed by the item's file_path (the folder
// root / zip), but the extractor reads a folder book's sidecars through
// the sentinel INSIDE the root — whose basename yields no "<base>.<ext>"
// cover candidates — so the manifest must stat the representative's name
// list to never disagree with what extraction consumes.
func (s *MediaScanner) detectSidecarChangeWithRepresentative(ctx context.Context, entry *scanEntry, libraryID pgtype.UUID, representative, dir string) bool {
	computed := statSidecarManifest(representative, dir)
	stored := decodeSidecarManifest(entry.Manifest)

	// Extractor-version gate (faceted plan Phase 10): a manifest stamped
	// by an older parser re-extracts even when every sidecar byte is
	// unchanged — the fingerprint says "disk same", ev says "parser
	// newer". Budget-bounded: over budget → fall through, the stale ev
	// stays stored, a later poll resumes. Checked FIRST so books with
	// no sidecars (nothing to reconcile) are still caught up.
	if stored != nil && manifestEv(stored) != s.GetExtractorVersion() && s.evBudgetTake() {
		return true
	}

	if len(computed.Entries) == 0 {
		if stored != nil && len(stored.Entries) > 0 {
			return true // every sidecar was removed — fall back to embedded
		}
		if stored == nil {
			// Never had sidecars: nothing to fingerprint, but the book
			// still needs an ev home so future parser bumps see it. A
			// first-sighting store follows no extraction → baseline
			// (ev unset → eligible for the next bump).
			s.storeManifestBaseline(ctx, libraryID, entry, &sidecarManifest{Version: sidecarManifestVersion})
		}
		return false
	}

	if stored == nil {
		// Deploy-day backfill: hash + store WITHOUT re-extraction. The
		// row was extracted by an unknown parser version — the stamp is
		// deliberately OMITTED (ev unset → "v1-or-unknown") so a later
		// scan_extractor_version bump still catches it.
		s.storeManifestBaseline(ctx, libraryID, entry, hashSidecarManifest(representative, dir, computed))
		return false
	}

	fresh, changed := reconcileSidecarManifest(dir, stored, computed)
	if changed {
		return true
	}
	if fresh != nil {
		// Repoint reconciliation refreshed stat pairs only — the row was
		// NOT re-extracted, so the manifest must keep the row's stored
		// ev (a stamp here would claim parser work that never ran;
		// possible when the ev budget was exhausted this pass).
		s.storeManifestBaseline(ctx, libraryID, entry, fresh)
	}
	return false
}