From 6750e9e0dc1ea0899c7828ab1c810c71b64b4735 Mon Sep 17 00:00:00 2001 From: John O'Keefe Date: Mon, 19 Oct 2026 20:17:57 -0400 Subject: [PATCH] feat(scanner): manifest-gated sidecar change detection + playlist sync on change path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Part 2 of the faceted plan, Phase 4 (steps 3-4). Size-only detection was blind to metadata-sidecar edits; a size-stable book is changed by EDITING its metadata.opf/metadata.json/cover — previously invisible until a force rescan. - Sidecar manifest fingerprints (media_scanner_manifest.go): the manifest stats exactly the names the extractors consume (metadata.opf, metadata.json, findSidecarCover patterns incl. .), so manifest and extractors can never disagree. Shape v1: {v, entries:[{n, s, m (unix nanos — second granularity would hide edits landing in the backfill scan's own second), h (sha256 — covers included: the repoint ladder needs a baseline hash for any stat- drifted sidecar; a hashless cover turns harmless mtime drift into a re-extraction, the exact flood this design kills)]}. - Evaluation ladder in cost order: stat pairs identical → skip with ZERO bytes read; stat drift → hash that sidecar (equal = host repoint reconciliation, stat pair refreshed, no re-extraction; differing = changed); NULL stored manifest → backfill only, NO re-extraction (deploy-day rule: the first scan after release must not look like an mtime flood). - Change path: re-read full row, extractMetadata once and pass it through updateMediaItemWithMetadata (preExtracted — metadata_overrides stay protected), run syncPlaylistCollectionsAfterCreate (Correction 2: a reading-order entry added to an EXISTING book now creates its collection; add-if-missing is idempotent), store a fresh full manifest. Creates seed their manifest at insert time. - Multifile folder/zip books intentionally keep Phase 3 behavior: their change path does not consume sidecar extractors, so gating them would flip books 'changed' without picking anything up (plan correction — the plan's §3 multifile note assumed otherwise). - UpsertMediaScanState + GetMediaScanState queries. Tests: media_scanner_manifest_test.go (ladder unit tests — fast path, repoint, same-size drift, appear/vanish, unhashable legacy entries); TestScanManifestLifecycle (seeding, unchanged-skip stability, repoint reconciliation with stat-pair refresh assertion, same-size edit consumed, playlist sync on the change path + idempotency, no-leak guard). Container runner. --- cmd/server/tests/scan_state_test.go | 113 ++++++++ database/schema/schema.sql | 11 +- docs/contract_pins.md | 17 ++ internal/database/querier.go | 8 + internal/database/queries.sql.go | 33 +++ internal/database/queries/queries.sql | 14 + internal/services/media_scanner.go | 60 ++++- internal/services/media_scanner_manifest.go | 255 ++++++++++++++++++ .../services/media_scanner_manifest_test.go | 136 ++++++++++ 9 files changed, 633 insertions(+), 14 deletions(-) create mode 100644 internal/services/media_scanner_manifest.go create mode 100644 internal/services/media_scanner_manifest_test.go diff --git a/cmd/server/tests/scan_state_test.go b/cmd/server/tests/scan_state_test.go index 6699c66b..8ecb6099 100644 --- a/cmd/server/tests/scan_state_test.go +++ b/cmd/server/tests/scan_state_test.go @@ -11,7 +11,10 @@ import ( "context" "crypto/sha256" "encoding/hex" + "encoding/json" "fmt" + "io" + "net/http" "os" "path/filepath" "testing" @@ -117,3 +120,113 @@ func TestScanStateBulkLoad(t *testing.T) { } } } + +// TestScanManifestLifecycle exercises the manifest-gated change ladder +// end to end (faceted plan Part 2 Phase 4): seeding at create, repoint +// reconciliation (mtime drift, identical content → no re-extraction), +// same-size sidecar edits (the size-only detection blind spot), +// playlist sync on the change path, and the no-leak guard. +func TestScanManifestLifecycle(t *testing.T) { + setup := setupTestServer(t) + t.Cleanup(func() { _ = setup.Close() }) + + admin, _ := moveTestAdminID(t, setup) + libID, folder := createTypedTestLibrary(t, setup, fmt.Sprintf("Test Manifest Lifecycle %d", time.Now().UnixNano()), "ebooks") + t.Cleanup(func() { deleteLibraries(t, setup, libID) }) + + // The book ships WITH an ABS sidecar: the create path seeds the + // manifest so the next scan compares instead of backfilling. + bookDir := filepath.Join(folder, "Manifest Author", "Manifest Book") + require.NoError(t, os.MkdirAll(bookDir, 0755)) + require.NoError(t, os.WriteFile(filepath.Join(bookDir, "book.epub"), []byte("manifest book content"), 0644)) + opfPath := filepath.Join(bookDir, "metadata.json") + require.NoError(t, os.WriteFile(opfPath, []byte(`{"title":"Manifest Book"}`), 0644)) + + scanTestFolder(t, setup, admin, folder) + + var itemID pgtype.UUID + var itemIDStr, updatedStr string + require.NoError(t, setup.DBPool.QueryRow(context.Background(), + `SELECT id::text, updated_at::text FROM media_items WHERE title = 'Manifest Book' AND library_id = $1`, + libID, + ).Scan(&itemIDStr, &updatedStr)) + require.NoError(t, itemID.Scan(itemIDStr)) + + // Seeded manifest exists. + seeded, err := setup.DB.GetMediaScanState(context.Background(), itemID) + require.NoError(t, err, "create path must seed the manifest") + require.NotEmpty(t, seeded.Manifest) + + // Unchanged rescan: skip path, nothing moves. + scanTestFolder(t, setup, admin, folder) + var updated2 string + require.NoError(t, setup.DBPool.QueryRow(context.Background(), + `SELECT updated_at::text FROM media_items WHERE id = $1`, itemID).Scan(&updated2)) + assert.Equal(t, updatedStr, updated2, "unchanged rescan is a pure skip") + + // Repoint: mtime drift with identical content → reconciled, ZERO + // re-extraction; the stored stat pair refreshes. + past := time.Now().Add(-2 * time.Hour) + require.NoError(t, os.Chtimes(opfPath, past, past)) + scanTestFolder(t, setup, admin, folder) + var updated3 string + require.NoError(t, setup.DBPool.QueryRow(context.Background(), + `SELECT updated_at::text FROM media_items WHERE id = $1`, itemID).Scan(&updated3)) + assert.Equal(t, updatedStr, updated3, "repoint reconciliation must not re-extract") + reconciled, err := setup.DB.GetMediaScanState(context.Background(), itemID) + require.NoError(t, err) + var storedManifest struct { + Entries []struct { + Name string `json:"n"` + MTime int64 `json:"m"` + } `json:"entries"` + } + require.NoError(t, json.Unmarshal(reconciled.Manifest, &storedManifest)) + var sawRefreshed bool + for _, e := range storedManifest.Entries { + if e.Name == "metadata.json" && e.MTime == past.UnixNano() { + sawRefreshed = true + } + } + assert.True(t, sawRefreshed, "repoint refreshes the stored stat pair") + + // Same-size sidecar edit — the size-only detection blind spot: the + // stat pair moves, the hash differs → re-extract consumes the edit. + sameLen := []byte(`{"title":"Manifesu Book"}`) + require.Len(t, sameLen, len(`{"title":"Manifest Book"}`), "fixture must be a same-size edit") + require.NoError(t, os.WriteFile(opfPath, sameLen, 0644)) + scanTestFolder(t, setup, admin, folder) + var title4 string + require.NoError(t, setup.DBPool.QueryRow(context.Background(), + `SELECT title FROM media_items WHERE id = $1`, itemID).Scan(&title4)) + assert.Equal(t, "Manifesu Book", title4, "same-size sidecar edit is detected and consumed") + + // Playlist sync on the change path (Correction 2): the sidecar gains + // a reading-order series → the collection is created for an + // EXISTING book, and a rescan does not duplicate it. + require.NoError(t, os.WriteFile(opfPath, []byte(`{"title":"Manifesu Book","series":["Lifecycle Saga (Reading Order) #1"]}`), 0644)) + scanTestFolder(t, setup, admin, folder) + var collCount int + require.NoError(t, setup.DBPool.QueryRow(context.Background(), + `SELECT COUNT(*) FROM collections WHERE name = 'Lifecycle Saga (Reading Order)' AND user_id = $1`, admin, + ).Scan(&collCount)) + assert.Equal(t, 1, collCount, "reading-order sidecar added to an existing book creates its collection") + scanTestFolder(t, setup, admin, folder) + require.NoError(t, setup.DBPool.QueryRow(context.Background(), + `SELECT COUNT(*) FROM collections WHERE name = 'Lifecycle Saga (Reading Order)' AND user_id = $1`, admin, + ).Scan(&collCount)) + assert.Equal(t, 1, collCount, "rescan is idempotent — no duplicate collection") + + // No-leak guard (plan Part 2 test 7): the library-scoped list emits + // raw rows — scanner state must never ride along (structural by + // design; this assertion keeps it honest). + req, _ := http.NewRequest("GET", setup.Server.URL+"/api/media-items?library_id="+libID.String(), nil) + req.Header.Set("Authorization", "Bearer "+setup.RegularToken) + resp, err := http.DefaultClient.Do(req) + require.NoError(t, err) + defer func() { _ = resp.Body.Close() }() + body, err := io.ReadAll(resp.Body) + require.NoError(t, err) + assert.NotContains(t, string(body), `"manifest":`, "scanner state must not leak into media list JSON") + assert.NotContains(t, string(body), `"sidecar`, "scanner state must not leak into media list JSON") +} diff --git a/database/schema/schema.sql b/database/schema/schema.sql index 994ed978..4775fe9e 100644 --- a/database/schema/schema.sql +++ b/database/schema/schema.sql @@ -1253,10 +1253,13 @@ CREATE INDEX IF NOT EXISTS idx_media_item_formats_sha256 ON media_item_formats(f -- bookkeeping must never leak into API payloads (nor churn the model -- or the contract pins). One row per item; cascade on delete. -- manifest shape v1: --- {"v":1,"entries":[{"n":"metadata.opf","s":3013013,"m":1697500000,"h":""}, ...]} --- n basename, s size, m mtime unix (CACHE ONLY — never a change --- signal), h sha256 for text sidecars (omitted for cover images). --- "ev" (extractor version) reserved for parser-fix re-extraction. +-- {"v":1,"entries":[{"n":"metadata.opf","s":3013013,"m":1697500000123456789,"h":""}, ...]} +-- n basename, s size, m mtime as unix NANOS (second granularity +-- would hide edits landing in the backfill scan's own second), h +-- sha256 of the content — present for ALL entries, covers included: +-- the repoint reconciliation needs a baseline hash for any stat- +-- drifted sidecar. "ev" (extractor version) reserved for parser-fix +-- re-extraction. CREATE TABLE IF NOT EXISTS media_scan_state ( media_item_id UUID PRIMARY KEY REFERENCES media_items(id) ON DELETE CASCADE, manifest JSONB diff --git a/docs/contract_pins.md b/docs/contract_pins.md index e92431f2..999bb5aa 100644 --- a/docs/contract_pins.md +++ b/docs/contract_pins.md @@ -202,3 +202,20 @@ cover_image_path, has_conflict, format_group, badge`. - Web: the bookshelf gains an active-filter chips bar (client-local form state rendered by Alpine; templ owns the markup) and `applyURLFilters` accepts the new facet params. + +## 2026-10-19 — Scanner Phase 4: manifest-gated sidecar detection + +- No wire change: `media_scan_state` lives in its own table, so the + raw-row list JSON cannot contain scanner state (guarded by an + assertion in TestScanManifestLifecycle). +- Scanner BEHAVIOR: size-stable books are now checked against a stored + sidecar fingerprint (`{"v":1,"entries":[{n,s,m-nanos,h-sha256},...]}`). + Stat-pair-identical → skip with zero reads; stat drift → hash + reconciliation (equal hash = host repoint, stat pair refreshed, no + re-extraction; differing hash = metadata re-extract + fresh manifest, + metadata_overrides protected, playlist collections synced on the + change path — a "(Reading Order)" entry added to an EXISTING book now + creates its collection). NULL stored manifest → backfill only, no + re-extraction (deploy-day rule). Multifile folder/zip books keep the + Phase 3 behavior (their change path does not consume sidecar + extractors yet — gating them would flip books "changed" for nothing). diff --git a/internal/database/querier.go b/internal/database/querier.go index d171b18f..17ba6d97 100644 --- a/internal/database/querier.go +++ b/internal/database/querier.go @@ -380,6 +380,9 @@ type Querier interface { GetMediaRating(ctx context.Context, arg GetMediaRatingParams) (MediaRatings, error) GetMediaRatings(ctx context.Context, mediaItemID pgtype.UUID) ([]GetMediaRatingsRow, error) GetMediaReview(ctx context.Context, arg GetMediaReviewParams) (MediaReviews, error) + // GetMediaScanState reads one item's manifest (per-item rescan paths + // and the manifest integration tests). + GetMediaScanState(ctx context.Context, mediaItemID pgtype.UUID) (MediaScanState, error) GetNextRetryTime(ctx context.Context) (interface{}, error) GetNotStartedItems(ctx context.Context, arg GetNotStartedItemsParams) ([]MediaItems, error) // Get OPDS token @@ -685,6 +688,11 @@ type Querier interface { UpsertDashboardPreferences(ctx context.Context, arg UpsertDashboardPreferencesParams) (UserDashboardPreferences, error) UpsertGlobalDashboardPreferences(ctx context.Context, arg UpsertGlobalDashboardPreferencesParams) (UserDashboardPreferencesGlobal, error) UpsertMediaReview(ctx context.Context, arg UpsertMediaReviewParams) (MediaReviews, error) + // UpsertMediaScanState stores a book's sidecar manifest fingerprint + // (faceted plan Part 2 §3). Written on backfill (first sighting — + // no re-extraction), on repoint reconciliation (stat pairs refreshed, + // content identical), and after a manifest-gated re-extract. + UpsertMediaScanState(ctx context.Context, arg UpsertMediaScanStateParams) error UpsertPanelData(ctx context.Context, arg UpsertPanelDataParams) (PanelData, error) UpsertReaderSettings(ctx context.Context, arg UpsertReaderSettingsParams) (ReaderSettings, error) UpsertSystemSetting(ctx context.Context, arg UpsertSystemSettingParams) (SystemSettings, error) diff --git a/internal/database/queries.sql.go b/internal/database/queries.sql.go index 007d5e86..3eb0f0b8 100644 --- a/internal/database/queries.sql.go +++ b/internal/database/queries.sql.go @@ -7796,6 +7796,19 @@ func (q *Queries) GetMediaReview(ctx context.Context, arg GetMediaReviewParams) return i, err } +const GetMediaScanState = `-- name: GetMediaScanState :one +SELECT media_item_id, manifest FROM media_scan_state WHERE media_item_id = $1 +` + +// GetMediaScanState reads one item's manifest (per-item rescan paths +// and the manifest integration tests). +func (q *Queries) GetMediaScanState(ctx context.Context, mediaItemID pgtype.UUID) (MediaScanState, error) { + row := q.db.QueryRow(ctx, GetMediaScanState, mediaItemID) + var i MediaScanState + err := row.Scan(&i.MediaItemID, &i.Manifest) + return i, err +} + const GetNextRetryTime = `-- name: GetNextRetryTime :one SELECT CASE @@ -16920,6 +16933,26 @@ func (q *Queries) UpsertMediaReview(ctx context.Context, arg UpsertMediaReviewPa return i, err } +const UpsertMediaScanState = `-- name: UpsertMediaScanState :exec +INSERT INTO media_scan_state (media_item_id, manifest) +VALUES ($1, $2) +ON CONFLICT (media_item_id) DO UPDATE SET manifest = EXCLUDED.manifest +` + +type UpsertMediaScanStateParams struct { + MediaItemID pgtype.UUID `db:"media_item_id" json:"media_item_id"` + Manifest []byte `db:"manifest" json:"manifest"` +} + +// UpsertMediaScanState stores a book's sidecar manifest fingerprint +// (faceted plan Part 2 §3). Written on backfill (first sighting — +// no re-extraction), on repoint reconciliation (stat pairs refreshed, +// content identical), and after a manifest-gated re-extract. +func (q *Queries) UpsertMediaScanState(ctx context.Context, arg UpsertMediaScanStateParams) error { + _, err := q.db.Exec(ctx, UpsertMediaScanState, arg.MediaItemID, arg.Manifest) + return err +} + const UpsertPanelData = `-- name: UpsertPanelData :one INSERT INTO panel_data (media_item_id, page_number, detection_method, panels) VALUES ($1, $2, $3, $4) diff --git a/internal/database/queries/queries.sql b/internal/database/queries/queries.sql index 0f9d24b1..9ac968c2 100644 --- a/internal/database/queries/queries.sql +++ b/internal/database/queries/queries.sql @@ -452,6 +452,20 @@ FROM media_items mi LEFT JOIN media_scan_state s ON s.media_item_id = mi.id WHERE mi.library_id = sqlc.narg('library_id'); +-- UpsertMediaScanState stores a book's sidecar manifest fingerprint +-- (faceted plan Part 2 §3). Written on backfill (first sighting — +-- no re-extraction), on repoint reconciliation (stat pairs refreshed, +-- content identical), and after a manifest-gated re-extract. +-- name: UpsertMediaScanState :exec +INSERT INTO media_scan_state (media_item_id, manifest) +VALUES ($1, $2) +ON CONFLICT (media_item_id) DO UPDATE SET manifest = EXCLUDED.manifest; + +-- GetMediaScanState reads one item's manifest (per-item rescan paths +-- and the manifest integration tests). +-- name: GetMediaScanState :one +SELECT media_item_id, manifest FROM media_scan_state WHERE media_item_id = $1; + -- name: GetMediaItemByFilePathAnyLibrary :one SELECT * FROM media_items WHERE file_path = $1 LIMIT 1; diff --git a/internal/services/media_scanner.go b/internal/services/media_scanner.go index 5893a0f0..968a5c3c 100644 --- a/internal/services/media_scanner.go +++ b/internal/services/media_scanner.go @@ -331,10 +331,10 @@ func (s *MediaScanner) syncPlaylistCollectionsAfterCreate(ctx context.Context, i } position := cleanIntegerPosition(entry.Sequence) if _, err := s.db.AddBookToCollectionOrdered(ctx, database.AddBookToCollectionOrderedParams{ - CollectionID: collection.ID, - MediaItemID: itemID, - AddedByUserID: s.adminID, - Position: pgtype.Int4{Int32: position, Valid: position > 0}, + CollectionID: collection.ID, + MediaItemID: itemID, + AddedByUserID: s.adminID, + Position: pgtype.Int4{Int32: position, Valid: position > 0}, }); err != nil { fmt.Printf("Warning: playlist collection add failed for %q: %v\n", entry.Name, err) continue @@ -350,7 +350,7 @@ func (s *MediaScanner) syncPlaylistCollectionsAfterCreate(ctx context.Context, i func (s *MediaScanner) upsertPlaylistCollection(ctx context.Context, entry SeriesCandidate) (database.Collections, bool, error) { owner := s.adminID if collection, err := s.db.GetCollectionByImportedSeries(ctx, database.GetCollectionByImportedSeriesParams{ - UserID: owner, + UserID: owner, ImportedFromSeries: entry.Name, }); err == nil { return collection, false, nil @@ -1311,12 +1311,21 @@ func (s *MediaScanner) processMediaFile(ctx context.Context, path string) (bool, if metadata.FileHashInfo != nil { sha = metadata.FileHashInfo.FileSHA256 } - s.scanStateUpsert(libraryID, &scanEntry{ - ID: createdItem.ID, - FilePath: relativePath, - FileSize: info.Size(), + newEntry := &scanEntry{ + ID: createdItem.ID, + FilePath: relativePath, + FileSize: info.Size(), FileSha256: sha, - }) + } + s.scanStateUpsert(libraryID, newEntry) + + // Seed the sidecar manifest at create time (Part 2 §3): the next + // scan compares instead of backfilling, so an edit is detected on + // its first post-seed scan. + dir := bookSidecarDir(path) + if computed := statSidecarManifest(path, dir); computed != nil && len(computed.Entries) > 0 { + s.storeManifest(ctx, libraryID, newEntry, hashSidecarManifest(path, dir, computed)) + } return true, nil } @@ -1418,6 +1427,37 @@ func (s *MediaScanner) processExistingViaScanState(ctx context.Context, entry *s return false, nil } + // Manifest-gated sidecar detection (Part 2 §3): a size-stable book + // is changed by EDITING its sidecars. NULL stored manifest → + // backfill only; stat drift with equal hashes → repoint + // reconciliation; content drift → re-extract below. + if s.detectSidecarChange(ctx, entry, libraryID, path) { + fmt.Printf("Sidecar changed, re-extracting metadata: %s\n", path) + existingItem, err := s.getMediaItemByFilePath(ctx, path, libraryID) + if err != nil { + return false, fmt.Errorf("sidecar change re-read failed: %v", err) + } + // Extract once and pass it in, so the playlist sync below sees + // the sidecar's ordering entries (metadata_overrides stay + // protected inside the update path either way). + metadata, extractErr := s.extractMetadata(path) + if extractErr != nil || metadata == nil { + metadata = &MediaMetadata{} + } + if err := s.updateMediaItemWithMetadata(ctx, existingItem, path, metadata); err != nil { + fmt.Printf("Warning: failed to update media item after sidecar change %s: %v\n", path, err) + } + // Playlist sync on the change path (plan Correction 2): a + // "(Reading Order)" entry added to an EXISTING book's sidecar + // still creates its collection — add-if-missing is idempotent. + s.syncPlaylistCollectionsAfterCreate(ctx, entry.ID, metadata.PlaylistSeries) + // Fresh full manifest: the next scan compares against TODAY's + // sidecars, not the stale pre-edit fingerprint. + dir := bookSidecarDir(path) + s.storeManifest(ctx, libraryID, entry, hashSidecarManifest(path, dir, statSidecarManifest(path, dir))) + return false, nil + } + // Self-heal items imported before hashing existed: even an unchanged // file gets its hash computed if missing. recomputeHashInfo persists // it; the next scan's bulk load carries it — no write-through needed diff --git a/internal/services/media_scanner_manifest.go b/internal/services/media_scanner_manifest.go new file mode 100644 index 00000000..702ab4e3 --- /dev/null +++ b/internal/services/media_scanner_manifest.go @@ -0,0 +1,255 @@ +package services + +// Sidecar manifest fingerprints (faceted plan Part 2 §3): size-stable +// books are changed by EDITING their metadata sidecars, which the old +// size-only detection never noticed. One manifest per book (JSONB in +// media_scan_state) lists every sidecar the extractors consume, so the +// manifest and the extractors can never disagree — the manifest stats +// exactly the names extractCalibreSidecar, extractAudiobookshelfSidecar, +// and findSidecarCover look for. +// +// Change evaluation ladder, in cost order: +// 1. stat pairs all equal → unchanged, ZERO bytes read (steady state); +// 2. a stat pair differs → read + hash that one sidecar: +// hash equal → refresh the stored stat pair (host repoint +// reconciliation: bounded reads, zero re-extraction); +// hash differs → the item changed → re-extract (caller); +// 3. stored manifest NULL → backfill only: hash + store, NO +// re-extraction (deploy-day rule — the first scan after release +// must not look like an mtime flood). +// +// mtime is a CACHE field only — it is never a change signal. Docker +// bind-mount repoints reset every mtime, which is exactly why the hash +// reconciliation ladder exists. + +import ( + "bookhoard/internal/database" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/jackc/pgx/v5/pgtype" +) + +const sidecarManifestVersion = 1 + +type manifestSidecar struct { + Name string `json:"n"` + Size int64 `json:"s"` + MTime int64 `json:"m"` + // Sha is the content hash. Present for ALL entries — covers too: + // the repoint reconciliation needs a baseline hash for any stat- + // drifted sidecar, and a hashless cover would turn harmless mtime + // drift into a re-extraction (the exact flood this design kills). + Sha string `json:"h,omitempty"` +} + +type sidecarManifest struct { + Version int `json:"v"` + Entries []manifestSidecar `json:"entries"` +} + +// coverSidecarNames mirrors findSidecarCover's priority list — the +// manifest must stat the same names the extractor would consume. +var coverSidecarNames = []string{ + "cover.jpg", "cover.jpeg", "cover.png", "cover.webp", + "folder.jpg", "folder.jpeg", "folder.png", "folder.webp", + ".folder.jpg", ".folder.jpeg", ".folder.png", +} + +// bookSidecarDir is the directory whose sidecars feed a book: the book +// file's directory (multifile book roots resolve to themselves because +// file_path IS the directory). +func bookSidecarDir(path string) string { + info, err := os.Stat(path) + if err == nil && info.IsDir() { + return path + } + return filepath.Dir(path) +} + +// sidecarNamesForBook lists every sidecar filename the extractors +// consume for a book at path. The book basename covers the +// findSidecarCover "." pattern. +func sidecarNamesForBook(path string) []string { + names := make([]string, 0, len(coverSidecarNames)+5) + names = append(names, "metadata.opf", "metadata.json") + names = append(names, coverSidecarNames...) + base := strings.TrimSuffix(filepath.Base(path), filepath.Ext(path)) + if base != "" && base != "." { + for _, ext := range []string{".jpg", ".jpeg", ".png", ".webp"} { + names = append(names, base+ext) + } + } + return names +} + +// statSidecarManifest stats every candidate sidecar (ReadDir-free: one +// lstat per candidate name — the walk already visited the directory). +// Returns nil when the directory itself is gone (deleted book mid-scan). +func statSidecarManifest(path, dir string) *sidecarManifest { + manifest := &sidecarManifest{Version: sidecarManifestVersion} + for _, name := range sidecarNamesForBook(path) { + info, err := os.Stat(filepath.Join(dir, name)) + if err != nil || info.IsDir() { + continue + } + manifest.Entries = append(manifest.Entries, manifestSidecar{ + Name: name, + Size: info.Size(), + MTime: info.ModTime().UnixNano(), + }) + } + return manifest +} + +// hashSidecarManifest fills the content hashes of every entry — text +// sidecars (KBs) and covers (a one-time bounded read at backfill; the +// repoint ladder needs the baseline). Returns the same manifest for +// chaining. +func hashSidecarManifest(path, dir string, manifest *sidecarManifest) *sidecarManifest { + for i := range manifest.Entries { + entry := &manifest.Entries[i] + sha, err := sha256File(filepath.Join(dir, entry.Name)) + if err != nil { + continue // unreadable: stays size-only this pass + } + entry.Sha = sha + } + return manifest +} + +func sha256File(path string) (string, error) { + data, err := os.ReadFile(path) + if err != nil { + return "", err + } + sum := sha256.Sum256(data) + return hex.EncodeToString(sum[:]), nil +} + +// decodeSidecarManifest parses a stored manifest; nil when absent or +// corrupt (the caller then treats it as a backfill, self-healing any +// shape breakage). +func decodeSidecarManifest(raw []byte) *sidecarManifest { + if len(raw) == 0 { + return nil + } + var m sidecarManifest + if err := json.Unmarshal(raw, &m); err != nil { + return nil + } + return &m +} + +// reconcileSidecarManifest runs the evaluation ladder against the +// freshly-statted manifest. Returns the manifest to store and whether +// the item's metadata must be re-extracted: +// - stat pairs all equal → fresh=nil, changed=false (zero reads); +// - stat drift + equal hash → fresh with the refreshed pair +// (repoint reconciliation); +// - hash differs, a sidecar appeared/vanished, or a differing text +// sidecar has no stored hash → changed. +func reconcileSidecarManifest(dir string, stored, computed *sidecarManifest) (fresh *sidecarManifest, changed bool) { + if stored == nil { + return nil, true + } + if len(stored.Entries) != len(computed.Entries) { + return nil, true // sidecar appeared or vanished + } + storedByName := make(map[string]manifestSidecar, len(stored.Entries)) + for _, e := range stored.Entries { + storedByName[e.Name] = e + } + // Fast path: stat pairs all equal → unchanged, zero bytes read, + // nothing to store. + drifted := false + for _, cur := range computed.Entries { + old, ok := storedByName[cur.Name] + if !ok || cur.Size != old.Size || cur.MTime != old.MTime { + drifted = true + break + } + } + if !drifted { + return nil, false + } + fresh = &sidecarManifest{Version: sidecarManifestVersion} + for _, cur := range computed.Entries { + old, ok := storedByName[cur.Name] + if !ok { + return nil, true + } + if cur.Size == old.Size && cur.MTime == old.MTime { + fresh.Entries = append(fresh.Entries, old) // untouched: keep the stored hash + continue + } + if old.Sha == "" { + return nil, true // unhashable drift: fall back to re-extract + } + sha, err := sha256File(filepath.Join(dir, cur.Name)) + if err != nil || sha != old.Sha { + return nil, true + } + cur.Sha = old.Sha // repoint: content identical, stat pair refreshed + fresh.Entries = append(fresh.Entries, cur) + } + return fresh, false +} + +// storeManifest persists the manifest and write-throughs the map entry. +func (s *MediaScanner) storeManifest(ctx context.Context, libraryID pgtype.UUID, entry *scanEntry, manifest *sidecarManifest) { + b, err := json.Marshal(manifest) + if err != nil { + fmt.Printf("Warning: failed to encode sidecar manifest for %s: %v\n", entry.FilePath, err) + return + } + if err := s.db.UpsertMediaScanState(ctx, database.UpsertMediaScanStateParams{ + MediaItemID: entry.ID, + Manifest: b, + }); err != nil { + fmt.Printf("Warning: failed to store sidecar manifest for %s: %v\n", entry.FilePath, err) + return + } + entry.Manifest = b + s.scanStateUpsert(libraryID, entry) +} + +// detectSidecarChange fingerprints the book's sidecar directory and +// reconciles against the stored manifest. Returns true when the item's +// metadata must be re-extracted (a sidecar's CONTENT changed, or +// sidecars appeared/vanished). Backfill and repoint reconciliation are +// handled here without re-extraction; the caller stores a fresh full +// manifest after a change-path update. +func (s *MediaScanner) detectSidecarChange(ctx context.Context, entry *scanEntry, libraryID pgtype.UUID, path string) bool { + dir := bookSidecarDir(path) + computed := statSidecarManifest(path, dir) + stored := decodeSidecarManifest(entry.Manifest) + + if len(computed.Entries) == 0 { + if stored != nil && len(stored.Entries) > 0 { + return true // every sidecar was removed — fall back to embedded + } + return false // never had sidecars: nothing to detect or store + } + + if stored == nil { + // Deploy-day backfill: hash + store WITHOUT re-extraction. + s.storeManifest(ctx, libraryID, entry, hashSidecarManifest(path, dir, computed)) + return false + } + + fresh, changed := reconcileSidecarManifest(dir, stored, computed) + if changed { + return true + } + if fresh != nil { + s.storeManifest(ctx, libraryID, entry, fresh) + } + return false +} diff --git a/internal/services/media_scanner_manifest_test.go b/internal/services/media_scanner_manifest_test.go new file mode 100644 index 00000000..02aee692 --- /dev/null +++ b/internal/services/media_scanner_manifest_test.go @@ -0,0 +1,136 @@ +package services + +// Unit tests for the sidecar manifest ladder (faceted plan Part 2 §3): +// stat-pair fast compare, hash-based repoint reconciliation, and the +// change verdicts. Real files only where the hash compare needs them. + +import ( + "os" + "path/filepath" + "testing" + "time" +) + +func TestDecodeSidecarManifest(t *testing.T) { + if decodeSidecarManifest(nil) != nil { + t.Error("NULL manifest decodes to nil (backfill territory)") + } + if decodeSidecarManifest([]byte("{not json")) != nil { + t.Error("corrupt manifest decodes to nil (self-healing backfill)") + } + m := decodeSidecarManifest([]byte(`{"v":1,"entries":[{"n":"metadata.opf","s":10,"m":100,"h":"abc"}]}`)) + if m == nil || len(m.Entries) != 1 || m.Entries[0].Sha != "abc" { + t.Errorf("valid manifest round-trip failed: %+v", m) + } +} + +func TestReconcileSidecarManifest(t *testing.T) { + dir := t.TempDir() + write := func(name, content string) { + t.Helper() + if err := os.WriteFile(filepath.Join(dir, name), []byte(content), 0644); err != nil { + t.Fatal(err) + } + } + + // The book: one hashed sidecar + one size-only cover. + write("metadata.opf", `original content`) + originalSha, err := sha256File(filepath.Join(dir, "metadata.opf")) + if err != nil { + t.Fatal(err) + } + write("cover.jpg", "coverbytes") + // stored is seeded exactly the way production backfill does: a real + // stat pass + hash fill. Hand-built fake mtimes would drift on the + // first compare (real files' mtimes never equal a constant). + stored := hashSidecarManifest("book.epub", dir, statSidecarManifest(filepath.Join(dir, "book.epub"), dir)) + + stat := func() *sidecarManifest { + m := statSidecarManifest(filepath.Join(dir, "book.epub"), dir) + if m == nil || len(m.Entries) < 2 { + t.Fatalf("stat manifest should find both sidecars, got %+v", m) + } + return m + } + + // Stat pairs identical → unchanged, zero reads, nothing to store. + fresh, changed := reconcileSidecarManifest(dir, stored, stat()) + if changed || fresh != nil { + t.Errorf("identical stats must be unchanged, got changed=%v fresh=%v", changed, fresh) + } + + // Repoint: mtime drifts, content identical → reconciled fresh + // manifest with the refreshed stat pair and the KEPT hash, no change. + if err := os.Chtimes(filepath.Join(dir, "metadata.opf"), time.Now(), time.Now().Add(time.Minute)); err != nil { + t.Fatal(err) + } + fresh, changed = reconcileSidecarManifest(dir, stored, stat()) + if changed { + t.Error("mtime drift with identical content is repoint reconciliation, not a change") + } + if fresh == nil || len(fresh.Entries) != 2 { + t.Fatalf("repoint must produce a refreshed manifest, got %+v", fresh) + } + if fresh.Entries[0].Sha != originalSha { + t.Error("repointed entry keeps its stored hash") + } + if fresh.Entries[0].MTime == stored.Entries[0].MTime { + t.Error("repointed entry refreshes its stat pair") + } + + // Same-size edit: drift + hash differs → changed. + write("metadata.opf", `mutated XYY content`) // +1 byte, and hash differs + _, changed = reconcileSidecarManifest(dir, stored, stat()) + if !changed { + t.Error("content drift must be a change") + } + + // Same-LENGTH edit (the size-only detection blind spot) → changed. + write("metadata.opf", `original Xontent`) + _, changed = reconcileSidecarManifest(dir, stored, stat()) + if !changed { + t.Error("same-size content drift must be a change (hash compare)") + } + + // New sidecar appears → changed. + write("metadata.json", `{"title":"X"}`) + _, changed = reconcileSidecarManifest(dir, stored, stat()) + if !changed { + t.Error("an appearing sidecar must be a change") + } + + // Sidecar vanishes AND everything else returns to the stored state + // (opf restored; its mtime drifts but the hash proves equality) → + // the manifest reconciles, not a change. + if err := os.Remove(filepath.Join(dir, "metadata.json")); err != nil { + t.Fatal(err) + } + write("metadata.opf", `original content`) + fresh, changed = reconcileSidecarManifest(dir, stored, stat()) + if changed { + t.Errorf("returning to the stored state is not a change: %+v", fresh) + } + if fresh == nil || len(fresh.Entries) != 2 { + t.Errorf("reconciled manifest covers both sidecars: %+v", fresh) + } +} + +// An unhashable drift (a stored entry with no baseline hash — legacy +// or partially-written manifest) is a change: the manifest cannot prove +// content equality for it. +func TestReconcileUnhashableDrift(t *testing.T) { + dir := t.TempDir() + stored := &sidecarManifest{Version: 1, Entries: []manifestSidecar{ + {Name: "cover.jpg", Size: 10, MTime: 1000}, + }} + computed := &sidecarManifest{Version: 1, Entries: []manifestSidecar{ + {Name: "cover.jpg", Size: 11, MTime: 1001}, + }} + fresh, changed := reconcileSidecarManifest(dir, stored, computed) + if !changed { + t.Error("stat drift without a stored baseline hash must be a change") + } + if fresh != nil { + t.Error("a change never returns a fresh manifest (caller builds one)") + } +}