feat(scanner): manifest-gated sidecar change detection + playlist sync on change path

Part 2 of the faceted plan, Phase 4 (steps 3-4). Size-only detection
was blind to metadata-sidecar edits; a size-stable book is changed by
EDITING its metadata.opf/metadata.json/cover — previously invisible
until a force rescan.

- Sidecar manifest fingerprints (media_scanner_manifest.go): the
  manifest stats exactly the names the extractors consume (metadata.opf,
  metadata.json, findSidecarCover patterns incl. <basename>.<ext>), so
  manifest and extractors can never disagree. Shape v1:
  {v, entries:[{n, s, m (unix nanos — second granularity would hide
  edits landing in the backfill scan's own second), h (sha256 — covers
  included: the repoint ladder needs a baseline hash for any stat-
  drifted sidecar; a hashless cover turns harmless mtime drift into a
  re-extraction, the exact flood this design kills)]}.
- Evaluation ladder in cost order: stat pairs identical → skip with
  ZERO bytes read; stat drift → hash that sidecar (equal = host repoint
  reconciliation, stat pair refreshed, no re-extraction; differing =
  changed); NULL stored manifest → backfill only, NO re-extraction
  (deploy-day rule: the first scan after release must not look like an
  mtime flood).
- Change path: re-read full row, extractMetadata once and pass it
  through updateMediaItemWithMetadata (preExtracted — metadata_overrides
  stay protected), run syncPlaylistCollectionsAfterCreate (Correction 2:
  a reading-order entry added to an EXISTING book now creates its
  collection; add-if-missing is idempotent), store a fresh full
  manifest. Creates seed their manifest at insert time.
- Multifile folder/zip books intentionally keep Phase 3 behavior: their
  change path does not consume sidecar extractors, so gating them would
  flip books 'changed' without picking anything up (plan correction —
  the plan's §3 multifile note assumed otherwise).
- UpsertMediaScanState + GetMediaScanState queries.

Tests: media_scanner_manifest_test.go (ladder unit tests — fast path,
repoint, same-size drift, appear/vanish, unhashable legacy entries);
TestScanManifestLifecycle (seeding, unchanged-skip stability, repoint
reconciliation with stat-pair refresh assertion, same-size edit
consumed, playlist sync on the change path + idempotency, no-leak
guard). Container runner.
This commit is contained in:
John O'Keefe committed 2026-10-19 20:17:57 -04:00
1 parent 693053a9f5
commit 6750e9e0dc
9 files changed
+633 -14

No files matched your search

+113
View File
@@ -11,7 +11,10 @@ import (
"context" "context"
"crypto/sha256" "crypto/sha256"
"encoding/hex" "encoding/hex"
"encoding/json"
"fmt" "fmt"
"io"
"net/http"
"os" "os"
"path/filepath" "path/filepath"
"testing" "testing"
@@ -117,3 +120,113 @@ func TestScanStateBulkLoad(t *testing.T) {
} }
} }
} }
// TestScanManifestLifecycle exercises the manifest-gated change ladder
// end to end (faceted plan Part 2 Phase 4): seeding at create, repoint
// reconciliation (mtime drift, identical content → no re-extraction),
// same-size sidecar edits (the size-only detection blind spot),
// playlist sync on the change path, and the no-leak guard.
func TestScanManifestLifecycle(t *testing.T) {
setup := setupTestServer(t)
t.Cleanup(func() { _ = setup.Close() })
admin, _ := moveTestAdminID(t, setup)
libID, folder := createTypedTestLibrary(t, setup, fmt.Sprintf("Test Manifest Lifecycle %d", time.Now().UnixNano()), "ebooks")
t.Cleanup(func() { deleteLibraries(t, setup, libID) })
// The book ships WITH an ABS sidecar: the create path seeds the
// manifest so the next scan compares instead of backfilling.
bookDir := filepath.Join(folder, "Manifest Author", "Manifest Book")
require.NoError(t, os.MkdirAll(bookDir, 0755))
require.NoError(t, os.WriteFile(filepath.Join(bookDir, "book.epub"), []byte("manifest book content"), 0644))
opfPath := filepath.Join(bookDir, "metadata.json")
require.NoError(t, os.WriteFile(opfPath, []byte(`{"title":"Manifest Book"}`), 0644))
scanTestFolder(t, setup, admin, folder)
var itemID pgtype.UUID
var itemIDStr, updatedStr string
require.NoError(t, setup.DBPool.QueryRow(context.Background(),
`SELECT id::text, updated_at::text FROM media_items WHERE title = 'Manifest Book' AND library_id = $1`,
libID,
).Scan(&itemIDStr, &updatedStr))
require.NoError(t, itemID.Scan(itemIDStr))
// Seeded manifest exists.
seeded, err := setup.DB.GetMediaScanState(context.Background(), itemID)
require.NoError(t, err, "create path must seed the manifest")
require.NotEmpty(t, seeded.Manifest)
// Unchanged rescan: skip path, nothing moves.
scanTestFolder(t, setup, admin, folder)
var updated2 string
require.NoError(t, setup.DBPool.QueryRow(context.Background(),
`SELECT updated_at::text FROM media_items WHERE id = $1`, itemID).Scan(&updated2))
assert.Equal(t, updatedStr, updated2, "unchanged rescan is a pure skip")
// Repoint: mtime drift with identical content → reconciled, ZERO
// re-extraction; the stored stat pair refreshes.
past := time.Now().Add(-2 * time.Hour)
require.NoError(t, os.Chtimes(opfPath, past, past))
scanTestFolder(t, setup, admin, folder)
var updated3 string
require.NoError(t, setup.DBPool.QueryRow(context.Background(),
`SELECT updated_at::text FROM media_items WHERE id = $1`, itemID).Scan(&updated3))
assert.Equal(t, updatedStr, updated3, "repoint reconciliation must not re-extract")
reconciled, err := setup.DB.GetMediaScanState(context.Background(), itemID)
require.NoError(t, err)
var storedManifest struct {
Entries []struct {
Name string `json:"n"`
MTime int64 `json:"m"`
} `json:"entries"`
}
require.NoError(t, json.Unmarshal(reconciled.Manifest, &storedManifest))
var sawRefreshed bool
for _, e := range storedManifest.Entries {
if e.Name == "metadata.json" && e.MTime == past.UnixNano() {
sawRefreshed = true
}
}
assert.True(t, sawRefreshed, "repoint refreshes the stored stat pair")
// Same-size sidecar edit — the size-only detection blind spot: the
// stat pair moves, the hash differs → re-extract consumes the edit.
sameLen := []byte(`{"title":"Manifesu Book"}`)
require.Len(t, sameLen, len(`{"title":"Manifest Book"}`), "fixture must be a same-size edit")
require.NoError(t, os.WriteFile(opfPath, sameLen, 0644))
scanTestFolder(t, setup, admin, folder)
var title4 string
require.NoError(t, setup.DBPool.QueryRow(context.Background(),
`SELECT title FROM media_items WHERE id = $1`, itemID).Scan(&title4))
assert.Equal(t, "Manifesu Book", title4, "same-size sidecar edit is detected and consumed")
// Playlist sync on the change path (Correction 2): the sidecar gains
// a reading-order series → the collection is created for an
// EXISTING book, and a rescan does not duplicate it.
require.NoError(t, os.WriteFile(opfPath, []byte(`{"title":"Manifesu Book","series":["Lifecycle Saga (Reading Order) #1"]}`), 0644))
scanTestFolder(t, setup, admin, folder)
var collCount int
require.NoError(t, setup.DBPool.QueryRow(context.Background(),
`SELECT COUNT(*) FROM collections WHERE name = 'Lifecycle Saga (Reading Order)' AND user_id = $1`, admin,
).Scan(&collCount))
assert.Equal(t, 1, collCount, "reading-order sidecar added to an existing book creates its collection")
scanTestFolder(t, setup, admin, folder)
require.NoError(t, setup.DBPool.QueryRow(context.Background(),
`SELECT COUNT(*) FROM collections WHERE name = 'Lifecycle Saga (Reading Order)' AND user_id = $1`, admin,
).Scan(&collCount))
assert.Equal(t, 1, collCount, "rescan is idempotent — no duplicate collection")
// No-leak guard (plan Part 2 test 7): the library-scoped list emits
// raw rows — scanner state must never ride along (structural by
// design; this assertion keeps it honest).
req, _ := http.NewRequest("GET", setup.Server.URL+"/api/media-items?library_id="+libID.String(), nil)
req.Header.Set("Authorization", "Bearer "+setup.RegularToken)
resp, err := http.DefaultClient.Do(req)
require.NoError(t, err)
defer func() { _ = resp.Body.Close() }()
body, err := io.ReadAll(resp.Body)
require.NoError(t, err)
assert.NotContains(t, string(body), `"manifest":`, "scanner state must not leak into media list JSON")
assert.NotContains(t, string(body), `"sidecar`, "scanner state must not leak into media list JSON")
}
+7 -4
View File
@@ -1253,10 +1253,13 @@ CREATE INDEX IF NOT EXISTS idx_media_item_formats_sha256 ON media_item_formats(f
-- bookkeeping must never leak into API payloads (nor churn the model -- bookkeeping must never leak into API payloads (nor churn the model
-- or the contract pins). One row per item; cascade on delete. -- or the contract pins). One row per item; cascade on delete.
-- manifest shape v1: -- manifest shape v1:
-- {"v":1,"entries":[{"n":"metadata.opf","s":3013013,"m":1697500000,"h":"<sha256>"}, ...]} -- {"v":1,"entries":[{"n":"metadata.opf","s":3013013,"m":1697500000123456789,"h":"<sha256>"}, ...]}
-- n basename, s size, m mtime unix (CACHE ONLY — never a change -- n basename, s size, m mtime as unix NANOS (second granularity
-- signal), h sha256 for text sidecars (omitted for cover images). -- would hide edits landing in the backfill scan's own second), h
-- "ev" (extractor version) reserved for parser-fix re-extraction. -- sha256 of the content — present for ALL entries, covers included:
-- the repoint reconciliation needs a baseline hash for any stat-
-- drifted sidecar. "ev" (extractor version) reserved for parser-fix
-- re-extraction.
CREATE TABLE IF NOT EXISTS media_scan_state ( CREATE TABLE IF NOT EXISTS media_scan_state (
media_item_id UUID PRIMARY KEY REFERENCES media_items(id) ON DELETE CASCADE, media_item_id UUID PRIMARY KEY REFERENCES media_items(id) ON DELETE CASCADE,
manifest JSONB manifest JSONB
+17
View File
@@ -202,3 +202,20 @@ cover_image_path, has_conflict, format_group, badge`.
- Web: the bookshelf gains an active-filter chips bar (client-local - Web: the bookshelf gains an active-filter chips bar (client-local
form state rendered by Alpine; templ owns the markup) and form state rendered by Alpine; templ owns the markup) and
`applyURLFilters` accepts the new facet params. `applyURLFilters` accepts the new facet params.
## 2026-10-19 — Scanner Phase 4: manifest-gated sidecar detection
- No wire change: `media_scan_state` lives in its own table, so the
raw-row list JSON cannot contain scanner state (guarded by an
assertion in TestScanManifestLifecycle).
- Scanner BEHAVIOR: size-stable books are now checked against a stored
sidecar fingerprint (`{"v":1,"entries":[{n,s,m-nanos,h-sha256},...]}`).
Stat-pair-identical → skip with zero reads; stat drift → hash
reconciliation (equal hash = host repoint, stat pair refreshed, no
re-extraction; differing hash = metadata re-extract + fresh manifest,
metadata_overrides protected, playlist collections synced on the
change path — a "(Reading Order)" entry added to an EXISTING book now
creates its collection). NULL stored manifest → backfill only, no
re-extraction (deploy-day rule). Multifile folder/zip books keep the
Phase 3 behavior (their change path does not consume sidecar
extractors yet — gating them would flip books "changed" for nothing).
+8
View File
@@ -380,6 +380,9 @@ type Querier interface {
GetMediaRating(ctx context.Context, arg GetMediaRatingParams) (MediaRatings, error) GetMediaRating(ctx context.Context, arg GetMediaRatingParams) (MediaRatings, error)
GetMediaRatings(ctx context.Context, mediaItemID pgtype.UUID) ([]GetMediaRatingsRow, error) GetMediaRatings(ctx context.Context, mediaItemID pgtype.UUID) ([]GetMediaRatingsRow, error)
GetMediaReview(ctx context.Context, arg GetMediaReviewParams) (MediaReviews, error) GetMediaReview(ctx context.Context, arg GetMediaReviewParams) (MediaReviews, error)
// GetMediaScanState reads one item's manifest (per-item rescan paths
// and the manifest integration tests).
GetMediaScanState(ctx context.Context, mediaItemID pgtype.UUID) (MediaScanState, error)
GetNextRetryTime(ctx context.Context) (interface{}, error) GetNextRetryTime(ctx context.Context) (interface{}, error)
GetNotStartedItems(ctx context.Context, arg GetNotStartedItemsParams) ([]MediaItems, error) GetNotStartedItems(ctx context.Context, arg GetNotStartedItemsParams) ([]MediaItems, error)
// Get OPDS token // Get OPDS token
@@ -685,6 +688,11 @@ type Querier interface {
UpsertDashboardPreferences(ctx context.Context, arg UpsertDashboardPreferencesParams) (UserDashboardPreferences, error) UpsertDashboardPreferences(ctx context.Context, arg UpsertDashboardPreferencesParams) (UserDashboardPreferences, error)
UpsertGlobalDashboardPreferences(ctx context.Context, arg UpsertGlobalDashboardPreferencesParams) (UserDashboardPreferencesGlobal, error) UpsertGlobalDashboardPreferences(ctx context.Context, arg UpsertGlobalDashboardPreferencesParams) (UserDashboardPreferencesGlobal, error)
UpsertMediaReview(ctx context.Context, arg UpsertMediaReviewParams) (MediaReviews, error) UpsertMediaReview(ctx context.Context, arg UpsertMediaReviewParams) (MediaReviews, error)
// UpsertMediaScanState stores a book's sidecar manifest fingerprint
// (faceted plan Part 2 §3). Written on backfill (first sighting —
// no re-extraction), on repoint reconciliation (stat pairs refreshed,
// content identical), and after a manifest-gated re-extract.
UpsertMediaScanState(ctx context.Context, arg UpsertMediaScanStateParams) error
UpsertPanelData(ctx context.Context, arg UpsertPanelDataParams) (PanelData, error) UpsertPanelData(ctx context.Context, arg UpsertPanelDataParams) (PanelData, error)
UpsertReaderSettings(ctx context.Context, arg UpsertReaderSettingsParams) (ReaderSettings, error) UpsertReaderSettings(ctx context.Context, arg UpsertReaderSettingsParams) (ReaderSettings, error)
UpsertSystemSetting(ctx context.Context, arg UpsertSystemSettingParams) (SystemSettings, error) UpsertSystemSetting(ctx context.Context, arg UpsertSystemSettingParams) (SystemSettings, error)
+33
View File
@@ -7796,6 +7796,19 @@ func (q *Queries) GetMediaReview(ctx context.Context, arg GetMediaReviewParams)
return i, err return i, err
} }
const GetMediaScanState = `-- name: GetMediaScanState :one
SELECT media_item_id, manifest FROM media_scan_state WHERE media_item_id = $1
`
// GetMediaScanState reads one item's manifest (per-item rescan paths
// and the manifest integration tests).
func (q *Queries) GetMediaScanState(ctx context.Context, mediaItemID pgtype.UUID) (MediaScanState, error) {
row := q.db.QueryRow(ctx, GetMediaScanState, mediaItemID)
var i MediaScanState
err := row.Scan(&i.MediaItemID, &i.Manifest)
return i, err
}
const GetNextRetryTime = `-- name: GetNextRetryTime :one const GetNextRetryTime = `-- name: GetNextRetryTime :one
SELECT SELECT
CASE CASE
@@ -16920,6 +16933,26 @@ func (q *Queries) UpsertMediaReview(ctx context.Context, arg UpsertMediaReviewPa
return i, err return i, err
} }
const UpsertMediaScanState = `-- name: UpsertMediaScanState :exec
INSERT INTO media_scan_state (media_item_id, manifest)
VALUES ($1, $2)
ON CONFLICT (media_item_id) DO UPDATE SET manifest = EXCLUDED.manifest
`
type UpsertMediaScanStateParams struct {
MediaItemID pgtype.UUID `db:"media_item_id" json:"media_item_id"`
Manifest []byte `db:"manifest" json:"manifest"`
}
// UpsertMediaScanState stores a book's sidecar manifest fingerprint
// (faceted plan Part 2 §3). Written on backfill (first sighting —
// no re-extraction), on repoint reconciliation (stat pairs refreshed,
// content identical), and after a manifest-gated re-extract.
func (q *Queries) UpsertMediaScanState(ctx context.Context, arg UpsertMediaScanStateParams) error {
_, err := q.db.Exec(ctx, UpsertMediaScanState, arg.MediaItemID, arg.Manifest)
return err
}
const UpsertPanelData = `-- name: UpsertPanelData :one const UpsertPanelData = `-- name: UpsertPanelData :one
INSERT INTO panel_data (media_item_id, page_number, detection_method, panels) INSERT INTO panel_data (media_item_id, page_number, detection_method, panels)
VALUES ($1, $2, $3, $4) VALUES ($1, $2, $3, $4)
+14
View File
@@ -452,6 +452,20 @@ FROM media_items mi
LEFT JOIN media_scan_state s ON s.media_item_id = mi.id LEFT JOIN media_scan_state s ON s.media_item_id = mi.id
WHERE mi.library_id = sqlc.narg('library_id'); WHERE mi.library_id = sqlc.narg('library_id');
-- UpsertMediaScanState stores a book's sidecar manifest fingerprint
-- (faceted plan Part 2 §3). Written on backfill (first sighting —
-- no re-extraction), on repoint reconciliation (stat pairs refreshed,
-- content identical), and after a manifest-gated re-extract.
-- name: UpsertMediaScanState :exec
INSERT INTO media_scan_state (media_item_id, manifest)
VALUES ($1, $2)
ON CONFLICT (media_item_id) DO UPDATE SET manifest = EXCLUDED.manifest;
-- GetMediaScanState reads one item's manifest (per-item rescan paths
-- and the manifest integration tests).
-- name: GetMediaScanState :one
SELECT media_item_id, manifest FROM media_scan_state WHERE media_item_id = $1;
-- name: GetMediaItemByFilePathAnyLibrary :one -- name: GetMediaItemByFilePathAnyLibrary :one
SELECT * FROM media_items WHERE file_path = $1 LIMIT 1; SELECT * FROM media_items WHERE file_path = $1 LIMIT 1;
+50 -10
View File
@@ -331,10 +331,10 @@ func (s *MediaScanner) syncPlaylistCollectionsAfterCreate(ctx context.Context, i
} }
position := cleanIntegerPosition(entry.Sequence) position := cleanIntegerPosition(entry.Sequence)
if _, err := s.db.AddBookToCollectionOrdered(ctx, database.AddBookToCollectionOrderedParams{ if _, err := s.db.AddBookToCollectionOrdered(ctx, database.AddBookToCollectionOrderedParams{
CollectionID: collection.ID, CollectionID: collection.ID,
MediaItemID: itemID, MediaItemID: itemID,
AddedByUserID: s.adminID, AddedByUserID: s.adminID,
Position: pgtype.Int4{Int32: position, Valid: position > 0}, Position: pgtype.Int4{Int32: position, Valid: position > 0},
}); err != nil { }); err != nil {
fmt.Printf("Warning: playlist collection add failed for %q: %v\n", entry.Name, err) fmt.Printf("Warning: playlist collection add failed for %q: %v\n", entry.Name, err)
continue continue
@@ -350,7 +350,7 @@ func (s *MediaScanner) syncPlaylistCollectionsAfterCreate(ctx context.Context, i
func (s *MediaScanner) upsertPlaylistCollection(ctx context.Context, entry SeriesCandidate) (database.Collections, bool, error) { func (s *MediaScanner) upsertPlaylistCollection(ctx context.Context, entry SeriesCandidate) (database.Collections, bool, error) {
owner := s.adminID owner := s.adminID
if collection, err := s.db.GetCollectionByImportedSeries(ctx, database.GetCollectionByImportedSeriesParams{ if collection, err := s.db.GetCollectionByImportedSeries(ctx, database.GetCollectionByImportedSeriesParams{
UserID: owner, UserID: owner,
ImportedFromSeries: entry.Name, ImportedFromSeries: entry.Name,
}); err == nil { }); err == nil {
return collection, false, nil return collection, false, nil
@@ -1311,12 +1311,21 @@ func (s *MediaScanner) processMediaFile(ctx context.Context, path string) (bool,
if metadata.FileHashInfo != nil { if metadata.FileHashInfo != nil {
sha = metadata.FileHashInfo.FileSHA256 sha = metadata.FileHashInfo.FileSHA256
} }
s.scanStateUpsert(libraryID, &scanEntry{ newEntry := &scanEntry{
ID: createdItem.ID, ID: createdItem.ID,
FilePath: relativePath, FilePath: relativePath,
FileSize: info.Size(), FileSize: info.Size(),
FileSha256: sha, FileSha256: sha,
}) }
s.scanStateUpsert(libraryID, newEntry)
// Seed the sidecar manifest at create time (Part 2 §3): the next
// scan compares instead of backfilling, so an edit is detected on
// its first post-seed scan.
dir := bookSidecarDir(path)
if computed := statSidecarManifest(path, dir); computed != nil && len(computed.Entries) > 0 {
s.storeManifest(ctx, libraryID, newEntry, hashSidecarManifest(path, dir, computed))
}
return true, nil return true, nil
} }
@@ -1418,6 +1427,37 @@ func (s *MediaScanner) processExistingViaScanState(ctx context.Context, entry *s
return false, nil return false, nil
} }
// Manifest-gated sidecar detection (Part 2 §3): a size-stable book
// is changed by EDITING its sidecars. NULL stored manifest →
// backfill only; stat drift with equal hashes → repoint
// reconciliation; content drift → re-extract below.
if s.detectSidecarChange(ctx, entry, libraryID, path) {
fmt.Printf("Sidecar changed, re-extracting metadata: %s\n", path)
existingItem, err := s.getMediaItemByFilePath(ctx, path, libraryID)
if err != nil {
return false, fmt.Errorf("sidecar change re-read failed: %v", err)
}
// Extract once and pass it in, so the playlist sync below sees
// the sidecar's ordering entries (metadata_overrides stay
// protected inside the update path either way).
metadata, extractErr := s.extractMetadata(path)
if extractErr != nil || metadata == nil {
metadata = &MediaMetadata{}
}
if err := s.updateMediaItemWithMetadata(ctx, existingItem, path, metadata); err != nil {
fmt.Printf("Warning: failed to update media item after sidecar change %s: %v\n", path, err)
}
// Playlist sync on the change path (plan Correction 2): a
// "(Reading Order)" entry added to an EXISTING book's sidecar
// still creates its collection — add-if-missing is idempotent.
s.syncPlaylistCollectionsAfterCreate(ctx, entry.ID, metadata.PlaylistSeries)
// Fresh full manifest: the next scan compares against TODAY's
// sidecars, not the stale pre-edit fingerprint.
dir := bookSidecarDir(path)
s.storeManifest(ctx, libraryID, entry, hashSidecarManifest(path, dir, statSidecarManifest(path, dir)))
return false, nil
}
// Self-heal items imported before hashing existed: even an unchanged // Self-heal items imported before hashing existed: even an unchanged
// file gets its hash computed if missing. recomputeHashInfo persists // file gets its hash computed if missing. recomputeHashInfo persists
// it; the next scan's bulk load carries it — no write-through needed // it; the next scan's bulk load carries it — no write-through needed
+255
View File
@@ -0,0 +1,255 @@
package services
// Sidecar manifest fingerprints (faceted plan Part 2 §3): size-stable
// books are changed by EDITING their metadata sidecars, which the old
// size-only detection never noticed. One manifest per book (JSONB in
// media_scan_state) lists every sidecar the extractors consume, so the
// manifest and the extractors can never disagree — the manifest stats
// exactly the names extractCalibreSidecar, extractAudiobookshelfSidecar,
// and findSidecarCover look for.
//
// Change evaluation ladder, in cost order:
// 1. stat pairs all equal → unchanged, ZERO bytes read (steady state);
// 2. a stat pair differs → read + hash that one sidecar:
// hash equal → refresh the stored stat pair (host repoint
// reconciliation: bounded reads, zero re-extraction);
// hash differs → the item changed → re-extract (caller);
// 3. stored manifest NULL → backfill only: hash + store, NO
// re-extraction (deploy-day rule — the first scan after release
// must not look like an mtime flood).
//
// mtime is a CACHE field only — it is never a change signal. Docker
// bind-mount repoints reset every mtime, which is exactly why the hash
// reconciliation ladder exists.
import (
"bookhoard/internal/database"
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"fmt"
"os"
"path/filepath"
"strings"
"github.com/jackc/pgx/v5/pgtype"
)
const sidecarManifestVersion = 1
type manifestSidecar struct {
Name string `json:"n"`
Size int64 `json:"s"`
MTime int64 `json:"m"`
// Sha is the content hash. Present for ALL entries — covers too:
// the repoint reconciliation needs a baseline hash for any stat-
// drifted sidecar, and a hashless cover would turn harmless mtime
// drift into a re-extraction (the exact flood this design kills).
Sha string `json:"h,omitempty"`
}
type sidecarManifest struct {
Version int `json:"v"`
Entries []manifestSidecar `json:"entries"`
}
// coverSidecarNames mirrors findSidecarCover's priority list — the
// manifest must stat the same names the extractor would consume.
var coverSidecarNames = []string{
"cover.jpg", "cover.jpeg", "cover.png", "cover.webp",
"folder.jpg", "folder.jpeg", "folder.png", "folder.webp",
".folder.jpg", ".folder.jpeg", ".folder.png",
}
// bookSidecarDir is the directory whose sidecars feed a book: the book
// file's directory (multifile book roots resolve to themselves because
// file_path IS the directory).
func bookSidecarDir(path string) string {
info, err := os.Stat(path)
if err == nil && info.IsDir() {
return path
}
return filepath.Dir(path)
}
// sidecarNamesForBook lists every sidecar filename the extractors
// consume for a book at path. The book basename covers the
// findSidecarCover "<basename>.<ext>" pattern.
func sidecarNamesForBook(path string) []string {
names := make([]string, 0, len(coverSidecarNames)+5)
names = append(names, "metadata.opf", "metadata.json")
names = append(names, coverSidecarNames...)
base := strings.TrimSuffix(filepath.Base(path), filepath.Ext(path))
if base != "" && base != "." {
for _, ext := range []string{".jpg", ".jpeg", ".png", ".webp"} {
names = append(names, base+ext)
}
}
return names
}
// statSidecarManifest stats every candidate sidecar (ReadDir-free: one
// lstat per candidate name — the walk already visited the directory).
// Returns nil when the directory itself is gone (deleted book mid-scan).
func statSidecarManifest(path, dir string) *sidecarManifest {
manifest := &sidecarManifest{Version: sidecarManifestVersion}
for _, name := range sidecarNamesForBook(path) {
info, err := os.Stat(filepath.Join(dir, name))
if err != nil || info.IsDir() {
continue
}
manifest.Entries = append(manifest.Entries, manifestSidecar{
Name: name,
Size: info.Size(),
MTime: info.ModTime().UnixNano(),
})
}
return manifest
}
// hashSidecarManifest fills the content hashes of every entry — text
// sidecars (KBs) and covers (a one-time bounded read at backfill; the
// repoint ladder needs the baseline). Returns the same manifest for
// chaining.
func hashSidecarManifest(path, dir string, manifest *sidecarManifest) *sidecarManifest {
for i := range manifest.Entries {
entry := &manifest.Entries[i]
sha, err := sha256File(filepath.Join(dir, entry.Name))
if err != nil {
continue // unreadable: stays size-only this pass
}
entry.Sha = sha
}
return manifest
}
func sha256File(path string) (string, error) {
data, err := os.ReadFile(path)
if err != nil {
return "", err
}
sum := sha256.Sum256(data)
return hex.EncodeToString(sum[:]), nil
}
// decodeSidecarManifest parses a stored manifest; nil when absent or
// corrupt (the caller then treats it as a backfill, self-healing any
// shape breakage).
func decodeSidecarManifest(raw []byte) *sidecarManifest {
if len(raw) == 0 {
return nil
}
var m sidecarManifest
if err := json.Unmarshal(raw, &m); err != nil {
return nil
}
return &m
}
// reconcileSidecarManifest runs the evaluation ladder against the
// freshly-statted manifest. Returns the manifest to store and whether
// the item's metadata must be re-extracted:
// - stat pairs all equal → fresh=nil, changed=false (zero reads);
// - stat drift + equal hash → fresh with the refreshed pair
// (repoint reconciliation);
// - hash differs, a sidecar appeared/vanished, or a differing text
// sidecar has no stored hash → changed.
func reconcileSidecarManifest(dir string, stored, computed *sidecarManifest) (fresh *sidecarManifest, changed bool) {
if stored == nil {
return nil, true
}
if len(stored.Entries) != len(computed.Entries) {
return nil, true // sidecar appeared or vanished
}
storedByName := make(map[string]manifestSidecar, len(stored.Entries))
for _, e := range stored.Entries {
storedByName[e.Name] = e
}
// Fast path: stat pairs all equal → unchanged, zero bytes read,
// nothing to store.
drifted := false
for _, cur := range computed.Entries {
old, ok := storedByName[cur.Name]
if !ok || cur.Size != old.Size || cur.MTime != old.MTime {
drifted = true
break
}
}
if !drifted {
return nil, false
}
fresh = &sidecarManifest{Version: sidecarManifestVersion}
for _, cur := range computed.Entries {
old, ok := storedByName[cur.Name]
if !ok {
return nil, true
}
if cur.Size == old.Size && cur.MTime == old.MTime {
fresh.Entries = append(fresh.Entries, old) // untouched: keep the stored hash
continue
}
if old.Sha == "" {
return nil, true // unhashable drift: fall back to re-extract
}
sha, err := sha256File(filepath.Join(dir, cur.Name))
if err != nil || sha != old.Sha {
return nil, true
}
cur.Sha = old.Sha // repoint: content identical, stat pair refreshed
fresh.Entries = append(fresh.Entries, cur)
}
return fresh, false
}
// storeManifest persists the manifest and write-throughs the map entry.
func (s *MediaScanner) storeManifest(ctx context.Context, libraryID pgtype.UUID, entry *scanEntry, manifest *sidecarManifest) {
b, err := json.Marshal(manifest)
if err != nil {
fmt.Printf("Warning: failed to encode sidecar manifest for %s: %v\n", entry.FilePath, err)
return
}
if err := s.db.UpsertMediaScanState(ctx, database.UpsertMediaScanStateParams{
MediaItemID: entry.ID,
Manifest: b,
}); err != nil {
fmt.Printf("Warning: failed to store sidecar manifest for %s: %v\n", entry.FilePath, err)
return
}
entry.Manifest = b
s.scanStateUpsert(libraryID, entry)
}
// detectSidecarChange fingerprints the book's sidecar directory and
// reconciles against the stored manifest. Returns true when the item's
// metadata must be re-extracted (a sidecar's CONTENT changed, or
// sidecars appeared/vanished). Backfill and repoint reconciliation are
// handled here without re-extraction; the caller stores a fresh full
// manifest after a change-path update.
func (s *MediaScanner) detectSidecarChange(ctx context.Context, entry *scanEntry, libraryID pgtype.UUID, path string) bool {
dir := bookSidecarDir(path)
computed := statSidecarManifest(path, dir)
stored := decodeSidecarManifest(entry.Manifest)
if len(computed.Entries) == 0 {
if stored != nil && len(stored.Entries) > 0 {
return true // every sidecar was removed — fall back to embedded
}
return false // never had sidecars: nothing to detect or store
}
if stored == nil {
// Deploy-day backfill: hash + store WITHOUT re-extraction.
s.storeManifest(ctx, libraryID, entry, hashSidecarManifest(path, dir, computed))
return false
}
fresh, changed := reconcileSidecarManifest(dir, stored, computed)
if changed {
return true
}
if fresh != nil {
s.storeManifest(ctx, libraryID, entry, fresh)
}
return false
}
@@ -0,0 +1,136 @@
package services
// Unit tests for the sidecar manifest ladder (faceted plan Part 2 §3):
// stat-pair fast compare, hash-based repoint reconciliation, and the
// change verdicts. Real files only where the hash compare needs them.
import (
"os"
"path/filepath"
"testing"
"time"
)
func TestDecodeSidecarManifest(t *testing.T) {
if decodeSidecarManifest(nil) != nil {
t.Error("NULL manifest decodes to nil (backfill territory)")
}
if decodeSidecarManifest([]byte("{not json")) != nil {
t.Error("corrupt manifest decodes to nil (self-healing backfill)")
}
m := decodeSidecarManifest([]byte(`{"v":1,"entries":[{"n":"metadata.opf","s":10,"m":100,"h":"abc"}]}`))
if m == nil || len(m.Entries) != 1 || m.Entries[0].Sha != "abc" {
t.Errorf("valid manifest round-trip failed: %+v", m)
}
}
func TestReconcileSidecarManifest(t *testing.T) {
dir := t.TempDir()
write := func(name, content string) {
t.Helper()
if err := os.WriteFile(filepath.Join(dir, name), []byte(content), 0644); err != nil {
t.Fatal(err)
}
}
// The book: one hashed sidecar + one size-only cover.
write("metadata.opf", `<opf>original content</opf>`)
originalSha, err := sha256File(filepath.Join(dir, "metadata.opf"))
if err != nil {
t.Fatal(err)
}
write("cover.jpg", "coverbytes")
// stored is seeded exactly the way production backfill does: a real
// stat pass + hash fill. Hand-built fake mtimes would drift on the
// first compare (real files' mtimes never equal a constant).
stored := hashSidecarManifest("book.epub", dir, statSidecarManifest(filepath.Join(dir, "book.epub"), dir))
stat := func() *sidecarManifest {
m := statSidecarManifest(filepath.Join(dir, "book.epub"), dir)
if m == nil || len(m.Entries) < 2 {
t.Fatalf("stat manifest should find both sidecars, got %+v", m)
}
return m
}
// Stat pairs identical → unchanged, zero reads, nothing to store.
fresh, changed := reconcileSidecarManifest(dir, stored, stat())
if changed || fresh != nil {
t.Errorf("identical stats must be unchanged, got changed=%v fresh=%v", changed, fresh)
}
// Repoint: mtime drifts, content identical → reconciled fresh
// manifest with the refreshed stat pair and the KEPT hash, no change.
if err := os.Chtimes(filepath.Join(dir, "metadata.opf"), time.Now(), time.Now().Add(time.Minute)); err != nil {
t.Fatal(err)
}
fresh, changed = reconcileSidecarManifest(dir, stored, stat())
if changed {
t.Error("mtime drift with identical content is repoint reconciliation, not a change")
}
if fresh == nil || len(fresh.Entries) != 2 {
t.Fatalf("repoint must produce a refreshed manifest, got %+v", fresh)
}
if fresh.Entries[0].Sha != originalSha {
t.Error("repointed entry keeps its stored hash")
}
if fresh.Entries[0].MTime == stored.Entries[0].MTime {
t.Error("repointed entry refreshes its stat pair")
}
// Same-size edit: drift + hash differs → changed.
write("metadata.opf", `<opf>mutated XYY content</opf>`) // +1 byte, and hash differs
_, changed = reconcileSidecarManifest(dir, stored, stat())
if !changed {
t.Error("content drift must be a change")
}
// Same-LENGTH edit (the size-only detection blind spot) → changed.
write("metadata.opf", `<opf>original Xontent</opf>`)
_, changed = reconcileSidecarManifest(dir, stored, stat())
if !changed {
t.Error("same-size content drift must be a change (hash compare)")
}
// New sidecar appears → changed.
write("metadata.json", `{"title":"X"}`)
_, changed = reconcileSidecarManifest(dir, stored, stat())
if !changed {
t.Error("an appearing sidecar must be a change")
}
// Sidecar vanishes AND everything else returns to the stored state
// (opf restored; its mtime drifts but the hash proves equality) →
// the manifest reconciles, not a change.
if err := os.Remove(filepath.Join(dir, "metadata.json")); err != nil {
t.Fatal(err)
}
write("metadata.opf", `<opf>original content</opf>`)
fresh, changed = reconcileSidecarManifest(dir, stored, stat())
if changed {
t.Errorf("returning to the stored state is not a change: %+v", fresh)
}
if fresh == nil || len(fresh.Entries) != 2 {
t.Errorf("reconciled manifest covers both sidecars: %+v", fresh)
}
}
// An unhashable drift (a stored entry with no baseline hash — legacy
// or partially-written manifest) is a change: the manifest cannot prove
// content equality for it.
func TestReconcileUnhashableDrift(t *testing.T) {
dir := t.TempDir()
stored := &sidecarManifest{Version: 1, Entries: []manifestSidecar{
{Name: "cover.jpg", Size: 10, MTime: 1000},
}}
computed := &sidecarManifest{Version: 1, Entries: []manifestSidecar{
{Name: "cover.jpg", Size: 11, MTime: 1001},
}}
fresh, changed := reconcileSidecarManifest(dir, stored, computed)
if !changed {
t.Error("stat drift without a stored baseline hash must be a change")
}
if fresh != nil {
t.Error("a change never returns a fresh manifest (caller builds one)")
}
}