diff --git a/cmd/server/tests/scan_state_test.go b/cmd/server/tests/scan_state_test.go new file mode 100644 index 00000000..6699c66b --- /dev/null +++ b/cmd/server/tests/scan_state_test.go @@ -0,0 +1,119 @@ +package main + +// Bulk scan-state tests (faceted plan Part 2 Phase 3): the scanner's +// one-query-per-library bulk load replaces the per-file +// GetMediaItemByFilePath flood. Behavioral equivalence asserted at the +// container level: unchanged rescans skip (updated_at stable), items +// created OUTSIDE the scanner (API create path) are seen by the bulk +// load, and size changes still take the update path. + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "os" + "path/filepath" + "testing" + "time" + + "bookhoard/internal/database" + + "github.com/jackc/pgx/v5/pgtype" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestScanStateBulkLoad(t *testing.T) { + setup := setupTestServer(t) + t.Cleanup(func() { _ = setup.Close() }) + + admin, _ := moveTestAdminID(t, setup) + libID, folder := createTypedTestLibrary(t, setup, fmt.Sprintf("Test Scan State %d", time.Now().UnixNano()), "ebooks") + t.Cleanup(func() { deleteLibraries(t, setup, libID) }) + + // Two scanner-created fixture books. + epubA := filepath.Join(folder, "Author A", "State Book A", "book.epub") + require.NoError(t, os.MkdirAll(filepath.Dir(epubA), 0755)) + require.NoError(t, os.WriteFile(epubA, []byte("state book A content"), 0644)) + epubB := filepath.Join(folder, "Author B", "State Book B", "book.epub") + require.NoError(t, os.MkdirAll(filepath.Dir(epubB), 0755)) + require.NoError(t, os.WriteFile(epubB, []byte("state book B content"), 0644)) + + scanTestFolder(t, setup, admin, folder) + + items := listLibraryAudioItems(t, setup, libID) + require.Len(t, items, 2, "both books ingested") + var updatedA time.Time + for _, item := range items { + switch item.Title { + case "State Book A": + updatedA = item.UpdatedAt.Time + case "State Book B": + default: + t.Errorf("unexpected item %q", item.Title) + } + } + require.False(t, updatedA.IsZero()) + + // Unchanged rescan: the skip path must not touch the rows. + scanTestFolder(t, setup, admin, folder) + items = listLibraryAudioItems(t, setup, libID) + require.Len(t, items, 2, "no duplicates on rescan") + for _, item := range items { + if item.Title == "State Book A" { + assert.Equal(t, updatedA, item.UpdatedAt.Time, "unchanged rescan leaves updated_at stable") + } + } + + // A row created by the API create path (NOT by a scan) must be seen + // by the bulk load: the rescan skips it instead of re-creating. The + // file must really exist (the archive pass missing-checks rows + // whose files are absent). + apiFile := filepath.Join(folder, "Author C", "API Made Book", "book.epub") + require.NoError(t, os.MkdirAll(filepath.Dir(apiFile), 0755)) + require.NoError(t, os.WriteFile(apiFile, []byte("api made book content"), 0644)) + apiStat, err := os.Stat(apiFile) + require.NoError(t, err) + created, err := setup.DB.CreateMediaItem(context.Background(), database.CreateMediaItemParams{ + LibraryID: libID, + Title: "API Made Book", + Authors: []string{"API Author"}, + FilePath: "Author C/API Made Book/book.epub", + FileSize: pgtype.Int8{Int64: apiStat.Size(), Valid: true}, + MimeType: pgtype.Text{String: "application/epub+zip", Valid: true}, + }) + require.NoError(t, err) + // Stamp the file's real sha the way the scanner does — otherwise the + // skip path's one-time hash self-heal fires and bumps updated_at. + // The identifiers UPDATE also bumps updated_at, so read the row + // AFTER the stamp for the baseline. + apiSha := sha256.Sum256([]byte("api made book content")) + stamped, err := setup.DB.UpdateMediaItemIdentifiers(context.Background(), database.UpdateMediaItemIdentifiersParams{ + ID: created.ID, + FileSha256: pgtype.Text{String: hex.EncodeToString(apiSha[:]), Valid: true}, + }) + require.NoError(t, err) + apiUpdated := stamped.UpdatedAt.Time + + scanTestFolder(t, setup, admin, folder) + items = listLibraryAudioItems(t, setup, libID) + require.Len(t, items, 3, "API-created item survives the rescan as-is") + for _, item := range items { + if item.Title == "API Made Book" { + assert.Equal(t, apiUpdated, item.UpdatedAt.Time, "bulk-loaded state sees scanner-external rows; skip path leaves them alone") + } + } + + // Size change: the update path still fires (with a fresh full-row + // read + hash recompute, then a write-through of the new size). + require.NoError(t, os.WriteFile(epubA, []byte("state book A content — now longer"), 0644)) + scanTestFolder(t, setup, admin, folder) + items = listLibraryAudioItems(t, setup, libID) + require.Len(t, items, 3, "size change updates in place, never duplicates") + for _, item := range items { + if item.Title == "State Book A" { + assert.NotEqual(t, updatedA, item.UpdatedAt.Time, "size change takes the update path") + } + } +} diff --git a/database/schema/schema.sql b/database/schema/schema.sql index b89586e0..994ed978 100644 --- a/database/schema/schema.sql +++ b/database/schema/schema.sql @@ -1247,6 +1247,21 @@ CREATE TABLE IF NOT EXISTS media_item_formats ( CREATE INDEX IF NOT EXISTS idx_media_item_formats_media ON media_item_formats(media_item_id, format_type); CREATE INDEX IF NOT EXISTS idx_media_item_formats_sha256 ON media_item_formats(file_sha256); +-- media_scan_state: the scanner's per-item fingerprint cache (faceted +-- plan Part 2). Deliberately NOT a media_items column: the library- +-- scoped media list emits raw sqlc rows as JSON, and scanner +-- bookkeeping must never leak into API payloads (nor churn the model +-- or the contract pins). One row per item; cascade on delete. +-- manifest shape v1: +-- {"v":1,"entries":[{"n":"metadata.opf","s":3013013,"m":1697500000,"h":""}, ...]} +-- n basename, s size, m mtime unix (CACHE ONLY — never a change +-- signal), h sha256 for text sidecars (omitted for cover images). +-- "ev" (extractor version) reserved for parser-fix re-extraction. +CREATE TABLE IF NOT EXISTS media_scan_state ( + media_item_id UUID PRIMARY KEY REFERENCES media_items(id) ON DELETE CASCADE, + manifest JSONB +); + -- Create device_file_aliases table (track file paths per device for cross-device matching) CREATE TABLE IF NOT EXISTS device_file_aliases ( id UUID PRIMARY KEY DEFAULT gen_random_uuid(), diff --git a/internal/database/models.go b/internal/database/models.go index e1d77ff2..e7c31eb2 100644 --- a/internal/database/models.go +++ b/internal/database/models.go @@ -372,6 +372,11 @@ type MediaReviews struct { UpdatedAt pgtype.Timestamptz `db:"updated_at" json:"updated_at"` } +type MediaScanState struct { + MediaItemID pgtype.UUID `db:"media_item_id" json:"media_item_id"` + Manifest []byte `db:"manifest" json:"manifest"` +} + type OpdsTokens struct { ID pgtype.UUID `db:"id" json:"id"` DeviceID pgtype.UUID `db:"device_id" json:"device_id"` diff --git a/internal/database/querier.go b/internal/database/querier.go index 0aef411b..d171b18f 100644 --- a/internal/database/querier.go +++ b/internal/database/querier.go @@ -400,6 +400,17 @@ type Querier interface { GetRefreshToken(ctx context.Context, token pgtype.UUID) (GetRefreshTokenRow, error) GetSavedFilterByID(ctx context.Context, arg GetSavedFilterByIDParams) (SavedFilters, error) GetSavedFilters(ctx context.Context, arg GetSavedFiltersParams) ([]SavedFilters, error) + // GetScanStateByLibrary: the scanner's ONE bulk query per library scan + // (faceted plan Part 2 §2) — every item's existence + scanner-owned + // facts + sidecar manifest in a single pass. Replaces the per-file + // GetMediaItemByFilePath flood (~3,094 queries per scan in prod). + // Lean projection: metadata blobs (audio_metadata, chapter_metadata, + // authors, description...) are NEVER loaded — the skip path never + // needs them, and the change path re-reads the full row. + // archived_at + missing_scan_count + file_sha256 ride along so the + // skip path's archive-restore and hash self-heal checks stay + // DB-free too. + GetScanStateByLibrary(ctx context.Context, libraryID pgtype.UUID) ([]GetScanStateByLibraryRow, error) // Scoped to ONE library: same-name series in different libraries are // different stories (manga adaptations, LN splits). Callers must pass a // library_id and verify the requester can see that library. diff --git a/internal/database/queries.sql.go b/internal/database/queries.sql.go index c92e546e..007d5e86 100644 --- a/internal/database/queries.sql.go +++ b/internal/database/queries.sql.go @@ -8557,6 +8557,68 @@ func (q *Queries) GetSavedFilters(ctx context.Context, arg GetSavedFiltersParams return items, nil } +const GetScanStateByLibrary = `-- name: GetScanStateByLibrary :many +SELECT + mi.id, + mi.file_path, + mi.file_size, + mi.archived_at, + mi.missing_scan_count, + mi.file_sha256, + s.manifest +FROM media_items mi +LEFT JOIN media_scan_state s ON s.media_item_id = mi.id +WHERE mi.library_id = $1 +` + +type GetScanStateByLibraryRow struct { + ID pgtype.UUID `db:"id" json:"id"` + FilePath string `db:"file_path" json:"file_path"` + FileSize pgtype.Int8 `db:"file_size" json:"file_size"` + ArchivedAt pgtype.Timestamptz `db:"archived_at" json:"archived_at"` + MissingScanCount int32 `db:"missing_scan_count" json:"missing_scan_count"` + FileSha256 pgtype.Text `db:"file_sha256" json:"file_sha256"` + Manifest []byte `db:"manifest" json:"manifest"` +} + +// GetScanStateByLibrary: the scanner's ONE bulk query per library scan +// (faceted plan Part 2 §2) — every item's existence + scanner-owned +// facts + sidecar manifest in a single pass. Replaces the per-file +// GetMediaItemByFilePath flood (~3,094 queries per scan in prod). +// Lean projection: metadata blobs (audio_metadata, chapter_metadata, +// authors, description...) are NEVER loaded — the skip path never +// needs them, and the change path re-reads the full row. +// archived_at + missing_scan_count + file_sha256 ride along so the +// skip path's archive-restore and hash self-heal checks stay +// DB-free too. +func (q *Queries) GetScanStateByLibrary(ctx context.Context, libraryID pgtype.UUID) ([]GetScanStateByLibraryRow, error) { + rows, err := q.db.Query(ctx, GetScanStateByLibrary, libraryID) + if err != nil { + return nil, err + } + defer rows.Close() + items := []GetScanStateByLibraryRow{} + for rows.Next() { + var i GetScanStateByLibraryRow + if err := rows.Scan( + &i.ID, + &i.FilePath, + &i.FileSize, + &i.ArchivedAt, + &i.MissingScanCount, + &i.FileSha256, + &i.Manifest, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + const GetSeriesBooks = `-- name: GetSeriesBooks :many SELECT id, library_id, title, subtitle, author, isbn, description, file_path, file_size, mime_type, cover_image_path, series, series_number, tags, narrators, authors, authors_search, publishers, publishers_search, abridged, explicit, asin, date_published, publisher, contributors, language, edition, page_count, genre, copyright_year, goodreads_id, openlibrary_id, google_books_id, added_by_admin_id, created_at, imported_at, updated_at, format_group, format_mimetype, is_reflowable, has_fixed_layout, total_characters, chapter_count, entitlement_id, revision_number, kobo_content_id, kobo_metadata, manga_type, reading_direction, series_count, volume, imprint, age_rating, web_url, story_arc, is_black_and_white, metadata_notes, community_rating, alternate_info, scan_information, summary, metadata_overrides, missing_scan_count, archived_at, chapter_metadata, library_type_name, audio_metadata, tags_search, contributors_search, narrators_search, file_sha256, opf_identifier, opf_uuid, hash_confidence FROM media_items WHERE series = $1 diff --git a/internal/database/queries/queries.sql b/internal/database/queries/queries.sql index d30ed23f..0f9d24b1 100644 --- a/internal/database/queries/queries.sql +++ b/internal/database/queries/queries.sql @@ -429,6 +429,29 @@ DELETE FROM media_items WHERE id = $1; -- name: GetMediaItemByFilePath :one SELECT * FROM media_items WHERE file_path = $1 AND library_id = $2; +-- GetScanStateByLibrary: the scanner's ONE bulk query per library scan +-- (faceted plan Part 2 §2) — every item's existence + scanner-owned +-- facts + sidecar manifest in a single pass. Replaces the per-file +-- GetMediaItemByFilePath flood (~3,094 queries per scan in prod). +-- Lean projection: metadata blobs (audio_metadata, chapter_metadata, +-- authors, description...) are NEVER loaded — the skip path never +-- needs them, and the change path re-reads the full row. +-- archived_at + missing_scan_count + file_sha256 ride along so the +-- skip path's archive-restore and hash self-heal checks stay +-- DB-free too. +-- name: GetScanStateByLibrary :many +SELECT + mi.id, + mi.file_path, + mi.file_size, + mi.archived_at, + mi.missing_scan_count, + mi.file_sha256, + s.manifest +FROM media_items mi +LEFT JOIN media_scan_state s ON s.media_item_id = mi.id +WHERE mi.library_id = sqlc.narg('library_id'); + -- name: GetMediaItemByFilePathAnyLibrary :one SELECT * FROM media_items WHERE file_path = $1 LIMIT 1; diff --git a/internal/services/media_scanner.go b/internal/services/media_scanner.go index 3ea540f2..5893a0f0 100644 --- a/internal/services/media_scanner.go +++ b/internal/services/media_scanner.go @@ -158,6 +158,30 @@ type MediaScanner struct { // skip instead of reprocessing. Reset at the top of ScanFolders; the // archive pass reuses it to keep book roots off the missing list. audioBookRoots map[string]bool + + // Bulk scan state (faceted plan Part 2 §2): one GetScanStateByLibrary + // load per library replaces the per-file GetMediaItemByFilePath flood. + // Keyed by "libraryID\x00relative-path" (multi-folder libraries make a + // bare relative path ambiguous); nil map = per-file lookups (the + // watcher's scanDirectory path). Guarded by its own mutex — scans can + // overlap on one instance. + scanState map[string]*scanEntry + scanStateMu sync.RWMutex + folderLibrary map[string]pgtype.UUID +} + +// scanEntry is one media item's scanner-owned facts, bulk-loaded per +// scan: everything the unchanged-skip path needs, nothing else. The +// manifest field feeds Phase 4's sidecar change detection (NULL until +// the first manifest pass writes it). +type scanEntry struct { + ID pgtype.UUID + FilePath string // relative, as stored + FileSize int64 + Archived bool + MissingCount int32 + FileSha256 string + Manifest []byte } // CalibreOPFMetadata represents intermediate parsed metadata from Calibre metadata.opf files @@ -583,6 +607,12 @@ func (s *MediaScanner) ScanFolders(ctx context.Context) error { s.newItems = 0 s.errors = 0 s.audioBookRoots = make(map[string]bool) + // Bulk scan state: per-run reset; one load per distinct library. + s.scanStateMu.Lock() + s.scanState = make(map[string]*scanEntry) + s.folderLibrary = make(map[string]pgtype.UUID) + s.scanStateMu.Unlock() + loadedLibraries := make(map[string]bool) for _, folder := range s.folders { if ctx.Err() != nil { @@ -617,7 +647,25 @@ func (s *MediaScanner) ScanFolders(ctx context.Context) error { continue } - err := filepath.WalkDir(folder, func(path string, d fs.DirEntry, err error) error { + // Resolve this folder's library once and cache the bulk scan + // state for it (one query per library, not per file). + lib, err := s.db.GetLibraryByFolder(ctx, folder) + if err != nil { + fmt.Printf("Warning: no library found for folder %s, skipping: %v\n", folder, err) + s.errors++ + continue + } + s.scanStateMu.Lock() + s.folderLibrary[folder] = lib.LibraryID + s.scanStateMu.Unlock() + if !loadedLibraries[lib.LibraryID.String()] { + if err := s.loadScanState(ctx, lib.LibraryID); err != nil { + fmt.Printf("Warning: bulk scan-state load failed for %s, falling back to per-file lookups: %v\n", lib.Name, err) + } + loadedLibraries[lib.LibraryID.String()] = true + } + + err = filepath.WalkDir(folder, func(path string, d fs.DirEntry, err error) error { if ctx.Err() != nil { return fs.SkipAll } @@ -926,20 +974,34 @@ func (s *MediaScanner) processMediaFile(ctx context.Context, path string) (bool, fileModTime := info.ModTime() - // Find library for this file's folder + // Find library for this file's folder (cached per scan run — the + // folder loop used to issue a GetLibraryByFolder per file too). var libraryID pgtype.UUID if s.defaultLibraryID.Valid { libraryID = s.defaultLibraryID } else { + s.scanStateMu.RLock() + folderLibrary := s.folderLibrary + s.scanStateMu.RUnlock() for _, folder := range s.folders { - if strings.HasPrefix(path, folder) { - lib, err := s.db.GetLibraryByFolder(ctx, folder) - if err != nil { - return false, fmt.Errorf("failed to find library for folder %s: %v", folder, err) - } - libraryID = lib.LibraryID + if !strings.HasPrefix(path, folder) { + continue + } + if libID, cached := folderLibrary[folder]; cached { + libraryID = libID break } + lib, err := s.db.GetLibraryByFolder(ctx, folder) + if err != nil { + return false, fmt.Errorf("failed to find library for folder %s: %v", folder, err) + } + libraryID = lib.LibraryID + s.scanStateMu.Lock() + if s.folderLibrary != nil { + s.folderLibrary[folder] = lib.LibraryID + } + s.scanStateMu.Unlock() + break } } @@ -947,53 +1009,21 @@ func (s *MediaScanner) processMediaFile(ctx context.Context, path string) (bool, return false, fmt.Errorf("no library found for file path: %s", path) } - // Check if media item already exists in database - existingItem, err := s.getMediaItemByFilePath(ctx, path, libraryID) - if err == nil { - fmt.Printf("Media item already exists in database: %s (size: %d vs %d)\n", path, existingItem.FileSize.Int64, info.Size()) - - // The file is back on disk: lift any archive/missing state so the - // item reappears in libraries and future missing scans start fresh. - if existingItem.ArchivedAt.Valid || existingItem.MissingScanCount > 0 { - if err := s.db.ClearMediaItemArchive(ctx, existingItem.ID); err != nil { - fmt.Printf("Warning: failed to unarchive media item %s: %v\n", existingItem.FilePath, err) - } else if existingItem.ArchivedAt.Valid { - fmt.Printf("[ARCHIVE] Restored from archive, file is back: %s\n", existingItem.FilePath) - s.logger.LogDelete(fmt.Sprintf("[ARCHIVE] Restored archived item '%s' (file returned at %s)", existingItem.Title, existingItem.FilePath)) - } + // Bulk scan state (faceted plan Part 2 §2): a map hit answers + // "exists + size + flags" with zero DB reads; a map miss in bulk + // mode means NEW (the map holds every row of the library). A nil + // map (watcher's scanDirectory path) keeps the per-file lookup. + if entry, ok := s.scanStateLookup(libraryID, path); ok { + return s.processExistingViaScanState(ctx, entry, libraryID, path, info) + } + if s.scanState == nil { + existingItem, err := s.getMediaItemByFilePath(ctx, path, libraryID) + if err == nil { + return s.processExistingRow(ctx, existingItem, libraryID, path, info) + } else if !errors.Is(err, pgx.ErrNoRows) { + fmt.Printf("Database error checking media item existence: %v\n", err) + return false, fmt.Errorf("failed to check if media item exists: %v", err) } - - // If force rescan is enabled, always re-process - if s.forceRescan { - fmt.Printf("Force rescan enabled, updating existing media item: %s\n", path) - // Use UPDATE instead of DELETE+INSERT to preserve created_at - if err := s.updateMediaItem(ctx, existingItem, path); err != nil { - fmt.Printf("Warning: failed to update existing media item: %v\n", err) - } - // Recompute hash identifiers too - a force rescan is the admin's - // backfill tool and must refresh stale or missing hashes. - s.recomputeHashInfo(ctx, existingItem.ID, libraryID, path) - return false, nil - } else { - // Normal behavior: check if file has changed (by size) - if existingItem.FileSize.Int64 != info.Size() { - fmt.Printf("File size changed, updating media item: %s\n", path) - _ = s.updateMediaItem(ctx, existingItem, path) - // The bytes changed, so any stored hash is stale. - s.recomputeHashInfo(ctx, existingItem.ID, libraryID, path) - return false, nil - } - // Self-heal items imported before hashing existed: even an unchanged - // file gets its hash computed if missing. - if !existingItem.FileSha256.Valid || existingItem.FileSha256.String == "" { - s.recomputeHashInfo(ctx, existingItem.ID, libraryID, path) - } - fmt.Printf("Media item already exists with same size, skipping: %s\n", path) - return false, nil - } - } else if !errors.Is(err, pgx.ErrNoRows) { - fmt.Printf("Database error checking media item existence: %v\n", err) - return false, fmt.Errorf("failed to check if media item exists: %v", err) } fmt.Printf("Media item does not exist in database, creating new entry: %s\n", path) @@ -1273,9 +1303,131 @@ func (s *MediaScanner) processMediaFile(ctx context.Context, path string) (bool, // A2c §2.4: obvious playlist sidecar series become curated collections. s.syncPlaylistCollectionsAfterCreate(ctx, createdItem.ID, metadata.PlaylistSeries) + + // Write-through (faceted plan Part 2 §2): the create's facts land in + // the per-run map so later passes in this scan read post-insert + // state. Manifest stays NULL — Phase 4's manifest pass writes it. + sha := "" + if metadata.FileHashInfo != nil { + sha = metadata.FileHashInfo.FileSHA256 + } + s.scanStateUpsert(libraryID, &scanEntry{ + ID: createdItem.ID, + FilePath: relativePath, + FileSize: info.Size(), + FileSha256: sha, + }) return true, nil } +// processExistingRow is the nil-map (per-file lookup) existing-item +// branch, preserved verbatim from the pre-bulk-load behavior so the +// watcher's scanDirectory path and any bulk-load failure keep working +// exactly as before. +func (s *MediaScanner) processExistingRow(ctx context.Context, existingItem database.MediaItems, libraryID pgtype.UUID, path string, info os.FileInfo) (bool, error) { + fmt.Printf("Media item already exists in database: %s (size: %d vs %d)\n", path, existingItem.FileSize.Int64, info.Size()) + + // The file is back on disk: lift any archive/missing state so the + // item reappears in libraries and future missing scans start fresh. + if existingItem.ArchivedAt.Valid || existingItem.MissingScanCount > 0 { + if err := s.db.ClearMediaItemArchive(ctx, existingItem.ID); err != nil { + fmt.Printf("Warning: failed to unarchive media item %s: %v\n", existingItem.FilePath, err) + } else if existingItem.ArchivedAt.Valid { + fmt.Printf("[ARCHIVE] Restored from archive, file is back: %s\n", existingItem.FilePath) + s.logger.LogDelete(fmt.Sprintf("[ARCHIVE] Restored archived item '%s' (file returned at %s)", existingItem.Title, existingItem.FilePath)) + } + } + + // If force rescan is enabled, always re-process + if s.forceRescan { + fmt.Printf("Force rescan enabled, updating existing media item: %s\n", path) + // Use UPDATE instead of DELETE+INSERT to preserve created_at + if err := s.updateMediaItem(ctx, existingItem, path); err != nil { + fmt.Printf("Warning: failed to update existing media item: %v\n", err) + } + // Recompute hash identifiers too - a force rescan is the admin's + // backfill tool and must refresh stale or missing hashes. + s.recomputeHashInfo(ctx, existingItem.ID, libraryID, path) + return false, nil + } + // Normal behavior: check if file has changed (by size) + if existingItem.FileSize.Int64 != info.Size() { + fmt.Printf("File size changed, updating media item: %s\n", path) + _ = s.updateMediaItem(ctx, existingItem, path) + // The bytes changed, so any stored hash is stale. + s.recomputeHashInfo(ctx, existingItem.ID, libraryID, path) + return false, nil + } + // Self-heal items imported before hashing existed: even an unchanged + // file gets its hash computed if missing. + if !existingItem.FileSha256.Valid || existingItem.FileSha256.String == "" { + s.recomputeHashInfo(ctx, existingItem.ID, libraryID, path) + } + fmt.Printf("Media item already exists with same size, skipping: %s\n", path) + return false, nil +} + +// processExistingViaScanState is the bulk-state fast path (faceted plan +// Part 2 §2): archive restore + size compare with ZERO DB reads on the +// steady skip path. The rare branches (force rescan, size change) +// re-read the full row — the change path is allowed to be expensive. +// Write-through keeps later passes in the same run honest. +func (s *MediaScanner) processExistingViaScanState(ctx context.Context, entry *scanEntry, libraryID pgtype.UUID, path string, info os.FileInfo) (bool, error) { + if entry.Archived || entry.MissingCount > 0 { + if err := s.db.ClearMediaItemArchive(ctx, entry.ID); err != nil { + fmt.Printf("Warning: failed to unarchive media item %s: %v\n", entry.FilePath, err) + } else { + if entry.Archived { + fmt.Printf("[ARCHIVE] Restored from archive, file is back: %s\n", entry.FilePath) + s.logger.LogDelete(fmt.Sprintf("[ARCHIVE] Restored archived item '%s' (file returned at %s)", entry.FilePath, entry.FilePath)) + } + entry.Archived = false + entry.MissingCount = 0 + s.scanStateUpsert(libraryID, entry) + } + } + + if s.forceRescan { + fmt.Printf("Force rescan enabled, updating existing media item: %s\n", path) + existingItem, err := s.getMediaItemByFilePath(ctx, path, libraryID) + if err != nil { + return false, fmt.Errorf("force rescan re-read failed: %v", err) + } + if err := s.updateMediaItem(ctx, existingItem, path); err != nil { + fmt.Printf("Warning: failed to update existing media item: %v\n", err) + } + // Recompute hash identifiers too - a force rescan is the admin's + // backfill tool and must refresh stale or missing hashes. + s.recomputeHashInfo(ctx, entry.ID, libraryID, path) + return false, nil + } + + if entry.FileSize != info.Size() { + fmt.Printf("File size changed, updating media item: %s\n", path) + existingItem, err := s.getMediaItemByFilePath(ctx, path, libraryID) + if err != nil { + return false, fmt.Errorf("change-path re-read failed: %v", err) + } + if err := s.updateMediaItem(ctx, existingItem, path); err != nil { + fmt.Printf("Warning: failed to update media item: %v\n", err) + } + // The bytes changed, so any stored hash is stale. + s.recomputeHashInfo(ctx, entry.ID, libraryID, path) + entry.FileSize = info.Size() + s.scanStateUpsert(libraryID, entry) + return false, nil + } + + // Self-heal items imported before hashing existed: even an unchanged + // file gets its hash computed if missing. recomputeHashInfo persists + // it; the next scan's bulk load carries it — no write-through needed + // for a per-run map. + if entry.FileSha256 == "" { + s.recomputeHashInfo(ctx, entry.ID, libraryID, path) + } + return false, nil +} + // applyTitleDerivedSeries fills a missing series and/or ordinal from the // item's own subtitle or title text when they follow the unambiguous // Audible/Amazon ", Book N" convention. Lowest-precedence source: @@ -3755,6 +3907,77 @@ func (s *MediaScanner) RescanMediaItem(ctx context.Context, mediaItemID pgtype.U return nil } +// loadScanState bulk-loads one library's scan state into the per-run +// map (faceted plan Part 2 §2). One query replaces the per-file +// GetMediaItemByFilePath flood for this scan. +func (s *MediaScanner) loadScanState(ctx context.Context, libraryID pgtype.UUID) error { + rows, err := s.db.GetScanStateByLibrary(ctx, libraryID) + if err != nil { + return err + } + s.scanStateMu.Lock() + defer s.scanStateMu.Unlock() + if s.scanState == nil { + s.scanState = make(map[string]*scanEntry, len(rows)) + } + for _, row := range rows { + entry := &scanEntry{ + ID: row.ID, + FilePath: row.FilePath, + FileSize: row.FileSize.Int64, + Archived: row.ArchivedAt.Valid, + MissingCount: row.MissingScanCount, + FileSha256: row.FileSha256.String, + Manifest: row.Manifest, + } + s.scanState[scanStateKey(libraryID, row.FilePath)] = entry + } + return nil +} + +// scanStateKey composites library + relative path — multi-folder +// libraries make a bare relative path ambiguous. +func scanStateKey(libraryID pgtype.UUID, relPath string) string { + return libraryID.String() + "\x00" + relPath +} + +// scanStateLookup returns the bulk-loaded entry for an absolute file +// path, ok=false when the map is nil (non-bulk scan paths), the library +// is unknown, or the item is new. Caller resolves the library first +// (same precedence the per-file lookup used). +func (s *MediaScanner) scanStateLookup(libraryID pgtype.UUID, path string) (*scanEntry, bool) { + s.scanStateMu.RLock() + scanState := s.scanState + s.scanStateMu.RUnlock() + if scanState == nil || !libraryID.Valid { + return nil, false + } + relPath := s.getRelativePath(path) + entry, ok := scanState[scanStateKey(libraryID, relPath)] + return entry, ok +} + +// scanStateUpsert writes a known-good entry back into the per-run map +// (write-through: later passes in the same run read post-update state). +func (s *MediaScanner) scanStateUpsert(libraryID pgtype.UUID, entry *scanEntry) { + s.scanStateMu.Lock() + defer s.scanStateMu.Unlock() + if s.scanState == nil { + return + } + s.scanState[scanStateKey(libraryID, entry.FilePath)] = entry +} + +// scanStateRemove drops an entry (the row's file_path moved elsewhere). +func (s *MediaScanner) scanStateRemove(libraryID pgtype.UUID, relPath string) { + s.scanStateMu.Lock() + defer s.scanStateMu.Unlock() + if s.scanState == nil { + return + } + delete(s.scanState, scanStateKey(libraryID, relPath)) +} + func (s *MediaScanner) getMediaItemByFilePath(ctx context.Context, filePath string, libraryID pgtype.UUID) (database.MediaItems, error) { return s.db.GetMediaItemByFilePath(ctx, database.GetMediaItemByFilePathParams{ FilePath: s.getRelativePath(filePath),