perf(scanner): bulk scan-state load — one query per library scan
Part 2 of the faceted plan, Phase 3 (steps 1-2). The periodic scan
issued one GetMediaItemByFilePath per scannable file (~3,094 queries
per scan in prod, nearly all answering 'unchanged, skip') plus one
GetLibraryByFolder per file for folder resolution.
- NEW media_scan_state table: the scanner's per-item fingerprint
cache. Deliberately NOT a media_items column — the library-scoped
list emits raw sqlc rows as JSON (7537063c), so scanner bookkeeping
must never leak into API payloads nor churn the model or pins.
Manifest JSONB shape v1 documented in the schema (n/s/m/h; ev
reserved for the Phase 4+ extractor-version mechanism).
- GetScanStateByLibrary: lean bulk projection (id, file_path,
file_size, archived_at, missing_scan_count, file_sha256, manifest).
archived/hash columns ride along so the skip path's archive-restore
and hash self-heal checks stay DB-free too (refines the plan's
4-column sketch; same goal, narrower invariant).
- ScanFolders: per-run reset, per-folder library cache, one bulk load
per distinct library. processMediaFile: map hit answers existence +
size + flags with zero DB reads; force-rescan and size-change
branches re-read the full row (the change path may be expensive);
write-through on create/size-change keeps later passes honest.
- Multifile folder/zip books keep their full-row lookup: it is one
query per BOOK ROOT (not per track) and needs audio_metadata for
the duration-probe check. They join the map in Phase 4, when the
manifest compare replaces their fingerprint check.
- nil map (watcher's scanDirectory path, bulk-load failure) falls
back to per-file lookups — behavior-preserving; the failure case
logged once per scan.
Test: TestScanStateBulkLoad (unchanged rescan leaves updated_at
stable; API-created rows are seen by the bulk load; size changes
still update in place). Container runner.
This commit is contained in:
1 parent
bbedbfb13f
commit
693053a9f5
7 files changed
+512
-54
No files matched your search
@@ -0,0 +1,119 @@
|
||||
package main
|
||||
|
||||
// Bulk scan-state tests (faceted plan Part 2 Phase 3): the scanner's
|
||||
// one-query-per-library bulk load replaces the per-file
|
||||
// GetMediaItemByFilePath flood. Behavioral equivalence asserted at the
|
||||
// container level: unchanged rescans skip (updated_at stable), items
|
||||
// created OUTSIDE the scanner (API create path) are seen by the bulk
|
||||
// load, and size changes still take the update path.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"bookhoard/internal/database"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgtype"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestScanStateBulkLoad(t *testing.T) {
|
||||
setup := setupTestServer(t)
|
||||
t.Cleanup(func() { _ = setup.Close() })
|
||||
|
||||
admin, _ := moveTestAdminID(t, setup)
|
||||
libID, folder := createTypedTestLibrary(t, setup, fmt.Sprintf("Test Scan State %d", time.Now().UnixNano()), "ebooks")
|
||||
t.Cleanup(func() { deleteLibraries(t, setup, libID) })
|
||||
|
||||
// Two scanner-created fixture books.
|
||||
epubA := filepath.Join(folder, "Author A", "State Book A", "book.epub")
|
||||
require.NoError(t, os.MkdirAll(filepath.Dir(epubA), 0755))
|
||||
require.NoError(t, os.WriteFile(epubA, []byte("state book A content"), 0644))
|
||||
epubB := filepath.Join(folder, "Author B", "State Book B", "book.epub")
|
||||
require.NoError(t, os.MkdirAll(filepath.Dir(epubB), 0755))
|
||||
require.NoError(t, os.WriteFile(epubB, []byte("state book B content"), 0644))
|
||||
|
||||
scanTestFolder(t, setup, admin, folder)
|
||||
|
||||
items := listLibraryAudioItems(t, setup, libID)
|
||||
require.Len(t, items, 2, "both books ingested")
|
||||
var updatedA time.Time
|
||||
for _, item := range items {
|
||||
switch item.Title {
|
||||
case "State Book A":
|
||||
updatedA = item.UpdatedAt.Time
|
||||
case "State Book B":
|
||||
default:
|
||||
t.Errorf("unexpected item %q", item.Title)
|
||||
}
|
||||
}
|
||||
require.False(t, updatedA.IsZero())
|
||||
|
||||
// Unchanged rescan: the skip path must not touch the rows.
|
||||
scanTestFolder(t, setup, admin, folder)
|
||||
items = listLibraryAudioItems(t, setup, libID)
|
||||
require.Len(t, items, 2, "no duplicates on rescan")
|
||||
for _, item := range items {
|
||||
if item.Title == "State Book A" {
|
||||
assert.Equal(t, updatedA, item.UpdatedAt.Time, "unchanged rescan leaves updated_at stable")
|
||||
}
|
||||
}
|
||||
|
||||
// A row created by the API create path (NOT by a scan) must be seen
|
||||
// by the bulk load: the rescan skips it instead of re-creating. The
|
||||
// file must really exist (the archive pass missing-checks rows
|
||||
// whose files are absent).
|
||||
apiFile := filepath.Join(folder, "Author C", "API Made Book", "book.epub")
|
||||
require.NoError(t, os.MkdirAll(filepath.Dir(apiFile), 0755))
|
||||
require.NoError(t, os.WriteFile(apiFile, []byte("api made book content"), 0644))
|
||||
apiStat, err := os.Stat(apiFile)
|
||||
require.NoError(t, err)
|
||||
created, err := setup.DB.CreateMediaItem(context.Background(), database.CreateMediaItemParams{
|
||||
LibraryID: libID,
|
||||
Title: "API Made Book",
|
||||
Authors: []string{"API Author"},
|
||||
FilePath: "Author C/API Made Book/book.epub",
|
||||
FileSize: pgtype.Int8{Int64: apiStat.Size(), Valid: true},
|
||||
MimeType: pgtype.Text{String: "application/epub+zip", Valid: true},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
// Stamp the file's real sha the way the scanner does — otherwise the
|
||||
// skip path's one-time hash self-heal fires and bumps updated_at.
|
||||
// The identifiers UPDATE also bumps updated_at, so read the row
|
||||
// AFTER the stamp for the baseline.
|
||||
apiSha := sha256.Sum256([]byte("api made book content"))
|
||||
stamped, err := setup.DB.UpdateMediaItemIdentifiers(context.Background(), database.UpdateMediaItemIdentifiersParams{
|
||||
ID: created.ID,
|
||||
FileSha256: pgtype.Text{String: hex.EncodeToString(apiSha[:]), Valid: true},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
apiUpdated := stamped.UpdatedAt.Time
|
||||
|
||||
scanTestFolder(t, setup, admin, folder)
|
||||
items = listLibraryAudioItems(t, setup, libID)
|
||||
require.Len(t, items, 3, "API-created item survives the rescan as-is")
|
||||
for _, item := range items {
|
||||
if item.Title == "API Made Book" {
|
||||
assert.Equal(t, apiUpdated, item.UpdatedAt.Time, "bulk-loaded state sees scanner-external rows; skip path leaves them alone")
|
||||
}
|
||||
}
|
||||
|
||||
// Size change: the update path still fires (with a fresh full-row
|
||||
// read + hash recompute, then a write-through of the new size).
|
||||
require.NoError(t, os.WriteFile(epubA, []byte("state book A content — now longer"), 0644))
|
||||
scanTestFolder(t, setup, admin, folder)
|
||||
items = listLibraryAudioItems(t, setup, libID)
|
||||
require.Len(t, items, 3, "size change updates in place, never duplicates")
|
||||
for _, item := range items {
|
||||
if item.Title == "State Book A" {
|
||||
assert.NotEqual(t, updatedA, item.UpdatedAt.Time, "size change takes the update path")
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user