John O'Keefe
6750e9e0dc
Part 2 of the faceted plan, Phase 4 (steps 3-4). Size-only detection
was blind to metadata-sidecar edits; a size-stable book is changed by
EDITING its metadata.opf/metadata.json/cover — previously invisible
until a force rescan.
- Sidecar manifest fingerprints (media_scanner_manifest.go): the
manifest stats exactly the names the extractors consume (metadata.opf,
metadata.json, findSidecarCover patterns incl. <basename>.<ext>), so
manifest and extractors can never disagree. Shape v1:
{v, entries:[{n, s, m (unix nanos — second granularity would hide
edits landing in the backfill scan's own second), h (sha256 — covers
included: the repoint ladder needs a baseline hash for any stat-
drifted sidecar; a hashless cover turns harmless mtime drift into a
re-extraction, the exact flood this design kills)]}.
- Evaluation ladder in cost order: stat pairs identical → skip with
ZERO bytes read; stat drift → hash that sidecar (equal = host repoint
reconciliation, stat pair refreshed, no re-extraction; differing =
changed); NULL stored manifest → backfill only, NO re-extraction
(deploy-day rule: the first scan after release must not look like an
mtime flood).
- Change path: re-read full row, extractMetadata once and pass it
through updateMediaItemWithMetadata (preExtracted — metadata_overrides
stay protected), run syncPlaylistCollectionsAfterCreate (Correction 2:
a reading-order entry added to an EXISTING book now creates its
collection; add-if-missing is idempotent), store a fresh full
manifest. Creates seed their manifest at insert time.
- Multifile folder/zip books intentionally keep Phase 3 behavior: their
change path does not consume sidecar extractors, so gating them would
flip books 'changed' without picking anything up (plan correction —
the plan's §3 multifile note assumed otherwise).
- UpsertMediaScanState + GetMediaScanState queries.
Tests: media_scanner_manifest_test.go (ladder unit tests — fast path,
repoint, same-size drift, appear/vanish, unhashable legacy entries);
TestScanManifestLifecycle (seeding, unchanged-skip stability, repoint
reconciliation with stat-pair refresh assertion, same-size edit
consumed, playlist sync on the change path + idempotency, no-leak
guard). Container runner.
233 lines
9.9 KiB
Go
233 lines
9.9 KiB
Go
package main
|
|
|
|
// Bulk scan-state tests (faceted plan Part 2 Phase 3): the scanner's
|
|
// one-query-per-library bulk load replaces the per-file
|
|
// GetMediaItemByFilePath flood. Behavioral equivalence asserted at the
|
|
// container level: unchanged rescans skip (updated_at stable), items
|
|
// created OUTSIDE the scanner (API create path) are seen by the bulk
|
|
// load, and size changes still take the update path.
|
|
|
|
import (
|
|
"context"
|
|
"crypto/sha256"
|
|
"encoding/hex"
|
|
"encoding/json"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"os"
|
|
"path/filepath"
|
|
"testing"
|
|
"time"
|
|
|
|
"bookhoard/internal/database"
|
|
|
|
"github.com/jackc/pgx/v5/pgtype"
|
|
"github.com/stretchr/testify/assert"
|
|
"github.com/stretchr/testify/require"
|
|
)
|
|
|
|
func TestScanStateBulkLoad(t *testing.T) {
|
|
setup := setupTestServer(t)
|
|
t.Cleanup(func() { _ = setup.Close() })
|
|
|
|
admin, _ := moveTestAdminID(t, setup)
|
|
libID, folder := createTypedTestLibrary(t, setup, fmt.Sprintf("Test Scan State %d", time.Now().UnixNano()), "ebooks")
|
|
t.Cleanup(func() { deleteLibraries(t, setup, libID) })
|
|
|
|
// Two scanner-created fixture books.
|
|
epubA := filepath.Join(folder, "Author A", "State Book A", "book.epub")
|
|
require.NoError(t, os.MkdirAll(filepath.Dir(epubA), 0755))
|
|
require.NoError(t, os.WriteFile(epubA, []byte("state book A content"), 0644))
|
|
epubB := filepath.Join(folder, "Author B", "State Book B", "book.epub")
|
|
require.NoError(t, os.MkdirAll(filepath.Dir(epubB), 0755))
|
|
require.NoError(t, os.WriteFile(epubB, []byte("state book B content"), 0644))
|
|
|
|
scanTestFolder(t, setup, admin, folder)
|
|
|
|
items := listLibraryAudioItems(t, setup, libID)
|
|
require.Len(t, items, 2, "both books ingested")
|
|
var updatedA time.Time
|
|
for _, item := range items {
|
|
switch item.Title {
|
|
case "State Book A":
|
|
updatedA = item.UpdatedAt.Time
|
|
case "State Book B":
|
|
default:
|
|
t.Errorf("unexpected item %q", item.Title)
|
|
}
|
|
}
|
|
require.False(t, updatedA.IsZero())
|
|
|
|
// Unchanged rescan: the skip path must not touch the rows.
|
|
scanTestFolder(t, setup, admin, folder)
|
|
items = listLibraryAudioItems(t, setup, libID)
|
|
require.Len(t, items, 2, "no duplicates on rescan")
|
|
for _, item := range items {
|
|
if item.Title == "State Book A" {
|
|
assert.Equal(t, updatedA, item.UpdatedAt.Time, "unchanged rescan leaves updated_at stable")
|
|
}
|
|
}
|
|
|
|
// A row created by the API create path (NOT by a scan) must be seen
|
|
// by the bulk load: the rescan skips it instead of re-creating. The
|
|
// file must really exist (the archive pass missing-checks rows
|
|
// whose files are absent).
|
|
apiFile := filepath.Join(folder, "Author C", "API Made Book", "book.epub")
|
|
require.NoError(t, os.MkdirAll(filepath.Dir(apiFile), 0755))
|
|
require.NoError(t, os.WriteFile(apiFile, []byte("api made book content"), 0644))
|
|
apiStat, err := os.Stat(apiFile)
|
|
require.NoError(t, err)
|
|
created, err := setup.DB.CreateMediaItem(context.Background(), database.CreateMediaItemParams{
|
|
LibraryID: libID,
|
|
Title: "API Made Book",
|
|
Authors: []string{"API Author"},
|
|
FilePath: "Author C/API Made Book/book.epub",
|
|
FileSize: pgtype.Int8{Int64: apiStat.Size(), Valid: true},
|
|
MimeType: pgtype.Text{String: "application/epub+zip", Valid: true},
|
|
})
|
|
require.NoError(t, err)
|
|
// Stamp the file's real sha the way the scanner does — otherwise the
|
|
// skip path's one-time hash self-heal fires and bumps updated_at.
|
|
// The identifiers UPDATE also bumps updated_at, so read the row
|
|
// AFTER the stamp for the baseline.
|
|
apiSha := sha256.Sum256([]byte("api made book content"))
|
|
stamped, err := setup.DB.UpdateMediaItemIdentifiers(context.Background(), database.UpdateMediaItemIdentifiersParams{
|
|
ID: created.ID,
|
|
FileSha256: pgtype.Text{String: hex.EncodeToString(apiSha[:]), Valid: true},
|
|
})
|
|
require.NoError(t, err)
|
|
apiUpdated := stamped.UpdatedAt.Time
|
|
|
|
scanTestFolder(t, setup, admin, folder)
|
|
items = listLibraryAudioItems(t, setup, libID)
|
|
require.Len(t, items, 3, "API-created item survives the rescan as-is")
|
|
for _, item := range items {
|
|
if item.Title == "API Made Book" {
|
|
assert.Equal(t, apiUpdated, item.UpdatedAt.Time, "bulk-loaded state sees scanner-external rows; skip path leaves them alone")
|
|
}
|
|
}
|
|
|
|
// Size change: the update path still fires (with a fresh full-row
|
|
// read + hash recompute, then a write-through of the new size).
|
|
require.NoError(t, os.WriteFile(epubA, []byte("state book A content — now longer"), 0644))
|
|
scanTestFolder(t, setup, admin, folder)
|
|
items = listLibraryAudioItems(t, setup, libID)
|
|
require.Len(t, items, 3, "size change updates in place, never duplicates")
|
|
for _, item := range items {
|
|
if item.Title == "State Book A" {
|
|
assert.NotEqual(t, updatedA, item.UpdatedAt.Time, "size change takes the update path")
|
|
}
|
|
}
|
|
}
|
|
|
|
// TestScanManifestLifecycle exercises the manifest-gated change ladder
|
|
// end to end (faceted plan Part 2 Phase 4): seeding at create, repoint
|
|
// reconciliation (mtime drift, identical content → no re-extraction),
|
|
// same-size sidecar edits (the size-only detection blind spot),
|
|
// playlist sync on the change path, and the no-leak guard.
|
|
func TestScanManifestLifecycle(t *testing.T) {
|
|
setup := setupTestServer(t)
|
|
t.Cleanup(func() { _ = setup.Close() })
|
|
|
|
admin, _ := moveTestAdminID(t, setup)
|
|
libID, folder := createTypedTestLibrary(t, setup, fmt.Sprintf("Test Manifest Lifecycle %d", time.Now().UnixNano()), "ebooks")
|
|
t.Cleanup(func() { deleteLibraries(t, setup, libID) })
|
|
|
|
// The book ships WITH an ABS sidecar: the create path seeds the
|
|
// manifest so the next scan compares instead of backfilling.
|
|
bookDir := filepath.Join(folder, "Manifest Author", "Manifest Book")
|
|
require.NoError(t, os.MkdirAll(bookDir, 0755))
|
|
require.NoError(t, os.WriteFile(filepath.Join(bookDir, "book.epub"), []byte("manifest book content"), 0644))
|
|
opfPath := filepath.Join(bookDir, "metadata.json")
|
|
require.NoError(t, os.WriteFile(opfPath, []byte(`{"title":"Manifest Book"}`), 0644))
|
|
|
|
scanTestFolder(t, setup, admin, folder)
|
|
|
|
var itemID pgtype.UUID
|
|
var itemIDStr, updatedStr string
|
|
require.NoError(t, setup.DBPool.QueryRow(context.Background(),
|
|
`SELECT id::text, updated_at::text FROM media_items WHERE title = 'Manifest Book' AND library_id = $1`,
|
|
libID,
|
|
).Scan(&itemIDStr, &updatedStr))
|
|
require.NoError(t, itemID.Scan(itemIDStr))
|
|
|
|
// Seeded manifest exists.
|
|
seeded, err := setup.DB.GetMediaScanState(context.Background(), itemID)
|
|
require.NoError(t, err, "create path must seed the manifest")
|
|
require.NotEmpty(t, seeded.Manifest)
|
|
|
|
// Unchanged rescan: skip path, nothing moves.
|
|
scanTestFolder(t, setup, admin, folder)
|
|
var updated2 string
|
|
require.NoError(t, setup.DBPool.QueryRow(context.Background(),
|
|
`SELECT updated_at::text FROM media_items WHERE id = $1`, itemID).Scan(&updated2))
|
|
assert.Equal(t, updatedStr, updated2, "unchanged rescan is a pure skip")
|
|
|
|
// Repoint: mtime drift with identical content → reconciled, ZERO
|
|
// re-extraction; the stored stat pair refreshes.
|
|
past := time.Now().Add(-2 * time.Hour)
|
|
require.NoError(t, os.Chtimes(opfPath, past, past))
|
|
scanTestFolder(t, setup, admin, folder)
|
|
var updated3 string
|
|
require.NoError(t, setup.DBPool.QueryRow(context.Background(),
|
|
`SELECT updated_at::text FROM media_items WHERE id = $1`, itemID).Scan(&updated3))
|
|
assert.Equal(t, updatedStr, updated3, "repoint reconciliation must not re-extract")
|
|
reconciled, err := setup.DB.GetMediaScanState(context.Background(), itemID)
|
|
require.NoError(t, err)
|
|
var storedManifest struct {
|
|
Entries []struct {
|
|
Name string `json:"n"`
|
|
MTime int64 `json:"m"`
|
|
} `json:"entries"`
|
|
}
|
|
require.NoError(t, json.Unmarshal(reconciled.Manifest, &storedManifest))
|
|
var sawRefreshed bool
|
|
for _, e := range storedManifest.Entries {
|
|
if e.Name == "metadata.json" && e.MTime == past.UnixNano() {
|
|
sawRefreshed = true
|
|
}
|
|
}
|
|
assert.True(t, sawRefreshed, "repoint refreshes the stored stat pair")
|
|
|
|
// Same-size sidecar edit — the size-only detection blind spot: the
|
|
// stat pair moves, the hash differs → re-extract consumes the edit.
|
|
sameLen := []byte(`{"title":"Manifesu Book"}`)
|
|
require.Len(t, sameLen, len(`{"title":"Manifest Book"}`), "fixture must be a same-size edit")
|
|
require.NoError(t, os.WriteFile(opfPath, sameLen, 0644))
|
|
scanTestFolder(t, setup, admin, folder)
|
|
var title4 string
|
|
require.NoError(t, setup.DBPool.QueryRow(context.Background(),
|
|
`SELECT title FROM media_items WHERE id = $1`, itemID).Scan(&title4))
|
|
assert.Equal(t, "Manifesu Book", title4, "same-size sidecar edit is detected and consumed")
|
|
|
|
// Playlist sync on the change path (Correction 2): the sidecar gains
|
|
// a reading-order series → the collection is created for an
|
|
// EXISTING book, and a rescan does not duplicate it.
|
|
require.NoError(t, os.WriteFile(opfPath, []byte(`{"title":"Manifesu Book","series":["Lifecycle Saga (Reading Order) #1"]}`), 0644))
|
|
scanTestFolder(t, setup, admin, folder)
|
|
var collCount int
|
|
require.NoError(t, setup.DBPool.QueryRow(context.Background(),
|
|
`SELECT COUNT(*) FROM collections WHERE name = 'Lifecycle Saga (Reading Order)' AND user_id = $1`, admin,
|
|
).Scan(&collCount))
|
|
assert.Equal(t, 1, collCount, "reading-order sidecar added to an existing book creates its collection")
|
|
scanTestFolder(t, setup, admin, folder)
|
|
require.NoError(t, setup.DBPool.QueryRow(context.Background(),
|
|
`SELECT COUNT(*) FROM collections WHERE name = 'Lifecycle Saga (Reading Order)' AND user_id = $1`, admin,
|
|
).Scan(&collCount))
|
|
assert.Equal(t, 1, collCount, "rescan is idempotent — no duplicate collection")
|
|
|
|
// No-leak guard (plan Part 2 test 7): the library-scoped list emits
|
|
// raw rows — scanner state must never ride along (structural by
|
|
// design; this assertion keeps it honest).
|
|
req, _ := http.NewRequest("GET", setup.Server.URL+"/api/media-items?library_id="+libID.String(), nil)
|
|
req.Header.Set("Authorization", "Bearer "+setup.RegularToken)
|
|
resp, err := http.DefaultClient.Do(req)
|
|
require.NoError(t, err)
|
|
defer func() { _ = resp.Body.Close() }()
|
|
body, err := io.ReadAll(resp.Body)
|
|
require.NoError(t, err)
|
|
assert.NotContains(t, string(body), `"manifest":`, "scanner state must not leak into media list JSON")
|
|
assert.NotContains(t, string(body), `"sidecar`, "scanner state must not leak into media list JSON")
|
|
}
|