feat(scanner): manifest-gated sidecar change detection + playlist sync on change path
Part 2 of the faceted plan, Phase 4 (steps 3-4). Size-only detection
was blind to metadata-sidecar edits; a size-stable book is changed by
EDITING its metadata.opf/metadata.json/cover — previously invisible
until a force rescan.
- Sidecar manifest fingerprints (media_scanner_manifest.go): the
manifest stats exactly the names the extractors consume (metadata.opf,
metadata.json, findSidecarCover patterns incl. <basename>.<ext>), so
manifest and extractors can never disagree. Shape v1:
{v, entries:[{n, s, m (unix nanos — second granularity would hide
edits landing in the backfill scan's own second), h (sha256 — covers
included: the repoint ladder needs a baseline hash for any stat-
drifted sidecar; a hashless cover turns harmless mtime drift into a
re-extraction, the exact flood this design kills)]}.
- Evaluation ladder in cost order: stat pairs identical → skip with
ZERO bytes read; stat drift → hash that sidecar (equal = host repoint
reconciliation, stat pair refreshed, no re-extraction; differing =
changed); NULL stored manifest → backfill only, NO re-extraction
(deploy-day rule: the first scan after release must not look like an
mtime flood).
- Change path: re-read full row, extractMetadata once and pass it
through updateMediaItemWithMetadata (preExtracted — metadata_overrides
stay protected), run syncPlaylistCollectionsAfterCreate (Correction 2:
a reading-order entry added to an EXISTING book now creates its
collection; add-if-missing is idempotent), store a fresh full
manifest. Creates seed their manifest at insert time.
- Multifile folder/zip books intentionally keep Phase 3 behavior: their
change path does not consume sidecar extractors, so gating them would
flip books 'changed' without picking anything up (plan correction —
the plan's §3 multifile note assumed otherwise).
- UpsertMediaScanState + GetMediaScanState queries.
Tests: media_scanner_manifest_test.go (ladder unit tests — fast path,
repoint, same-size drift, appear/vanish, unhashable legacy entries);
TestScanManifestLifecycle (seeding, unchanged-skip stability, repoint
reconciliation with stat-pair refresh assertion, same-size edit
consumed, playlist sync on the change path + idempotency, no-leak
guard). Container runner.
This commit is contained in:
1 parent
693053a9f5
commit
6750e9e0dc
9 files changed
+633
-14
No files matched your search
@@ -380,6 +380,9 @@ type Querier interface {
|
||||
GetMediaRating(ctx context.Context, arg GetMediaRatingParams) (MediaRatings, error)
|
||||
GetMediaRatings(ctx context.Context, mediaItemID pgtype.UUID) ([]GetMediaRatingsRow, error)
|
||||
GetMediaReview(ctx context.Context, arg GetMediaReviewParams) (MediaReviews, error)
|
||||
// GetMediaScanState reads one item's manifest (per-item rescan paths
|
||||
// and the manifest integration tests).
|
||||
GetMediaScanState(ctx context.Context, mediaItemID pgtype.UUID) (MediaScanState, error)
|
||||
GetNextRetryTime(ctx context.Context) (interface{}, error)
|
||||
GetNotStartedItems(ctx context.Context, arg GetNotStartedItemsParams) ([]MediaItems, error)
|
||||
// Get OPDS token
|
||||
@@ -685,6 +688,11 @@ type Querier interface {
|
||||
UpsertDashboardPreferences(ctx context.Context, arg UpsertDashboardPreferencesParams) (UserDashboardPreferences, error)
|
||||
UpsertGlobalDashboardPreferences(ctx context.Context, arg UpsertGlobalDashboardPreferencesParams) (UserDashboardPreferencesGlobal, error)
|
||||
UpsertMediaReview(ctx context.Context, arg UpsertMediaReviewParams) (MediaReviews, error)
|
||||
// UpsertMediaScanState stores a book's sidecar manifest fingerprint
|
||||
// (faceted plan Part 2 §3). Written on backfill (first sighting —
|
||||
// no re-extraction), on repoint reconciliation (stat pairs refreshed,
|
||||
// content identical), and after a manifest-gated re-extract.
|
||||
UpsertMediaScanState(ctx context.Context, arg UpsertMediaScanStateParams) error
|
||||
UpsertPanelData(ctx context.Context, arg UpsertPanelDataParams) (PanelData, error)
|
||||
UpsertReaderSettings(ctx context.Context, arg UpsertReaderSettingsParams) (ReaderSettings, error)
|
||||
UpsertSystemSetting(ctx context.Context, arg UpsertSystemSettingParams) (SystemSettings, error)
|
||||
|
||||
@@ -7796,6 +7796,19 @@ func (q *Queries) GetMediaReview(ctx context.Context, arg GetMediaReviewParams)
|
||||
return i, err
|
||||
}
|
||||
|
||||
const GetMediaScanState = `-- name: GetMediaScanState :one
|
||||
SELECT media_item_id, manifest FROM media_scan_state WHERE media_item_id = $1
|
||||
`
|
||||
|
||||
// GetMediaScanState reads one item's manifest (per-item rescan paths
|
||||
// and the manifest integration tests).
|
||||
func (q *Queries) GetMediaScanState(ctx context.Context, mediaItemID pgtype.UUID) (MediaScanState, error) {
|
||||
row := q.db.QueryRow(ctx, GetMediaScanState, mediaItemID)
|
||||
var i MediaScanState
|
||||
err := row.Scan(&i.MediaItemID, &i.Manifest)
|
||||
return i, err
|
||||
}
|
||||
|
||||
const GetNextRetryTime = `-- name: GetNextRetryTime :one
|
||||
SELECT
|
||||
CASE
|
||||
@@ -16920,6 +16933,26 @@ func (q *Queries) UpsertMediaReview(ctx context.Context, arg UpsertMediaReviewPa
|
||||
return i, err
|
||||
}
|
||||
|
||||
const UpsertMediaScanState = `-- name: UpsertMediaScanState :exec
|
||||
INSERT INTO media_scan_state (media_item_id, manifest)
|
||||
VALUES ($1, $2)
|
||||
ON CONFLICT (media_item_id) DO UPDATE SET manifest = EXCLUDED.manifest
|
||||
`
|
||||
|
||||
type UpsertMediaScanStateParams struct {
|
||||
MediaItemID pgtype.UUID `db:"media_item_id" json:"media_item_id"`
|
||||
Manifest []byte `db:"manifest" json:"manifest"`
|
||||
}
|
||||
|
||||
// UpsertMediaScanState stores a book's sidecar manifest fingerprint
|
||||
// (faceted plan Part 2 §3). Written on backfill (first sighting —
|
||||
// no re-extraction), on repoint reconciliation (stat pairs refreshed,
|
||||
// content identical), and after a manifest-gated re-extract.
|
||||
func (q *Queries) UpsertMediaScanState(ctx context.Context, arg UpsertMediaScanStateParams) error {
|
||||
_, err := q.db.Exec(ctx, UpsertMediaScanState, arg.MediaItemID, arg.Manifest)
|
||||
return err
|
||||
}
|
||||
|
||||
const UpsertPanelData = `-- name: UpsertPanelData :one
|
||||
INSERT INTO panel_data (media_item_id, page_number, detection_method, panels)
|
||||
VALUES ($1, $2, $3, $4)
|
||||
|
||||
@@ -452,6 +452,20 @@ FROM media_items mi
|
||||
LEFT JOIN media_scan_state s ON s.media_item_id = mi.id
|
||||
WHERE mi.library_id = sqlc.narg('library_id');
|
||||
|
||||
-- UpsertMediaScanState stores a book's sidecar manifest fingerprint
|
||||
-- (faceted plan Part 2 §3). Written on backfill (first sighting —
|
||||
-- no re-extraction), on repoint reconciliation (stat pairs refreshed,
|
||||
-- content identical), and after a manifest-gated re-extract.
|
||||
-- name: UpsertMediaScanState :exec
|
||||
INSERT INTO media_scan_state (media_item_id, manifest)
|
||||
VALUES ($1, $2)
|
||||
ON CONFLICT (media_item_id) DO UPDATE SET manifest = EXCLUDED.manifest;
|
||||
|
||||
-- GetMediaScanState reads one item's manifest (per-item rescan paths
|
||||
-- and the manifest integration tests).
|
||||
-- name: GetMediaScanState :one
|
||||
SELECT media_item_id, manifest FROM media_scan_state WHERE media_item_id = $1;
|
||||
|
||||
-- name: GetMediaItemByFilePathAnyLibrary :one
|
||||
SELECT * FROM media_items WHERE file_path = $1 LIMIT 1;
|
||||
|
||||
|
||||
Reference in new issue
Block a user