fix(scanner): prevent duplicate media item imports

A read-then-write race in processMediaFile allowed the same file to be imported twice: two concurrent scan jobs (startup scan, fsnotify dirty- directory scan, periodic backup poll, or a manual scan each run on separate worker goroutines with separate MediaScanner instances) could both SELECT 'not found' and both INSERT. There was no transaction, no row lock, no unique constraint on (library_id, file_path), and no ON CONFLICT clause, so nothing stopped the double insert. Observed in production as two identical 'Head First SQL' rows created in the same second (same sha256, size, path, library). Database enforcement: - schema.sql: add UNIQUE(library_id, file_path) constraint, guarded so re-runs don't error - schema.sql: add self-healing migration that runs on every startup - dedup_media_items_by_path() collapses existing path-duplicates and reparent_media_item_children() moves all child rows (progress, highlights, bookmarks, notes, collections, formats, aliases, kobo entitlements, etc.) onto a survivor before deleting losers, so the constraint applies cleanly on already-duplicated servers without losing reading history. Survivor picks the row with the most user data, ties broken by lowest id - CreateMediaItem: upsert via ON CONFLICT (library_id, file_path) DO UPDATE so concurrent inserts collapse to one row and return it - CreateMediaItemFormat: upsert via ON CONFLICT (media_item_id, format_type), closing the same race on format rows Application-level guards: - media_scanner processMediaFile: after computing the file hash, check GetMediaItemBySHA256AndLibrary (new query) and treat the file as existing when identical content is already in the library under a different path (content dedup, library-scoped so multi-library setups still work) Ops tooling: - scripts/dedup_media_items.sql: standalone idempotent maintenance script with a dry-run report (path + content duplicate groups, child row counts) and transactional cleanup, for servers that prefer to dedup manually before upgrading Verified against the live database: the duplicate pair was collapsed (reading_progress preserved on the survivor), schema.sql re-runs are a no-op, and the constraint is in place with 62 unique books remaining.
2026-08-14 08:18:36 -04:00
parent b7a9b470a7
commit 9b171a0060
7 changed files with 468 additions and 1 deletions
@@ -537,4 +537,3 @@ type Users struct {
 	Timezone     pgtype.Text        `db:"timezone" json:"timezone"`
 	UpdatedAt    pgtype.Timestamptz `db:"updated_at" json:"updated_at"`
 }
-
@@ -232,6 +232,8 @@ type Querier interface {
 	GetMediaItemByOPFUUID(ctx context.Context, opfUuid pgtype.Text) (MediaItems, error)
 	// Get media item by SHA-256 hash
 	GetMediaItemBySHA256(ctx context.Context, fileSha256 pgtype.Text) (MediaItems, error)
+	// Get media item by SHA-256 hash within a specific library (content dedup)
+	GetMediaItemBySHA256AndLibrary(ctx context.Context, arg GetMediaItemBySHA256AndLibraryParams) (MediaItems, error)
 	// Get media item format by SHA-256
 	GetMediaItemFormatBySHA256(ctx context.Context, fileSha256 pgtype.Text) (MediaItemFormats, error)
 	// Get media item format by type
@@ -875,6 +875,7 @@ func (q *Queries) CreateMediaHighlightFull(ctx context.Context, arg CreateMediaH
 const CreateMediaItem = `-- name: CreateMediaItem :one
 INSERT INTO media_items (library_id, title, author, isbn, description, file_path, file_size, mime_type, cover_image_path, series, series_number, tags, tags_search, asin, date_published, publisher, contributors, contributors_search, language, edition, page_count, genre, copyright_year, goodreads_id, openlibrary_id, google_books_id, added_by_admin_id, created_at, imported_at, manga_type, reading_direction, series_count, volume, imprint, age_rating, web_url, story_arc, is_black_and_white, metadata_notes, community_rating, alternate_info, scan_information, summary, library_type_name)
 VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28, $29, $30, $31, $32, $33, $34, $35, $36, $37, $38, $39, $40, $41, $42, $43, $44)
+ON CONFLICT (library_id, file_path) DO UPDATE SET updated_at = NOW()
 RETURNING id, library_id, title, author, isbn, description, file_path, file_size, mime_type, cover_image_path, series, series_number, tags, asin, date_published, publisher, contributors, language, edition, page_count, genre, copyright_year, goodreads_id, openlibrary_id, google_books_id, added_by_admin_id, created_at, imported_at, updated_at, format_group, format_mimetype, is_reflowable, has_fixed_layout, total_characters, chapter_count, entitlement_id, revision_number, kobo_content_id, kobo_metadata, manga_type, reading_direction, series_count, volume, imprint, age_rating, web_url, story_arc, is_black_and_white, metadata_notes, community_rating, alternate_info, scan_information, summary, chapter_metadata, library_type_name, tags_search, contributors_search, file_sha256, opf_identifier, opf_uuid, hash_confidence
 `

@@ -1044,6 +1045,11 @@ const CreateMediaItemFormat = `-- name: CreateMediaItemFormat :one

 INSERT INTO media_item_formats (media_item_id, format_type, file_path, file_sha256, file_size_bytes, mime_type, converted_from_format_id)
 VALUES ($1, $2, $3, $4, $5, $6, $7)
+ON CONFLICT (media_item_id, format_type) DO UPDATE SET
+    file_path = EXCLUDED.file_path,
+    file_sha256 = EXCLUDED.file_sha256,
+    file_size_bytes = EXCLUDED.file_size_bytes,
+    mime_type = EXCLUDED.mime_type
 RETURNING id, media_item_id, format_type, file_path, file_sha256, file_size_bytes, mime_type, created_at, converted_from_format_id
 `

@@ -5312,6 +5318,85 @@ func (q *Queries) GetMediaItemBySHA256(ctx context.Context, fileSha256 pgtype.Te
 	return i, err
 }

+const GetMediaItemBySHA256AndLibrary = `-- name: GetMediaItemBySHA256AndLibrary :one
+SELECT id, library_id, title, author, isbn, description, file_path, file_size, mime_type, cover_image_path, series, series_number, tags, asin, date_published, publisher, contributors, language, edition, page_count, genre, copyright_year, goodreads_id, openlibrary_id, google_books_id, added_by_admin_id, created_at, imported_at, updated_at, format_group, format_mimetype, is_reflowable, has_fixed_layout, total_characters, chapter_count, entitlement_id, revision_number, kobo_content_id, kobo_metadata, manga_type, reading_direction, series_count, volume, imprint, age_rating, web_url, story_arc, is_black_and_white, metadata_notes, community_rating, alternate_info, scan_information, summary, chapter_metadata, library_type_name, tags_search, contributors_search, file_sha256, opf_identifier, opf_uuid, hash_confidence FROM media_items WHERE file_sha256 = $1 AND library_id = $2
+`
+
+type GetMediaItemBySHA256AndLibraryParams struct {
+	FileSha256 pgtype.Text `db:"file_sha256" json:"file_sha256"`
+	LibraryID  pgtype.UUID `db:"library_id" json:"library_id"`
+}
+
+// Get media item by SHA-256 hash within a specific library (content dedup)
+func (q *Queries) GetMediaItemBySHA256AndLibrary(ctx context.Context, arg GetMediaItemBySHA256AndLibraryParams) (MediaItems, error) {
+	row := q.db.QueryRow(ctx, GetMediaItemBySHA256AndLibrary, arg.FileSha256, arg.LibraryID)
+	var i MediaItems
+	err := row.Scan(
+		&i.ID,
+		&i.LibraryID,
+		&i.Title,
+		&i.Author,
+		&i.Isbn,
+		&i.Description,
+		&i.FilePath,
+		&i.FileSize,
+		&i.MimeType,
+		&i.CoverImagePath,
+		&i.Series,
+		&i.SeriesNumber,
+		&i.Tags,
+		&i.Asin,
+		&i.DatePublished,
+		&i.Publisher,
+		&i.Contributors,
+		&i.Language,
+		&i.Edition,
+		&i.PageCount,
+		&i.Genre,
+		&i.CopyrightYear,
+		&i.GoodreadsID,
+		&i.OpenlibraryID,
+		&i.GoogleBooksID,
+		&i.AddedByAdminID,
+		&i.CreatedAt,
+		&i.ImportedAt,
+		&i.UpdatedAt,
+		&i.FormatGroup,
+		&i.FormatMimetype,
+		&i.IsReflowable,
+		&i.HasFixedLayout,
+		&i.TotalCharacters,
+		&i.ChapterCount,
+		&i.EntitlementID,
+		&i.RevisionNumber,
+		&i.KoboContentID,
+		&i.KoboMetadata,
+		&i.MangaType,
+		&i.ReadingDirection,
+		&i.SeriesCount,
+		&i.Volume,
+		&i.Imprint,
+		&i.AgeRating,
+		&i.WebUrl,
+		&i.StoryArc,
+		&i.IsBlackAndWhite,
+		&i.MetadataNotes,
+		&i.CommunityRating,
+		&i.AlternateInfo,
+		&i.ScanInformation,
+		&i.Summary,
+		&i.ChapterMetadata,
+		&i.LibraryTypeName,
+		&i.TagsSearch,
+		&i.ContributorsSearch,
+		&i.FileSha256,
+		&i.OpfIdentifier,
+		&i.OpfUuid,
+		&i.HashConfidence,
+	)
+	return i, err
+}
+
 const GetMediaItemFormatBySHA256 = `-- name: GetMediaItemFormatBySHA256 :one
 SELECT id, media_item_id, format_type, file_path, file_sha256, file_size_bytes, mime_type, created_at, converted_from_format_id FROM media_item_formats WHERE file_sha256 = $1
 `
@@ -149,6 +149,7 @@ GROUP BY l.id;
 -- name: CreateMediaItem :one
 INSERT INTO media_items (library_id, title, author, isbn, description, file_path, file_size, mime_type, cover_image_path, series, series_number, tags, tags_search, asin, date_published, publisher, contributors, contributors_search, language, edition, page_count, genre, copyright_year, goodreads_id, openlibrary_id, google_books_id, added_by_admin_id, created_at, imported_at, manga_type, reading_direction, series_count, volume, imprint, age_rating, web_url, story_arc, is_black_and_white, metadata_notes, community_rating, alternate_info, scan_information, summary, library_type_name)
 VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28, $29, $30, $31, $32, $33, $34, $35, $36, $37, $38, $39, $40, $41, $42, $43, $44)
+ON CONFLICT (library_id, file_path) DO UPDATE SET updated_at = NOW()
 RETURNING *;

 -- name: GetMediaItem :one
@@ -1707,6 +1708,10 @@ RETURNING *;
 -- name: GetMediaItemBySHA256 :one
 SELECT * FROM media_items WHERE file_sha256 = $1;

+-- Get media item by SHA-256 hash within a specific library (content dedup)
+-- name: GetMediaItemBySHA256AndLibrary :one
+SELECT * FROM media_items WHERE file_sha256 = $1 AND library_id = $2;
+
 -- Get media item by OPF identifier
 -- name: GetMediaItemByOPFIdentifier :one
 SELECT * FROM media_items WHERE opf_identifier = $1;
@@ -1754,6 +1759,11 @@ ORDER BY confidence_score DESC;
 -- name: CreateMediaItemFormat :one
 INSERT INTO media_item_formats (media_item_id, format_type, file_path, file_sha256, file_size_bytes, mime_type, converted_from_format_id)
 VALUES ($1, $2, $3, $4, $5, $6, $7)
+ON CONFLICT (media_item_id, format_type) DO UPDATE SET
+    file_path = EXCLUDED.file_path,
+    file_sha256 = EXCLUDED.file_sha256,
+    file_size_bytes = EXCLUDED.file_size_bytes,
+    mime_type = EXCLUDED.mime_type
 RETURNING *;

 -- Get media item formats
@@ -735,6 +735,26 @@ func (s *MediaScanner) processMediaFile(ctx context.Context, path string) (bool,
 			path, hashInfo.FileSHA256, hashInfo.OPFIdentifier, hashInfo.OPFUUID, hashInfo.HashConfidence)
 	}

+	// Content dedup: if an item with the same SHA-256 already exists in this
+	// library (same file at a different path), treat it as existing rather than
+	// creating a duplicate. The file bytes are identical, so metadata matches.
+	if hashInfo.FileSHA256 != "" {
+		existingByHash, err := s.db.GetMediaItemBySHA256AndLibrary(ctx, database.GetMediaItemBySHA256AndLibraryParams{
+			FileSha256: pgtype.Text{String: hashInfo.FileSHA256, Valid: true},
+			LibraryID:  libraryID,
+		})
+		if err == nil && existingByHash.ID.Valid {
+			fmt.Printf("Media item with same SHA-256 already exists in library (path %q), skipping duplicate: %s\n",
+				existingByHash.FilePath, path)
+			if s.forceRescan {
+				_ = s.updateMediaItem(ctx, existingByHash.ID, path, info)
+			}
+			return false, nil
+		} else if err != nil && !errors.Is(err, pgx.ErrNoRows) {
+			fmt.Printf("Warning: failed to check media item by SHA-256 for %s: %v\n", path, err)
+		}
+	}
+
 	// REMOVED: Comic metadata extraction now handled by mergeMetadata()
 	// This avoids duplicate extraction and ensures smart merging happens