diff --git a/database/schema/schema.sql b/database/schema/schema.sql index af2f44f..be7abfc 100644 --- a/database/schema/schema.sql +++ b/database/schema/schema.sql @@ -1395,3 +1395,141 @@ CREATE UNIQUE INDEX IF NOT EXISTS idx_media_bookmarks_dedup CREATE INDEX IF NOT EXISTS idx_media_highlights_deleted_at ON media_highlights(deleted_at) WHERE deleted = TRUE; CREATE INDEX IF NOT EXISTS idx_media_notes_deleted_at ON media_notes(deleted_at) WHERE deleted = TRUE; CREATE INDEX IF NOT EXISTS idx_media_bookmarks_deleted_at ON media_bookmarks(deleted_at) WHERE deleted = TRUE; + +-- ============================================ +--: MEDIA ITEM DEDUPLICATION + PATH UNIQUENESS +-- ============================================ +-- A read-then-write race in the scanner historically allowed the same +-- (library_id, file_path) to be inserted twice. This block is self-healing: +-- it collapses any existing path-duplicates (re-parenting child rows onto a +-- survivor so no reading history is lost), then enforces uniqueness going +-- forward. Idempotent — safe to re-run on every startup. + +-- Move every child row that points at p_source so it points at p_target, +-- deleting source rows that would violate a UNIQUE constraint on the target. +CREATE OR REPLACE FUNCTION reparent_media_item_children(p_target UUID, p_source UUID) +RETURNS void +LANGUAGE plpgsql +AS $$ +BEGIN + IF p_target IS NULL OR p_source IS NULL OR p_target = p_source THEN + RETURN; + END IF; + + DELETE FROM reading_progress + WHERE media_item_id = p_source + AND user_id IN (SELECT user_id FROM reading_progress WHERE media_item_id = p_target); + UPDATE reading_progress SET media_item_id = p_target WHERE media_item_id = p_source; + + DELETE FROM reading_speed + WHERE media_item_id = p_source + AND user_id IN (SELECT user_id FROM reading_speed WHERE media_item_id = p_target); + UPDATE reading_speed SET media_item_id = p_target WHERE media_item_id = p_source; + + DELETE FROM media_ratings + WHERE media_item_id = p_source + AND user_id IN (SELECT user_id FROM media_ratings WHERE media_item_id = p_target); + UPDATE media_ratings SET media_item_id = p_target WHERE media_item_id = p_source; + + DELETE FROM media_bookmarks + WHERE media_item_id = p_source + AND (user_id, title) IN (SELECT user_id, title FROM media_bookmarks WHERE media_item_id = p_target); + UPDATE media_bookmarks SET media_item_id = p_target WHERE media_item_id = p_source; + + DELETE FROM media_item_formats + WHERE media_item_id = p_source + AND format_type IN (SELECT format_type FROM media_item_formats WHERE media_item_id = p_target); + UPDATE media_item_formats SET media_item_id = p_target WHERE media_item_id = p_source; + + DELETE FROM collection_items + WHERE media_item_id = p_source + AND collection_id IN (SELECT collection_id FROM collection_items WHERE media_item_id = p_target); + UPDATE collection_items SET media_item_id = p_target WHERE media_item_id = p_source; + + DELETE FROM kobo_shelves + WHERE media_item_id = p_source + AND device_id IN (SELECT device_id FROM kobo_shelves WHERE media_item_id = p_target); + UPDATE kobo_shelves SET media_item_id = p_target WHERE media_item_id = p_source; + + DELETE FROM panel_data + WHERE media_item_id = p_source + AND page_number IN (SELECT page_number FROM panel_data WHERE media_item_id = p_target); + UPDATE panel_data SET media_item_id = p_target WHERE media_item_id = p_source; + + DELETE FROM processing_issues + WHERE media_item_id = p_source + AND issue_type IN (SELECT issue_type FROM processing_issues WHERE media_item_id = p_target); + UPDATE processing_issues SET media_item_id = p_target WHERE media_item_id = p_source; + + DELETE FROM device_file_aliases + WHERE media_item_id = p_source + AND (device_id, file_path) IN (SELECT device_id, file_path FROM device_file_aliases WHERE media_item_id = p_target); + UPDATE device_file_aliases SET media_item_id = p_target WHERE media_item_id = p_source; + + -- Tables whose UNIQUE keys do not include media_item_id. + UPDATE device_catalogs SET media_item_id = p_target WHERE media_item_id = p_source; + UPDATE kobo_entitlements SET media_item_id = p_target WHERE media_item_id = p_source; + UPDATE media_highlights SET media_item_id = p_target WHERE media_item_id = p_source; + UPDATE media_notes SET media_item_id = p_target WHERE media_item_id = p_source; + UPDATE reading_history SET media_item_id = p_target WHERE media_item_id = p_source; + UPDATE sync_conflicts SET media_item_id = p_target WHERE media_item_id = p_source; + UPDATE sync_queue SET media_item_id = p_target WHERE media_item_id = p_source; +END; +$$; + +-- Collapse every (library_id, file_path) group into a single row. +-- Survivor = the row with the most user data; ties broken by lowest id. +CREATE OR REPLACE FUNCTION dedup_media_items_by_path() RETURNS void +LANGUAGE plpgsql +AS $$ +DECLARE + g RECORD; + v_surv UUID; + v_loser UUID; +BEGIN + FOR g IN + SELECT library_id, file_path + FROM media_items + GROUP BY library_id, file_path + HAVING COUNT(*) > 1 + LOOP + SELECT mi.id INTO v_surv + FROM media_items mi + WHERE mi.library_id = g.library_id AND mi.file_path = g.file_path + ORDER BY + ((SELECT COUNT(*) FROM reading_progress rp WHERE rp.media_item_id = mi.id) + + (SELECT COUNT(*) FROM media_highlights mh WHERE mh.media_item_id = mi.id) + + (SELECT COUNT(*) FROM media_bookmarks mb WHERE mb.media_item_id = mi.id) + + (SELECT COUNT(*) FROM media_notes mn WHERE mn.media_item_id = mi.id) + + (SELECT COUNT(*) FROM reading_history rh WHERE rh.media_item_id = mi.id) + + (SELECT COUNT(*) FROM collection_items ci WHERE ci.media_item_id = mi.id)) DESC, + mi.id ASC + LIMIT 1; + + FOR v_loser IN + SELECT id FROM media_items + WHERE library_id = g.library_id AND file_path = g.file_path AND id <> v_surv + ORDER BY id + LOOP + PERFORM reparent_media_item_children(v_surv, v_loser); + DELETE FROM media_items WHERE id = v_loser; + END LOOP; + END LOOP; +END; +$$; + +-- Collapse any existing path-duplicates so the constraint below can be created. +SELECT dedup_media_items_by_path(); + +-- Enforce path uniqueness going forward (guarded so re-runs don't error). +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conname = 'media_items_library_id_file_path_key' + AND conrelid = 'media_items'::regclass + ) THEN + ALTER TABLE media_items + ADD CONSTRAINT media_items_library_id_file_path_key UNIQUE (library_id, file_path); + END IF; +END $$; diff --git a/internal/database/models.go b/internal/database/models.go index 714627a..86cd90c 100644 --- a/internal/database/models.go +++ b/internal/database/models.go @@ -537,4 +537,3 @@ type Users struct { Timezone pgtype.Text `db:"timezone" json:"timezone"` UpdatedAt pgtype.Timestamptz `db:"updated_at" json:"updated_at"` } - diff --git a/internal/database/querier.go b/internal/database/querier.go index 28c4b45..7044830 100644 --- a/internal/database/querier.go +++ b/internal/database/querier.go @@ -232,6 +232,8 @@ type Querier interface { GetMediaItemByOPFUUID(ctx context.Context, opfUuid pgtype.Text) (MediaItems, error) // Get media item by SHA-256 hash GetMediaItemBySHA256(ctx context.Context, fileSha256 pgtype.Text) (MediaItems, error) + // Get media item by SHA-256 hash within a specific library (content dedup) + GetMediaItemBySHA256AndLibrary(ctx context.Context, arg GetMediaItemBySHA256AndLibraryParams) (MediaItems, error) // Get media item format by SHA-256 GetMediaItemFormatBySHA256(ctx context.Context, fileSha256 pgtype.Text) (MediaItemFormats, error) // Get media item format by type diff --git a/internal/database/queries.sql.go b/internal/database/queries.sql.go index 076afdd..ac3da9a 100644 --- a/internal/database/queries.sql.go +++ b/internal/database/queries.sql.go @@ -875,6 +875,7 @@ func (q *Queries) CreateMediaHighlightFull(ctx context.Context, arg CreateMediaH const CreateMediaItem = `-- name: CreateMediaItem :one INSERT INTO media_items (library_id, title, author, isbn, description, file_path, file_size, mime_type, cover_image_path, series, series_number, tags, tags_search, asin, date_published, publisher, contributors, contributors_search, language, edition, page_count, genre, copyright_year, goodreads_id, openlibrary_id, google_books_id, added_by_admin_id, created_at, imported_at, manga_type, reading_direction, series_count, volume, imprint, age_rating, web_url, story_arc, is_black_and_white, metadata_notes, community_rating, alternate_info, scan_information, summary, library_type_name) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28, $29, $30, $31, $32, $33, $34, $35, $36, $37, $38, $39, $40, $41, $42, $43, $44) +ON CONFLICT (library_id, file_path) DO UPDATE SET updated_at = NOW() RETURNING id, library_id, title, author, isbn, description, file_path, file_size, mime_type, cover_image_path, series, series_number, tags, asin, date_published, publisher, contributors, language, edition, page_count, genre, copyright_year, goodreads_id, openlibrary_id, google_books_id, added_by_admin_id, created_at, imported_at, updated_at, format_group, format_mimetype, is_reflowable, has_fixed_layout, total_characters, chapter_count, entitlement_id, revision_number, kobo_content_id, kobo_metadata, manga_type, reading_direction, series_count, volume, imprint, age_rating, web_url, story_arc, is_black_and_white, metadata_notes, community_rating, alternate_info, scan_information, summary, chapter_metadata, library_type_name, tags_search, contributors_search, file_sha256, opf_identifier, opf_uuid, hash_confidence ` @@ -1044,6 +1045,11 @@ const CreateMediaItemFormat = `-- name: CreateMediaItemFormat :one INSERT INTO media_item_formats (media_item_id, format_type, file_path, file_sha256, file_size_bytes, mime_type, converted_from_format_id) VALUES ($1, $2, $3, $4, $5, $6, $7) +ON CONFLICT (media_item_id, format_type) DO UPDATE SET + file_path = EXCLUDED.file_path, + file_sha256 = EXCLUDED.file_sha256, + file_size_bytes = EXCLUDED.file_size_bytes, + mime_type = EXCLUDED.mime_type RETURNING id, media_item_id, format_type, file_path, file_sha256, file_size_bytes, mime_type, created_at, converted_from_format_id ` @@ -5312,6 +5318,85 @@ func (q *Queries) GetMediaItemBySHA256(ctx context.Context, fileSha256 pgtype.Te return i, err } +const GetMediaItemBySHA256AndLibrary = `-- name: GetMediaItemBySHA256AndLibrary :one +SELECT id, library_id, title, author, isbn, description, file_path, file_size, mime_type, cover_image_path, series, series_number, tags, asin, date_published, publisher, contributors, language, edition, page_count, genre, copyright_year, goodreads_id, openlibrary_id, google_books_id, added_by_admin_id, created_at, imported_at, updated_at, format_group, format_mimetype, is_reflowable, has_fixed_layout, total_characters, chapter_count, entitlement_id, revision_number, kobo_content_id, kobo_metadata, manga_type, reading_direction, series_count, volume, imprint, age_rating, web_url, story_arc, is_black_and_white, metadata_notes, community_rating, alternate_info, scan_information, summary, chapter_metadata, library_type_name, tags_search, contributors_search, file_sha256, opf_identifier, opf_uuid, hash_confidence FROM media_items WHERE file_sha256 = $1 AND library_id = $2 +` + +type GetMediaItemBySHA256AndLibraryParams struct { + FileSha256 pgtype.Text `db:"file_sha256" json:"file_sha256"` + LibraryID pgtype.UUID `db:"library_id" json:"library_id"` +} + +// Get media item by SHA-256 hash within a specific library (content dedup) +func (q *Queries) GetMediaItemBySHA256AndLibrary(ctx context.Context, arg GetMediaItemBySHA256AndLibraryParams) (MediaItems, error) { + row := q.db.QueryRow(ctx, GetMediaItemBySHA256AndLibrary, arg.FileSha256, arg.LibraryID) + var i MediaItems + err := row.Scan( + &i.ID, + &i.LibraryID, + &i.Title, + &i.Author, + &i.Isbn, + &i.Description, + &i.FilePath, + &i.FileSize, + &i.MimeType, + &i.CoverImagePath, + &i.Series, + &i.SeriesNumber, + &i.Tags, + &i.Asin, + &i.DatePublished, + &i.Publisher, + &i.Contributors, + &i.Language, + &i.Edition, + &i.PageCount, + &i.Genre, + &i.CopyrightYear, + &i.GoodreadsID, + &i.OpenlibraryID, + &i.GoogleBooksID, + &i.AddedByAdminID, + &i.CreatedAt, + &i.ImportedAt, + &i.UpdatedAt, + &i.FormatGroup, + &i.FormatMimetype, + &i.IsReflowable, + &i.HasFixedLayout, + &i.TotalCharacters, + &i.ChapterCount, + &i.EntitlementID, + &i.RevisionNumber, + &i.KoboContentID, + &i.KoboMetadata, + &i.MangaType, + &i.ReadingDirection, + &i.SeriesCount, + &i.Volume, + &i.Imprint, + &i.AgeRating, + &i.WebUrl, + &i.StoryArc, + &i.IsBlackAndWhite, + &i.MetadataNotes, + &i.CommunityRating, + &i.AlternateInfo, + &i.ScanInformation, + &i.Summary, + &i.ChapterMetadata, + &i.LibraryTypeName, + &i.TagsSearch, + &i.ContributorsSearch, + &i.FileSha256, + &i.OpfIdentifier, + &i.OpfUuid, + &i.HashConfidence, + ) + return i, err +} + const GetMediaItemFormatBySHA256 = `-- name: GetMediaItemFormatBySHA256 :one SELECT id, media_item_id, format_type, file_path, file_sha256, file_size_bytes, mime_type, created_at, converted_from_format_id FROM media_item_formats WHERE file_sha256 = $1 ` diff --git a/internal/database/queries/queries.sql b/internal/database/queries/queries.sql index fb8e871..630884c 100644 --- a/internal/database/queries/queries.sql +++ b/internal/database/queries/queries.sql @@ -149,6 +149,7 @@ GROUP BY l.id; -- name: CreateMediaItem :one INSERT INTO media_items (library_id, title, author, isbn, description, file_path, file_size, mime_type, cover_image_path, series, series_number, tags, tags_search, asin, date_published, publisher, contributors, contributors_search, language, edition, page_count, genre, copyright_year, goodreads_id, openlibrary_id, google_books_id, added_by_admin_id, created_at, imported_at, manga_type, reading_direction, series_count, volume, imprint, age_rating, web_url, story_arc, is_black_and_white, metadata_notes, community_rating, alternate_info, scan_information, summary, library_type_name) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28, $29, $30, $31, $32, $33, $34, $35, $36, $37, $38, $39, $40, $41, $42, $43, $44) +ON CONFLICT (library_id, file_path) DO UPDATE SET updated_at = NOW() RETURNING *; -- name: GetMediaItem :one @@ -1707,6 +1708,10 @@ RETURNING *; -- name: GetMediaItemBySHA256 :one SELECT * FROM media_items WHERE file_sha256 = $1; +-- Get media item by SHA-256 hash within a specific library (content dedup) +-- name: GetMediaItemBySHA256AndLibrary :one +SELECT * FROM media_items WHERE file_sha256 = $1 AND library_id = $2; + -- Get media item by OPF identifier -- name: GetMediaItemByOPFIdentifier :one SELECT * FROM media_items WHERE opf_identifier = $1; @@ -1754,6 +1759,11 @@ ORDER BY confidence_score DESC; -- name: CreateMediaItemFormat :one INSERT INTO media_item_formats (media_item_id, format_type, file_path, file_sha256, file_size_bytes, mime_type, converted_from_format_id) VALUES ($1, $2, $3, $4, $5, $6, $7) +ON CONFLICT (media_item_id, format_type) DO UPDATE SET + file_path = EXCLUDED.file_path, + file_sha256 = EXCLUDED.file_sha256, + file_size_bytes = EXCLUDED.file_size_bytes, + mime_type = EXCLUDED.mime_type RETURNING *; -- Get media item formats diff --git a/internal/services/media_scanner.go b/internal/services/media_scanner.go index 0a4d07c..73d0b8b 100644 --- a/internal/services/media_scanner.go +++ b/internal/services/media_scanner.go @@ -735,6 +735,26 @@ func (s *MediaScanner) processMediaFile(ctx context.Context, path string) (bool, path, hashInfo.FileSHA256, hashInfo.OPFIdentifier, hashInfo.OPFUUID, hashInfo.HashConfidence) } + // Content dedup: if an item with the same SHA-256 already exists in this + // library (same file at a different path), treat it as existing rather than + // creating a duplicate. The file bytes are identical, so metadata matches. + if hashInfo.FileSHA256 != "" { + existingByHash, err := s.db.GetMediaItemBySHA256AndLibrary(ctx, database.GetMediaItemBySHA256AndLibraryParams{ + FileSha256: pgtype.Text{String: hashInfo.FileSHA256, Valid: true}, + LibraryID: libraryID, + }) + if err == nil && existingByHash.ID.Valid { + fmt.Printf("Media item with same SHA-256 already exists in library (path %q), skipping duplicate: %s\n", + existingByHash.FilePath, path) + if s.forceRescan { + _ = s.updateMediaItem(ctx, existingByHash.ID, path, info) + } + return false, nil + } else if err != nil && !errors.Is(err, pgx.ErrNoRows) { + fmt.Printf("Warning: failed to check media item by SHA-256 for %s: %v\n", path, err) + } + } + // REMOVED: Comic metadata extraction now handled by mergeMetadata() // This avoids duplicate extraction and ensures smart merging happens diff --git a/scripts/dedup_media_items.sql b/scripts/dedup_media_items.sql new file mode 100644 index 0000000..4e59654 --- /dev/null +++ b/scripts/dedup_media_items.sql @@ -0,0 +1,213 @@ +-- scripts/dedup_media_items.sql +-- +-- Detects and removes duplicate media_items, re-parenting all child rows +-- (reading progress, highlights, collections, etc.) onto a single survivor +-- before deleting the losers. +-- +-- Two kinds of duplicates are handled: +-- 1. PATH duplicates — same (library_id, file_path), multiple rows. +-- These block the UNIQUE(library_id, file_path) +-- constraint added by the schema migration. +-- 2. CONTENT duplicates — same file_sha256 within a library, different paths. +-- Same file imported twice under two names. +-- +-- This script is IDEMPOTENT: re-running it is a no-op once the data is clean. +-- It is safe to run against any Bookhoard database, before or after upgrading. +-- +-- Usage: +-- psql -h -U postgres -d bookhoard -f scripts/dedup_media_items.sql +-- +-- The first section is a DRY-RUN report (SELECTs only, no writes). The cleanup +-- runs inside an explicit transaction. Comment out the cleanup block to inspect +-- first. + +-- ===================================================================== +-- DRY RUN: report duplicates (no writes) +-- ====================================================================== + +\echo '=== PATH duplicates (library_id + file_path) ===' +SELECT library_id, + file_path, + COUNT(*) AS dupes, + array_agg(id::text) AS media_item_ids, + array_agg(COALESCE(file_sha256::text, 'NULL')) AS hashes +FROM media_items +GROUP BY library_id, file_path +HAVING COUNT(*) > 1 +ORDER BY COUNT(*) DESC; + +\echo '=== CONTENT duplicates (same file_sha256 within a library, different paths) ===' +SELECT library_id, + file_sha256::text AS hash, + COUNT(*) AS dupes, + array_agg(file_path) AS paths, + array_agg(id::text) AS media_item_ids +FROM media_items +WHERE file_sha256 IS NOT NULL +GROUP BY library_id, file_sha256 +HAVING COUNT(*) > 1 +ORDER BY COUNT(*) DESC; + +\echo '=== Child-row counts per duplicate candidate (helps confirm survivor choice) ===' +SELECT mi.id, + mi.library_id, + mi.file_path, + (SELECT COUNT(*) FROM reading_progress rp WHERE rp.media_item_id = mi.id) AS progress, + (SELECT COUNT(*) FROM media_highlights mh WHERE mh.media_item_id = mi.id) AS highlights, + (SELECT COUNT(*) FROM media_bookmarks mb WHERE mb.media_item_id = mi.id) AS bookmarks, + (SELECT COUNT(*) FROM media_notes mn WHERE mn.media_item_id = mi.id) AS notes, + (SELECT COUNT(*) FROM reading_history rh WHERE rh.media_item_id = mi.id) AS history, + (SELECT COUNT(*) FROM collection_items ci WHERE ci.media_item_id = mi.id) AS collections +FROM media_items mi +WHERE (mi.library_id, mi.file_path) IN ( + SELECT library_id, file_path FROM media_items + GROUP BY library_id, file_path HAVING COUNT(*) > 1 +) +ORDER BY mi.library_id, mi.file_path, mi.id; + +-- ===================================================================== +-- HELPER FUNCTIONS (also defined by schema.sql; CREATE OR REPLACE keeps them in sync) +-- ====================================================================== + +-- Move every child row that points at p_source so it points at p_target, +-- deleting any source rows that would violate a UNIQUE constraint on the +-- target. Idempotent; no-op when p_target = p_source. +CREATE OR REPLACE FUNCTION reparent_media_item_children(p_target UUID, p_source UUID) +RETURNS void +LANGUAGE plpgsql +AS $$ +BEGIN + IF p_target IS NULL OR p_source IS NULL OR p_target = p_source THEN + RETURN; + END IF; + + -- reading_progress (UNIQUE media_item_id, user_id) + DELETE FROM reading_progress + WHERE media_item_id = p_source + AND user_id IN (SELECT user_id FROM reading_progress WHERE media_item_id = p_target); + UPDATE reading_progress SET media_item_id = p_target WHERE media_item_id = p_source; + + -- reading_speed (UNIQUE user_id, media_item_id) + DELETE FROM reading_speed + WHERE media_item_id = p_source + AND user_id IN (SELECT user_id FROM reading_speed WHERE media_item_id = p_target); + UPDATE reading_speed SET media_item_id = p_target WHERE media_item_id = p_source; + + -- media_ratings (UNIQUE media_item_id, user_id) + DELETE FROM media_ratings + WHERE media_item_id = p_source + AND user_id IN (SELECT user_id FROM media_ratings WHERE media_item_id = p_target); + UPDATE media_ratings SET media_item_id = p_target WHERE media_item_id = p_source; + + -- media_bookmarks (UNIQUE media_item_id, user_id, title) + DELETE FROM media_bookmarks + WHERE media_item_id = p_source + AND (user_id, title) IN (SELECT user_id, title FROM media_bookmarks WHERE media_item_id = p_target); + UPDATE media_bookmarks SET media_item_id = p_target WHERE media_item_id = p_source; + + -- media_item_formats (UNIQUE media_item_id, format_type) + DELETE FROM media_item_formats + WHERE media_item_id = p_source + AND format_type IN (SELECT format_type FROM media_item_formats WHERE media_item_id = p_target); + UPDATE media_item_formats SET media_item_id = p_target WHERE media_item_id = p_source; + + -- collection_items (UNIQUE collection_id, media_item_id) + DELETE FROM collection_items + WHERE media_item_id = p_source + AND collection_id IN (SELECT collection_id FROM collection_items WHERE media_item_id = p_target); + UPDATE collection_items SET media_item_id = p_target WHERE media_item_id = p_source; + + -- kobo_shelves (UNIQUE device_id, media_item_id) + DELETE FROM kobo_shelves + WHERE media_item_id = p_source + AND device_id IN (SELECT device_id FROM kobo_shelves WHERE media_item_id = p_target); + UPDATE kobo_shelves SET media_item_id = p_target WHERE media_item_id = p_source; + + -- panel_data (UNIQUE media_item_id, page_number) + DELETE FROM panel_data + WHERE media_item_id = p_source + AND page_number IN (SELECT page_number FROM panel_data WHERE media_item_id = p_target); + UPDATE panel_data SET media_item_id = p_target WHERE media_item_id = p_source; + + -- processing_issues (UNIQUE media_item_id, issue_type) + DELETE FROM processing_issues + WHERE media_item_id = p_source + AND issue_type IN (SELECT issue_type FROM processing_issues WHERE media_item_id = p_target); + UPDATE processing_issues SET media_item_id = p_target WHERE media_item_id = p_source; + + -- device_file_aliases (UNIQUE device_id, file_path) — paths may collide + DELETE FROM device_file_aliases + WHERE media_item_id = p_source + AND (device_id, file_path) IN (SELECT device_id, file_path FROM device_file_aliases WHERE media_item_id = p_target); + UPDATE device_file_aliases SET media_item_id = p_target WHERE media_item_id = p_source; + + -- Tables whose UNIQUE keys do not include media_item_id: plain re-parent. + UPDATE device_catalogs SET media_item_id = p_target WHERE media_item_id = p_source; + UPDATE kobo_entitlements SET media_item_id = p_target WHERE media_item_id = p_source; + UPDATE media_highlights SET media_item_id = p_target WHERE media_item_id = p_source; + UPDATE media_notes SET media_item_id = p_target WHERE media_item_id = p_source; + UPDATE reading_history SET media_item_id = p_target WHERE media_item_id = p_source; + UPDATE sync_conflicts SET media_item_id = p_target WHERE media_item_id = p_source; + UPDATE sync_queue SET media_item_id = p_target WHERE media_item_id = p_source; +END; +$$; + +-- Collapse every (library_id, file_path) group into a single row. +-- Survivor = the row with the most user data; ties broken by lowest id. +CREATE OR REPLACE FUNCTION dedup_media_items_by_path() RETURNS void +LANGUAGE plpgsql +AS $$ +DECLARE + g RECORD; + v_surv UUID; + v_loser UUID; +BEGIN + FOR g IN + SELECT library_id, file_path + FROM media_items + GROUP BY library_id, file_path + HAVING COUNT(*) > 1 + LOOP + SELECT mi.id INTO v_surv + FROM media_items mi + WHERE mi.library_id = g.library_id AND mi.file_path = g.file_path + ORDER BY + ((SELECT COUNT(*) FROM reading_progress rp WHERE rp.media_item_id = mi.id) + + (SELECT COUNT(*) FROM media_highlights mh WHERE mh.media_item_id = mi.id) + + (SELECT COUNT(*) FROM media_bookmarks mb WHERE mb.media_item_id = mi.id) + + (SELECT COUNT(*) FROM media_notes mn WHERE mn.media_item_id = mi.id) + + (SELECT COUNT(*) FROM reading_history rh WHERE rh.media_item_id = mi.id) + + (SELECT COUNT(*) FROM collection_items ci WHERE ci.media_item_id = mi.id)) DESC, + mi.id ASC + LIMIT 1; + + FOR v_loser IN + SELECT id FROM media_items + WHERE library_id = g.library_id AND file_path = g.file_path AND id <> v_surv + ORDER BY id + LOOP + PERFORM reparent_media_item_children(v_surv, v_loser); + DELETE FROM media_items WHERE id = v_loser; + END LOOP; + END LOOP; +END; +$$; + +-- ====================================================================== +-- CLEANUP: collapse path duplicates (required before the UNIQUE constraint) +-- ====================================================================== + +\echo '=== Collapsing path duplicates ==='; +BEGIN; + SELECT dedup_media_items_by_path(); +COMMIT; + +\echo '=== Done. Remaining PATH duplicates (should be empty): ==='; +SELECT library_id, file_path, COUNT(*) AS dupes +FROM media_items +GROUP BY library_id, file_path +HAVING COUNT(*) > 1; + +\echo 'NOTE: CONTENT duplicates (same hash, different paths) are NOT auto-deleted.' +\echo ' They do not violate the UNIQUE constraint. Review the dry-run output' +\echo ' above and merge them manually if desired.'