fix(scanner): prevent duplicate media item imports
A read-then-write race in processMediaFile allowed the same file to be imported twice: two concurrent scan jobs (startup scan, fsnotify dirty- directory scan, periodic backup poll, or a manual scan each run on separate worker goroutines with separate MediaScanner instances) could both SELECT 'not found' and both INSERT. There was no transaction, no row lock, no unique constraint on (library_id, file_path), and no ON CONFLICT clause, so nothing stopped the double insert. Observed in production as two identical 'Head First SQL' rows created in the same second (same sha256, size, path, library). Database enforcement: - schema.sql: add UNIQUE(library_id, file_path) constraint, guarded so re-runs don't error - schema.sql: add self-healing migration that runs on every startup - dedup_media_items_by_path() collapses existing path-duplicates and reparent_media_item_children() moves all child rows (progress, highlights, bookmarks, notes, collections, formats, aliases, kobo entitlements, etc.) onto a survivor before deleting losers, so the constraint applies cleanly on already-duplicated servers without losing reading history. Survivor picks the row with the most user data, ties broken by lowest id - CreateMediaItem: upsert via ON CONFLICT (library_id, file_path) DO UPDATE so concurrent inserts collapse to one row and return it - CreateMediaItemFormat: upsert via ON CONFLICT (media_item_id, format_type), closing the same race on format rows Application-level guards: - media_scanner processMediaFile: after computing the file hash, check GetMediaItemBySHA256AndLibrary (new query) and treat the file as existing when identical content is already in the library under a different path (content dedup, library-scoped so multi-library setups still work) Ops tooling: - scripts/dedup_media_items.sql: standalone idempotent maintenance script with a dry-run report (path + content duplicate groups, child row counts) and transactional cleanup, for servers that prefer to dedup manually before upgrading Verified against the live database: the duplicate pair was collapsed (reading_progress preserved on the survivor), schema.sql re-runs are a no-op, and the constraint is in place with 62 unique books remaining.
This commit is contained in:
@@ -1395,3 +1395,141 @@ CREATE UNIQUE INDEX IF NOT EXISTS idx_media_bookmarks_dedup
|
||||
CREATE INDEX IF NOT EXISTS idx_media_highlights_deleted_at ON media_highlights(deleted_at) WHERE deleted = TRUE;
|
||||
CREATE INDEX IF NOT EXISTS idx_media_notes_deleted_at ON media_notes(deleted_at) WHERE deleted = TRUE;
|
||||
CREATE INDEX IF NOT EXISTS idx_media_bookmarks_deleted_at ON media_bookmarks(deleted_at) WHERE deleted = TRUE;
|
||||
|
||||
-- ============================================
|
||||
--: MEDIA ITEM DEDUPLICATION + PATH UNIQUENESS
|
||||
-- ============================================
|
||||
-- A read-then-write race in the scanner historically allowed the same
|
||||
-- (library_id, file_path) to be inserted twice. This block is self-healing:
|
||||
-- it collapses any existing path-duplicates (re-parenting child rows onto a
|
||||
-- survivor so no reading history is lost), then enforces uniqueness going
|
||||
-- forward. Idempotent — safe to re-run on every startup.
|
||||
|
||||
-- Move every child row that points at p_source so it points at p_target,
|
||||
-- deleting source rows that would violate a UNIQUE constraint on the target.
|
||||
CREATE OR REPLACE FUNCTION reparent_media_item_children(p_target UUID, p_source UUID)
|
||||
RETURNS void
|
||||
LANGUAGE plpgsql
|
||||
AS $$
|
||||
BEGIN
|
||||
IF p_target IS NULL OR p_source IS NULL OR p_target = p_source THEN
|
||||
RETURN;
|
||||
END IF;
|
||||
|
||||
DELETE FROM reading_progress
|
||||
WHERE media_item_id = p_source
|
||||
AND user_id IN (SELECT user_id FROM reading_progress WHERE media_item_id = p_target);
|
||||
UPDATE reading_progress SET media_item_id = p_target WHERE media_item_id = p_source;
|
||||
|
||||
DELETE FROM reading_speed
|
||||
WHERE media_item_id = p_source
|
||||
AND user_id IN (SELECT user_id FROM reading_speed WHERE media_item_id = p_target);
|
||||
UPDATE reading_speed SET media_item_id = p_target WHERE media_item_id = p_source;
|
||||
|
||||
DELETE FROM media_ratings
|
||||
WHERE media_item_id = p_source
|
||||
AND user_id IN (SELECT user_id FROM media_ratings WHERE media_item_id = p_target);
|
||||
UPDATE media_ratings SET media_item_id = p_target WHERE media_item_id = p_source;
|
||||
|
||||
DELETE FROM media_bookmarks
|
||||
WHERE media_item_id = p_source
|
||||
AND (user_id, title) IN (SELECT user_id, title FROM media_bookmarks WHERE media_item_id = p_target);
|
||||
UPDATE media_bookmarks SET media_item_id = p_target WHERE media_item_id = p_source;
|
||||
|
||||
DELETE FROM media_item_formats
|
||||
WHERE media_item_id = p_source
|
||||
AND format_type IN (SELECT format_type FROM media_item_formats WHERE media_item_id = p_target);
|
||||
UPDATE media_item_formats SET media_item_id = p_target WHERE media_item_id = p_source;
|
||||
|
||||
DELETE FROM collection_items
|
||||
WHERE media_item_id = p_source
|
||||
AND collection_id IN (SELECT collection_id FROM collection_items WHERE media_item_id = p_target);
|
||||
UPDATE collection_items SET media_item_id = p_target WHERE media_item_id = p_source;
|
||||
|
||||
DELETE FROM kobo_shelves
|
||||
WHERE media_item_id = p_source
|
||||
AND device_id IN (SELECT device_id FROM kobo_shelves WHERE media_item_id = p_target);
|
||||
UPDATE kobo_shelves SET media_item_id = p_target WHERE media_item_id = p_source;
|
||||
|
||||
DELETE FROM panel_data
|
||||
WHERE media_item_id = p_source
|
||||
AND page_number IN (SELECT page_number FROM panel_data WHERE media_item_id = p_target);
|
||||
UPDATE panel_data SET media_item_id = p_target WHERE media_item_id = p_source;
|
||||
|
||||
DELETE FROM processing_issues
|
||||
WHERE media_item_id = p_source
|
||||
AND issue_type IN (SELECT issue_type FROM processing_issues WHERE media_item_id = p_target);
|
||||
UPDATE processing_issues SET media_item_id = p_target WHERE media_item_id = p_source;
|
||||
|
||||
DELETE FROM device_file_aliases
|
||||
WHERE media_item_id = p_source
|
||||
AND (device_id, file_path) IN (SELECT device_id, file_path FROM device_file_aliases WHERE media_item_id = p_target);
|
||||
UPDATE device_file_aliases SET media_item_id = p_target WHERE media_item_id = p_source;
|
||||
|
||||
-- Tables whose UNIQUE keys do not include media_item_id.
|
||||
UPDATE device_catalogs SET media_item_id = p_target WHERE media_item_id = p_source;
|
||||
UPDATE kobo_entitlements SET media_item_id = p_target WHERE media_item_id = p_source;
|
||||
UPDATE media_highlights SET media_item_id = p_target WHERE media_item_id = p_source;
|
||||
UPDATE media_notes SET media_item_id = p_target WHERE media_item_id = p_source;
|
||||
UPDATE reading_history SET media_item_id = p_target WHERE media_item_id = p_source;
|
||||
UPDATE sync_conflicts SET media_item_id = p_target WHERE media_item_id = p_source;
|
||||
UPDATE sync_queue SET media_item_id = p_target WHERE media_item_id = p_source;
|
||||
END;
|
||||
$$;
|
||||
|
||||
-- Collapse every (library_id, file_path) group into a single row.
|
||||
-- Survivor = the row with the most user data; ties broken by lowest id.
|
||||
CREATE OR REPLACE FUNCTION dedup_media_items_by_path() RETURNS void
|
||||
LANGUAGE plpgsql
|
||||
AS $$
|
||||
DECLARE
|
||||
g RECORD;
|
||||
v_surv UUID;
|
||||
v_loser UUID;
|
||||
BEGIN
|
||||
FOR g IN
|
||||
SELECT library_id, file_path
|
||||
FROM media_items
|
||||
GROUP BY library_id, file_path
|
||||
HAVING COUNT(*) > 1
|
||||
LOOP
|
||||
SELECT mi.id INTO v_surv
|
||||
FROM media_items mi
|
||||
WHERE mi.library_id = g.library_id AND mi.file_path = g.file_path
|
||||
ORDER BY
|
||||
((SELECT COUNT(*) FROM reading_progress rp WHERE rp.media_item_id = mi.id)
|
||||
+ (SELECT COUNT(*) FROM media_highlights mh WHERE mh.media_item_id = mi.id)
|
||||
+ (SELECT COUNT(*) FROM media_bookmarks mb WHERE mb.media_item_id = mi.id)
|
||||
+ (SELECT COUNT(*) FROM media_notes mn WHERE mn.media_item_id = mi.id)
|
||||
+ (SELECT COUNT(*) FROM reading_history rh WHERE rh.media_item_id = mi.id)
|
||||
+ (SELECT COUNT(*) FROM collection_items ci WHERE ci.media_item_id = mi.id)) DESC,
|
||||
mi.id ASC
|
||||
LIMIT 1;
|
||||
|
||||
FOR v_loser IN
|
||||
SELECT id FROM media_items
|
||||
WHERE library_id = g.library_id AND file_path = g.file_path AND id <> v_surv
|
||||
ORDER BY id
|
||||
LOOP
|
||||
PERFORM reparent_media_item_children(v_surv, v_loser);
|
||||
DELETE FROM media_items WHERE id = v_loser;
|
||||
END LOOP;
|
||||
END LOOP;
|
||||
END;
|
||||
$$;
|
||||
|
||||
-- Collapse any existing path-duplicates so the constraint below can be created.
|
||||
SELECT dedup_media_items_by_path();
|
||||
|
||||
-- Enforce path uniqueness going forward (guarded so re-runs don't error).
|
||||
DO $$
|
||||
BEGIN
|
||||
IF NOT EXISTS (
|
||||
SELECT 1 FROM pg_constraint
|
||||
WHERE conname = 'media_items_library_id_file_path_key'
|
||||
AND conrelid = 'media_items'::regclass
|
||||
) THEN
|
||||
ALTER TABLE media_items
|
||||
ADD CONSTRAINT media_items_library_id_file_path_key UNIQUE (library_id, file_path);
|
||||
END IF;
|
||||
END $$;
|
||||
|
||||
@@ -537,4 +537,3 @@ type Users struct {
|
||||
Timezone pgtype.Text `db:"timezone" json:"timezone"`
|
||||
UpdatedAt pgtype.Timestamptz `db:"updated_at" json:"updated_at"`
|
||||
}
|
||||
|
||||
|
||||
@@ -232,6 +232,8 @@ type Querier interface {
|
||||
GetMediaItemByOPFUUID(ctx context.Context, opfUuid pgtype.Text) (MediaItems, error)
|
||||
// Get media item by SHA-256 hash
|
||||
GetMediaItemBySHA256(ctx context.Context, fileSha256 pgtype.Text) (MediaItems, error)
|
||||
// Get media item by SHA-256 hash within a specific library (content dedup)
|
||||
GetMediaItemBySHA256AndLibrary(ctx context.Context, arg GetMediaItemBySHA256AndLibraryParams) (MediaItems, error)
|
||||
// Get media item format by SHA-256
|
||||
GetMediaItemFormatBySHA256(ctx context.Context, fileSha256 pgtype.Text) (MediaItemFormats, error)
|
||||
// Get media item format by type
|
||||
|
||||
@@ -875,6 +875,7 @@ func (q *Queries) CreateMediaHighlightFull(ctx context.Context, arg CreateMediaH
|
||||
const CreateMediaItem = `-- name: CreateMediaItem :one
|
||||
INSERT INTO media_items (library_id, title, author, isbn, description, file_path, file_size, mime_type, cover_image_path, series, series_number, tags, tags_search, asin, date_published, publisher, contributors, contributors_search, language, edition, page_count, genre, copyright_year, goodreads_id, openlibrary_id, google_books_id, added_by_admin_id, created_at, imported_at, manga_type, reading_direction, series_count, volume, imprint, age_rating, web_url, story_arc, is_black_and_white, metadata_notes, community_rating, alternate_info, scan_information, summary, library_type_name)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28, $29, $30, $31, $32, $33, $34, $35, $36, $37, $38, $39, $40, $41, $42, $43, $44)
|
||||
ON CONFLICT (library_id, file_path) DO UPDATE SET updated_at = NOW()
|
||||
RETURNING id, library_id, title, author, isbn, description, file_path, file_size, mime_type, cover_image_path, series, series_number, tags, asin, date_published, publisher, contributors, language, edition, page_count, genre, copyright_year, goodreads_id, openlibrary_id, google_books_id, added_by_admin_id, created_at, imported_at, updated_at, format_group, format_mimetype, is_reflowable, has_fixed_layout, total_characters, chapter_count, entitlement_id, revision_number, kobo_content_id, kobo_metadata, manga_type, reading_direction, series_count, volume, imprint, age_rating, web_url, story_arc, is_black_and_white, metadata_notes, community_rating, alternate_info, scan_information, summary, chapter_metadata, library_type_name, tags_search, contributors_search, file_sha256, opf_identifier, opf_uuid, hash_confidence
|
||||
`
|
||||
|
||||
@@ -1044,6 +1045,11 @@ const CreateMediaItemFormat = `-- name: CreateMediaItemFormat :one
|
||||
|
||||
INSERT INTO media_item_formats (media_item_id, format_type, file_path, file_sha256, file_size_bytes, mime_type, converted_from_format_id)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7)
|
||||
ON CONFLICT (media_item_id, format_type) DO UPDATE SET
|
||||
file_path = EXCLUDED.file_path,
|
||||
file_sha256 = EXCLUDED.file_sha256,
|
||||
file_size_bytes = EXCLUDED.file_size_bytes,
|
||||
mime_type = EXCLUDED.mime_type
|
||||
RETURNING id, media_item_id, format_type, file_path, file_sha256, file_size_bytes, mime_type, created_at, converted_from_format_id
|
||||
`
|
||||
|
||||
@@ -5312,6 +5318,85 @@ func (q *Queries) GetMediaItemBySHA256(ctx context.Context, fileSha256 pgtype.Te
|
||||
return i, err
|
||||
}
|
||||
|
||||
const GetMediaItemBySHA256AndLibrary = `-- name: GetMediaItemBySHA256AndLibrary :one
|
||||
SELECT id, library_id, title, author, isbn, description, file_path, file_size, mime_type, cover_image_path, series, series_number, tags, asin, date_published, publisher, contributors, language, edition, page_count, genre, copyright_year, goodreads_id, openlibrary_id, google_books_id, added_by_admin_id, created_at, imported_at, updated_at, format_group, format_mimetype, is_reflowable, has_fixed_layout, total_characters, chapter_count, entitlement_id, revision_number, kobo_content_id, kobo_metadata, manga_type, reading_direction, series_count, volume, imprint, age_rating, web_url, story_arc, is_black_and_white, metadata_notes, community_rating, alternate_info, scan_information, summary, chapter_metadata, library_type_name, tags_search, contributors_search, file_sha256, opf_identifier, opf_uuid, hash_confidence FROM media_items WHERE file_sha256 = $1 AND library_id = $2
|
||||
`
|
||||
|
||||
type GetMediaItemBySHA256AndLibraryParams struct {
|
||||
FileSha256 pgtype.Text `db:"file_sha256" json:"file_sha256"`
|
||||
LibraryID pgtype.UUID `db:"library_id" json:"library_id"`
|
||||
}
|
||||
|
||||
// Get media item by SHA-256 hash within a specific library (content dedup)
|
||||
func (q *Queries) GetMediaItemBySHA256AndLibrary(ctx context.Context, arg GetMediaItemBySHA256AndLibraryParams) (MediaItems, error) {
|
||||
row := q.db.QueryRow(ctx, GetMediaItemBySHA256AndLibrary, arg.FileSha256, arg.LibraryID)
|
||||
var i MediaItems
|
||||
err := row.Scan(
|
||||
&i.ID,
|
||||
&i.LibraryID,
|
||||
&i.Title,
|
||||
&i.Author,
|
||||
&i.Isbn,
|
||||
&i.Description,
|
||||
&i.FilePath,
|
||||
&i.FileSize,
|
||||
&i.MimeType,
|
||||
&i.CoverImagePath,
|
||||
&i.Series,
|
||||
&i.SeriesNumber,
|
||||
&i.Tags,
|
||||
&i.Asin,
|
||||
&i.DatePublished,
|
||||
&i.Publisher,
|
||||
&i.Contributors,
|
||||
&i.Language,
|
||||
&i.Edition,
|
||||
&i.PageCount,
|
||||
&i.Genre,
|
||||
&i.CopyrightYear,
|
||||
&i.GoodreadsID,
|
||||
&i.OpenlibraryID,
|
||||
&i.GoogleBooksID,
|
||||
&i.AddedByAdminID,
|
||||
&i.CreatedAt,
|
||||
&i.ImportedAt,
|
||||
&i.UpdatedAt,
|
||||
&i.FormatGroup,
|
||||
&i.FormatMimetype,
|
||||
&i.IsReflowable,
|
||||
&i.HasFixedLayout,
|
||||
&i.TotalCharacters,
|
||||
&i.ChapterCount,
|
||||
&i.EntitlementID,
|
||||
&i.RevisionNumber,
|
||||
&i.KoboContentID,
|
||||
&i.KoboMetadata,
|
||||
&i.MangaType,
|
||||
&i.ReadingDirection,
|
||||
&i.SeriesCount,
|
||||
&i.Volume,
|
||||
&i.Imprint,
|
||||
&i.AgeRating,
|
||||
&i.WebUrl,
|
||||
&i.StoryArc,
|
||||
&i.IsBlackAndWhite,
|
||||
&i.MetadataNotes,
|
||||
&i.CommunityRating,
|
||||
&i.AlternateInfo,
|
||||
&i.ScanInformation,
|
||||
&i.Summary,
|
||||
&i.ChapterMetadata,
|
||||
&i.LibraryTypeName,
|
||||
&i.TagsSearch,
|
||||
&i.ContributorsSearch,
|
||||
&i.FileSha256,
|
||||
&i.OpfIdentifier,
|
||||
&i.OpfUuid,
|
||||
&i.HashConfidence,
|
||||
)
|
||||
return i, err
|
||||
}
|
||||
|
||||
const GetMediaItemFormatBySHA256 = `-- name: GetMediaItemFormatBySHA256 :one
|
||||
SELECT id, media_item_id, format_type, file_path, file_sha256, file_size_bytes, mime_type, created_at, converted_from_format_id FROM media_item_formats WHERE file_sha256 = $1
|
||||
`
|
||||
|
||||
@@ -149,6 +149,7 @@ GROUP BY l.id;
|
||||
-- name: CreateMediaItem :one
|
||||
INSERT INTO media_items (library_id, title, author, isbn, description, file_path, file_size, mime_type, cover_image_path, series, series_number, tags, tags_search, asin, date_published, publisher, contributors, contributors_search, language, edition, page_count, genre, copyright_year, goodreads_id, openlibrary_id, google_books_id, added_by_admin_id, created_at, imported_at, manga_type, reading_direction, series_count, volume, imprint, age_rating, web_url, story_arc, is_black_and_white, metadata_notes, community_rating, alternate_info, scan_information, summary, library_type_name)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28, $29, $30, $31, $32, $33, $34, $35, $36, $37, $38, $39, $40, $41, $42, $43, $44)
|
||||
ON CONFLICT (library_id, file_path) DO UPDATE SET updated_at = NOW()
|
||||
RETURNING *;
|
||||
|
||||
-- name: GetMediaItem :one
|
||||
@@ -1707,6 +1708,10 @@ RETURNING *;
|
||||
-- name: GetMediaItemBySHA256 :one
|
||||
SELECT * FROM media_items WHERE file_sha256 = $1;
|
||||
|
||||
-- Get media item by SHA-256 hash within a specific library (content dedup)
|
||||
-- name: GetMediaItemBySHA256AndLibrary :one
|
||||
SELECT * FROM media_items WHERE file_sha256 = $1 AND library_id = $2;
|
||||
|
||||
-- Get media item by OPF identifier
|
||||
-- name: GetMediaItemByOPFIdentifier :one
|
||||
SELECT * FROM media_items WHERE opf_identifier = $1;
|
||||
@@ -1754,6 +1759,11 @@ ORDER BY confidence_score DESC;
|
||||
-- name: CreateMediaItemFormat :one
|
||||
INSERT INTO media_item_formats (media_item_id, format_type, file_path, file_sha256, file_size_bytes, mime_type, converted_from_format_id)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7)
|
||||
ON CONFLICT (media_item_id, format_type) DO UPDATE SET
|
||||
file_path = EXCLUDED.file_path,
|
||||
file_sha256 = EXCLUDED.file_sha256,
|
||||
file_size_bytes = EXCLUDED.file_size_bytes,
|
||||
mime_type = EXCLUDED.mime_type
|
||||
RETURNING *;
|
||||
|
||||
-- Get media item formats
|
||||
|
||||
@@ -735,6 +735,26 @@ func (s *MediaScanner) processMediaFile(ctx context.Context, path string) (bool,
|
||||
path, hashInfo.FileSHA256, hashInfo.OPFIdentifier, hashInfo.OPFUUID, hashInfo.HashConfidence)
|
||||
}
|
||||
|
||||
// Content dedup: if an item with the same SHA-256 already exists in this
|
||||
// library (same file at a different path), treat it as existing rather than
|
||||
// creating a duplicate. The file bytes are identical, so metadata matches.
|
||||
if hashInfo.FileSHA256 != "" {
|
||||
existingByHash, err := s.db.GetMediaItemBySHA256AndLibrary(ctx, database.GetMediaItemBySHA256AndLibraryParams{
|
||||
FileSha256: pgtype.Text{String: hashInfo.FileSHA256, Valid: true},
|
||||
LibraryID: libraryID,
|
||||
})
|
||||
if err == nil && existingByHash.ID.Valid {
|
||||
fmt.Printf("Media item with same SHA-256 already exists in library (path %q), skipping duplicate: %s\n",
|
||||
existingByHash.FilePath, path)
|
||||
if s.forceRescan {
|
||||
_ = s.updateMediaItem(ctx, existingByHash.ID, path, info)
|
||||
}
|
||||
return false, nil
|
||||
} else if err != nil && !errors.Is(err, pgx.ErrNoRows) {
|
||||
fmt.Printf("Warning: failed to check media item by SHA-256 for %s: %v\n", path, err)
|
||||
}
|
||||
}
|
||||
|
||||
// REMOVED: Comic metadata extraction now handled by mergeMetadata()
|
||||
// This avoids duplicate extraction and ensures smart merging happens
|
||||
|
||||
|
||||
@@ -0,0 +1,213 @@
|
||||
-- scripts/dedup_media_items.sql
|
||||
--
|
||||
-- Detects and removes duplicate media_items, re-parenting all child rows
|
||||
-- (reading progress, highlights, collections, etc.) onto a single survivor
|
||||
-- before deleting the losers.
|
||||
--
|
||||
-- Two kinds of duplicates are handled:
|
||||
-- 1. PATH duplicates — same (library_id, file_path), multiple rows.
|
||||
-- These block the UNIQUE(library_id, file_path)
|
||||
-- constraint added by the schema migration.
|
||||
-- 2. CONTENT duplicates — same file_sha256 within a library, different paths.
|
||||
-- Same file imported twice under two names.
|
||||
--
|
||||
-- This script is IDEMPOTENT: re-running it is a no-op once the data is clean.
|
||||
-- It is safe to run against any Bookhoard database, before or after upgrading.
|
||||
--
|
||||
-- Usage:
|
||||
-- psql -h <host> -U postgres -d bookhoard -f scripts/dedup_media_items.sql
|
||||
--
|
||||
-- The first section is a DRY-RUN report (SELECTs only, no writes). The cleanup
|
||||
-- runs inside an explicit transaction. Comment out the cleanup block to inspect
|
||||
-- first.
|
||||
|
||||
-- =====================================================================
|
||||
-- DRY RUN: report duplicates (no writes)
|
||||
-- ======================================================================
|
||||
|
||||
\echo '=== PATH duplicates (library_id + file_path) ==='
|
||||
SELECT library_id,
|
||||
file_path,
|
||||
COUNT(*) AS dupes,
|
||||
array_agg(id::text) AS media_item_ids,
|
||||
array_agg(COALESCE(file_sha256::text, 'NULL')) AS hashes
|
||||
FROM media_items
|
||||
GROUP BY library_id, file_path
|
||||
HAVING COUNT(*) > 1
|
||||
ORDER BY COUNT(*) DESC;
|
||||
|
||||
\echo '=== CONTENT duplicates (same file_sha256 within a library, different paths) ==='
|
||||
SELECT library_id,
|
||||
file_sha256::text AS hash,
|
||||
COUNT(*) AS dupes,
|
||||
array_agg(file_path) AS paths,
|
||||
array_agg(id::text) AS media_item_ids
|
||||
FROM media_items
|
||||
WHERE file_sha256 IS NOT NULL
|
||||
GROUP BY library_id, file_sha256
|
||||
HAVING COUNT(*) > 1
|
||||
ORDER BY COUNT(*) DESC;
|
||||
|
||||
\echo '=== Child-row counts per duplicate candidate (helps confirm survivor choice) ==='
|
||||
SELECT mi.id,
|
||||
mi.library_id,
|
||||
mi.file_path,
|
||||
(SELECT COUNT(*) FROM reading_progress rp WHERE rp.media_item_id = mi.id) AS progress,
|
||||
(SELECT COUNT(*) FROM media_highlights mh WHERE mh.media_item_id = mi.id) AS highlights,
|
||||
(SELECT COUNT(*) FROM media_bookmarks mb WHERE mb.media_item_id = mi.id) AS bookmarks,
|
||||
(SELECT COUNT(*) FROM media_notes mn WHERE mn.media_item_id = mi.id) AS notes,
|
||||
(SELECT COUNT(*) FROM reading_history rh WHERE rh.media_item_id = mi.id) AS history,
|
||||
(SELECT COUNT(*) FROM collection_items ci WHERE ci.media_item_id = mi.id) AS collections
|
||||
FROM media_items mi
|
||||
WHERE (mi.library_id, mi.file_path) IN (
|
||||
SELECT library_id, file_path FROM media_items
|
||||
GROUP BY library_id, file_path HAVING COUNT(*) > 1
|
||||
)
|
||||
ORDER BY mi.library_id, mi.file_path, mi.id;
|
||||
|
||||
-- =====================================================================
|
||||
-- HELPER FUNCTIONS (also defined by schema.sql; CREATE OR REPLACE keeps them in sync)
|
||||
-- ======================================================================
|
||||
|
||||
-- Move every child row that points at p_source so it points at p_target,
|
||||
-- deleting any source rows that would violate a UNIQUE constraint on the
|
||||
-- target. Idempotent; no-op when p_target = p_source.
|
||||
CREATE OR REPLACE FUNCTION reparent_media_item_children(p_target UUID, p_source UUID)
|
||||
RETURNS void
|
||||
LANGUAGE plpgsql
|
||||
AS $$
|
||||
BEGIN
|
||||
IF p_target IS NULL OR p_source IS NULL OR p_target = p_source THEN
|
||||
RETURN;
|
||||
END IF;
|
||||
|
||||
-- reading_progress (UNIQUE media_item_id, user_id)
|
||||
DELETE FROM reading_progress
|
||||
WHERE media_item_id = p_source
|
||||
AND user_id IN (SELECT user_id FROM reading_progress WHERE media_item_id = p_target);
|
||||
UPDATE reading_progress SET media_item_id = p_target WHERE media_item_id = p_source;
|
||||
|
||||
-- reading_speed (UNIQUE user_id, media_item_id)
|
||||
DELETE FROM reading_speed
|
||||
WHERE media_item_id = p_source
|
||||
AND user_id IN (SELECT user_id FROM reading_speed WHERE media_item_id = p_target);
|
||||
UPDATE reading_speed SET media_item_id = p_target WHERE media_item_id = p_source;
|
||||
|
||||
-- media_ratings (UNIQUE media_item_id, user_id)
|
||||
DELETE FROM media_ratings
|
||||
WHERE media_item_id = p_source
|
||||
AND user_id IN (SELECT user_id FROM media_ratings WHERE media_item_id = p_target);
|
||||
UPDATE media_ratings SET media_item_id = p_target WHERE media_item_id = p_source;
|
||||
|
||||
-- media_bookmarks (UNIQUE media_item_id, user_id, title)
|
||||
DELETE FROM media_bookmarks
|
||||
WHERE media_item_id = p_source
|
||||
AND (user_id, title) IN (SELECT user_id, title FROM media_bookmarks WHERE media_item_id = p_target);
|
||||
UPDATE media_bookmarks SET media_item_id = p_target WHERE media_item_id = p_source;
|
||||
|
||||
-- media_item_formats (UNIQUE media_item_id, format_type)
|
||||
DELETE FROM media_item_formats
|
||||
WHERE media_item_id = p_source
|
||||
AND format_type IN (SELECT format_type FROM media_item_formats WHERE media_item_id = p_target);
|
||||
UPDATE media_item_formats SET media_item_id = p_target WHERE media_item_id = p_source;
|
||||
|
||||
-- collection_items (UNIQUE collection_id, media_item_id)
|
||||
DELETE FROM collection_items
|
||||
WHERE media_item_id = p_source
|
||||
AND collection_id IN (SELECT collection_id FROM collection_items WHERE media_item_id = p_target);
|
||||
UPDATE collection_items SET media_item_id = p_target WHERE media_item_id = p_source;
|
||||
|
||||
-- kobo_shelves (UNIQUE device_id, media_item_id)
|
||||
DELETE FROM kobo_shelves
|
||||
WHERE media_item_id = p_source
|
||||
AND device_id IN (SELECT device_id FROM kobo_shelves WHERE media_item_id = p_target);
|
||||
UPDATE kobo_shelves SET media_item_id = p_target WHERE media_item_id = p_source;
|
||||
|
||||
-- panel_data (UNIQUE media_item_id, page_number)
|
||||
DELETE FROM panel_data
|
||||
WHERE media_item_id = p_source
|
||||
AND page_number IN (SELECT page_number FROM panel_data WHERE media_item_id = p_target);
|
||||
UPDATE panel_data SET media_item_id = p_target WHERE media_item_id = p_source;
|
||||
|
||||
-- processing_issues (UNIQUE media_item_id, issue_type)
|
||||
DELETE FROM processing_issues
|
||||
WHERE media_item_id = p_source
|
||||
AND issue_type IN (SELECT issue_type FROM processing_issues WHERE media_item_id = p_target);
|
||||
UPDATE processing_issues SET media_item_id = p_target WHERE media_item_id = p_source;
|
||||
|
||||
-- device_file_aliases (UNIQUE device_id, file_path) — paths may collide
|
||||
DELETE FROM device_file_aliases
|
||||
WHERE media_item_id = p_source
|
||||
AND (device_id, file_path) IN (SELECT device_id, file_path FROM device_file_aliases WHERE media_item_id = p_target);
|
||||
UPDATE device_file_aliases SET media_item_id = p_target WHERE media_item_id = p_source;
|
||||
|
||||
-- Tables whose UNIQUE keys do not include media_item_id: plain re-parent.
|
||||
UPDATE device_catalogs SET media_item_id = p_target WHERE media_item_id = p_source;
|
||||
UPDATE kobo_entitlements SET media_item_id = p_target WHERE media_item_id = p_source;
|
||||
UPDATE media_highlights SET media_item_id = p_target WHERE media_item_id = p_source;
|
||||
UPDATE media_notes SET media_item_id = p_target WHERE media_item_id = p_source;
|
||||
UPDATE reading_history SET media_item_id = p_target WHERE media_item_id = p_source;
|
||||
UPDATE sync_conflicts SET media_item_id = p_target WHERE media_item_id = p_source;
|
||||
UPDATE sync_queue SET media_item_id = p_target WHERE media_item_id = p_source;
|
||||
END;
|
||||
$$;
|
||||
|
||||
-- Collapse every (library_id, file_path) group into a single row.
|
||||
-- Survivor = the row with the most user data; ties broken by lowest id.
|
||||
CREATE OR REPLACE FUNCTION dedup_media_items_by_path() RETURNS void
|
||||
LANGUAGE plpgsql
|
||||
AS $$
|
||||
DECLARE
|
||||
g RECORD;
|
||||
v_surv UUID;
|
||||
v_loser UUID;
|
||||
BEGIN
|
||||
FOR g IN
|
||||
SELECT library_id, file_path
|
||||
FROM media_items
|
||||
GROUP BY library_id, file_path
|
||||
HAVING COUNT(*) > 1
|
||||
LOOP
|
||||
SELECT mi.id INTO v_surv
|
||||
FROM media_items mi
|
||||
WHERE mi.library_id = g.library_id AND mi.file_path = g.file_path
|
||||
ORDER BY
|
||||
((SELECT COUNT(*) FROM reading_progress rp WHERE rp.media_item_id = mi.id)
|
||||
+ (SELECT COUNT(*) FROM media_highlights mh WHERE mh.media_item_id = mi.id)
|
||||
+ (SELECT COUNT(*) FROM media_bookmarks mb WHERE mb.media_item_id = mi.id)
|
||||
+ (SELECT COUNT(*) FROM media_notes mn WHERE mn.media_item_id = mi.id)
|
||||
+ (SELECT COUNT(*) FROM reading_history rh WHERE rh.media_item_id = mi.id)
|
||||
+ (SELECT COUNT(*) FROM collection_items ci WHERE ci.media_item_id = mi.id)) DESC,
|
||||
mi.id ASC
|
||||
LIMIT 1;
|
||||
|
||||
FOR v_loser IN
|
||||
SELECT id FROM media_items
|
||||
WHERE library_id = g.library_id AND file_path = g.file_path AND id <> v_surv
|
||||
ORDER BY id
|
||||
LOOP
|
||||
PERFORM reparent_media_item_children(v_surv, v_loser);
|
||||
DELETE FROM media_items WHERE id = v_loser;
|
||||
END LOOP;
|
||||
END LOOP;
|
||||
END;
|
||||
$$;
|
||||
|
||||
-- ======================================================================
|
||||
-- CLEANUP: collapse path duplicates (required before the UNIQUE constraint)
|
||||
-- ======================================================================
|
||||
|
||||
\echo '=== Collapsing path duplicates ===';
|
||||
BEGIN;
|
||||
SELECT dedup_media_items_by_path();
|
||||
COMMIT;
|
||||
|
||||
\echo '=== Done. Remaining PATH duplicates (should be empty): ===';
|
||||
SELECT library_id, file_path, COUNT(*) AS dupes
|
||||
FROM media_items
|
||||
GROUP BY library_id, file_path
|
||||
HAVING COUNT(*) > 1;
|
||||
|
||||
\echo 'NOTE: CONTENT duplicates (same hash, different paths) are NOT auto-deleted.'
|
||||
\echo ' They do not violate the UNIQUE constraint. Review the dry-run output'
|
||||
\echo ' above and merge them manually if desired.'
|
||||
Reference in New Issue
Block a user