fix(scanner): prevent duplicate media item imports
A read-then-write race in processMediaFile allowed the same file to be imported twice: two concurrent scan jobs (startup scan, fsnotify dirty- directory scan, periodic backup poll, or a manual scan each run on separate worker goroutines with separate MediaScanner instances) could both SELECT 'not found' and both INSERT. There was no transaction, no row lock, no unique constraint on (library_id, file_path), and no ON CONFLICT clause, so nothing stopped the double insert. Observed in production as two identical 'Head First SQL' rows created in the same second (same sha256, size, path, library). Database enforcement: - schema.sql: add UNIQUE(library_id, file_path) constraint, guarded so re-runs don't error - schema.sql: add self-healing migration that runs on every startup - dedup_media_items_by_path() collapses existing path-duplicates and reparent_media_item_children() moves all child rows (progress, highlights, bookmarks, notes, collections, formats, aliases, kobo entitlements, etc.) onto a survivor before deleting losers, so the constraint applies cleanly on already-duplicated servers without losing reading history. Survivor picks the row with the most user data, ties broken by lowest id - CreateMediaItem: upsert via ON CONFLICT (library_id, file_path) DO UPDATE so concurrent inserts collapse to one row and return it - CreateMediaItemFormat: upsert via ON CONFLICT (media_item_id, format_type), closing the same race on format rows Application-level guards: - media_scanner processMediaFile: after computing the file hash, check GetMediaItemBySHA256AndLibrary (new query) and treat the file as existing when identical content is already in the library under a different path (content dedup, library-scoped so multi-library setups still work) Ops tooling: - scripts/dedup_media_items.sql: standalone idempotent maintenance script with a dry-run report (path + content duplicate groups, child row counts) and transactional cleanup, for servers that prefer to dedup manually before upgrading Verified against the live database: the duplicate pair was collapsed (reading_progress preserved on the survivor), schema.sql re-runs are a no-op, and the constraint is in place with 62 unique books remaining.
This commit is contained in:
@@ -1395,3 +1395,141 @@ CREATE UNIQUE INDEX IF NOT EXISTS idx_media_bookmarks_dedup
|
||||
CREATE INDEX IF NOT EXISTS idx_media_highlights_deleted_at ON media_highlights(deleted_at) WHERE deleted = TRUE;
|
||||
CREATE INDEX IF NOT EXISTS idx_media_notes_deleted_at ON media_notes(deleted_at) WHERE deleted = TRUE;
|
||||
CREATE INDEX IF NOT EXISTS idx_media_bookmarks_deleted_at ON media_bookmarks(deleted_at) WHERE deleted = TRUE;
|
||||
|
||||
-- ============================================
|
||||
--: MEDIA ITEM DEDUPLICATION + PATH UNIQUENESS
|
||||
-- ============================================
|
||||
-- A read-then-write race in the scanner historically allowed the same
|
||||
-- (library_id, file_path) to be inserted twice. This block is self-healing:
|
||||
-- it collapses any existing path-duplicates (re-parenting child rows onto a
|
||||
-- survivor so no reading history is lost), then enforces uniqueness going
|
||||
-- forward. Idempotent — safe to re-run on every startup.
|
||||
|
||||
-- Move every child row that points at p_source so it points at p_target,
|
||||
-- deleting source rows that would violate a UNIQUE constraint on the target.
|
||||
CREATE OR REPLACE FUNCTION reparent_media_item_children(p_target UUID, p_source UUID)
|
||||
RETURNS void
|
||||
LANGUAGE plpgsql
|
||||
AS $$
|
||||
BEGIN
|
||||
IF p_target IS NULL OR p_source IS NULL OR p_target = p_source THEN
|
||||
RETURN;
|
||||
END IF;
|
||||
|
||||
DELETE FROM reading_progress
|
||||
WHERE media_item_id = p_source
|
||||
AND user_id IN (SELECT user_id FROM reading_progress WHERE media_item_id = p_target);
|
||||
UPDATE reading_progress SET media_item_id = p_target WHERE media_item_id = p_source;
|
||||
|
||||
DELETE FROM reading_speed
|
||||
WHERE media_item_id = p_source
|
||||
AND user_id IN (SELECT user_id FROM reading_speed WHERE media_item_id = p_target);
|
||||
UPDATE reading_speed SET media_item_id = p_target WHERE media_item_id = p_source;
|
||||
|
||||
DELETE FROM media_ratings
|
||||
WHERE media_item_id = p_source
|
||||
AND user_id IN (SELECT user_id FROM media_ratings WHERE media_item_id = p_target);
|
||||
UPDATE media_ratings SET media_item_id = p_target WHERE media_item_id = p_source;
|
||||
|
||||
DELETE FROM media_bookmarks
|
||||
WHERE media_item_id = p_source
|
||||
AND (user_id, title) IN (SELECT user_id, title FROM media_bookmarks WHERE media_item_id = p_target);
|
||||
UPDATE media_bookmarks SET media_item_id = p_target WHERE media_item_id = p_source;
|
||||
|
||||
DELETE FROM media_item_formats
|
||||
WHERE media_item_id = p_source
|
||||
AND format_type IN (SELECT format_type FROM media_item_formats WHERE media_item_id = p_target);
|
||||
UPDATE media_item_formats SET media_item_id = p_target WHERE media_item_id = p_source;
|
||||
|
||||
DELETE FROM collection_items
|
||||
WHERE media_item_id = p_source
|
||||
AND collection_id IN (SELECT collection_id FROM collection_items WHERE media_item_id = p_target);
|
||||
UPDATE collection_items SET media_item_id = p_target WHERE media_item_id = p_source;
|
||||
|
||||
DELETE FROM kobo_shelves
|
||||
WHERE media_item_id = p_source
|
||||
AND device_id IN (SELECT device_id FROM kobo_shelves WHERE media_item_id = p_target);
|
||||
UPDATE kobo_shelves SET media_item_id = p_target WHERE media_item_id = p_source;
|
||||
|
||||
DELETE FROM panel_data
|
||||
WHERE media_item_id = p_source
|
||||
AND page_number IN (SELECT page_number FROM panel_data WHERE media_item_id = p_target);
|
||||
UPDATE panel_data SET media_item_id = p_target WHERE media_item_id = p_source;
|
||||
|
||||
DELETE FROM processing_issues
|
||||
WHERE media_item_id = p_source
|
||||
AND issue_type IN (SELECT issue_type FROM processing_issues WHERE media_item_id = p_target);
|
||||
UPDATE processing_issues SET media_item_id = p_target WHERE media_item_id = p_source;
|
||||
|
||||
DELETE FROM device_file_aliases
|
||||
WHERE media_item_id = p_source
|
||||
AND (device_id, file_path) IN (SELECT device_id, file_path FROM device_file_aliases WHERE media_item_id = p_target);
|
||||
UPDATE device_file_aliases SET media_item_id = p_target WHERE media_item_id = p_source;
|
||||
|
||||
-- Tables whose UNIQUE keys do not include media_item_id.
|
||||
UPDATE device_catalogs SET media_item_id = p_target WHERE media_item_id = p_source;
|
||||
UPDATE kobo_entitlements SET media_item_id = p_target WHERE media_item_id = p_source;
|
||||
UPDATE media_highlights SET media_item_id = p_target WHERE media_item_id = p_source;
|
||||
UPDATE media_notes SET media_item_id = p_target WHERE media_item_id = p_source;
|
||||
UPDATE reading_history SET media_item_id = p_target WHERE media_item_id = p_source;
|
||||
UPDATE sync_conflicts SET media_item_id = p_target WHERE media_item_id = p_source;
|
||||
UPDATE sync_queue SET media_item_id = p_target WHERE media_item_id = p_source;
|
||||
END;
|
||||
$$;
|
||||
|
||||
-- Collapse every (library_id, file_path) group into a single row.
|
||||
-- Survivor = the row with the most user data; ties broken by lowest id.
|
||||
CREATE OR REPLACE FUNCTION dedup_media_items_by_path() RETURNS void
|
||||
LANGUAGE plpgsql
|
||||
AS $$
|
||||
DECLARE
|
||||
g RECORD;
|
||||
v_surv UUID;
|
||||
v_loser UUID;
|
||||
BEGIN
|
||||
FOR g IN
|
||||
SELECT library_id, file_path
|
||||
FROM media_items
|
||||
GROUP BY library_id, file_path
|
||||
HAVING COUNT(*) > 1
|
||||
LOOP
|
||||
SELECT mi.id INTO v_surv
|
||||
FROM media_items mi
|
||||
WHERE mi.library_id = g.library_id AND mi.file_path = g.file_path
|
||||
ORDER BY
|
||||
((SELECT COUNT(*) FROM reading_progress rp WHERE rp.media_item_id = mi.id)
|
||||
+ (SELECT COUNT(*) FROM media_highlights mh WHERE mh.media_item_id = mi.id)
|
||||
+ (SELECT COUNT(*) FROM media_bookmarks mb WHERE mb.media_item_id = mi.id)
|
||||
+ (SELECT COUNT(*) FROM media_notes mn WHERE mn.media_item_id = mi.id)
|
||||
+ (SELECT COUNT(*) FROM reading_history rh WHERE rh.media_item_id = mi.id)
|
||||
+ (SELECT COUNT(*) FROM collection_items ci WHERE ci.media_item_id = mi.id)) DESC,
|
||||
mi.id ASC
|
||||
LIMIT 1;
|
||||
|
||||
FOR v_loser IN
|
||||
SELECT id FROM media_items
|
||||
WHERE library_id = g.library_id AND file_path = g.file_path AND id <> v_surv
|
||||
ORDER BY id
|
||||
LOOP
|
||||
PERFORM reparent_media_item_children(v_surv, v_loser);
|
||||
DELETE FROM media_items WHERE id = v_loser;
|
||||
END LOOP;
|
||||
END LOOP;
|
||||
END;
|
||||
$$;
|
||||
|
||||
-- Collapse any existing path-duplicates so the constraint below can be created.
|
||||
SELECT dedup_media_items_by_path();
|
||||
|
||||
-- Enforce path uniqueness going forward (guarded so re-runs don't error).
|
||||
DO $$
|
||||
BEGIN
|
||||
IF NOT EXISTS (
|
||||
SELECT 1 FROM pg_constraint
|
||||
WHERE conname = 'media_items_library_id_file_path_key'
|
||||
AND conrelid = 'media_items'::regclass
|
||||
) THEN
|
||||
ALTER TABLE media_items
|
||||
ADD CONSTRAINT media_items_library_id_file_path_key UNIQUE (library_id, file_path);
|
||||
END IF;
|
||||
END $$;
|
||||
|
||||
Reference in New Issue
Block a user