From 77990d0dc00510eb99220ee5341552ac7e49a65f Mon Sep 17 00:00:00 2001 From: John O'Keefe Date: Fri, 14 Aug 2026 08:52:18 -0400 Subject: [PATCH] feat(scanner): recompute hashes on rescan and flag content duplicates Force rescan was metadata-only: updateMediaItem never touched the hash identifiers, so a force scan could not backfill file_sha256 for items imported before hashing existed (or where extraction originally failed). Those items were invisible to content dedup and SHA-256 device matching with no way to fix short of delete + re-import. processMediaFile now refreshes hash identifiers in three cases: - force rescan (the admin Scan button becomes the backfill tool) - file size change (stored hash is stale - the bytes changed) - unchanged file with no stored hash (ordinary scans self-heal the legacy backlog incrementally, no admin action required) Each recompute runs recordHashConflictIfAny: when the freshly stored hash is now shared by more than one item in the library, the group is upserted into hash_conflicts for the admin Hash Conflicts page. The upsert is a no-op for already-tracked groups, so resolved 'keep both' decisions stick. Also extract a package-level computeFileSHA256 (the scanner method now delegates to it) so the startup backfill service can hash files without a scanner instance. --- internal/services/media_scanner.go | 78 ++++++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) diff --git a/internal/services/media_scanner.go b/internal/services/media_scanner.go index 73d0b8b..5fd2215 100644 --- a/internal/services/media_scanner.go +++ b/internal/services/media_scanner.go @@ -699,14 +699,24 @@ func (s *MediaScanner) processMediaFile(ctx context.Context, path string) (bool, if err := s.updateMediaItem(ctx, existingItem.ID, path, info); err != nil { fmt.Printf("Warning: failed to update existing media item: %v\n", err) } + // Recompute hash identifiers too - a force rescan is the admin's + // backfill tool and must refresh stale or missing hashes. + s.recomputeHashInfo(ctx, existingItem.ID, libraryID, path) return false, nil } else { // Normal behavior: check if file has changed (by size) if existingItem.FileSize.Int64 != info.Size() { fmt.Printf("File size changed, updating media item: %s\n", path) _ = s.updateMediaItem(ctx, existingItem.ID, path, info) + // The bytes changed, so any stored hash is stale. + s.recomputeHashInfo(ctx, existingItem.ID, libraryID, path) return false, nil } + // Self-heal items imported before hashing existed: even an unchanged + // file gets its hash computed if missing. + if !existingItem.FileSha256.Valid || existingItem.FileSha256.String == "" { + s.recomputeHashInfo(ctx, existingItem.ID, libraryID, path) + } fmt.Printf("Media item already exists with same size, skipping: %s\n", path) return false, nil } @@ -2615,6 +2625,68 @@ func (s *MediaScanner) getMediaItemByFilePath(ctx context.Context, filePath stri }) } +// recomputeHashInfo recomputes the file's hash identifiers and stores them on +// the media item (plus its per-format row). Called on force rescan, on file +// size change, and when an unchanged item is found with no stored hash, so +// items imported before hashing existed are backfilled by ordinary scans. +// After storing, it records a hash conflict if the same content now exists at +// more than one path in the library. +func (s *MediaScanner) recomputeHashInfo(ctx context.Context, mediaItemID pgtype.UUID, libraryID pgtype.UUID, path string) { + hashInfo, formatInfo, err := s.extractHashInfo(path) + if err != nil { + fmt.Printf("Warning: failed to extract hash info from %s: %v\n", path, err) + return + } + if hashInfo == nil || hashInfo.FileSHA256 == "" { + return + } + + _, err = s.db.UpdateMediaItemIdentifiers(ctx, database.UpdateMediaItemIdentifiersParams{ + ID: mediaItemID, + FileSha256: pgtype.Text{String: hashInfo.FileSHA256, Valid: true}, + OpfIdentifier: pgtype.Text{String: hashInfo.OPFIdentifier, Valid: hashInfo.OPFIdentifier != ""}, + OpfUuid: pgtype.Text{String: hashInfo.OPFUUID, Valid: hashInfo.OPFUUID != ""}, + HashConfidence: pgtype.Text{String: hashInfo.HashConfidence, Valid: hashInfo.HashConfidence != ""}, + }) + if err != nil { + fmt.Printf("Warning: failed to update hash identifiers for %s: %v\n", path, err) + return + } + + if formatInfo != nil { + _, _ = s.db.CreateMediaItemFormat(ctx, database.CreateMediaItemFormatParams{ + MediaItemID: mediaItemID, + FormatType: formatInfo.FormatType, + FilePath: pgtype.Text{String: s.getRelativePath(formatInfo.FilePath), Valid: true}, + FileSha256: pgtype.Text{String: formatInfo.FileSHA256, Valid: true}, + FileSizeBytes: pgtype.Int8{Int64: formatInfo.FileSizeBytes, Valid: true}, + MimeType: pgtype.Text{String: formatInfo.MimeType, Valid: true}, + }) + } + + s.recordHashConflictIfAny(ctx, libraryID, hashInfo.FileSHA256) +} + +// recordHashConflictIfAny flags a pending hash conflict when the given content +// hash is now shared by more than one media item in the same library. The +// upsert is a no-op for already-tracked (including resolved) groups. +func (s *MediaScanner) recordHashConflictIfAny(ctx context.Context, libraryID pgtype.UUID, fileSHA256 string) { + items, err := s.db.ListMediaItemsBySHA256AndLibrary(ctx, database.ListMediaItemsBySHA256AndLibraryParams{ + FileSha256: pgtype.Text{String: fileSHA256, Valid: true}, + LibraryID: libraryID, + }) + if err != nil { + return + } + if len(items) > 1 { + fmt.Printf("Hash conflict: %d media items share SHA-256 %s in one library\n", len(items), fileSHA256) + _ = s.db.CreateHashConflict(ctx, database.CreateHashConflictParams{ + LibraryID: libraryID, + FileSha256: fileSHA256, + }) + } +} + func (s *MediaScanner) getMimeType(path string) string { ext := strings.ToLower(filepath.Ext(path)) if mime, ok := MimeTypes[ext]; ok { @@ -3038,6 +3110,12 @@ func (s *MediaScanner) startBackupScan(ctx context.Context) { } func (s *MediaScanner) calculateFileSHA256(filePath string) (string, error) { + return computeFileSHA256(filePath) +} + +// computeFileSHA256 is the package-level full-file SHA-256 used by the hash +// backfill service; the MediaScanner method delegates to it. +func computeFileSHA256(filePath string) (string, error) { file, err := os.Open(filePath) if err != nil { return "", fmt.Errorf("failed to open file: %v", err)