feat(scanner): recompute hashes on rescan and flag content duplicates

Force rescan was metadata-only: updateMediaItem never touched the
hash identifiers, so a force scan could not backfill file_sha256 for
items imported before hashing existed (or where extraction originally
failed). Those items were invisible to content dedup and SHA-256
device matching with no way to fix short of delete + re-import.

processMediaFile now refreshes hash identifiers in three cases:
- force rescan (the admin Scan button becomes the backfill tool)
- file size change (stored hash is stale - the bytes changed)
- unchanged file with no stored hash (ordinary scans self-heal the
  legacy backlog incrementally, no admin action required)

Each recompute runs recordHashConflictIfAny: when the freshly stored
hash is now shared by more than one item in the library, the group is
upserted into hash_conflicts for the admin Hash Conflicts page. The
upsert is a no-op for already-tracked groups, so resolved 'keep both'
decisions stick.

Also extract a package-level computeFileSHA256 (the scanner method
now delegates to it) so the startup backfill service can hash files
without a scanner instance.
This commit is contained in:
2026-08-14 08:52:18 -04:00
parent 0c39e04e4a
commit 77990d0dc0
+78
View File
@@ -699,14 +699,24 @@ func (s *MediaScanner) processMediaFile(ctx context.Context, path string) (bool,
if err := s.updateMediaItem(ctx, existingItem.ID, path, info); err != nil {
fmt.Printf("Warning: failed to update existing media item: %v\n", err)
}
// Recompute hash identifiers too - a force rescan is the admin's
// backfill tool and must refresh stale or missing hashes.
s.recomputeHashInfo(ctx, existingItem.ID, libraryID, path)
return false, nil
} else {
// Normal behavior: check if file has changed (by size)
if existingItem.FileSize.Int64 != info.Size() {
fmt.Printf("File size changed, updating media item: %s\n", path)
_ = s.updateMediaItem(ctx, existingItem.ID, path, info)
// The bytes changed, so any stored hash is stale.
s.recomputeHashInfo(ctx, existingItem.ID, libraryID, path)
return false, nil
}
// Self-heal items imported before hashing existed: even an unchanged
// file gets its hash computed if missing.
if !existingItem.FileSha256.Valid || existingItem.FileSha256.String == "" {
s.recomputeHashInfo(ctx, existingItem.ID, libraryID, path)
}
fmt.Printf("Media item already exists with same size, skipping: %s\n", path)
return false, nil
}
@@ -2615,6 +2625,68 @@ func (s *MediaScanner) getMediaItemByFilePath(ctx context.Context, filePath stri
})
}
// recomputeHashInfo recomputes the file's hash identifiers and stores them on
// the media item (plus its per-format row). Called on force rescan, on file
// size change, and when an unchanged item is found with no stored hash, so
// items imported before hashing existed are backfilled by ordinary scans.
// After storing, it records a hash conflict if the same content now exists at
// more than one path in the library.
func (s *MediaScanner) recomputeHashInfo(ctx context.Context, mediaItemID pgtype.UUID, libraryID pgtype.UUID, path string) {
hashInfo, formatInfo, err := s.extractHashInfo(path)
if err != nil {
fmt.Printf("Warning: failed to extract hash info from %s: %v\n", path, err)
return
}
if hashInfo == nil || hashInfo.FileSHA256 == "" {
return
}
_, err = s.db.UpdateMediaItemIdentifiers(ctx, database.UpdateMediaItemIdentifiersParams{
ID: mediaItemID,
FileSha256: pgtype.Text{String: hashInfo.FileSHA256, Valid: true},
OpfIdentifier: pgtype.Text{String: hashInfo.OPFIdentifier, Valid: hashInfo.OPFIdentifier != ""},
OpfUuid: pgtype.Text{String: hashInfo.OPFUUID, Valid: hashInfo.OPFUUID != ""},
HashConfidence: pgtype.Text{String: hashInfo.HashConfidence, Valid: hashInfo.HashConfidence != ""},
})
if err != nil {
fmt.Printf("Warning: failed to update hash identifiers for %s: %v\n", path, err)
return
}
if formatInfo != nil {
_, _ = s.db.CreateMediaItemFormat(ctx, database.CreateMediaItemFormatParams{
MediaItemID: mediaItemID,
FormatType: formatInfo.FormatType,
FilePath: pgtype.Text{String: s.getRelativePath(formatInfo.FilePath), Valid: true},
FileSha256: pgtype.Text{String: formatInfo.FileSHA256, Valid: true},
FileSizeBytes: pgtype.Int8{Int64: formatInfo.FileSizeBytes, Valid: true},
MimeType: pgtype.Text{String: formatInfo.MimeType, Valid: true},
})
}
s.recordHashConflictIfAny(ctx, libraryID, hashInfo.FileSHA256)
}
// recordHashConflictIfAny flags a pending hash conflict when the given content
// hash is now shared by more than one media item in the same library. The
// upsert is a no-op for already-tracked (including resolved) groups.
func (s *MediaScanner) recordHashConflictIfAny(ctx context.Context, libraryID pgtype.UUID, fileSHA256 string) {
items, err := s.db.ListMediaItemsBySHA256AndLibrary(ctx, database.ListMediaItemsBySHA256AndLibraryParams{
FileSha256: pgtype.Text{String: fileSHA256, Valid: true},
LibraryID: libraryID,
})
if err != nil {
return
}
if len(items) > 1 {
fmt.Printf("Hash conflict: %d media items share SHA-256 %s in one library\n", len(items), fileSHA256)
_ = s.db.CreateHashConflict(ctx, database.CreateHashConflictParams{
LibraryID: libraryID,
FileSha256: fileSHA256,
})
}
}
func (s *MediaScanner) getMimeType(path string) string {
ext := strings.ToLower(filepath.Ext(path))
if mime, ok := MimeTypes[ext]; ok {
@@ -3038,6 +3110,12 @@ func (s *MediaScanner) startBackupScan(ctx context.Context) {
}
func (s *MediaScanner) calculateFileSHA256(filePath string) (string, error) {
return computeFileSHA256(filePath)
}
// computeFileSHA256 is the package-level full-file SHA-256 used by the hash
// backfill service; the MediaScanner method delegates to it.
func computeFileSHA256(filePath string) (string, error) {
file, err := os.Open(filePath)
if err != nil {
return "", fmt.Errorf("failed to open file: %v", err)