feat(scanner): recompute hashes on rescan and flag content duplicates
Force rescan was metadata-only: updateMediaItem never touched the hash identifiers, so a force scan could not backfill file_sha256 for items imported before hashing existed (or where extraction originally failed). Those items were invisible to content dedup and SHA-256 device matching with no way to fix short of delete + re-import. processMediaFile now refreshes hash identifiers in three cases: - force rescan (the admin Scan button becomes the backfill tool) - file size change (stored hash is stale - the bytes changed) - unchanged file with no stored hash (ordinary scans self-heal the legacy backlog incrementally, no admin action required) Each recompute runs recordHashConflictIfAny: when the freshly stored hash is now shared by more than one item in the library, the group is upserted into hash_conflicts for the admin Hash Conflicts page. The upsert is a no-op for already-tracked groups, so resolved 'keep both' decisions stick. Also extract a package-level computeFileSHA256 (the scanner method now delegates to it) so the startup backfill service can hash files without a scanner instance.
This commit is contained in:
@@ -699,14 +699,24 @@ func (s *MediaScanner) processMediaFile(ctx context.Context, path string) (bool,
|
||||
if err := s.updateMediaItem(ctx, existingItem.ID, path, info); err != nil {
|
||||
fmt.Printf("Warning: failed to update existing media item: %v\n", err)
|
||||
}
|
||||
// Recompute hash identifiers too - a force rescan is the admin's
|
||||
// backfill tool and must refresh stale or missing hashes.
|
||||
s.recomputeHashInfo(ctx, existingItem.ID, libraryID, path)
|
||||
return false, nil
|
||||
} else {
|
||||
// Normal behavior: check if file has changed (by size)
|
||||
if existingItem.FileSize.Int64 != info.Size() {
|
||||
fmt.Printf("File size changed, updating media item: %s\n", path)
|
||||
_ = s.updateMediaItem(ctx, existingItem.ID, path, info)
|
||||
// The bytes changed, so any stored hash is stale.
|
||||
s.recomputeHashInfo(ctx, existingItem.ID, libraryID, path)
|
||||
return false, nil
|
||||
}
|
||||
// Self-heal items imported before hashing existed: even an unchanged
|
||||
// file gets its hash computed if missing.
|
||||
if !existingItem.FileSha256.Valid || existingItem.FileSha256.String == "" {
|
||||
s.recomputeHashInfo(ctx, existingItem.ID, libraryID, path)
|
||||
}
|
||||
fmt.Printf("Media item already exists with same size, skipping: %s\n", path)
|
||||
return false, nil
|
||||
}
|
||||
@@ -2615,6 +2625,68 @@ func (s *MediaScanner) getMediaItemByFilePath(ctx context.Context, filePath stri
|
||||
})
|
||||
}
|
||||
|
||||
// recomputeHashInfo recomputes the file's hash identifiers and stores them on
|
||||
// the media item (plus its per-format row). Called on force rescan, on file
|
||||
// size change, and when an unchanged item is found with no stored hash, so
|
||||
// items imported before hashing existed are backfilled by ordinary scans.
|
||||
// After storing, it records a hash conflict if the same content now exists at
|
||||
// more than one path in the library.
|
||||
func (s *MediaScanner) recomputeHashInfo(ctx context.Context, mediaItemID pgtype.UUID, libraryID pgtype.UUID, path string) {
|
||||
hashInfo, formatInfo, err := s.extractHashInfo(path)
|
||||
if err != nil {
|
||||
fmt.Printf("Warning: failed to extract hash info from %s: %v\n", path, err)
|
||||
return
|
||||
}
|
||||
if hashInfo == nil || hashInfo.FileSHA256 == "" {
|
||||
return
|
||||
}
|
||||
|
||||
_, err = s.db.UpdateMediaItemIdentifiers(ctx, database.UpdateMediaItemIdentifiersParams{
|
||||
ID: mediaItemID,
|
||||
FileSha256: pgtype.Text{String: hashInfo.FileSHA256, Valid: true},
|
||||
OpfIdentifier: pgtype.Text{String: hashInfo.OPFIdentifier, Valid: hashInfo.OPFIdentifier != ""},
|
||||
OpfUuid: pgtype.Text{String: hashInfo.OPFUUID, Valid: hashInfo.OPFUUID != ""},
|
||||
HashConfidence: pgtype.Text{String: hashInfo.HashConfidence, Valid: hashInfo.HashConfidence != ""},
|
||||
})
|
||||
if err != nil {
|
||||
fmt.Printf("Warning: failed to update hash identifiers for %s: %v\n", path, err)
|
||||
return
|
||||
}
|
||||
|
||||
if formatInfo != nil {
|
||||
_, _ = s.db.CreateMediaItemFormat(ctx, database.CreateMediaItemFormatParams{
|
||||
MediaItemID: mediaItemID,
|
||||
FormatType: formatInfo.FormatType,
|
||||
FilePath: pgtype.Text{String: s.getRelativePath(formatInfo.FilePath), Valid: true},
|
||||
FileSha256: pgtype.Text{String: formatInfo.FileSHA256, Valid: true},
|
||||
FileSizeBytes: pgtype.Int8{Int64: formatInfo.FileSizeBytes, Valid: true},
|
||||
MimeType: pgtype.Text{String: formatInfo.MimeType, Valid: true},
|
||||
})
|
||||
}
|
||||
|
||||
s.recordHashConflictIfAny(ctx, libraryID, hashInfo.FileSHA256)
|
||||
}
|
||||
|
||||
// recordHashConflictIfAny flags a pending hash conflict when the given content
|
||||
// hash is now shared by more than one media item in the same library. The
|
||||
// upsert is a no-op for already-tracked (including resolved) groups.
|
||||
func (s *MediaScanner) recordHashConflictIfAny(ctx context.Context, libraryID pgtype.UUID, fileSHA256 string) {
|
||||
items, err := s.db.ListMediaItemsBySHA256AndLibrary(ctx, database.ListMediaItemsBySHA256AndLibraryParams{
|
||||
FileSha256: pgtype.Text{String: fileSHA256, Valid: true},
|
||||
LibraryID: libraryID,
|
||||
})
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
if len(items) > 1 {
|
||||
fmt.Printf("Hash conflict: %d media items share SHA-256 %s in one library\n", len(items), fileSHA256)
|
||||
_ = s.db.CreateHashConflict(ctx, database.CreateHashConflictParams{
|
||||
LibraryID: libraryID,
|
||||
FileSha256: fileSHA256,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func (s *MediaScanner) getMimeType(path string) string {
|
||||
ext := strings.ToLower(filepath.Ext(path))
|
||||
if mime, ok := MimeTypes[ext]; ok {
|
||||
@@ -3038,6 +3110,12 @@ func (s *MediaScanner) startBackupScan(ctx context.Context) {
|
||||
}
|
||||
|
||||
func (s *MediaScanner) calculateFileSHA256(filePath string) (string, error) {
|
||||
return computeFileSHA256(filePath)
|
||||
}
|
||||
|
||||
// computeFileSHA256 is the package-level full-file SHA-256 used by the hash
|
||||
// backfill service; the MediaScanner method delegates to it.
|
||||
func computeFileSHA256(filePath string) (string, error) {
|
||||
file, err := os.Open(filePath)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to open file: %v", err)
|
||||
|
||||
Reference in New Issue
Block a user