From 14445a7c3fc395cd9e5024cc30aff0b5aa82597b Mon Sep 17 00:00:00 2001 From: John O'Keefe Date: Sat, 12 Sep 2026 17:50:08 -0400 Subject: [PATCH] feat(scanner): archive-at-two-scans lifecycle for files missing from disk Replace the silent hard-delete orphan cleanup (which logged only through ScannerLogger file logs and whose failure paths left rows undetected) with an archive lifecycle that preserves reading history: - A file missing in one scan is marked (missing_scan_count = 1); missing in a second consecutive scan archives it (archived_at, hidden from browsing, progress/notes/highlights survive). Every branch logs to stdout with an [ARCHIVE] prefix so skips are always visible. - When a file reappears - same path, or identical content at a new path via the SHA-256 dedup match - the archived state clears automatically and the item returns with its history intact. - Archived rows older than ARCHIVE_RETENTION_DAYS are hard-purged at scan time (cascading deletes); 0 disables auto-purge for manual-only management. Retention is read from the environment in NewMediaScanner. --- internal/config/config.go | 11 +++ internal/services/media_scanner.go | 141 +++++++++++++++++++---------- 2 files changed, 105 insertions(+), 47 deletions(-) diff --git a/internal/config/config.go b/internal/config/config.go index 137334f..64db42d 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -84,3 +84,14 @@ func getEnvInt(key string, defaultValue int) int { } return defaultValue } + +// ArchiveRetentionDays returns how many days a media item stays archived +// (file missing from disk for two consecutive scans) before library scans +// purge it for good. Reading progress, notes, and highlights survive the +// archive window and are restored if the file returns; the purge deletes +// them along with the row. +// Configure via ARCHIVE_RETENTION_DAYS (default 90); 0 keeps archived items +// until an admin purges them manually from the library admin page. +func ArchiveRetentionDays() int { + return getEnvInt("ARCHIVE_RETENTION_DAYS", 90) +} diff --git a/internal/services/media_scanner.go b/internal/services/media_scanner.go index 2b20766..2a29a92 100644 --- a/internal/services/media_scanner.go +++ b/internal/services/media_scanner.go @@ -5,6 +5,7 @@ package services import ( "archive/tar" "archive/zip" + "bookhoard/internal/config" "bookhoard/internal/database" "bookhoard/internal/utils" "bytes" @@ -105,22 +106,23 @@ type FormatInfo struct { // MediaScanner scans library folders for media files (ebooks, comics, manga) type MediaScanner struct { - db *database.Queries - watcher *fsnotify.Watcher - folders []string - adminID pgtype.UUID - defaultLibraryID pgtype.UUID - libraryTypes map[string][]string - forceRescan bool - logger *ScannerLogger - dirtyDirs map[string]time.Time - dirtyDirsMu sync.RWMutex - fileStability map[string]*atomic.Bool - fileStabilityMu sync.RWMutex - scanMutex sync.Mutex - scanInProgress atomic.Bool - watching atomic.Bool - settingsCache *SettingsCache + db *database.Queries + watcher *fsnotify.Watcher + folders []string + adminID pgtype.UUID + defaultLibraryID pgtype.UUID + libraryTypes map[string][]string + forceRescan bool + archiveRetentionDays int + logger *ScannerLogger + dirtyDirs map[string]time.Time + dirtyDirsMu sync.RWMutex + fileStability map[string]*atomic.Bool + fileStabilityMu sync.RWMutex + scanMutex sync.Mutex + scanInProgress atomic.Bool + watching atomic.Bool + settingsCache *SettingsCache totalFiles int newItems int @@ -157,18 +159,19 @@ type CalibreOPFMetadata struct { // never closed. func NewMediaScanner(db *database.Queries) *MediaScanner { return &MediaScanner{ - db: db, - watcher: nil, - settingsCache: NewSettingsCache(30 * time.Second), - dirtyDirs: make(map[string]time.Time), - fileStability: make(map[string]*atomic.Bool), - watching: atomic.Bool{}, - scanInProgress: atomic.Bool{}, - folders: []string{}, - adminID: pgtype.UUID{}, - defaultLibraryID: pgtype.UUID{Valid: false}, - libraryTypes: make(map[string][]string), - logger: NewScannerLogger(), + db: db, + watcher: nil, + archiveRetentionDays: config.ArchiveRetentionDays(), + settingsCache: NewSettingsCache(30 * time.Second), + dirtyDirs: make(map[string]time.Time), + fileStability: make(map[string]*atomic.Bool), + watching: atomic.Bool{}, + scanInProgress: atomic.Bool{}, + folders: []string{}, + adminID: pgtype.UUID{}, + defaultLibraryID: pgtype.UUID{Valid: false}, + libraryTypes: make(map[string][]string), + logger: NewScannerLogger(), } } @@ -476,17 +479,23 @@ func (s *MediaScanner) ScanFolders(ctx context.Context) error { fmt.Printf("Scan completed: %d total files scanned, %d media files found, %d new items, %d errors\n", processedFiles, mediaFiles, s.newItems, s.errors) - // Clean up: Find media items in DB that no longer exist on filesystem + // Archive lifecycle pass. Items whose files vanished from disk are + // archived after two consecutive missing scans (reading history kept, + // item hidden), and purged for good once archived older than the + // retention window (ARCHIVE_RETENTION_DAYS; 0 = manual purge only). + // Every branch logs - the previous hard-delete cleanup failed silently + // and left orphaned rows undetected. for _, folder := range s.folders { lib, err := s.db.GetLibraryByFolder(ctx, folder) if err != nil { + fmt.Printf("[ARCHIVE] Warning: no library found for folder %s, skipping archive pass: %v\n", folder, err) continue } libraryID := lib.LibraryID - dbItems, err := s.db.ListMediaItemsByLibrary(ctx, libraryID) + dbItems, err := s.db.ListMediaItemsByLibraryIncludingArchived(ctx, libraryID) if err != nil { - fmt.Printf("Warning: failed to get library items for cleanup: %v\n", err) + fmt.Printf("[ARCHIVE] Warning: failed to get library items for archive pass: %v\n", err) continue } @@ -501,28 +510,46 @@ func (s *MediaScanner) ScanFolders(ctx context.Context) error { } return nil }); err != nil { - fmt.Printf("[RESCAN-CLEANUP] Warning: failed to walk directory %s, skipping orphan cleanup: %v\n", folder, err) - continue // Skip to next folder to avoid false deletions + fmt.Printf("[ARCHIVE] Warning: failed to walk directory %s, skipping archive pass: %v\n", folder, err) + continue // Skip to next folder to avoid false archivals } - // Delete items whose files no longer exist - with safety logging for _, item := range dbItems { - filePath := item.FilePath - if filePath != "" && !scannedPaths[filePath] { - msg := fmt.Sprintf("[RESCAN-CLEANUP] Orphaned media item found: ID=%s, Title=%s, Path=%s", - item.ID, item.Title, filePath) - s.logger.LogDelete(msg) + if item.FilePath == "" || scannedPaths[item.FilePath] { + continue // File present; unarchive is handled in processMediaFile + } - delMsg := fmt.Sprintf("[RESCAN-CLEANUP] Deleting orphaned item '%s' (file no longer exists at %s)", - item.Title, filePath) - s.logger.LogDelete(delMsg) + if item.ArchivedAt.Valid { + // Still missing and already archived: purge once past the + // retention window (0 = keep until manual purge). + if s.archiveRetentionDays > 0 && time.Now().AddDate(0, 0, -s.archiveRetentionDays).After(item.ArchivedAt.Time) { + if err := s.db.DeleteMediaItem(ctx, item.ID); err != nil { + fmt.Printf("[ARCHIVE] Error: failed to purge archived item %s: %v\n", item.Title, err) + s.logger.LogError(fmt.Sprintf("[ARCHIVE] ERROR: failed to purge archived item '%s': %v", item.Title, err)) + } else { + fmt.Printf("[ARCHIVE] Purged archived item '%s' (retention %d days): %s\n", item.Title, s.archiveRetentionDays, item.FilePath) + s.logger.LogDelete(fmt.Sprintf("[ARCHIVE] Purged archived item '%s' after retention window (file missing at %s)", item.Title, item.FilePath)) + } + } + continue + } - if err := s.db.DeleteMediaItem(ctx, item.ID); err != nil { - errMsg := fmt.Sprintf("[RESCAN-CLEANUP] ERROR: failed to delete orphaned item %s: %v", item.Title, err) - s.logger.LogDelete(errMsg) - s.logger.LogError(errMsg) + // Missing but not yet archived. + if item.MissingScanCount >= 1 { + // Second consecutive missing scan: archive it. + if err := s.db.ArchiveMediaItem(ctx, item.ID); err != nil { + fmt.Printf("[ARCHIVE] Error: failed to archive item %s: %v\n", item.Title, err) + s.logger.LogError(fmt.Sprintf("[ARCHIVE] ERROR: failed to archive item '%s': %v", item.Title, err)) } else { - s.logger.LogDelete(fmt.Sprintf("[RESCAN-CLEANUP] SUCCESS: deleted orphaned item '%s'", item.Title)) + fmt.Printf("[ARCHIVE] Archived item '%s' (missing from disk for %d scans): %s\n", item.Title, item.MissingScanCount+1, item.FilePath) + s.logger.LogDelete(fmt.Sprintf("[ARCHIVE] Archived item '%s' (file missing from disk at %s)", item.Title, item.FilePath)) + } + } else { + // First missing scan: mark, archive on the next one. + if err := s.db.MarkMediaItemMissing(ctx, item.ID); err != nil { + fmt.Printf("[ARCHIVE] Warning: failed to mark item missing %s: %v\n", item.Title, err) + } else { + fmt.Printf("[ARCHIVE] Item missing from disk (1/2 scans before archiving): %s\n", item.FilePath) } } } @@ -693,6 +720,17 @@ func (s *MediaScanner) processMediaFile(ctx context.Context, path string) (bool, if err == nil { fmt.Printf("Media item already exists in database: %s (size: %d vs %d)\n", path, existingItem.FileSize.Int64, info.Size()) + // The file is back on disk: lift any archive/missing state so the + // item reappears in libraries and future missing scans start fresh. + if existingItem.ArchivedAt.Valid || existingItem.MissingScanCount > 0 { + if err := s.db.ClearMediaItemArchive(ctx, existingItem.ID); err != nil { + fmt.Printf("Warning: failed to unarchive media item %s: %v\n", existingItem.FilePath, err) + } else if existingItem.ArchivedAt.Valid { + fmt.Printf("[ARCHIVE] Restored from archive, file is back: %s\n", existingItem.FilePath) + s.logger.LogDelete(fmt.Sprintf("[ARCHIVE] Restored archived item '%s' (file returned at %s)", existingItem.Title, existingItem.FilePath)) + } + } + // If force rescan is enabled, always re-process if s.forceRescan { fmt.Printf("Force rescan enabled, updating existing media item: %s\n", path) @@ -757,6 +795,15 @@ func (s *MediaScanner) processMediaFile(ctx context.Context, path string) (bool, if err == nil && existingByHash.ID.Valid { fmt.Printf("Media item with same SHA-256 already exists in library (path %q), skipping duplicate: %s\n", existingByHash.FilePath, path) + // Content returned (possibly at a new path): restore archived rows. + if existingByHash.ArchivedAt.Valid || existingByHash.MissingScanCount > 0 { + if err := s.db.ClearMediaItemArchive(ctx, existingByHash.ID); err != nil { + fmt.Printf("Warning: failed to unarchive media item %s: %v\n", existingByHash.FilePath, err) + } else if existingByHash.ArchivedAt.Valid { + fmt.Printf("[ARCHIVE] Restored from archive, identical content found at %s\n", path) + s.logger.LogDelete(fmt.Sprintf("[ARCHIVE] Restored archived item '%s' (identical content found at %s)", existingByHash.Title, path)) + } + } if s.forceRescan { _ = s.updateMediaItem(ctx, existingByHash, path) }