Bookhoard/bookhoard
0
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases 18 Wiki Activity

feat(scanner): archive-at-two-scans lifecycle for files missing from disk

Browse Source 
Replace the silent hard-delete orphan cleanup (which logged only through
ScannerLogger file logs and whose failure paths left rows undetected)
with an archive lifecycle that preserves reading history:

- A file missing in one scan is marked (missing_scan_count = 1); missing
  in a second consecutive scan archives it (archived_at, hidden from
  browsing, progress/notes/highlights survive). Every branch logs to
  stdout with an [ARCHIVE] prefix so skips are always visible.
- When a file reappears - same path, or identical content at a new path
  via the SHA-256 dedup match - the archived state clears automatically
  and the item returns with its history intact.
- Archived rows older than ARCHIVE_RETENTION_DAYS are hard-purged at
  scan time (cascading deletes); 0 disables auto-purge for manual-only
  management. Retention is read from the environment in NewMediaScanner.
This commit is contained in:
John O'Keefe
2026-09-12 17:50:15 -04:00
parent e6aceae0da
commit 14445a7c3f
2 changed files with 105 additions and 47 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/config/config.go
+11
View File
@@ -84,3 +84,14 @@ func getEnvInt(key string, defaultValue int) int {
}
return defaultValue
}
// ArchiveRetentionDays returns how many days a media item stays archived
// (file missing from disk for two consecutive scans) before library scans
// purge it for good. Reading progress, notes, and highlights survive the
// archive window and are restored if the file returns; the purge deletes
// them along with the row.
// Configure via ARCHIVE_RETENTION_DAYS (default 90); 0 keeps archived items
// until an admin purges them manually from the library admin page.
func ArchiveRetentionDays() int {
return getEnvInt("ARCHIVE_RETENTION_DAYS", 90)
}
internal/services/media_scanner.go
+94 -47
View File
@@ -5,6 +5,7 @@ package services
import (
"archive/tar"
"archive/zip"
"bookhoard/internal/config"
"bookhoard/internal/database"
"bookhoard/internal/utils"
"bytes"
@@ -105,22 +106,23 @@ type FormatInfo struct {
// MediaScanner scans library folders for media files (ebooks, comics, manga)
type MediaScanner struct {
db *database.Queries
watcher *fsnotify.Watcher
folders []string
adminID pgtype.UUID
defaultLibraryID pgtype.UUID
libraryTypes map[string][]string
forceRescan bool
logger *ScannerLogger
dirtyDirs map[string]time.Time
dirtyDirsMu sync.RWMutex
fileStability map[string]*atomic.Bool
fileStabilityMu sync.RWMutex
scanMutex sync.Mutex
scanInProgress atomic.Bool
watching atomic.Bool
settingsCache *SettingsCache
db *database.Queries
watcher *fsnotify.Watcher
folders []string
adminID pgtype.UUID
defaultLibraryID pgtype.UUID
libraryTypes map[string][]string
forceRescan bool
archiveRetentionDays int
logger *ScannerLogger
dirtyDirs map[string]time.Time
dirtyDirsMu sync.RWMutex
fileStability map[string]*atomic.Bool
fileStabilityMu sync.RWMutex
scanMutex sync.Mutex
scanInProgress atomic.Bool
watching atomic.Bool
settingsCache *SettingsCache
totalFiles int
newItems int
@@ -157,18 +159,19 @@ type CalibreOPFMetadata struct {
// never closed.
func NewMediaScanner(db *database.Queries) *MediaScanner {
return &MediaScanner{
db: db,
watcher: nil,
settingsCache: NewSettingsCache(30 * time.Second),
dirtyDirs: make(map[string]time.Time),
fileStability: make(map[string]*atomic.Bool),
watching: atomic.Bool{},
scanInProgress: atomic.Bool{},
folders: []string{},
adminID: pgtype.UUID{},
defaultLibraryID: pgtype.UUID{Valid: false},
libraryTypes: make(map[string][]string),
logger: NewScannerLogger(),
db: db,
watcher: nil,
archiveRetentionDays: config.ArchiveRetentionDays(),
settingsCache: NewSettingsCache(30 * time.Second),
dirtyDirs: make(map[string]time.Time),
fileStability: make(map[string]*atomic.Bool),
watching: atomic.Bool{},
scanInProgress: atomic.Bool{},
folders: []string{},
adminID: pgtype.UUID{},
defaultLibraryID: pgtype.UUID{Valid: false},
libraryTypes: make(map[string][]string),
logger: NewScannerLogger(),
}
}
@@ -476,17 +479,23 @@ func (s *MediaScanner) ScanFolders(ctx context.Context) error {
fmt.Printf("Scan completed: %d total files scanned, %d media files found, %d new items, %d errors\n",
processedFiles, mediaFiles, s.newItems, s.errors)
// Clean up: Find media items in DB that no longer exist on filesystem
// Archive lifecycle pass. Items whose files vanished from disk are
// archived after two consecutive missing scans (reading history kept,
// item hidden), and purged for good once archived older than the
// retention window (ARCHIVE_RETENTION_DAYS; 0 = manual purge only).
// Every branch logs - the previous hard-delete cleanup failed silently
// and left orphaned rows undetected.
for _, folder := range s.folders {
lib, err := s.db.GetLibraryByFolder(ctx, folder)
if err != nil {
fmt.Printf("[ARCHIVE] Warning: no library found for folder %s, skipping archive pass: %v\n", folder, err)
continue
}
libraryID := lib.LibraryID
dbItems, err := s.db.ListMediaItemsByLibrary(ctx, libraryID)
dbItems, err := s.db.ListMediaItemsByLibraryIncludingArchived(ctx, libraryID)
if err != nil {
fmt.Printf("Warning: failed to get library items for cleanup: %v\n", err)
fmt.Printf("[ARCHIVE] Warning: failed to get library items for archive pass: %v\n", err)
continue
}
@@ -501,28 +510,46 @@ func (s *MediaScanner) ScanFolders(ctx context.Context) error {
}
return nil
}); err != nil {
fmt.Printf("[RESCAN-CLEANUP] Warning: failed to walk directory %s, skipping orphan cleanup: %v\n", folder, err)
continue // Skip to next folder to avoid false deletions
fmt.Printf("[ARCHIVE] Warning: failed to walk directory %s, skipping archive pass: %v\n", folder, err)
continue // Skip to next folder to avoid false archivals
}
// Delete items whose files no longer exist - with safety logging
for _, item := range dbItems {
filePath := item.FilePath
if filePath != "" && !scannedPaths[filePath] {
msg := fmt.Sprintf("[RESCAN-CLEANUP] Orphaned media item found: ID=%s, Title=%s, Path=%s",
item.ID, item.Title, filePath)
s.logger.LogDelete(msg)
if item.FilePath == "" || scannedPaths[item.FilePath] {
continue // File present; unarchive is handled in processMediaFile
}
delMsg := fmt.Sprintf("[RESCAN-CLEANUP] Deleting orphaned item '%s' (file no longer exists at %s)",
item.Title, filePath)
s.logger.LogDelete(delMsg)
if item.ArchivedAt.Valid {
// Still missing and already archived: purge once past the
// retention window (0 = keep until manual purge).
if s.archiveRetentionDays > 0 && time.Now().AddDate(0, 0, -s.archiveRetentionDays).After(item.ArchivedAt.Time) {
if err := s.db.DeleteMediaItem(ctx, item.ID); err != nil {
fmt.Printf("[ARCHIVE] Error: failed to purge archived item %s: %v\n", item.Title, err)
s.logger.LogError(fmt.Sprintf("[ARCHIVE] ERROR: failed to purge archived item '%s': %v", item.Title, err))
} else {
fmt.Printf("[ARCHIVE] Purged archived item '%s' (retention %d days): %s\n", item.Title, s.archiveRetentionDays, item.FilePath)
s.logger.LogDelete(fmt.Sprintf("[ARCHIVE] Purged archived item '%s' after retention window (file missing at %s)", item.Title, item.FilePath))
}
}
continue
}
if err := s.db.DeleteMediaItem(ctx, item.ID); err != nil {
errMsg := fmt.Sprintf("[RESCAN-CLEANUP] ERROR: failed to delete orphaned item %s: %v", item.Title, err)
s.logger.LogDelete(errMsg)
s.logger.LogError(errMsg)
// Missing but not yet archived.
if item.MissingScanCount >= 1 {
// Second consecutive missing scan: archive it.
if err := s.db.ArchiveMediaItem(ctx, item.ID); err != nil {
fmt.Printf("[ARCHIVE] Error: failed to archive item %s: %v\n", item.Title, err)
s.logger.LogError(fmt.Sprintf("[ARCHIVE] ERROR: failed to archive item '%s': %v", item.Title, err))
} else {
s.logger.LogDelete(fmt.Sprintf("[RESCAN-CLEANUP] SUCCESS: deleted orphaned item '%s'", item.Title))
fmt.Printf("[ARCHIVE] Archived item '%s' (missing from disk for %d scans): %s\n", item.Title, item.MissingScanCount+1, item.FilePath)
s.logger.LogDelete(fmt.Sprintf("[ARCHIVE] Archived item '%s' (file missing from disk at %s)", item.Title, item.FilePath))
}
} else {
// First missing scan: mark, archive on the next one.
if err := s.db.MarkMediaItemMissing(ctx, item.ID); err != nil {
fmt.Printf("[ARCHIVE] Warning: failed to mark item missing %s: %v\n", item.Title, err)
} else {
fmt.Printf("[ARCHIVE] Item missing from disk (1/2 scans before archiving): %s\n", item.FilePath)
}
}
}
@@ -693,6 +720,17 @@ func (s *MediaScanner) processMediaFile(ctx context.Context, path string) (bool,
if err == nil {
fmt.Printf("Media item already exists in database: %s (size: %d vs %d)\n", path, existingItem.FileSize.Int64, info.Size())
// The file is back on disk: lift any archive/missing state so the
// item reappears in libraries and future missing scans start fresh.
if existingItem.ArchivedAt.Valid || existingItem.MissingScanCount > 0 {
if err := s.db.ClearMediaItemArchive(ctx, existingItem.ID); err != nil {
fmt.Printf("Warning: failed to unarchive media item %s: %v\n", existingItem.FilePath, err)
} else if existingItem.ArchivedAt.Valid {
fmt.Printf("[ARCHIVE] Restored from archive, file is back: %s\n", existingItem.FilePath)
s.logger.LogDelete(fmt.Sprintf("[ARCHIVE] Restored archived item '%s' (file returned at %s)", existingItem.Title, existingItem.FilePath))
}
}
// If force rescan is enabled, always re-process
if s.forceRescan {
fmt.Printf("Force rescan enabled, updating existing media item: %s\n", path)
@@ -757,6 +795,15 @@ func (s *MediaScanner) processMediaFile(ctx context.Context, path string) (bool,
if err == nil && existingByHash.ID.Valid {
fmt.Printf("Media item with same SHA-256 already exists in library (path %q), skipping duplicate: %s\n",
existingByHash.FilePath, path)
// Content returned (possibly at a new path): restore archived rows.
if existingByHash.ArchivedAt.Valid || existingByHash.MissingScanCount > 0 {
if err := s.db.ClearMediaItemArchive(ctx, existingByHash.ID); err != nil {
fmt.Printf("Warning: failed to unarchive media item %s: %v\n", existingByHash.FilePath, err)
} else if existingByHash.ArchivedAt.Valid {
fmt.Printf("[ARCHIVE] Restored from archive, identical content found at %s\n", path)
s.logger.LogDelete(fmt.Sprintf("[ARCHIVE] Restored archived item '%s' (identical content found at %s)", existingByHash.Title, path))
}
}
if s.forceRescan {
_ = s.updateMediaItem(ctx, existingByHash, path)
}