Bookhoard/bookhoard
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 12 Wiki Activity

scanner: fix library isolation, file mtime, force rescan, and deletion handling

Browse Source 
Fix 1 - File modification time for created_at:
- Get file.ModTime() in processMediaFile and pass to CreateMediaItem
- Modified SQL INSERT to include created_at column

Fix 2 - Force rescan UPDATE instead of DELETE+INSERT:
- Changed force rescan logic to call updateMediaItem instead of delete + create
- Preserves created_at timestamp on force rescan

Fix 3 - GetMediaItemByFilePath filters by library_id:
- Added library_id to WHERE clause in SQL query
- Created GetMediaItemByFilePathAnyLibrary for cross-library lookups (KOReader)
- Added SetLibraryID method to MediaScanner
- Updated handler to call SetLibraryID for watch mode

Fix 4 - File deletion handling with persistent logging:
- Added fsnotify.Remove handler in WatchChanges
- Added orphan cleanup in ScanFolders after scan completes
- Created scanner_logger.go with daily log rotation (7 days)
- Logs to /app/logs/scanner-deletes-YYYY-MM-DD.log and scanner-errors-YYYY-MM-DD.log
- Individual deletes with enhanced safety logging

Note: Integration tests can now safely scan /app/uploads because
GetMediaItemByFilePath now filters by library_id, preventing
cross-library interference.
This commit is contained in:
john-okeefe
2026-02-26 16:39:42 -05:00
parent 56c80fbe14
commit d802236874
14 changed files with 1432 additions and 83 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/services/media_scanner.go
+170 -27
View File
@@ -77,6 +77,7 @@ type MediaScanner struct {
defaultLibraryID pgtype.UUID
libraryTypes map[string][]string
forceRescan bool
logger *ScannerLogger
totalFiles int
newItems int
@@ -98,6 +99,7 @@ func NewMediaScanner(db *database.Queries) *MediaScanner {
adminID: pgtype.UUID{},
defaultLibraryID: pgtype.UUID{Valid: false},
libraryTypes: make(map[string][]string),
logger: NewScannerLogger(),
}
}
@@ -105,6 +107,10 @@ func (s *MediaScanner) SetAdminID(adminID pgtype.UUID) {
s.adminID = adminID
}
func (s *MediaScanner) SetLibraryID(libraryID pgtype.UUID) {
s.defaultLibraryID = libraryID
}
func (s *MediaScanner) SetForce(force bool) {
s.forceRescan = force
}
@@ -242,6 +248,55 @@ func (s *MediaScanner) ScanFolders(ctx context.Context) error {
fmt.Printf("Scan completed: %d total files scanned, %d media files found, %d new items, %d errors\n",
processedFiles, mediaFiles, s.newItems, s.errors)
// Clean up: Find media items in DB that no longer exist on filesystem
for _, folder := range s.folders {
lib, err := s.db.GetLibraryByFolder(ctx, folder)
if err != nil {
continue
}
libraryID := lib.LibraryID
dbItems, err := s.db.ListMediaItemsByLibrary(ctx, libraryID)
if err != nil {
fmt.Printf("Warning: failed to get library items for cleanup: %v\n", err)
continue
}
// Build set of scanned file paths for this folder
scannedPaths := make(map[string]bool)
filepath.WalkDir(folder, func(path string, d os.DirEntry, walkErr error) error {
if walkErr != nil {
return walkErr
}
if !d.IsDir() && s.isScannableFile(path) {
scannedPaths[path] = true
}
return nil
})
// Delete items whose files no longer exist - with safety logging
for _, item := range dbItems {
filePath := item.FilePath
if filePath != "" && !scannedPaths[filePath] {
msg := fmt.Sprintf("[RESCAN-CLEANUP] Orphaned media item found: ID=%s, Title=%s, Path=%s",
item.ID, item.Title, filePath)
s.logger.LogDelete(msg)
delMsg := fmt.Sprintf("[RESCAN-CLEANUP] Deleting orphaned item '%s' (file no longer exists at %s)",
item.Title, filePath)
s.logger.LogDelete(delMsg)
if err := s.db.DeleteMediaItem(ctx, item.ID); err != nil {
errMsg := fmt.Sprintf("[RESCAN-CLEANUP] ERROR: failed to delete orphaned item %s: %v", item.Title, err)
s.logger.LogDelete(errMsg)
s.logger.LogError(errMsg)
} else {
s.logger.LogDelete(fmt.Sprintf("[RESCAN-CLEANUP] SUCCESS: deleted orphaned item '%s'", item.Title))
}
}
}
}
if s.job != nil && s.totalFiles > 0 {
s.job.UpdateProgress(1.0, processedFiles, s.newItems, s.errors)
}
@@ -357,19 +412,39 @@ func (s *MediaScanner) processMediaFile(ctx context.Context, path string) (bool,
fmt.Printf("File info for %s: size=%d\n", path, info.Size())
// Get file modification time for created_at
fileModTime := info.ModTime()
// Find library for this file's folder
var libraryID pgtype.UUID
for _, folder := range s.folders {
if strings.HasPrefix(path, folder) {
lib, err := s.db.GetLibraryByFolder(ctx, folder)
if err != nil {
return false, fmt.Errorf("failed to find library for folder %s: %v", folder, err)
}
libraryID = lib.LibraryID
break
}
}
if !libraryID.Valid {
return false, fmt.Errorf("no library found for file path: %s", path)
}
// Check if media item already exists in database
existingItem, err := s.getMediaItemByFilePath(ctx, path)
existingItem, err := s.getMediaItemByFilePath(ctx, path, libraryID)
if err == nil {
fmt.Printf("Media item already exists in database: %s (size: %d vs %d)\n", path, existingItem.FileSize.Int64, info.Size())
// If force rescan is enabled, always re-process
if s.forceRescan {
fmt.Printf("Force rescan enabled, re-processing existing media item: %s\n", path)
// Force update: delete existing and re-create
if err := s.db.DeleteMediaItem(ctx, existingItem.ID); err != nil {
fmt.Printf("Warning: failed to delete existing media item: %v\n", err)
fmt.Printf("Force rescan enabled, updating existing media item: %s\n", path)
// Use UPDATE instead of DELETE+INSERT to preserve created_at
if err := s.updateMediaItem(ctx, existingItem.ID, path, info); err != nil {
fmt.Printf("Warning: failed to update existing media item: %v\n", err)
}
// Continue to create new entry below
return false, nil
} else {
// Normal behavior: check if file has changed (by size)
if existingItem.FileSize.Int64 != info.Size() {
@@ -483,22 +558,7 @@ func (s *MediaScanner) processMediaFile(ctx context.Context, path string) (bool,
metadata.Author = "Unknown"
}
// Find library for this folder
var libraryID pgtype.UUID
for _, folder := range s.folders {
if strings.HasPrefix(path, folder) {
lib, err := s.db.GetLibraryByFolder(ctx, folder)
if err != nil {
return false, fmt.Errorf("failed to find library for folder %s: %v", folder, err)
}
libraryID = lib.LibraryID
break
}
}
if !libraryID.Valid {
return false, fmt.Errorf("no library found for file path: %s", path)
}
// libraryID already determined at start of function
// Normalize metadata fields for display
metadata.Contributors = utils.NormalizeContributors(metadata.Contributors)
@@ -529,6 +589,7 @@ func (s *MediaScanner) processMediaFile(ctx context.Context, path string) (bool,
Tags: metadata.Tags,
TagsSearch: tagsSearch,
AddedByAdminID: s.adminID,
CreatedAt: pgtype.Timestamptz{Time: fileModTime, Valid: true},
})
if err != nil {
return false, fmt.Errorf("failed to create media item: %v", err)
@@ -1387,13 +1448,46 @@ func isImageFile(filename string) bool {
return ext == ".jpg" || ext == ".jpeg" || ext == ".png" || ext == ".gif"
}
func (s *MediaScanner) updateMediaItem(ctx context.Context, mediaItemID pgtype.UUID, filePath string, info os.FileInfo) error {
// Update disabled - scanner creates items but doesn't update
return nil
func (s *MediaScanner) updateMediaItem(ctx context.Context, mediaItemID pgtype.UUID, path string, info os.FileInfo) error {
// Re-extract metadata for the update
metadata, err := s.extractMetadata(path)
if err != nil {
fmt.Printf("Warning: failed to extract metadata for force rescan %s: %v\n", path, err)
metadata = &MediaMetadata{}
}
// Normalize metadata fields
metadata.Contributors = utils.NormalizeContributors(metadata.Contributors)
metadata.Tags = utils.NormalizeTags(metadata.Tags)
contributorsSearch := utils.NormalizeContributorsSearch(metadata.Contributors)
tagsSearch := utils.NormalizeTagsSearch(metadata.Tags)
// Call the database update - only update fields available in MediaMetadata
_, err = s.db.UpdateMediaItem(ctx, database.UpdateMediaItemParams{
ID: mediaItemID,
Title: metadata.Title,
Author: pgtype.Text{String: metadata.Author, Valid: metadata.Author != ""},
Isbn: pgtype.Text{String: utils.NormalizeISBNSafe(metadata.ISBN), Valid: metadata.ISBN != ""},
Description: pgtype.Text{String: metadata.Description, Valid: metadata.Description != ""},
CoverImagePath: pgtype.Text{String: metadata.CoverPath, Valid: metadata.CoverPath != ""},
Series: pgtype.Text{String: metadata.Series, Valid: metadata.Series != ""},
SeriesNumber: pgtype.Int4{Int32: metadata.SeriesNumber, Valid: metadata.SeriesNumber > 0},
Tags: metadata.Tags,
TagsSearch: tagsSearch,
Asin: pgtype.Text{String: metadata.ASIN, Valid: metadata.ASIN != ""},
DatePublished: pgtype.Date{Time: metadata.PublishDate, Valid: !metadata.PublishDate.IsZero()},
Publisher: pgtype.Text{String: metadata.Publisher, Valid: metadata.Publisher != ""},
Contributors: metadata.Contributors,
ContributorsSearch: contributorsSearch,
})
return err
}
func (s *MediaScanner) getMediaItemByFilePath(ctx context.Context, filePath string) (database.MediaItems, error) {
return s.db.GetMediaItemByFilePath(ctx, filePath)
func (s *MediaScanner) getMediaItemByFilePath(ctx context.Context, filePath string, libraryID pgtype.UUID) (database.MediaItems, error) {
return s.db.GetMediaItemByFilePath(ctx, database.GetMediaItemByFilePathParams{
FilePath: filePath,
LibraryID: libraryID,
})
}
func (s *MediaScanner) getMimeType(path string) string {
@@ -1454,6 +1548,55 @@ func (s *MediaScanner) WatchChanges(ctx context.Context) {
}
}
// Handle file deletions
if event.Has(fsnotify.Remove) && s.isScannableFile(event.Name) {
s.logger.LogDelete(fmt.Sprintf("[WATCH-DELETE] File removed from filesystem: %s", event.Name))
// Determine libraryID for this file
var libraryID pgtype.UUID
for _, folder := range s.folders {
if strings.HasPrefix(event.Name, folder) {
lib, err := s.db.GetLibraryByFolder(ctx, folder)
if err == nil {
libraryID = lib.LibraryID
break
}
}
}
if !libraryID.Valid {
msg := fmt.Sprintf("[WATCH-DELETE] WARNING: could not determine library for deleted file: %s", event.Name)
s.logger.LogDelete(msg)
s.logger.LogError(msg)
return
}
// Look up media item BEFORE deleting - log for safety
existingItem, err := s.db.GetMediaItemByFilePath(ctx, database.GetMediaItemByFilePathParams{
FilePath: event.Name,
LibraryID: libraryID,
})
if err == nil {
msg := fmt.Sprintf("[WATCH-DELETE] Found media item to delete: ID=%s, Title=%s, Path=%s",
existingItem.ID, existingItem.Title, existingItem.FilePath)
s.logger.LogDelete(msg)
if err := s.db.DeleteMediaItem(ctx, existingItem.ID); err != nil {
errMsg := fmt.Sprintf("[WATCH-DELETE] ERROR: failed to delete media item %s: %v", existingItem.ID, err)
s.logger.LogDelete(errMsg)
s.logger.LogError(errMsg)
} else {
s.logger.LogDelete(fmt.Sprintf("[WATCH-DELETE] SUCCESS: deleted media item '%s' (was at %s)",
existingItem.Title, existingItem.FilePath))
}
} else if err != pgx.ErrNoRows {
errMsg := fmt.Sprintf("[WATCH-DELETE] ERROR: failed to look up media item for %s: %v", event.Name, err)
s.logger.LogDelete(errMsg)
s.logger.LogError(errMsg)
} else {
s.logger.LogDelete(fmt.Sprintf("[WATCH-DELETE] No media item found in database for deleted file: %s", event.Name))
}
}
case err, ok := <-s.watcher.Errors:
if !ok {
return
internal/services/scanner_logger.go
+107
View File
@@ -0,0 +1,107 @@
package services
import (
"fmt"
"os"
"path/filepath"
"time"
)
const (
logDir = "/app/logs"
maxLogAgeDays = 7
)
type ScannerLogger struct {
deletesFile *os.File
errorsFile *os.File
currentDate string
}
func NewScannerLogger() *ScannerLogger {
return &ScannerLogger{}
}
func (l *ScannerLogger) ensureLogFiles() error {
today := time.Now().Format("2006-01-02")
if l.currentDate == today && l.deletesFile != nil {
return nil
}
if l.deletesFile != nil {
l.deletesFile.Close()
}
if l.errorsFile != nil {
l.errorsFile.Close()
}
if err := os.MkdirAll(logDir, 0755); err != nil {
return fmt.Errorf("failed to create log directory: %v", err)
}
deletesPath := filepath.Join(logDir, fmt.Sprintf("scanner-deletes-%s.log", today))
errorsPath := filepath.Join(logDir, fmt.Sprintf("scanner-errors-%s.log", today))
deletesFile, err := os.OpenFile(deletesPath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
if err != nil {
return fmt.Errorf("failed to open deletes log file: %v", err)
}
errorsFile, err := os.OpenFile(errorsPath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
if err != nil {
deletesFile.Close()
return fmt.Errorf("failed to open errors log file: %v", err)
}
l.deletesFile = deletesFile
l.errorsFile = errorsFile
l.currentDate = today
l.cleanupOldLogs()
return nil
}
func (l *ScannerLogger) cleanupOldLogs() {
cutoff := time.Now().AddDate(0, 0, -maxLogAgeDays)
filepath.Walk(logDir, func(path string, info os.FileInfo, walkErr error) error {
if walkErr != nil {
return walkErr
}
if !info.IsDir() && info.ModTime().Before(cutoff) {
os.Remove(path)
}
return nil
})
}
func (l *ScannerLogger) LogDelete(message string) {
if err := l.ensureLogFiles(); err != nil {
fmt.Printf("ERROR: Failed to ensure log files: %v\n", err)
return
}
timestamp := time.Now().Format("2006-01-02 15:04:05")
logLine := fmt.Sprintf("[%s] %s\n", timestamp, message)
l.deletesFile.WriteString(logLine)
}
func (l *ScannerLogger) LogError(message string) {
if err := l.ensureLogFiles(); err != nil {
fmt.Printf("ERROR: Failed to ensure log files: %v\n", err)
return
}
timestamp := time.Now().Format("2006-01-02 15:04:05")
logLine := fmt.Sprintf("[%s] %s\n", timestamp, message)
l.errorsFile.WriteString(logLine)
}
func (l *ScannerLogger) Close() {
if l.deletesFile != nil {
l.deletesFile.Close()
}
if l.errorsFile != nil {
l.errorsFile.Close()
}
}