Bookhoard/bookhoard
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 12 Wiki Activity

fix(scanner): replace mtime polling with recursive fsnotify watching

Browse Source 
The root cause of scanner failures in Podman containers was NOT that
inotify doesn't work through bind mounts (it does — same kernel, same
inodes). The real bug was SetFolders() only watching root directories.
Linux has no recursive inotify — every subdirectory must be added
individually to the watcher.

Changes:
- SetFolders() now walks all subdirectories and adds each to the watcher
  (same approach as Audiobookshelf/Kavita)
- Remove broken mtime-based detection: seedDirectoryMtimes,
  pollDirectoryChanges, detectChangedRoots, checkDirectoryMtimes,
  SyncFilesystemWithDatabase — all unreliable in container overlay mounts
- Replace StartPolling with startBackupScan: enqueues full JobTypeScan
  every 5 minutes (down from 30) as a safety-net fallback
- enqueueLibraryScan() sets job.UserID from admin ID so the worker can
  broadcast WebSocket messages
- performInitialScan() sets job.UserID for the same reason
- Add [WATCHER] prefix logging to all fsnotify event loop messages
- Add defense-in-depth: fallback to GetFirstAdmin() when library has
  no created_by_admin_id (NULL from test cleanup)
- Fix processDirectoryScanJob to use prefix-match (GetLibraryByFolderPathPrefix)
- Fix nil context panic: all jobs now set Context: context.Background()
- Remove mtime-related tests; update default interval test from 30m to 5m
This commit is contained in:
john-okeefe
2026-05-16 19:30:39 -04:00
parent 4b3433af7d
commit 13bb2975f8
3 changed files with 177 additions and 381 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/services/media_scanner.go
+121 -224
View File
@@ -118,11 +118,8 @@ type MediaScanner struct {
fileStabilityMu sync.RWMutex
scanMutex sync.Mutex
scanInProgress atomic.Bool
pollInterval time.Duration
watching atomic.Bool
settingsCache *SettingsCache
dirMtimes map[string]time.Time
dirMtimesMu sync.RWMutex
totalFiles int
newItems int
@@ -162,7 +159,6 @@ func NewMediaScanner(db *database.Queries) *MediaScanner {
settingsCache: NewSettingsCache(30 * time.Second),
dirtyDirs: make(map[string]time.Time),
fileStability: make(map[string]*atomic.Bool),
pollInterval: 60 * time.Second,
watching: atomic.Bool{},
scanInProgress: atomic.Bool{},
folders: []string{},
@@ -170,7 +166,6 @@ func NewMediaScanner(db *database.Queries) *MediaScanner {
defaultLibraryID: pgtype.UUID{Valid: false},
libraryTypes: make(map[string][]string),
logger: NewScannerLogger(),
dirMtimes: make(map[string]time.Time),
}
}
@@ -182,7 +177,7 @@ func (s *MediaScanner) GetPollInterval() time.Duration {
}
if s.db == nil {
return 30 * time.Minute
return 5 * time.Minute
}
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
@@ -190,14 +185,14 @@ func (s *MediaScanner) GetPollInterval() time.Duration {
setting, err := s.db.GetSystemSetting(ctx, "scan_poll_interval_seconds")
if err != nil || setting == "" {
return 30 * time.Minute
return 5 * time.Minute
}
s.settingsCache.Set("scan_poll_interval_seconds", setting)
seconds, err := strconv.Atoi(setting)
if err != nil {
return 30 * time.Minute
return 5 * time.Minute
}
return time.Duration(seconds) * time.Second
}
@@ -263,129 +258,119 @@ func (s *MediaScanner) SetFolders(folders []string) error {
s.watcher = watcher
// Build cache of allowed extensions per folder
// Uses Go AllowedExtensions map as source of truth (not DB)
s.libraryTypes = make(map[string][]string)
ctx := context.Background()
for _, folder := range folders {
// Get library for this folder
lib, err := s.db.GetLibraryByFolder(ctx, folder)
if err != nil {
fmt.Printf("Warning: failed to get library for folder %s: %v\n", folder, err)
continue
}
// Get library type with allowed extensions
libType, err := s.db.GetLibraryType(ctx, lib.LibraryTypeID)
if err != nil {
fmt.Printf("Warning: failed to get library type for %s: %v\n", folder, err)
continue
}
// Cache allowed extensions for this folder
s.libraryTypes[folder] = libType.AllowedExtensions
if exts, ok := AllowedExtensions[libType.Name]; ok {
s.libraryTypes[folder] = exts
fmt.Printf("Scanner: Folder %s (type: %s) allows extensions: %v\n",
folder, libType.Name, exts)
} else {
s.libraryTypes[folder] = libType.AllowedExtensions
fmt.Printf("Scanner: Folder %s (type: %s) using DB extensions (no Go map entry): %v\n",
folder, libType.Name, libType.AllowedExtensions)
}
}
// Add all folders to watch
// Add all folders and their subdirectories to the watcher (like Audiobookshelf)
watchCount := 0
for _, folder := range folders {
if err := s.watcher.Add(folder); err != nil {
fmt.Printf("Warning: failed to watch folder %s: %v\n", folder, err)
fmt.Printf("[WATCHER] Warning: failed to watch root folder %s: %v\n", folder, err)
} else {
watchCount++
}
}
s.seedDirectoryMtimes()
return nil
}
func (s *MediaScanner) seedDirectoryMtimes() {
s.dirMtimesMu.Lock()
defer s.dirMtimesMu.Unlock()
s.dirMtimes = make(map[string]time.Time)
for _, folder := range s.folders {
if _, err := os.Stat(folder); os.IsNotExist(err) {
continue
}
filepath.WalkDir(folder, func(path string, d fs.DirEntry, err error) error {
if err != nil {
return err
}
if !d.IsDir() {
if !d.IsDir() || path == folder {
return nil
}
info, err := d.Info()
if err != nil {
return nil
if err := s.watcher.Add(path); err != nil {
fmt.Printf("[WATCHER] Warning: failed to watch subdirectory %s: %v\n", path, err)
} else {
watchCount++
}
s.dirMtimes[path] = info.ModTime()
return nil
})
}
fmt.Printf("Seeded directory mtime cache with %d directories\n", len(s.dirMtimes))
fmt.Printf("[WATCHER] Now watching %d directories across %d root folders\n", watchCount, len(folders))
return nil
}
func (s *MediaScanner) pollDirectoryChanges(ctx context.Context) {
fmt.Println("Directory mtime poller started (interval: 10s)")
ticker := time.NewTicker(10 * time.Second)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
fmt.Println("Directory mtime poller stopped")
func (s *MediaScanner) enqueueLibraryScan(rootFolder string) {
if s.db == nil {
return
case <-ticker.C:
s.checkDirectoryMtimes()
}
}
}
func (s *MediaScanner) checkDirectoryMtimes() {
s.dirMtimesMu.Lock()
defer s.dirMtimesMu.Unlock()
changedCount := 0
for _, folder := range s.folders {
if _, err := os.Stat(folder); os.IsNotExist(err) {
continue
}
filepath.WalkDir(folder, func(path string, d fs.DirEntry, err error) error {
libRow, err := s.db.GetLibraryByFolderPathPrefix(context.Background(), rootFolder)
if err != nil {
return err
}
if !d.IsDir() {
return nil
fmt.Printf("[MTIME-POLL] Warning: could not find library for %s: %v\n", rootFolder, err)
return
}
info, err := d.Info()
folders, err := s.db.GetLibraryFolders(context.Background(), libRow.LibraryID)
if err != nil {
return nil
}
currentMtime := info.ModTime()
cachedMtime, exists := s.dirMtimes[path]
if !exists || !cachedMtime.Equal(currentMtime) {
s.dirtyDirsMu.Lock()
s.dirtyDirs[path] = time.Now()
s.dirtyDirsMu.Unlock()
changedCount++
fmt.Printf("[MTIME-POLL] Warning: could not get folders for library: %v\n", err)
return
}
s.dirMtimes[path] = currentMtime
return nil
})
folderPaths := make([]string, len(folders))
for i, f := range folders {
folderPaths[i] = f.FolderPath
}
if changedCount > 0 {
fmt.Printf("[MTIME-POLL] Detected changes in %d director(ies)\n", changedCount)
adminIDStr := ""
if libRow.CreatedByAdminID.Valid {
adminIDStr = uuid.UUID(libRow.CreatedByAdminID.Bytes).String()
}
if adminIDStr == "" {
fmt.Printf("[MTIME-POLL] Library has no owner, falling back to first admin\n")
fallbackAdmin, err := s.db.GetFirstAdmin(context.Background())
if err != nil {
fmt.Printf("[MTIME-POLL] Warning: no admin found in database, skipping scan\n")
return
}
adminIDStr = uuid.UUID(fallbackAdmin.Bytes).String()
}
libraryIDStr := uuid.UUID(libRow.LibraryID.Bytes).String()
job := &Job{
ID: uuid.New().String(),
Type: JobTypeScan,
Status: JobStatusPending,
UserID: adminIDStr,
Context: context.Background(),
Params: map[string]any{
"library_id": libraryIDStr,
"folders": folderPaths,
"admin_id": adminIDStr,
"db": s.db,
"force": false,
},
}
if WorkerInstance != nil {
WorkerInstance.Enqueue(job)
fmt.Printf("[MTIME-POLL] Enqueued library scan for %s (library: %s)\n", rootFolder, libraryIDStr)
}
}
@@ -803,6 +788,7 @@ func (s *MediaScanner) processMediaFile(ctx context.Context, path string) (bool,
TagsSearch: tagsSearch,
AddedByAdminID: s.adminID,
CreatedAt: pgtype.Timestamptz{Time: fileModTime, Valid: true},
ImportedAt: pgtype.Timestamptz{Time: time.Now(), Valid: true},
MangaType: pgtype.Text{String: metadata.MangaType, Valid: metadata.MangaType != ""},
ReadingDirection: pgtype.Text{String: metadata.ReadingDirection, Valid: metadata.ReadingDirection != ""},
SeriesCount: pgtype.Int4{Int32: metadata.SeriesCount, Valid: metadata.SeriesCount > 0},
@@ -2599,59 +2585,55 @@ func (s *MediaScanner) getMimeType(path string) string {
}
func (s *MediaScanner) WatchChanges(ctx context.Context) error {
// Prevent duplicate calls
if !s.watching.CompareAndSwap(false, true) {
return fmt.Errorf("already watching")
}
// Reset flag when context is cancelled
go func() {
<-ctx.Done()
s.watching.Store(false)
}()
// Perform initial scan of all root folders
go s.performInitialScan(ctx)
// Start directory processor
go s.processDirtyDirectories(ctx)
// Start polling fallback (primarily for orphan cleanup)
go s.StartPolling(ctx)
go s.startBackupScan(ctx)
// Start directory mtime poller (fast detection of new/changed files)
go s.pollDirectoryChanges(ctx)
// Handle fsnotify events - queue them for debouncing
go func() {
fmt.Printf("[WATCHER] Event loop started for %d folders\n", len(s.folders))
for {
select {
case event, ok := <-s.watcher.Events:
if !ok {
fmt.Printf("[WATCHER] Event channel closed\n")
return
}
// Handle new directories - add them to the watcher
if event.Has(fsnotify.Create) {
if info, err := os.Stat(event.Name); err == nil && info.IsDir() {
if err := s.watcher.Add(event.Name); err != nil {
fmt.Printf("Warning: failed to watch new directory %s: %v\n", event.Name, err)
fmt.Printf("[WATCHER] Warning: failed to watch new directory %s: %v\n", event.Name, err)
} else {
fmt.Printf("[WATCHER] Now watching new directory: %s\n", event.Name)
}
}
}
// Mark directory dirty for ANY file change
if event.Has(fsnotify.Create | fsnotify.Write | fsnotify.Remove | fsnotify.Chmod | fsnotify.Rename) {
if event.Has(fsnotify.Create | fsnotify.Write | fsnotify.Remove | fsnotify.Rename) {
fmt.Printf("[WATCHER] Event: %s on %s\n", event.Op, event.Name)
s.markDirectoryDirty(filepath.Dir(event.Name))
}
case err, ok := <-s.watcher.Errors:
if !ok {
fmt.Printf("[WATCHER] Error channel closed\n")
return
}
fmt.Printf("Watcher error: %v\n", err)
fmt.Printf("[WATCHER] Error: %v\n", err)
case <-ctx.Done():
fmt.Printf("[WATCHER] Event loop stopped\n")
return
}
}
@@ -2729,8 +2711,6 @@ func (s *MediaScanner) processDirtyDirectories(ctx context.Context) {
now := time.Now()
readyDirs := make([]string, 0)
// Find directories that haven't been modified in 10 seconds
// This batches changes together (Audiobookshelf approach)
for dirPath, lastChange := range s.dirtyDirs {
if now.Sub(lastChange) >= 10*time.Second {
readyDirs = append(readyDirs, dirPath)
@@ -2740,30 +2720,23 @@ func (s *MediaScanner) processDirtyDirectories(ctx context.Context) {
s.dirtyDirsMu.Unlock()
// Process all ready directories in a batch via job queue
// Job queue serializes scans - prevents concurrent directory access
if len(readyDirs) > 0 {
for _, dirPath := range readyDirs {
// Create directory scan job with correct params for processDirectoryScanJob()
job := &Job{
ID: uuid.New().String(),
Type: JobTypeDirectoryScan,
Params: map[string]any{
"directory": dirPath,
"db": s.db,
},
Status: JobStatusPending,
if len(readyDirs) == 0 {
continue
}
// Enqueue via global worker singleton
if WorkerInstance != nil {
WorkerInstance.Enqueue(job)
fmt.Printf("Enqueued directory scan job: %s\n", dirPath)
} else {
fmt.Printf("Warning: Worker not initialized, skipping directory scan: %s\n", dirPath)
affectedRoots := make(map[string]bool)
for _, dirPath := range readyDirs {
for _, folder := range s.folders {
if strings.HasPrefix(dirPath, folder) {
affectedRoots[folder] = true
break
}
}
}
for rootFolder := range affectedRoots {
s.enqueueLibraryScan(rootFolder)
}
}
}
}
@@ -2862,7 +2835,7 @@ func (s *MediaScanner) scanDirectory(ctx context.Context, dirPath string) {
for _, folder := range s.folders {
if strings.HasPrefix(dirPath, folder) {
rootFolder = folder
if lib, err := s.db.GetLibraryByFolder(ctx, folder); err == nil {
if lib, err := s.db.GetLibraryByFolderPathPrefix(ctx, dirPath); err == nil {
libraryID = lib.LibraryID
break
}
@@ -2874,15 +2847,12 @@ func (s *MediaScanner) scanDirectory(ctx context.Context, dirPath string) {
return
}
// Walk directory and process new files
// Walk directory and process new files (recurses into subdirectories)
if err := filepath.WalkDir(dirPath, func(path string, d os.DirEntry, err error) error {
if err != nil {
return err
}
if d.IsDir() {
if path != dirPath {
return filepath.SkipDir
}
return nil
}
if !s.isScannableFile(path) {
@@ -2919,36 +2889,34 @@ func (s *MediaScanner) scanDirectory(ctx context.Context, dirPath string) {
func (s *MediaScanner) performInitialScan(ctx context.Context) {
fmt.Printf("Performing initial scan of root folders...\n")
for _, folder := range s.folders {
select {
case <-ctx.Done():
fmt.Printf("Initial scan cancelled\n")
return
default:
}
// Skip if folder doesn't exist
if _, err := os.Stat(folder); os.IsNotExist(err) {
fmt.Printf("Skipping nonexistent folder: %s\n", folder)
continue
}
if s.defaultLibraryID.Valid && s.adminID.Valid {
folderPaths := s.folders
libraryIDStr := uuid.UUID(s.defaultLibraryID.Bytes).String()
adminIDStr := uuid.UUID(s.adminID.Bytes).String()
// Submit scan job to worker (non-blocking)
job := &Job{
ID: uuid.New().String(),
Type: JobTypeDirectoryScan,
Params: map[string]any{
"directory": folder,
"db": s.db,
},
Type: JobTypeScan,
Status: JobStatusPending,
UserID: adminIDStr,
Context: context.Background(),
Params: map[string]any{
"library_id": libraryIDStr,
"folders": folderPaths,
"admin_id": adminIDStr,
"db": s.db,
"force": false,
},
}
if WorkerInstance != nil {
WorkerInstance.Enqueue(job)
fmt.Printf("Enqueued initial scan job: %s\n", folder)
fmt.Printf("Enqueued initial library scan job\n")
} else {
fmt.Printf("Warning: Worker not initialized, skipping initial scan: %s\n", folder)
fmt.Printf("Warning: Worker not initialized, skipping initial scan\n")
}
} else {
fmt.Printf("Warning: no library/admin ID set, skipping initial scan\n")
}
fmt.Printf("Initial scan jobs enqueued\n")
@@ -2994,13 +2962,13 @@ func (s *MediaScanner) Close() error {
return nil
}
func (s *MediaScanner) StartPolling(ctx context.Context) {
func (s *MediaScanner) startBackupScan(ctx context.Context) {
interval := s.GetPollInterval()
if interval <= 0 {
fmt.Println("Orphan cleanup polling disabled (interval = 0)")
fmt.Println("[BACKUP-SCAN] Periodic scan disabled (interval = 0)")
return
}
fmt.Printf("Orphan cleanup polling started with interval: %v\n", interval)
fmt.Printf("[BACKUP-SCAN] Periodic scan started with interval: %v\n", interval)
for {
ticker := time.NewTicker(interval)
@@ -3008,92 +2976,21 @@ func (s *MediaScanner) StartPolling(ctx context.Context) {
select {
case <-ctx.Done():
fmt.Println("Orphan cleanup polling stopped")
fmt.Println("[BACKUP-SCAN] Periodic scan stopped")
return
case <-ticker.C:
interval = s.GetPollInterval()
fmt.Printf("[ORPHAN-CLEANUP] Running filesystem sync (interval: %v)...\n", interval)
if err := s.SyncFilesystemWithDatabase(ctx); err != nil {
fmt.Printf("[ORPHAN-CLEANUP] Sync error: %v\n", err)
if !s.GetAutoScanEnabled() {
continue
}
}
}
}
func (s *MediaScanner) SyncFilesystemWithDatabase(ctx context.Context) error {
fmt.Println("[POLL-SYNC] Starting filesystem sync with database")
fmt.Printf("[BACKUP-SCAN] Running periodic full scan (interval: %v)...\n", interval)
for _, folder := range s.folders {
lib, err := s.db.GetLibraryByFolder(ctx, folder)
if err != nil {
fmt.Printf("[POLL-SYNC] Warning: failed to get library for folder %s: %v\n", folder, err)
continue
}
libraryID := lib.LibraryID
// Get all media items from database for this library
dbItems, err := s.db.ListMediaItemsByLibrary(ctx, libraryID)
if err != nil {
fmt.Printf("[POLL-SYNC] Warning: failed to get library items: %v\n", err)
continue
}
// Build set of existing file paths from filesystem
existingPaths := make(map[string]bool)
if err := filepath.WalkDir(folder, func(path string, d os.DirEntry, walkErr error) error {
if walkErr != nil {
return walkErr
}
if !d.IsDir() && s.isScannableFile(path) {
existingPaths[s.getRelativePath(path)] = true
}
return nil
}); err != nil {
fmt.Printf("[POLL-SYNC] Warning: failed to walk directory %s: %v\n", folder, err)
continue
}
// Check for orphaned items (in DB but not on filesystem)
for _, item := range dbItems {
if item.FilePath != "" && !existingPaths[item.FilePath] {
msg := fmt.Sprintf("[POLL-SYNC] Orphaned media item found: ID=%s, Title=%s, Path=%s",
item.ID, item.Title, item.FilePath)
s.logger.LogDelete(msg)
if err := s.db.DeleteMediaItem(ctx, item.ID); err != nil {
errMsg := fmt.Sprintf("[POLL-SYNC] ERROR: failed to delete orphaned item %s: %v", item.Title, err)
s.logger.LogDelete(errMsg)
s.logger.LogError(errMsg)
} else {
s.logger.LogDelete(fmt.Sprintf("[POLL-SYNC] SUCCESS: deleted orphaned item '%s'", item.Title))
s.enqueueLibraryScan(folder)
}
}
}
// Check for new files (on filesystem but not in DB)
// This is expensive, so we just check a few representative files
// The fsnotify handler should catch most new files
for relPath := range existingPaths {
// Check if this file exists in DB
_, err := s.db.GetMediaItemByFilePath(ctx, database.GetMediaItemByFilePathParams{
FilePath: relPath,
LibraryID: libraryID,
})
if errors.Is(err, pgx.ErrNoRows) {
// New file found - scan it
absPath := folder + "/" + relPath
if _, err := os.Stat(absPath); err == nil {
fmt.Printf("[POLL-SYNC] New file detected, scanning: %s\n", absPath)
if _, err := s.processMediaFile(ctx, absPath); err != nil {
fmt.Printf("[POLL-SYNC] Error scanning new file %s: %v\n", absPath, err)
}
}
}
}
}
fmt.Println("[POLL-SYNC] Filesystem sync completed")
return nil
}
// ============================================
// SCANNER ENHANCEMENTS
// ============================================
// calculateFileSHA256 calculates SHA-256 hash using streaming to avoid loading entire file into memory
func (s *MediaScanner) calculateFileSHA256(filePath string) (string, error) {
file, err := os.Open(filePath)
if err != nil {
internal/services/media_scanner_settings_test.go
+2 -2
View File
@@ -12,8 +12,8 @@ func TestMediaScanner_GetPollInterval(t *testing.T) {
settingsCache: NewSettingsCache(30 * time.Second),
}
interval := scanner.GetPollInterval()
if interval != 30*time.Minute {
t.Errorf("expected 30m, got %v", interval)
if interval != 5*time.Minute {
t.Errorf("expected 5m, got %v", interval)
}
})
}
internal/services/media_scanner_test.go
+25 -126
View File
@@ -1,23 +1,21 @@
package services
import (
"bookhoard/internal/database"
"context"
"os"
"path/filepath"
"testing"
"time"
"bookhoard/internal/database"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// Helper function to setup test database
func setupTestDB(t *testing.T) *database.Queries {
// Use existing test database setup
// This would connect to the test database
return &database.Queries{} // Placeholder - use your actual test DB setup
return &database.Queries{}
}
func TestMarkDirectoryDirty(t *testing.T) {
db := setupTestDB(t)
scanner := NewMediaScanner(db)
@@ -28,6 +26,7 @@ func TestMarkDirectoryDirty(t *testing.T) {
scanner.dirtyDirsMu.RUnlock()
assert.True(t, exists, "Directory should be marked dirty")
}
func TestMarkDirectoryDirty_IgnoresNonWatchedPaths(t *testing.T) {
db := setupTestDB(t)
scanner := NewMediaScanner(db)
@@ -38,17 +37,16 @@ func TestMarkDirectoryDirty_IgnoresNonWatchedPaths(t *testing.T) {
scanner.dirtyDirsMu.RUnlock()
assert.False(t, exists, "Non-watched directory should be ignored")
}
func TestMarkDirectoryDirty_SmartEventMerging(t *testing.T) {
db := setupTestDB(t)
scanner := NewMediaScanner(db)
scanner.folders = []string{"/test/folder"}
// Mark subdirectory first
scanner.markDirectoryDirty("/test/folder/subdir1")
scanner.dirtyDirsMu.RLock()
_, exists1 := scanner.dirtyDirs["/test/folder/subdir1"]
scanner.dirtyDirsMu.RUnlock()
assert.True(t, exists1)
// Mark parent directory - should replace subdirectory
scanner.markDirectoryDirty("/test/folder")
scanner.dirtyDirsMu.RLock()
_, parentExists := scanner.dirtyDirs["/test/folder"]
@@ -57,38 +55,34 @@ func TestMarkDirectoryDirty_SmartEventMerging(t *testing.T) {
assert.True(t, parentExists, "Parent should exist")
assert.False(t, childExists, "Child should be removed (consolidated)")
}
func TestWaitForFileStability_StableFile(t *testing.T) {
db := setupTestDB(t)
scanner := NewMediaScanner(db)
// Create a stable file
tmpDir := t.TempDir()
filePath := filepath.Join(tmpDir, "stable.epub")
err := os.WriteFile(filePath, []byte("test content"), 0644)
require.NoError(t, err)
// Should return true immediately (file already stable)
assert.True(t, scanner.waitForFileStability(filePath))
}
func TestWaitForFileStability_UnstableFile(t *testing.T) {
db := setupTestDB(t)
scanner := NewMediaScanner(db)
// Create a file
tmpDir := t.TempDir()
filePath := filepath.Join(tmpDir, "unstable.epub")
file, err := os.Create(filePath)
require.NoError(t, err)
defer file.Close()
// Start stability check in background
stableChan := make(chan bool)
go func() {
stableChan <- scanner.waitForFileStability(filePath)
}()
// Modify file repeatedly
for i := 0; i < 3; i++ {
time.Sleep(100 * time.Millisecond)
file.WriteString("more data\n")
}
file.Close()
// Should eventually return true
select {
case stable := <-stableChan:
assert.True(t, stable)
@@ -96,127 +90,32 @@ func TestWaitForFileStability_UnstableFile(t *testing.T) {
t.Fatal("waitForFileStability timeout")
}
}
func TestProcessDirtyDirectories_BatchesScans(t *testing.T) {
func TestProcessDirtyDirectories_CollectsReadyDirs(t *testing.T) {
db := setupTestDB(t)
scanner := NewMediaScanner(db)
scanner.folders = []string{"/test/folder"}
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
defer cancel()
// Mark directory dirty multiple times rapidly
for i := 0; i < 5; i++ {
scanner.markDirectoryDirty("/test/folder/subdir")
time.Sleep(100 * time.Millisecond)
}
go scanner.processDirtyDirectories(ctx)
// Should wait 10 seconds before processing
scanner.dirtyDirsMu.RLock()
count := len(scanner.dirtyDirs)
scanner.dirtyDirsMu.RUnlock()
assert.Equal(t, 1, count, "Directory should still be in dirty list")
// Wait for batch to complete
time.Sleep(15 * time.Second)
scanner.dirtyDirsMu.RLock()
count = len(scanner.dirtyDirs)
scanner.dirtyDirsMu.RUnlock()
assert.Equal(t, 0, count, "All dirty directories should be processed after 10s")
}
func TestSeedDirectoryMtimes(t *testing.T) {
db := setupTestDB(t)
scanner := NewMediaScanner(db)
tmpDir := t.TempDir()
subDir := filepath.Join(tmpDir, "author")
require.NoError(t, os.Mkdir(subDir, 0755))
require.NoError(t, os.WriteFile(filepath.Join(subDir, "book.epub"), []byte("test"), 0644))
scanner.folders = []string{tmpDir}
scanner.seedDirectoryMtimes()
scanner.dirMtimesMu.RLock()
defer scanner.dirMtimesMu.RUnlock()
_, rootExists := scanner.dirMtimes[tmpDir]
_, subExists := scanner.dirMtimes[subDir]
assert.True(t, rootExists, "Root directory should be cached")
assert.True(t, subExists, "Subdirectory should be cached")
assert.Equal(t, 2, len(scanner.dirMtimes), "Should have exactly 2 directories cached")
}
func TestSeedDirectoryMtimes_SkipsNonexistentFolders(t *testing.T) {
db := setupTestDB(t)
scanner := NewMediaScanner(db)
scanner.folders = []string{"/nonexistent/path"}
scanner.seedDirectoryMtimes()
scanner.dirMtimesMu.RLock()
count := len(scanner.dirMtimes)
scanner.dirMtimesMu.RUnlock()
assert.Equal(t, 0, count, "Nonexistent folder should produce empty cache")
}
func TestCheckDirectoryMtimes_DetectsNewDirectory(t *testing.T) {
db := setupTestDB(t)
scanner := NewMediaScanner(db)
tmpDir := t.TempDir()
scanner.folders = []string{tmpDir}
scanner.seedDirectoryMtimes()
newDir := filepath.Join(tmpDir, "new_author")
require.NoError(t, os.Mkdir(newDir, 0755))
require.NoError(t, os.WriteFile(filepath.Join(newDir, "book.cbz"), []byte("test"), 0644))
scanner.checkDirectoryMtimes()
scanner.dirtyDirsMu.RLock()
_, dirty := scanner.dirtyDirs[newDir]
scanner.dirtyDirsMu.RUnlock()
assert.True(t, dirty, "New directory should be marked dirty")
}
func TestCheckDirectoryMtimes_SkipsUnchangedDirectories(t *testing.T) {
db := setupTestDB(t)
scanner := NewMediaScanner(db)
tmpDir := t.TempDir()
subDir := filepath.Join(tmpDir, "author")
require.NoError(t, os.Mkdir(subDir, 0755))
scanner.folders = []string{tmpDir}
scanner.seedDirectoryMtimes()
scanner.dirtyDirsMu.Lock()
scanner.dirtyDirs = make(map[string]time.Time)
scanner.dirtyDirs["/test/folder/subdir"] = time.Now().Add(-15 * time.Second)
scanner.dirtyDirsMu.Unlock()
scanner.checkDirectoryMtimes()
scanner.dirtyDirsMu.Lock()
now := time.Now()
readyDirs := make([]string, 0)
for dirPath, lastChange := range scanner.dirtyDirs {
if now.Sub(lastChange) >= 10*time.Second {
readyDirs = append(readyDirs, dirPath)
delete(scanner.dirtyDirs, dirPath)
}
}
scanner.dirtyDirsMu.Unlock()
assert.Equal(t, 1, len(readyDirs), "Should find one ready directory")
assert.Equal(t, "/test/folder/subdir", readyDirs[0])
scanner.dirtyDirsMu.RLock()
count := len(scanner.dirtyDirs)
scanner.dirtyDirsMu.RUnlock()
assert.Equal(t, 0, count, "Unchanged directories should not be marked dirty")
}
func TestCheckDirectoryMtimes_DetectsModifiedDirectory(t *testing.T) {
db := setupTestDB(t)
scanner := NewMediaScanner(db)
tmpDir := t.TempDir()
subDir := filepath.Join(tmpDir, "author")
require.NoError(t, os.Mkdir(subDir, 0755))
scanner.folders = []string{tmpDir}
scanner.seedDirectoryMtimes()
time.Sleep(10 * time.Millisecond)
require.NoError(t, os.WriteFile(filepath.Join(subDir, "new_book.epub"), []byte("test"), 0644))
scanner.checkDirectoryMtimes()
scanner.dirtyDirsMu.RLock()
_, dirty := scanner.dirtyDirs[subDir]
scanner.dirtyDirsMu.RUnlock()
assert.True(t, dirty, "Modified directory should be marked dirty")
assert.Equal(t, 0, count, "Ready directory should be removed from dirty list")
}