Bookhoard/bookhoard
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 12 Wiki Activity

fix(scanner): replace mtime polling with recursive fsnotify watching

Browse Source 
The root cause of scanner failures in Podman containers was NOT that
inotify doesn't work through bind mounts (it does — same kernel, same
inodes). The real bug was SetFolders() only watching root directories.
Linux has no recursive inotify — every subdirectory must be added
individually to the watcher.

Changes:
- SetFolders() now walks all subdirectories and adds each to the watcher
  (same approach as Audiobookshelf/Kavita)
- Remove broken mtime-based detection: seedDirectoryMtimes,
  pollDirectoryChanges, detectChangedRoots, checkDirectoryMtimes,
  SyncFilesystemWithDatabase — all unreliable in container overlay mounts
- Replace StartPolling with startBackupScan: enqueues full JobTypeScan
  every 5 minutes (down from 30) as a safety-net fallback
- enqueueLibraryScan() sets job.UserID from admin ID so the worker can
  broadcast WebSocket messages
- performInitialScan() sets job.UserID for the same reason
- Add [WATCHER] prefix logging to all fsnotify event loop messages
- Add defense-in-depth: fallback to GetFirstAdmin() when library has
  no created_by_admin_id (NULL from test cleanup)
- Fix processDirectoryScanJob to use prefix-match (GetLibraryByFolderPathPrefix)
- Fix nil context panic: all jobs now set Context: context.Background()
- Remove mtime-related tests; update default interval test from 30m to 5m
This commit is contained in:
john-okeefe
2026-05-16 19:30:39 -04:00
parent 4b3433af7d
commit 13bb2975f8
3 changed files with 177 additions and 381 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/services/media_scanner_test.go
+25 -126
View File
@@ -1,23 +1,21 @@
package services
import (
"bookhoard/internal/database"
"context"
"os"
"path/filepath"
"testing"
"time"
"bookhoard/internal/database"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// Helper function to setup test database
func setupTestDB(t *testing.T) *database.Queries {
// Use existing test database setup
// This would connect to the test database
return &database.Queries{} // Placeholder - use your actual test DB setup
return &database.Queries{}
}
func TestMarkDirectoryDirty(t *testing.T) {
db := setupTestDB(t)
scanner := NewMediaScanner(db)
@@ -28,6 +26,7 @@ func TestMarkDirectoryDirty(t *testing.T) {
scanner.dirtyDirsMu.RUnlock()
assert.True(t, exists, "Directory should be marked dirty")
}
func TestMarkDirectoryDirty_IgnoresNonWatchedPaths(t *testing.T) {
db := setupTestDB(t)
scanner := NewMediaScanner(db)
@@ -38,17 +37,16 @@ func TestMarkDirectoryDirty_IgnoresNonWatchedPaths(t *testing.T) {
scanner.dirtyDirsMu.RUnlock()
assert.False(t, exists, "Non-watched directory should be ignored")
}
func TestMarkDirectoryDirty_SmartEventMerging(t *testing.T) {
db := setupTestDB(t)
scanner := NewMediaScanner(db)
scanner.folders = []string{"/test/folder"}
// Mark subdirectory first
scanner.markDirectoryDirty("/test/folder/subdir1")
scanner.dirtyDirsMu.RLock()
_, exists1 := scanner.dirtyDirs["/test/folder/subdir1"]
scanner.dirtyDirsMu.RUnlock()
assert.True(t, exists1)
// Mark parent directory - should replace subdirectory
scanner.markDirectoryDirty("/test/folder")
scanner.dirtyDirsMu.RLock()
_, parentExists := scanner.dirtyDirs["/test/folder"]
@@ -57,38 +55,34 @@ func TestMarkDirectoryDirty_SmartEventMerging(t *testing.T) {
assert.True(t, parentExists, "Parent should exist")
assert.False(t, childExists, "Child should be removed (consolidated)")
}
func TestWaitForFileStability_StableFile(t *testing.T) {
db := setupTestDB(t)
scanner := NewMediaScanner(db)
// Create a stable file
tmpDir := t.TempDir()
filePath := filepath.Join(tmpDir, "stable.epub")
err := os.WriteFile(filePath, []byte("test content"), 0644)
require.NoError(t, err)
// Should return true immediately (file already stable)
assert.True(t, scanner.waitForFileStability(filePath))
}
func TestWaitForFileStability_UnstableFile(t *testing.T) {
db := setupTestDB(t)
scanner := NewMediaScanner(db)
// Create a file
tmpDir := t.TempDir()
filePath := filepath.Join(tmpDir, "unstable.epub")
file, err := os.Create(filePath)
require.NoError(t, err)
defer file.Close()
// Start stability check in background
stableChan := make(chan bool)
go func() {
stableChan <- scanner.waitForFileStability(filePath)
}()
// Modify file repeatedly
for i := 0; i < 3; i++ {
time.Sleep(100 * time.Millisecond)
file.WriteString("more data\n")
}
file.Close()
// Should eventually return true
select {
case stable := <-stableChan:
assert.True(t, stable)
@@ -96,127 +90,32 @@ func TestWaitForFileStability_UnstableFile(t *testing.T) {
t.Fatal("waitForFileStability timeout")
}
}
func TestProcessDirtyDirectories_BatchesScans(t *testing.T) {
func TestProcessDirtyDirectories_CollectsReadyDirs(t *testing.T) {
db := setupTestDB(t)
scanner := NewMediaScanner(db)
scanner.folders = []string{"/test/folder"}
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
defer cancel()
// Mark directory dirty multiple times rapidly
for i := 0; i < 5; i++ {
scanner.markDirectoryDirty("/test/folder/subdir")
time.Sleep(100 * time.Millisecond)
}
go scanner.processDirtyDirectories(ctx)
// Should wait 10 seconds before processing
scanner.dirtyDirsMu.RLock()
count := len(scanner.dirtyDirs)
scanner.dirtyDirsMu.RUnlock()
assert.Equal(t, 1, count, "Directory should still be in dirty list")
// Wait for batch to complete
time.Sleep(15 * time.Second)
scanner.dirtyDirsMu.RLock()
count = len(scanner.dirtyDirs)
scanner.dirtyDirsMu.RUnlock()
assert.Equal(t, 0, count, "All dirty directories should be processed after 10s")
}
func TestSeedDirectoryMtimes(t *testing.T) {
db := setupTestDB(t)
scanner := NewMediaScanner(db)
tmpDir := t.TempDir()
subDir := filepath.Join(tmpDir, "author")
require.NoError(t, os.Mkdir(subDir, 0755))
require.NoError(t, os.WriteFile(filepath.Join(subDir, "book.epub"), []byte("test"), 0644))
scanner.folders = []string{tmpDir}
scanner.seedDirectoryMtimes()
scanner.dirMtimesMu.RLock()
defer scanner.dirMtimesMu.RUnlock()
_, rootExists := scanner.dirMtimes[tmpDir]
_, subExists := scanner.dirMtimes[subDir]
assert.True(t, rootExists, "Root directory should be cached")
assert.True(t, subExists, "Subdirectory should be cached")
assert.Equal(t, 2, len(scanner.dirMtimes), "Should have exactly 2 directories cached")
}
func TestSeedDirectoryMtimes_SkipsNonexistentFolders(t *testing.T) {
db := setupTestDB(t)
scanner := NewMediaScanner(db)
scanner.folders = []string{"/nonexistent/path"}
scanner.seedDirectoryMtimes()
scanner.dirMtimesMu.RLock()
count := len(scanner.dirMtimes)
scanner.dirMtimesMu.RUnlock()
assert.Equal(t, 0, count, "Nonexistent folder should produce empty cache")
}
func TestCheckDirectoryMtimes_DetectsNewDirectory(t *testing.T) {
db := setupTestDB(t)
scanner := NewMediaScanner(db)
tmpDir := t.TempDir()
scanner.folders = []string{tmpDir}
scanner.seedDirectoryMtimes()
newDir := filepath.Join(tmpDir, "new_author")
require.NoError(t, os.Mkdir(newDir, 0755))
require.NoError(t, os.WriteFile(filepath.Join(newDir, "book.cbz"), []byte("test"), 0644))
scanner.checkDirectoryMtimes()
scanner.dirtyDirsMu.RLock()
_, dirty := scanner.dirtyDirs[newDir]
scanner.dirtyDirsMu.RUnlock()
assert.True(t, dirty, "New directory should be marked dirty")
}
func TestCheckDirectoryMtimes_SkipsUnchangedDirectories(t *testing.T) {
db := setupTestDB(t)
scanner := NewMediaScanner(db)
tmpDir := t.TempDir()
subDir := filepath.Join(tmpDir, "author")
require.NoError(t, os.Mkdir(subDir, 0755))
scanner.folders = []string{tmpDir}
scanner.seedDirectoryMtimes()
scanner.dirtyDirsMu.Lock()
scanner.dirtyDirs = make(map[string]time.Time)
scanner.dirtyDirs["/test/folder/subdir"] = time.Now().Add(-15 * time.Second)
scanner.dirtyDirsMu.Unlock()
scanner.checkDirectoryMtimes()
scanner.dirtyDirsMu.Lock()
now := time.Now()
readyDirs := make([]string, 0)
for dirPath, lastChange := range scanner.dirtyDirs {
if now.Sub(lastChange) >= 10*time.Second {
readyDirs = append(readyDirs, dirPath)
delete(scanner.dirtyDirs, dirPath)
}
}
scanner.dirtyDirsMu.Unlock()
assert.Equal(t, 1, len(readyDirs), "Should find one ready directory")
assert.Equal(t, "/test/folder/subdir", readyDirs[0])
scanner.dirtyDirsMu.RLock()
count := len(scanner.dirtyDirs)
scanner.dirtyDirsMu.RUnlock()
assert.Equal(t, 0, count, "Unchanged directories should not be marked dirty")
}
func TestCheckDirectoryMtimes_DetectsModifiedDirectory(t *testing.T) {
db := setupTestDB(t)
scanner := NewMediaScanner(db)
tmpDir := t.TempDir()
subDir := filepath.Join(tmpDir, "author")
require.NoError(t, os.Mkdir(subDir, 0755))
scanner.folders = []string{tmpDir}
scanner.seedDirectoryMtimes()
time.Sleep(10 * time.Millisecond)
require.NoError(t, os.WriteFile(filepath.Join(subDir, "new_book.epub"), []byte("test"), 0644))
scanner.checkDirectoryMtimes()
scanner.dirtyDirsMu.RLock()
_, dirty := scanner.dirtyDirs[subDir]
scanner.dirtyDirsMu.RUnlock()
assert.True(t, dirty, "Modified directory should be marked dirty")
assert.Equal(t, 0, count, "Ready directory should be removed from dirty list")
}