Podman rootless containers with overlay storage do not propagate inotify events through bind mounts, making the fsnotify file watcher ineffective. This caused new files added on the host to go undetected until the 5-minute full-filesystem-walk polling fallback caught them. Add a lightweight directory mtime polling mechanism that runs every 10 seconds, checking stat() on all subdirectories under watched library folders against a cached mtime value. When a directory's mtime changes (indicating files were added/removed/renamed), it feeds into the existing markDirectoryDirty() → processDirtyDirectories() → job queue pipeline. Changes: - Add dirMtimes cache + mutex to MediaScanner struct - Add seedDirectoryMtimes() to populate cache on startup (prevents false-positive flood on first poll) - Add pollDirectoryChanges() goroutine (10s ticker) and checkDirectoryMtimes() (walks directories, compares mtimes) - Launch mtime poller from WatchChanges() alongside existing goroutines - Rename StartPolling logs to [ORPHAN-CLEANUP] to clarify its role - Change default poll interval from 60s → 30m (new file detection now handled by the fast mtime poll; full sync focuses on orphan cleanup) - Update GetScanSettings default from 60 → 1800 seconds - Add 5 tests: seed cache, skip nonexistent, detect new dir, skip unchanged, detect modified dir Expected result: new files detected in ~20 seconds (10s poll + 10s debounce) regardless of inotify/container support.
223 lines
6.9 KiB
Go
223 lines
6.9 KiB
Go
package services
|
|
|
|
import (
|
|
"bookhoard/internal/database"
|
|
"context"
|
|
"os"
|
|
"path/filepath"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/stretchr/testify/assert"
|
|
"github.com/stretchr/testify/require"
|
|
)
|
|
|
|
// Helper function to setup test database
|
|
func setupTestDB(t *testing.T) *database.Queries {
|
|
// Use existing test database setup
|
|
// This would connect to the test database
|
|
return &database.Queries{} // Placeholder - use your actual test DB setup
|
|
}
|
|
func TestMarkDirectoryDirty(t *testing.T) {
|
|
db := setupTestDB(t)
|
|
scanner := NewMediaScanner(db)
|
|
scanner.folders = []string{"/test/folder"}
|
|
scanner.markDirectoryDirty("/test/folder/subdir")
|
|
scanner.dirtyDirsMu.RLock()
|
|
_, exists := scanner.dirtyDirs["/test/folder/subdir"]
|
|
scanner.dirtyDirsMu.RUnlock()
|
|
assert.True(t, exists, "Directory should be marked dirty")
|
|
}
|
|
func TestMarkDirectoryDirty_IgnoresNonWatchedPaths(t *testing.T) {
|
|
db := setupTestDB(t)
|
|
scanner := NewMediaScanner(db)
|
|
scanner.folders = []string{"/test/folder"}
|
|
scanner.markDirectoryDirty("/other/folder")
|
|
scanner.dirtyDirsMu.RLock()
|
|
_, exists := scanner.dirtyDirs["/other/folder"]
|
|
scanner.dirtyDirsMu.RUnlock()
|
|
assert.False(t, exists, "Non-watched directory should be ignored")
|
|
}
|
|
func TestMarkDirectoryDirty_SmartEventMerging(t *testing.T) {
|
|
db := setupTestDB(t)
|
|
scanner := NewMediaScanner(db)
|
|
scanner.folders = []string{"/test/folder"}
|
|
// Mark subdirectory first
|
|
scanner.markDirectoryDirty("/test/folder/subdir1")
|
|
scanner.dirtyDirsMu.RLock()
|
|
_, exists1 := scanner.dirtyDirs["/test/folder/subdir1"]
|
|
scanner.dirtyDirsMu.RUnlock()
|
|
assert.True(t, exists1)
|
|
// Mark parent directory - should replace subdirectory
|
|
scanner.markDirectoryDirty("/test/folder")
|
|
scanner.dirtyDirsMu.RLock()
|
|
_, parentExists := scanner.dirtyDirs["/test/folder"]
|
|
_, childExists := scanner.dirtyDirs["/test/folder/subdir1"]
|
|
scanner.dirtyDirsMu.RUnlock()
|
|
assert.True(t, parentExists, "Parent should exist")
|
|
assert.False(t, childExists, "Child should be removed (consolidated)")
|
|
}
|
|
func TestWaitForFileStability_StableFile(t *testing.T) {
|
|
db := setupTestDB(t)
|
|
scanner := NewMediaScanner(db)
|
|
// Create a stable file
|
|
tmpDir := t.TempDir()
|
|
filePath := filepath.Join(tmpDir, "stable.epub")
|
|
err := os.WriteFile(filePath, []byte("test content"), 0644)
|
|
require.NoError(t, err)
|
|
// Should return true immediately (file already stable)
|
|
assert.True(t, scanner.waitForFileStability(filePath))
|
|
}
|
|
func TestWaitForFileStability_UnstableFile(t *testing.T) {
|
|
db := setupTestDB(t)
|
|
scanner := NewMediaScanner(db)
|
|
// Create a file
|
|
tmpDir := t.TempDir()
|
|
filePath := filepath.Join(tmpDir, "unstable.epub")
|
|
file, err := os.Create(filePath)
|
|
require.NoError(t, err)
|
|
defer file.Close()
|
|
// Start stability check in background
|
|
stableChan := make(chan bool)
|
|
go func() {
|
|
stableChan <- scanner.waitForFileStability(filePath)
|
|
}()
|
|
// Modify file repeatedly
|
|
for i := 0; i < 3; i++ {
|
|
time.Sleep(100 * time.Millisecond)
|
|
file.WriteString("more data\n")
|
|
}
|
|
file.Close()
|
|
// Should eventually return true
|
|
select {
|
|
case stable := <-stableChan:
|
|
assert.True(t, stable)
|
|
case <-time.After(7 * time.Second):
|
|
t.Fatal("waitForFileStability timeout")
|
|
}
|
|
}
|
|
func TestProcessDirtyDirectories_BatchesScans(t *testing.T) {
|
|
db := setupTestDB(t)
|
|
scanner := NewMediaScanner(db)
|
|
scanner.folders = []string{"/test/folder"}
|
|
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
|
|
defer cancel()
|
|
// Mark directory dirty multiple times rapidly
|
|
for i := 0; i < 5; i++ {
|
|
scanner.markDirectoryDirty("/test/folder/subdir")
|
|
time.Sleep(100 * time.Millisecond)
|
|
}
|
|
go scanner.processDirtyDirectories(ctx)
|
|
// Should wait 10 seconds before processing
|
|
scanner.dirtyDirsMu.RLock()
|
|
count := len(scanner.dirtyDirs)
|
|
scanner.dirtyDirsMu.RUnlock()
|
|
assert.Equal(t, 1, count, "Directory should still be in dirty list")
|
|
// Wait for batch to complete
|
|
time.Sleep(15 * time.Second)
|
|
scanner.dirtyDirsMu.RLock()
|
|
count = len(scanner.dirtyDirs)
|
|
scanner.dirtyDirsMu.RUnlock()
|
|
assert.Equal(t, 0, count, "All dirty directories should be processed after 10s")
|
|
}
|
|
|
|
func TestSeedDirectoryMtimes(t *testing.T) {
|
|
db := setupTestDB(t)
|
|
scanner := NewMediaScanner(db)
|
|
|
|
tmpDir := t.TempDir()
|
|
subDir := filepath.Join(tmpDir, "author")
|
|
require.NoError(t, os.Mkdir(subDir, 0755))
|
|
require.NoError(t, os.WriteFile(filepath.Join(subDir, "book.epub"), []byte("test"), 0644))
|
|
|
|
scanner.folders = []string{tmpDir}
|
|
scanner.seedDirectoryMtimes()
|
|
|
|
scanner.dirMtimesMu.RLock()
|
|
defer scanner.dirMtimesMu.RUnlock()
|
|
|
|
_, rootExists := scanner.dirMtimes[tmpDir]
|
|
_, subExists := scanner.dirMtimes[subDir]
|
|
assert.True(t, rootExists, "Root directory should be cached")
|
|
assert.True(t, subExists, "Subdirectory should be cached")
|
|
assert.Equal(t, 2, len(scanner.dirMtimes), "Should have exactly 2 directories cached")
|
|
}
|
|
|
|
func TestSeedDirectoryMtimes_SkipsNonexistentFolders(t *testing.T) {
|
|
db := setupTestDB(t)
|
|
scanner := NewMediaScanner(db)
|
|
scanner.folders = []string{"/nonexistent/path"}
|
|
scanner.seedDirectoryMtimes()
|
|
|
|
scanner.dirMtimesMu.RLock()
|
|
count := len(scanner.dirMtimes)
|
|
scanner.dirMtimesMu.RUnlock()
|
|
|
|
assert.Equal(t, 0, count, "Nonexistent folder should produce empty cache")
|
|
}
|
|
|
|
func TestCheckDirectoryMtimes_DetectsNewDirectory(t *testing.T) {
|
|
db := setupTestDB(t)
|
|
scanner := NewMediaScanner(db)
|
|
|
|
tmpDir := t.TempDir()
|
|
scanner.folders = []string{tmpDir}
|
|
scanner.seedDirectoryMtimes()
|
|
|
|
newDir := filepath.Join(tmpDir, "new_author")
|
|
require.NoError(t, os.Mkdir(newDir, 0755))
|
|
require.NoError(t, os.WriteFile(filepath.Join(newDir, "book.cbz"), []byte("test"), 0644))
|
|
|
|
scanner.checkDirectoryMtimes()
|
|
|
|
scanner.dirtyDirsMu.RLock()
|
|
_, dirty := scanner.dirtyDirs[newDir]
|
|
scanner.dirtyDirsMu.RUnlock()
|
|
assert.True(t, dirty, "New directory should be marked dirty")
|
|
}
|
|
|
|
func TestCheckDirectoryMtimes_SkipsUnchangedDirectories(t *testing.T) {
|
|
db := setupTestDB(t)
|
|
scanner := NewMediaScanner(db)
|
|
|
|
tmpDir := t.TempDir()
|
|
subDir := filepath.Join(tmpDir, "author")
|
|
require.NoError(t, os.Mkdir(subDir, 0755))
|
|
|
|
scanner.folders = []string{tmpDir}
|
|
scanner.seedDirectoryMtimes()
|
|
|
|
scanner.dirtyDirsMu.Lock()
|
|
scanner.dirtyDirs = make(map[string]time.Time)
|
|
scanner.dirtyDirsMu.Unlock()
|
|
|
|
scanner.checkDirectoryMtimes()
|
|
|
|
scanner.dirtyDirsMu.RLock()
|
|
count := len(scanner.dirtyDirs)
|
|
scanner.dirtyDirsMu.RUnlock()
|
|
assert.Equal(t, 0, count, "Unchanged directories should not be marked dirty")
|
|
}
|
|
|
|
func TestCheckDirectoryMtimes_DetectsModifiedDirectory(t *testing.T) {
|
|
db := setupTestDB(t)
|
|
scanner := NewMediaScanner(db)
|
|
|
|
tmpDir := t.TempDir()
|
|
subDir := filepath.Join(tmpDir, "author")
|
|
require.NoError(t, os.Mkdir(subDir, 0755))
|
|
|
|
scanner.folders = []string{tmpDir}
|
|
scanner.seedDirectoryMtimes()
|
|
|
|
time.Sleep(10 * time.Millisecond)
|
|
require.NoError(t, os.WriteFile(filepath.Join(subDir, "new_book.epub"), []byte("test"), 0644))
|
|
|
|
scanner.checkDirectoryMtimes()
|
|
|
|
scanner.dirtyDirsMu.RLock()
|
|
_, dirty := scanner.dirtyDirs[subDir]
|
|
scanner.dirtyDirsMu.RUnlock()
|
|
assert.True(t, dirty, "Modified directory should be marked dirty")
|
|
}
|