From ced90cd1f4081d4a3cc8784855942f0443a14ae6 Mon Sep 17 00:00:00 2001 From: John O'Keefe Date: Tue, 12 May 2026 16:54:35 -0400 Subject: [PATCH] feat(scanner): add directory mtime-based fast polling for container environments MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Podman rootless containers with overlay storage do not propagate inotify events through bind mounts, making the fsnotify file watcher ineffective. This caused new files added on the host to go undetected until the 5-minute full-filesystem-walk polling fallback caught them. Add a lightweight directory mtime polling mechanism that runs every 10 seconds, checking stat() on all subdirectories under watched library folders against a cached mtime value. When a directory's mtime changes (indicating files were added/removed/renamed), it feeds into the existing markDirectoryDirty() → processDirtyDirectories() → job queue pipeline. Changes: - Add dirMtimes cache + mutex to MediaScanner struct - Add seedDirectoryMtimes() to populate cache on startup (prevents false-positive flood on first poll) - Add pollDirectoryChanges() goroutine (10s ticker) and checkDirectoryMtimes() (walks directories, compares mtimes) - Launch mtime poller from WatchChanges() alongside existing goroutines - Rename StartPolling logs to [ORPHAN-CLEANUP] to clarify its role - Change default poll interval from 60s → 30m (new file detection now handled by the fast mtime poll; full sync focuses on orphan cleanup) - Update GetScanSettings default from 60 → 1800 seconds - Add 5 tests: seed cache, skip nonexistent, detect new dir, skip unchanged, detect modified dir Expected result: new files detected in ~20 seconds (10s poll + 10s debounce) regardless of inotify/container support. --- internal/handlers/system_settings.go | 4 +- internal/services/media_scanner.go | 121 ++++++++++++++++-- .../services/media_scanner_settings_test.go | 4 +- internal/services/media_scanner_test.go | 100 +++++++++++++++ 4 files changed, 211 insertions(+), 18 deletions(-) diff --git a/internal/handlers/system_settings.go b/internal/handlers/system_settings.go index f2af571..7e3c1eb 100644 --- a/internal/handlers/system_settings.go +++ b/internal/handlers/system_settings.go @@ -100,7 +100,7 @@ func (h *SystemSettingsHandler) GetScanSettings(c *echo.Context) error { if err != nil { if errors.Is(err, pgx.ErrNoRows) { return c.JSON(http.StatusOK, ScanSettingsResponse{ - ScanPollIntervalSeconds: 60, + ScanPollIntervalSeconds: 1800, AutoScanEnabled: true, }) } @@ -111,7 +111,7 @@ func (h *SystemSettingsHandler) GetScanSettings(c *echo.Context) error { if err != nil { if errors.Is(err, pgx.ErrNoRows) { return c.JSON(http.StatusOK, ScanSettingsResponse{ - ScanPollIntervalSeconds: 60, + ScanPollIntervalSeconds: 1800, AutoScanEnabled: true, }) } diff --git a/internal/services/media_scanner.go b/internal/services/media_scanner.go index dd8cf73..2864e3c 100644 --- a/internal/services/media_scanner.go +++ b/internal/services/media_scanner.go @@ -121,6 +121,8 @@ type MediaScanner struct { pollInterval time.Duration watching atomic.Bool settingsCache *SettingsCache + dirMtimes map[string]time.Time + dirMtimesMu sync.RWMutex totalFiles int newItems int @@ -168,11 +170,11 @@ func NewMediaScanner(db *database.Queries) *MediaScanner { defaultLibraryID: pgtype.UUID{Valid: false}, libraryTypes: make(map[string][]string), logger: NewScannerLogger(), + dirMtimes: make(map[string]time.Time), } } func (s *MediaScanner) GetPollInterval() time.Duration { - // Check cache first if cached, ok := s.settingsCache.Get("scan_poll_interval_seconds"); ok { if seconds, err := strconv.Atoi(cached); err == nil { return time.Duration(seconds) * time.Second @@ -180,25 +182,22 @@ func (s *MediaScanner) GetPollInterval() time.Duration { } if s.db == nil { - return 60 * time.Second + return 30 * time.Minute } - // Cache miss - query database ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() setting, err := s.db.GetSystemSetting(ctx, "scan_poll_interval_seconds") if err != nil || setting == "" { - return 60 * time.Second + return 30 * time.Minute } - // Store in cache s.settingsCache.Set("scan_poll_interval_seconds", setting) - // Convert to duration seconds, err := strconv.Atoi(setting) if err != nil { - return 60 * time.Second + return 30 * time.Minute } return time.Duration(seconds) * time.Second } @@ -295,9 +294,101 @@ func (s *MediaScanner) SetFolders(folders []string) error { } } + s.seedDirectoryMtimes() + return nil } +func (s *MediaScanner) seedDirectoryMtimes() { + s.dirMtimesMu.Lock() + defer s.dirMtimesMu.Unlock() + + s.dirMtimes = make(map[string]time.Time) + + for _, folder := range s.folders { + if _, err := os.Stat(folder); os.IsNotExist(err) { + continue + } + + filepath.WalkDir(folder, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if !d.IsDir() { + return nil + } + info, err := d.Info() + if err != nil { + return nil + } + s.dirMtimes[path] = info.ModTime() + return nil + }) + } + + fmt.Printf("Seeded directory mtime cache with %d directories\n", len(s.dirMtimes)) +} + +func (s *MediaScanner) pollDirectoryChanges(ctx context.Context) { + fmt.Println("Directory mtime poller started (interval: 10s)") + + ticker := time.NewTicker(10 * time.Second) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + fmt.Println("Directory mtime poller stopped") + return + case <-ticker.C: + s.checkDirectoryMtimes() + } + } +} + +func (s *MediaScanner) checkDirectoryMtimes() { + s.dirMtimesMu.Lock() + defer s.dirMtimesMu.Unlock() + + changedCount := 0 + + for _, folder := range s.folders { + if _, err := os.Stat(folder); os.IsNotExist(err) { + continue + } + + filepath.WalkDir(folder, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if !d.IsDir() { + return nil + } + + info, err := d.Info() + if err != nil { + return nil + } + currentMtime := info.ModTime() + + cachedMtime, exists := s.dirMtimes[path] + if !exists || !cachedMtime.Equal(currentMtime) { + s.dirtyDirsMu.Lock() + s.dirtyDirs[path] = time.Now() + s.dirtyDirsMu.Unlock() + changedCount++ + } + + s.dirMtimes[path] = currentMtime + return nil + }) + } + + if changedCount > 0 { + fmt.Printf("[MTIME-POLL] Detected changes in %d director(ies)\n", changedCount) + } +} + func (s *MediaScanner) ScanFolders(ctx context.Context) error { if len(s.folders) == 0 { return fmt.Errorf("no folders set") @@ -2525,9 +2616,12 @@ func (s *MediaScanner) WatchChanges(ctx context.Context) error { // Start directory processor go s.processDirtyDirectories(ctx) - // Start polling fallback + // Start polling fallback (primarily for orphan cleanup) go s.StartPolling(ctx) + // Start directory mtime poller (fast detection of new/changed files) + go s.pollDirectoryChanges(ctx) + // Handle fsnotify events - queue them for debouncing go func() { for { @@ -2903,10 +2997,10 @@ func (s *MediaScanner) Close() error { func (s *MediaScanner) StartPolling(ctx context.Context) { interval := s.GetPollInterval() if interval <= 0 { - fmt.Println("Polling fallback disabled (interval = 0") + fmt.Println("Orphan cleanup polling disabled (interval = 0)") return } - fmt.Printf("Polling fallback started with interval: %v\n", interval) + fmt.Printf("Orphan cleanup polling started with interval: %v\n", interval) for { ticker := time.NewTicker(interval) @@ -2914,14 +3008,13 @@ func (s *MediaScanner) StartPolling(ctx context.Context) { select { case <-ctx.Done(): - fmt.Println("Polling fallback stopped") + fmt.Println("Orphan cleanup polling stopped") return case <-ticker.C: - //Re-read interval each tick for dynamic updates interval = s.GetPollInterval() - fmt.Printf("Running polling fallback sync (interval: %v)...\n", interval) + fmt.Printf("[ORPHAN-CLEANUP] Running filesystem sync (interval: %v)...\n", interval) if err := s.SyncFilesystemWithDatabase(ctx); err != nil { - fmt.Printf("Polling sync error: %v\n", err) + fmt.Printf("[ORPHAN-CLEANUP] Sync error: %v\n", err) } } } diff --git a/internal/services/media_scanner_settings_test.go b/internal/services/media_scanner_settings_test.go index ffa32c4..194a141 100644 --- a/internal/services/media_scanner_settings_test.go +++ b/internal/services/media_scanner_settings_test.go @@ -12,8 +12,8 @@ func TestMediaScanner_GetPollInterval(t *testing.T) { settingsCache: NewSettingsCache(30 * time.Second), } interval := scanner.GetPollInterval() - if interval != 60*time.Second { - t.Errorf("expected 60s, got %v", interval) + if interval != 30*time.Minute { + t.Errorf("expected 30m, got %v", interval) } }) } diff --git a/internal/services/media_scanner_test.go b/internal/services/media_scanner_test.go index dd668d8..6c8a916 100644 --- a/internal/services/media_scanner_test.go +++ b/internal/services/media_scanner_test.go @@ -120,3 +120,103 @@ func TestProcessDirtyDirectories_BatchesScans(t *testing.T) { scanner.dirtyDirsMu.RUnlock() assert.Equal(t, 0, count, "All dirty directories should be processed after 10s") } + +func TestSeedDirectoryMtimes(t *testing.T) { + db := setupTestDB(t) + scanner := NewMediaScanner(db) + + tmpDir := t.TempDir() + subDir := filepath.Join(tmpDir, "author") + require.NoError(t, os.Mkdir(subDir, 0755)) + require.NoError(t, os.WriteFile(filepath.Join(subDir, "book.epub"), []byte("test"), 0644)) + + scanner.folders = []string{tmpDir} + scanner.seedDirectoryMtimes() + + scanner.dirMtimesMu.RLock() + defer scanner.dirMtimesMu.RUnlock() + + _, rootExists := scanner.dirMtimes[tmpDir] + _, subExists := scanner.dirMtimes[subDir] + assert.True(t, rootExists, "Root directory should be cached") + assert.True(t, subExists, "Subdirectory should be cached") + assert.Equal(t, 2, len(scanner.dirMtimes), "Should have exactly 2 directories cached") +} + +func TestSeedDirectoryMtimes_SkipsNonexistentFolders(t *testing.T) { + db := setupTestDB(t) + scanner := NewMediaScanner(db) + scanner.folders = []string{"/nonexistent/path"} + scanner.seedDirectoryMtimes() + + scanner.dirMtimesMu.RLock() + count := len(scanner.dirMtimes) + scanner.dirMtimesMu.RUnlock() + + assert.Equal(t, 0, count, "Nonexistent folder should produce empty cache") +} + +func TestCheckDirectoryMtimes_DetectsNewDirectory(t *testing.T) { + db := setupTestDB(t) + scanner := NewMediaScanner(db) + + tmpDir := t.TempDir() + scanner.folders = []string{tmpDir} + scanner.seedDirectoryMtimes() + + newDir := filepath.Join(tmpDir, "new_author") + require.NoError(t, os.Mkdir(newDir, 0755)) + require.NoError(t, os.WriteFile(filepath.Join(newDir, "book.cbz"), []byte("test"), 0644)) + + scanner.checkDirectoryMtimes() + + scanner.dirtyDirsMu.RLock() + _, dirty := scanner.dirtyDirs[newDir] + scanner.dirtyDirsMu.RUnlock() + assert.True(t, dirty, "New directory should be marked dirty") +} + +func TestCheckDirectoryMtimes_SkipsUnchangedDirectories(t *testing.T) { + db := setupTestDB(t) + scanner := NewMediaScanner(db) + + tmpDir := t.TempDir() + subDir := filepath.Join(tmpDir, "author") + require.NoError(t, os.Mkdir(subDir, 0755)) + + scanner.folders = []string{tmpDir} + scanner.seedDirectoryMtimes() + + scanner.dirtyDirsMu.Lock() + scanner.dirtyDirs = make(map[string]time.Time) + scanner.dirtyDirsMu.Unlock() + + scanner.checkDirectoryMtimes() + + scanner.dirtyDirsMu.RLock() + count := len(scanner.dirtyDirs) + scanner.dirtyDirsMu.RUnlock() + assert.Equal(t, 0, count, "Unchanged directories should not be marked dirty") +} + +func TestCheckDirectoryMtimes_DetectsModifiedDirectory(t *testing.T) { + db := setupTestDB(t) + scanner := NewMediaScanner(db) + + tmpDir := t.TempDir() + subDir := filepath.Join(tmpDir, "author") + require.NoError(t, os.Mkdir(subDir, 0755)) + + scanner.folders = []string{tmpDir} + scanner.seedDirectoryMtimes() + + time.Sleep(10 * time.Millisecond) + require.NoError(t, os.WriteFile(filepath.Join(subDir, "new_book.epub"), []byte("test"), 0644)) + + scanner.checkDirectoryMtimes() + + scanner.dirtyDirsMu.RLock() + _, dirty := scanner.dirtyDirs[subDir] + scanner.dirtyDirsMu.RUnlock() + assert.True(t, dirty, "Modified directory should be marked dirty") +}