fix(scanner): eliminate fsnotify watcher leak and harden worker against panics

The bookhoard container crashed with 'panic: Failed to create file
watcher: too many open files' (media_scanner.go) after running for a few
hours, preceded by floods of 'no space left on device' from watcher.Add.

Root cause: every scan job called NewMediaScanner(), which eagerly
created an fsnotify watcher. SetFolders() then walked the entire library
tree and registered one inotify watch per directory (~3,000+ across the
libraries), and ScanFolders() registered them again during its walk. The
worker never called scanner.Close() on these ephemeral per-job scanners,
and the worker loop had no recover(), so:

  1. Leaked watchers accumulated until the kernel inotify watch cap was
     hit (ENOSPC -> 'no space left on device'), then
  2. the process fd limit (ulimit -n 1024) was exhausted, causing
     fsnotify.NewWatcher() to fail with EMFILE, and
  3. NewMediaScanner panicked on that error, taking down the whole
     process (exit code 2). With no restart policy the container stayed
     down.

The scan jobs run frequently (scan_poll_interval), so the leak built up
within hours. Note this was NOT a disk-space issue; df showed plenty free.

Fix:

- media_scanner.go: NewMediaScanner no longer creates a watcher eagerly
  (s.watcher starts nil), which removes the panic site entirely -- there
  is nothing to fail at construction. The watcher is created lazily only
  when needed.

- media_scanner.go: SetFolders gains a [?1049h(B[?7h[?25lEvery 2.0s: boolgaruda-ser8: Fri 31 Jul 2026 10:45:41 AM EDTin 0.002s (127)
sh: line 1: bool: command not found
[?12l[?25h[?1049l
[?1l> parameter. It creates
  and populates a watcher (returning an error instead of panicking) only
  when watch=true; otherwise it skips all watcher.Add calls. ScanFolders
  guards its watcher.Add with a nil check, and the WatchChanges event
  loop exits cleanly when there is no watcher (polling still runs).

- worker.go: the worker() loop now wraps each job in defer/recover() so a
  panicking job is recorded as failed and can never kill the process.

- worker.go: the three ephemeral scan handlers (processScanJob,
  processSetFoldersJob, processDirectoryScanJob) now defer scanner.Close()
  and call SetFolders(..., false), so scan jobs allocate zero watchers and
  zero inotify watches. Any pre-existing leak is also bounded by Close().

- handlers/scanner.go: the long-lived watch-mode scanners (StartScanner
  and StartWatchModeForLibrary) pass watch=true since they actually read
  watcher.Events for live change detection.

- calibre_integration_test.go: updated to the new SetFolders signature
  (watch=false, matching one-off scan usage).

Auto-add is fully preserved: new files are still detected by the periodic
poller (startBackupScan), which is independent of fsnotify and unaffected
by these changes. The watch-mode event loop remains as bonus responsiveness
when inotify is available; through Docker bind mounts where inotify is
unreliable, polling is what catches new books.

cmd/server/tests/calibre_integration_test.go

@@ -90,7 +90,7 @@ func TestCalibreLibraryScan(t *testing.T) {
 	// Create scanner and configure it
 	scanner := services.NewMediaScanner(setup.DB)
 	scanner.SetAdminID(adminID)
-	err = scanner.SetFolders([]string{tmpDir})
+	err = scanner.SetFolders([]string{tmpDir}, false)
 	require.NoError(t, err, "Failed to set scanner folders")
 
 	// Scan library
@@ -167,7 +167,7 @@ func TestCalibreLibraryScanWithoutSidecar(t *testing.T) {
 	// Create scanner and configure it
 	scanner := services.NewMediaScanner(setup.DB)
 	scanner.SetAdminID(adminID)
-	err = scanner.SetFolders([]string{tmpDir})
+	err = scanner.SetFolders([]string{tmpDir}, false)
 	require.NoError(t, err, "Failed to set scanner folders")
 
 	// Scan library

internal/handlers/scanner.go

@@ -128,8 +128,8 @@ func (h *Handler) StartScanner(c *echo.Context) error {
 		return c.JSON(http.StatusBadRequest, map[string]string{"error": "invalid user id"})
 	}
 
-	// Set the folder paths
-	if err := h.scanner.SetFolders(req.FolderPaths); err != nil {
+	// Set the folder paths (watch=true: this long-lived scanner reads events)
+	if err := h.scanner.SetFolders(req.FolderPaths, true); err != nil {
 		return c.JSON(http.StatusBadRequest, map[string]string{"error": "invalid folder paths: " + err.Error()})
 	}
 
@@ -202,7 +202,7 @@ func (h *Handler) StartWatchModeForLibrary(ctx context.Context, libraryID pgtype
 	}
 
 	scanner := services.NewMediaScanner(h.db)
-	if err := scanner.SetFolders(folderPaths); err != nil {
+	if err := scanner.SetFolders(folderPaths, true); err != nil {
 		return fmt.Errorf("failed to set scanner folders: %v", err)
 	}
 

internal/services/media_scanner.go

@@ -146,16 +146,18 @@ type CalibreOPFMetadata struct {
 	Timestamp    *time.Time
 }
 
-// NewMediaScanner creates a new media scanner instance
+// NewMediaScanner creates a new media scanner instance.
+//
+// The fsnotify watcher is NOT created here. It is created lazily inside
+// SetFolders only when watch=true (the long-lived watch-mode scanner).
+// Ephemeral one-off scan jobs pass watch=false, so they never allocate a
+// watcher (and thus can never panic on EMFILE/ENOSPC). This fixes the
+// fd/inotify-watch leak where every scan job created a watcher that was
+// never closed.
 func NewMediaScanner(db *database.Queries) *MediaScanner {
-	watcher, err := fsnotify.NewWatcher()
-	if err != nil {
-		panic(fmt.Sprintf("Failed to create file watcher: %v", err))
-	}
-
 	return &MediaScanner{
 		db:             db,
-		watcher:        watcher,
+		watcher:        nil,
 		settingsCache:  NewSettingsCache(30 * time.Second),
 		dirtyDirs:      make(map[string]time.Time),
 		fileStability:  make(map[string]*atomic.Bool),
@@ -238,24 +240,34 @@ func (s *MediaScanner) GetStats() (int, int, int) {
 	return s.totalFiles, s.newItems, s.errors
 }
 
-func (s *MediaScanner) SetFolders(folders []string) error {
+// SetFolders configures the scanner's folders and (optionally) sets up an
+// fsnotify watcher over the full directory tree.
+//
+// watch should be true only for the single long-lived watch-mode scanner that
+// actually consumes watcher.Events. Ephemeral scan jobs must pass false so no
+// watcher (and thus no fd/inotify watches) is allocated — the watcher is never
+// read by scan jobs and previously leaked one watcher per job.
+func (s *MediaScanner) SetFolders(folders []string, watch bool) error {
 	s.folders = folders
 
-	// Remove old watch if exists
-	if s.watcher != nil {
+	// Always close any previously-owned watcher so reconfiguration doesn't leak.
 	if s.watcher != nil {
 		if err := s.watcher.Close(); err != nil {
 			fmt.Printf("Warning: failed to close old watcher during folder reconfiguration: %v\n", err)
 		}
-		}
+		s.watcher = nil
 	}
 
-	// Create new watcher
+	// Create + populate a fresh watcher only when the caller intends to read events.
+	if watch {
 		watcher, err := fsnotify.NewWatcher()
 		if err != nil {
-		return fmt.Errorf("failed to create watcher: %v", err)
+			// Return an error instead of panicking so a failed watcher can't
+			// take down the whole process.
+			return fmt.Errorf("failed to create watcher: %w", err)
 		}
 		s.watcher = watcher
+	}
 
 	// Build cache of allowed extensions per folder
 	// Uses Go AllowedExtensions map as source of truth (not DB)
@@ -286,7 +298,9 @@ func (s *MediaScanner) SetFolders(folders []string) error {
 		}
 	}
 
-	// Add all folders and their subdirectories to the watcher (like Audiobookshelf)
+	// Add all folders and their subdirectories to the watcher (like Audiobookshelf).
+	// Only when watching; scan jobs (watch=false) skip this entirely.
+	if s.watcher != nil {
 		watchCount := 0
 		for _, folder := range folders {
 			if err := s.watcher.Add(folder); err != nil {
@@ -311,6 +325,9 @@ func (s *MediaScanner) SetFolders(folders []string) error {
 		}
 
 		fmt.Printf("[WATCHER] Now watching %d directories across %d root folders\n", watchCount, len(folders))
+	} else {
+		fmt.Printf("[SCANNER] Configured %d root folders (watch mode disabled, no inotify watcher)\n", len(folders))
+	}
 
 	return nil
 }
@@ -417,9 +434,11 @@ func (s *MediaScanner) ScanFolders(ctx context.Context) error {
 			}
 
 			if d.IsDir() {
+				if s.watcher != nil {
 					if err := s.watcher.Add(path); err != nil {
 						fmt.Printf("Warning: failed to watch subdirectory %s: %v\n", path, err)
 					}
+				}
 				return nil
 			}
 
@@ -2601,6 +2620,13 @@ func (s *MediaScanner) WatchChanges(ctx context.Context) error {
 	go s.startBackupScan(ctx)
 
 	go func() {
+		// The event loop only runs if a real watcher was set up (watch=true).
+		// If watching with no watcher (e.g. inotify unavailable through a Docker
+		// bind mount), polling via startBackupScan above still handles detection.
+		if s.watcher == nil {
+			fmt.Printf("[WATCHER] No inotify watcher configured; relying on periodic polling for change detection\n")
+			return
+		}
 		fmt.Printf("[WATCHER] Event loop started for %d folders\n", len(s.folders))
 		for {
 			select {

internal/services/worker.go

@@ -205,7 +205,23 @@ func (w *Worker) worker() {
 				return
 			}
 
+			// Recover from any panic inside a job so a single failing job can
+			// never crash the whole worker goroutine (and thus the process).
+			func() {
+				defer func() {
+					if r := recover(); r != nil {
+						fmt.Printf("[WORKER] panic in job %s (%s): %v\n", job.ID, job.Type, r)
+						w.mu.Lock()
+						w.results[job.ID] = &JobResult{
+							JobID:  job.ID,
+							Status: JobStatusFailed,
+							Error:  fmt.Sprintf("panic: %v", r),
+						}
+						w.mu.Unlock()
+					}
+				}()
 				w.processJob(job)
+			}()
 
 		case <-w.ctx.Done():
 			return
@@ -348,6 +364,7 @@ func (w *Worker) processScanJob(job *Job) (interface{}, error) {
 	}
 
 	scanner := NewMediaScanner(db)
+	defer scanner.Close()
 	scanner.job = job
 
 	job.ProgressCallback = func(progress float64, filesScanned, newItems, errors int) {
@@ -377,7 +394,7 @@ func (w *Worker) processScanJob(job *Job) (interface{}, error) {
 		}
 	}
 
-	if err := scanner.SetFolders(folders); err != nil {
+	if err := scanner.SetFolders(folders, false); err != nil {
 		return nil, err
 	}
 
@@ -521,7 +538,8 @@ func (w *Worker) processSetFoldersJob(job *Job) (interface{}, error) {
 
 	// Create scanner and configure folders
 	scanner := NewMediaScanner(db)
-	if err := scanner.SetFolders(folders); err != nil {
+	defer scanner.Close()
+	if err := scanner.SetFolders(folders, false); err != nil {
 		return nil, fmt.Errorf("failed to set folders: %w", err)
 	}
 
@@ -900,6 +918,7 @@ func (w *Worker) processDirectoryScanJob(job *Job) (interface{}, error) {
 
 	// Create temporary scanner instance for this job
 	scanner := NewMediaScanner(db)
+	defer scanner.Close()
 	scanner.job = job
 	// Find which library owns this directory (prefix match for subdirectories)
 	ctx := context.Background()
@@ -918,7 +937,7 @@ func (w *Worker) processDirectoryScanJob(job *Job) (interface{}, error) {
 		folderPaths = append(folderPaths, f.FolderPath)
 	}
 	// Configure scanner with folders
-	if err := scanner.SetFolders(folderPaths); err != nil {
+	if err := scanner.SetFolders(folderPaths, false); err != nil {
 		return nil, fmt.Errorf("failed to set folders: %w", err)
 	}
 	// Now scan the directory