2 Commits
Author SHA1 Message Date
john-okeefe d0040fe428 chore(deploy): add restart: unless-stopped to app container
Release / build-and-push (push) Successful in 2m26s
The app service in docker-compose.yml had no restart policy (defaults to
'no'), so if the process exited -- e.g. the watcher-leak panic fixed in
the previous commit -- the container stayed down until a manual
restart. Adding restart: unless-stopped makes the container self-recover
from crashes or host reboots, while still honoring explicit 'docker
compose down'.

Defense-in-depth alongside the scanner leak/panic fix: even if a future
unforeseen panic occurs, the app comes back automatically.
2026-07-31 10:45:53 -04:00
john-okeefe 59d5de3607 fix(scanner): eliminate fsnotify watcher leak and harden worker against panics
The bookhoard container crashed with 'panic: Failed to create file
watcher: too many open files' (media_scanner.go) after running for a few
hours, preceded by floods of 'no space left on device' from watcher.Add.

Root cause: every scan job called NewMediaScanner(), which eagerly
created an fsnotify watcher. SetFolders() then walked the entire library
tree and registered one inotify watch per directory (~3,000+ across the
libraries), and ScanFolders() registered them again during its walk. The
worker never called scanner.Close() on these ephemeral per-job scanners,
and the worker loop had no recover(), so:

  1. Leaked watchers accumulated until the kernel inotify watch cap was
     hit (ENOSPC -> 'no space left on device'), then
  2. the process fd limit (ulimit -n 1024) was exhausted, causing
     fsnotify.NewWatcher() to fail with EMFILE, and
  3. NewMediaScanner panicked on that error, taking down the whole
     process (exit code 2). With no restart policy the container stayed
     down.

The scan jobs run frequently (scan_poll_interval), so the leak built up
within hours. Note this was NOT a disk-space issue; df showed plenty free.

Fix:

- media_scanner.go: NewMediaScanner no longer creates a watcher eagerly
  (s.watcher starts nil), which removes the panic site entirely -- there
  is nothing to fail at construction. The watcher is created lazily only
  when needed.

- media_scanner.go: SetFolders gains a [?1049h(B[?7h[?25lEvery 2.0s: boolgaruda-ser8: Fri 31 Jul 2026 10:45:41 AM EDTin 0.002s (127)
sh: line 1: bool: command not found
[?12l[?25h[?1049l
[?1l> parameter. It creates
  and populates a watcher (returning an error instead of panicking) only
  when watch=true; otherwise it skips all watcher.Add calls. ScanFolders
  guards its watcher.Add with a nil check, and the WatchChanges event
  loop exits cleanly when there is no watcher (polling still runs).

- worker.go: the worker() loop now wraps each job in defer/recover() so a
  panicking job is recorded as failed and can never kill the process.

- worker.go: the three ephemeral scan handlers (processScanJob,
  processSetFoldersJob, processDirectoryScanJob) now defer scanner.Close()
  and call SetFolders(..., false), so scan jobs allocate zero watchers and
  zero inotify watches. Any pre-existing leak is also bounded by Close().

- handlers/scanner.go: the long-lived watch-mode scanners (StartScanner
  and StartWatchModeForLibrary) pass watch=true since they actually read
  watcher.Events for live change detection.

- calibre_integration_test.go: updated to the new SetFolders signature
  (watch=false, matching one-off scan usage).

Auto-add is fully preserved: new files are still detected by the periodic
poller (startBackupScan), which is independent of fsnotify and unaffected
by these changes. The watch-mode event loop remains as bonus responsiveness
when inotify is available; through Docker bind mounts where inotify is
unreliable, polling is what catches new books.
2026-07-31 10:45:41 -04:00
5 changed files with 96 additions and 50 deletions
+2 -2
View File
@@ -90,7 +90,7 @@ func TestCalibreLibraryScan(t *testing.T) {
// Create scanner and configure it
scanner := services.NewMediaScanner(setup.DB)
scanner.SetAdminID(adminID)
err = scanner.SetFolders([]string{tmpDir})
err = scanner.SetFolders([]string{tmpDir}, false)
require.NoError(t, err, "Failed to set scanner folders")
// Scan library
@@ -167,7 +167,7 @@ func TestCalibreLibraryScanWithoutSidecar(t *testing.T) {
// Create scanner and configure it
scanner := services.NewMediaScanner(setup.DB)
scanner.SetAdminID(adminID)
err = scanner.SetFolders([]string{tmpDir})
err = scanner.SetFolders([]string{tmpDir}, false)
require.NoError(t, err, "Failed to set scanner folders")
// Scan library
+1
View File
@@ -31,6 +31,7 @@ services:
app:
image: git.linuxhg.com/bookhoard/bookhoard:${IMAGE_TAG:-latest}
container_name: bookhoard
restart: unless-stopped
environment:
# Database Configuration
DATABASE_HOST: db
+3 -3
View File
@@ -128,8 +128,8 @@ func (h *Handler) StartScanner(c *echo.Context) error {
return c.JSON(http.StatusBadRequest, map[string]string{"error": "invalid user id"})
}
// Set the folder paths
if err := h.scanner.SetFolders(req.FolderPaths); err != nil {
// Set the folder paths (watch=true: this long-lived scanner reads events)
if err := h.scanner.SetFolders(req.FolderPaths, true); err != nil {
return c.JSON(http.StatusBadRequest, map[string]string{"error": "invalid folder paths: " + err.Error()})
}
@@ -202,7 +202,7 @@ func (h *Handler) StartWatchModeForLibrary(ctx context.Context, libraryID pgtype
}
scanner := services.NewMediaScanner(h.db)
if err := scanner.SetFolders(folderPaths); err != nil {
if err := scanner.SetFolders(folderPaths, true); err != nil {
return fmt.Errorf("failed to set scanner folders: %v", err)
}
+67 -41
View File
@@ -146,16 +146,18 @@ type CalibreOPFMetadata struct {
Timestamp *time.Time
}
// NewMediaScanner creates a new media scanner instance
// NewMediaScanner creates a new media scanner instance.
//
// The fsnotify watcher is NOT created here. It is created lazily inside
// SetFolders only when watch=true (the long-lived watch-mode scanner).
// Ephemeral one-off scan jobs pass watch=false, so they never allocate a
// watcher (and thus can never panic on EMFILE/ENOSPC). This fixes the
// fd/inotify-watch leak where every scan job created a watcher that was
// never closed.
func NewMediaScanner(db *database.Queries) *MediaScanner {
watcher, err := fsnotify.NewWatcher()
if err != nil {
panic(fmt.Sprintf("Failed to create file watcher: %v", err))
}
return &MediaScanner{
db: db,
watcher: watcher,
watcher: nil,
settingsCache: NewSettingsCache(30 * time.Second),
dirtyDirs: make(map[string]time.Time),
fileStability: make(map[string]*atomic.Bool),
@@ -238,24 +240,34 @@ func (s *MediaScanner) GetStats() (int, int, int) {
return s.totalFiles, s.newItems, s.errors
}
func (s *MediaScanner) SetFolders(folders []string) error {
// SetFolders configures the scanner's folders and (optionally) sets up an
// fsnotify watcher over the full directory tree.
//
// watch should be true only for the single long-lived watch-mode scanner that
// actually consumes watcher.Events. Ephemeral scan jobs must pass false so no
// watcher (and thus no fd/inotify watches) is allocated — the watcher is never
// read by scan jobs and previously leaked one watcher per job.
func (s *MediaScanner) SetFolders(folders []string, watch bool) error {
s.folders = folders
// Remove old watch if exists
// Always close any previously-owned watcher so reconfiguration doesn't leak.
if s.watcher != nil {
if s.watcher != nil {
if err := s.watcher.Close(); err != nil {
fmt.Printf("Warning: failed to close old watcher during folder reconfiguration: %v\n", err)
}
if err := s.watcher.Close(); err != nil {
fmt.Printf("Warning: failed to close old watcher during folder reconfiguration: %v\n", err)
}
s.watcher = nil
}
// Create new watcher
watcher, err := fsnotify.NewWatcher()
if err != nil {
return fmt.Errorf("failed to create watcher: %v", err)
// Create + populate a fresh watcher only when the caller intends to read events.
if watch {
watcher, err := fsnotify.NewWatcher()
if err != nil {
// Return an error instead of panicking so a failed watcher can't
// take down the whole process.
return fmt.Errorf("failed to create watcher: %w", err)
}
s.watcher = watcher
}
s.watcher = watcher
// Build cache of allowed extensions per folder
// Uses Go AllowedExtensions map as source of truth (not DB)
@@ -286,31 +298,36 @@ func (s *MediaScanner) SetFolders(folders []string) error {
}
}
// Add all folders and their subdirectories to the watcher (like Audiobookshelf)
watchCount := 0
for _, folder := range folders {
if err := s.watcher.Add(folder); err != nil {
fmt.Printf("[WATCHER] Warning: failed to watch root folder %s: %v\n", folder, err)
} else {
watchCount++
}
filepath.WalkDir(folder, func(path string, d fs.DirEntry, err error) error {
if err != nil {
return err
}
if !d.IsDir() || path == folder {
return nil
}
if err := s.watcher.Add(path); err != nil {
fmt.Printf("[WATCHER] Warning: failed to watch subdirectory %s: %v\n", path, err)
// Add all folders and their subdirectories to the watcher (like Audiobookshelf).
// Only when watching; scan jobs (watch=false) skip this entirely.
if s.watcher != nil {
watchCount := 0
for _, folder := range folders {
if err := s.watcher.Add(folder); err != nil {
fmt.Printf("[WATCHER] Warning: failed to watch root folder %s: %v\n", folder, err)
} else {
watchCount++
}
return nil
})
}
filepath.WalkDir(folder, func(path string, d fs.DirEntry, err error) error {
if err != nil {
return err
}
if !d.IsDir() || path == folder {
return nil
}
if err := s.watcher.Add(path); err != nil {
fmt.Printf("[WATCHER] Warning: failed to watch subdirectory %s: %v\n", path, err)
} else {
watchCount++
}
return nil
})
}
fmt.Printf("[WATCHER] Now watching %d directories across %d root folders\n", watchCount, len(folders))
fmt.Printf("[WATCHER] Now watching %d directories across %d root folders\n", watchCount, len(folders))
} else {
fmt.Printf("[SCANNER] Configured %d root folders (watch mode disabled, no inotify watcher)\n", len(folders))
}
return nil
}
@@ -417,8 +434,10 @@ func (s *MediaScanner) ScanFolders(ctx context.Context) error {
}
if d.IsDir() {
if err := s.watcher.Add(path); err != nil {
fmt.Printf("Warning: failed to watch subdirectory %s: %v\n", path, err)
if s.watcher != nil {
if err := s.watcher.Add(path); err != nil {
fmt.Printf("Warning: failed to watch subdirectory %s: %v\n", path, err)
}
}
return nil
}
@@ -2601,6 +2620,13 @@ func (s *MediaScanner) WatchChanges(ctx context.Context) error {
go s.startBackupScan(ctx)
go func() {
// The event loop only runs if a real watcher was set up (watch=true).
// If watching with no watcher (e.g. inotify unavailable through a Docker
// bind mount), polling via startBackupScan above still handles detection.
if s.watcher == nil {
fmt.Printf("[WATCHER] No inotify watcher configured; relying on periodic polling for change detection\n")
return
}
fmt.Printf("[WATCHER] Event loop started for %d folders\n", len(s.folders))
for {
select {
+23 -4
View File
@@ -205,7 +205,23 @@ func (w *Worker) worker() {
return
}
w.processJob(job)
// Recover from any panic inside a job so a single failing job can
// never crash the whole worker goroutine (and thus the process).
func() {
defer func() {
if r := recover(); r != nil {
fmt.Printf("[WORKER] panic in job %s (%s): %v\n", job.ID, job.Type, r)
w.mu.Lock()
w.results[job.ID] = &JobResult{
JobID: job.ID,
Status: JobStatusFailed,
Error: fmt.Sprintf("panic: %v", r),
}
w.mu.Unlock()
}
}()
w.processJob(job)
}()
case <-w.ctx.Done():
return
@@ -348,6 +364,7 @@ func (w *Worker) processScanJob(job *Job) (interface{}, error) {
}
scanner := NewMediaScanner(db)
defer scanner.Close()
scanner.job = job
job.ProgressCallback = func(progress float64, filesScanned, newItems, errors int) {
@@ -377,7 +394,7 @@ func (w *Worker) processScanJob(job *Job) (interface{}, error) {
}
}
if err := scanner.SetFolders(folders); err != nil {
if err := scanner.SetFolders(folders, false); err != nil {
return nil, err
}
@@ -521,7 +538,8 @@ func (w *Worker) processSetFoldersJob(job *Job) (interface{}, error) {
// Create scanner and configure folders
scanner := NewMediaScanner(db)
if err := scanner.SetFolders(folders); err != nil {
defer scanner.Close()
if err := scanner.SetFolders(folders, false); err != nil {
return nil, fmt.Errorf("failed to set folders: %w", err)
}
@@ -900,6 +918,7 @@ func (w *Worker) processDirectoryScanJob(job *Job) (interface{}, error) {
// Create temporary scanner instance for this job
scanner := NewMediaScanner(db)
defer scanner.Close()
scanner.job = job
// Find which library owns this directory (prefix match for subdirectories)
ctx := context.Background()
@@ -918,7 +937,7 @@ func (w *Worker) processDirectoryScanJob(job *Job) (interface{}, error) {
folderPaths = append(folderPaths, f.FolderPath)
}
// Configure scanner with folders
if err := scanner.SetFolders(folderPaths); err != nil {
if err := scanner.SetFolders(folderPaths, false); err != nil {
return nil, fmt.Errorf("failed to set folders: %w", err)
}
// Now scan the directory