Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
d0040fe428 | ||
|
|
59d5de3607 |
@@ -90,7 +90,7 @@ func TestCalibreLibraryScan(t *testing.T) {
|
||||
// Create scanner and configure it
|
||||
scanner := services.NewMediaScanner(setup.DB)
|
||||
scanner.SetAdminID(adminID)
|
||||
err = scanner.SetFolders([]string{tmpDir})
|
||||
err = scanner.SetFolders([]string{tmpDir}, false)
|
||||
require.NoError(t, err, "Failed to set scanner folders")
|
||||
|
||||
// Scan library
|
||||
@@ -167,7 +167,7 @@ func TestCalibreLibraryScanWithoutSidecar(t *testing.T) {
|
||||
// Create scanner and configure it
|
||||
scanner := services.NewMediaScanner(setup.DB)
|
||||
scanner.SetAdminID(adminID)
|
||||
err = scanner.SetFolders([]string{tmpDir})
|
||||
err = scanner.SetFolders([]string{tmpDir}, false)
|
||||
require.NoError(t, err, "Failed to set scanner folders")
|
||||
|
||||
// Scan library
|
||||
|
||||
@@ -31,6 +31,7 @@ services:
|
||||
app:
|
||||
image: git.linuxhg.com/bookhoard/bookhoard:${IMAGE_TAG:-latest}
|
||||
container_name: bookhoard
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
# Database Configuration
|
||||
DATABASE_HOST: db
|
||||
|
||||
@@ -128,8 +128,8 @@ func (h *Handler) StartScanner(c *echo.Context) error {
|
||||
return c.JSON(http.StatusBadRequest, map[string]string{"error": "invalid user id"})
|
||||
}
|
||||
|
||||
// Set the folder paths
|
||||
if err := h.scanner.SetFolders(req.FolderPaths); err != nil {
|
||||
// Set the folder paths (watch=true: this long-lived scanner reads events)
|
||||
if err := h.scanner.SetFolders(req.FolderPaths, true); err != nil {
|
||||
return c.JSON(http.StatusBadRequest, map[string]string{"error": "invalid folder paths: " + err.Error()})
|
||||
}
|
||||
|
||||
@@ -202,7 +202,7 @@ func (h *Handler) StartWatchModeForLibrary(ctx context.Context, libraryID pgtype
|
||||
}
|
||||
|
||||
scanner := services.NewMediaScanner(h.db)
|
||||
if err := scanner.SetFolders(folderPaths); err != nil {
|
||||
if err := scanner.SetFolders(folderPaths, true); err != nil {
|
||||
return fmt.Errorf("failed to set scanner folders: %v", err)
|
||||
}
|
||||
|
||||
|
||||
@@ -146,16 +146,18 @@ type CalibreOPFMetadata struct {
|
||||
Timestamp *time.Time
|
||||
}
|
||||
|
||||
// NewMediaScanner creates a new media scanner instance
|
||||
// NewMediaScanner creates a new media scanner instance.
|
||||
//
|
||||
// The fsnotify watcher is NOT created here. It is created lazily inside
|
||||
// SetFolders only when watch=true (the long-lived watch-mode scanner).
|
||||
// Ephemeral one-off scan jobs pass watch=false, so they never allocate a
|
||||
// watcher (and thus can never panic on EMFILE/ENOSPC). This fixes the
|
||||
// fd/inotify-watch leak where every scan job created a watcher that was
|
||||
// never closed.
|
||||
func NewMediaScanner(db *database.Queries) *MediaScanner {
|
||||
watcher, err := fsnotify.NewWatcher()
|
||||
if err != nil {
|
||||
panic(fmt.Sprintf("Failed to create file watcher: %v", err))
|
||||
}
|
||||
|
||||
return &MediaScanner{
|
||||
db: db,
|
||||
watcher: watcher,
|
||||
watcher: nil,
|
||||
settingsCache: NewSettingsCache(30 * time.Second),
|
||||
dirtyDirs: make(map[string]time.Time),
|
||||
fileStability: make(map[string]*atomic.Bool),
|
||||
@@ -238,24 +240,34 @@ func (s *MediaScanner) GetStats() (int, int, int) {
|
||||
return s.totalFiles, s.newItems, s.errors
|
||||
}
|
||||
|
||||
func (s *MediaScanner) SetFolders(folders []string) error {
|
||||
// SetFolders configures the scanner's folders and (optionally) sets up an
|
||||
// fsnotify watcher over the full directory tree.
|
||||
//
|
||||
// watch should be true only for the single long-lived watch-mode scanner that
|
||||
// actually consumes watcher.Events. Ephemeral scan jobs must pass false so no
|
||||
// watcher (and thus no fd/inotify watches) is allocated — the watcher is never
|
||||
// read by scan jobs and previously leaked one watcher per job.
|
||||
func (s *MediaScanner) SetFolders(folders []string, watch bool) error {
|
||||
s.folders = folders
|
||||
|
||||
// Remove old watch if exists
|
||||
// Always close any previously-owned watcher so reconfiguration doesn't leak.
|
||||
if s.watcher != nil {
|
||||
if s.watcher != nil {
|
||||
if err := s.watcher.Close(); err != nil {
|
||||
fmt.Printf("Warning: failed to close old watcher during folder reconfiguration: %v\n", err)
|
||||
}
|
||||
if err := s.watcher.Close(); err != nil {
|
||||
fmt.Printf("Warning: failed to close old watcher during folder reconfiguration: %v\n", err)
|
||||
}
|
||||
s.watcher = nil
|
||||
}
|
||||
|
||||
// Create new watcher
|
||||
watcher, err := fsnotify.NewWatcher()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create watcher: %v", err)
|
||||
// Create + populate a fresh watcher only when the caller intends to read events.
|
||||
if watch {
|
||||
watcher, err := fsnotify.NewWatcher()
|
||||
if err != nil {
|
||||
// Return an error instead of panicking so a failed watcher can't
|
||||
// take down the whole process.
|
||||
return fmt.Errorf("failed to create watcher: %w", err)
|
||||
}
|
||||
s.watcher = watcher
|
||||
}
|
||||
s.watcher = watcher
|
||||
|
||||
// Build cache of allowed extensions per folder
|
||||
// Uses Go AllowedExtensions map as source of truth (not DB)
|
||||
@@ -286,31 +298,36 @@ func (s *MediaScanner) SetFolders(folders []string) error {
|
||||
}
|
||||
}
|
||||
|
||||
// Add all folders and their subdirectories to the watcher (like Audiobookshelf)
|
||||
watchCount := 0
|
||||
for _, folder := range folders {
|
||||
if err := s.watcher.Add(folder); err != nil {
|
||||
fmt.Printf("[WATCHER] Warning: failed to watch root folder %s: %v\n", folder, err)
|
||||
} else {
|
||||
watchCount++
|
||||
}
|
||||
filepath.WalkDir(folder, func(path string, d fs.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !d.IsDir() || path == folder {
|
||||
return nil
|
||||
}
|
||||
if err := s.watcher.Add(path); err != nil {
|
||||
fmt.Printf("[WATCHER] Warning: failed to watch subdirectory %s: %v\n", path, err)
|
||||
// Add all folders and their subdirectories to the watcher (like Audiobookshelf).
|
||||
// Only when watching; scan jobs (watch=false) skip this entirely.
|
||||
if s.watcher != nil {
|
||||
watchCount := 0
|
||||
for _, folder := range folders {
|
||||
if err := s.watcher.Add(folder); err != nil {
|
||||
fmt.Printf("[WATCHER] Warning: failed to watch root folder %s: %v\n", folder, err)
|
||||
} else {
|
||||
watchCount++
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
filepath.WalkDir(folder, func(path string, d fs.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !d.IsDir() || path == folder {
|
||||
return nil
|
||||
}
|
||||
if err := s.watcher.Add(path); err != nil {
|
||||
fmt.Printf("[WATCHER] Warning: failed to watch subdirectory %s: %v\n", path, err)
|
||||
} else {
|
||||
watchCount++
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
fmt.Printf("[WATCHER] Now watching %d directories across %d root folders\n", watchCount, len(folders))
|
||||
fmt.Printf("[WATCHER] Now watching %d directories across %d root folders\n", watchCount, len(folders))
|
||||
} else {
|
||||
fmt.Printf("[SCANNER] Configured %d root folders (watch mode disabled, no inotify watcher)\n", len(folders))
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -417,8 +434,10 @@ func (s *MediaScanner) ScanFolders(ctx context.Context) error {
|
||||
}
|
||||
|
||||
if d.IsDir() {
|
||||
if err := s.watcher.Add(path); err != nil {
|
||||
fmt.Printf("Warning: failed to watch subdirectory %s: %v\n", path, err)
|
||||
if s.watcher != nil {
|
||||
if err := s.watcher.Add(path); err != nil {
|
||||
fmt.Printf("Warning: failed to watch subdirectory %s: %v\n", path, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -2601,6 +2620,13 @@ func (s *MediaScanner) WatchChanges(ctx context.Context) error {
|
||||
go s.startBackupScan(ctx)
|
||||
|
||||
go func() {
|
||||
// The event loop only runs if a real watcher was set up (watch=true).
|
||||
// If watching with no watcher (e.g. inotify unavailable through a Docker
|
||||
// bind mount), polling via startBackupScan above still handles detection.
|
||||
if s.watcher == nil {
|
||||
fmt.Printf("[WATCHER] No inotify watcher configured; relying on periodic polling for change detection\n")
|
||||
return
|
||||
}
|
||||
fmt.Printf("[WATCHER] Event loop started for %d folders\n", len(s.folders))
|
||||
for {
|
||||
select {
|
||||
|
||||
@@ -205,7 +205,23 @@ func (w *Worker) worker() {
|
||||
return
|
||||
}
|
||||
|
||||
w.processJob(job)
|
||||
// Recover from any panic inside a job so a single failing job can
|
||||
// never crash the whole worker goroutine (and thus the process).
|
||||
func() {
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
fmt.Printf("[WORKER] panic in job %s (%s): %v\n", job.ID, job.Type, r)
|
||||
w.mu.Lock()
|
||||
w.results[job.ID] = &JobResult{
|
||||
JobID: job.ID,
|
||||
Status: JobStatusFailed,
|
||||
Error: fmt.Sprintf("panic: %v", r),
|
||||
}
|
||||
w.mu.Unlock()
|
||||
}
|
||||
}()
|
||||
w.processJob(job)
|
||||
}()
|
||||
|
||||
case <-w.ctx.Done():
|
||||
return
|
||||
@@ -348,6 +364,7 @@ func (w *Worker) processScanJob(job *Job) (interface{}, error) {
|
||||
}
|
||||
|
||||
scanner := NewMediaScanner(db)
|
||||
defer scanner.Close()
|
||||
scanner.job = job
|
||||
|
||||
job.ProgressCallback = func(progress float64, filesScanned, newItems, errors int) {
|
||||
@@ -377,7 +394,7 @@ func (w *Worker) processScanJob(job *Job) (interface{}, error) {
|
||||
}
|
||||
}
|
||||
|
||||
if err := scanner.SetFolders(folders); err != nil {
|
||||
if err := scanner.SetFolders(folders, false); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
@@ -521,7 +538,8 @@ func (w *Worker) processSetFoldersJob(job *Job) (interface{}, error) {
|
||||
|
||||
// Create scanner and configure folders
|
||||
scanner := NewMediaScanner(db)
|
||||
if err := scanner.SetFolders(folders); err != nil {
|
||||
defer scanner.Close()
|
||||
if err := scanner.SetFolders(folders, false); err != nil {
|
||||
return nil, fmt.Errorf("failed to set folders: %w", err)
|
||||
}
|
||||
|
||||
@@ -900,6 +918,7 @@ func (w *Worker) processDirectoryScanJob(job *Job) (interface{}, error) {
|
||||
|
||||
// Create temporary scanner instance for this job
|
||||
scanner := NewMediaScanner(db)
|
||||
defer scanner.Close()
|
||||
scanner.job = job
|
||||
// Find which library owns this directory (prefix match for subdirectories)
|
||||
ctx := context.Background()
|
||||
@@ -918,7 +937,7 @@ func (w *Worker) processDirectoryScanJob(job *Job) (interface{}, error) {
|
||||
folderPaths = append(folderPaths, f.FolderPath)
|
||||
}
|
||||
// Configure scanner with folders
|
||||
if err := scanner.SetFolders(folderPaths); err != nil {
|
||||
if err := scanner.SetFolders(folderPaths, false); err != nil {
|
||||
return nil, fmt.Errorf("failed to set folders: %w", err)
|
||||
}
|
||||
// Now scan the directory
|
||||
|
||||
Reference in New Issue
Block a user