john-okeefe
59d5de3607
The bookhoard container crashed with 'panic: Failed to create file
watcher: too many open files' (media_scanner.go) after running for a few
hours, preceded by floods of 'no space left on device' from watcher.Add.
Root cause: every scan job called NewMediaScanner(), which eagerly
created an fsnotify watcher. SetFolders() then walked the entire library
tree and registered one inotify watch per directory (~3,000+ across the
libraries), and ScanFolders() registered them again during its walk. The
worker never called scanner.Close() on these ephemeral per-job scanners,
and the worker loop had no recover(), so:
1. Leaked watchers accumulated until the kernel inotify watch cap was
hit (ENOSPC -> 'no space left on device'), then
2. the process fd limit (ulimit -n 1024) was exhausted, causing
fsnotify.NewWatcher() to fail with EMFILE, and
3. NewMediaScanner panicked on that error, taking down the whole
process (exit code 2). With no restart policy the container stayed
down.
The scan jobs run frequently (scan_poll_interval), so the leak built up
within hours. Note this was NOT a disk-space issue; df showed plenty free.
Fix:
- media_scanner.go: NewMediaScanner no longer creates a watcher eagerly
(s.watcher starts nil), which removes the panic site entirely -- there
is nothing to fail at construction. The watcher is created lazily only
when needed.
- media_scanner.go: SetFolders gains a [?1049h[22;0;0t[1;24r(B[m[4l[?7h[?25l[H[2JEvery 2.0s: bool[1;37Hgaruda-ser8: Fri 31 Jul 2026 10:45:41 AM EDT[2;66Hin 0.002s (127)[2;80H
[3dsh: line 1: bool: command not found
[4d[24;1H[?12l[?25h[?1049l[23;0;0t
[?1l> parameter. It creates
and populates a watcher (returning an error instead of panicking) only
when watch=true; otherwise it skips all watcher.Add calls. ScanFolders
guards its watcher.Add with a nil check, and the WatchChanges event
loop exits cleanly when there is no watcher (polling still runs).
- worker.go: the worker() loop now wraps each job in defer/recover() so a
panicking job is recorded as failed and can never kill the process.
- worker.go: the three ephemeral scan handlers (processScanJob,
processSetFoldersJob, processDirectoryScanJob) now defer scanner.Close()
and call SetFolders(..., false), so scan jobs allocate zero watchers and
zero inotify watches. Any pre-existing leak is also bounded by Close().
- handlers/scanner.go: the long-lived watch-mode scanners (StartScanner
and StartWatchModeForLibrary) pass watch=true since they actually read
watcher.Events for live change detection.
- calibre_integration_test.go: updated to the new SetFolders signature
(watch=false, matching one-off scan usage).
Auto-add is fully preserved: new files are still detected by the periodic
poller (startBackupScan), which is independent of fsnotify and unaffected
by these changes. The watch-mode event loop remains as bonus responsiveness
when inotify is available; through Docker bind mounts where inotify is
unreliable, polling is what catches new books.
187 lines
7.2 KiB
Go
187 lines
7.2 KiB
Go
package main
|
|
|
|
import (
|
|
"bookhoard/internal/database"
|
|
"bookhoard/internal/services"
|
|
"context"
|
|
"os"
|
|
"path/filepath"
|
|
"testing"
|
|
|
|
"github.com/google/uuid"
|
|
"github.com/jackc/pgx/v5/pgtype"
|
|
"github.com/stretchr/testify/require"
|
|
)
|
|
|
|
// TestCalibreLibraryScan tests importing books from a Calibre library with metadata.opf sidecar files
|
|
func TestCalibreLibraryScan(t *testing.T) {
|
|
setup := setupTestServer(t)
|
|
defer setup.Close()
|
|
|
|
ctx := context.Background()
|
|
|
|
// Get admin user ID from database
|
|
adminUser, err := setup.DB.GetUserByEmail(ctx, "testuser@tests.bookhoard.internal")
|
|
require.NoError(t, err, "Failed to get admin user")
|
|
adminUUID, err := uuid.FromBytes(adminUser.ID.Bytes[:])
|
|
require.NoError(t, err, "Failed to parse admin UUID")
|
|
adminID := pgtype.UUID{Bytes: adminUUID, Valid: true}
|
|
|
|
// Create test Calibre library structure
|
|
tmpDir := t.TempDir()
|
|
|
|
// Create author directory
|
|
authorDir := filepath.Join(tmpDir, "Test Author")
|
|
require.NoError(t, os.Mkdir(authorDir, 0755), "Failed to create author directory")
|
|
|
|
// Create book directory
|
|
bookDir := filepath.Join(authorDir, "Test Book")
|
|
require.NoError(t, os.Mkdir(bookDir, 0755), "Failed to create book directory")
|
|
|
|
// Create metadata.opf with Calibre metadata
|
|
opfPath := filepath.Join(bookDir, "metadata.opf")
|
|
opfContent := `<?xml version='1.0' encoding='utf-8'?>
|
|
<package xmlns="http://www.idpf.org/2007/opf" version="2.0">
|
|
<metadata xmlns:dc="http://purl.org/dc/elements/1.1/">
|
|
<dc:title>Test Book</dc:title>
|
|
<dc:creator>Test Author</dc:creator>
|
|
<dc:subject>Fantasy</dc:subject>
|
|
<dc:subject>Adventure</dc:subject>
|
|
<dc:description>Test description</dc:description>
|
|
<dc:publisher>Test Publisher</dc:publisher>
|
|
<dc:date>2024-01-15</dc:date>
|
|
<dc:language>en</dc:language>
|
|
<dc:identifier opf:scheme="ISBN">978-0-123456-78-9</dc:identifier>
|
|
<dc:contributor>Contributor Name</dc:contributor>
|
|
<meta name="calibre:series" content="Test Series"/>
|
|
<meta name="calibre:series_index" content="1"/>
|
|
</metadata>
|
|
</package>`
|
|
require.NoError(t, os.WriteFile(opfPath, []byte(opfContent), 0644), "Failed to create metadata.opf")
|
|
|
|
// Create dummy EPUB file
|
|
epubPath := filepath.Join(bookDir, "Test Book.epub")
|
|
require.NoError(t, os.WriteFile(epubPath, []byte("dummy epub content"), 0644), "Failed to create EPUB file")
|
|
|
|
// Create library via API
|
|
libID := createTestLibraryWithFolder(t, setup.Server, setup.Token, "Calibre Test Library", true)
|
|
|
|
// Update the folder path to our temp directory
|
|
libUUID, err := uuid.Parse(libID)
|
|
require.NoError(t, err, "Failed to parse library UUID")
|
|
|
|
// Get and delete the default folder, then add our temp directory
|
|
folders, err := setup.DB.GetLibraryFolders(ctx, pgtype.UUID{Bytes: libUUID, Valid: true})
|
|
require.NoError(t, err, "Failed to list library folders")
|
|
if len(folders) > 0 {
|
|
_, err = setup.DB.DeleteLibraryFolder(ctx, database.DeleteLibraryFolderParams{
|
|
LibraryID: pgtype.UUID{Bytes: libUUID, Valid: true},
|
|
FolderPath: folders[0].FolderPath,
|
|
})
|
|
require.NoError(t, err, "Failed to delete default folder")
|
|
}
|
|
|
|
_, err = setup.DB.AddLibraryFolder(ctx, database.AddLibraryFolderParams{
|
|
LibraryID: pgtype.UUID{Bytes: libUUID, Valid: true},
|
|
FolderPath: tmpDir,
|
|
})
|
|
require.NoError(t, err, "Failed to add folder to library")
|
|
|
|
// Create scanner and configure it
|
|
scanner := services.NewMediaScanner(setup.DB)
|
|
scanner.SetAdminID(adminID)
|
|
err = scanner.SetFolders([]string{tmpDir}, false)
|
|
require.NoError(t, err, "Failed to set scanner folders")
|
|
|
|
// Scan library
|
|
err = scanner.ScanFolders(ctx)
|
|
require.NoError(t, err, "ScanFolders should succeed")
|
|
|
|
// Verify imported book
|
|
books, err := setup.DB.ListMediaItemsByLibrary(ctx, pgtype.UUID{Bytes: libUUID, Valid: true})
|
|
require.NoError(t, err, "ListMediaItemsByLibrary should succeed")
|
|
require.Len(t, books, 1, "Should have imported 1 book")
|
|
|
|
book := books[0]
|
|
|
|
// Verify metadata from sidecar
|
|
require.Equal(t, "Test Book", book.Title, "Title should match sidecar")
|
|
require.Equal(t, "Test Author", book.Author.String, "Author should match sidecar")
|
|
require.Equal(t, "Test Series", book.Series.String, "Series should match sidecar")
|
|
require.Equal(t, int32(1), book.SeriesNumber.Int32, "Series number should match sidecar")
|
|
require.Equal(t, "Test Publisher", book.Publisher.String, "Publisher should match sidecar")
|
|
require.Len(t, book.Tags, 2, "Should have 2 tags from sidecar")
|
|
require.Contains(t, book.Tags, "Fantasy", "Should have Fantasy tag")
|
|
require.Contains(t, book.Tags, "Adventure", "Should have Adventure tag")
|
|
}
|
|
|
|
// TestCalibreLibraryScanWithoutSidecar tests that books without metadata.opf still work (backward compatibility)
|
|
func TestCalibreLibraryScanWithoutSidecar(t *testing.T) {
|
|
setup := setupTestServer(t)
|
|
defer setup.Close()
|
|
|
|
ctx := context.Background()
|
|
|
|
// Get admin user ID from database
|
|
adminUser, err := setup.DB.GetUserByEmail(ctx, "testuser@tests.bookhoard.internal")
|
|
require.NoError(t, err, "Failed to get admin user")
|
|
adminUUID, err := uuid.FromBytes(adminUser.ID.Bytes[:])
|
|
require.NoError(t, err, "Failed to parse admin UUID")
|
|
adminID := pgtype.UUID{Bytes: adminUUID, Valid: true}
|
|
|
|
// Create test directory structure (non-Calibre)
|
|
tmpDir := t.TempDir()
|
|
|
|
// Create book directory
|
|
bookDir := filepath.Join(tmpDir, "Plain Book")
|
|
require.NoError(t, os.Mkdir(bookDir, 0755), "Failed to create book directory")
|
|
|
|
// Create EPUB file WITHOUT metadata.opf sidecar
|
|
epubPath := filepath.Join(bookDir, "Plain Book.epub")
|
|
require.NoError(t, os.WriteFile(epubPath, []byte("dummy epub content"), 0644), "Failed to create EPUB file")
|
|
|
|
// Create library via API
|
|
libID := createTestLibraryWithFolder(t, setup.Server, setup.Token, "Non-Calibre Test Library", true)
|
|
|
|
// Update the folder path to our temp directory
|
|
libUUID, err := uuid.Parse(libID)
|
|
require.NoError(t, err, "Failed to parse library UUID")
|
|
|
|
// Get and delete the default folder, then add our temp directory
|
|
folders, err := setup.DB.GetLibraryFolders(ctx, pgtype.UUID{Bytes: libUUID, Valid: true})
|
|
require.NoError(t, err, "Failed to list library folders")
|
|
if len(folders) > 0 {
|
|
_, err = setup.DB.DeleteLibraryFolder(ctx, database.DeleteLibraryFolderParams{
|
|
LibraryID: pgtype.UUID{Bytes: libUUID, Valid: true},
|
|
FolderPath: folders[0].FolderPath,
|
|
})
|
|
require.NoError(t, err, "Failed to delete default folder")
|
|
}
|
|
|
|
_, err = setup.DB.AddLibraryFolder(ctx, database.AddLibraryFolderParams{
|
|
LibraryID: pgtype.UUID{Bytes: libUUID, Valid: true},
|
|
FolderPath: tmpDir,
|
|
})
|
|
require.NoError(t, err, "Failed to add folder to library")
|
|
|
|
// Create scanner and configure it
|
|
scanner := services.NewMediaScanner(setup.DB)
|
|
scanner.SetAdminID(adminID)
|
|
err = scanner.SetFolders([]string{tmpDir}, false)
|
|
require.NoError(t, err, "Failed to set scanner folders")
|
|
|
|
// Scan library
|
|
err = scanner.ScanFolders(ctx)
|
|
require.NoError(t, err, "ScanFolders should succeed")
|
|
|
|
// Verify imported book (using fallback metadata)
|
|
books, err := setup.DB.ListMediaItemsByLibrary(ctx, pgtype.UUID{Bytes: libUUID, Valid: true})
|
|
require.NoError(t, err, "ListMediaItemsByLibrary should succeed")
|
|
require.Len(t, books, 1, "Should have imported 1 book")
|
|
|
|
book := books[0]
|
|
|
|
// Verify fallback metadata (from filename)
|
|
require.Equal(t, "Plain Book", book.Title, "Title should fallback to filename")
|
|
}
|