feat(scanner): add debounced file watching with polling fallback

- Implement event queue with 3-second debouncing for file system events
- Add configurable polling fallback (default 3 min) via SCAN_POLL_INTERVAL_MINUTES
- Add SyncFilesystemWithDatabase to detect orphaned DB entries and new files
- Integrate utils.ResolveMediaURL for consistent media file path resolution
- Add COOKIE_SECURE env var with SameSite=LaxMode for session cookies
- Update media handler to properly decode URL paths for file serving
- Refactor scanner initialization to accept poll interval configuration

internal/services/dashboard_service.go

@@ -2,6 +2,7 @@ package services
 
 import (
 	"bookhoard/internal/database"
+	"bookhoard/internal/utils"
 	"context"
 	"encoding/json"
 	"fmt"
@@ -18,10 +19,10 @@ func mediaItemsToListMediaItemsRow(item database.MediaItems) database.ListMediaI
 		Author:             item.Author,
 		Isbn:               item.Isbn,
 		Description:        item.Description,
-		FilePath:           item.FilePath,
+		FilePath:           utils.ResolveMediaURL(item.LibraryID, pgtype.Text{String: item.FilePath, Valid: true}),
 		FileSize:           item.FileSize,
 		MimeType:           item.MimeType,
-		CoverImagePath:     item.CoverImagePath,
+		CoverImagePath:     pgtype.Text{String: utils.ResolveMediaURL(item.LibraryID, item.CoverImagePath), Valid: true},
 		Series:             item.Series,
 		SeriesNumber:       item.SeriesNumber,
 		Tags:               item.Tags,
@@ -69,10 +70,10 @@ func getCollectionItemsRowToMediaItems(item database.GetCollectionItemsForDashbo
 		Author:             item.Author,
 		Isbn:               item.Isbn,
 		Description:        item.Description,
-		FilePath:           item.FilePath,
+		FilePath:           utils.ResolveMediaURL(item.LibraryID, pgtype.Text{String: item.FilePath, Valid: true}),
 		FileSize:           item.FileSize,
 		MimeType:           item.MimeType,
-		CoverImagePath:     item.CoverImagePath,
+		CoverImagePath:     pgtype.Text{String: utils.ResolveMediaURL(item.LibraryID, item.CoverImagePath), Valid: true},
 		Series:             item.Series,
 		SeriesNumber:       item.SeriesNumber,
 		Tags:               item.Tags,

internal/services/media_scanner.go

@@ -79,6 +79,9 @@ type MediaScanner struct {
 	libraryTypes     map[string][]string
 	forceRescan      bool
 	logger           *ScannerLogger
+	eventQueue       chan string
+	debounceTimer    *time.Timer
+	pollInterval     time.Duration
 
 	totalFiles int
 	newItems   int
@@ -87,12 +90,17 @@ type MediaScanner struct {
 }
 
 // NewMediaScanner creates a new media scanner instance
-func NewMediaScanner(db *database.Queries) *MediaScanner {
+func NewMediaScanner(db *database.Queries, pollIntervalMinutes int) *MediaScanner {
 	watcher, err := fsnotify.NewWatcher()
 	if err != nil {
 		panic(fmt.Sprintf("Failed to create file watcher: %v", err))
 	}
 
+	pollInterval := 3 * time.Minute
+	if pollInterval > 0 {
+		pollInterval = time.Duration(pollIntervalMinutes) * time.Minute
+	}
+
 	return &MediaScanner{
 		db:               db,
 		watcher:          watcher,
@@ -101,6 +109,8 @@ func NewMediaScanner(db *database.Queries) *MediaScanner {
 		defaultLibraryID: pgtype.UUID{Valid: false},
 		libraryTypes:     make(map[string][]string),
 		logger:           NewScannerLogger(),
+		eventQueue:       make(chan string, 500),
+		pollInterval:     pollInterval,
 	}
 }
 
@@ -270,7 +280,7 @@ func (s *MediaScanner) ScanFolders(ctx context.Context) error {
 				return walkErr
 			}
 			if !d.IsDir() && s.isScannableFile(path) {
-				scannedPaths[path] = true
+				scannedPaths[s.getRelativePath(path)] = true
 			}
 			return nil
 		})
@@ -570,6 +580,7 @@ func (s *MediaScanner) processMediaFile(ctx context.Context, path string) (bool,
 	tagsSearch := utils.NormalizeTagsSearch(metadata.Tags)
 
 	// Create media item in database
+	relativePath := s.getRelativePath(path)
 	createdItem, err := s.db.CreateMediaItem(ctx, database.CreateMediaItemParams{
 		LibraryID:          libraryID,
 		Title:              metadata.Title,
@@ -577,7 +588,7 @@ func (s *MediaScanner) processMediaFile(ctx context.Context, path string) (bool,
 		Isbn:               pgtype.Text{String: utils.NormalizeISBNSafe(metadata.ISBN), Valid: metadata.ISBN != ""},
 		Asin:               pgtype.Text{String: metadata.ASIN, Valid: metadata.ASIN != ""},
 		Description:        pgtype.Text{String: metadata.Description, Valid: metadata.Description != ""},
-		FilePath:           s.getRelativePath(path),
+		FilePath:           relativePath,
 		FileSize:           pgtype.Int8{Int64: info.Size(), Valid: true},
 		MimeType:           pgtype.Text{String: s.getMimeType(path), Valid: true},
 		CoverImagePath:     pgtype.Text{String: metadata.CoverPath, Valid: metadata.CoverPath != ""},
@@ -1486,7 +1497,7 @@ func (s *MediaScanner) updateMediaItem(ctx context.Context, mediaItemID pgtype.U
 
 func (s *MediaScanner) getMediaItemByFilePath(ctx context.Context, filePath string, libraryID pgtype.UUID) (database.MediaItems, error) {
 	return s.db.GetMediaItemByFilePath(ctx, database.GetMediaItemByFilePathParams{
-		FilePath:  filePath,
+		FilePath:  s.getRelativePath(filePath),
 		LibraryID: libraryID,
 	})
 }
@@ -1500,6 +1511,11 @@ func (s *MediaScanner) getMimeType(path string) string {
 }
 
 func (s *MediaScanner) WatchChanges(ctx context.Context) {
+	// Start the debounced event processor
+	go s.processEventQueue(ctx)
+	// Start polling fallback
+	go s.StartPolling(ctx)
+	// Handle fsnotify events - queue them for debouncing
 	go func() {
 		for {
 			select {
@@ -1507,12 +1523,10 @@ func (s *MediaScanner) WatchChanges(ctx context.Context) {
 				if !ok {
 					return
 				}
-
 				// Handle new directories - add them to the watcher
 				if event.Has(fsnotify.Create) {
 					info, err := os.Stat(event.Name)
 					if err == nil && info.IsDir() {
-						// Add the new directory to the watcher
 						if err := s.watcher.Add(event.Name); err != nil {
 							fmt.Printf("Warning: failed to watch new directory %s: %v\n", event.Name, err)
 						} else {
@@ -1520,64 +1534,15 @@ func (s *MediaScanner) WatchChanges(ctx context.Context) {
 						}
 					}
 				}
-
-				// Handle file modifications and creations
-				if (event.Has(fsnotify.Create) || event.Has(fsnotify.Write)) && s.isScannableFile(event.Name) {
-					fmt.Printf("New/modified media file detected: %s\n", event.Name)
-					if _, err := s.processMediaFile(ctx, event.Name); err != nil {
-						fmt.Printf("Error processing modified media file %s: %v\n", event.Name, err)
+				// Queue file events for debounced processing
+				if (event.Has(fsnotify.Create) || event.Has(fsnotify.Write) || event.Has(fsnotify.Remove)) && s.isScannableFile(event.Name) {
+					select {
+					case s.eventQueue <- event.Name:
+						// Event queued
+					default:
+						fmt.Printf("Warning: event queue full, dropping event for %s\n", event.Name)
 					}
 				}
-
-				// Handle file deletions
-				if event.Has(fsnotify.Remove) && s.isScannableFile(event.Name) {
-					s.logger.LogDelete(fmt.Sprintf("[WATCH-DELETE] File removed from filesystem: %s", event.Name))
-
-					// Determine libraryID for this file
-					var libraryID pgtype.UUID
-					for _, folder := range s.folders {
-						if strings.HasPrefix(event.Name, folder) {
-							lib, err := s.db.GetLibraryByFolder(ctx, folder)
-							if err == nil {
-								libraryID = lib.LibraryID
-								break
-							}
-						}
-					}
-					if !libraryID.Valid {
-						msg := fmt.Sprintf("[WATCH-DELETE] WARNING: could not determine library for deleted file: %s", event.Name)
-						s.logger.LogDelete(msg)
-						s.logger.LogError(msg)
-						return
-					}
-
-					// Look up media item BEFORE deleting - log for safety
-					existingItem, err := s.db.GetMediaItemByFilePath(ctx, database.GetMediaItemByFilePathParams{
-						FilePath:  event.Name,
-						LibraryID: libraryID,
-					})
-					if err == nil {
-						msg := fmt.Sprintf("[WATCH-DELETE] Found media item to delete: ID=%s, Title=%s, Path=%s",
-							existingItem.ID, existingItem.Title, existingItem.FilePath)
-						s.logger.LogDelete(msg)
-
-						if err := s.db.DeleteMediaItem(ctx, existingItem.ID); err != nil {
-							errMsg := fmt.Sprintf("[WATCH-DELETE] ERROR: failed to delete media item %s: %v", existingItem.ID, err)
-							s.logger.LogDelete(errMsg)
-							s.logger.LogError(errMsg)
-						} else {
-							s.logger.LogDelete(fmt.Sprintf("[WATCH-DELETE] SUCCESS: deleted media item '%s' (was at %s)",
-								existingItem.Title, existingItem.FilePath))
-						}
-					} else if err != pgx.ErrNoRows {
-						errMsg := fmt.Sprintf("[WATCH-DELETE] ERROR: failed to look up media item for %s: %v", event.Name, err)
-						s.logger.LogDelete(errMsg)
-						s.logger.LogError(errMsg)
-					} else {
-						s.logger.LogDelete(fmt.Sprintf("[WATCH-DELETE] No media item found in database for deleted file: %s", event.Name))
-					}
-				}
-
 			case err, ok := <-s.watcher.Errors:
 				if !ok {
 					return
@@ -1589,6 +1554,188 @@ func (s *MediaScanner) WatchChanges(ctx context.Context) {
 		}
 	}()
 }
+func (s *MediaScanner) processEventQueue(ctx context.Context) {
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case path := <-s.eventQueue:
+			// Reset debounce timer - wait for more events
+			if s.debounceTimer != nil {
+				s.debounceTimer.Stop()
+			}
+			s.debounceTimer = time.AfterFunc(3*time.Second, func() {
+				s.flushEventQueue(ctx, path)
+			})
+		}
+	}
+}
+func (s *MediaScanner) flushEventQueue(ctx context.Context, initialPath string) {
+	// Collect all pending events from queue
+	paths := make(map[string]bool)
+	paths[initialPath] = true
+	// Drain remaining events (with short timeout to batch them)
+	timeout := time.After(500 * time.Millisecond)
+DrainLoop:
+	for {
+		select {
+		case path := <-s.eventQueue:
+			paths[path] = true
+		case <-timeout:
+			break DrainLoop
+		}
+	}
+	fmt.Printf("Processing %d file events after debounce\n", len(paths))
+	// Process each unique path
+	for path := range paths {
+		// Determine if file exists or was deleted
+		_, err := os.Stat(path)
+		if os.IsNotExist(err) {
+			// File was deleted
+			s.handleFileDelete(ctx, path)
+		} else if err == nil {
+			// File exists (new or modified)
+			s.handleFileAdd(ctx, path)
+		}
+	}
+}
+func (s *MediaScanner) handleFileAdd(ctx context.Context, filePath string) {
+	fmt.Printf("New/modified media file detected: %s\n", filePath)
+	if _, err := s.processMediaFile(ctx, filePath); err != nil {
+		fmt.Printf("Error processing modified media file %s: %v\n", filePath, err)
+	}
+}
+func (s *MediaScanner) handleFileDelete(ctx context.Context, filePath string) {
+	s.logger.LogDelete(fmt.Sprintf("[WATCH-DELETE] File removed from filesystem: %s", filePath))
+	// Determine libraryID for this file
+	var libraryID pgtype.UUID
+	for _, folder := range s.folders {
+		if strings.HasPrefix(filePath, folder) {
+			lib, err := s.db.GetLibraryByFolder(ctx, folder)
+			if err == nil {
+				libraryID = lib.LibraryID
+				break
+			}
+		}
+	}
+	if !libraryID.Valid {
+		msg := fmt.Sprintf("[WATCH-DELETE] WARNING: could not determine library for deleted file: %s", filePath)
+		s.logger.LogDelete(msg)
+		s.logger.LogError(msg)
+		return
+	}
+	// Look up media item
+	existingItem, err := s.db.GetMediaItemByFilePath(ctx, database.GetMediaItemByFilePathParams{
+		FilePath:  s.getRelativePath(filePath),
+		LibraryID: libraryID,
+	})
+	if err == nil {
+		msg := fmt.Sprintf("[WATCH-DELETE] Found media item to delete: ID=%s, Title=%s, Path=%s",
+			existingItem.ID, existingItem.Title, existingItem.FilePath)
+		s.logger.LogDelete(msg)
+		if err := s.db.DeleteMediaItem(ctx, existingItem.ID); err != nil {
+			errMsg := fmt.Sprintf("[WATCH-DELETE] ERROR: failed to delete media item %s: %v", existingItem.ID, err)
+			s.logger.LogDelete(errMsg)
+			s.logger.LogError(errMsg)
+		} else {
+			s.logger.LogDelete(fmt.Sprintf("[WATCH-DELETE] SUCCESS: deleted media item '%s' (was at %s)",
+				existingItem.Title, existingItem.FilePath))
+		}
+	} else if err != pgx.ErrNoRows {
+		errMsg := fmt.Sprintf("[WATCH-DELETE] ERROR: failed to look up media item for %s: %v", filePath, err)
+		s.logger.LogDelete(errMsg)
+		s.logger.LogError(errMsg)
+	} else {
+		s.logger.LogDelete(fmt.Sprintf("[WATCH-DELETE] No media item found in database for deleted file: %s", filePath))
+	}
+}
+
+func (s *MediaScanner) StartPolling(ctx context.Context) {
+	if s.pollInterval <= 0 {
+		fmt.Println("Polling fallback disabled (interval = 0")
+		return
+	}
+	ticker := time.NewTicker(s.pollInterval)
+	defer ticker.Stop()
+	fmt.Printf("Polling fallback started with interval: %v\n", s.pollInterval)
+	for {
+		select {
+		case <-ctx.Done():
+			fmt.Println("Polling fallback stopped")
+			return
+		case <-ticker.C:
+			fmt.Println("Running polling fallback sync...")
+			if err := s.SyncFilesystemWithDatabase(ctx); err != nil {
+				fmt.Printf("Polling sync error: %v\n", err)
+			}
+		}
+	}
+}
+func (s *MediaScanner) SyncFilesystemWithDatabase(ctx context.Context) error {
+	fmt.Println("[POLL-SYNC] Starting filesystem sync with database")
+	for _, folder := range s.folders {
+		lib, err := s.db.GetLibraryByFolder(ctx, folder)
+		if err != nil {
+			fmt.Printf("[POLL-SYNC] Warning: failed to get library for folder %s: %v\n", folder, err)
+			continue
+		}
+		libraryID := lib.LibraryID
+		// Get all media items from database for this library
+		dbItems, err := s.db.ListMediaItemsByLibrary(ctx, libraryID)
+		if err != nil {
+			fmt.Printf("[POLL-SYNC] Warning: failed to get library items: %v\n", err)
+			continue
+		}
+		// Build set of existing file paths from filesystem
+		existingPaths := make(map[string]bool)
+		filepath.WalkDir(folder, func(path string, d os.DirEntry, walkErr error) error {
+			if walkErr != nil {
+				return walkErr
+			}
+			if !d.IsDir() && s.isScannableFile(path) {
+				existingPaths[s.getRelativePath(path)] = true
+			}
+			return nil
+		})
+		// Check for orphaned items (in DB but not on filesystem)
+		for _, item := range dbItems {
+			if item.FilePath != "" && !existingPaths[item.FilePath] {
+				msg := fmt.Sprintf("[POLL-SYNC] Orphaned media item found: ID=%s, Title=%s, Path=%s",
+					item.ID, item.Title, item.FilePath)
+				s.logger.LogDelete(msg)
+				if err := s.db.DeleteMediaItem(ctx, item.ID); err != nil {
+					errMsg := fmt.Sprintf("[POLL-SYNC] ERROR: failed to delete orphaned item %s: %v", item.Title, err)
+					s.logger.LogDelete(errMsg)
+					s.logger.LogError(errMsg)
+				} else {
+					s.logger.LogDelete(fmt.Sprintf("[POLL-SYNC] SUCCESS: deleted orphaned item '%s'", item.Title))
+				}
+			}
+		}
+		// Check for new files (on filesystem but not in DB)
+		// This is expensive, so we just check a few representative files
+		// The fsnotify handler should catch most new files
+		for relPath := range existingPaths {
+			// Check if this file exists in DB
+			_, err := s.db.GetMediaItemByFilePath(ctx, database.GetMediaItemByFilePathParams{
+				FilePath:  relPath,
+				LibraryID: libraryID,
+			})
+			if err == pgx.ErrNoRows {
+				// New file found - scan it
+				absPath := folder + "/" + relPath
+				if _, err := os.Stat(absPath); err == nil {
+					fmt.Printf("[POLL-SYNC] New file detected, scanning: %s\n", absPath)
+					if _, err := s.processMediaFile(ctx, absPath); err != nil {
+						fmt.Printf("[POLL-SYNC] Error scanning new file %s: %v\n", absPath, err)
+					}
+				}
+			}
+		}
+	}
+	fmt.Println("[POLL-SYNC] Filesystem sync completed")
+	return nil
+}
 
 func (s *MediaScanner) Close() error {
 	if s.watcher != nil {

internal/services/worker.go

@@ -199,7 +199,7 @@ func (w *Worker) processScanJob(job *Job) (interface{}, error) {
 		force = forceVal
 	}
 
-	scanner := NewMediaScanner(db)
+	scanner := NewMediaScanner(db, 0)
 	scanner.job = job
 
 	job.ProgressCallback = func(progress float64, filesScanned, newItems, errors int) {