diff --git a/AUTOSCANNER_IMPROVEMENTS.md b/AUTOSCANNER_IMPROVEMENTS.md new file mode 100644 index 0000000..1939bad --- /dev/null +++ b/AUTOSCANNER_IMPROVEMENTS.md @@ -0,0 +1,207 @@ +# Autoscanner Improvements - Implementation Plan + +## Problem Statement + +The current autoscanner has two issues: + +1. **Bulk file detection unreliable**: When files are copied in bulk (e.g., 6 files at once), fsnotify only detects 1 or 0 files +2. **Delete detection broken**: When files are deleted from the filesystem, they remain in the database + +### Root Causes + +1. **fsnotify limitations**: The file system watcher can miss events during bulk file operations +2. **Relative path mismatch**: Delete detection uses absolute paths for database lookups, but database stores relative paths (partially fixed) + +## Solution Architecture + +``` +[fsnotify Events] → [Event Queue] → [Debounce Timer (3s)] → [Process Queue] + ↓ +[Polling Fallback (3 min)] → [Full Sync Check] → [Add missing / Remove deleted] +``` + +Two complementary systems working together: +- **fsnotify + debounce**: Handles most file changes in real-time +- **Polling fallback**: Safety net that catches anything fsnotify misses + +## Implementation Plan + +### Phase 1: Add Debounce to fsnotify Handler + +**File**: `internal/services/media_scanner.go` + +**Changes**: +1. Create an event queue to accumulate fsnotify events +2. Add debounce timer (3 seconds) that resets on each new event +3. When timer fires, process all queued events in batch +4. Process each event: new files → scan, deleted files → remove from DB + +**Implementation Details**: +- Use a buffered channel as the event queue +- Use `time.After()` or `time.Timer` for debounce +- Process events in order, skip duplicates for same file + +```go +// Pseudo-code structure +type FileEvent struct { + path string + isDelete bool +} + +eventQueue := make(chan FileEvent, 100) +var debounceTimer *time.Timer + +func handleFsEvent(event fsnotify.Event) { + select { + case eventQueue <- FileEvent{path: event.Name, isDelete: event.Has(fsnotify.Remove)}: + default: + // Queue full, log warning + } + + // Reset debounce timer + if debounceTimer != nil { + debounceTimer.Stop() + } + debounceTimer = time.AfterFunc(3*time.Second, processEventQueue) +} + +func processEventQueue() { + // Drain queue and process unique paths +} +``` + +### Phase 2: Add Polling Fallback + +**File**: `internal/services/media_scanner.go` (or new file) + +**Changes**: +1. Add polling interval configuration (default: 3 minutes) +2. Create sync function that: + - Walks all library folders + - Compares filesystem against database + - Adds missing files (triggers scan for new files) + - Removes orphaned database entries (files no longer exist) +3. Start polling goroutine alongside existing fsnotify watcher + +**Configuration**: +- Environment variable: `SCAN_POLL_INTERVAL_MINUTES` (default: 3) +- Use existing config system or add to `internal/config/config.go` + +**Implementation Details**: +```go +type ScannerSyncOptions struct { + PollInterval time.Duration // default: 3 minutes +} + +func (s *MediaScanner) StartPolling(ctx context.Context, opts ScannerSyncOptions) { + ticker := time.NewTicker(opts.PollInterval) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + s.SyncFilesystemWithDatabase(ctx) + } + } +} + +func (s *MediaScanner) SyncFilesystemWithDatabase(ctx context.Context) error { + // 1. Get all media items from database + // 2. For each library folder: + // - Walk filesystem, build map of existing files (relative paths) + // - Compare against database + // - Add: file in filesystem but not in DB → scan + // - Remove: file in DB but not on filesystem → delete +} +``` + +### Phase 3: Reuse Existing Cleanup Logic + +**File**: `internal/services/media_scanner.go` + +The cleanup logic already exists (lines 252-299 in `ScanFolders`) - it removes orphaned items after each manual scan. We can refactor this into a reusable function called by both: +- Manual scan (existing behavior) +- Polling fallback (new behavior) + +```go +func (s *MediaScanner) CleanupOrphanedItems(ctx context.Context) error { + // Existing cleanup code from ScanFolders + // Extract to reusable function +} +``` + +### Phase 4: Configuration + +**File**: `internal/config/config.go` + +Add new configuration option: + +```go +type Config struct { + // ... existing fields ... + ScanPollIntervalMinutes int `env:"SCAN_POLL_INTERVAL_MINUTES"` +} + +func LoadConfig() *Config { + return &Config{ + // ... existing fields ... + ScanPollIntervalMinutes: getEnvInt("SCAN_POLL_INTERVAL_MINUTES", 3), // 3 minutes default + } +} +``` + +### Phase 5: Integration with Scheduler + +**File**: `internal/services/scheduler.go` + +The scheduler already handles periodic tasks. Consider: +1. Adding polling fallback to scheduler, OR +2. Starting polling directly in MediaScanner initialization + +## Files to Modify + +| File | Changes | +|------|---------| +| `internal/services/media_scanner.go` | Add debounce queue, polling fallback, reuse cleanup | +| `internal/config/config.go` | Add `SCAN_POLL_INTERVAL_MINUTES` config | +| `docker-compose.yml` | Add environment variable (optional) | + +## Testing Plan + +1. **Bulk file addition**: Copy 10+ files at once, verify all detected within 3 seconds (fsnotify) or 3 minutes (polling) +2. **Bulk file deletion**: Delete 5+ files, verify all removed from database within 3 minutes +3. **Mixed operations**: Add some, delete some, verify correct state +4. **Large library**: Test with 100+ files to ensure performance is acceptable + +## Backward Compatibility + +- Default polling interval: 3 minutes (user can configure) +- Existing manual scan functionality unchanged +- fsnotify continues to work as before (with debounce improvement) + +## Security Considerations + +- Polling runs on same goroutine as scanner (no new attack surface) +- File operations are read-only until changes detected +- Database operations use existing service layer (already authorized) + +## Timeline Estimate + +| Phase | Complexity | Estimate | +|-------|-------------|----------| +| Phase 1: Debounce | Medium | 1-2 hours | +| Phase 2: Polling | Medium | 1-2 hours | +| Phase 3: Reuse cleanup | Low | 30 min | +| Phase 4: Config | Low | 15 min | +| Phase 5: Integration | Low | 15 min | +| Testing | Medium | 1 hour | +| **Total** | - | **4-6 hours** | + +## Future Improvements (Out of Scope) + +1. **Configurable debounce duration** +2. **Per-library polling intervals** +3. **Event history/logging for debugging** +4. **Manual trigger for full sync** diff --git a/docker-compose.yml b/docker-compose.yml index 4063285..7653367 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -9,9 +9,12 @@ services: POSTGRES_DB: bookhoard POSTGRES_USER: postgres POSTGRES_PASSWORD: ${DBPASS} + COOKIE_SECURE: false # make true in production with HTTPS + SCAN_POLL_INTERVAL_MINUTES: 3 volumes: - postgres_data:/var/lib/postgresql/data - ./database/schema:/docker-entrypoint-initdb.d + # Make other volumes as needed - ./uploads:/app/uploads ports: - "5432:5432" @@ -84,6 +87,7 @@ services: DATABASE_USER: postgres DATABASE_PASSWORD: ${DBPASS} DATABASE_NAME: bookhoard + COOKIE_SECURE: false # Application Configuration JWT_SECRET: ${JWT_SECRET} diff --git a/internal/config/config.go b/internal/config/config.go index e3cb35d..e5ac3da 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -7,35 +7,37 @@ import ( ) type Config struct { - ServerPort string - BaseURL string - JWTSecret string - UploadPath string - DatabaseHost string - DatabasePort string - DatabaseUser string - DatabasePassword string - DatabaseName string - TestMode bool - RateLimitEnabled bool - RequestsPerMinute int + ServerPort string + BaseURL string + JWTSecret string + UploadPath string + DatabaseHost string + DatabasePort string + DatabaseUser string + DatabasePassword string + DatabaseName string + TestMode bool + RateLimitEnabled bool + RequestsPerMinute int + ScanPollIntervalMinutes int `env:"SCAN_POLL_INTERVAL_MINUTES" default:"3"` } func LoadConfig() *Config { port := getEnv("SERVER_PORT", "8765") return &Config{ - ServerPort: port, - BaseURL: getEnv("BASE_URL", "http://localhost:"+port), - DatabaseHost: getEnv("DATABASE_HOST", "localhost"), - DatabasePort: getEnv("DATABASE_PORT", "5432"), - DatabaseUser: getEnv("DATABASE_USER", "postgres"), - DatabasePassword: getEnv("DATABASE_PASSWORD", "password"), - DatabaseName: getEnv("DATABASE_NAME", "bookhoard"), - JWTSecret: getEnv("JWT_SECRET", "your-secret-key"), - UploadPath: getEnv("UPLOAD_PATH", "./uploads"), - TestMode: getEnvBool("TEST_MODE", false), - RateLimitEnabled: getEnvBool("RATE_LIMIT_ENABLED", true), - RequestsPerMinute: getEnvInt("REQUESTS_PER_MINUTE", 10), + ServerPort: port, + BaseURL: getEnv("BASE_URL", "http://localhost:"+port), + DatabaseHost: getEnv("DATABASE_HOST", "localhost"), + DatabasePort: getEnv("DATABASE_PORT", "5432"), + DatabaseUser: getEnv("DATABASE_USER", "postgres"), + DatabasePassword: getEnv("DATABASE_PASSWORD", "password"), + DatabaseName: getEnv("DATABASE_NAME", "bookhoard"), + JWTSecret: getEnv("JWT_SECRET", "your-secret-key"), + UploadPath: getEnv("UPLOAD_PATH", "./uploads"), + TestMode: getEnvBool("TEST_MODE", false), + RateLimitEnabled: getEnvBool("RATE_LIMIT_ENABLED", true), + RequestsPerMinute: getEnvInt("REQUESTS_PER_MINUTE", 10), + ScanPollIntervalMinutes: getEnvInt("SCAN_POLL_INTERVAL_MINUTES", 3), } } diff --git a/internal/handlers/auth.go b/internal/handlers/auth.go index eabd7d7..87a4b3c 100644 --- a/internal/handlers/auth.go +++ b/internal/handlers/auth.go @@ -6,6 +6,7 @@ import ( "context" "fmt" "net/http" + "os" "path/filepath" "strings" "time" @@ -29,6 +30,8 @@ const ( // Note: This is computed from SessionDuration to avoid magic numbers var SessionDurationSec = int(SessionDuration.Seconds()) +var secure = os.Getenv("COOKIE_SECURE") + type AuthHandler struct { db *database.Queries jwtKey []byte @@ -249,7 +252,8 @@ func (h *AuthHandler) Register(c echo.Context) error { Value: accessToken, Path: "/", HttpOnly: true, - Secure: false, // TODO: Set to true in production with HTTPS + Secure: secure == "true", // TODO: Set to true in production with HTTPS + SameSite: http.SameSiteLaxMode, MaxAge: SessionDurationSec, } c.SetCookie(cookie) @@ -394,7 +398,8 @@ func (h *AuthHandler) Login(c echo.Context) error { Value: accessToken, Path: "/", HttpOnly: true, - Secure: false, // TODO: Set to true in production with HTTPS + Secure: secure == "true", // TODO: Set to true in production with HTTPS + SameSite: http.SameSiteLaxMode, MaxAge: SessionDurationSec, } c.SetCookie(cookie) diff --git a/internal/handlers/collections.go b/internal/handlers/collections.go index f069ac1..e9011ea 100644 --- a/internal/handlers/collections.go +++ b/internal/handlers/collections.go @@ -928,10 +928,10 @@ func mediaItemsToListMediaItemsRow(item database.MediaItems) database.ListMediaI Author: item.Author, Isbn: item.Isbn, Description: item.Description, - FilePath: item.FilePath, + FilePath: utils.ResolveMediaURL(item.LibraryID, pgtype.Text{String: item.FilePath, Valid: true}), FileSize: item.FileSize, MimeType: item.MimeType, - CoverImagePath: item.CoverImagePath, + CoverImagePath: pgtype.Text{String: utils.ResolveMediaURL(item.LibraryID, item.CoverImagePath), Valid: true}, Series: item.Series, SeriesNumber: item.SeriesNumber, Tags: item.Tags, diff --git a/internal/handlers/commonhandlers.go b/internal/handlers/commonhandlers.go index 3da4fb4..40dfe95 100644 --- a/internal/handlers/commonhandlers.go +++ b/internal/handlers/commonhandlers.go @@ -1,6 +1,7 @@ package handlers import ( + "bookhoard/internal/config" "bookhoard/internal/database" "bookhoard/internal/services" wsync "bookhoard/internal/sync" @@ -27,9 +28,10 @@ type Handler struct { watchingLibraries map[string]bool connManager *wsync.ConnectionManager cleanupTaskCancel context.CancelFunc + config *config.Config } -func NewHandler(db *database.Queries, connManager *wsync.ConnectionManager, queueProcessor *wsync.SyncQueueProcessor) *Handler { +func NewHandler(db *database.Queries, connManager *wsync.ConnectionManager, queueProcessor *wsync.SyncQueueProcessor, cfg *config.Config) *Handler { ctx, cancel := context.WithCancel(context.Background()) worker := services.NewWorker(3) scheduler := services.NewScheduler(worker, db) @@ -40,7 +42,7 @@ func NewHandler(db *database.Queries, connManager *wsync.ConnectionManager, queu return &Handler{ db: db, - scanner: services.NewMediaScanner(db), + scanner: services.NewMediaScanner(db, cfg.ScanPollIntervalMinutes), worker: worker, scheduler: scheduler, queueProcessor: queueProcessor, @@ -66,6 +68,6 @@ func parseDate(dateStr string) time.Time { return time.Time{} } -func SetupRoutes(g *echo.Group, db *database.Queries, connManager *wsync.ConnectionManager, queueProcessor *wsync.SyncQueueProcessor) *Handler { - return NewHandler(db, connManager, queueProcessor) +func SetupRoutes(g *echo.Group, db *database.Queries, connManager *wsync.ConnectionManager, queueProcessor *wsync.SyncQueueProcessor, cfg *config.Config) *Handler { + return NewHandler(db, connManager, queueProcessor, cfg) } diff --git a/internal/handlers/dashboard.go b/internal/handlers/dashboard.go index 245e417..2f53b82 100644 --- a/internal/handlers/dashboard.go +++ b/internal/handlers/dashboard.go @@ -3,6 +3,7 @@ package handlers import ( "bookhoard/internal/database" "bookhoard/internal/services" + "bookhoard/internal/utils" "net/http" "strconv" @@ -151,7 +152,7 @@ func BuildSections(sections []services.DashboardSection) []SectionData { MediaItemID: itemUUID.String(), Title: item.Title, Author: textToString(item.Author), - CoverImagePath: textToString(item.CoverImagePath), + CoverImagePath: utils.ResolveMediaURL(item.LibraryID, item.CoverImagePath), } } diff --git a/internal/handlers/media.go b/internal/handlers/media.go index 7c71428..a484096 100644 --- a/internal/handlers/media.go +++ b/internal/handlers/media.go @@ -9,6 +9,7 @@ import ( "io" "mime" "net/http" + "net/url" "os" "path/filepath" "strconv" @@ -1490,21 +1491,23 @@ func (mh *MediaHandler) getFullFilePath(ctx context.Context, libraryID pgtype.UU // Requires JWT authentication func (mh *MediaHandler) ServeFile(c echo.Context) error { // URL format: /uploads/library-{libraryID}/{relativePath} - path := c.Param("*") // Gets everything after /uploads/library-{id}/ - - // Extract library ID from path - parts := strings.SplitN(path, "/", 2) - if len(parts) < 2 { - return c.JSON(http.StatusBadRequest, map[string]string{"error": "invalid path"}) - } - - libraryIDStr := strings.TrimPrefix(parts[0], "library-") + // Get library ID directly from route parameter + libraryIDStr := c.Param("id") libraryUUID, err := uuid.Parse(libraryIDStr) if err != nil { return c.JSON(http.StatusBadRequest, map[string]string{"error": "invalid library ID"}) } - relativePath := parts[1] + // Get remaining path from URL + rawPath := c.Param("*") + relativePath, err := url.QueryUnescape(rawPath) + if err != nil { + relativePath = rawPath + } + + if relativePath == "" { + return c.JSON(http.StatusBadRequest, map[string]string{"error": "invalid path"}) + } // Resolve using service fullPath, err := mh.getFullFilePath(c.Request().Context(), pgtype.UUID{Bytes: libraryUUID, Valid: true}, relativePath) diff --git a/internal/handlers/scanner.go b/internal/handlers/scanner.go index 60f92d0..f7acd8e 100644 --- a/internal/handlers/scanner.go +++ b/internal/handlers/scanner.go @@ -197,7 +197,7 @@ func (h *Handler) StartWatchModeForLibrary(ctx context.Context, libraryID pgtype folderPaths[i] = folder.FolderPath } - scanner := services.NewMediaScanner(h.db) + scanner := services.NewMediaScanner(h.db, 3) if err := scanner.SetFolders(folderPaths); err != nil { return fmt.Errorf("failed to set scanner folders: %v", err) } diff --git a/internal/router/router.go b/internal/router/router.go index 9475cd0..420b022 100644 --- a/internal/router/router.go +++ b/internal/router/router.go @@ -200,7 +200,7 @@ func RegisterRoutes(cfg *Config) *handlers.Handler { protected := e.Group("/api", jwtMiddleware) // Create scanner handler for scanner routes and progress routes - scannerHandler := handlers.SetupRoutes(protected, cfg.Queries, cfg.ConnManager, cfg.QueueProcessor) + scannerHandler := handlers.SetupRoutes(protected, cfg.Queries, cfg.ConnManager, cfg.QueueProcessor, cfg.Cfg) cfg.ScannerHandler = scannerHandler // Register route groups diff --git a/internal/router/sync.go b/internal/router/sync.go index dd0ef0f..6b8dc0f 100644 --- a/internal/router/sync.go +++ b/internal/router/sync.go @@ -13,7 +13,7 @@ func registerSyncRoutes(cfg *Config) { protected := e.Group("/api", jwtMiddleware) // Create handler for sync-specific routes - h := handlers.NewHandler(cfg.Queries, cfg.ConnManager, cfg.QueueProcessor) + h := handlers.NewHandler(cfg.Queries, cfg.ConnManager, cfg.QueueProcessor, cfg.Cfg) // Book matching and unlinked book resolution routes sync := protected.Group("/sync") diff --git a/internal/services/dashboard_service.go b/internal/services/dashboard_service.go index ec3e6e2..d03796f 100644 --- a/internal/services/dashboard_service.go +++ b/internal/services/dashboard_service.go @@ -2,6 +2,7 @@ package services import ( "bookhoard/internal/database" + "bookhoard/internal/utils" "context" "encoding/json" "fmt" @@ -18,10 +19,10 @@ func mediaItemsToListMediaItemsRow(item database.MediaItems) database.ListMediaI Author: item.Author, Isbn: item.Isbn, Description: item.Description, - FilePath: item.FilePath, + FilePath: utils.ResolveMediaURL(item.LibraryID, pgtype.Text{String: item.FilePath, Valid: true}), FileSize: item.FileSize, MimeType: item.MimeType, - CoverImagePath: item.CoverImagePath, + CoverImagePath: pgtype.Text{String: utils.ResolveMediaURL(item.LibraryID, item.CoverImagePath), Valid: true}, Series: item.Series, SeriesNumber: item.SeriesNumber, Tags: item.Tags, @@ -69,10 +70,10 @@ func getCollectionItemsRowToMediaItems(item database.GetCollectionItemsForDashbo Author: item.Author, Isbn: item.Isbn, Description: item.Description, - FilePath: item.FilePath, + FilePath: utils.ResolveMediaURL(item.LibraryID, pgtype.Text{String: item.FilePath, Valid: true}), FileSize: item.FileSize, MimeType: item.MimeType, - CoverImagePath: item.CoverImagePath, + CoverImagePath: pgtype.Text{String: utils.ResolveMediaURL(item.LibraryID, item.CoverImagePath), Valid: true}, Series: item.Series, SeriesNumber: item.SeriesNumber, Tags: item.Tags, diff --git a/internal/services/media_scanner.go b/internal/services/media_scanner.go index 35cbd87..1f99c5c 100644 --- a/internal/services/media_scanner.go +++ b/internal/services/media_scanner.go @@ -79,6 +79,9 @@ type MediaScanner struct { libraryTypes map[string][]string forceRescan bool logger *ScannerLogger + eventQueue chan string + debounceTimer *time.Timer + pollInterval time.Duration totalFiles int newItems int @@ -87,12 +90,17 @@ type MediaScanner struct { } // NewMediaScanner creates a new media scanner instance -func NewMediaScanner(db *database.Queries) *MediaScanner { +func NewMediaScanner(db *database.Queries, pollIntervalMinutes int) *MediaScanner { watcher, err := fsnotify.NewWatcher() if err != nil { panic(fmt.Sprintf("Failed to create file watcher: %v", err)) } + pollInterval := 3 * time.Minute + if pollInterval > 0 { + pollInterval = time.Duration(pollIntervalMinutes) * time.Minute + } + return &MediaScanner{ db: db, watcher: watcher, @@ -101,6 +109,8 @@ func NewMediaScanner(db *database.Queries) *MediaScanner { defaultLibraryID: pgtype.UUID{Valid: false}, libraryTypes: make(map[string][]string), logger: NewScannerLogger(), + eventQueue: make(chan string, 500), + pollInterval: pollInterval, } } @@ -270,7 +280,7 @@ func (s *MediaScanner) ScanFolders(ctx context.Context) error { return walkErr } if !d.IsDir() && s.isScannableFile(path) { - scannedPaths[path] = true + scannedPaths[s.getRelativePath(path)] = true } return nil }) @@ -570,6 +580,7 @@ func (s *MediaScanner) processMediaFile(ctx context.Context, path string) (bool, tagsSearch := utils.NormalizeTagsSearch(metadata.Tags) // Create media item in database + relativePath := s.getRelativePath(path) createdItem, err := s.db.CreateMediaItem(ctx, database.CreateMediaItemParams{ LibraryID: libraryID, Title: metadata.Title, @@ -577,7 +588,7 @@ func (s *MediaScanner) processMediaFile(ctx context.Context, path string) (bool, Isbn: pgtype.Text{String: utils.NormalizeISBNSafe(metadata.ISBN), Valid: metadata.ISBN != ""}, Asin: pgtype.Text{String: metadata.ASIN, Valid: metadata.ASIN != ""}, Description: pgtype.Text{String: metadata.Description, Valid: metadata.Description != ""}, - FilePath: s.getRelativePath(path), + FilePath: relativePath, FileSize: pgtype.Int8{Int64: info.Size(), Valid: true}, MimeType: pgtype.Text{String: s.getMimeType(path), Valid: true}, CoverImagePath: pgtype.Text{String: metadata.CoverPath, Valid: metadata.CoverPath != ""}, @@ -1486,7 +1497,7 @@ func (s *MediaScanner) updateMediaItem(ctx context.Context, mediaItemID pgtype.U func (s *MediaScanner) getMediaItemByFilePath(ctx context.Context, filePath string, libraryID pgtype.UUID) (database.MediaItems, error) { return s.db.GetMediaItemByFilePath(ctx, database.GetMediaItemByFilePathParams{ - FilePath: filePath, + FilePath: s.getRelativePath(filePath), LibraryID: libraryID, }) } @@ -1500,6 +1511,11 @@ func (s *MediaScanner) getMimeType(path string) string { } func (s *MediaScanner) WatchChanges(ctx context.Context) { + // Start the debounced event processor + go s.processEventQueue(ctx) + // Start polling fallback + go s.StartPolling(ctx) + // Handle fsnotify events - queue them for debouncing go func() { for { select { @@ -1507,12 +1523,10 @@ func (s *MediaScanner) WatchChanges(ctx context.Context) { if !ok { return } - // Handle new directories - add them to the watcher if event.Has(fsnotify.Create) { info, err := os.Stat(event.Name) if err == nil && info.IsDir() { - // Add the new directory to the watcher if err := s.watcher.Add(event.Name); err != nil { fmt.Printf("Warning: failed to watch new directory %s: %v\n", event.Name, err) } else { @@ -1520,64 +1534,15 @@ func (s *MediaScanner) WatchChanges(ctx context.Context) { } } } - - // Handle file modifications and creations - if (event.Has(fsnotify.Create) || event.Has(fsnotify.Write)) && s.isScannableFile(event.Name) { - fmt.Printf("New/modified media file detected: %s\n", event.Name) - if _, err := s.processMediaFile(ctx, event.Name); err != nil { - fmt.Printf("Error processing modified media file %s: %v\n", event.Name, err) + // Queue file events for debounced processing + if (event.Has(fsnotify.Create) || event.Has(fsnotify.Write) || event.Has(fsnotify.Remove)) && s.isScannableFile(event.Name) { + select { + case s.eventQueue <- event.Name: + // Event queued + default: + fmt.Printf("Warning: event queue full, dropping event for %s\n", event.Name) } } - - // Handle file deletions - if event.Has(fsnotify.Remove) && s.isScannableFile(event.Name) { - s.logger.LogDelete(fmt.Sprintf("[WATCH-DELETE] File removed from filesystem: %s", event.Name)) - - // Determine libraryID for this file - var libraryID pgtype.UUID - for _, folder := range s.folders { - if strings.HasPrefix(event.Name, folder) { - lib, err := s.db.GetLibraryByFolder(ctx, folder) - if err == nil { - libraryID = lib.LibraryID - break - } - } - } - if !libraryID.Valid { - msg := fmt.Sprintf("[WATCH-DELETE] WARNING: could not determine library for deleted file: %s", event.Name) - s.logger.LogDelete(msg) - s.logger.LogError(msg) - return - } - - // Look up media item BEFORE deleting - log for safety - existingItem, err := s.db.GetMediaItemByFilePath(ctx, database.GetMediaItemByFilePathParams{ - FilePath: event.Name, - LibraryID: libraryID, - }) - if err == nil { - msg := fmt.Sprintf("[WATCH-DELETE] Found media item to delete: ID=%s, Title=%s, Path=%s", - existingItem.ID, existingItem.Title, existingItem.FilePath) - s.logger.LogDelete(msg) - - if err := s.db.DeleteMediaItem(ctx, existingItem.ID); err != nil { - errMsg := fmt.Sprintf("[WATCH-DELETE] ERROR: failed to delete media item %s: %v", existingItem.ID, err) - s.logger.LogDelete(errMsg) - s.logger.LogError(errMsg) - } else { - s.logger.LogDelete(fmt.Sprintf("[WATCH-DELETE] SUCCESS: deleted media item '%s' (was at %s)", - existingItem.Title, existingItem.FilePath)) - } - } else if err != pgx.ErrNoRows { - errMsg := fmt.Sprintf("[WATCH-DELETE] ERROR: failed to look up media item for %s: %v", event.Name, err) - s.logger.LogDelete(errMsg) - s.logger.LogError(errMsg) - } else { - s.logger.LogDelete(fmt.Sprintf("[WATCH-DELETE] No media item found in database for deleted file: %s", event.Name)) - } - } - case err, ok := <-s.watcher.Errors: if !ok { return @@ -1589,6 +1554,188 @@ func (s *MediaScanner) WatchChanges(ctx context.Context) { } }() } +func (s *MediaScanner) processEventQueue(ctx context.Context) { + for { + select { + case <-ctx.Done(): + return + case path := <-s.eventQueue: + // Reset debounce timer - wait for more events + if s.debounceTimer != nil { + s.debounceTimer.Stop() + } + s.debounceTimer = time.AfterFunc(3*time.Second, func() { + s.flushEventQueue(ctx, path) + }) + } + } +} +func (s *MediaScanner) flushEventQueue(ctx context.Context, initialPath string) { + // Collect all pending events from queue + paths := make(map[string]bool) + paths[initialPath] = true + // Drain remaining events (with short timeout to batch them) + timeout := time.After(500 * time.Millisecond) +DrainLoop: + for { + select { + case path := <-s.eventQueue: + paths[path] = true + case <-timeout: + break DrainLoop + } + } + fmt.Printf("Processing %d file events after debounce\n", len(paths)) + // Process each unique path + for path := range paths { + // Determine if file exists or was deleted + _, err := os.Stat(path) + if os.IsNotExist(err) { + // File was deleted + s.handleFileDelete(ctx, path) + } else if err == nil { + // File exists (new or modified) + s.handleFileAdd(ctx, path) + } + } +} +func (s *MediaScanner) handleFileAdd(ctx context.Context, filePath string) { + fmt.Printf("New/modified media file detected: %s\n", filePath) + if _, err := s.processMediaFile(ctx, filePath); err != nil { + fmt.Printf("Error processing modified media file %s: %v\n", filePath, err) + } +} +func (s *MediaScanner) handleFileDelete(ctx context.Context, filePath string) { + s.logger.LogDelete(fmt.Sprintf("[WATCH-DELETE] File removed from filesystem: %s", filePath)) + // Determine libraryID for this file + var libraryID pgtype.UUID + for _, folder := range s.folders { + if strings.HasPrefix(filePath, folder) { + lib, err := s.db.GetLibraryByFolder(ctx, folder) + if err == nil { + libraryID = lib.LibraryID + break + } + } + } + if !libraryID.Valid { + msg := fmt.Sprintf("[WATCH-DELETE] WARNING: could not determine library for deleted file: %s", filePath) + s.logger.LogDelete(msg) + s.logger.LogError(msg) + return + } + // Look up media item + existingItem, err := s.db.GetMediaItemByFilePath(ctx, database.GetMediaItemByFilePathParams{ + FilePath: s.getRelativePath(filePath), + LibraryID: libraryID, + }) + if err == nil { + msg := fmt.Sprintf("[WATCH-DELETE] Found media item to delete: ID=%s, Title=%s, Path=%s", + existingItem.ID, existingItem.Title, existingItem.FilePath) + s.logger.LogDelete(msg) + if err := s.db.DeleteMediaItem(ctx, existingItem.ID); err != nil { + errMsg := fmt.Sprintf("[WATCH-DELETE] ERROR: failed to delete media item %s: %v", existingItem.ID, err) + s.logger.LogDelete(errMsg) + s.logger.LogError(errMsg) + } else { + s.logger.LogDelete(fmt.Sprintf("[WATCH-DELETE] SUCCESS: deleted media item '%s' (was at %s)", + existingItem.Title, existingItem.FilePath)) + } + } else if err != pgx.ErrNoRows { + errMsg := fmt.Sprintf("[WATCH-DELETE] ERROR: failed to look up media item for %s: %v", filePath, err) + s.logger.LogDelete(errMsg) + s.logger.LogError(errMsg) + } else { + s.logger.LogDelete(fmt.Sprintf("[WATCH-DELETE] No media item found in database for deleted file: %s", filePath)) + } +} + +func (s *MediaScanner) StartPolling(ctx context.Context) { + if s.pollInterval <= 0 { + fmt.Println("Polling fallback disabled (interval = 0") + return + } + ticker := time.NewTicker(s.pollInterval) + defer ticker.Stop() + fmt.Printf("Polling fallback started with interval: %v\n", s.pollInterval) + for { + select { + case <-ctx.Done(): + fmt.Println("Polling fallback stopped") + return + case <-ticker.C: + fmt.Println("Running polling fallback sync...") + if err := s.SyncFilesystemWithDatabase(ctx); err != nil { + fmt.Printf("Polling sync error: %v\n", err) + } + } + } +} +func (s *MediaScanner) SyncFilesystemWithDatabase(ctx context.Context) error { + fmt.Println("[POLL-SYNC] Starting filesystem sync with database") + for _, folder := range s.folders { + lib, err := s.db.GetLibraryByFolder(ctx, folder) + if err != nil { + fmt.Printf("[POLL-SYNC] Warning: failed to get library for folder %s: %v\n", folder, err) + continue + } + libraryID := lib.LibraryID + // Get all media items from database for this library + dbItems, err := s.db.ListMediaItemsByLibrary(ctx, libraryID) + if err != nil { + fmt.Printf("[POLL-SYNC] Warning: failed to get library items: %v\n", err) + continue + } + // Build set of existing file paths from filesystem + existingPaths := make(map[string]bool) + filepath.WalkDir(folder, func(path string, d os.DirEntry, walkErr error) error { + if walkErr != nil { + return walkErr + } + if !d.IsDir() && s.isScannableFile(path) { + existingPaths[s.getRelativePath(path)] = true + } + return nil + }) + // Check for orphaned items (in DB but not on filesystem) + for _, item := range dbItems { + if item.FilePath != "" && !existingPaths[item.FilePath] { + msg := fmt.Sprintf("[POLL-SYNC] Orphaned media item found: ID=%s, Title=%s, Path=%s", + item.ID, item.Title, item.FilePath) + s.logger.LogDelete(msg) + if err := s.db.DeleteMediaItem(ctx, item.ID); err != nil { + errMsg := fmt.Sprintf("[POLL-SYNC] ERROR: failed to delete orphaned item %s: %v", item.Title, err) + s.logger.LogDelete(errMsg) + s.logger.LogError(errMsg) + } else { + s.logger.LogDelete(fmt.Sprintf("[POLL-SYNC] SUCCESS: deleted orphaned item '%s'", item.Title)) + } + } + } + // Check for new files (on filesystem but not in DB) + // This is expensive, so we just check a few representative files + // The fsnotify handler should catch most new files + for relPath := range existingPaths { + // Check if this file exists in DB + _, err := s.db.GetMediaItemByFilePath(ctx, database.GetMediaItemByFilePathParams{ + FilePath: relPath, + LibraryID: libraryID, + }) + if err == pgx.ErrNoRows { + // New file found - scan it + absPath := folder + "/" + relPath + if _, err := os.Stat(absPath); err == nil { + fmt.Printf("[POLL-SYNC] New file detected, scanning: %s\n", absPath) + if _, err := s.processMediaFile(ctx, absPath); err != nil { + fmt.Printf("[POLL-SYNC] Error scanning new file %s: %v\n", absPath, err) + } + } + } + } + } + fmt.Println("[POLL-SYNC] Filesystem sync completed") + return nil +} func (s *MediaScanner) Close() error { if s.watcher != nil { diff --git a/internal/services/worker.go b/internal/services/worker.go index fd0d1da..ed88271 100644 --- a/internal/services/worker.go +++ b/internal/services/worker.go @@ -199,7 +199,7 @@ func (w *Worker) processScanJob(job *Job) (interface{}, error) { force = forceVal } - scanner := NewMediaScanner(db) + scanner := NewMediaScanner(db, 0) scanner.job = job job.ProgressCallback = func(progress float64, filesScanned, newItems, errors int) {