Bookhoard/bookhoard
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 12 Wiki Activity

feat(scanner): add debounced file watching with polling fallback

Browse Source 
- Implement event queue with 3-second debouncing for file system events
- Add configurable polling fallback (default 3 min) via SCAN_POLL_INTERVAL_MINUTES
- Add SyncFilesystemWithDatabase to detect orphaned DB entries and new files
- Integrate utils.ResolveMediaURL for consistent media file path resolution
- Add COOKIE_SECURE env var with SameSite=LaxMode for session cookies
- Update media handler to properly decode URL paths for file serving
- Refactor scanner initialization to accept poll interval configuration
This commit is contained in:
john-okeefe
2026-02-28 01:16:27 -05:00
parent 594c630b99
commit 037e7c1189
14 changed files with 485 additions and 113 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/config/config.go
+26 -24
View File
@@ -7,35 +7,37 @@ import (
)
type Config struct {
ServerPort string
BaseURL string
JWTSecret string
UploadPath string
DatabaseHost string
DatabasePort string
DatabaseUser string
DatabasePassword string
DatabaseName string
TestMode bool
RateLimitEnabled bool
RequestsPerMinute int
ServerPort string
BaseURL string
JWTSecret string
UploadPath string
DatabaseHost string
DatabasePort string
DatabaseUser string
DatabasePassword string
DatabaseName string
TestMode bool
RateLimitEnabled bool
RequestsPerMinute int
ScanPollIntervalMinutes int `env:"SCAN_POLL_INTERVAL_MINUTES" default:"3"`
}
func LoadConfig() *Config {
port := getEnv("SERVER_PORT", "8765")
return &Config{
ServerPort: port,
BaseURL: getEnv("BASE_URL", "http://localhost:"+port),
DatabaseHost: getEnv("DATABASE_HOST", "localhost"),
DatabasePort: getEnv("DATABASE_PORT", "5432"),
DatabaseUser: getEnv("DATABASE_USER", "postgres"),
DatabasePassword: getEnv("DATABASE_PASSWORD", "password"),
DatabaseName: getEnv("DATABASE_NAME", "bookhoard"),
JWTSecret: getEnv("JWT_SECRET", "your-secret-key"),
UploadPath: getEnv("UPLOAD_PATH", "./uploads"),
TestMode: getEnvBool("TEST_MODE", false),
RateLimitEnabled: getEnvBool("RATE_LIMIT_ENABLED", true),
RequestsPerMinute: getEnvInt("REQUESTS_PER_MINUTE", 10),
ServerPort: port,
BaseURL: getEnv("BASE_URL", "http://localhost:"+port),
DatabaseHost: getEnv("DATABASE_HOST", "localhost"),
DatabasePort: getEnv("DATABASE_PORT", "5432"),
DatabaseUser: getEnv("DATABASE_USER", "postgres"),
DatabasePassword: getEnv("DATABASE_PASSWORD", "password"),
DatabaseName: getEnv("DATABASE_NAME", "bookhoard"),
JWTSecret: getEnv("JWT_SECRET", "your-secret-key"),
UploadPath: getEnv("UPLOAD_PATH", "./uploads"),
TestMode: getEnvBool("TEST_MODE", false),
RateLimitEnabled: getEnvBool("RATE_LIMIT_ENABLED", true),
RequestsPerMinute: getEnvInt("REQUESTS_PER_MINUTE", 10),
ScanPollIntervalMinutes: getEnvInt("SCAN_POLL_INTERVAL_MINUTES", 3),
}
}
internal/handlers/auth.go
+7 -2
View File
@@ -6,6 +6,7 @@ import (
"context"
"fmt"
"net/http"
"os"
"path/filepath"
"strings"
"time"
@@ -29,6 +30,8 @@ const (
// Note: This is computed from SessionDuration to avoid magic numbers
var SessionDurationSec = int(SessionDuration.Seconds())
var secure = os.Getenv("COOKIE_SECURE")
type AuthHandler struct {
db *database.Queries
jwtKey []byte
@@ -249,7 +252,8 @@ func (h *AuthHandler) Register(c echo.Context) error {
Value: accessToken,
Path: "/",
HttpOnly: true,
Secure: false, // TODO: Set to true in production with HTTPS
Secure: secure == "true", // TODO: Set to true in production with HTTPS
SameSite: http.SameSiteLaxMode,
MaxAge: SessionDurationSec,
}
c.SetCookie(cookie)
@@ -394,7 +398,8 @@ func (h *AuthHandler) Login(c echo.Context) error {
Value: accessToken,
Path: "/",
HttpOnly: true,
Secure: false, // TODO: Set to true in production with HTTPS
Secure: secure == "true", // TODO: Set to true in production with HTTPS
SameSite: http.SameSiteLaxMode,
MaxAge: SessionDurationSec,
}
c.SetCookie(cookie)
internal/handlers/collections.go
+2 -2
View File
@@ -928,10 +928,10 @@ func mediaItemsToListMediaItemsRow(item database.MediaItems) database.ListMediaI
Author: item.Author,
Isbn: item.Isbn,
Description: item.Description,
FilePath: item.FilePath,
FilePath: utils.ResolveMediaURL(item.LibraryID, pgtype.Text{String: item.FilePath, Valid: true}),
FileSize: item.FileSize,
MimeType: item.MimeType,
CoverImagePath: item.CoverImagePath,
CoverImagePath: pgtype.Text{String: utils.ResolveMediaURL(item.LibraryID, item.CoverImagePath), Valid: true},
Series: item.Series,
SeriesNumber: item.SeriesNumber,
Tags: item.Tags,
internal/handlers/commonhandlers.go
+6 -4
View File
@@ -1,6 +1,7 @@
package handlers
import (
"bookhoard/internal/config"
"bookhoard/internal/database"
"bookhoard/internal/services"
wsync "bookhoard/internal/sync"
@@ -27,9 +28,10 @@ type Handler struct {
watchingLibraries map[string]bool
connManager *wsync.ConnectionManager
cleanupTaskCancel context.CancelFunc
config *config.Config
}
func NewHandler(db *database.Queries, connManager *wsync.ConnectionManager, queueProcessor *wsync.SyncQueueProcessor) *Handler {
func NewHandler(db *database.Queries, connManager *wsync.ConnectionManager, queueProcessor *wsync.SyncQueueProcessor, cfg *config.Config) *Handler {
ctx, cancel := context.WithCancel(context.Background())
worker := services.NewWorker(3)
scheduler := services.NewScheduler(worker, db)
@@ -40,7 +42,7 @@ func NewHandler(db *database.Queries, connManager *wsync.ConnectionManager, queu
return &Handler{
db: db,
scanner: services.NewMediaScanner(db),
scanner: services.NewMediaScanner(db, cfg.ScanPollIntervalMinutes),
worker: worker,
scheduler: scheduler,
queueProcessor: queueProcessor,
@@ -66,6 +68,6 @@ func parseDate(dateStr string) time.Time {
return time.Time{}
}
func SetupRoutes(g *echo.Group, db *database.Queries, connManager *wsync.ConnectionManager, queueProcessor *wsync.SyncQueueProcessor) *Handler {
return NewHandler(db, connManager, queueProcessor)
func SetupRoutes(g *echo.Group, db *database.Queries, connManager *wsync.ConnectionManager, queueProcessor *wsync.SyncQueueProcessor, cfg *config.Config) *Handler {
return NewHandler(db, connManager, queueProcessor, cfg)
}
internal/handlers/dashboard.go
+2 -1
View File
@@ -3,6 +3,7 @@ package handlers
import (
"bookhoard/internal/database"
"bookhoard/internal/services"
"bookhoard/internal/utils"
"net/http"
"strconv"
@@ -151,7 +152,7 @@ func BuildSections(sections []services.DashboardSection) []SectionData {
MediaItemID: itemUUID.String(),
Title: item.Title,
Author: textToString(item.Author),
CoverImagePath: textToString(item.CoverImagePath),
CoverImagePath: utils.ResolveMediaURL(item.LibraryID, item.CoverImagePath),
}
}
internal/handlers/media.go
+13 -10
View File
@@ -9,6 +9,7 @@ import (
"io"
"mime"
"net/http"
"net/url"
"os"
"path/filepath"
"strconv"
@@ -1490,21 +1491,23 @@ func (mh *MediaHandler) getFullFilePath(ctx context.Context, libraryID pgtype.UU
// Requires JWT authentication
func (mh *MediaHandler) ServeFile(c echo.Context) error {
// URL format: /uploads/library-{libraryID}/{relativePath}
path := c.Param("*") // Gets everything after /uploads/library-{id}/
// Extract library ID from path
parts := strings.SplitN(path, "/", 2)
if len(parts) < 2 {
return c.JSON(http.StatusBadRequest, map[string]string{"error": "invalid path"})
}
libraryIDStr := strings.TrimPrefix(parts[0], "library-")
// Get library ID directly from route parameter
libraryIDStr := c.Param("id")
libraryUUID, err := uuid.Parse(libraryIDStr)
if err != nil {
return c.JSON(http.StatusBadRequest, map[string]string{"error": "invalid library ID"})
}
relativePath := parts[1]
// Get remaining path from URL
rawPath := c.Param("*")
relativePath, err := url.QueryUnescape(rawPath)
if err != nil {
relativePath = rawPath
}
if relativePath == "" {
return c.JSON(http.StatusBadRequest, map[string]string{"error": "invalid path"})
}
// Resolve using service
fullPath, err := mh.getFullFilePath(c.Request().Context(), pgtype.UUID{Bytes: libraryUUID, Valid: true}, relativePath)
internal/handlers/scanner.go
+1 -1
View File
@@ -197,7 +197,7 @@ func (h *Handler) StartWatchModeForLibrary(ctx context.Context, libraryID pgtype
folderPaths[i] = folder.FolderPath
}
scanner := services.NewMediaScanner(h.db)
scanner := services.NewMediaScanner(h.db, 3)
if err := scanner.SetFolders(folderPaths); err != nil {
return fmt.Errorf("failed to set scanner folders: %v", err)
}
internal/router/router.go
+1 -1
View File
@@ -200,7 +200,7 @@ func RegisterRoutes(cfg *Config) *handlers.Handler {
protected := e.Group("/api", jwtMiddleware)
// Create scanner handler for scanner routes and progress routes
scannerHandler := handlers.SetupRoutes(protected, cfg.Queries, cfg.ConnManager, cfg.QueueProcessor)
scannerHandler := handlers.SetupRoutes(protected, cfg.Queries, cfg.ConnManager, cfg.QueueProcessor, cfg.Cfg)
cfg.ScannerHandler = scannerHandler
// Register route groups
internal/router/sync.go
+1 -1
View File
@@ -13,7 +13,7 @@ func registerSyncRoutes(cfg *Config) {
protected := e.Group("/api", jwtMiddleware)
// Create handler for sync-specific routes
h := handlers.NewHandler(cfg.Queries, cfg.ConnManager, cfg.QueueProcessor)
h := handlers.NewHandler(cfg.Queries, cfg.ConnManager, cfg.QueueProcessor, cfg.Cfg)
// Book matching and unlinked book resolution routes
sync := protected.Group("/sync")
internal/services/dashboard_service.go
+5 -4
View File
@@ -2,6 +2,7 @@ package services
import (
"bookhoard/internal/database"
"bookhoard/internal/utils"
"context"
"encoding/json"
"fmt"
@@ -18,10 +19,10 @@ func mediaItemsToListMediaItemsRow(item database.MediaItems) database.ListMediaI
Author: item.Author,
Isbn: item.Isbn,
Description: item.Description,
FilePath: item.FilePath,
FilePath: utils.ResolveMediaURL(item.LibraryID, pgtype.Text{String: item.FilePath, Valid: true}),
FileSize: item.FileSize,
MimeType: item.MimeType,
CoverImagePath: item.CoverImagePath,
CoverImagePath: pgtype.Text{String: utils.ResolveMediaURL(item.LibraryID, item.CoverImagePath), Valid: true},
Series: item.Series,
SeriesNumber: item.SeriesNumber,
Tags: item.Tags,
@@ -69,10 +70,10 @@ func getCollectionItemsRowToMediaItems(item database.GetCollectionItemsForDashbo
Author: item.Author,
Isbn: item.Isbn,
Description: item.Description,
FilePath: item.FilePath,
FilePath: utils.ResolveMediaURL(item.LibraryID, pgtype.Text{String: item.FilePath, Valid: true}),
FileSize: item.FileSize,
MimeType: item.MimeType,
CoverImagePath: item.CoverImagePath,
CoverImagePath: pgtype.Text{String: utils.ResolveMediaURL(item.LibraryID, item.CoverImagePath), Valid: true},
Series: item.Series,
SeriesNumber: item.SeriesNumber,
Tags: item.Tags,
internal/services/media_scanner.go
+209 -62
View File
@@ -79,6 +79,9 @@ type MediaScanner struct {
libraryTypes map[string][]string
forceRescan bool
logger *ScannerLogger
eventQueue chan string
debounceTimer *time.Timer
pollInterval time.Duration
totalFiles int
newItems int
@@ -87,12 +90,17 @@ type MediaScanner struct {
}
// NewMediaScanner creates a new media scanner instance
func NewMediaScanner(db *database.Queries) *MediaScanner {
func NewMediaScanner(db *database.Queries, pollIntervalMinutes int) *MediaScanner {
watcher, err := fsnotify.NewWatcher()
if err != nil {
panic(fmt.Sprintf("Failed to create file watcher: %v", err))
}
pollInterval := 3 * time.Minute
if pollInterval > 0 {
pollInterval = time.Duration(pollIntervalMinutes) * time.Minute
}
return &MediaScanner{
db: db,
watcher: watcher,
@@ -101,6 +109,8 @@ func NewMediaScanner(db *database.Queries) *MediaScanner {
defaultLibraryID: pgtype.UUID{Valid: false},
libraryTypes: make(map[string][]string),
logger: NewScannerLogger(),
eventQueue: make(chan string, 500),
pollInterval: pollInterval,
}
}
@@ -270,7 +280,7 @@ func (s *MediaScanner) ScanFolders(ctx context.Context) error {
return walkErr
}
if !d.IsDir() && s.isScannableFile(path) {
scannedPaths[path] = true
scannedPaths[s.getRelativePath(path)] = true
}
return nil
})
@@ -570,6 +580,7 @@ func (s *MediaScanner) processMediaFile(ctx context.Context, path string) (bool,
tagsSearch := utils.NormalizeTagsSearch(metadata.Tags)
// Create media item in database
relativePath := s.getRelativePath(path)
createdItem, err := s.db.CreateMediaItem(ctx, database.CreateMediaItemParams{
LibraryID: libraryID,
Title: metadata.Title,
@@ -577,7 +588,7 @@ func (s *MediaScanner) processMediaFile(ctx context.Context, path string) (bool,
Isbn: pgtype.Text{String: utils.NormalizeISBNSafe(metadata.ISBN), Valid: metadata.ISBN != ""},
Asin: pgtype.Text{String: metadata.ASIN, Valid: metadata.ASIN != ""},
Description: pgtype.Text{String: metadata.Description, Valid: metadata.Description != ""},
FilePath: s.getRelativePath(path),
FilePath: relativePath,
FileSize: pgtype.Int8{Int64: info.Size(), Valid: true},
MimeType: pgtype.Text{String: s.getMimeType(path), Valid: true},
CoverImagePath: pgtype.Text{String: metadata.CoverPath, Valid: metadata.CoverPath != ""},
@@ -1486,7 +1497,7 @@ func (s *MediaScanner) updateMediaItem(ctx context.Context, mediaItemID pgtype.U
func (s *MediaScanner) getMediaItemByFilePath(ctx context.Context, filePath string, libraryID pgtype.UUID) (database.MediaItems, error) {
return s.db.GetMediaItemByFilePath(ctx, database.GetMediaItemByFilePathParams{
FilePath: filePath,
FilePath: s.getRelativePath(filePath),
LibraryID: libraryID,
})
}
@@ -1500,6 +1511,11 @@ func (s *MediaScanner) getMimeType(path string) string {
}
func (s *MediaScanner) WatchChanges(ctx context.Context) {
// Start the debounced event processor
go s.processEventQueue(ctx)
// Start polling fallback
go s.StartPolling(ctx)
// Handle fsnotify events - queue them for debouncing
go func() {
for {
select {
@@ -1507,12 +1523,10 @@ func (s *MediaScanner) WatchChanges(ctx context.Context) {
if !ok {
return
}
// Handle new directories - add them to the watcher
if event.Has(fsnotify.Create) {
info, err := os.Stat(event.Name)
if err == nil && info.IsDir() {
// Add the new directory to the watcher
if err := s.watcher.Add(event.Name); err != nil {
fmt.Printf("Warning: failed to watch new directory %s: %v\n", event.Name, err)
} else {
@@ -1520,64 +1534,15 @@ func (s *MediaScanner) WatchChanges(ctx context.Context) {
}
}
}
// Handle file modifications and creations
if (event.Has(fsnotify.Create) || event.Has(fsnotify.Write)) && s.isScannableFile(event.Name) {
fmt.Printf("New/modified media file detected: %s\n", event.Name)
if _, err := s.processMediaFile(ctx, event.Name); err != nil {
fmt.Printf("Error processing modified media file %s: %v\n", event.Name, err)
// Queue file events for debounced processing
if (event.Has(fsnotify.Create) || event.Has(fsnotify.Write) || event.Has(fsnotify.Remove)) && s.isScannableFile(event.Name) {
select {
case s.eventQueue <- event.Name:
// Event queued
default:
fmt.Printf("Warning: event queue full, dropping event for %s\n", event.Name)
}
}
// Handle file deletions
if event.Has(fsnotify.Remove) && s.isScannableFile(event.Name) {
s.logger.LogDelete(fmt.Sprintf("[WATCH-DELETE] File removed from filesystem: %s", event.Name))
// Determine libraryID for this file
var libraryID pgtype.UUID
for _, folder := range s.folders {
if strings.HasPrefix(event.Name, folder) {
lib, err := s.db.GetLibraryByFolder(ctx, folder)
if err == nil {
libraryID = lib.LibraryID
break
}
}
}
if !libraryID.Valid {
msg := fmt.Sprintf("[WATCH-DELETE] WARNING: could not determine library for deleted file: %s", event.Name)
s.logger.LogDelete(msg)
s.logger.LogError(msg)
return
}
// Look up media item BEFORE deleting - log for safety
existingItem, err := s.db.GetMediaItemByFilePath(ctx, database.GetMediaItemByFilePathParams{
FilePath: event.Name,
LibraryID: libraryID,
})
if err == nil {
msg := fmt.Sprintf("[WATCH-DELETE] Found media item to delete: ID=%s, Title=%s, Path=%s",
existingItem.ID, existingItem.Title, existingItem.FilePath)
s.logger.LogDelete(msg)
if err := s.db.DeleteMediaItem(ctx, existingItem.ID); err != nil {
errMsg := fmt.Sprintf("[WATCH-DELETE] ERROR: failed to delete media item %s: %v", existingItem.ID, err)
s.logger.LogDelete(errMsg)
s.logger.LogError(errMsg)
} else {
s.logger.LogDelete(fmt.Sprintf("[WATCH-DELETE] SUCCESS: deleted media item '%s' (was at %s)",
existingItem.Title, existingItem.FilePath))
}
} else if err != pgx.ErrNoRows {
errMsg := fmt.Sprintf("[WATCH-DELETE] ERROR: failed to look up media item for %s: %v", event.Name, err)
s.logger.LogDelete(errMsg)
s.logger.LogError(errMsg)
} else {
s.logger.LogDelete(fmt.Sprintf("[WATCH-DELETE] No media item found in database for deleted file: %s", event.Name))
}
}
case err, ok := <-s.watcher.Errors:
if !ok {
return
@@ -1589,6 +1554,188 @@ func (s *MediaScanner) WatchChanges(ctx context.Context) {
}
}()
}
func (s *MediaScanner) processEventQueue(ctx context.Context) {
for {
select {
case <-ctx.Done():
return
case path := <-s.eventQueue:
// Reset debounce timer - wait for more events
if s.debounceTimer != nil {
s.debounceTimer.Stop()
}
s.debounceTimer = time.AfterFunc(3*time.Second, func() {
s.flushEventQueue(ctx, path)
})
}
}
}
func (s *MediaScanner) flushEventQueue(ctx context.Context, initialPath string) {
// Collect all pending events from queue
paths := make(map[string]bool)
paths[initialPath] = true
// Drain remaining events (with short timeout to batch them)
timeout := time.After(500 * time.Millisecond)
DrainLoop:
for {
select {
case path := <-s.eventQueue:
paths[path] = true
case <-timeout:
break DrainLoop
}
}
fmt.Printf("Processing %d file events after debounce\n", len(paths))
// Process each unique path
for path := range paths {
// Determine if file exists or was deleted
_, err := os.Stat(path)
if os.IsNotExist(err) {
// File was deleted
s.handleFileDelete(ctx, path)
} else if err == nil {
// File exists (new or modified)
s.handleFileAdd(ctx, path)
}
}
}
func (s *MediaScanner) handleFileAdd(ctx context.Context, filePath string) {
fmt.Printf("New/modified media file detected: %s\n", filePath)
if _, err := s.processMediaFile(ctx, filePath); err != nil {
fmt.Printf("Error processing modified media file %s: %v\n", filePath, err)
}
}
func (s *MediaScanner) handleFileDelete(ctx context.Context, filePath string) {
s.logger.LogDelete(fmt.Sprintf("[WATCH-DELETE] File removed from filesystem: %s", filePath))
// Determine libraryID for this file
var libraryID pgtype.UUID
for _, folder := range s.folders {
if strings.HasPrefix(filePath, folder) {
lib, err := s.db.GetLibraryByFolder(ctx, folder)
if err == nil {
libraryID = lib.LibraryID
break
}
}
}
if !libraryID.Valid {
msg := fmt.Sprintf("[WATCH-DELETE] WARNING: could not determine library for deleted file: %s", filePath)
s.logger.LogDelete(msg)
s.logger.LogError(msg)
return
}
// Look up media item
existingItem, err := s.db.GetMediaItemByFilePath(ctx, database.GetMediaItemByFilePathParams{
FilePath: s.getRelativePath(filePath),
LibraryID: libraryID,
})
if err == nil {
msg := fmt.Sprintf("[WATCH-DELETE] Found media item to delete: ID=%s, Title=%s, Path=%s",
existingItem.ID, existingItem.Title, existingItem.FilePath)
s.logger.LogDelete(msg)
if err := s.db.DeleteMediaItem(ctx, existingItem.ID); err != nil {
errMsg := fmt.Sprintf("[WATCH-DELETE] ERROR: failed to delete media item %s: %v", existingItem.ID, err)
s.logger.LogDelete(errMsg)
s.logger.LogError(errMsg)
} else {
s.logger.LogDelete(fmt.Sprintf("[WATCH-DELETE] SUCCESS: deleted media item '%s' (was at %s)",
existingItem.Title, existingItem.FilePath))
}
} else if err != pgx.ErrNoRows {
errMsg := fmt.Sprintf("[WATCH-DELETE] ERROR: failed to look up media item for %s: %v", filePath, err)
s.logger.LogDelete(errMsg)
s.logger.LogError(errMsg)
} else {
s.logger.LogDelete(fmt.Sprintf("[WATCH-DELETE] No media item found in database for deleted file: %s", filePath))
}
}
func (s *MediaScanner) StartPolling(ctx context.Context) {
if s.pollInterval <= 0 {
fmt.Println("Polling fallback disabled (interval = 0")
return
}
ticker := time.NewTicker(s.pollInterval)
defer ticker.Stop()
fmt.Printf("Polling fallback started with interval: %v\n", s.pollInterval)
for {
select {
case <-ctx.Done():
fmt.Println("Polling fallback stopped")
return
case <-ticker.C:
fmt.Println("Running polling fallback sync...")
if err := s.SyncFilesystemWithDatabase(ctx); err != nil {
fmt.Printf("Polling sync error: %v\n", err)
}
}
}
}
func (s *MediaScanner) SyncFilesystemWithDatabase(ctx context.Context) error {
fmt.Println("[POLL-SYNC] Starting filesystem sync with database")
for _, folder := range s.folders {
lib, err := s.db.GetLibraryByFolder(ctx, folder)
if err != nil {
fmt.Printf("[POLL-SYNC] Warning: failed to get library for folder %s: %v\n", folder, err)
continue
}
libraryID := lib.LibraryID
// Get all media items from database for this library
dbItems, err := s.db.ListMediaItemsByLibrary(ctx, libraryID)
if err != nil {
fmt.Printf("[POLL-SYNC] Warning: failed to get library items: %v\n", err)
continue
}
// Build set of existing file paths from filesystem
existingPaths := make(map[string]bool)
filepath.WalkDir(folder, func(path string, d os.DirEntry, walkErr error) error {
if walkErr != nil {
return walkErr
}
if !d.IsDir() && s.isScannableFile(path) {
existingPaths[s.getRelativePath(path)] = true
}
return nil
})
// Check for orphaned items (in DB but not on filesystem)
for _, item := range dbItems {
if item.FilePath != "" && !existingPaths[item.FilePath] {
msg := fmt.Sprintf("[POLL-SYNC] Orphaned media item found: ID=%s, Title=%s, Path=%s",
item.ID, item.Title, item.FilePath)
s.logger.LogDelete(msg)
if err := s.db.DeleteMediaItem(ctx, item.ID); err != nil {
errMsg := fmt.Sprintf("[POLL-SYNC] ERROR: failed to delete orphaned item %s: %v", item.Title, err)
s.logger.LogDelete(errMsg)
s.logger.LogError(errMsg)
} else {
s.logger.LogDelete(fmt.Sprintf("[POLL-SYNC] SUCCESS: deleted orphaned item '%s'", item.Title))
}
}
}
// Check for new files (on filesystem but not in DB)
// This is expensive, so we just check a few representative files
// The fsnotify handler should catch most new files
for relPath := range existingPaths {
// Check if this file exists in DB
_, err := s.db.GetMediaItemByFilePath(ctx, database.GetMediaItemByFilePathParams{
FilePath: relPath,
LibraryID: libraryID,
})
if err == pgx.ErrNoRows {
// New file found - scan it
absPath := folder + "/" + relPath
if _, err := os.Stat(absPath); err == nil {
fmt.Printf("[POLL-SYNC] New file detected, scanning: %s\n", absPath)
if _, err := s.processMediaFile(ctx, absPath); err != nil {
fmt.Printf("[POLL-SYNC] Error scanning new file %s: %v\n", absPath, err)
}
}
}
}
}
fmt.Println("[POLL-SYNC] Filesystem sync completed")
return nil
}
func (s *MediaScanner) Close() error {
if s.watcher != nil {
internal/services/worker.go
+1 -1
View File
@@ -199,7 +199,7 @@ func (w *Worker) processScanJob(job *Job) (interface{}, error) {
force = forceVal
}
scanner := NewMediaScanner(db)
scanner := NewMediaScanner(db, 0)
scanner.job = job
job.ProgressCallback = func(progress float64, filesScanned, newItems, errors int) {