feat(scanner): add debounced file watching with polling fallback
- Implement event queue with 3-second debouncing for file system events - Add configurable polling fallback (default 3 min) via SCAN_POLL_INTERVAL_MINUTES - Add SyncFilesystemWithDatabase to detect orphaned DB entries and new files - Integrate utils.ResolveMediaURL for consistent media file path resolution - Add COOKIE_SECURE env var with SameSite=LaxMode for session cookies - Update media handler to properly decode URL paths for file serving - Refactor scanner initialization to accept poll interval configuration
This commit is contained in:
@@ -0,0 +1,207 @@
|
|||||||
|
# Autoscanner Improvements - Implementation Plan
|
||||||
|
|
||||||
|
## Problem Statement
|
||||||
|
|
||||||
|
The current autoscanner has two issues:
|
||||||
|
|
||||||
|
1. **Bulk file detection unreliable**: When files are copied in bulk (e.g., 6 files at once), fsnotify only detects 1 or 0 files
|
||||||
|
2. **Delete detection broken**: When files are deleted from the filesystem, they remain in the database
|
||||||
|
|
||||||
|
### Root Causes
|
||||||
|
|
||||||
|
1. **fsnotify limitations**: The file system watcher can miss events during bulk file operations
|
||||||
|
2. **Relative path mismatch**: Delete detection uses absolute paths for database lookups, but database stores relative paths (partially fixed)
|
||||||
|
|
||||||
|
## Solution Architecture
|
||||||
|
|
||||||
|
```
|
||||||
|
[fsnotify Events] → [Event Queue] → [Debounce Timer (3s)] → [Process Queue]
|
||||||
|
↓
|
||||||
|
[Polling Fallback (3 min)] → [Full Sync Check] → [Add missing / Remove deleted]
|
||||||
|
```
|
||||||
|
|
||||||
|
Two complementary systems working together:
|
||||||
|
- **fsnotify + debounce**: Handles most file changes in real-time
|
||||||
|
- **Polling fallback**: Safety net that catches anything fsnotify misses
|
||||||
|
|
||||||
|
## Implementation Plan
|
||||||
|
|
||||||
|
### Phase 1: Add Debounce to fsnotify Handler
|
||||||
|
|
||||||
|
**File**: `internal/services/media_scanner.go`
|
||||||
|
|
||||||
|
**Changes**:
|
||||||
|
1. Create an event queue to accumulate fsnotify events
|
||||||
|
2. Add debounce timer (3 seconds) that resets on each new event
|
||||||
|
3. When timer fires, process all queued events in batch
|
||||||
|
4. Process each event: new files → scan, deleted files → remove from DB
|
||||||
|
|
||||||
|
**Implementation Details**:
|
||||||
|
- Use a buffered channel as the event queue
|
||||||
|
- Use `time.After()` or `time.Timer` for debounce
|
||||||
|
- Process events in order, skip duplicates for same file
|
||||||
|
|
||||||
|
```go
|
||||||
|
// Pseudo-code structure
|
||||||
|
type FileEvent struct {
|
||||||
|
path string
|
||||||
|
isDelete bool
|
||||||
|
}
|
||||||
|
|
||||||
|
eventQueue := make(chan FileEvent, 100)
|
||||||
|
var debounceTimer *time.Timer
|
||||||
|
|
||||||
|
func handleFsEvent(event fsnotify.Event) {
|
||||||
|
select {
|
||||||
|
case eventQueue <- FileEvent{path: event.Name, isDelete: event.Has(fsnotify.Remove)}:
|
||||||
|
default:
|
||||||
|
// Queue full, log warning
|
||||||
|
}
|
||||||
|
|
||||||
|
// Reset debounce timer
|
||||||
|
if debounceTimer != nil {
|
||||||
|
debounceTimer.Stop()
|
||||||
|
}
|
||||||
|
debounceTimer = time.AfterFunc(3*time.Second, processEventQueue)
|
||||||
|
}
|
||||||
|
|
||||||
|
func processEventQueue() {
|
||||||
|
// Drain queue and process unique paths
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
### Phase 2: Add Polling Fallback
|
||||||
|
|
||||||
|
**File**: `internal/services/media_scanner.go` (or new file)
|
||||||
|
|
||||||
|
**Changes**:
|
||||||
|
1. Add polling interval configuration (default: 3 minutes)
|
||||||
|
2. Create sync function that:
|
||||||
|
- Walks all library folders
|
||||||
|
- Compares filesystem against database
|
||||||
|
- Adds missing files (triggers scan for new files)
|
||||||
|
- Removes orphaned database entries (files no longer exist)
|
||||||
|
3. Start polling goroutine alongside existing fsnotify watcher
|
||||||
|
|
||||||
|
**Configuration**:
|
||||||
|
- Environment variable: `SCAN_POLL_INTERVAL_MINUTES` (default: 3)
|
||||||
|
- Use existing config system or add to `internal/config/config.go`
|
||||||
|
|
||||||
|
**Implementation Details**:
|
||||||
|
```go
|
||||||
|
type ScannerSyncOptions struct {
|
||||||
|
PollInterval time.Duration // default: 3 minutes
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *MediaScanner) StartPolling(ctx context.Context, opts ScannerSyncOptions) {
|
||||||
|
ticker := time.NewTicker(opts.PollInterval)
|
||||||
|
defer ticker.Stop()
|
||||||
|
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return
|
||||||
|
case <-ticker.C:
|
||||||
|
s.SyncFilesystemWithDatabase(ctx)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *MediaScanner) SyncFilesystemWithDatabase(ctx context.Context) error {
|
||||||
|
// 1. Get all media items from database
|
||||||
|
// 2. For each library folder:
|
||||||
|
// - Walk filesystem, build map of existing files (relative paths)
|
||||||
|
// - Compare against database
|
||||||
|
// - Add: file in filesystem but not in DB → scan
|
||||||
|
// - Remove: file in DB but not on filesystem → delete
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
### Phase 3: Reuse Existing Cleanup Logic
|
||||||
|
|
||||||
|
**File**: `internal/services/media_scanner.go`
|
||||||
|
|
||||||
|
The cleanup logic already exists (lines 252-299 in `ScanFolders`) - it removes orphaned items after each manual scan. We can refactor this into a reusable function called by both:
|
||||||
|
- Manual scan (existing behavior)
|
||||||
|
- Polling fallback (new behavior)
|
||||||
|
|
||||||
|
```go
|
||||||
|
func (s *MediaScanner) CleanupOrphanedItems(ctx context.Context) error {
|
||||||
|
// Existing cleanup code from ScanFolders
|
||||||
|
// Extract to reusable function
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
### Phase 4: Configuration
|
||||||
|
|
||||||
|
**File**: `internal/config/config.go`
|
||||||
|
|
||||||
|
Add new configuration option:
|
||||||
|
|
||||||
|
```go
|
||||||
|
type Config struct {
|
||||||
|
// ... existing fields ...
|
||||||
|
ScanPollIntervalMinutes int `env:"SCAN_POLL_INTERVAL_MINUTES"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func LoadConfig() *Config {
|
||||||
|
return &Config{
|
||||||
|
// ... existing fields ...
|
||||||
|
ScanPollIntervalMinutes: getEnvInt("SCAN_POLL_INTERVAL_MINUTES", 3), // 3 minutes default
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
### Phase 5: Integration with Scheduler
|
||||||
|
|
||||||
|
**File**: `internal/services/scheduler.go`
|
||||||
|
|
||||||
|
The scheduler already handles periodic tasks. Consider:
|
||||||
|
1. Adding polling fallback to scheduler, OR
|
||||||
|
2. Starting polling directly in MediaScanner initialization
|
||||||
|
|
||||||
|
## Files to Modify
|
||||||
|
|
||||||
|
| File | Changes |
|
||||||
|
|------|---------|
|
||||||
|
| `internal/services/media_scanner.go` | Add debounce queue, polling fallback, reuse cleanup |
|
||||||
|
| `internal/config/config.go` | Add `SCAN_POLL_INTERVAL_MINUTES` config |
|
||||||
|
| `docker-compose.yml` | Add environment variable (optional) |
|
||||||
|
|
||||||
|
## Testing Plan
|
||||||
|
|
||||||
|
1. **Bulk file addition**: Copy 10+ files at once, verify all detected within 3 seconds (fsnotify) or 3 minutes (polling)
|
||||||
|
2. **Bulk file deletion**: Delete 5+ files, verify all removed from database within 3 minutes
|
||||||
|
3. **Mixed operations**: Add some, delete some, verify correct state
|
||||||
|
4. **Large library**: Test with 100+ files to ensure performance is acceptable
|
||||||
|
|
||||||
|
## Backward Compatibility
|
||||||
|
|
||||||
|
- Default polling interval: 3 minutes (user can configure)
|
||||||
|
- Existing manual scan functionality unchanged
|
||||||
|
- fsnotify continues to work as before (with debounce improvement)
|
||||||
|
|
||||||
|
## Security Considerations
|
||||||
|
|
||||||
|
- Polling runs on same goroutine as scanner (no new attack surface)
|
||||||
|
- File operations are read-only until changes detected
|
||||||
|
- Database operations use existing service layer (already authorized)
|
||||||
|
|
||||||
|
## Timeline Estimate
|
||||||
|
|
||||||
|
| Phase | Complexity | Estimate |
|
||||||
|
|-------|-------------|----------|
|
||||||
|
| Phase 1: Debounce | Medium | 1-2 hours |
|
||||||
|
| Phase 2: Polling | Medium | 1-2 hours |
|
||||||
|
| Phase 3: Reuse cleanup | Low | 30 min |
|
||||||
|
| Phase 4: Config | Low | 15 min |
|
||||||
|
| Phase 5: Integration | Low | 15 min |
|
||||||
|
| Testing | Medium | 1 hour |
|
||||||
|
| **Total** | - | **4-6 hours** |
|
||||||
|
|
||||||
|
## Future Improvements (Out of Scope)
|
||||||
|
|
||||||
|
1. **Configurable debounce duration**
|
||||||
|
2. **Per-library polling intervals**
|
||||||
|
3. **Event history/logging for debugging**
|
||||||
|
4. **Manual trigger for full sync**
|
||||||
@@ -9,9 +9,12 @@ services:
|
|||||||
POSTGRES_DB: bookhoard
|
POSTGRES_DB: bookhoard
|
||||||
POSTGRES_USER: postgres
|
POSTGRES_USER: postgres
|
||||||
POSTGRES_PASSWORD: ${DBPASS}
|
POSTGRES_PASSWORD: ${DBPASS}
|
||||||
|
COOKIE_SECURE: false # make true in production with HTTPS
|
||||||
|
SCAN_POLL_INTERVAL_MINUTES: 3
|
||||||
volumes:
|
volumes:
|
||||||
- postgres_data:/var/lib/postgresql/data
|
- postgres_data:/var/lib/postgresql/data
|
||||||
- ./database/schema:/docker-entrypoint-initdb.d
|
- ./database/schema:/docker-entrypoint-initdb.d
|
||||||
|
# Make other volumes as needed
|
||||||
- ./uploads:/app/uploads
|
- ./uploads:/app/uploads
|
||||||
ports:
|
ports:
|
||||||
- "5432:5432"
|
- "5432:5432"
|
||||||
@@ -84,6 +87,7 @@ services:
|
|||||||
DATABASE_USER: postgres
|
DATABASE_USER: postgres
|
||||||
DATABASE_PASSWORD: ${DBPASS}
|
DATABASE_PASSWORD: ${DBPASS}
|
||||||
DATABASE_NAME: bookhoard
|
DATABASE_NAME: bookhoard
|
||||||
|
COOKIE_SECURE: false
|
||||||
|
|
||||||
# Application Configuration
|
# Application Configuration
|
||||||
JWT_SECRET: ${JWT_SECRET}
|
JWT_SECRET: ${JWT_SECRET}
|
||||||
|
|||||||
+26
-24
@@ -7,35 +7,37 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
type Config struct {
|
type Config struct {
|
||||||
ServerPort string
|
ServerPort string
|
||||||
BaseURL string
|
BaseURL string
|
||||||
JWTSecret string
|
JWTSecret string
|
||||||
UploadPath string
|
UploadPath string
|
||||||
DatabaseHost string
|
DatabaseHost string
|
||||||
DatabasePort string
|
DatabasePort string
|
||||||
DatabaseUser string
|
DatabaseUser string
|
||||||
DatabasePassword string
|
DatabasePassword string
|
||||||
DatabaseName string
|
DatabaseName string
|
||||||
TestMode bool
|
TestMode bool
|
||||||
RateLimitEnabled bool
|
RateLimitEnabled bool
|
||||||
RequestsPerMinute int
|
RequestsPerMinute int
|
||||||
|
ScanPollIntervalMinutes int `env:"SCAN_POLL_INTERVAL_MINUTES" default:"3"`
|
||||||
}
|
}
|
||||||
|
|
||||||
func LoadConfig() *Config {
|
func LoadConfig() *Config {
|
||||||
port := getEnv("SERVER_PORT", "8765")
|
port := getEnv("SERVER_PORT", "8765")
|
||||||
return &Config{
|
return &Config{
|
||||||
ServerPort: port,
|
ServerPort: port,
|
||||||
BaseURL: getEnv("BASE_URL", "http://localhost:"+port),
|
BaseURL: getEnv("BASE_URL", "http://localhost:"+port),
|
||||||
DatabaseHost: getEnv("DATABASE_HOST", "localhost"),
|
DatabaseHost: getEnv("DATABASE_HOST", "localhost"),
|
||||||
DatabasePort: getEnv("DATABASE_PORT", "5432"),
|
DatabasePort: getEnv("DATABASE_PORT", "5432"),
|
||||||
DatabaseUser: getEnv("DATABASE_USER", "postgres"),
|
DatabaseUser: getEnv("DATABASE_USER", "postgres"),
|
||||||
DatabasePassword: getEnv("DATABASE_PASSWORD", "password"),
|
DatabasePassword: getEnv("DATABASE_PASSWORD", "password"),
|
||||||
DatabaseName: getEnv("DATABASE_NAME", "bookhoard"),
|
DatabaseName: getEnv("DATABASE_NAME", "bookhoard"),
|
||||||
JWTSecret: getEnv("JWT_SECRET", "your-secret-key"),
|
JWTSecret: getEnv("JWT_SECRET", "your-secret-key"),
|
||||||
UploadPath: getEnv("UPLOAD_PATH", "./uploads"),
|
UploadPath: getEnv("UPLOAD_PATH", "./uploads"),
|
||||||
TestMode: getEnvBool("TEST_MODE", false),
|
TestMode: getEnvBool("TEST_MODE", false),
|
||||||
RateLimitEnabled: getEnvBool("RATE_LIMIT_ENABLED", true),
|
RateLimitEnabled: getEnvBool("RATE_LIMIT_ENABLED", true),
|
||||||
RequestsPerMinute: getEnvInt("REQUESTS_PER_MINUTE", 10),
|
RequestsPerMinute: getEnvInt("REQUESTS_PER_MINUTE", 10),
|
||||||
|
ScanPollIntervalMinutes: getEnvInt("SCAN_POLL_INTERVAL_MINUTES", 3),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -6,6 +6,7 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"fmt"
|
"fmt"
|
||||||
"net/http"
|
"net/http"
|
||||||
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
@@ -29,6 +30,8 @@ const (
|
|||||||
// Note: This is computed from SessionDuration to avoid magic numbers
|
// Note: This is computed from SessionDuration to avoid magic numbers
|
||||||
var SessionDurationSec = int(SessionDuration.Seconds())
|
var SessionDurationSec = int(SessionDuration.Seconds())
|
||||||
|
|
||||||
|
var secure = os.Getenv("COOKIE_SECURE")
|
||||||
|
|
||||||
type AuthHandler struct {
|
type AuthHandler struct {
|
||||||
db *database.Queries
|
db *database.Queries
|
||||||
jwtKey []byte
|
jwtKey []byte
|
||||||
@@ -249,7 +252,8 @@ func (h *AuthHandler) Register(c echo.Context) error {
|
|||||||
Value: accessToken,
|
Value: accessToken,
|
||||||
Path: "/",
|
Path: "/",
|
||||||
HttpOnly: true,
|
HttpOnly: true,
|
||||||
Secure: false, // TODO: Set to true in production with HTTPS
|
Secure: secure == "true", // TODO: Set to true in production with HTTPS
|
||||||
|
SameSite: http.SameSiteLaxMode,
|
||||||
MaxAge: SessionDurationSec,
|
MaxAge: SessionDurationSec,
|
||||||
}
|
}
|
||||||
c.SetCookie(cookie)
|
c.SetCookie(cookie)
|
||||||
@@ -394,7 +398,8 @@ func (h *AuthHandler) Login(c echo.Context) error {
|
|||||||
Value: accessToken,
|
Value: accessToken,
|
||||||
Path: "/",
|
Path: "/",
|
||||||
HttpOnly: true,
|
HttpOnly: true,
|
||||||
Secure: false, // TODO: Set to true in production with HTTPS
|
Secure: secure == "true", // TODO: Set to true in production with HTTPS
|
||||||
|
SameSite: http.SameSiteLaxMode,
|
||||||
MaxAge: SessionDurationSec,
|
MaxAge: SessionDurationSec,
|
||||||
}
|
}
|
||||||
c.SetCookie(cookie)
|
c.SetCookie(cookie)
|
||||||
|
|||||||
@@ -928,10 +928,10 @@ func mediaItemsToListMediaItemsRow(item database.MediaItems) database.ListMediaI
|
|||||||
Author: item.Author,
|
Author: item.Author,
|
||||||
Isbn: item.Isbn,
|
Isbn: item.Isbn,
|
||||||
Description: item.Description,
|
Description: item.Description,
|
||||||
FilePath: item.FilePath,
|
FilePath: utils.ResolveMediaURL(item.LibraryID, pgtype.Text{String: item.FilePath, Valid: true}),
|
||||||
FileSize: item.FileSize,
|
FileSize: item.FileSize,
|
||||||
MimeType: item.MimeType,
|
MimeType: item.MimeType,
|
||||||
CoverImagePath: item.CoverImagePath,
|
CoverImagePath: pgtype.Text{String: utils.ResolveMediaURL(item.LibraryID, item.CoverImagePath), Valid: true},
|
||||||
Series: item.Series,
|
Series: item.Series,
|
||||||
SeriesNumber: item.SeriesNumber,
|
SeriesNumber: item.SeriesNumber,
|
||||||
Tags: item.Tags,
|
Tags: item.Tags,
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
package handlers
|
package handlers
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"bookhoard/internal/config"
|
||||||
"bookhoard/internal/database"
|
"bookhoard/internal/database"
|
||||||
"bookhoard/internal/services"
|
"bookhoard/internal/services"
|
||||||
wsync "bookhoard/internal/sync"
|
wsync "bookhoard/internal/sync"
|
||||||
@@ -27,9 +28,10 @@ type Handler struct {
|
|||||||
watchingLibraries map[string]bool
|
watchingLibraries map[string]bool
|
||||||
connManager *wsync.ConnectionManager
|
connManager *wsync.ConnectionManager
|
||||||
cleanupTaskCancel context.CancelFunc
|
cleanupTaskCancel context.CancelFunc
|
||||||
|
config *config.Config
|
||||||
}
|
}
|
||||||
|
|
||||||
func NewHandler(db *database.Queries, connManager *wsync.ConnectionManager, queueProcessor *wsync.SyncQueueProcessor) *Handler {
|
func NewHandler(db *database.Queries, connManager *wsync.ConnectionManager, queueProcessor *wsync.SyncQueueProcessor, cfg *config.Config) *Handler {
|
||||||
ctx, cancel := context.WithCancel(context.Background())
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
worker := services.NewWorker(3)
|
worker := services.NewWorker(3)
|
||||||
scheduler := services.NewScheduler(worker, db)
|
scheduler := services.NewScheduler(worker, db)
|
||||||
@@ -40,7 +42,7 @@ func NewHandler(db *database.Queries, connManager *wsync.ConnectionManager, queu
|
|||||||
|
|
||||||
return &Handler{
|
return &Handler{
|
||||||
db: db,
|
db: db,
|
||||||
scanner: services.NewMediaScanner(db),
|
scanner: services.NewMediaScanner(db, cfg.ScanPollIntervalMinutes),
|
||||||
worker: worker,
|
worker: worker,
|
||||||
scheduler: scheduler,
|
scheduler: scheduler,
|
||||||
queueProcessor: queueProcessor,
|
queueProcessor: queueProcessor,
|
||||||
@@ -66,6 +68,6 @@ func parseDate(dateStr string) time.Time {
|
|||||||
return time.Time{}
|
return time.Time{}
|
||||||
}
|
}
|
||||||
|
|
||||||
func SetupRoutes(g *echo.Group, db *database.Queries, connManager *wsync.ConnectionManager, queueProcessor *wsync.SyncQueueProcessor) *Handler {
|
func SetupRoutes(g *echo.Group, db *database.Queries, connManager *wsync.ConnectionManager, queueProcessor *wsync.SyncQueueProcessor, cfg *config.Config) *Handler {
|
||||||
return NewHandler(db, connManager, queueProcessor)
|
return NewHandler(db, connManager, queueProcessor, cfg)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -3,6 +3,7 @@ package handlers
|
|||||||
import (
|
import (
|
||||||
"bookhoard/internal/database"
|
"bookhoard/internal/database"
|
||||||
"bookhoard/internal/services"
|
"bookhoard/internal/services"
|
||||||
|
"bookhoard/internal/utils"
|
||||||
"net/http"
|
"net/http"
|
||||||
"strconv"
|
"strconv"
|
||||||
|
|
||||||
@@ -151,7 +152,7 @@ func BuildSections(sections []services.DashboardSection) []SectionData {
|
|||||||
MediaItemID: itemUUID.String(),
|
MediaItemID: itemUUID.String(),
|
||||||
Title: item.Title,
|
Title: item.Title,
|
||||||
Author: textToString(item.Author),
|
Author: textToString(item.Author),
|
||||||
CoverImagePath: textToString(item.CoverImagePath),
|
CoverImagePath: utils.ResolveMediaURL(item.LibraryID, item.CoverImagePath),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
+13
-10
@@ -9,6 +9,7 @@ import (
|
|||||||
"io"
|
"io"
|
||||||
"mime"
|
"mime"
|
||||||
"net/http"
|
"net/http"
|
||||||
|
"net/url"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"strconv"
|
"strconv"
|
||||||
@@ -1490,21 +1491,23 @@ func (mh *MediaHandler) getFullFilePath(ctx context.Context, libraryID pgtype.UU
|
|||||||
// Requires JWT authentication
|
// Requires JWT authentication
|
||||||
func (mh *MediaHandler) ServeFile(c echo.Context) error {
|
func (mh *MediaHandler) ServeFile(c echo.Context) error {
|
||||||
// URL format: /uploads/library-{libraryID}/{relativePath}
|
// URL format: /uploads/library-{libraryID}/{relativePath}
|
||||||
path := c.Param("*") // Gets everything after /uploads/library-{id}/
|
// Get library ID directly from route parameter
|
||||||
|
libraryIDStr := c.Param("id")
|
||||||
// Extract library ID from path
|
|
||||||
parts := strings.SplitN(path, "/", 2)
|
|
||||||
if len(parts) < 2 {
|
|
||||||
return c.JSON(http.StatusBadRequest, map[string]string{"error": "invalid path"})
|
|
||||||
}
|
|
||||||
|
|
||||||
libraryIDStr := strings.TrimPrefix(parts[0], "library-")
|
|
||||||
libraryUUID, err := uuid.Parse(libraryIDStr)
|
libraryUUID, err := uuid.Parse(libraryIDStr)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return c.JSON(http.StatusBadRequest, map[string]string{"error": "invalid library ID"})
|
return c.JSON(http.StatusBadRequest, map[string]string{"error": "invalid library ID"})
|
||||||
}
|
}
|
||||||
|
|
||||||
relativePath := parts[1]
|
// Get remaining path from URL
|
||||||
|
rawPath := c.Param("*")
|
||||||
|
relativePath, err := url.QueryUnescape(rawPath)
|
||||||
|
if err != nil {
|
||||||
|
relativePath = rawPath
|
||||||
|
}
|
||||||
|
|
||||||
|
if relativePath == "" {
|
||||||
|
return c.JSON(http.StatusBadRequest, map[string]string{"error": "invalid path"})
|
||||||
|
}
|
||||||
|
|
||||||
// Resolve using service
|
// Resolve using service
|
||||||
fullPath, err := mh.getFullFilePath(c.Request().Context(), pgtype.UUID{Bytes: libraryUUID, Valid: true}, relativePath)
|
fullPath, err := mh.getFullFilePath(c.Request().Context(), pgtype.UUID{Bytes: libraryUUID, Valid: true}, relativePath)
|
||||||
|
|||||||
@@ -197,7 +197,7 @@ func (h *Handler) StartWatchModeForLibrary(ctx context.Context, libraryID pgtype
|
|||||||
folderPaths[i] = folder.FolderPath
|
folderPaths[i] = folder.FolderPath
|
||||||
}
|
}
|
||||||
|
|
||||||
scanner := services.NewMediaScanner(h.db)
|
scanner := services.NewMediaScanner(h.db, 3)
|
||||||
if err := scanner.SetFolders(folderPaths); err != nil {
|
if err := scanner.SetFolders(folderPaths); err != nil {
|
||||||
return fmt.Errorf("failed to set scanner folders: %v", err)
|
return fmt.Errorf("failed to set scanner folders: %v", err)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -200,7 +200,7 @@ func RegisterRoutes(cfg *Config) *handlers.Handler {
|
|||||||
protected := e.Group("/api", jwtMiddleware)
|
protected := e.Group("/api", jwtMiddleware)
|
||||||
|
|
||||||
// Create scanner handler for scanner routes and progress routes
|
// Create scanner handler for scanner routes and progress routes
|
||||||
scannerHandler := handlers.SetupRoutes(protected, cfg.Queries, cfg.ConnManager, cfg.QueueProcessor)
|
scannerHandler := handlers.SetupRoutes(protected, cfg.Queries, cfg.ConnManager, cfg.QueueProcessor, cfg.Cfg)
|
||||||
cfg.ScannerHandler = scannerHandler
|
cfg.ScannerHandler = scannerHandler
|
||||||
|
|
||||||
// Register route groups
|
// Register route groups
|
||||||
|
|||||||
@@ -13,7 +13,7 @@ func registerSyncRoutes(cfg *Config) {
|
|||||||
protected := e.Group("/api", jwtMiddleware)
|
protected := e.Group("/api", jwtMiddleware)
|
||||||
|
|
||||||
// Create handler for sync-specific routes
|
// Create handler for sync-specific routes
|
||||||
h := handlers.NewHandler(cfg.Queries, cfg.ConnManager, cfg.QueueProcessor)
|
h := handlers.NewHandler(cfg.Queries, cfg.ConnManager, cfg.QueueProcessor, cfg.Cfg)
|
||||||
|
|
||||||
// Book matching and unlinked book resolution routes
|
// Book matching and unlinked book resolution routes
|
||||||
sync := protected.Group("/sync")
|
sync := protected.Group("/sync")
|
||||||
|
|||||||
@@ -2,6 +2,7 @@ package services
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"bookhoard/internal/database"
|
"bookhoard/internal/database"
|
||||||
|
"bookhoard/internal/utils"
|
||||||
"context"
|
"context"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"fmt"
|
"fmt"
|
||||||
@@ -18,10 +19,10 @@ func mediaItemsToListMediaItemsRow(item database.MediaItems) database.ListMediaI
|
|||||||
Author: item.Author,
|
Author: item.Author,
|
||||||
Isbn: item.Isbn,
|
Isbn: item.Isbn,
|
||||||
Description: item.Description,
|
Description: item.Description,
|
||||||
FilePath: item.FilePath,
|
FilePath: utils.ResolveMediaURL(item.LibraryID, pgtype.Text{String: item.FilePath, Valid: true}),
|
||||||
FileSize: item.FileSize,
|
FileSize: item.FileSize,
|
||||||
MimeType: item.MimeType,
|
MimeType: item.MimeType,
|
||||||
CoverImagePath: item.CoverImagePath,
|
CoverImagePath: pgtype.Text{String: utils.ResolveMediaURL(item.LibraryID, item.CoverImagePath), Valid: true},
|
||||||
Series: item.Series,
|
Series: item.Series,
|
||||||
SeriesNumber: item.SeriesNumber,
|
SeriesNumber: item.SeriesNumber,
|
||||||
Tags: item.Tags,
|
Tags: item.Tags,
|
||||||
@@ -69,10 +70,10 @@ func getCollectionItemsRowToMediaItems(item database.GetCollectionItemsForDashbo
|
|||||||
Author: item.Author,
|
Author: item.Author,
|
||||||
Isbn: item.Isbn,
|
Isbn: item.Isbn,
|
||||||
Description: item.Description,
|
Description: item.Description,
|
||||||
FilePath: item.FilePath,
|
FilePath: utils.ResolveMediaURL(item.LibraryID, pgtype.Text{String: item.FilePath, Valid: true}),
|
||||||
FileSize: item.FileSize,
|
FileSize: item.FileSize,
|
||||||
MimeType: item.MimeType,
|
MimeType: item.MimeType,
|
||||||
CoverImagePath: item.CoverImagePath,
|
CoverImagePath: pgtype.Text{String: utils.ResolveMediaURL(item.LibraryID, item.CoverImagePath), Valid: true},
|
||||||
Series: item.Series,
|
Series: item.Series,
|
||||||
SeriesNumber: item.SeriesNumber,
|
SeriesNumber: item.SeriesNumber,
|
||||||
Tags: item.Tags,
|
Tags: item.Tags,
|
||||||
|
|||||||
@@ -79,6 +79,9 @@ type MediaScanner struct {
|
|||||||
libraryTypes map[string][]string
|
libraryTypes map[string][]string
|
||||||
forceRescan bool
|
forceRescan bool
|
||||||
logger *ScannerLogger
|
logger *ScannerLogger
|
||||||
|
eventQueue chan string
|
||||||
|
debounceTimer *time.Timer
|
||||||
|
pollInterval time.Duration
|
||||||
|
|
||||||
totalFiles int
|
totalFiles int
|
||||||
newItems int
|
newItems int
|
||||||
@@ -87,12 +90,17 @@ type MediaScanner struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// NewMediaScanner creates a new media scanner instance
|
// NewMediaScanner creates a new media scanner instance
|
||||||
func NewMediaScanner(db *database.Queries) *MediaScanner {
|
func NewMediaScanner(db *database.Queries, pollIntervalMinutes int) *MediaScanner {
|
||||||
watcher, err := fsnotify.NewWatcher()
|
watcher, err := fsnotify.NewWatcher()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
panic(fmt.Sprintf("Failed to create file watcher: %v", err))
|
panic(fmt.Sprintf("Failed to create file watcher: %v", err))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pollInterval := 3 * time.Minute
|
||||||
|
if pollInterval > 0 {
|
||||||
|
pollInterval = time.Duration(pollIntervalMinutes) * time.Minute
|
||||||
|
}
|
||||||
|
|
||||||
return &MediaScanner{
|
return &MediaScanner{
|
||||||
db: db,
|
db: db,
|
||||||
watcher: watcher,
|
watcher: watcher,
|
||||||
@@ -101,6 +109,8 @@ func NewMediaScanner(db *database.Queries) *MediaScanner {
|
|||||||
defaultLibraryID: pgtype.UUID{Valid: false},
|
defaultLibraryID: pgtype.UUID{Valid: false},
|
||||||
libraryTypes: make(map[string][]string),
|
libraryTypes: make(map[string][]string),
|
||||||
logger: NewScannerLogger(),
|
logger: NewScannerLogger(),
|
||||||
|
eventQueue: make(chan string, 500),
|
||||||
|
pollInterval: pollInterval,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -270,7 +280,7 @@ func (s *MediaScanner) ScanFolders(ctx context.Context) error {
|
|||||||
return walkErr
|
return walkErr
|
||||||
}
|
}
|
||||||
if !d.IsDir() && s.isScannableFile(path) {
|
if !d.IsDir() && s.isScannableFile(path) {
|
||||||
scannedPaths[path] = true
|
scannedPaths[s.getRelativePath(path)] = true
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
})
|
})
|
||||||
@@ -570,6 +580,7 @@ func (s *MediaScanner) processMediaFile(ctx context.Context, path string) (bool,
|
|||||||
tagsSearch := utils.NormalizeTagsSearch(metadata.Tags)
|
tagsSearch := utils.NormalizeTagsSearch(metadata.Tags)
|
||||||
|
|
||||||
// Create media item in database
|
// Create media item in database
|
||||||
|
relativePath := s.getRelativePath(path)
|
||||||
createdItem, err := s.db.CreateMediaItem(ctx, database.CreateMediaItemParams{
|
createdItem, err := s.db.CreateMediaItem(ctx, database.CreateMediaItemParams{
|
||||||
LibraryID: libraryID,
|
LibraryID: libraryID,
|
||||||
Title: metadata.Title,
|
Title: metadata.Title,
|
||||||
@@ -577,7 +588,7 @@ func (s *MediaScanner) processMediaFile(ctx context.Context, path string) (bool,
|
|||||||
Isbn: pgtype.Text{String: utils.NormalizeISBNSafe(metadata.ISBN), Valid: metadata.ISBN != ""},
|
Isbn: pgtype.Text{String: utils.NormalizeISBNSafe(metadata.ISBN), Valid: metadata.ISBN != ""},
|
||||||
Asin: pgtype.Text{String: metadata.ASIN, Valid: metadata.ASIN != ""},
|
Asin: pgtype.Text{String: metadata.ASIN, Valid: metadata.ASIN != ""},
|
||||||
Description: pgtype.Text{String: metadata.Description, Valid: metadata.Description != ""},
|
Description: pgtype.Text{String: metadata.Description, Valid: metadata.Description != ""},
|
||||||
FilePath: s.getRelativePath(path),
|
FilePath: relativePath,
|
||||||
FileSize: pgtype.Int8{Int64: info.Size(), Valid: true},
|
FileSize: pgtype.Int8{Int64: info.Size(), Valid: true},
|
||||||
MimeType: pgtype.Text{String: s.getMimeType(path), Valid: true},
|
MimeType: pgtype.Text{String: s.getMimeType(path), Valid: true},
|
||||||
CoverImagePath: pgtype.Text{String: metadata.CoverPath, Valid: metadata.CoverPath != ""},
|
CoverImagePath: pgtype.Text{String: metadata.CoverPath, Valid: metadata.CoverPath != ""},
|
||||||
@@ -1486,7 +1497,7 @@ func (s *MediaScanner) updateMediaItem(ctx context.Context, mediaItemID pgtype.U
|
|||||||
|
|
||||||
func (s *MediaScanner) getMediaItemByFilePath(ctx context.Context, filePath string, libraryID pgtype.UUID) (database.MediaItems, error) {
|
func (s *MediaScanner) getMediaItemByFilePath(ctx context.Context, filePath string, libraryID pgtype.UUID) (database.MediaItems, error) {
|
||||||
return s.db.GetMediaItemByFilePath(ctx, database.GetMediaItemByFilePathParams{
|
return s.db.GetMediaItemByFilePath(ctx, database.GetMediaItemByFilePathParams{
|
||||||
FilePath: filePath,
|
FilePath: s.getRelativePath(filePath),
|
||||||
LibraryID: libraryID,
|
LibraryID: libraryID,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
@@ -1500,6 +1511,11 @@ func (s *MediaScanner) getMimeType(path string) string {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (s *MediaScanner) WatchChanges(ctx context.Context) {
|
func (s *MediaScanner) WatchChanges(ctx context.Context) {
|
||||||
|
// Start the debounced event processor
|
||||||
|
go s.processEventQueue(ctx)
|
||||||
|
// Start polling fallback
|
||||||
|
go s.StartPolling(ctx)
|
||||||
|
// Handle fsnotify events - queue them for debouncing
|
||||||
go func() {
|
go func() {
|
||||||
for {
|
for {
|
||||||
select {
|
select {
|
||||||
@@ -1507,12 +1523,10 @@ func (s *MediaScanner) WatchChanges(ctx context.Context) {
|
|||||||
if !ok {
|
if !ok {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
// Handle new directories - add them to the watcher
|
// Handle new directories - add them to the watcher
|
||||||
if event.Has(fsnotify.Create) {
|
if event.Has(fsnotify.Create) {
|
||||||
info, err := os.Stat(event.Name)
|
info, err := os.Stat(event.Name)
|
||||||
if err == nil && info.IsDir() {
|
if err == nil && info.IsDir() {
|
||||||
// Add the new directory to the watcher
|
|
||||||
if err := s.watcher.Add(event.Name); err != nil {
|
if err := s.watcher.Add(event.Name); err != nil {
|
||||||
fmt.Printf("Warning: failed to watch new directory %s: %v\n", event.Name, err)
|
fmt.Printf("Warning: failed to watch new directory %s: %v\n", event.Name, err)
|
||||||
} else {
|
} else {
|
||||||
@@ -1520,64 +1534,15 @@ func (s *MediaScanner) WatchChanges(ctx context.Context) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
// Queue file events for debounced processing
|
||||||
// Handle file modifications and creations
|
if (event.Has(fsnotify.Create) || event.Has(fsnotify.Write) || event.Has(fsnotify.Remove)) && s.isScannableFile(event.Name) {
|
||||||
if (event.Has(fsnotify.Create) || event.Has(fsnotify.Write)) && s.isScannableFile(event.Name) {
|
select {
|
||||||
fmt.Printf("New/modified media file detected: %s\n", event.Name)
|
case s.eventQueue <- event.Name:
|
||||||
if _, err := s.processMediaFile(ctx, event.Name); err != nil {
|
// Event queued
|
||||||
fmt.Printf("Error processing modified media file %s: %v\n", event.Name, err)
|
default:
|
||||||
|
fmt.Printf("Warning: event queue full, dropping event for %s\n", event.Name)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Handle file deletions
|
|
||||||
if event.Has(fsnotify.Remove) && s.isScannableFile(event.Name) {
|
|
||||||
s.logger.LogDelete(fmt.Sprintf("[WATCH-DELETE] File removed from filesystem: %s", event.Name))
|
|
||||||
|
|
||||||
// Determine libraryID for this file
|
|
||||||
var libraryID pgtype.UUID
|
|
||||||
for _, folder := range s.folders {
|
|
||||||
if strings.HasPrefix(event.Name, folder) {
|
|
||||||
lib, err := s.db.GetLibraryByFolder(ctx, folder)
|
|
||||||
if err == nil {
|
|
||||||
libraryID = lib.LibraryID
|
|
||||||
break
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if !libraryID.Valid {
|
|
||||||
msg := fmt.Sprintf("[WATCH-DELETE] WARNING: could not determine library for deleted file: %s", event.Name)
|
|
||||||
s.logger.LogDelete(msg)
|
|
||||||
s.logger.LogError(msg)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
// Look up media item BEFORE deleting - log for safety
|
|
||||||
existingItem, err := s.db.GetMediaItemByFilePath(ctx, database.GetMediaItemByFilePathParams{
|
|
||||||
FilePath: event.Name,
|
|
||||||
LibraryID: libraryID,
|
|
||||||
})
|
|
||||||
if err == nil {
|
|
||||||
msg := fmt.Sprintf("[WATCH-DELETE] Found media item to delete: ID=%s, Title=%s, Path=%s",
|
|
||||||
existingItem.ID, existingItem.Title, existingItem.FilePath)
|
|
||||||
s.logger.LogDelete(msg)
|
|
||||||
|
|
||||||
if err := s.db.DeleteMediaItem(ctx, existingItem.ID); err != nil {
|
|
||||||
errMsg := fmt.Sprintf("[WATCH-DELETE] ERROR: failed to delete media item %s: %v", existingItem.ID, err)
|
|
||||||
s.logger.LogDelete(errMsg)
|
|
||||||
s.logger.LogError(errMsg)
|
|
||||||
} else {
|
|
||||||
s.logger.LogDelete(fmt.Sprintf("[WATCH-DELETE] SUCCESS: deleted media item '%s' (was at %s)",
|
|
||||||
existingItem.Title, existingItem.FilePath))
|
|
||||||
}
|
|
||||||
} else if err != pgx.ErrNoRows {
|
|
||||||
errMsg := fmt.Sprintf("[WATCH-DELETE] ERROR: failed to look up media item for %s: %v", event.Name, err)
|
|
||||||
s.logger.LogDelete(errMsg)
|
|
||||||
s.logger.LogError(errMsg)
|
|
||||||
} else {
|
|
||||||
s.logger.LogDelete(fmt.Sprintf("[WATCH-DELETE] No media item found in database for deleted file: %s", event.Name))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
case err, ok := <-s.watcher.Errors:
|
case err, ok := <-s.watcher.Errors:
|
||||||
if !ok {
|
if !ok {
|
||||||
return
|
return
|
||||||
@@ -1589,6 +1554,188 @@ func (s *MediaScanner) WatchChanges(ctx context.Context) {
|
|||||||
}
|
}
|
||||||
}()
|
}()
|
||||||
}
|
}
|
||||||
|
func (s *MediaScanner) processEventQueue(ctx context.Context) {
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return
|
||||||
|
case path := <-s.eventQueue:
|
||||||
|
// Reset debounce timer - wait for more events
|
||||||
|
if s.debounceTimer != nil {
|
||||||
|
s.debounceTimer.Stop()
|
||||||
|
}
|
||||||
|
s.debounceTimer = time.AfterFunc(3*time.Second, func() {
|
||||||
|
s.flushEventQueue(ctx, path)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
func (s *MediaScanner) flushEventQueue(ctx context.Context, initialPath string) {
|
||||||
|
// Collect all pending events from queue
|
||||||
|
paths := make(map[string]bool)
|
||||||
|
paths[initialPath] = true
|
||||||
|
// Drain remaining events (with short timeout to batch them)
|
||||||
|
timeout := time.After(500 * time.Millisecond)
|
||||||
|
DrainLoop:
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case path := <-s.eventQueue:
|
||||||
|
paths[path] = true
|
||||||
|
case <-timeout:
|
||||||
|
break DrainLoop
|
||||||
|
}
|
||||||
|
}
|
||||||
|
fmt.Printf("Processing %d file events after debounce\n", len(paths))
|
||||||
|
// Process each unique path
|
||||||
|
for path := range paths {
|
||||||
|
// Determine if file exists or was deleted
|
||||||
|
_, err := os.Stat(path)
|
||||||
|
if os.IsNotExist(err) {
|
||||||
|
// File was deleted
|
||||||
|
s.handleFileDelete(ctx, path)
|
||||||
|
} else if err == nil {
|
||||||
|
// File exists (new or modified)
|
||||||
|
s.handleFileAdd(ctx, path)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
func (s *MediaScanner) handleFileAdd(ctx context.Context, filePath string) {
|
||||||
|
fmt.Printf("New/modified media file detected: %s\n", filePath)
|
||||||
|
if _, err := s.processMediaFile(ctx, filePath); err != nil {
|
||||||
|
fmt.Printf("Error processing modified media file %s: %v\n", filePath, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
func (s *MediaScanner) handleFileDelete(ctx context.Context, filePath string) {
|
||||||
|
s.logger.LogDelete(fmt.Sprintf("[WATCH-DELETE] File removed from filesystem: %s", filePath))
|
||||||
|
// Determine libraryID for this file
|
||||||
|
var libraryID pgtype.UUID
|
||||||
|
for _, folder := range s.folders {
|
||||||
|
if strings.HasPrefix(filePath, folder) {
|
||||||
|
lib, err := s.db.GetLibraryByFolder(ctx, folder)
|
||||||
|
if err == nil {
|
||||||
|
libraryID = lib.LibraryID
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !libraryID.Valid {
|
||||||
|
msg := fmt.Sprintf("[WATCH-DELETE] WARNING: could not determine library for deleted file: %s", filePath)
|
||||||
|
s.logger.LogDelete(msg)
|
||||||
|
s.logger.LogError(msg)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
// Look up media item
|
||||||
|
existingItem, err := s.db.GetMediaItemByFilePath(ctx, database.GetMediaItemByFilePathParams{
|
||||||
|
FilePath: s.getRelativePath(filePath),
|
||||||
|
LibraryID: libraryID,
|
||||||
|
})
|
||||||
|
if err == nil {
|
||||||
|
msg := fmt.Sprintf("[WATCH-DELETE] Found media item to delete: ID=%s, Title=%s, Path=%s",
|
||||||
|
existingItem.ID, existingItem.Title, existingItem.FilePath)
|
||||||
|
s.logger.LogDelete(msg)
|
||||||
|
if err := s.db.DeleteMediaItem(ctx, existingItem.ID); err != nil {
|
||||||
|
errMsg := fmt.Sprintf("[WATCH-DELETE] ERROR: failed to delete media item %s: %v", existingItem.ID, err)
|
||||||
|
s.logger.LogDelete(errMsg)
|
||||||
|
s.logger.LogError(errMsg)
|
||||||
|
} else {
|
||||||
|
s.logger.LogDelete(fmt.Sprintf("[WATCH-DELETE] SUCCESS: deleted media item '%s' (was at %s)",
|
||||||
|
existingItem.Title, existingItem.FilePath))
|
||||||
|
}
|
||||||
|
} else if err != pgx.ErrNoRows {
|
||||||
|
errMsg := fmt.Sprintf("[WATCH-DELETE] ERROR: failed to look up media item for %s: %v", filePath, err)
|
||||||
|
s.logger.LogDelete(errMsg)
|
||||||
|
s.logger.LogError(errMsg)
|
||||||
|
} else {
|
||||||
|
s.logger.LogDelete(fmt.Sprintf("[WATCH-DELETE] No media item found in database for deleted file: %s", filePath))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *MediaScanner) StartPolling(ctx context.Context) {
|
||||||
|
if s.pollInterval <= 0 {
|
||||||
|
fmt.Println("Polling fallback disabled (interval = 0")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
ticker := time.NewTicker(s.pollInterval)
|
||||||
|
defer ticker.Stop()
|
||||||
|
fmt.Printf("Polling fallback started with interval: %v\n", s.pollInterval)
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
fmt.Println("Polling fallback stopped")
|
||||||
|
return
|
||||||
|
case <-ticker.C:
|
||||||
|
fmt.Println("Running polling fallback sync...")
|
||||||
|
if err := s.SyncFilesystemWithDatabase(ctx); err != nil {
|
||||||
|
fmt.Printf("Polling sync error: %v\n", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
func (s *MediaScanner) SyncFilesystemWithDatabase(ctx context.Context) error {
|
||||||
|
fmt.Println("[POLL-SYNC] Starting filesystem sync with database")
|
||||||
|
for _, folder := range s.folders {
|
||||||
|
lib, err := s.db.GetLibraryByFolder(ctx, folder)
|
||||||
|
if err != nil {
|
||||||
|
fmt.Printf("[POLL-SYNC] Warning: failed to get library for folder %s: %v\n", folder, err)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
libraryID := lib.LibraryID
|
||||||
|
// Get all media items from database for this library
|
||||||
|
dbItems, err := s.db.ListMediaItemsByLibrary(ctx, libraryID)
|
||||||
|
if err != nil {
|
||||||
|
fmt.Printf("[POLL-SYNC] Warning: failed to get library items: %v\n", err)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
// Build set of existing file paths from filesystem
|
||||||
|
existingPaths := make(map[string]bool)
|
||||||
|
filepath.WalkDir(folder, func(path string, d os.DirEntry, walkErr error) error {
|
||||||
|
if walkErr != nil {
|
||||||
|
return walkErr
|
||||||
|
}
|
||||||
|
if !d.IsDir() && s.isScannableFile(path) {
|
||||||
|
existingPaths[s.getRelativePath(path)] = true
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
// Check for orphaned items (in DB but not on filesystem)
|
||||||
|
for _, item := range dbItems {
|
||||||
|
if item.FilePath != "" && !existingPaths[item.FilePath] {
|
||||||
|
msg := fmt.Sprintf("[POLL-SYNC] Orphaned media item found: ID=%s, Title=%s, Path=%s",
|
||||||
|
item.ID, item.Title, item.FilePath)
|
||||||
|
s.logger.LogDelete(msg)
|
||||||
|
if err := s.db.DeleteMediaItem(ctx, item.ID); err != nil {
|
||||||
|
errMsg := fmt.Sprintf("[POLL-SYNC] ERROR: failed to delete orphaned item %s: %v", item.Title, err)
|
||||||
|
s.logger.LogDelete(errMsg)
|
||||||
|
s.logger.LogError(errMsg)
|
||||||
|
} else {
|
||||||
|
s.logger.LogDelete(fmt.Sprintf("[POLL-SYNC] SUCCESS: deleted orphaned item '%s'", item.Title))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Check for new files (on filesystem but not in DB)
|
||||||
|
// This is expensive, so we just check a few representative files
|
||||||
|
// The fsnotify handler should catch most new files
|
||||||
|
for relPath := range existingPaths {
|
||||||
|
// Check if this file exists in DB
|
||||||
|
_, err := s.db.GetMediaItemByFilePath(ctx, database.GetMediaItemByFilePathParams{
|
||||||
|
FilePath: relPath,
|
||||||
|
LibraryID: libraryID,
|
||||||
|
})
|
||||||
|
if err == pgx.ErrNoRows {
|
||||||
|
// New file found - scan it
|
||||||
|
absPath := folder + "/" + relPath
|
||||||
|
if _, err := os.Stat(absPath); err == nil {
|
||||||
|
fmt.Printf("[POLL-SYNC] New file detected, scanning: %s\n", absPath)
|
||||||
|
if _, err := s.processMediaFile(ctx, absPath); err != nil {
|
||||||
|
fmt.Printf("[POLL-SYNC] Error scanning new file %s: %v\n", absPath, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
fmt.Println("[POLL-SYNC] Filesystem sync completed")
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
func (s *MediaScanner) Close() error {
|
func (s *MediaScanner) Close() error {
|
||||||
if s.watcher != nil {
|
if s.watcher != nil {
|
||||||
|
|||||||
@@ -199,7 +199,7 @@ func (w *Worker) processScanJob(job *Job) (interface{}, error) {
|
|||||||
force = forceVal
|
force = forceVal
|
||||||
}
|
}
|
||||||
|
|
||||||
scanner := NewMediaScanner(db)
|
scanner := NewMediaScanner(db, 0)
|
||||||
scanner.job = job
|
scanner.job = job
|
||||||
|
|
||||||
job.ProgressCallback = func(progress float64, filesScanned, newItems, errors int) {
|
job.ProgressCallback = func(progress float64, filesScanned, newItems, errors int) {
|
||||||
|
|||||||
Reference in New Issue
Block a user