docs: remove obsolete implementation plan
Remove IMPLEMENTATION_PLAN.md as the implementation phase has been completed and the document is no longer needed for reference. The changes described in the plan have been successfully integrated into the codebase.
This commit is contained in:
@@ -1,904 +0,0 @@
|
||||
# Scanner Implementation Plan - Detailed Steps
|
||||
|
||||
This document provides precise, line-by-line steps to implement the scanner fixes. Follow in order.
|
||||
|
||||
---
|
||||
|
||||
## Fix 1: Use File Modification Time for `created_at` on INSERT
|
||||
|
||||
**Goal:** When inserting a new media item, use the file's actual modification time instead of database default (which uses scan time).
|
||||
|
||||
### Step 1.1: Modify `processMediaFile` to get file mtime
|
||||
|
||||
**File:** `internal/services/media_scanner.go`
|
||||
|
||||
**Current code (around line 348-360):**
|
||||
|
||||
```go
|
||||
func (s *MediaScanner) processMediaFile(ctx context.Context, path string) (bool, error) {
|
||||
// Get file info
|
||||
info, err := os.Stat(path)
|
||||
if err != nil {
|
||||
fmt.Printf("Failed to get file info for %s: %v\n", path, err)
|
||||
return false, fmt.Errorf("failed to get file info: %v", err)
|
||||
}
|
||||
|
||||
// Check if media item already exists in database
|
||||
existingItem, err := s.getMediaItemByFilePath(ctx, path)
|
||||
```
|
||||
|
||||
**Add AFTER line 357 (after getting file info, before existingItem check):**
|
||||
|
||||
```go
|
||||
// Get file modification time for created_at
|
||||
fileModTime := info.ModTime()
|
||||
```
|
||||
|
||||
### Step 1.2: Modify SQL INSERT to include created_at column
|
||||
|
||||
**File:** `internal/database/queries/queries.sql`
|
||||
|
||||
**Current code (line 131-133):**
|
||||
|
||||
```sql
|
||||
-- name: CreateMediaItem :one
|
||||
INSERT INTO media_items (library_id, title, author, isbn, description, file_path, file_size, mime_type, cover_image_path, series, series_number, tags, tags_search, asin, date_published, publisher, contributors, contributors_search, language, edition, page_count, genre, copyright_year, goodreads_id, openlibrary_id, google_books_id, added_by_admin_id)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27)
|
||||
RETURNING *;
|
||||
```
|
||||
|
||||
**Change TO:**
|
||||
|
||||
```sql
|
||||
-- name: CreateMediaItem :one
|
||||
INSERT INTO media_items (library_id, title, author, isbn, description, file_path, file_size, mime_type, cover_image_path, series, series_number, tags, tags_search, asin, date_published, publisher, contributors, contributors_search, language, edition, page_count, genre, copyright_year, goodreads_id, openlibrary_id, google_books_id, added_by_admin_id, created_at)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28)
|
||||
RETURNING *;
|
||||
```
|
||||
|
||||
### Step 1.3: Regenerate Go code from SQL OR manually update queries.sql.go
|
||||
|
||||
**Option A - Run SQL generation (recommended):**
|
||||
|
||||
```bash
|
||||
cd internal/database && go generate ./...
|
||||
```
|
||||
|
||||
**Option B - Manual update (if Option A fails):**
|
||||
|
||||
**File:** `internal/database/queries.sql.go`
|
||||
|
||||
**Find `CreateMediaItemParams` struct (around line 557):**
|
||||
|
||||
**Add to struct (after AddedByAdminID):**
|
||||
|
||||
```go
|
||||
CreatedAt pgtype.Timestamp `db:"created_at" json:"created_at"`
|
||||
```
|
||||
|
||||
**Find `CreateMediaItem` function (around line 588):**
|
||||
|
||||
**Add to the query parameters (after AddedByAdminID in the VALUES):**
|
||||
|
||||
```go
|
||||
arg.CreatedAt,
|
||||
```
|
||||
|
||||
### Step 1.4: Pass `fileModTime` to CreateMediaItem in scanner
|
||||
|
||||
**File:** `internal/services/media_scanner.go`
|
||||
|
||||
**Find the CreateMediaItem call** - around line 512.
|
||||
|
||||
**Current code (line 512-532):**
|
||||
|
||||
```go
|
||||
createdItem, err := s.db.CreateMediaItem(ctx, database.CreateMediaItemParams{
|
||||
LibraryID: libraryID,
|
||||
Title: metadata.Title,
|
||||
// ... other fields
|
||||
})
|
||||
```
|
||||
|
||||
**Add to the params (after AddedByAdminID):**
|
||||
|
||||
```go
|
||||
CreatedAt: pgtype.Timestamp{Time: fileModTime, Valid: true},
|
||||
```
|
||||
|
||||
**Note:** You'll need to import `"github.com/jackc/pgx/v5/pgtype"` if not already present.
|
||||
|
||||
```go
|
||||
CreatedAt: pgtype.Timestamp{Time: fileModTime, Valid: true},
|
||||
```
|
||||
|
||||
**Note:** Verify `CreatedAt` field exists in `CreateMediaItemParams`. If not, the schema uses `DEFAULT NOW()` and the column is `created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW()`. The INSERT may need to explicitly include the column name and value.
|
||||
|
||||
---
|
||||
|
||||
## Fix 2: Force Rescan Uses UPDATE Instead of DELETE+INSERT
|
||||
|
||||
**Goal:** Preserve `created_at` when force rescan runs by using UPDATE instead of delete + insert.
|
||||
|
||||
### Step 2.1: Find the force rescan delete logic
|
||||
|
||||
**File:** `internal/services/media_scanner.go`
|
||||
|
||||
**Current code (around lines 365-372):**
|
||||
|
||||
```go
|
||||
if s.forceRescan {
|
||||
fmt.Printf("Force rescan enabled, re-processing existing media item: %s\n", path)
|
||||
// Force update: delete existing and re-create
|
||||
if err := s.db.DeleteMediaItem(ctx, existingItem.ID); err != nil {
|
||||
fmt.Printf("Warning: failed to delete existing media item: %v\n", err)
|
||||
}
|
||||
// Continue to create new entry below
|
||||
}
|
||||
```
|
||||
|
||||
### Step 2.2: Replace DELETE+INSERT with UPDATE
|
||||
|
||||
**Replace lines 365-372 with:**
|
||||
|
||||
```go
|
||||
if s.forceRescan {
|
||||
fmt.Printf("Force rescan enabled, updating existing media item: %s\n", path)
|
||||
// Use UPDATE instead of DELETE+INSERT to preserve created_at
|
||||
// Re-extract metadata and update
|
||||
metadata, err := s.extractMetadata(path)
|
||||
if err != nil {
|
||||
fmt.Printf("Warning: failed to extract metadata for force rescan %s: %v\n", path, err)
|
||||
metadata = &MediaMetadata{}
|
||||
}
|
||||
if err := s.updateMediaItem(ctx, existingItem.ID, path, info); err != nil {
|
||||
fmt.Printf("Warning: failed to update existing media item: %v\n", err)
|
||||
}
|
||||
return false, nil
|
||||
}
|
||||
```
|
||||
|
||||
**Note:** The existing `updateMediaItem` function (line 1390) handles updating most fields. Verify it doesn't overwrite `created_at`.
|
||||
|
||||
---
|
||||
|
||||
## Fix 3: `GetMediaItemByFilePath` Filters by `library_id`
|
||||
|
||||
**Goal:** Prevent cross-library interference - scanning one library shouldn't affect another library's entries.
|
||||
|
||||
**Important Context:** The `libraryID` is already available in `StartWatchModeForLibrary` at `scanner.go:385`:
|
||||
|
||||
```go
|
||||
h.StartWatchModeForLibrary(ctx, library.ID, library.CreatedByAdminID)
|
||||
```
|
||||
|
||||
The function receives `libraryID` but doesn't store it in the scanner. We need to store it.
|
||||
|
||||
### Step 3.1: Add SetLibraryID method to MediaScanner
|
||||
|
||||
**File:** `internal/services/media_scanner.go`
|
||||
|
||||
**Find the SetAdminID function (around line 104):**
|
||||
|
||||
**Add AFTER SetAdminID:**
|
||||
|
||||
```go
|
||||
func (s *MediaScanner) SetLibraryID(libraryID pgtype.UUID) {
|
||||
s.defaultLibraryID = libraryID
|
||||
}
|
||||
```
|
||||
|
||||
### Step 3.2: Store libraryID in StartWatchModeForLibrary
|
||||
|
||||
**File:** `internal/handlers/scanner.go`
|
||||
|
||||
**Current code (around line 268):**
|
||||
|
||||
```go
|
||||
scanner.SetAdminID(adminID)
|
||||
scanner.WatchChanges(h.watchModeCtx)
|
||||
```
|
||||
|
||||
**Add AFTER line 268:**
|
||||
|
||||
```go
|
||||
scanner.SetLibraryID(libraryID)
|
||||
```
|
||||
|
||||
### Step 3.3: Modify the SQL query
|
||||
|
||||
**File:** `internal/database/queries/queries.sql`
|
||||
|
||||
**Current code (line 302-303):**
|
||||
|
||||
```sql
|
||||
-- name: GetMediaItemByFilePath :one
|
||||
SELECT * FROM media_items WHERE file_path = $1;
|
||||
```
|
||||
|
||||
**Change TO:**
|
||||
|
||||
```sql
|
||||
-- name: GetMediaItemByFilePath :one
|
||||
SELECT * FROM media_items WHERE file_path = $1 AND library_id = $2;
|
||||
```
|
||||
|
||||
### Step 3.4: Update the Go code (generated or manual)
|
||||
|
||||
**File:** `internal/database/queries.sql.go`
|
||||
|
||||
Find `GetMediaItemByFilePath` function and update:
|
||||
|
||||
1. Add `LibraryID pgtype.UUID` parameter to the function and params struct
|
||||
2. Add the parameter to the query call
|
||||
|
||||
**OR run SQL generation:**
|
||||
|
||||
```bash
|
||||
cd internal/database && go generate ./...
|
||||
```
|
||||
|
||||
### Step 3.5: Update getMediaItemByFilePath in scanner
|
||||
|
||||
**File:** `internal/services/media_scanner.go`
|
||||
|
||||
**Current code (lines 1395-1397):**
|
||||
|
||||
```go
|
||||
func (s *MediaScanner) getMediaItemByFilePath(ctx context.Context, filePath string) (database.MediaItems, error) {
|
||||
return s.db.GetMediaItemByFilePath(ctx, filePath)
|
||||
}
|
||||
```
|
||||
|
||||
**Change TO:**
|
||||
|
||||
```go
|
||||
func (s *MediaScanner) getMediaItemByFilePath(ctx context.Context, filePath string) (database.MediaItems, error) {
|
||||
return s.db.GetMediaItemByFilePath(ctx, filePath, s.defaultLibraryID)
|
||||
}
|
||||
```
|
||||
|
||||
### Step 3.6: Update all callers
|
||||
|
||||
**File:** `internal/services/media_scanner.go`
|
||||
|
||||
Update all places that call `getMediaItemByFilePath` to pass the libraryID:
|
||||
|
||||
- Line 361: In `processMediaFile` - already has access to libraryID via folder lookup
|
||||
|
||||
**Note:** The `processMediaFile` function already determines libraryID from the folder path (lines 487-501). Use that libraryID instead of `s.defaultLibraryID` for better accuracy.
|
||||
|
||||
---
|
||||
|
||||
## Fix 4: Add Deletion Handling
|
||||
|
||||
**Goal:** Delete media items when files are removed from the filesystem.
|
||||
|
||||
**Safety Note:** Individual deletes are used (not bulk) to minimize risk of accidental mass deletion. Each deletion is logged before execution for traceability.
|
||||
|
||||
### Step 4.0: Add File Logging Infrastructure
|
||||
|
||||
**New File:** `internal/services/scanner_logger.go`
|
||||
|
||||
Create a new file with the following structure:
|
||||
|
||||
```go
|
||||
package services
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
logDir = "/app/logs"
|
||||
maxLogAgeDays = 7
|
||||
)
|
||||
|
||||
type ScannerLogger struct {
|
||||
deletesFile *os.File
|
||||
errorsFile *os.File
|
||||
currentDate string
|
||||
}
|
||||
|
||||
// NewScannerLogger creates a new scanner logger instance
|
||||
func NewScannerLogger() *ScannerLogger {
|
||||
return &ScannerLogger{}
|
||||
}
|
||||
|
||||
// ensureLogFiles creates/opens log files for today
|
||||
func (l *ScannerLogger) ensureLogFiles() error {
|
||||
today := time.Now().Format("2006-01-02")
|
||||
|
||||
// Check if we need to rotate (new day)
|
||||
if l.currentDate == today && l.deletesFile != nil {
|
||||
return nil // Already have today's files open
|
||||
}
|
||||
|
||||
// Close existing files
|
||||
if l.deletesFile != nil {
|
||||
l.deletesFile.Close()
|
||||
}
|
||||
if l.errorsFile != nil {
|
||||
l.errorsFile.Close()
|
||||
}
|
||||
|
||||
// Create log directory if it doesn't exist
|
||||
if err := os.MkdirAll(logDir, 0755); err != nil {
|
||||
return fmt.Errorf("failed to create log directory: %v", err)
|
||||
}
|
||||
|
||||
// Open new files for today
|
||||
deletesPath := filepath.Join(logDir, fmt.Sprintf("scanner-deletes-%s.log", today))
|
||||
errorsPath := filepath.Join(logDir, fmt.Sprintf("scanner-errors-%s.log", today))
|
||||
|
||||
deletesFile, err := os.OpenFile(deletesPath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to open deletes log file: %v", err)
|
||||
}
|
||||
|
||||
errorsFile, err := os.OpenFile(errorsPath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
|
||||
if err != nil {
|
||||
deletesFile.Close()
|
||||
return fmt.Errorf("failed to open errors log file: %v", err)
|
||||
}
|
||||
|
||||
l.deletesFile = deletesFile
|
||||
l.errorsFile = errorsFile
|
||||
l.currentDate = today
|
||||
|
||||
// Clean up old log files
|
||||
l.cleanupOldLogs()
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// cleanupOldLogs removes log files older than maxLogAgeDays
|
||||
func (l *ScannerLogger) cleanupOldLogs() {
|
||||
cutoff := time.Now().AddDate(0, 0, -maxLogAgeDays)
|
||||
|
||||
filepath.Walk(logDir, func(path string, info os.FileInfo) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !info.IsDir() && info.ModTime().Before(cutoff) {
|
||||
os.Remove(path)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
// LogDelete logs a deletion event
|
||||
func (l *ScannerLogger) LogDelete(message string) {
|
||||
if err := l.ensureLogFiles(); err != nil {
|
||||
fmt.Printf("ERROR: Failed to ensure log files: %v\n", err)
|
||||
return
|
||||
}
|
||||
timestamp := time.Now().Format("2006-01-02 15:04:05")
|
||||
logLine := fmt.Sprintf("[%s] %s\n", timestamp, message)
|
||||
l.deletesFile.WriteString(logLine)
|
||||
}
|
||||
|
||||
// LogError logs an error event
|
||||
func (l *ScannerLogger) LogError(message string) {
|
||||
if err := l.ensureLogFiles(); err != nil {
|
||||
fmt.Printf("ERROR: Failed to ensure log files: %v\n", err)
|
||||
return
|
||||
}
|
||||
timestamp := time.Now().Format("2006-01-02 15:04:05")
|
||||
logLine := fmt.Sprintf("[%s] %s\n", timestamp, message)
|
||||
l.errorsFile.WriteString(logLine)
|
||||
}
|
||||
|
||||
// Close closes the log files
|
||||
func (l *ScannerLogger) Close() {
|
||||
if l.deletesFile != nil {
|
||||
l.deletesFile.Close()
|
||||
}
|
||||
if l.errorsFile != nil {
|
||||
l.errorsFile.Close()
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
**Add to MediaScanner struct:**
|
||||
|
||||
- Add `logger *ScannerLogger` field to track logger instance
|
||||
|
||||
**Update NewMediaScanner function:**
|
||||
|
||||
- Initialize logger: `logger: NewScannerLogger()`
|
||||
|
||||
### Step 4.1: Ensure libraryID is available in scanner
|
||||
|
||||
**Verify:** Steps 3.1-3.2 are complete - scanner has `SetLibraryID` and handler calls it.
|
||||
|
||||
### Step 4.2: Add deletion logic in Watch mode
|
||||
|
||||
**File:** `internal/services/media_scanner.go`
|
||||
|
||||
**Find:** `WatchChanges` function (around line 1427).
|
||||
|
||||
**Find the event handling section (around lines 1449-1455):**
|
||||
|
||||
```go
|
||||
// Handle file modifications and creations
|
||||
if (event.Has(fsnotify.Create) || event.Has(fsnotify.Write)) && s.isScannableFile(event.Name) {
|
||||
fmt.Printf("New/modified media file detected: %s\n", event.Name)
|
||||
if _, err := s.processMediaFile(ctx, event.Name); err != nil {
|
||||
fmt.Printf("Error processing modified media file %s: %v\n", event.Name, err)
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
**Add AFTER that block (before line 1457):**
|
||||
|
||||
```go
|
||||
// Handle file deletions
|
||||
if event.Has(fsnotify.Remove) && s.isScannableFile(event.Name) {
|
||||
// Use file logger for persistence
|
||||
s.logger.LogDelete(fmt.Sprintf("[WATCH-DELETE] File removed from filesystem: %s", event.Name))
|
||||
|
||||
// Determine libraryID for this file
|
||||
var libraryID pgtype.UUID
|
||||
for _, folder := range s.folders {
|
||||
if strings.HasPrefix(event.Name, folder) {
|
||||
lib, err := s.db.GetLibraryByFolder(ctx, folder)
|
||||
if err == nil {
|
||||
libraryID = lib.LibraryID
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
if !libraryID.Valid {
|
||||
msg := fmt.Sprintf("[WATCH-DELETE] WARNING: could not determine library for deleted file: %s", event.Name)
|
||||
s.logger.LogDelete(msg)
|
||||
s.logger.LogError(msg)
|
||||
return
|
||||
}
|
||||
|
||||
// Look up media item BEFORE deleting - log for safety
|
||||
existingItem, err := s.db.GetMediaItemByFilePath(ctx, event.Name, libraryID)
|
||||
if err == nil {
|
||||
msg := fmt.Sprintf("[WATCH-DELETE] Found media item to delete: ID=%s, Title=%s, Path=%s",
|
||||
existingItem.ID, existingItem.Title.String, existingItem.FilePath.String)
|
||||
s.logger.LogDelete(msg)
|
||||
|
||||
if err := s.db.DeleteMediaItem(ctx, existingItem.ID); err != nil {
|
||||
errMsg := fmt.Sprintf("[WATCH-DELETE] ERROR: failed to delete media item %s: %v", existingItem.ID, err)
|
||||
s.logger.LogDelete(errMsg)
|
||||
s.logger.LogError(errMsg)
|
||||
} else {
|
||||
s.logger.LogDelete(fmt.Sprintf("[WATCH-DELETE] SUCCESS: deleted media item '%s' (was at %s)",
|
||||
existingItem.Title.String, existingItem.FilePath.String))
|
||||
}
|
||||
} else if err != pgx.ErrNoRows {
|
||||
errMsg := fmt.Sprintf("[WATCH-DELETE] ERROR: failed to look up media item for %s: %v", event.Name, err)
|
||||
s.logger.LogDelete(errMsg)
|
||||
s.logger.LogError(errMsg)
|
||||
} else {
|
||||
s.logger.LogDelete(fmt.Sprintf("[WATCH-DELETE] No media item found in database for deleted file: %s", event.Name))
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
**Usage:** Replace `fmt.Printf` with `s.logger.LogDelete()` and `s.logger.LogError()` for persistent file logging.
|
||||
|
||||
**Note:** This requires Step 3.3-3.5 to be complete (GetMediaItemByFilePath accepts libraryID).
|
||||
|
||||
### Step 4.3: Add deletion logic in Rescan
|
||||
|
||||
**File:** `internal/services/media_scanner.go`
|
||||
|
||||
**Find:** End of `ScanFolders` function (after line 242).
|
||||
|
||||
**Current code (around line 242-248):**
|
||||
|
||||
```go
|
||||
fmt.Printf("Scan completed: %d total files scanned, %d media files found, %d new items, %d errors\n",
|
||||
processedFiles, mediaFiles, s.newItems, s.errors)
|
||||
|
||||
if s.job != nil && s.totalFiles > 0 {
|
||||
s.job.UpdateProgress(1.0, processedFiles, s.newItems, s.errors)
|
||||
}
|
||||
|
||||
return nil
|
||||
```
|
||||
|
||||
**Add BEFORE `return nil`:**
|
||||
|
||||
```go
|
||||
// Clean up: Find media items in DB that no longer exist on filesystem
|
||||
for _, folder := range s.folders {
|
||||
lib, err := s.db.GetLibraryByFolder(ctx, folder)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
libraryID := lib.LibraryID
|
||||
|
||||
dbItems, err := s.db.ListMediaItemsByLibrary(ctx, libraryID)
|
||||
if err != nil {
|
||||
fmt.Printf("Warning: failed to get library items for cleanup: %v\n", err)
|
||||
continue
|
||||
}
|
||||
|
||||
// Build set of scanned file paths for this folder
|
||||
scannedPaths := make(map[string]bool)
|
||||
filepath.WalkDir(folder, func(path string, d os.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !d.IsDir() && s.isScannableFile(path) {
|
||||
scannedPaths[path] = true
|
||||
}
|
||||
return nil
|
||||
})
|
||||
|
||||
// Delete items whose files no longer exist
|
||||
// ENHANCED SAFETY LOGGING: Log BEFORE each delete - use file logger for persistence
|
||||
for _, item := range dbItems {
|
||||
filePath := item.FilePath.String
|
||||
if filePath != "" && !scannedPaths[filePath] {
|
||||
msg := fmt.Sprintf("[RESCAN-CLEANUP] Orphaned media item found: ID=%s, Title=%s, Path=%s",
|
||||
item.ID, item.Title.String, filePath)
|
||||
s.logger.LogDelete(msg)
|
||||
|
||||
delMsg := fmt.Sprintf("[RESCAN-CLEANUP] Deleting orphaned item '%s' (file no longer exists at %s)",
|
||||
item.Title.String, filePath)
|
||||
s.logger.LogDelete(delMsg)
|
||||
|
||||
if err := s.db.DeleteMediaItem(ctx, item.ID); err != nil {
|
||||
errMsg := fmt.Sprintf("[RESCAN-CLEANUP] ERROR: failed to delete orphaned item %s: %v", item.Title.String, err)
|
||||
s.logger.LogDelete(errMsg)
|
||||
s.logger.LogError(errMsg)
|
||||
} else {
|
||||
s.logger.LogDelete(fmt.Sprintf("[RESCAN-CLEANUP] SUCCESS: deleted orphaned item '%s'", item.Title.String))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
**Log Files Location:** `/app/logs/`
|
||||
|
||||
- `scanner-deletes-YYYY-MM-DD.log` - All deletion events (watch mode + rescan)
|
||||
- `scanner-errors-YYYY-MM-DD.log` - All error events
|
||||
- Rotation: Daily, keeps 7 days of history
|
||||
- Volume mount in docker-compose: `./logs:/app/logs`
|
||||
|
||||
---
|
||||
|
||||
## Verification Steps After Implementation
|
||||
|
||||
1. **Compile the code:**
|
||||
|
||||
```bash
|
||||
go build ./...
|
||||
```
|
||||
|
||||
2. **Run tests:**
|
||||
|
||||
```bash
|
||||
go test ./... -v
|
||||
```
|
||||
|
||||
3. **Test manually:**
|
||||
- Add a new book to the uploads folder
|
||||
- Verify it appears in the database with correct `created_at` (should be file's actual modification time)
|
||||
- Delete a book from the filesystem
|
||||
- Verify it's removed from the database (via watch mode OR rescan)
|
||||
- Run force rescan on a library
|
||||
- Verify `created_at` is preserved (not reset)
|
||||
|
||||
4. **Check the logs** for expected output:
|
||||
- Log files are at `/app/logs/scanner-deletes-YYYY-MM-DD.log` and `/app/logs/scanner-errors-YYYY-MM-DD.log`
|
||||
- Look for `[WATCH-DELETE]` entries for watch mode deletions
|
||||
- Look for `[RESCAN-CLEANUP]` entries for rescan cleanup deletions
|
||||
- Check both files to verify all operations were logged
|
||||
|
||||
---
|
||||
|
||||
## Safety Checks (IMPORTANT)
|
||||
|
||||
### Prevent deleting ALL books:
|
||||
|
||||
- The cleanup logic MUST check `scannedPaths[filePath]` - this ensures we only delete items whose paths were NOT found during the filesystem scan
|
||||
- The key condition is: `if filePath != "" && !scannedPaths[filePath]` - meaning "if this file was NOT found in our scan, delete it"
|
||||
- This is correct because:
|
||||
1. We scan the filesystem → get all current file paths
|
||||
2. We query DB → get all stored file paths
|
||||
3. We compare → only delete if DB path is NOT in filesystem paths
|
||||
|
||||
### Before running against production:
|
||||
|
||||
- Test with a small subset of books first
|
||||
- Verify the delete queries target specific library_id (not all libraries)
|
||||
- Check logs show only expected deletions
|
||||
|
||||
---
|
||||
|
||||
## Files to Modify
|
||||
|
||||
| Fix | File | Changes |
|
||||
| --- | --------------------------------------- | -------------------------------------------------------- |
|
||||
| 1a | `internal/database/queries/queries.sql` | Add created_at to INSERT columns |
|
||||
| 1b | `internal/database/queries.sql.go` | Add CreatedAt to CreateMediaItemParams struct + query |
|
||||
| 1c | `internal/services/media_scanner.go` | Get file.ModTime() + pass to CreateMediaItem |
|
||||
| 2 | `internal/services/media_scanner.go` | Change force rescan from DELETE+INSERT to UPDATE |
|
||||
| 3a | `internal/services/media_scanner.go` | Add SetLibraryID() method |
|
||||
| 3b | `internal/handlers/scanner.go` | Call scanner.SetLibraryID(libraryID) |
|
||||
| 3c | `internal/database/queries/queries.sql` | Add library_id to WHERE clause |
|
||||
| 3d | `internal/database/queries.sql.go` | Update GetMediaItemByFilePath params |
|
||||
| 3e | `internal/services/media_scanner.go` | Pass libraryID to getMediaItemByFilePath |
|
||||
| 4.0 | `internal/services/scanner_logger.go` | **NEW FILE** - File logging infrastructure |
|
||||
| 4a | `internal/services/media_scanner.go` | Add fsnotify.Remove handler in WatchChanges + use logger |
|
||||
| 4b | `internal/services/media_scanner.go` | Add cleanup loop in ScanFolders + use logger |
|
||||
|
||||
### Docker Compose Volume Mount
|
||||
|
||||
Add to your `docker-compose.yml` to persist logs:
|
||||
|
||||
```yaml
|
||||
volumes:
|
||||
- ./logs:/app/logs
|
||||
```
|
||||
|
||||
### Important: SQL Code Generation
|
||||
|
||||
**The file `internal/database/queries.sql.go` is AUTO-GENERATED from `internal/database/queries/queries.sql`**
|
||||
|
||||
After editing `queries.sql`, you MUST regenerate the Go code:
|
||||
|
||||
**Option A - Run SQL code generation (recommended):**
|
||||
|
||||
```bash
|
||||
cd internal/database && go generate ./...
|
||||
```
|
||||
|
||||
**Option B - Manual update (if Option A fails):**
|
||||
|
||||
If `go generate` fails or is not available, manually update `queries.sql.go`:
|
||||
|
||||
1. Add `LibraryID pgtype.UUID` parameter to `GetMediaItemByFilePathParams` struct
|
||||
2. Add parameter to the query function call
|
||||
|
||||
For Fix 1, manually add `CreatedAt` to:
|
||||
|
||||
- `CreateMediaItemParams` struct
|
||||
- The query VALUES
|
||||
|
||||
---
|
||||
|
||||
## Implementation Order
|
||||
|
||||
1. Fix 3a, 3b, 3c, 3d, 3e - Add library_id filtering (foundational for Fix 4)
|
||||
2. Fix 1 - File mtime for created_at
|
||||
3. Fix 2 - Force rescan UPDATE
|
||||
4. Fix 4 - Watch mode + Rescan deletion handling
|
||||
|
||||
---
|
||||
|
||||
## Testing Requirements
|
||||
|
||||
### Existing Tests Analysis
|
||||
|
||||
**Current scanner integration tests** (`cmd/server/tests/scanner_integration_test.go`):
|
||||
|
||||
- Uses `/app/uploads` as test folder
|
||||
- Tests scan, progress tracking, watch mode start/stop
|
||||
- **After Fix 3:** Tests can safely use `/app/uploads` because GetMediaItemByFilePath now filters by library_id - test library's entries are isolated from user's library
|
||||
|
||||
**Current unit tests** (`internal/services/*_test.go`):
|
||||
|
||||
- `media_scanner_epub_cover_test.go` - Tests cover extraction
|
||||
- `media_scanner_hash_test.go` - Tests hash calculation
|
||||
- `media_scanner_library_type_test.go` - Tests library type detection
|
||||
- `media_scanner_comic_test.go` - Tests comic handling
|
||||
|
||||
These unit tests test specific functions but don't test the new features we're adding.
|
||||
|
||||
### Required Test Changes
|
||||
|
||||
#### Fix 1 & 2 Tests (File mtime + Force rescan UPDATE)
|
||||
|
||||
**Unit Test:** Add test in `internal/services/media_scanner_test.go` (create if doesn't exist)
|
||||
|
||||
```go
|
||||
// TestProcessMediaFile_UsesFileMtime tests that created_at uses file modification time
|
||||
func TestProcessMediaFile_UsesFileMtime(t *testing.T) {
|
||||
// Create test file with specific modification time
|
||||
testFile := createTestEpub(t, "test-book.epub")
|
||||
defer os.Remove(testFile)
|
||||
|
||||
// Set specific mtime
|
||||
pastTime := time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC)
|
||||
os.Chtimes(testFile, pastTime, pastTime)
|
||||
|
||||
// Process file
|
||||
scanner := NewMediaScanner(db)
|
||||
scanner.SetFolders([]string{filepath.Dir(testFile)})
|
||||
scanner.SetLibraryID(testLibraryID)
|
||||
|
||||
wasNew, err := scanner.ProcessMediaFile(ctx, testFile)
|
||||
require.NoError(t, err)
|
||||
require.True(t, wasNew)
|
||||
|
||||
// Verify created_at matches file mtime, not scan time
|
||||
item, err := db.GetMediaItemByFilePath(ctx, testFile, testLibraryID)
|
||||
require.NoError(t, err)
|
||||
assert.WithinDuration(t, pastTime, item.CreatedAt.Time, time.Second)
|
||||
}
|
||||
|
||||
// TestForceRescan_PreservesCreatedAt tests that force rescan doesn't reset created_at
|
||||
func TestForceRescan_PreservesCreatedAt(t *testing.T) {
|
||||
// Create and process file
|
||||
testFile := createTestEpub(t, "test-book.epub")
|
||||
defer os.Remove(testFile)
|
||||
|
||||
scanner := NewMediaScanner(db)
|
||||
scanner.SetFolders([]string{filepath.Dir(testFile)})
|
||||
scanner.SetLibraryID(testLibraryID)
|
||||
|
||||
_, _ = scanner.ProcessMediaFile(ctx, testFile)
|
||||
|
||||
// Get original created_at
|
||||
item, _ := db.GetMediaItemByFilePath(ctx, testFile, testLibraryID)
|
||||
originalCreatedAt := item.CreatedAt.Time
|
||||
|
||||
// Wait a moment to ensure time difference
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
|
||||
// Force rescan
|
||||
scanner.SetForce(true)
|
||||
_, _ = scanner.ProcessMediaFile(ctx, testFile)
|
||||
|
||||
// Verify created_at is preserved
|
||||
item, _ = db.GetMediaItemByFilePath(ctx, testFile, testLibraryID)
|
||||
assert.Equal(t, originalCreatedAt, item.CreatedAt.Time)
|
||||
}
|
||||
```
|
||||
|
||||
#### Fix 3 Tests (library_id filtering)
|
||||
|
||||
**Unit Test:** Add to same test file
|
||||
|
||||
```go
|
||||
// TestGetMediaItemByFilePath_FiltersByLibrary tests that GetMediaItemByFilePath respects library_id
|
||||
func TestGetMediaItemByFilePath_FiltersByLibrary(t *testing.T) {
|
||||
// Same file path in two different libraries
|
||||
testFile := createTestEpub(t, "shared-book.epub")
|
||||
defer os.Remove(testFile)
|
||||
|
||||
// Add to library A
|
||||
scannerA := NewMediaScanner(db)
|
||||
scannerA.SetLibraryID(libraryAID)
|
||||
_, _ = scannerA.ProcessMediaFile(ctx, testFile)
|
||||
|
||||
// Add same path to library B (simulating shared folder scenario)
|
||||
scannerB := NewMediaScanner(db)
|
||||
scannerB.SetLibraryID(libraryBID)
|
||||
_, _ = scannerB.ProcessMediaFile(ctx, testFile)
|
||||
|
||||
// Verify each library has its own entry
|
||||
itemA, errA := db.GetMediaItemByFilePath(ctx, testFile, libraryAID)
|
||||
itemB, errB := db.GetMediaItemByFilePath(ctx, testFile, libraryBID)
|
||||
|
||||
require.NoError(t, errA)
|
||||
require.NoError(t, errB)
|
||||
assert.Equal(t, libraryAID, itemA.LibraryID)
|
||||
assert.Equal(t, libraryBID, itemB.LibraryID)
|
||||
assert.NotEqual(t, itemA.ID, itemB.ID)
|
||||
}
|
||||
```
|
||||
|
||||
#### Fix 4 Tests (Deletion handling)
|
||||
|
||||
**Integration Test:** Modify existing scanner_integration_test.go
|
||||
|
||||
**Note:** Tests can continue to use `/app/uploads` - Fix 3 ensures test library's entries are isolated from user's library.
|
||||
|
||||
```go
|
||||
// TestScan_DeletesOrphanedBooks tests that rescan removes books no longer on filesystem
|
||||
func TestScan_DeletesOrphanedBooks(t *testing.T) {
|
||||
// Create test library pointing to /app/uploads (same as existing tests)
|
||||
testFolder := "/app/uploads"
|
||||
|
||||
// Create test library with test folder
|
||||
libraryID := createTestLibrary(t, s.setup.Token, "Orphan Test Library", testFolder)
|
||||
|
||||
// Add a test file
|
||||
testFile := createTestEpubFile(t, testFolder, "test-orphan-book.epub")
|
||||
defer os.Remove(testFile) // Cleanup after test
|
||||
|
||||
// Add a test file
|
||||
testFile := createTestEpubFile(t, testFolder, "test-book.epub")
|
||||
|
||||
// Initial scan
|
||||
scanLibrary(t, s.setup.Server.URL, libraryID, s.setup.Token)
|
||||
|
||||
// Verify book exists
|
||||
items, _ := s.setup.DB.ListMediaItemsByLibrary(ctx, libraryID)
|
||||
require.Len(t, items, 1)
|
||||
|
||||
// Delete file from filesystem (simulating user deletion)
|
||||
os.Remove(testFile)
|
||||
|
||||
// Rescan
|
||||
scanLibrary(t, s.setup.Server.URL, libraryID, s.setup.Token)
|
||||
|
||||
// Verify book was deleted from DB
|
||||
items, _ = s.setup.DB.ListMediaItemsByLibrary(ctx, libraryID)
|
||||
assert.Len(t, items, 0, "Orphaned book should be removed from database")
|
||||
}
|
||||
```
|
||||
|
||||
**Integration Test for Watch Mode deletion:**
|
||||
|
||||
**Note:** Tests can continue to use `/app/uploads` - Fix 3 ensures test library's entries are isolated.
|
||||
|
||||
```go
|
||||
// TestWatchMode_DeletesRemovedFile tests that watch mode detects and deletes removed files
|
||||
func TestWatchMode_DeletesRemovedFile(t *testing.T) {
|
||||
// Use /app/uploads - Fix 3 handles isolation
|
||||
testFolder := "/app/uploads"
|
||||
|
||||
// Create test library
|
||||
libraryID := createTestLibrary(t, s.setup.Token, "Watch Delete Test", testFolder)
|
||||
|
||||
// Add test file
|
||||
testFile := createTestEpubFile(t, testFolder, "watch-test.epub")
|
||||
|
||||
// Start watch mode
|
||||
startWatchMode(t, s.setup.Server.URL, libraryID, s.setup.Token)
|
||||
|
||||
// Wait for initial scan
|
||||
time.Sleep(2 * time.Second)
|
||||
|
||||
// Verify book exists
|
||||
items, _ := s.setup.DB.ListMediaItemsByLibrary(ctx, libraryID)
|
||||
require.Len(t, items, 1)
|
||||
|
||||
// Delete file
|
||||
os.Remove(testFile)
|
||||
|
||||
// Wait for watch mode to detect
|
||||
time.Sleep(2 * time.Second)
|
||||
|
||||
// Verify book was deleted
|
||||
items, _ = s.setup.DB.ListMediaItemsByLibrary(ctx, libraryID)
|
||||
assert.Len(t, items, 0, "Book should be deleted when file removed from filesystem")
|
||||
|
||||
// Stop watch mode
|
||||
stopWatchMode(t, s.setup.Server.URL, libraryID, s.setup.Token)
|
||||
}
|
||||
```
|
||||
|
||||
### Test Infrastructure Changes Required
|
||||
|
||||
1. **No folder changes needed:** The Fix 3 (library_id filtering) ENABLEs tests to safely use `/app/uploads` without affecting user's library
|
||||
2. **Existing tests should work as-is** after Fix 3 is implemented
|
||||
3. **Use test_helpers:** Ensure all tests use `setupTestServer()` from `test_helpers.go`
|
||||
|
||||
### Documentation Updates
|
||||
|
||||
If API behavior changes, update:
|
||||
|
||||
- `docs/developer/api/scanner.md` - For any endpoint changes
|
||||
- `docs/user/` - If user-facing behavior changes
|
||||
|
||||
### Running Tests
|
||||
|
||||
After implementation, run:
|
||||
|
||||
```bash
|
||||
# Unit tests
|
||||
go test ./internal/services/... -v -run "TestProcessMediaFile|TestGetMediaItemByFilePath|TestForceRescan"
|
||||
|
||||
# Integration tests
|
||||
go test ./cmd/server/tests/... -v -run "Scanner"
|
||||
|
||||
# All tests
|
||||
go test ./... -v
|
||||
```
|
||||
Reference in New Issue
Block a user