scanner: fix library isolation, file mtime, force rescan, and deletion handling
Fix 1 - File modification time for created_at: - Get file.ModTime() in processMediaFile and pass to CreateMediaItem - Modified SQL INSERT to include created_at column Fix 2 - Force rescan UPDATE instead of DELETE+INSERT: - Changed force rescan logic to call updateMediaItem instead of delete + create - Preserves created_at timestamp on force rescan Fix 3 - GetMediaItemByFilePath filters by library_id: - Added library_id to WHERE clause in SQL query - Created GetMediaItemByFilePathAnyLibrary for cross-library lookups (KOReader) - Added SetLibraryID method to MediaScanner - Updated handler to call SetLibraryID for watch mode Fix 4 - File deletion handling with persistent logging: - Added fsnotify.Remove handler in WatchChanges - Added orphan cleanup in ScanFolders after scan completes - Created scanner_logger.go with daily log rotation (7 days) - Logs to /app/logs/scanner-deletes-YYYY-MM-DD.log and scanner-errors-YYYY-MM-DD.log - Individual deletes with enhanced safety logging Note: Integration tests can now safely scan /app/uploads because GetMediaItemByFilePath now filters by library_id, preventing cross-library interference.
This commit is contained in:
@@ -175,7 +175,8 @@ type Querier interface {
|
||||
GetMediaHighlight(ctx context.Context, id pgtype.UUID) (MediaHighlights, error)
|
||||
GetMediaHighlights(ctx context.Context, arg GetMediaHighlightsParams) ([]MediaHighlights, error)
|
||||
GetMediaItem(ctx context.Context, id pgtype.UUID) (MediaItems, error)
|
||||
GetMediaItemByFilePath(ctx context.Context, filePath string) (MediaItems, error)
|
||||
GetMediaItemByFilePath(ctx context.Context, arg GetMediaItemByFilePathParams) (MediaItems, error)
|
||||
GetMediaItemByFilePathAnyLibrary(ctx context.Context, filePath string) (MediaItems, error)
|
||||
// ============================================
|
||||
// KOREADER SYNC PROTOCOL
|
||||
// ============================================
|
||||
|
||||
@@ -549,39 +549,40 @@ func (q *Queries) CreateMediaHighlight(ctx context.Context, arg CreateMediaHighl
|
||||
}
|
||||
|
||||
const CreateMediaItem = `-- name: CreateMediaItem :one
|
||||
INSERT INTO media_items (library_id, title, author, isbn, description, file_path, file_size, mime_type, cover_image_path, series, series_number, tags, tags_search, asin, date_published, publisher, contributors, contributors_search, language, edition, page_count, genre, copyright_year, goodreads_id, openlibrary_id, google_books_id, added_by_admin_id)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27)
|
||||
INSERT INTO media_items (library_id, title, author, isbn, description, file_path, file_size, mime_type, cover_image_path, series, series_number, tags, tags_search, asin, date_published, publisher, contributors, contributors_search, language, edition, page_count, genre, copyright_year, goodreads_id, openlibrary_id, google_books_id, added_by_admin_id, created_at)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28)
|
||||
RETURNING id, library_id, title, author, isbn, description, file_path, file_size, mime_type, cover_image_path, series, series_number, tags, asin, date_published, publisher, contributors, language, edition, page_count, genre, copyright_year, goodreads_id, openlibrary_id, google_books_id, added_by_admin_id, created_at, updated_at, format_group, format_mimetype, is_reflowable, has_fixed_layout, total_characters, chapter_count, entitlement_id, revision_number, kobo_content_id, kobo_metadata, tags_search, contributors_search, file_sha256, opf_identifier, opf_uuid, hash_confidence
|
||||
`
|
||||
|
||||
type CreateMediaItemParams struct {
|
||||
LibraryID pgtype.UUID `db:"library_id" json:"library_id"`
|
||||
Title string `db:"title" json:"title"`
|
||||
Author pgtype.Text `db:"author" json:"author"`
|
||||
Isbn pgtype.Text `db:"isbn" json:"isbn"`
|
||||
Description pgtype.Text `db:"description" json:"description"`
|
||||
FilePath string `db:"file_path" json:"file_path"`
|
||||
FileSize pgtype.Int8 `db:"file_size" json:"file_size"`
|
||||
MimeType pgtype.Text `db:"mime_type" json:"mime_type"`
|
||||
CoverImagePath pgtype.Text `db:"cover_image_path" json:"cover_image_path"`
|
||||
Series pgtype.Text `db:"series" json:"series"`
|
||||
SeriesNumber pgtype.Int4 `db:"series_number" json:"series_number"`
|
||||
Tags []string `db:"tags" json:"tags"`
|
||||
TagsSearch []string `db:"tags_search" json:"tags_search"`
|
||||
Asin pgtype.Text `db:"asin" json:"asin"`
|
||||
DatePublished pgtype.Date `db:"date_published" json:"date_published"`
|
||||
Publisher pgtype.Text `db:"publisher" json:"publisher"`
|
||||
Contributors []string `db:"contributors" json:"contributors"`
|
||||
ContributorsSearch []string `db:"contributors_search" json:"contributors_search"`
|
||||
Language pgtype.Text `db:"language" json:"language"`
|
||||
Edition pgtype.Text `db:"edition" json:"edition"`
|
||||
PageCount pgtype.Int4 `db:"page_count" json:"page_count"`
|
||||
Genre pgtype.Text `db:"genre" json:"genre"`
|
||||
CopyrightYear pgtype.Int4 `db:"copyright_year" json:"copyright_year"`
|
||||
GoodreadsID pgtype.Text `db:"goodreads_id" json:"goodreads_id"`
|
||||
OpenlibraryID pgtype.Text `db:"openlibrary_id" json:"openlibrary_id"`
|
||||
GoogleBooksID pgtype.Text `db:"google_books_id" json:"google_books_id"`
|
||||
AddedByAdminID pgtype.UUID `db:"added_by_admin_id" json:"added_by_admin_id"`
|
||||
LibraryID pgtype.UUID `db:"library_id" json:"library_id"`
|
||||
Title string `db:"title" json:"title"`
|
||||
Author pgtype.Text `db:"author" json:"author"`
|
||||
Isbn pgtype.Text `db:"isbn" json:"isbn"`
|
||||
Description pgtype.Text `db:"description" json:"description"`
|
||||
FilePath string `db:"file_path" json:"file_path"`
|
||||
FileSize pgtype.Int8 `db:"file_size" json:"file_size"`
|
||||
MimeType pgtype.Text `db:"mime_type" json:"mime_type"`
|
||||
CoverImagePath pgtype.Text `db:"cover_image_path" json:"cover_image_path"`
|
||||
Series pgtype.Text `db:"series" json:"series"`
|
||||
SeriesNumber pgtype.Int4 `db:"series_number" json:"series_number"`
|
||||
Tags []string `db:"tags" json:"tags"`
|
||||
TagsSearch []string `db:"tags_search" json:"tags_search"`
|
||||
Asin pgtype.Text `db:"asin" json:"asin"`
|
||||
DatePublished pgtype.Date `db:"date_published" json:"date_published"`
|
||||
Publisher pgtype.Text `db:"publisher" json:"publisher"`
|
||||
Contributors []string `db:"contributors" json:"contributors"`
|
||||
ContributorsSearch []string `db:"contributors_search" json:"contributors_search"`
|
||||
Language pgtype.Text `db:"language" json:"language"`
|
||||
Edition pgtype.Text `db:"edition" json:"edition"`
|
||||
PageCount pgtype.Int4 `db:"page_count" json:"page_count"`
|
||||
Genre pgtype.Text `db:"genre" json:"genre"`
|
||||
CopyrightYear pgtype.Int4 `db:"copyright_year" json:"copyright_year"`
|
||||
GoodreadsID pgtype.Text `db:"goodreads_id" json:"goodreads_id"`
|
||||
OpenlibraryID pgtype.Text `db:"openlibrary_id" json:"openlibrary_id"`
|
||||
GoogleBooksID pgtype.Text `db:"google_books_id" json:"google_books_id"`
|
||||
AddedByAdminID pgtype.UUID `db:"added_by_admin_id" json:"added_by_admin_id"`
|
||||
CreatedAt pgtype.Timestamptz `db:"created_at" json:"created_at"`
|
||||
}
|
||||
|
||||
// Media Items queries
|
||||
@@ -614,6 +615,7 @@ func (q *Queries) CreateMediaItem(ctx context.Context, arg CreateMediaItemParams
|
||||
arg.OpenlibraryID,
|
||||
arg.GoogleBooksID,
|
||||
arg.AddedByAdminID,
|
||||
arg.CreatedAt,
|
||||
)
|
||||
var i MediaItems
|
||||
err := row.Scan(
|
||||
@@ -3250,11 +3252,72 @@ func (q *Queries) GetMediaItem(ctx context.Context, id pgtype.UUID) (MediaItems,
|
||||
}
|
||||
|
||||
const GetMediaItemByFilePath = `-- name: GetMediaItemByFilePath :one
|
||||
SELECT id, library_id, title, author, isbn, description, file_path, file_size, mime_type, cover_image_path, series, series_number, tags, asin, date_published, publisher, contributors, language, edition, page_count, genre, copyright_year, goodreads_id, openlibrary_id, google_books_id, added_by_admin_id, created_at, updated_at, format_group, format_mimetype, is_reflowable, has_fixed_layout, total_characters, chapter_count, entitlement_id, revision_number, kobo_content_id, kobo_metadata, tags_search, contributors_search, file_sha256, opf_identifier, opf_uuid, hash_confidence FROM media_items WHERE file_path = $1
|
||||
SELECT id, library_id, title, author, isbn, description, file_path, file_size, mime_type, cover_image_path, series, series_number, tags, asin, date_published, publisher, contributors, language, edition, page_count, genre, copyright_year, goodreads_id, openlibrary_id, google_books_id, added_by_admin_id, created_at, updated_at, format_group, format_mimetype, is_reflowable, has_fixed_layout, total_characters, chapter_count, entitlement_id, revision_number, kobo_content_id, kobo_metadata, tags_search, contributors_search, file_sha256, opf_identifier, opf_uuid, hash_confidence FROM media_items WHERE file_path = $1 AND library_id = $2
|
||||
`
|
||||
|
||||
func (q *Queries) GetMediaItemByFilePath(ctx context.Context, filePath string) (MediaItems, error) {
|
||||
row := q.db.QueryRow(ctx, GetMediaItemByFilePath, filePath)
|
||||
type GetMediaItemByFilePathParams struct {
|
||||
FilePath string `db:"file_path" json:"file_path"`
|
||||
LibraryID pgtype.UUID `db:"library_id" json:"library_id"`
|
||||
}
|
||||
|
||||
func (q *Queries) GetMediaItemByFilePath(ctx context.Context, arg GetMediaItemByFilePathParams) (MediaItems, error) {
|
||||
row := q.db.QueryRow(ctx, GetMediaItemByFilePath, arg.FilePath, arg.LibraryID)
|
||||
var i MediaItems
|
||||
err := row.Scan(
|
||||
&i.ID,
|
||||
&i.LibraryID,
|
||||
&i.Title,
|
||||
&i.Author,
|
||||
&i.Isbn,
|
||||
&i.Description,
|
||||
&i.FilePath,
|
||||
&i.FileSize,
|
||||
&i.MimeType,
|
||||
&i.CoverImagePath,
|
||||
&i.Series,
|
||||
&i.SeriesNumber,
|
||||
&i.Tags,
|
||||
&i.Asin,
|
||||
&i.DatePublished,
|
||||
&i.Publisher,
|
||||
&i.Contributors,
|
||||
&i.Language,
|
||||
&i.Edition,
|
||||
&i.PageCount,
|
||||
&i.Genre,
|
||||
&i.CopyrightYear,
|
||||
&i.GoodreadsID,
|
||||
&i.OpenlibraryID,
|
||||
&i.GoogleBooksID,
|
||||
&i.AddedByAdminID,
|
||||
&i.CreatedAt,
|
||||
&i.UpdatedAt,
|
||||
&i.FormatGroup,
|
||||
&i.FormatMimetype,
|
||||
&i.IsReflowable,
|
||||
&i.HasFixedLayout,
|
||||
&i.TotalCharacters,
|
||||
&i.ChapterCount,
|
||||
&i.EntitlementID,
|
||||
&i.RevisionNumber,
|
||||
&i.KoboContentID,
|
||||
&i.KoboMetadata,
|
||||
&i.TagsSearch,
|
||||
&i.ContributorsSearch,
|
||||
&i.FileSha256,
|
||||
&i.OpfIdentifier,
|
||||
&i.OpfUuid,
|
||||
&i.HashConfidence,
|
||||
)
|
||||
return i, err
|
||||
}
|
||||
|
||||
const GetMediaItemByFilePathAnyLibrary = `-- name: GetMediaItemByFilePathAnyLibrary :one
|
||||
SELECT id, library_id, title, author, isbn, description, file_path, file_size, mime_type, cover_image_path, series, series_number, tags, asin, date_published, publisher, contributors, language, edition, page_count, genre, copyright_year, goodreads_id, openlibrary_id, google_books_id, added_by_admin_id, created_at, updated_at, format_group, format_mimetype, is_reflowable, has_fixed_layout, total_characters, chapter_count, entitlement_id, revision_number, kobo_content_id, kobo_metadata, tags_search, contributors_search, file_sha256, opf_identifier, opf_uuid, hash_confidence FROM media_items WHERE file_path = $1 LIMIT 1
|
||||
`
|
||||
|
||||
func (q *Queries) GetMediaItemByFilePathAnyLibrary(ctx context.Context, filePath string) (MediaItems, error) {
|
||||
row := q.db.QueryRow(ctx, GetMediaItemByFilePathAnyLibrary, filePath)
|
||||
var i MediaItems
|
||||
err := row.Scan(
|
||||
&i.ID,
|
||||
|
||||
@@ -128,8 +128,8 @@ ORDER BY l.created_at DESC;
|
||||
|
||||
-- Media Items queries
|
||||
-- name: CreateMediaItem :one
|
||||
INSERT INTO media_items (library_id, title, author, isbn, description, file_path, file_size, mime_type, cover_image_path, series, series_number, tags, tags_search, asin, date_published, publisher, contributors, contributors_search, language, edition, page_count, genre, copyright_year, goodreads_id, openlibrary_id, google_books_id, added_by_admin_id)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27)
|
||||
INSERT INTO media_items (library_id, title, author, isbn, description, file_path, file_size, mime_type, cover_image_path, series, series_number, tags, tags_search, asin, date_published, publisher, contributors, contributors_search, language, edition, page_count, genre, copyright_year, goodreads_id, openlibrary_id, google_books_id, added_by_admin_id, created_at)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28)
|
||||
RETURNING *;
|
||||
|
||||
-- name: GetMediaItem :one
|
||||
@@ -300,7 +300,10 @@ RETURNING *;
|
||||
DELETE FROM media_items WHERE id = $1;
|
||||
|
||||
-- name: GetMediaItemByFilePath :one
|
||||
SELECT * FROM media_items WHERE file_path = $1;
|
||||
SELECT * FROM media_items WHERE file_path = $1 AND library_id = $2;
|
||||
|
||||
-- name: GetMediaItemByFilePathAnyLibrary :one
|
||||
SELECT * FROM media_items WHERE file_path = $1 LIMIT 1;
|
||||
|
||||
-- name: GetReadingProgress :one
|
||||
SELECT * FROM reading_progress WHERE media_item_id = $1 AND user_id = $2;
|
||||
|
||||
@@ -264,8 +264,8 @@ func (h *KOReaderHandler) resolveBookToMediaItem(c echo.Context, deviceID pgtype
|
||||
return alias.MediaItemID, alias.ConfidenceScore.Float64
|
||||
}
|
||||
|
||||
// Try to find by file path
|
||||
mediaItem, err := h.db.GetMediaItemByFilePath(ctx, book.FilePath)
|
||||
// Try to find by file path (search any library)
|
||||
mediaItem, err := h.db.GetMediaItemByFilePathAnyLibrary(ctx, book.FilePath)
|
||||
if err == nil {
|
||||
// Create new alias
|
||||
confidence := 0.7
|
||||
|
||||
@@ -266,6 +266,7 @@ func (h *Handler) StartWatchModeForLibrary(ctx context.Context, libraryID pgtype
|
||||
}
|
||||
|
||||
scanner.SetAdminID(adminID)
|
||||
scanner.SetLibraryID(libraryID)
|
||||
scanner.WatchChanges(h.watchModeCtx)
|
||||
|
||||
h.watchingLibraries[libraryIDStr] = true
|
||||
@@ -382,7 +383,7 @@ func (h *Handler) StartWatchModeForAllLibraries(ctx context.Context) error {
|
||||
|
||||
for _, library := range libraries {
|
||||
libraryIDStr := fmt.Sprintf("%x", library.ID.Bytes)
|
||||
if err := h.StartWatchModeForLibrary(ctx, library.ID, library.ID); err != nil {
|
||||
if err := h.StartWatchModeForLibrary(ctx, library.ID, library.CreatedByAdminID); err != nil {
|
||||
fmt.Printf("Warning: failed to start watch mode for library %s: %v\n", libraryIDStr, err)
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -77,6 +77,7 @@ type MediaScanner struct {
|
||||
defaultLibraryID pgtype.UUID
|
||||
libraryTypes map[string][]string
|
||||
forceRescan bool
|
||||
logger *ScannerLogger
|
||||
|
||||
totalFiles int
|
||||
newItems int
|
||||
@@ -98,6 +99,7 @@ func NewMediaScanner(db *database.Queries) *MediaScanner {
|
||||
adminID: pgtype.UUID{},
|
||||
defaultLibraryID: pgtype.UUID{Valid: false},
|
||||
libraryTypes: make(map[string][]string),
|
||||
logger: NewScannerLogger(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -105,6 +107,10 @@ func (s *MediaScanner) SetAdminID(adminID pgtype.UUID) {
|
||||
s.adminID = adminID
|
||||
}
|
||||
|
||||
func (s *MediaScanner) SetLibraryID(libraryID pgtype.UUID) {
|
||||
s.defaultLibraryID = libraryID
|
||||
}
|
||||
|
||||
func (s *MediaScanner) SetForce(force bool) {
|
||||
s.forceRescan = force
|
||||
}
|
||||
@@ -242,6 +248,55 @@ func (s *MediaScanner) ScanFolders(ctx context.Context) error {
|
||||
fmt.Printf("Scan completed: %d total files scanned, %d media files found, %d new items, %d errors\n",
|
||||
processedFiles, mediaFiles, s.newItems, s.errors)
|
||||
|
||||
// Clean up: Find media items in DB that no longer exist on filesystem
|
||||
for _, folder := range s.folders {
|
||||
lib, err := s.db.GetLibraryByFolder(ctx, folder)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
libraryID := lib.LibraryID
|
||||
|
||||
dbItems, err := s.db.ListMediaItemsByLibrary(ctx, libraryID)
|
||||
if err != nil {
|
||||
fmt.Printf("Warning: failed to get library items for cleanup: %v\n", err)
|
||||
continue
|
||||
}
|
||||
|
||||
// Build set of scanned file paths for this folder
|
||||
scannedPaths := make(map[string]bool)
|
||||
filepath.WalkDir(folder, func(path string, d os.DirEntry, walkErr error) error {
|
||||
if walkErr != nil {
|
||||
return walkErr
|
||||
}
|
||||
if !d.IsDir() && s.isScannableFile(path) {
|
||||
scannedPaths[path] = true
|
||||
}
|
||||
return nil
|
||||
})
|
||||
|
||||
// Delete items whose files no longer exist - with safety logging
|
||||
for _, item := range dbItems {
|
||||
filePath := item.FilePath
|
||||
if filePath != "" && !scannedPaths[filePath] {
|
||||
msg := fmt.Sprintf("[RESCAN-CLEANUP] Orphaned media item found: ID=%s, Title=%s, Path=%s",
|
||||
item.ID, item.Title, filePath)
|
||||
s.logger.LogDelete(msg)
|
||||
|
||||
delMsg := fmt.Sprintf("[RESCAN-CLEANUP] Deleting orphaned item '%s' (file no longer exists at %s)",
|
||||
item.Title, filePath)
|
||||
s.logger.LogDelete(delMsg)
|
||||
|
||||
if err := s.db.DeleteMediaItem(ctx, item.ID); err != nil {
|
||||
errMsg := fmt.Sprintf("[RESCAN-CLEANUP] ERROR: failed to delete orphaned item %s: %v", item.Title, err)
|
||||
s.logger.LogDelete(errMsg)
|
||||
s.logger.LogError(errMsg)
|
||||
} else {
|
||||
s.logger.LogDelete(fmt.Sprintf("[RESCAN-CLEANUP] SUCCESS: deleted orphaned item '%s'", item.Title))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if s.job != nil && s.totalFiles > 0 {
|
||||
s.job.UpdateProgress(1.0, processedFiles, s.newItems, s.errors)
|
||||
}
|
||||
@@ -357,19 +412,39 @@ func (s *MediaScanner) processMediaFile(ctx context.Context, path string) (bool,
|
||||
|
||||
fmt.Printf("File info for %s: size=%d\n", path, info.Size())
|
||||
|
||||
// Get file modification time for created_at
|
||||
fileModTime := info.ModTime()
|
||||
|
||||
// Find library for this file's folder
|
||||
var libraryID pgtype.UUID
|
||||
for _, folder := range s.folders {
|
||||
if strings.HasPrefix(path, folder) {
|
||||
lib, err := s.db.GetLibraryByFolder(ctx, folder)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("failed to find library for folder %s: %v", folder, err)
|
||||
}
|
||||
libraryID = lib.LibraryID
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if !libraryID.Valid {
|
||||
return false, fmt.Errorf("no library found for file path: %s", path)
|
||||
}
|
||||
|
||||
// Check if media item already exists in database
|
||||
existingItem, err := s.getMediaItemByFilePath(ctx, path)
|
||||
existingItem, err := s.getMediaItemByFilePath(ctx, path, libraryID)
|
||||
if err == nil {
|
||||
fmt.Printf("Media item already exists in database: %s (size: %d vs %d)\n", path, existingItem.FileSize.Int64, info.Size())
|
||||
|
||||
// If force rescan is enabled, always re-process
|
||||
if s.forceRescan {
|
||||
fmt.Printf("Force rescan enabled, re-processing existing media item: %s\n", path)
|
||||
// Force update: delete existing and re-create
|
||||
if err := s.db.DeleteMediaItem(ctx, existingItem.ID); err != nil {
|
||||
fmt.Printf("Warning: failed to delete existing media item: %v\n", err)
|
||||
fmt.Printf("Force rescan enabled, updating existing media item: %s\n", path)
|
||||
// Use UPDATE instead of DELETE+INSERT to preserve created_at
|
||||
if err := s.updateMediaItem(ctx, existingItem.ID, path, info); err != nil {
|
||||
fmt.Printf("Warning: failed to update existing media item: %v\n", err)
|
||||
}
|
||||
// Continue to create new entry below
|
||||
return false, nil
|
||||
} else {
|
||||
// Normal behavior: check if file has changed (by size)
|
||||
if existingItem.FileSize.Int64 != info.Size() {
|
||||
@@ -483,22 +558,7 @@ func (s *MediaScanner) processMediaFile(ctx context.Context, path string) (bool,
|
||||
metadata.Author = "Unknown"
|
||||
}
|
||||
|
||||
// Find library for this folder
|
||||
var libraryID pgtype.UUID
|
||||
for _, folder := range s.folders {
|
||||
if strings.HasPrefix(path, folder) {
|
||||
lib, err := s.db.GetLibraryByFolder(ctx, folder)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("failed to find library for folder %s: %v", folder, err)
|
||||
}
|
||||
libraryID = lib.LibraryID
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if !libraryID.Valid {
|
||||
return false, fmt.Errorf("no library found for file path: %s", path)
|
||||
}
|
||||
// libraryID already determined at start of function
|
||||
|
||||
// Normalize metadata fields for display
|
||||
metadata.Contributors = utils.NormalizeContributors(metadata.Contributors)
|
||||
@@ -529,6 +589,7 @@ func (s *MediaScanner) processMediaFile(ctx context.Context, path string) (bool,
|
||||
Tags: metadata.Tags,
|
||||
TagsSearch: tagsSearch,
|
||||
AddedByAdminID: s.adminID,
|
||||
CreatedAt: pgtype.Timestamptz{Time: fileModTime, Valid: true},
|
||||
})
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("failed to create media item: %v", err)
|
||||
@@ -1387,13 +1448,46 @@ func isImageFile(filename string) bool {
|
||||
return ext == ".jpg" || ext == ".jpeg" || ext == ".png" || ext == ".gif"
|
||||
}
|
||||
|
||||
func (s *MediaScanner) updateMediaItem(ctx context.Context, mediaItemID pgtype.UUID, filePath string, info os.FileInfo) error {
|
||||
// Update disabled - scanner creates items but doesn't update
|
||||
return nil
|
||||
func (s *MediaScanner) updateMediaItem(ctx context.Context, mediaItemID pgtype.UUID, path string, info os.FileInfo) error {
|
||||
// Re-extract metadata for the update
|
||||
metadata, err := s.extractMetadata(path)
|
||||
if err != nil {
|
||||
fmt.Printf("Warning: failed to extract metadata for force rescan %s: %v\n", path, err)
|
||||
metadata = &MediaMetadata{}
|
||||
}
|
||||
|
||||
// Normalize metadata fields
|
||||
metadata.Contributors = utils.NormalizeContributors(metadata.Contributors)
|
||||
metadata.Tags = utils.NormalizeTags(metadata.Tags)
|
||||
contributorsSearch := utils.NormalizeContributorsSearch(metadata.Contributors)
|
||||
tagsSearch := utils.NormalizeTagsSearch(metadata.Tags)
|
||||
|
||||
// Call the database update - only update fields available in MediaMetadata
|
||||
_, err = s.db.UpdateMediaItem(ctx, database.UpdateMediaItemParams{
|
||||
ID: mediaItemID,
|
||||
Title: metadata.Title,
|
||||
Author: pgtype.Text{String: metadata.Author, Valid: metadata.Author != ""},
|
||||
Isbn: pgtype.Text{String: utils.NormalizeISBNSafe(metadata.ISBN), Valid: metadata.ISBN != ""},
|
||||
Description: pgtype.Text{String: metadata.Description, Valid: metadata.Description != ""},
|
||||
CoverImagePath: pgtype.Text{String: metadata.CoverPath, Valid: metadata.CoverPath != ""},
|
||||
Series: pgtype.Text{String: metadata.Series, Valid: metadata.Series != ""},
|
||||
SeriesNumber: pgtype.Int4{Int32: metadata.SeriesNumber, Valid: metadata.SeriesNumber > 0},
|
||||
Tags: metadata.Tags,
|
||||
TagsSearch: tagsSearch,
|
||||
Asin: pgtype.Text{String: metadata.ASIN, Valid: metadata.ASIN != ""},
|
||||
DatePublished: pgtype.Date{Time: metadata.PublishDate, Valid: !metadata.PublishDate.IsZero()},
|
||||
Publisher: pgtype.Text{String: metadata.Publisher, Valid: metadata.Publisher != ""},
|
||||
Contributors: metadata.Contributors,
|
||||
ContributorsSearch: contributorsSearch,
|
||||
})
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *MediaScanner) getMediaItemByFilePath(ctx context.Context, filePath string) (database.MediaItems, error) {
|
||||
return s.db.GetMediaItemByFilePath(ctx, filePath)
|
||||
func (s *MediaScanner) getMediaItemByFilePath(ctx context.Context, filePath string, libraryID pgtype.UUID) (database.MediaItems, error) {
|
||||
return s.db.GetMediaItemByFilePath(ctx, database.GetMediaItemByFilePathParams{
|
||||
FilePath: filePath,
|
||||
LibraryID: libraryID,
|
||||
})
|
||||
}
|
||||
|
||||
func (s *MediaScanner) getMimeType(path string) string {
|
||||
@@ -1454,6 +1548,55 @@ func (s *MediaScanner) WatchChanges(ctx context.Context) {
|
||||
}
|
||||
}
|
||||
|
||||
// Handle file deletions
|
||||
if event.Has(fsnotify.Remove) && s.isScannableFile(event.Name) {
|
||||
s.logger.LogDelete(fmt.Sprintf("[WATCH-DELETE] File removed from filesystem: %s", event.Name))
|
||||
|
||||
// Determine libraryID for this file
|
||||
var libraryID pgtype.UUID
|
||||
for _, folder := range s.folders {
|
||||
if strings.HasPrefix(event.Name, folder) {
|
||||
lib, err := s.db.GetLibraryByFolder(ctx, folder)
|
||||
if err == nil {
|
||||
libraryID = lib.LibraryID
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
if !libraryID.Valid {
|
||||
msg := fmt.Sprintf("[WATCH-DELETE] WARNING: could not determine library for deleted file: %s", event.Name)
|
||||
s.logger.LogDelete(msg)
|
||||
s.logger.LogError(msg)
|
||||
return
|
||||
}
|
||||
|
||||
// Look up media item BEFORE deleting - log for safety
|
||||
existingItem, err := s.db.GetMediaItemByFilePath(ctx, database.GetMediaItemByFilePathParams{
|
||||
FilePath: event.Name,
|
||||
LibraryID: libraryID,
|
||||
})
|
||||
if err == nil {
|
||||
msg := fmt.Sprintf("[WATCH-DELETE] Found media item to delete: ID=%s, Title=%s, Path=%s",
|
||||
existingItem.ID, existingItem.Title, existingItem.FilePath)
|
||||
s.logger.LogDelete(msg)
|
||||
|
||||
if err := s.db.DeleteMediaItem(ctx, existingItem.ID); err != nil {
|
||||
errMsg := fmt.Sprintf("[WATCH-DELETE] ERROR: failed to delete media item %s: %v", existingItem.ID, err)
|
||||
s.logger.LogDelete(errMsg)
|
||||
s.logger.LogError(errMsg)
|
||||
} else {
|
||||
s.logger.LogDelete(fmt.Sprintf("[WATCH-DELETE] SUCCESS: deleted media item '%s' (was at %s)",
|
||||
existingItem.Title, existingItem.FilePath))
|
||||
}
|
||||
} else if err != pgx.ErrNoRows {
|
||||
errMsg := fmt.Sprintf("[WATCH-DELETE] ERROR: failed to look up media item for %s: %v", event.Name, err)
|
||||
s.logger.LogDelete(errMsg)
|
||||
s.logger.LogError(errMsg)
|
||||
} else {
|
||||
s.logger.LogDelete(fmt.Sprintf("[WATCH-DELETE] No media item found in database for deleted file: %s", event.Name))
|
||||
}
|
||||
}
|
||||
|
||||
case err, ok := <-s.watcher.Errors:
|
||||
if !ok {
|
||||
return
|
||||
|
||||
@@ -0,0 +1,107 @@
|
||||
package services
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
logDir = "/app/logs"
|
||||
maxLogAgeDays = 7
|
||||
)
|
||||
|
||||
type ScannerLogger struct {
|
||||
deletesFile *os.File
|
||||
errorsFile *os.File
|
||||
currentDate string
|
||||
}
|
||||
|
||||
func NewScannerLogger() *ScannerLogger {
|
||||
return &ScannerLogger{}
|
||||
}
|
||||
|
||||
func (l *ScannerLogger) ensureLogFiles() error {
|
||||
today := time.Now().Format("2006-01-02")
|
||||
|
||||
if l.currentDate == today && l.deletesFile != nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
if l.deletesFile != nil {
|
||||
l.deletesFile.Close()
|
||||
}
|
||||
if l.errorsFile != nil {
|
||||
l.errorsFile.Close()
|
||||
}
|
||||
|
||||
if err := os.MkdirAll(logDir, 0755); err != nil {
|
||||
return fmt.Errorf("failed to create log directory: %v", err)
|
||||
}
|
||||
|
||||
deletesPath := filepath.Join(logDir, fmt.Sprintf("scanner-deletes-%s.log", today))
|
||||
errorsPath := filepath.Join(logDir, fmt.Sprintf("scanner-errors-%s.log", today))
|
||||
|
||||
deletesFile, err := os.OpenFile(deletesPath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to open deletes log file: %v", err)
|
||||
}
|
||||
|
||||
errorsFile, err := os.OpenFile(errorsPath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
|
||||
if err != nil {
|
||||
deletesFile.Close()
|
||||
return fmt.Errorf("failed to open errors log file: %v", err)
|
||||
}
|
||||
|
||||
l.deletesFile = deletesFile
|
||||
l.errorsFile = errorsFile
|
||||
l.currentDate = today
|
||||
|
||||
l.cleanupOldLogs()
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (l *ScannerLogger) cleanupOldLogs() {
|
||||
cutoff := time.Now().AddDate(0, 0, -maxLogAgeDays)
|
||||
|
||||
filepath.Walk(logDir, func(path string, info os.FileInfo, walkErr error) error {
|
||||
if walkErr != nil {
|
||||
return walkErr
|
||||
}
|
||||
if !info.IsDir() && info.ModTime().Before(cutoff) {
|
||||
os.Remove(path)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
func (l *ScannerLogger) LogDelete(message string) {
|
||||
if err := l.ensureLogFiles(); err != nil {
|
||||
fmt.Printf("ERROR: Failed to ensure log files: %v\n", err)
|
||||
return
|
||||
}
|
||||
timestamp := time.Now().Format("2006-01-02 15:04:05")
|
||||
logLine := fmt.Sprintf("[%s] %s\n", timestamp, message)
|
||||
l.deletesFile.WriteString(logLine)
|
||||
}
|
||||
|
||||
func (l *ScannerLogger) LogError(message string) {
|
||||
if err := l.ensureLogFiles(); err != nil {
|
||||
fmt.Printf("ERROR: Failed to ensure log files: %v\n", err)
|
||||
return
|
||||
}
|
||||
timestamp := time.Now().Format("2006-01-02 15:04:05")
|
||||
logLine := fmt.Sprintf("[%s] %s\n", timestamp, message)
|
||||
l.errorsFile.WriteString(logLine)
|
||||
}
|
||||
|
||||
func (l *ScannerLogger) Close() {
|
||||
if l.deletesFile != nil {
|
||||
l.deletesFile.Close()
|
||||
}
|
||||
if l.errorsFile != nil {
|
||||
l.errorsFile.Close()
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user