Files
bookhoard/internal/services/media_scanner.go
T
john-okeefe 54bfd778db Refactor MediaScanner for improved file watching and job queue integration
- Replace event queue with dirty directories tracking (Jellyfin approach)
- Add file stability checking to wait for file writes to complete
- Add initial scan on startup to detect existing files
- Integrate with Worker job queue for directory scanning
- Change WatchChanges to return error and use atomic.Bool for state
- Add scan_mutex to prevent concurrent scans
- Add Close method with proper cleanup of resources
- Enhance polling with configurable interval
2026-03-05 16:28:40 -05:00

2281 lines
65 KiB
Go

package services
import (
"archive/tar"
"archive/zip"
"bookhoard/internal/database"
"bookhoard/internal/utils"
"bytes"
"compress/bzip2"
"compress/gzip"
"context"
"crypto/sha256"
"encoding/hex"
"encoding/xml"
"fmt"
"image"
_ "image/jpeg"
_ "image/png"
"io"
"io/fs"
"os"
"path/filepath"
"regexp"
"strconv"
"strings"
"sync"
"sync/atomic"
"time"
"bookhoard/internal/sevenzip"
epub "github.com/ArcadiaLin/go-epub"
"github.com/fsnotify/fsnotify"
"github.com/google/uuid"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgtype"
"github.com/nwaples/rardecode"
pdfcpuapi "github.com/pdfcpu/pdfcpu/pkg/api"
)
// MediaMetadata contains extracted metadata for media files (ebooks, comics, manga)
type MediaMetadata struct {
Title string
Author string
Description string
Series string
SeriesNumber int32
Publisher string
PublishDate time.Time
Contributors []string
CoverPath string
ISBN string
ASIN string
Tags []string
FileHashInfo *HashInfo
FileFormats []*FormatInfo
}
type HashInfo struct {
FileSHA256 string
OPFIdentifier string
OPFUUID string
HashConfidence string
}
type FormatInfo struct {
FormatType string
FilePath string
FileSHA256 string
FileSizeBytes int64
MimeType string
}
// MediaScanner scans library folders for media files (ebooks, comics, manga)
type MediaScanner struct {
db *database.Queries
watcher *fsnotify.Watcher
folders []string
adminID pgtype.UUID
defaultLibraryID pgtype.UUID
libraryTypes map[string][]string
forceRescan bool
logger *ScannerLogger
dirtyDirs map[string]time.Time
dirtyDirsMu sync.RWMutex
fileStability map[string]*atomic.Bool
fileStabilityMu sync.RWMutex
scan_mutex sync.Mutex
pollInterval time.Duration
watching atomic.Bool
totalFiles int
newItems int
errors int
job *Job
}
// NewMediaScanner creates a new media scanner instance
func NewMediaScanner(db *database.Queries) *MediaScanner {
watcher, err := fsnotify.NewWatcher()
if err != nil {
panic(fmt.Sprintf("Failed to create file watcher: %v", err))
}
return &MediaScanner{
db: db,
watcher: watcher,
dirtyDirs: make(map[string]time.Time),
fileStability: make(map[string]*atomic.Bool),
pollInterval: 60 * time.Second,
watching: atomic.Bool{},
folders: []string{},
adminID: pgtype.UUID{},
defaultLibraryID: pgtype.UUID{Valid: false},
libraryTypes: make(map[string][]string),
logger: NewScannerLogger(),
}
}
func (s *MediaScanner) GetPollInterval() time.Duration {
if s.db == nil {
return 30 * time.Second
}
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
setting, err := s.db.GetSystemSetting(ctx, "scan_poll_interval_seconds")
if err != nil || setting == "" {
return 30 * time.Second
}
seconds, err := strconv.Atoi(setting)
if err != nil || seconds < 1 {
return 30 * time.Second
}
return time.Duration(seconds) * time.Second
}
func (s *MediaScanner) GetAutoScanEnabled() bool {
if s.db == nil {
return true // default to enabled
}
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
setting, err := s.db.GetSystemSetting(ctx, "auto_scan_enabled")
if err != nil || setting == "" {
return true
}
enabled, err := strconv.ParseBool(setting)
if err != nil {
return true
}
return enabled
}
func (s *MediaScanner) SetAdminID(adminID pgtype.UUID) {
s.adminID = adminID
}
func (s *MediaScanner) SetLibraryID(libraryID pgtype.UUID) {
s.defaultLibraryID = libraryID
}
func (s *MediaScanner) SetForce(force bool) {
s.forceRescan = force
}
func (s *MediaScanner) GetStats() (int, int, int) {
return s.totalFiles, s.newItems, s.errors
}
func (s *MediaScanner) SetFolders(folders []string) error {
s.folders = folders
// Remove old watch if exists
if s.watcher != nil {
s.watcher.Close()
}
// Create new watcher
watcher, err := fsnotify.NewWatcher()
if err != nil {
return fmt.Errorf("failed to create watcher: %v", err)
}
s.watcher = watcher
// Build cache of allowed extensions per folder
s.libraryTypes = make(map[string][]string)
ctx := context.Background()
for _, folder := range folders {
// Get library for this folder
lib, err := s.db.GetLibraryByFolder(ctx, folder)
if err != nil {
fmt.Printf("Warning: failed to get library for folder %s: %v\n", folder, err)
continue
}
// Get library type with allowed extensions
libType, err := s.db.GetLibraryType(ctx, lib.LibraryTypeID)
if err != nil {
fmt.Printf("Warning: failed to get library type for %s: %v\n", folder, err)
continue
}
// Cache allowed extensions for this folder
s.libraryTypes[folder] = libType.AllowedExtensions
fmt.Printf("Scanner: Folder %s (type: %s) allows extensions: %v\n",
folder, libType.Name, libType.AllowedExtensions)
}
// Add all folders to watch
for _, folder := range folders {
if err := s.watcher.Add(folder); err != nil {
fmt.Printf("Warning: failed to watch folder %s: %v\n", folder, err)
}
}
return nil
}
func (s *MediaScanner) ScanFolders(ctx context.Context) error {
if len(s.folders) == 0 {
return fmt.Errorf("no folders set")
}
s.totalFiles = 0
s.newItems = 0
s.errors = 0
for _, folder := range s.folders {
filepath.WalkDir(folder, func(path string, d fs.DirEntry, err error) error {
if !d.IsDir() && s.isScannableFile(path) {
s.totalFiles++
}
return nil
})
}
fmt.Printf("Starting scan of %d folders: %v (%d files to scan)\n", len(s.folders), s.folders, s.totalFiles)
processedFiles := 0
mediaFiles := 0
for _, folder := range s.folders {
fmt.Printf("Scanning folder: %s\n", folder)
if _, err := os.Stat(folder); os.IsNotExist(err) {
fmt.Printf("Folder does not exist: %s\n", folder)
s.errors++
continue
}
err := filepath.WalkDir(folder, func(path string, d fs.DirEntry, err error) error {
if err != nil {
fmt.Printf("Error accessing path %s: %v\n", path, err)
s.errors++
return err
}
if d.IsDir() {
if err := s.watcher.Add(path); err != nil {
fmt.Printf("Warning: failed to watch subdirectory %s: %v\n", path, err)
}
return nil
}
if s.isScannableFile(path) {
mediaFiles++
processedFiles++
if processedFiles%10 == 0 && s.totalFiles > 0 {
progress := float64(processedFiles) / float64(s.totalFiles)
if s.job != nil {
s.job.UpdateProgress(progress, processedFiles, s.newItems, s.errors)
}
}
wasNew, err := s.processMediaFile(ctx, path)
if err != nil {
fmt.Printf("Error processing media file %s: %v\n", path, err)
s.errors++
} else if wasNew {
fmt.Printf("Successfully processed media file: %s\n", path)
} else {
fmt.Printf("Successfully processed media file: %s\n", path)
}
}
return nil
})
if err != nil {
s.errors++
return fmt.Errorf("failed to scan folder %s: %v", folder, err)
}
}
fmt.Printf("Scan completed: %d total files scanned, %d media files found, %d new items, %d errors\n",
processedFiles, mediaFiles, s.newItems, s.errors)
// Clean up: Find media items in DB that no longer exist on filesystem
for _, folder := range s.folders {
lib, err := s.db.GetLibraryByFolder(ctx, folder)
if err != nil {
continue
}
libraryID := lib.LibraryID
dbItems, err := s.db.ListMediaItemsByLibrary(ctx, libraryID)
if err != nil {
fmt.Printf("Warning: failed to get library items for cleanup: %v\n", err)
continue
}
// Build set of scanned file paths for this folder
scannedPaths := make(map[string]bool)
filepath.WalkDir(folder, func(path string, d os.DirEntry, walkErr error) error {
if walkErr != nil {
return walkErr
}
if !d.IsDir() && s.isScannableFile(path) {
scannedPaths[s.getRelativePath(path)] = true
}
return nil
})
// Delete items whose files no longer exist - with safety logging
for _, item := range dbItems {
filePath := item.FilePath
if filePath != "" && !scannedPaths[filePath] {
msg := fmt.Sprintf("[RESCAN-CLEANUP] Orphaned media item found: ID=%s, Title=%s, Path=%s",
item.ID, item.Title, filePath)
s.logger.LogDelete(msg)
delMsg := fmt.Sprintf("[RESCAN-CLEANUP] Deleting orphaned item '%s' (file no longer exists at %s)",
item.Title, filePath)
s.logger.LogDelete(delMsg)
if err := s.db.DeleteMediaItem(ctx, item.ID); err != nil {
errMsg := fmt.Sprintf("[RESCAN-CLEANUP] ERROR: failed to delete orphaned item %s: %v", item.Title, err)
s.logger.LogDelete(errMsg)
s.logger.LogError(errMsg)
} else {
s.logger.LogDelete(fmt.Sprintf("[RESCAN-CLEANUP] SUCCESS: deleted orphaned item '%s'", item.Title))
}
}
}
}
if s.job != nil && s.totalFiles > 0 {
s.job.UpdateProgress(1.0, processedFiles, s.newItems, s.errors)
}
return nil
}
func (s *MediaScanner) isScannableFile(path string) bool {
ext := strings.ToLower(filepath.Ext(path))
// Find which folder this file belongs to
var folder string
for _, f := range s.folders {
if strings.HasPrefix(path, f) {
folder = f
break
}
}
// If no folder match, don't scan
if folder == "" {
return false
}
// Get allowed extensions for this folder's library
allowed, ok := s.libraryTypes[folder]
if !ok {
// No library type info, skip file
fmt.Printf("Warning: No library type info for folder %s, skipping %s\n", folder, path)
return false
}
// Check if file extension is allowed for this library type
for _, allowedExt := range allowed {
if ext == strings.ToLower(allowedExt) {
return true
}
}
return false
}
// extractFolderStructureMetadata extracts metadata from folder paths, prioritizing Calibre structure
func (s *MediaScanner) extractFolderStructureMetadata(path, rootFolder string) *MediaMetadata {
metadata := &MediaMetadata{}
// Get the relative path from root folder
relPath, err := filepath.Rel(rootFolder, path)
if err != nil {
return metadata
}
// Split into directory components
dir := filepath.Dir(relPath)
components := strings.Split(dir, string(filepath.Separator))
if len(components) < 2 {
return metadata // Not enough structure to extract
}
// Calibre structure detection
// Pattern 1: Author Name/Book Title/
// Pattern 2: Author Name/Series Name/Book Title/
// Pattern 3: Author Name/Series Name, Book #1 - Book Title/
author := strings.TrimSuffix(components[0], "_") // Remove trailing underscore if present
metadata.Author = strings.ReplaceAll(author, "_", " ")
if len(components) >= 3 {
// This might be a series structure
possibleSeries := components[1]
possibleTitle := components[2]
// Check for Calibre series format: "Series Name, Book #1 - Title"
seriesMatch := regexp.MustCompile(`^(.*),\s+Book\s+#(\d+)\s*-\s*(.*)$`).FindStringSubmatch(possibleSeries)
if len(seriesMatch) == 4 {
metadata.Series = strings.ReplaceAll(seriesMatch[1], "_", " ")
if seriesNum, err := strconv.ParseInt(seriesMatch[2], 10, 32); err == nil {
metadata.SeriesNumber = int32(seriesNum)
}
metadata.Title = strings.ReplaceAll(possibleTitle, "_", " ")
} else {
// Simple series structure: Author/Series/Title
metadata.Series = strings.ReplaceAll(possibleSeries, "_", " ")
metadata.Title = strings.ReplaceAll(possibleTitle, "_", " ")
// Try to extract series number from title
titleNumMatch := regexp.MustCompile(`^(.*)\s+(\d+)$`).FindStringSubmatch(metadata.Title)
if len(titleNumMatch) == 3 {
metadata.Title = titleNumMatch[1]
if seriesNum, err := strconv.ParseInt(titleNumMatch[2], 10, 32); err == nil {
metadata.SeriesNumber = int32(seriesNum)
}
}
}
} else {
// Simple structure: Author/Title
metadata.Title = strings.ReplaceAll(components[1], "_", " ")
}
return metadata
}
func (s *MediaScanner) processMediaFile(ctx context.Context, path string) (bool, error) {
fmt.Printf("Processing media file: %s\n", path)
// Get file info
info, err := os.Stat(path)
if err != nil {
fmt.Printf("Failed to get file info for %s: %v\n", path, err)
return false, fmt.Errorf("failed to get file info: %v", err)
}
fmt.Printf("File info for %s: size=%d\n", path, info.Size())
// Get file modification time for created_at
fileModTime := info.ModTime()
// Find library for this file's folder
var libraryID pgtype.UUID
if s.defaultLibraryID.Valid {
libraryID = s.defaultLibraryID
} else {
for _, folder := range s.folders {
if strings.HasPrefix(path, folder) {
lib, err := s.db.GetLibraryByFolder(ctx, folder)
if err != nil {
return false, fmt.Errorf("failed to find library for folder %s: %v", folder, err)
}
libraryID = lib.LibraryID
break
}
}
}
if !libraryID.Valid {
return false, fmt.Errorf("no library found for file path: %s", path)
}
// Check if media item already exists in database
existingItem, err := s.getMediaItemByFilePath(ctx, path, libraryID)
if err == nil {
fmt.Printf("Media item already exists in database: %s (size: %d vs %d)\n", path, existingItem.FileSize.Int64, info.Size())
// If force rescan is enabled, always re-process
if s.forceRescan {
fmt.Printf("Force rescan enabled, updating existing media item: %s\n", path)
// Use UPDATE instead of DELETE+INSERT to preserve created_at
if err := s.updateMediaItem(ctx, existingItem.ID, path, info); err != nil {
fmt.Printf("Warning: failed to update existing media item: %v\n", err)
}
return false, nil
} else {
// Normal behavior: check if file has changed (by size)
if existingItem.FileSize.Int64 != info.Size() {
fmt.Printf("File size changed, updating media item: %s\n", path)
_ = s.updateMediaItem(ctx, existingItem.ID, path, info)
return false, nil
}
fmt.Printf("Media item already exists with same size, skipping: %s\n", path)
return false, nil
}
} else if err != pgx.ErrNoRows {
fmt.Printf("Database error checking media item existence: %v\n", err)
return false, fmt.Errorf("failed to check if media item exists: %v", err)
}
fmt.Printf("Media item does not exist in database, creating new entry: %s\n", path)
// Extract metadata from file first
metadata, err := s.extractMetadata(path)
if err != nil {
fmt.Printf("Warning: failed to extract metadata from %s: %v\n", path, err)
metadata = &MediaMetadata{}
}
// Extract hash information during metadata extraction
hashInfo, formatInfo, err := s.extractHashInfo(path)
if err != nil {
fmt.Printf("Warning: failed to extract hash info from %s: %v\n", path, err)
hashInfo = &HashInfo{}
formatInfo = &FormatInfo{}
} else {
metadata.FileHashInfo = hashInfo
metadata.FileFormats = []*FormatInfo{formatInfo}
fmt.Printf("Hash info for %s: SHA256=%s, OPF_ID=%s, OPF_UUID=%s, Confidence=%s\n",
path, hashInfo.FileSHA256, hashInfo.OPFIdentifier, hashInfo.OPFUUID, hashInfo.HashConfidence)
}
var comicInfo *ComicInfo
var coverImage []byte
ext := strings.ToLower(filepath.Ext(path))
if ext == ".cbz" || ext == ".cbr" || ext == ".cb7" || ext == ".cbt" ||
strings.HasSuffix(strings.ToLower(path), ".tar.gz") ||
strings.HasSuffix(strings.ToLower(path), ".tar.bz2") ||
strings.HasSuffix(strings.ToLower(path), ".tgz") ||
strings.HasSuffix(strings.ToLower(path), ".tbz2") {
info, cover, err := extractComicMetadata(path)
if err != nil {
fmt.Printf("Warning: failed to extract comic metadata from %s: %v\n", path, err)
} else {
comicInfo = info
coverImage = cover
if comicInfo.Title != "" && metadata.Title == "" {
metadata.Title = comicInfo.Title
}
if comicInfo.Series != "" && metadata.Series == "" {
metadata.Series = comicInfo.Series
}
if comicInfo.Number > 0 && metadata.SeriesNumber == 0 {
metadata.SeriesNumber = int32(comicInfo.Number)
}
if comicInfo.Publisher != "" && metadata.Publisher == "" {
metadata.Publisher = comicInfo.Publisher
}
if comicInfo.Writer != "" && metadata.Author == "" {
metadata.Author = comicInfo.Writer
}
if len(coverImage) > 0 && metadata.CoverPath == "" {
coverPath := path + ".cover.jpg"
if err := os.WriteFile(coverPath, coverImage, 0644); err == nil {
metadata.CoverPath = s.getRelativePath(coverPath)
}
}
fmt.Printf("Extracted comic metadata from %s: title=%s, series=%s, issue=%d\n",
path, comicInfo.Title, comicInfo.Series, comicInfo.Number)
}
}
// Try to get metadata from folder structure as fallback/enhancement
// Use the root folder that contains this file
var rootFolder string
for _, folder := range s.folders {
if strings.HasPrefix(path, folder) {
rootFolder = folder
break
}
}
if rootFolder != "" {
folderMetadata := s.extractFolderStructureMetadata(path, rootFolder)
// Use folder metadata as fallback for missing information
if metadata.Title == "" && folderMetadata.Title != "" {
metadata.Title = folderMetadata.Title
}
if metadata.Author == "" && folderMetadata.Author != "" {
metadata.Author = folderMetadata.Author
}
if metadata.Series == "" && folderMetadata.Series != "" {
metadata.Series = folderMetadata.Series
}
if metadata.SeriesNumber == 0 && folderMetadata.SeriesNumber > 0 {
metadata.SeriesNumber = folderMetadata.SeriesNumber
}
}
// Final fallback if still missing essential metadata
if metadata.Title == "" {
metadata.Title = strings.TrimSuffix(filepath.Base(path), filepath.Ext(path))
}
if metadata.Author == "" {
metadata.Author = "Unknown"
}
// libraryID already determined at start of function
// Normalize metadata fields for display
metadata.Contributors = utils.NormalizeContributors(metadata.Contributors)
metadata.Tags = utils.NormalizeTags(metadata.Tags)
// Normalize search fields
contributorsSearch := utils.NormalizeContributorsSearch(metadata.Contributors)
tagsSearch := utils.NormalizeTagsSearch(metadata.Tags)
// Create media item in database
relativePath := s.getRelativePath(path)
createdItem, err := s.db.CreateMediaItem(ctx, database.CreateMediaItemParams{
LibraryID: libraryID,
Title: metadata.Title,
Author: pgtype.Text{String: metadata.Author, Valid: metadata.Author != ""},
Isbn: pgtype.Text{String: utils.NormalizeISBNSafe(metadata.ISBN), Valid: metadata.ISBN != ""},
Asin: pgtype.Text{String: metadata.ASIN, Valid: metadata.ASIN != ""},
Description: pgtype.Text{String: metadata.Description, Valid: metadata.Description != ""},
FilePath: relativePath,
FileSize: pgtype.Int8{Int64: info.Size(), Valid: true},
MimeType: pgtype.Text{String: s.getMimeType(path), Valid: true},
CoverImagePath: pgtype.Text{String: metadata.CoverPath, Valid: metadata.CoverPath != ""},
Series: pgtype.Text{String: metadata.Series, Valid: metadata.Series != ""},
SeriesNumber: pgtype.Int4{Int32: metadata.SeriesNumber, Valid: metadata.SeriesNumber > 0},
Publisher: pgtype.Text{String: metadata.Publisher, Valid: metadata.Publisher != ""},
DatePublished: pgtype.Date{Time: metadata.PublishDate, Valid: !metadata.PublishDate.IsZero()},
Contributors: metadata.Contributors,
ContributorsSearch: contributorsSearch,
Tags: metadata.Tags,
TagsSearch: tagsSearch,
AddedByAdminID: s.adminID,
CreatedAt: pgtype.Timestamptz{Time: fileModTime, Valid: true},
})
if err != nil {
return false, fmt.Errorf("failed to create media item: %v", err)
}
// Update hash information before database storage
if metadata.FileHashInfo != nil && metadata.FileHashInfo.FileSHA256 != "" {
_, err = s.db.UpdateMediaItemIdentifiers(ctx, database.UpdateMediaItemIdentifiersParams{
ID: createdItem.ID,
FileSha256: pgtype.Text{String: metadata.FileHashInfo.FileSHA256, Valid: true},
OpfIdentifier: pgtype.Text{String: metadata.FileHashInfo.OPFIdentifier, Valid: metadata.FileHashInfo.OPFIdentifier != ""},
OpfUuid: pgtype.Text{String: metadata.FileHashInfo.OPFUUID, Valid: metadata.FileHashInfo.OPFUUID != ""},
HashConfidence: pgtype.Text{String: metadata.FileHashInfo.HashConfidence, Valid: true},
})
if err != nil {
fmt.Printf("Warning: failed to update hash identifiers for %s: %v\n", path, err)
}
}
// Store format information in the database
for _, format := range metadata.FileFormats {
_, err = s.db.CreateMediaItemFormat(ctx, database.CreateMediaItemFormatParams{
MediaItemID: createdItem.ID,
FormatType: format.FormatType,
FilePath: pgtype.Text{String: s.getRelativePath(format.FilePath), Valid: true},
FileSha256: pgtype.Text{String: format.FileSHA256, Valid: true},
FileSizeBytes: pgtype.Int8{Int64: format.FileSizeBytes, Valid: true},
MimeType: pgtype.Text{String: format.MimeType, Valid: true},
})
if err != nil {
fmt.Printf("Warning: failed to create format entry for %s: %v\n", path, err)
}
}
s.newItems++
return true, nil
}
func (s *MediaScanner) extractMetadata(path string) (*MediaMetadata, error) {
ext := strings.ToLower(filepath.Ext(path))
switch ext {
case ".epub":
metadata, err := s.extractEPUBMetadata(path)
if err != nil {
return metadata, err
}
// Try to extract embedded cover
coverPath, err := s.extractEPUBCover(path)
if err != nil {
fmt.Printf("Warning: failed to extract EPUB cover from %s: %v\n", path, err)
} else if coverPath != "" {
metadata.CoverPath = s.getRelativePath(coverPath)
}
// If no embedded cover, try sidecar
if metadata.CoverPath == "" {
sidecarCover := findSidecarCover(path)
if sidecarCover != "" {
metadata.CoverPath = s.getRelativePath(sidecarCover)
}
}
return metadata, nil
case ".pdf":
return s.extractPDFMetadata(path)
default:
// For other formats, return basic metadata
return &MediaMetadata{
Title: strings.TrimSuffix(filepath.Base(path), ext),
}, nil
}
}
func (s *MediaScanner) extractEPUBMetadata(path string) (*MediaMetadata, error) {
book, err := epub.ReadBook(path)
if err != nil {
return nil, fmt.Errorf("failed to open EPUB: %v", err)
}
metadata := &MediaMetadata{}
// Title
if title, err := book.Title(); err == nil && title != "" {
metadata.Title = title
}
// Author
if authors, err := book.MetadataByKey("creator"); err == nil && len(authors) > 0 {
metadata.Author = authors[0]
}
// Description
if descriptions, err := book.MetadataByKey("description"); err == nil && len(descriptions) > 0 {
metadata.Description = descriptions[0]
}
// Publisher
if publishers, err := book.MetadataByKey("publisher"); err == nil && len(publishers) > 0 {
metadata.Publisher = publishers[0]
}
// Series and series number (Calibre specific metadata)
if series, err := book.MetadataByKey("calibre:series"); err == nil && len(series) > 0 {
metadata.Series = series[0]
}
if seriesIndex, err := book.MetadataByKey("calibre:series_index"); err == nil && len(seriesIndex) > 0 {
if index, err := strconv.ParseFloat(seriesIndex[0], 32); err == nil {
metadata.SeriesNumber = int32(index)
}
}
// Publish date
if dates, err := book.MetadataByKey("date"); err == nil && len(dates) > 0 {
if date, err := time.Parse("2006-01-02", dates[0]); err == nil {
metadata.PublishDate = date
} else {
// Try alternative date formats
if date, err := time.Parse("2006", dates[0]); err == nil {
metadata.PublishDate = date
}
}
}
// Contributors
if contributors, err := book.MetadataByKey("contributor"); err == nil && len(contributors) > 0 {
// Normalize contributors for display
metadata.Contributors = utils.NormalizeContributors(contributors)
}
// ISBN
if isbns, err := book.MetadataByKey("identifier"); err == nil && len(isbns) > 0 {
for _, isbn := range isbns {
if strings.Contains(strings.ToLower(isbn), "isbn") {
// Extract ISBN number from identifier like "isbn:978-3-16-148410-0"
isbnParts := strings.SplitN(isbn, ":", 2)
if len(isbnParts) == 2 {
metadata.ISBN = isbnParts[1]
break
}
}
if strings.Contains(strings.ToLower(isbn), "asin") {
// Extract ASIN from identifier like "asin:B08XXXXX"
asinParts := strings.SplitN(isbn, ":", 2)
if len(asinParts) == 2 {
metadata.ASIN = asinParts[1]
}
}
}
}
// Tags
if tags, err := book.MetadataByKey("subject"); err == nil && len(tags) > 0 {
// Normalize tags for display
metadata.Tags = utils.NormalizeTags(tags)
}
return metadata, nil
}
// extractEPUBCover extracts the cover image from an EPUB file.
// It looks for:
// 1. An item with properties="cover-image" in the manifest
// 2. A meta tag with name="cover" pointing to an image
// 3. Common cover image paths like cover.jpg, cover.jpeg, cover.png
// Returns the path to the saved cover image, or empty string if no cover found.
func (s *MediaScanner) extractEPUBCover(epubPath string) (string, error) {
// Open the EPUB as a zip file to extract images
r, err := zip.OpenReader(epubPath)
if err != nil {
return "", fmt.Errorf("failed to open EPUB as zip: %v", err)
}
defer r.Close()
// Try to find cover image from OPF metadata
coverImageName := ""
// Attempt to read the OPF file to find cover reference
// First, find container.xml to locate the OPF
var opfPath string
for _, f := range r.File {
if f.Name == "META-INF/container.xml" {
rc, err := f.Open()
if err != nil {
continue
}
content, err := io.ReadAll(rc)
rc.Close()
if err != nil {
continue
}
// Parse container.xml to find OPF path
// Simple string search since we just need the path
opfStart := bytes.Index(content, []byte("<rootfile "))
if opfStart == -1 {
continue
}
opfStartAttr := bytes.Index(content[opfStart:], []byte("full-path="))
if opfStartAttr == -1 {
continue
}
opfStartAttr += len("full-path=")
quote := content[opfStart+opfStartAttr]
opfStartQuote := opfStart + opfStartAttr + 1
opfEndQuote := bytes.Index(content[opfStartQuote:], []byte{byte(quote)})
if opfEndQuote == -1 {
continue
}
opfPath = string(content[opfStartQuote : opfStartQuote+opfEndQuote])
break
}
}
if opfPath == "" {
// No OPF found, try common cover image paths
coverImageName = findCoverImageInZip(r.File)
} else {
// Read OPF to find cover reference
opfContent, err := readFileFromZip(r.File, opfPath)
if err != nil {
// Fall back to common paths
coverImageName = findCoverImageInZip(r.File)
} else {
coverImageName = findCoverInOPF(opfContent, r.File, opfPath)
}
}
if coverImageName == "" {
return "", nil // No cover found
}
// Extract the cover image
coverImage, err := extractImageFromZip(r.File, coverImageName, opfPath)
if err != nil {
return "", fmt.Errorf("failed to extract cover image: %v", err)
}
if len(coverImage) == 0 {
return "", nil
}
// Save cover to disk (same pattern as comics: {book_path}.cover.jpg)
coverPath := epubPath + ".cover.jpg"
if err := os.WriteFile(coverPath, coverImage, 0644); err != nil {
return "", fmt.Errorf("failed to write cover file: %v", err)
}
return coverPath, nil
}
// findCoverImageInZip searches for common cover image filenames in the zip
func findCoverImageInZip(files []*zip.File) string {
coverNames := []string{"cover.jpg", "cover.jpeg", "cover.png", "cover.webp",
"Cover.jpg", "Cover.jpeg", "Cover.png", "Cover.webp",
"images/cover.jpg", "Images/cover.jpg", "OEBPS/images/cover.jpg"}
for _, name := range coverNames {
for _, f := range files {
if strings.EqualFold(f.Name, name) {
return f.Name
}
}
}
return ""
}
// findCoverInOPF parses OPF content to find cover image reference
func findCoverInOPF(opfContent []byte, files []*zip.File, opfDir string) string {
contentStr := string(opfContent)
// Look for item with properties="cover-image"
coverImageRE := regexp.MustCompile(`<item[^>]*properties="[^"]*cover-image[^"]*"[^>]*id="([^"]+)"`)
matches := coverImageRE.FindStringSubmatch(contentStr)
if len(matches) > 1 {
coverID := matches[1]
// Find the href for this ID
hrefRE := regexp.MustCompile(fmt.Sprintf(`<item[^>]*id="%s"[^>]*href="([^"]+)"`, coverID))
hrefMatches := hrefRE.FindStringSubmatch(contentStr)
if len(hrefMatches) > 1 {
return resolveOPFPath(opfDir, hrefMatches[1])
}
}
// Look for meta name="cover"
metaCoverRE := regexp.MustCompile(`<meta[^>]*name="cover"[^>]*content="([^"]+)"`)
metaMatches := metaCoverRE.FindStringSubmatch(contentStr)
if len(metaMatches) > 1 {
coverContent := metaMatches[1]
// Could be "image-id" format
if strings.HasPrefix(coverContent, "image-") {
coverID := strings.TrimPrefix(coverContent, "image-")
hrefRE := regexp.MustCompile(fmt.Sprintf(`<item[^>]*id="%s"[^>]*href="([^"]+)"`, coverID))
hrefMatches := hrefRE.FindStringSubmatch(contentStr)
if len(hrefMatches) > 1 {
return resolveOPFPath(opfDir, hrefMatches[1])
}
}
}
// Fall back to searching common paths
return findCoverImageInZip(files)
}
// resolveOPFPath resolves a relative path against the OPF directory
func resolveOPFPath(opfDir, href string) string {
if opfDir == "" {
return href
}
// Handle ../ in href
if strings.HasPrefix(href, "../") {
// Simple case: just use the href as-is for now
return href
}
// Join the directory with the href
return filepath.Join(filepath.Dir(opfDir), href)
}
// readFileFromZip reads a file from the zip by name
func readFileFromZip(files []*zip.File, name string) ([]byte, error) {
// Normalize the name for comparison
name = filepath.ToSlash(name)
for _, f := range files {
fName := filepath.ToSlash(f.Name)
if fName == name {
rc, err := f.Open()
if err != nil {
return nil, err
}
defer rc.Close()
return io.ReadAll(rc)
}
}
return nil, fmt.Errorf("file not found: %s", name)
}
// extractImageFromZip extracts an image file and returns its contents
func extractImageFromZip(files []*zip.File, imagePath, opfDir string) ([]byte, error) {
// Try direct match first
for _, f := range files {
if strings.EqualFold(f.Name, imagePath) {
rc, err := f.Open()
if err != nil {
return nil, err
}
defer rc.Close()
return io.ReadAll(rc)
}
}
// Try resolved path
resolvedPath := resolveOPFPath(opfDir, imagePath)
for _, f := range files {
if strings.EqualFold(f.Name, resolvedPath) {
rc, err := f.Open()
if err != nil {
return nil, err
}
defer rc.Close()
return io.ReadAll(rc)
}
}
return nil, fmt.Errorf("image not found: %s", imagePath)
}
// findSidecarCover looks for cover images in the same directory as the media file.
// It checks for common cover filename patterns in priority order:
// 1. cover.jpg, cover.jpeg, cover.png, cover.webp
// 2. folder.jpg, folder.jpeg, folder.png, folder.webp
// 3. {basename}.jpg, {basename}.jpeg, etc. (same name as media file)
// 4. .folder.jpg (hidden file)
// Returns the full path to the cover file, or empty string if not found.
func findSidecarCover(mediaPath string) string {
dir := filepath.Dir(mediaPath)
baseName := strings.TrimSuffix(filepath.Base(mediaPath), filepath.Ext(mediaPath))
// Priority order for cover filenames
coverPatterns := []string{
"cover.jpg",
"cover.jpeg",
"cover.png",
"cover.webp",
"folder.jpg",
"folder.jpeg",
"folder.png",
"folder.webp",
".folder.jpg",
".folder.jpeg",
".folder.png",
}
// First, check exact match cover/folder names
for _, coverName := range coverPatterns {
coverPath := filepath.Join(dir, coverName)
if _, err := os.Stat(coverPath); err == nil {
return coverPath
}
}
// Second, check for {basename}.{ext} pattern
extensions := []string{".jpg", ".jpeg", ".png", ".webp"}
for _, ext := range extensions {
coverPath := filepath.Join(dir, baseName+ext)
if _, err := os.Stat(coverPath); err == nil {
return coverPath
}
// Also check uppercase extension
coverPathUpper := filepath.Join(dir, baseName+strings.ToUpper(ext))
if _, err := os.Stat(coverPathUpper); err == nil {
return coverPathUpper
}
}
return ""
}
func (s *MediaScanner) extractPDFMetadata(path string) (*MediaMetadata, error) {
metadata := &MediaMetadata{}
// Open PDF file for reading metadata
f, err := os.Open(path)
if err != nil {
fmt.Printf("Warning: failed to open PDF file %s: %v\n", path, err)
metadata.Title = strings.TrimSuffix(filepath.Base(path), ".pdf")
return metadata, nil
}
defer f.Close()
// Use pdfcpu API to read PDF metadata
// Configuration: nil = default (lenient mode)
pdfInfo, err := pdfcpuapi.PDFInfo(f, filepath.Base(path), nil, nil)
if err != nil {
fmt.Printf("Warning: failed to read PDF info from %s: %v\n", path, err)
// Fall back to filename as title
metadata.Title = strings.TrimSuffix(filepath.Base(path), ".pdf")
return metadata, nil
}
// Extract title
if pdfInfo.Title != "" {
metadata.Title = pdfInfo.Title
} else {
metadata.Title = strings.TrimSuffix(filepath.Base(path), ".pdf")
}
// Extract author
if pdfInfo.Author != "" {
metadata.Author = pdfInfo.Author
}
// Extract subject (use as description)
if pdfInfo.Subject != "" {
metadata.Description = pdfInfo.Subject
}
// Extract creator (use as author fallback)
if pdfInfo.Creator != "" && metadata.Author == "" {
metadata.Author = pdfInfo.Creator
}
// Extract producer (use as publisher)
if pdfInfo.Producer != "" {
metadata.Publisher = pdfInfo.Producer
}
// Try to extract cover image
coverPath, err := s.extractPDFCover(path)
if err != nil {
fmt.Printf("Warning: failed to extract PDF cover from %s: %v\n", path, err)
} else if coverPath != "" {
metadata.CoverPath = s.getRelativePath(coverPath)
}
// If no embedded cover, try sidecar
if metadata.CoverPath == "" {
sidecarCover := findSidecarCover(path)
if sidecarCover != "" {
metadata.CoverPath = s.getRelativePath(sidecarCover)
}
}
return metadata, nil
}
// extractPDFCover extracts a cover image from a PDF file.
// It uses pdfcpu API to extract images from the first page.
// Returns the path to the saved cover, or empty string if no cover found.
func (s *MediaScanner) extractPDFCover(pdfPath string) (string, error) {
// Create a temporary directory for extracted images
tmpDir, err := os.MkdirTemp("", "pdf-cover-")
if err != nil {
return "", fmt.Errorf("failed to create temp dir: %v", err)
}
defer os.RemoveAll(tmpDir)
// Use pdfcpu API to extract images from first page
// ExtractImagesFile(inFile, outDir string, selectedPages []string, conf *model.Configuration) error
err = pdfcpuapi.ExtractImagesFile(pdfPath, tmpDir, []string{"1"}, nil)
if err != nil {
// No images found or extraction failed - this is OK, just return empty
return "", nil
}
// Check for extracted images in the temp directory
entries, err := os.ReadDir(tmpDir)
if err != nil || len(entries) == 0 {
return "", nil
}
// Find the largest image (likely the cover)
var largestImage string
var largestSize int64
for _, entry := range entries {
if entry.IsDir() {
continue
}
info, err := entry.Info()
if err != nil {
continue
}
// Skip very small files (likely thumbnails or icons)
if info.Size() < 1000 {
continue
}
if info.Size() > largestSize {
largestImage = filepath.Join(tmpDir, entry.Name())
largestSize = info.Size()
}
}
if largestImage == "" {
return "", nil
}
// Read the image
imageData, err := os.ReadFile(largestImage)
if err != nil || len(imageData) == 0 {
return "", nil
}
// Save cover to disk (same pattern as comics: {pdf_path}.cover.jpg)
coverPath := pdfPath + ".cover.jpg"
if err := os.WriteFile(coverPath, imageData, 0644); err != nil {
return "", fmt.Errorf("failed to write cover file: %v", err)
}
return coverPath, nil
}
// ComicInfo represents metadata from ComicInfo.xml
type ComicInfo struct {
XMLName xml.Name `xml:"ComicInfo"`
Title string `xml:"Title"`
Series string `xml:"Series"`
Number int `xml:"Number"`
Volume int `xml:"Volume"`
Publisher string `xml:"Publisher"`
Year int `xml:"Year"`
Month int `xml:"Month"`
Day int `xml:"Day"`
Writer string `xml:"Writer"`
Penciller string `xml:"Penciller"`
Inker string `xml:"Inker"`
Colorist string `xml:"Colorist"`
Letterer string `xml:"Letterer"`
CoverArtist string `xml:"CoverArtist"`
Genre string `xml:"Genre"`
Tags string `xml:"Tags"`
Web string `xml:"Web"`
Notes string `xml:"Notes"`
}
// extractComicMetadata extracts metadata from comic archive (supports .cbz, .cbr, .cb7, .cbt)
func extractComicMetadata(filePath string) (*ComicInfo, []byte, error) {
ext := strings.ToLower(filepath.Ext(filePath))
switch ext {
case ".cbz":
return extractZipMetadata(filePath)
case ".cbr":
return extractRarMetadata(filePath)
case ".cb7":
return extract7ZipMetadata(filePath)
case ".cbt":
return extractTarMetadata(filePath)
default:
return nil, nil, fmt.Errorf("unsupported comic format: %s", ext)
}
}
// archiveFile represents a file in an archive for uniform handling
type archiveFile interface {
Name() string
Open() (io.ReadCloser, error)
}
// extractMetadataFromArchive extracts ComicInfo.xml and cover image from any archive format
func extractMetadataFromArchive(files []archiveFile) (*ComicInfo, []byte, error) {
var comicInfo *ComicInfo
var coverImage []byte
for _, f := range files {
if f.Name() == "ComicInfo.xml" {
rc, err := f.Open()
if err != nil {
return nil, nil, fmt.Errorf("failed to open ComicInfo.xml: %w", err)
}
data, err := io.ReadAll(rc)
rc.Close()
if err != nil {
return nil, nil, fmt.Errorf("failed to read ComicInfo.xml: %w", err)
}
comicInfo = &ComicInfo{}
if err := xml.Unmarshal(data, comicInfo); err != nil {
return nil, nil, fmt.Errorf("failed to parse ComicInfo.xml: %w", err)
}
}
if coverImage == nil && isImageFile(f.Name()) {
if !strings.Contains(filepath.Dir(f.Name()), string(filepath.Separator)) ||
filepath.Dir(f.Name()) == "." {
rc, err := f.Open()
if err != nil {
continue
}
coverImage, err = io.ReadAll(rc)
rc.Close()
if err == nil {
_, _, err = image.Decode(bytes.NewReader(coverImage))
if err != nil {
coverImage = nil
}
}
}
}
}
if comicInfo == nil {
comicInfo = &ComicInfo{}
}
return comicInfo, coverImage, nil
}
// extractZipMetadata extracts metadata from ZIP archives (.cbz)
func extractZipMetadata(filePath string) (*ComicInfo, []byte, error) {
r, err := zip.OpenReader(filePath)
if err != nil {
return nil, nil, fmt.Errorf("failed to open ZIP archive: %w", err)
}
defer r.Close()
files := make([]archiveFile, 0, len(r.File))
for _, f := range r.File {
files = append(files, &zipFileAdapter{f})
}
comicInfo, coverImage, err := extractMetadataFromArchive(files)
if err != nil {
return nil, nil, err
}
if comicInfo.Title == "" {
basename := filepath.Base(filePath)
comicInfo.Title = strings.TrimSuffix(basename, filepath.Ext(basename))
}
return comicInfo, coverImage, nil
}
// zipFileAdapter adapts zip.File to archiveFile interface
type zipFileAdapter struct {
*zip.File
}
func (z *zipFileAdapter) Name() string {
return z.File.Name
}
func (z *zipFileAdapter) Open() (io.ReadCloser, error) {
return z.File.Open()
}
// extractRarMetadata extracts metadata from RAR archives (.cbr)
func extractRarMetadata(filePath string) (*ComicInfo, []byte, error) {
r, err := rardecode.OpenReader(filePath, "")
if err != nil {
return nil, nil, fmt.Errorf("failed to open RAR archive: %w", err)
}
defer r.Close()
files := make([]archiveFile, 0, 100)
for {
header, err := r.Next()
if err == io.EOF {
break
}
if err != nil {
return nil, nil, fmt.Errorf("failed to read RAR entry: %w", err)
}
data, err := io.ReadAll(r)
if err != nil {
continue
}
files = append(files, &rarFileAdapter{
name: header.Name,
data: data,
})
}
comicInfo, coverImage, err := extractMetadataFromArchive(files)
if err != nil {
return nil, nil, err
}
if comicInfo.Title == "" {
basename := filepath.Base(filePath)
comicInfo.Title = strings.TrimSuffix(basename, filepath.Ext(basename))
}
return comicInfo, coverImage, nil
}
// rarFileAdapter stores RAR file data in memory
type rarFileAdapter struct {
name string
data []byte
}
func (r *rarFileAdapter) Name() string {
return r.name
}
func (r *rarFileAdapter) Open() (io.ReadCloser, error) {
return io.NopCloser(bytes.NewReader(r.data)), nil
}
// extract7ZipMetadata extracts metadata from 7-Zip archives (.cb7)
func extract7ZipMetadata(filePath string) (*ComicInfo, []byte, error) {
r, err := sevenzip.OpenReader(filePath)
if err != nil {
return nil, nil, fmt.Errorf("failed to open 7-Zip archive: %w", err)
}
defer r.Close()
files := make([]archiveFile, 0, len(r.File))
for _, f := range r.File {
files = append(files, &sevenZipFileAdapter{file: f})
}
comicInfo, coverImage, err := extractMetadataFromArchive(files)
if err != nil {
return nil, nil, err
}
if comicInfo.Title == "" {
basename := filepath.Base(filePath)
comicInfo.Title = strings.TrimSuffix(basename, filepath.Ext(basename))
}
return comicInfo, coverImage, nil
}
// sevenZipFileAdapter adapts sevenzip.File to archiveFile interface
type sevenZipFileAdapter struct {
file *sevenzip.File
}
func (s *sevenZipFileAdapter) Name() string {
return s.file.Name
}
func (s *sevenZipFileAdapter) Open() (io.ReadCloser, error) {
return s.file.Open()
}
// extractTarMetadata extracts metadata from TAR archives (.cbt)
func extractTarMetadata(filePath string) (*ComicInfo, []byte, error) {
f, err := os.Open(filePath)
if err != nil {
return nil, nil, fmt.Errorf("failed to open TAR archive: %w", err)
}
defer f.Close()
var tarReader *tar.Reader
if strings.HasSuffix(strings.ToLower(filePath), ".tar.gz") ||
strings.HasSuffix(strings.ToLower(filePath), ".tgz") {
gzReader, err := gzip.NewReader(f)
if err != nil {
return nil, nil, fmt.Errorf("failed to create gzip reader: %w", err)
}
defer gzReader.Close()
tarReader = tar.NewReader(gzReader)
} else if strings.HasSuffix(strings.ToLower(filePath), ".tar.bz2") ||
strings.HasSuffix(strings.ToLower(filePath), ".tbz2") {
bz2Reader := bzip2.NewReader(f)
tarReader = tar.NewReader(bz2Reader)
} else {
tarReader = tar.NewReader(f)
}
files := make([]archiveFile, 0, 100)
for {
header, err := tarReader.Next()
if err == io.EOF {
break
}
if err != nil {
return nil, nil, fmt.Errorf("failed to read TAR entry: %w", err)
}
if header.Typeflag == tar.TypeReg {
data, err := io.ReadAll(tarReader)
if err != nil {
continue
}
files = append(files, &tarFileAdapter{
name: header.Name,
data: data,
})
}
}
comicInfo, coverImage, err := extractMetadataFromArchive(files)
if err != nil {
return nil, nil, err
}
if comicInfo.Title == "" {
basename := filepath.Base(filePath)
comicInfo.Title = strings.TrimSuffix(basename, filepath.Ext(basename))
}
return comicInfo, coverImage, nil
}
// tarFileAdapter stores TAR file data in memory
type tarFileAdapter struct {
name string
data []byte
}
func (t *tarFileAdapter) Name() string {
return t.name
}
func (t *tarFileAdapter) Open() (io.ReadCloser, error) {
return io.NopCloser(bytes.NewReader(t.data)), nil
}
// isImageFile checks if a file is an image based on extension
func isImageFile(filename string) bool {
ext := strings.ToLower(filepath.Ext(filename))
return ext == ".jpg" || ext == ".jpeg" || ext == ".png" || ext == ".gif"
}
func (s *MediaScanner) updateMediaItem(ctx context.Context, mediaItemID pgtype.UUID, path string, info os.FileInfo) error {
// Re-extract metadata for the update
metadata, err := s.extractMetadata(path)
if err != nil {
fmt.Printf("Warning: failed to extract metadata for force rescan %s: %v\n", path, err)
metadata = &MediaMetadata{}
}
// Normalize metadata fields
metadata.Contributors = utils.NormalizeContributors(metadata.Contributors)
metadata.Tags = utils.NormalizeTags(metadata.Tags)
contributorsSearch := utils.NormalizeContributorsSearch(metadata.Contributors)
tagsSearch := utils.NormalizeTagsSearch(metadata.Tags)
// Call the database update - only update fields available in MediaMetadata
_, err = s.db.UpdateMediaItem(ctx, database.UpdateMediaItemParams{
ID: mediaItemID,
Title: metadata.Title,
Author: pgtype.Text{String: metadata.Author, Valid: metadata.Author != ""},
Isbn: pgtype.Text{String: utils.NormalizeISBNSafe(metadata.ISBN), Valid: metadata.ISBN != ""},
Description: pgtype.Text{String: metadata.Description, Valid: metadata.Description != ""},
CoverImagePath: pgtype.Text{String: s.getRelativePath(metadata.CoverPath), Valid: metadata.CoverPath != ""},
Series: pgtype.Text{String: metadata.Series, Valid: metadata.Series != ""},
SeriesNumber: pgtype.Int4{Int32: metadata.SeriesNumber, Valid: metadata.SeriesNumber > 0},
Tags: metadata.Tags,
TagsSearch: tagsSearch,
Asin: pgtype.Text{String: metadata.ASIN, Valid: metadata.ASIN != ""},
DatePublished: pgtype.Date{Time: metadata.PublishDate, Valid: !metadata.PublishDate.IsZero()},
Publisher: pgtype.Text{String: metadata.Publisher, Valid: metadata.Publisher != ""},
Contributors: metadata.Contributors,
ContributorsSearch: contributorsSearch,
})
return err
}
func (s *MediaScanner) getMediaItemByFilePath(ctx context.Context, filePath string, libraryID pgtype.UUID) (database.MediaItems, error) {
return s.db.GetMediaItemByFilePath(ctx, database.GetMediaItemByFilePathParams{
FilePath: s.getRelativePath(filePath),
LibraryID: libraryID,
})
}
func (s *MediaScanner) getMimeType(path string) string {
ext := strings.ToLower(filepath.Ext(path))
if mime, ok := MimeTypes[ext]; ok {
return mime
}
return "application/octet-stream"
}
func (s *MediaScanner) WatchChanges(ctx context.Context) error {
// Prevent duplicate calls
if !s.watching.CompareAndSwap(false, true) {
return fmt.Errorf("already watching")
}
// Reset flag when context is cancelled
go func() {
<-ctx.Done()
s.watching.Store(false)
}()
// Perform initial scan of all root folders
go s.performInitialScan(ctx)
// Start directory processor
go s.processDirtyDirectories(ctx)
// Start polling fallback
go s.StartPolling(ctx)
// Handle fsnotify events - queue them for debouncing
go func() {
for {
select {
case event, ok := <-s.watcher.Events:
if !ok {
return
}
// Handle new directories - add them to the watcher
if event.Has(fsnotify.Create) {
if info, err := os.Stat(event.Name); err == nil && info.IsDir() {
s.watcher.Add(event.Name)
}
}
// Mark directory dirty for ANY file change
if event.Has(fsnotify.Create | fsnotify.Write | fsnotify.Remove | fsnotify.Chmod | fsnotify.Rename) {
s.markDirectoryDirty(filepath.Dir(event.Name))
}
case err, ok := <-s.watcher.Errors:
if !ok {
return
}
fmt.Printf("Watcher error: %v\n", err)
case <-ctx.Done():
return
}
}
}()
return nil
}
func (s *MediaScanner) handleFileAdd(ctx context.Context, filePath string) {
fmt.Printf("New/modified media file detected: %s\n", filePath)
if _, err := s.processMediaFile(ctx, filePath); err != nil {
fmt.Printf("Error processing modified media file %s: %v\n", filePath, err)
}
}
func (s *MediaScanner) handleFileDelete(ctx context.Context, filePath string) {
s.logger.LogDelete(fmt.Sprintf("[WATCH-DELETE] File removed from filesystem: %s", filePath))
// Determine libraryID for this file
var libraryID pgtype.UUID
for _, folder := range s.folders {
if strings.HasPrefix(filePath, folder) {
lib, err := s.db.GetLibraryByFolder(ctx, folder)
if err == nil {
libraryID = lib.LibraryID
break
}
}
}
if !libraryID.Valid {
msg := fmt.Sprintf("[WATCH-DELETE] WARNING: could not determine library for deleted file: %s", filePath)
s.logger.LogDelete(msg)
s.logger.LogError(msg)
return
}
// Look up media item
existingItem, err := s.db.GetMediaItemByFilePath(ctx, database.GetMediaItemByFilePathParams{
FilePath: s.getRelativePath(filePath),
LibraryID: libraryID,
})
if err == nil {
msg := fmt.Sprintf("[WATCH-DELETE] Found media item to delete: ID=%s, Title=%s, Path=%s",
existingItem.ID, existingItem.Title, existingItem.FilePath)
s.logger.LogDelete(msg)
if err := s.db.DeleteMediaItem(ctx, existingItem.ID); err != nil {
errMsg := fmt.Sprintf("[WATCH-DELETE] ERROR: failed to delete media item %s: %v", existingItem.ID, err)
s.logger.LogDelete(errMsg)
s.logger.LogError(errMsg)
} else {
s.logger.LogDelete(fmt.Sprintf("[WATCH-DELETE] SUCCESS: deleted media item '%s' (was at %s)",
existingItem.Title, existingItem.FilePath))
}
} else if err != pgx.ErrNoRows {
errMsg := fmt.Sprintf("[WATCH-DELETE] ERROR: failed to look up media item for %s: %v", filePath, err)
s.logger.LogDelete(errMsg)
s.logger.LogError(errMsg)
} else {
s.logger.LogDelete(fmt.Sprintf("[WATCH-DELETE] No media item found in database for deleted file: %s", filePath))
}
}
func (s *MediaScanner) markDirectoryDirty(dirPath string) {
s.dirtyDirsMu.Lock()
defer s.dirtyDirsMu.Unlock()
// Only mark if within watched folders
var isWatched bool
for _, folder := range s.folders {
if strings.HasPrefix(dirPath, folder) {
isWatched = true
break
}
}
if !isWatched {
return
}
// Smart event merging (Jellyfin approach):
// 1. If parent dir exists, replace with parent (consolidate)
// 2. If sibling dirs exist, replace with common parent
// 3. Otherwise, add this dir
// Check if parent directory is already dirty
parentDir := filepath.Dir(dirPath)
if parentDir != dirPath { // Not at root
if _, parentExists := s.dirtyDirs[parentDir]; parentExists {
// Parent already being watched, reset its timestamp
s.dirtyDirs[parentDir] = time.Now()
return
}
}
// Check if any subdirectories are dirty, replace with parent
for existingDir := range s.dirtyDirs {
if strings.HasPrefix(existingDir, dirPath+"/") {
// This is a subdirectory, replace it with parent
delete(s.dirtyDirs, existingDir)
}
}
// NEW: Check for sibling directories and consolidate to parent
parentDir = filepath.Dir(dirPath)
for existingDir := range s.dirtyDirs {
existingParent := filepath.Dir(existingDir)
if existingParent == parentDir && existingParent != dirPath && existingParent != "." {
// Found a sibling! Both should be replaced with parent
delete(s.dirtyDirs, existingDir)
s.dirtyDirs[parentDir] = time.Now()
return
}
}
// Add/update this directory
s.dirtyDirs[dirPath] = time.Now()
}
func (s *MediaScanner) processDirtyDirectories(ctx context.Context) {
ticker := time.NewTicker(1 * time.Second)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
return
case <-ticker.C:
s.dirtyDirsMu.Lock()
now := time.Now()
readyDirs := make([]string, 0)
// Find directories that haven't been modified in 10 seconds
// This batches changes together (Audiobookshelf approach)
for dirPath, lastChange := range s.dirtyDirs {
if now.Sub(lastChange) >= 10*time.Second {
readyDirs = append(readyDirs, dirPath)
delete(s.dirtyDirs, dirPath)
}
}
s.dirtyDirsMu.Unlock()
// Process all ready directories in a batch via job queue
// Job queue serializes scans - prevents concurrent directory access
if len(readyDirs) > 0 {
for _, dirPath := range readyDirs {
// Create directory scan job with correct params for processDirectoryScanJob()
job := &Job{
ID: uuid.New().String(),
Type: JobTypeDirectoryScan,
Params: map[string]interface{}{
"directory": dirPath,
"db": s.db,
},
Status: JobStatusPending,
}
// Enqueue via global worker singleton
if WorkerInstance != nil {
WorkerInstance.Enqueue(job)
fmt.Printf("Enqueued directory scan job: %s\n", dirPath)
} else {
fmt.Printf("Warning: Worker not initialized, skipping directory scan: %s\n", dirPath)
}
}
}
}
}
}
// waitForFileStability checks if a file's mtime has stabilized
// Returns true when file is stable (not being modified)
// Polls every 3 seconds, times out after 60 seconds
// Uses atomic.Bool to prevent race conditions with concurrent checks
func (s *MediaScanner) waitForFileStability(filePath string) bool {
s.fileStabilityMu.Lock()
// Check if already being checked (atomic.Bool prevents race condition)
tracking, exists := s.fileStability[filePath]
if exists {
s.fileStabilityMu.Unlock()
// Another goroutine is already checking this file
if tracking.Load() {
return false // Still being checked
}
// Tracking exists but completed, remove stale entry
s.fileStabilityMu.Lock()
delete(s.fileStability, filePath)
}
// Start tracking with atomic.Bool set to true (checking in progress)
trackingFlag := &atomic.Bool{}
trackingFlag.Store(true)
s.fileStability[filePath] = trackingFlag
s.fileStabilityMu.Unlock()
// Get initial mtime
info, err := os.Stat(filePath)
if err != nil {
// Clean up tracking entry if file doesn't exist
s.fileStabilityMu.Lock()
delete(s.fileStability, filePath)
s.fileStabilityMu.Unlock()
return false
}
lastMtime := info.ModTime()
// Poll every 3 seconds for up to 60 seconds
timeout := time.After(60 * time.Second)
ticker := time.NewTicker(3 * time.Second)
defer ticker.Stop()
for {
select {
case <-timeout:
// Timeout - mark as done and clean up
trackingFlag.Store(false)
s.fileStabilityMu.Lock()
delete(s.fileStability, filePath)
s.fileStabilityMu.Unlock()
return false // File never stabilized
case <-ticker.C:
info, err := os.Stat(filePath)
if err != nil {
// File deleted - mark as done and clean up
trackingFlag.Store(false)
s.fileStabilityMu.Lock()
delete(s.fileStability, filePath)
s.fileStabilityMu.Unlock()
return false
}
currentMtime := info.ModTime()
if currentMtime.Equal(lastMtime) {
// File is stable! Mark as done and clean up
trackingFlag.Store(false)
s.fileStabilityMu.Lock()
delete(s.fileStability, filePath)
s.fileStabilityMu.Unlock()
return true
}
lastMtime = currentMtime
}
}
}
func (s *MediaScanner) scanDirectory(ctx context.Context, dirPath string) {
// Prevent concurrent scans of ANY directory
// Simple mutex is enough - job queue already serializes by directory
s.scan_mutex.Lock()
defer s.scan_mutex.Unlock()
// Find library for this directory
var libraryID pgtype.UUID
var rootFolder string
for _, folder := range s.folders {
if strings.HasPrefix(dirPath, folder) {
rootFolder = folder
if lib, err := s.db.GetLibraryByFolder(ctx, folder); err == nil {
libraryID = lib.LibraryID
break
}
}
}
// Check if libraryID is valid before proceeding
if !libraryID.Valid {
return
}
// Walk directory and process new files
filepath.WalkDir(dirPath, func(path string, d os.DirEntry, err error) error {
if err != nil {
return err
}
if d.IsDir() {
return filepath.SkipDir
}
if !s.isScannableFile(path) {
return nil
}
// Check if file is stable before processing (Audiobookshelf approach)
if !s.waitForFileStability(path) {
return nil
}
relPath := strings.TrimPrefix(path, rootFolder+"/")
_, err = s.db.GetMediaItemByFilePath(ctx, database.GetMediaItemByFilePathParams{
FilePath: relPath,
LibraryID: libraryID,
})
if err == pgx.ErrNoRows {
if _, err := s.processMediaFile(ctx, path); err != nil {
s.errors++
} else {
s.newItems++
}
s.totalFiles++
}
return nil
})
}
// performInitialScan scans all root folders on startup
// This ensures existing files are detected before watching begins
func (s *MediaScanner) performInitialScan(ctx context.Context) {
fmt.Printf("Performing initial scan of root folders...\n")
for _, folder := range s.folders {
// Skip if folder doesn't exist
if _, err := os.Stat(folder); os.IsNotExist(err) {
fmt.Printf("Skipping nonexistent folder: %s\n", folder)
continue
}
// Submit scan job to worker (non-blocking)
job := &Job{
ID: uuid.New().String(),
Type: JobTypeDirectoryScan,
Params: map[string]interface{}{
"directory": folder,
"db": s.db,
},
Status: JobStatusPending,
}
if WorkerInstance != nil {
WorkerInstance.Enqueue(job)
fmt.Printf("Enqueued initial scan job: %s\n", folder)
} else {
fmt.Printf("Warning: Worker not initialized, skipping initial scan: %s\n", folder)
}
}
fmt.Printf("Initial scan jobs enqueued\n")
}
func (s *MediaScanner) Close() error {
fmt.Printf("Cleaning up scanner resources...\n")
// Stop watching
if s.watcher != nil {
s.watcher.Close()
}
// Clean up fileStability map to prevent memory leaks
s.fileStabilityMu.Lock()
s.fileStability = make(map[string]*atomic.Bool)
s.fileStabilityMu.Unlock()
// Clear dirty directories
s.dirtyDirsMu.Lock()
s.dirtyDirs = make(map[string]time.Time)
s.dirtyDirsMu.Unlock()
// Wait for in-progress scan to complete (with timeout)
timeout := time.After(5 * time.Second)
done := make(chan struct{})
go func() {
s.scan_mutex.Lock()
s.scan_mutex.Unlock()
close(done)
}()
select {
case <-done:
fmt.Printf("Scanner cleanup complete\n")
case <-timeout:
fmt.Printf("Timeout waiting for scan to complete\n")
}
return nil
}
func (s *MediaScanner) StartPolling(ctx context.Context) {
interval := s.GetPollInterval()
if interval <= 0 {
fmt.Println("Polling fallback disabled (interval = 0")
return
}
fmt.Printf("Polling fallback started with interval: %v\n", interval)
for {
ticker := time.NewTicker(interval)
defer ticker.Stop()
select {
case <-ctx.Done():
fmt.Println("Polling fallback stopped")
return
case <-ticker.C:
//Re-read interval each tick for dynamic updates
interval = s.GetPollInterval()
fmt.Printf("Running polling fallback sync (interval: %v)...\n", interval)
if err := s.SyncFilesystemWithDatabase(ctx); err != nil {
fmt.Printf("Polling sync error: %v\n", err)
}
}
}
}
func (s *MediaScanner) SyncFilesystemWithDatabase(ctx context.Context) error {
fmt.Println("[POLL-SYNC] Starting filesystem sync with database")
for _, folder := range s.folders {
lib, err := s.db.GetLibraryByFolder(ctx, folder)
if err != nil {
fmt.Printf("[POLL-SYNC] Warning: failed to get library for folder %s: %v\n", folder, err)
continue
}
libraryID := lib.LibraryID
// Get all media items from database for this library
dbItems, err := s.db.ListMediaItemsByLibrary(ctx, libraryID)
if err != nil {
fmt.Printf("[POLL-SYNC] Warning: failed to get library items: %v\n", err)
continue
}
// Build set of existing file paths from filesystem
existingPaths := make(map[string]bool)
filepath.WalkDir(folder, func(path string, d os.DirEntry, walkErr error) error {
if walkErr != nil {
return walkErr
}
if !d.IsDir() && s.isScannableFile(path) {
existingPaths[s.getRelativePath(path)] = true
}
return nil
})
// Check for orphaned items (in DB but not on filesystem)
for _, item := range dbItems {
if item.FilePath != "" && !existingPaths[item.FilePath] {
msg := fmt.Sprintf("[POLL-SYNC] Orphaned media item found: ID=%s, Title=%s, Path=%s",
item.ID, item.Title, item.FilePath)
s.logger.LogDelete(msg)
if err := s.db.DeleteMediaItem(ctx, item.ID); err != nil {
errMsg := fmt.Sprintf("[POLL-SYNC] ERROR: failed to delete orphaned item %s: %v", item.Title, err)
s.logger.LogDelete(errMsg)
s.logger.LogError(errMsg)
} else {
s.logger.LogDelete(fmt.Sprintf("[POLL-SYNC] SUCCESS: deleted orphaned item '%s'", item.Title))
}
}
}
// Check for new files (on filesystem but not in DB)
// This is expensive, so we just check a few representative files
// The fsnotify handler should catch most new files
for relPath := range existingPaths {
// Check if this file exists in DB
_, err := s.db.GetMediaItemByFilePath(ctx, database.GetMediaItemByFilePathParams{
FilePath: relPath,
LibraryID: libraryID,
})
if err == pgx.ErrNoRows {
// New file found - scan it
absPath := folder + "/" + relPath
if _, err := os.Stat(absPath); err == nil {
fmt.Printf("[POLL-SYNC] New file detected, scanning: %s\n", absPath)
if _, err := s.processMediaFile(ctx, absPath); err != nil {
fmt.Printf("[POLL-SYNC] Error scanning new file %s: %v\n", absPath, err)
}
}
}
}
}
fmt.Println("[POLL-SYNC] Filesystem sync completed")
return nil
}
// ============================================
// SCANNER ENHANCEMENTS
// ============================================
// calculateFileSHA256 calculates SHA-256 hash using streaming to avoid loading entire file into memory
func (s *MediaScanner) calculateFileSHA256(filePath string) (string, error) {
file, err := os.Open(filePath)
if err != nil {
return "", fmt.Errorf("failed to open file: %v", err)
}
defer file.Close()
hasher := sha256.New()
if _, err := io.Copy(hasher, file); err != nil {
return "", fmt.Errorf("failed to calculate hash: %v", err)
}
return hex.EncodeToString(hasher.Sum(nil)), nil
}
// OPFIdentifier represents an identifier from OPF metadata
type OPFIdentifier struct {
XMLName xml.Name `xml:"identifier"`
ID string `xml:"id,attr"`
Scheme string `xml:"scheme,attr"`
Content string `xml:",chardata"`
}
// OPFMetadata represents parsed OPF metadata
type OPFMetadata struct {
XMLName xml.Name `xml:"package"`
Version string `xml:"version,attr"`
Identifiers []OPFIdentifier `xml:"metadata>identifier"`
}
// extractOPFIdentifiers extracts identifiers from EPUB OPF file
func (s *MediaScanner) extractOPFIdentifiers(epubPath string) (opfIdentifier, opfUUID string, confidence string, err error) {
book, err := epub.ReadBook(epubPath)
if err != nil {
return "", "", "", fmt.Errorf("failed to open EPUB: %v", err)
}
// Extract OPF identifiers using go-epub library
identifiers, err := book.MetadataByKey("identifier")
if err != nil || len(identifiers) == 0 {
return "", "", "low", nil
}
var identifier, uuid string
for _, id := range identifiers {
id = strings.TrimSpace(id)
// Check for UUID format (urn:uuid:)
if strings.HasPrefix(strings.ToLower(id), "urn:uuid:") {
uuid = strings.TrimPrefix(strings.ToLower(id), "urn:uuid:")
continue
}
// Check if it's a plain UUID (8-4-4-4-12 format)
if isValidUUID(id) {
uuid = id
continue
}
// Check for ISBN
if strings.Contains(strings.ToLower(id), "isbn") {
isbn := s.extractISBNFromIdentifier(id)
if isbn != "" {
identifier = isbn
continue
}
}
// Use first identifier as fallback
if identifier == "" && id != "" {
identifier = id
}
}
confidence = s.determineHashConfidence(uuid, identifier)
return identifier, uuid, confidence, nil
}
// isValidUUID checks if string is a valid UUID (8-4-4-4-12 format)
func isValidUUID(idStr string) bool {
uuidRegex := regexp.MustCompile(`^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$`)
return uuidRegex.MatchString(idStr)
}
// extractISBNFromIdentifier extracts ISBN from identifier string
func (s *MediaScanner) extractISBNFromIdentifier(id string) string {
id = strings.TrimSpace(id)
// Remove "isbn:" prefix if present
if strings.HasPrefix(strings.ToLower(id), "isbn:") {
id = strings.TrimPrefix(strings.ToLower(id), "isbn:")
}
// Remove hyphens and spaces
isbn := regexp.MustCompile(`[\s-]`).ReplaceAllString(id, "")
// Check if it's a valid ISBN-10 or ISBN-13
if len(isbn) == 10 || len(isbn) == 13 {
return isbn
}
return ""
}
// determineHashConfidence determines confidence level based on available identifiers
func (s *MediaScanner) determineHashConfidence(uuid, identifier string) string {
if uuid != "" && isValidUUID(uuid) {
return "high"
}
if identifier != "" && (strings.Contains(strings.ToLower(identifier), "isbn") || len(identifier) >= 10) {
return "medium"
}
return "low"
}
// detectFormatType detects the format type based on file extension and content
func (s *MediaScanner) detectFormatType(filePath string) string {
base := strings.ToLower(filepath.Base(filePath))
// Check for compound extensions first (like .kepub.epub)
if strings.HasSuffix(base, ".kepub.epub") {
return "kepub"
}
ext := strings.ToLower(filepath.Ext(filePath))
switch ext {
case ".epub":
return "epub"
case ".kepub":
return "kepub"
case ".pdf":
return "pdf"
case ".cbz", ".cbr", ".cb7", ".cbt":
return "comic_archive"
case ".mobi":
return "mobi"
case ".azw", ".azw3":
return "kfx"
case ".txt":
return "txt"
case ".fb2":
return "fb2"
default:
return "unknown"
}
}
// extractHashInfo calculates hash and extracts OPF identifiers for a file
func (s *MediaScanner) extractHashInfo(filePath string) (*HashInfo, *FormatInfo, error) {
// Calculate SHA-256
fileSHA256, err := s.calculateFileSHA256(filePath)
if err != nil {
return nil, nil, fmt.Errorf("failed to calculate SHA-256: %v", err)
}
// Get file info
info, err := os.Stat(filePath)
if err != nil {
return nil, nil, fmt.Errorf("failed to get file info: %v", err)
}
// Extract OPF identifiers for EPUB files
var opfIdentifier, opfUUID, confidence string
if strings.HasSuffix(strings.ToLower(filePath), ".epub") {
opfIdentifier, opfUUID, confidence, err = s.extractOPFIdentifiers(filePath)
if err != nil {
// Non-fatal error, continue with low confidence
confidence = "low"
}
}
hashInfo := &HashInfo{
FileSHA256: fileSHA256,
OPFIdentifier: opfIdentifier,
OPFUUID: opfUUID,
HashConfidence: confidence,
}
formatInfo := &FormatInfo{
FormatType: s.detectFormatType(filePath),
FilePath: filePath,
FileSHA256: fileSHA256,
FileSizeBytes: info.Size(),
MimeType: s.getMimeType(filePath),
}
return hashInfo, formatInfo, nil
}
func (s *MediaScanner) getRelativePath(absolutePath string) string {
// Get the base folder paths from scanner
for _, baseFolder := range s.folders {
// Check if path is within this base folder
if relPath, ok := strings.CutPrefix(absolutePath, baseFolder); ok {
return strings.TrimPrefix(relPath, "/")
}
}
// Fallback: if no match, return as-is (shouldn't happen)
return absolutePath
}