Files
bookhoard/internal/services/media_scanner.go
T
john-okeefe 821cd3df4c refactor(services): remove debug logging and fix directory scanning
- Remove debug printf statements from media scanner and worker
- Remove unused debug tracking variables (filesSeen, filesProcessed)
- Fix directory walk logic to properly scan the root directory itself
  (previous implementation would skip the root path entirely)

Clean up production code by removing debug artifacts and improving
the directory scanning logic to handle root-level directories correctly.
2026-03-06 10:48:36 -05:00

2307 lines
66 KiB
Go

package services
import (
"archive/tar"
"archive/zip"
"bookhoard/internal/database"
"bookhoard/internal/utils"
"bytes"
"compress/bzip2"
"compress/gzip"
"context"
"crypto/sha256"
"encoding/hex"
"encoding/xml"
"fmt"
"image"
_ "image/jpeg"
_ "image/png"
"io"
"io/fs"
"os"
"path/filepath"
"regexp"
"strconv"
"strings"
"sync"
"sync/atomic"
"time"
"bookhoard/internal/sevenzip"
epub "github.com/ArcadiaLin/go-epub"
"github.com/fsnotify/fsnotify"
"github.com/google/uuid"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgtype"
"github.com/nwaples/rardecode"
pdfcpuapi "github.com/pdfcpu/pdfcpu/pkg/api"
)
// MediaMetadata contains extracted metadata for media files (ebooks, comics, manga)
type MediaMetadata struct {
Title string
Author string
Description string
Series string
SeriesNumber int32
Publisher string
PublishDate time.Time
Contributors []string
CoverPath string
ISBN string
ASIN string
Tags []string
FileHashInfo *HashInfo
FileFormats []*FormatInfo
}
type HashInfo struct {
FileSHA256 string
OPFIdentifier string
OPFUUID string
HashConfidence string
}
type FormatInfo struct {
FormatType string
FilePath string
FileSHA256 string
FileSizeBytes int64
MimeType string
}
// MediaScanner scans library folders for media files (ebooks, comics, manga)
type MediaScanner struct {
db *database.Queries
watcher *fsnotify.Watcher
folders []string
adminID pgtype.UUID
defaultLibraryID pgtype.UUID
libraryTypes map[string][]string
forceRescan bool
logger *ScannerLogger
dirtyDirs map[string]time.Time
dirtyDirsMu sync.RWMutex
fileStability map[string]*atomic.Bool
fileStabilityMu sync.RWMutex
scan_mutex sync.Mutex
pollInterval time.Duration
watching atomic.Bool
settingsCache *SettingsCache
totalFiles int
newItems int
errors int
job *Job
}
// NewMediaScanner creates a new media scanner instance
func NewMediaScanner(db *database.Queries) *MediaScanner {
watcher, err := fsnotify.NewWatcher()
if err != nil {
panic(fmt.Sprintf("Failed to create file watcher: %v", err))
}
return &MediaScanner{
db: db,
watcher: watcher,
settingsCache: NewSettingsCache(30 * time.Second),
dirtyDirs: make(map[string]time.Time),
fileStability: make(map[string]*atomic.Bool),
pollInterval: 60 * time.Second,
watching: atomic.Bool{},
folders: []string{},
adminID: pgtype.UUID{},
defaultLibraryID: pgtype.UUID{Valid: false},
libraryTypes: make(map[string][]string),
logger: NewScannerLogger(),
}
}
func (s *MediaScanner) GetPollInterval() time.Duration {
// Check cache first
if cached, ok := s.settingsCache.Get("scan_poll_interval_seconds"); ok {
if seconds, err := strconv.Atoi(cached); err == nil {
return time.Duration(seconds) * time.Second
}
}
if s.db == nil {
return 60 * time.Second
}
// Cache miss - query database
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
setting, err := s.db.GetSystemSetting(ctx, "scan_poll_interval_seconds")
if err != nil || setting == "" {
return 60 * time.Second
}
// Store in cache
s.settingsCache.Set("scan_poll_interval_seconds", setting)
// Convert to duration
seconds, err := strconv.Atoi(setting)
if err != nil {
return 60 * time.Second
}
return time.Duration(seconds) * time.Second
}
func (s *MediaScanner) GetAutoScanEnabled() bool {
// Check cache first
if cached, ok := s.settingsCache.Get("auto_scan_enabled"); ok {
return strings.ToLower(cached) == "true"
}
if s.db == nil {
return true
}
// Cache miss - query database
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
setting, err := s.db.GetSystemSetting(ctx, "auto_scan_enabled")
if err != nil || setting == "" {
return true
}
// Store in cache
s.settingsCache.Set("auto_scan_enabled", setting)
return strings.ToLower(setting) == "true"
}
func (s *MediaScanner) SetAdminID(adminID pgtype.UUID) {
s.adminID = adminID
}
func (s *MediaScanner) SetLibraryID(libraryID pgtype.UUID) {
s.defaultLibraryID = libraryID
}
func (s *MediaScanner) SetForce(force bool) {
s.forceRescan = force
}
func (s *MediaScanner) GetStats() (int, int, int) {
return s.totalFiles, s.newItems, s.errors
}
func (s *MediaScanner) SetFolders(folders []string) error {
s.folders = folders
// Remove old watch if exists
if s.watcher != nil {
s.watcher.Close()
}
// Create new watcher
watcher, err := fsnotify.NewWatcher()
if err != nil {
return fmt.Errorf("failed to create watcher: %v", err)
}
s.watcher = watcher
// Build cache of allowed extensions per folder
s.libraryTypes = make(map[string][]string)
ctx := context.Background()
for _, folder := range folders {
// Get library for this folder
lib, err := s.db.GetLibraryByFolder(ctx, folder)
if err != nil {
fmt.Printf("Warning: failed to get library for folder %s: %v\n", folder, err)
continue
}
// Get library type with allowed extensions
libType, err := s.db.GetLibraryType(ctx, lib.LibraryTypeID)
if err != nil {
fmt.Printf("Warning: failed to get library type for %s: %v\n", folder, err)
continue
}
// Cache allowed extensions for this folder
s.libraryTypes[folder] = libType.AllowedExtensions
fmt.Printf("Scanner: Folder %s (type: %s) allows extensions: %v\n",
folder, libType.Name, libType.AllowedExtensions)
}
// Add all folders to watch
for _, folder := range folders {
if err := s.watcher.Add(folder); err != nil {
fmt.Printf("Warning: failed to watch folder %s: %v\n", folder, err)
}
}
return nil
}
func (s *MediaScanner) ScanFolders(ctx context.Context) error {
if len(s.folders) == 0 {
return fmt.Errorf("no folders set")
}
s.totalFiles = 0
s.newItems = 0
s.errors = 0
for _, folder := range s.folders {
filepath.WalkDir(folder, func(path string, d fs.DirEntry, err error) error {
if !d.IsDir() && s.isScannableFile(path) {
s.totalFiles++
}
return nil
})
}
fmt.Printf("Starting scan of %d folders: %v (%d files to scan)\n", len(s.folders), s.folders, s.totalFiles)
processedFiles := 0
mediaFiles := 0
for _, folder := range s.folders {
fmt.Printf("Scanning folder: %s\n", folder)
if _, err := os.Stat(folder); os.IsNotExist(err) {
fmt.Printf("Folder does not exist: %s\n", folder)
s.errors++
continue
}
err := filepath.WalkDir(folder, func(path string, d fs.DirEntry, err error) error {
if err != nil {
fmt.Printf("Error accessing path %s: %v\n", path, err)
s.errors++
return err
}
if d.IsDir() {
if err := s.watcher.Add(path); err != nil {
fmt.Printf("Warning: failed to watch subdirectory %s: %v\n", path, err)
}
return nil
}
if s.isScannableFile(path) {
mediaFiles++
processedFiles++
if processedFiles%10 == 0 && s.totalFiles > 0 {
progress := float64(processedFiles) / float64(s.totalFiles)
if s.job != nil {
s.job.UpdateProgress(progress, processedFiles, s.newItems, s.errors)
}
}
wasNew, err := s.processMediaFile(ctx, path)
if err != nil {
fmt.Printf("Error processing media file %s: %v\n", path, err)
s.errors++
} else if wasNew {
fmt.Printf("Successfully processed media file: %s\n", path)
} else {
fmt.Printf("Successfully processed media file: %s\n", path)
}
}
return nil
})
if err != nil {
s.errors++
return fmt.Errorf("failed to scan folder %s: %v", folder, err)
}
}
fmt.Printf("Scan completed: %d total files scanned, %d media files found, %d new items, %d errors\n",
processedFiles, mediaFiles, s.newItems, s.errors)
// Clean up: Find media items in DB that no longer exist on filesystem
for _, folder := range s.folders {
lib, err := s.db.GetLibraryByFolder(ctx, folder)
if err != nil {
continue
}
libraryID := lib.LibraryID
dbItems, err := s.db.ListMediaItemsByLibrary(ctx, libraryID)
if err != nil {
fmt.Printf("Warning: failed to get library items for cleanup: %v\n", err)
continue
}
// Build set of scanned file paths for this folder
scannedPaths := make(map[string]bool)
filepath.WalkDir(folder, func(path string, d os.DirEntry, walkErr error) error {
if walkErr != nil {
return walkErr
}
if !d.IsDir() && s.isScannableFile(path) {
scannedPaths[s.getRelativePath(path)] = true
}
return nil
})
// Delete items whose files no longer exist - with safety logging
for _, item := range dbItems {
filePath := item.FilePath
if filePath != "" && !scannedPaths[filePath] {
msg := fmt.Sprintf("[RESCAN-CLEANUP] Orphaned media item found: ID=%s, Title=%s, Path=%s",
item.ID, item.Title, filePath)
s.logger.LogDelete(msg)
delMsg := fmt.Sprintf("[RESCAN-CLEANUP] Deleting orphaned item '%s' (file no longer exists at %s)",
item.Title, filePath)
s.logger.LogDelete(delMsg)
if err := s.db.DeleteMediaItem(ctx, item.ID); err != nil {
errMsg := fmt.Sprintf("[RESCAN-CLEANUP] ERROR: failed to delete orphaned item %s: %v", item.Title, err)
s.logger.LogDelete(errMsg)
s.logger.LogError(errMsg)
} else {
s.logger.LogDelete(fmt.Sprintf("[RESCAN-CLEANUP] SUCCESS: deleted orphaned item '%s'", item.Title))
}
}
}
}
if s.job != nil && s.totalFiles > 0 {
s.job.UpdateProgress(1.0, processedFiles, s.newItems, s.errors)
}
return nil
}
func (s *MediaScanner) isScannableFile(path string) bool {
ext := strings.ToLower(filepath.Ext(path))
// Find which folder this file belongs to
var folder string
for _, f := range s.folders {
if strings.HasPrefix(path, f) {
folder = f
break
}
}
// If no folder match, don't scan
if folder == "" {
return false
}
// Get allowed extensions for this folder's library
allowed, ok := s.libraryTypes[folder]
if !ok {
// No library type info, skip file
fmt.Printf("Warning: No library type info for folder %s, skipping %s\n", folder, path)
return false
}
// Check if file extension is allowed for this library type
for _, allowedExt := range allowed {
if ext == strings.ToLower(allowedExt) {
return true
}
}
return false
}
// extractFolderStructureMetadata extracts metadata from folder paths, prioritizing Calibre structure
func (s *MediaScanner) extractFolderStructureMetadata(path, rootFolder string) *MediaMetadata {
metadata := &MediaMetadata{}
// Get the relative path from root folder
relPath, err := filepath.Rel(rootFolder, path)
if err != nil {
return metadata
}
// Split into directory components
dir := filepath.Dir(relPath)
components := strings.Split(dir, string(filepath.Separator))
if len(components) < 2 {
return metadata // Not enough structure to extract
}
// Calibre structure detection
// Pattern 1: Author Name/Book Title/
// Pattern 2: Author Name/Series Name/Book Title/
// Pattern 3: Author Name/Series Name, Book #1 - Book Title/
author := strings.TrimSuffix(components[0], "_") // Remove trailing underscore if present
metadata.Author = strings.ReplaceAll(author, "_", " ")
if len(components) >= 3 {
// This might be a series structure
possibleSeries := components[1]
possibleTitle := components[2]
// Check for Calibre series format: "Series Name, Book #1 - Title"
seriesMatch := regexp.MustCompile(`^(.*),\s+Book\s+#(\d+)\s*-\s*(.*)$`).FindStringSubmatch(possibleSeries)
if len(seriesMatch) == 4 {
metadata.Series = strings.ReplaceAll(seriesMatch[1], "_", " ")
if seriesNum, err := strconv.ParseInt(seriesMatch[2], 10, 32); err == nil {
metadata.SeriesNumber = int32(seriesNum)
}
metadata.Title = strings.ReplaceAll(possibleTitle, "_", " ")
} else {
// Simple series structure: Author/Series/Title
metadata.Series = strings.ReplaceAll(possibleSeries, "_", " ")
metadata.Title = strings.ReplaceAll(possibleTitle, "_", " ")
// Try to extract series number from title
titleNumMatch := regexp.MustCompile(`^(.*)\s+(\d+)$`).FindStringSubmatch(metadata.Title)
if len(titleNumMatch) == 3 {
metadata.Title = titleNumMatch[1]
if seriesNum, err := strconv.ParseInt(titleNumMatch[2], 10, 32); err == nil {
metadata.SeriesNumber = int32(seriesNum)
}
}
}
} else {
// Simple structure: Author/Title
metadata.Title = strings.ReplaceAll(components[1], "_", " ")
}
return metadata
}
func (s *MediaScanner) processMediaFile(ctx context.Context, path string) (bool, error) {
fmt.Printf("Processing media file: %s\n", path)
// Get file info
info, err := os.Stat(path)
if err != nil {
fmt.Printf("Failed to get file info for %s: %v\n", path, err)
return false, fmt.Errorf("failed to get file info: %v", err)
}
fmt.Printf("File info for %s: size=%d\n", path, info.Size())
// Get file modification time for created_at
fileModTime := info.ModTime()
// Find library for this file's folder
var libraryID pgtype.UUID
if s.defaultLibraryID.Valid {
libraryID = s.defaultLibraryID
} else {
for _, folder := range s.folders {
if strings.HasPrefix(path, folder) {
lib, err := s.db.GetLibraryByFolder(ctx, folder)
if err != nil {
return false, fmt.Errorf("failed to find library for folder %s: %v", folder, err)
}
libraryID = lib.LibraryID
break
}
}
}
if !libraryID.Valid {
return false, fmt.Errorf("no library found for file path: %s", path)
}
// Check if media item already exists in database
existingItem, err := s.getMediaItemByFilePath(ctx, path, libraryID)
if err == nil {
fmt.Printf("Media item already exists in database: %s (size: %d vs %d)\n", path, existingItem.FileSize.Int64, info.Size())
// If force rescan is enabled, always re-process
if s.forceRescan {
fmt.Printf("Force rescan enabled, updating existing media item: %s\n", path)
// Use UPDATE instead of DELETE+INSERT to preserve created_at
if err := s.updateMediaItem(ctx, existingItem.ID, path, info); err != nil {
fmt.Printf("Warning: failed to update existing media item: %v\n", err)
}
return false, nil
} else {
// Normal behavior: check if file has changed (by size)
if existingItem.FileSize.Int64 != info.Size() {
fmt.Printf("File size changed, updating media item: %s\n", path)
_ = s.updateMediaItem(ctx, existingItem.ID, path, info)
return false, nil
}
fmt.Printf("Media item already exists with same size, skipping: %s\n", path)
return false, nil
}
} else if err != pgx.ErrNoRows {
fmt.Printf("Database error checking media item existence: %v\n", err)
return false, fmt.Errorf("failed to check if media item exists: %v", err)
}
fmt.Printf("Media item does not exist in database, creating new entry: %s\n", path)
// Extract metadata from file first
metadata, err := s.extractMetadata(path)
if err != nil {
fmt.Printf("Warning: failed to extract metadata from %s: %v\n", path, err)
metadata = &MediaMetadata{}
}
// Extract hash information during metadata extraction
hashInfo, formatInfo, err := s.extractHashInfo(path)
if err != nil {
fmt.Printf("Warning: failed to extract hash info from %s: %v\n", path, err)
hashInfo = &HashInfo{}
formatInfo = &FormatInfo{}
} else {
metadata.FileHashInfo = hashInfo
metadata.FileFormats = []*FormatInfo{formatInfo}
fmt.Printf("Hash info for %s: SHA256=%s, OPF_ID=%s, OPF_UUID=%s, Confidence=%s\n",
path, hashInfo.FileSHA256, hashInfo.OPFIdentifier, hashInfo.OPFUUID, hashInfo.HashConfidence)
}
var comicInfo *ComicInfo
var coverImage []byte
ext := strings.ToLower(filepath.Ext(path))
if ext == ".cbz" || ext == ".cbr" || ext == ".cb7" || ext == ".cbt" ||
strings.HasSuffix(strings.ToLower(path), ".tar.gz") ||
strings.HasSuffix(strings.ToLower(path), ".tar.bz2") ||
strings.HasSuffix(strings.ToLower(path), ".tgz") ||
strings.HasSuffix(strings.ToLower(path), ".tbz2") {
info, cover, err := extractComicMetadata(path)
if err != nil {
fmt.Printf("Warning: failed to extract comic metadata from %s: %v\n", path, err)
} else {
comicInfo = info
coverImage = cover
if comicInfo.Title != "" && metadata.Title == "" {
metadata.Title = comicInfo.Title
}
if comicInfo.Series != "" && metadata.Series == "" {
metadata.Series = comicInfo.Series
}
if comicInfo.Number > 0 && metadata.SeriesNumber == 0 {
metadata.SeriesNumber = int32(comicInfo.Number)
}
if comicInfo.Publisher != "" && metadata.Publisher == "" {
metadata.Publisher = comicInfo.Publisher
}
if comicInfo.Writer != "" && metadata.Author == "" {
metadata.Author = comicInfo.Writer
}
if len(coverImage) > 0 && metadata.CoverPath == "" {
coverPath := path + ".cover.jpg"
if err := os.WriteFile(coverPath, coverImage, 0644); err == nil {
metadata.CoverPath = s.getRelativePath(coverPath)
}
}
fmt.Printf("Extracted comic metadata from %s: title=%s, series=%s, issue=%d\n",
path, comicInfo.Title, comicInfo.Series, comicInfo.Number)
}
}
// Try to get metadata from folder structure as fallback/enhancement
// Use the root folder that contains this file
var rootFolder string
for _, folder := range s.folders {
if strings.HasPrefix(path, folder) {
rootFolder = folder
break
}
}
if rootFolder != "" {
folderMetadata := s.extractFolderStructureMetadata(path, rootFolder)
// Use folder metadata as fallback for missing information
if metadata.Title == "" && folderMetadata.Title != "" {
metadata.Title = folderMetadata.Title
}
if metadata.Author == "" && folderMetadata.Author != "" {
metadata.Author = folderMetadata.Author
}
if metadata.Series == "" && folderMetadata.Series != "" {
metadata.Series = folderMetadata.Series
}
if metadata.SeriesNumber == 0 && folderMetadata.SeriesNumber > 0 {
metadata.SeriesNumber = folderMetadata.SeriesNumber
}
}
// Final fallback if still missing essential metadata
if metadata.Title == "" {
metadata.Title = strings.TrimSuffix(filepath.Base(path), filepath.Ext(path))
}
if metadata.Author == "" {
metadata.Author = "Unknown"
}
// libraryID already determined at start of function
// Normalize metadata fields for display
metadata.Contributors = utils.NormalizeContributors(metadata.Contributors)
metadata.Tags = utils.NormalizeTags(metadata.Tags)
// Normalize search fields
contributorsSearch := utils.NormalizeContributorsSearch(metadata.Contributors)
tagsSearch := utils.NormalizeTagsSearch(metadata.Tags)
// Create media item in database
relativePath := s.getRelativePath(path)
createdItem, err := s.db.CreateMediaItem(ctx, database.CreateMediaItemParams{
LibraryID: libraryID,
Title: metadata.Title,
Author: pgtype.Text{String: metadata.Author, Valid: metadata.Author != ""},
Isbn: pgtype.Text{String: utils.NormalizeISBNSafe(metadata.ISBN), Valid: metadata.ISBN != ""},
Asin: pgtype.Text{String: metadata.ASIN, Valid: metadata.ASIN != ""},
Description: pgtype.Text{String: metadata.Description, Valid: metadata.Description != ""},
FilePath: relativePath,
FileSize: pgtype.Int8{Int64: info.Size(), Valid: true},
MimeType: pgtype.Text{String: s.getMimeType(path), Valid: true},
CoverImagePath: pgtype.Text{String: metadata.CoverPath, Valid: metadata.CoverPath != ""},
Series: pgtype.Text{String: metadata.Series, Valid: metadata.Series != ""},
SeriesNumber: pgtype.Int4{Int32: metadata.SeriesNumber, Valid: metadata.SeriesNumber > 0},
Publisher: pgtype.Text{String: metadata.Publisher, Valid: metadata.Publisher != ""},
DatePublished: pgtype.Date{Time: metadata.PublishDate, Valid: !metadata.PublishDate.IsZero()},
Contributors: metadata.Contributors,
ContributorsSearch: contributorsSearch,
Tags: metadata.Tags,
TagsSearch: tagsSearch,
AddedByAdminID: s.adminID,
CreatedAt: pgtype.Timestamptz{Time: fileModTime, Valid: true},
})
if err != nil {
return false, fmt.Errorf("failed to create media item: %v", err)
}
// Update hash information before database storage
if metadata.FileHashInfo != nil && metadata.FileHashInfo.FileSHA256 != "" {
_, err = s.db.UpdateMediaItemIdentifiers(ctx, database.UpdateMediaItemIdentifiersParams{
ID: createdItem.ID,
FileSha256: pgtype.Text{String: metadata.FileHashInfo.FileSHA256, Valid: true},
OpfIdentifier: pgtype.Text{String: metadata.FileHashInfo.OPFIdentifier, Valid: metadata.FileHashInfo.OPFIdentifier != ""},
OpfUuid: pgtype.Text{String: metadata.FileHashInfo.OPFUUID, Valid: metadata.FileHashInfo.OPFUUID != ""},
HashConfidence: pgtype.Text{String: metadata.FileHashInfo.HashConfidence, Valid: true},
})
if err != nil {
fmt.Printf("Warning: failed to update hash identifiers for %s: %v\n", path, err)
}
}
// Store format information in the database
for _, format := range metadata.FileFormats {
_, err = s.db.CreateMediaItemFormat(ctx, database.CreateMediaItemFormatParams{
MediaItemID: createdItem.ID,
FormatType: format.FormatType,
FilePath: pgtype.Text{String: s.getRelativePath(format.FilePath), Valid: true},
FileSha256: pgtype.Text{String: format.FileSHA256, Valid: true},
FileSizeBytes: pgtype.Int8{Int64: format.FileSizeBytes, Valid: true},
MimeType: pgtype.Text{String: format.MimeType, Valid: true},
})
if err != nil {
fmt.Printf("Warning: failed to create format entry for %s: %v\n", path, err)
}
}
s.newItems++
return true, nil
}
func (s *MediaScanner) extractMetadata(path string) (*MediaMetadata, error) {
ext := strings.ToLower(filepath.Ext(path))
switch ext {
case ".epub":
metadata, err := s.extractEPUBMetadata(path)
if err != nil {
return metadata, err
}
// Try to extract embedded cover
coverPath, err := s.extractEPUBCover(path)
if err != nil {
fmt.Printf("Warning: failed to extract EPUB cover from %s: %v\n", path, err)
} else if coverPath != "" {
metadata.CoverPath = s.getRelativePath(coverPath)
}
// If no embedded cover, try sidecar
if metadata.CoverPath == "" {
sidecarCover := findSidecarCover(path)
if sidecarCover != "" {
metadata.CoverPath = s.getRelativePath(sidecarCover)
}
}
return metadata, nil
case ".pdf":
return s.extractPDFMetadata(path)
default:
// For other formats, return basic metadata
return &MediaMetadata{
Title: strings.TrimSuffix(filepath.Base(path), ext),
}, nil
}
}
func (s *MediaScanner) extractEPUBMetadata(path string) (*MediaMetadata, error) {
book, err := epub.ReadBook(path)
if err != nil {
return nil, fmt.Errorf("failed to open EPUB: %v", err)
}
metadata := &MediaMetadata{}
// Title
if title, err := book.Title(); err == nil && title != "" {
metadata.Title = title
}
// Author
if authors, err := book.MetadataByKey("creator"); err == nil && len(authors) > 0 {
metadata.Author = authors[0]
}
// Description
if descriptions, err := book.MetadataByKey("description"); err == nil && len(descriptions) > 0 {
metadata.Description = descriptions[0]
}
// Publisher
if publishers, err := book.MetadataByKey("publisher"); err == nil && len(publishers) > 0 {
metadata.Publisher = publishers[0]
}
// Series and series number (Calibre specific metadata)
if series, err := book.MetadataByKey("calibre:series"); err == nil && len(series) > 0 {
metadata.Series = series[0]
}
if seriesIndex, err := book.MetadataByKey("calibre:series_index"); err == nil && len(seriesIndex) > 0 {
if index, err := strconv.ParseFloat(seriesIndex[0], 32); err == nil {
metadata.SeriesNumber = int32(index)
}
}
// Publish date
if dates, err := book.MetadataByKey("date"); err == nil && len(dates) > 0 {
if date, err := time.Parse("2006-01-02", dates[0]); err == nil {
metadata.PublishDate = date
} else {
// Try alternative date formats
if date, err := time.Parse("2006", dates[0]); err == nil {
metadata.PublishDate = date
}
}
}
// Contributors
if contributors, err := book.MetadataByKey("contributor"); err == nil && len(contributors) > 0 {
// Normalize contributors for display
metadata.Contributors = utils.NormalizeContributors(contributors)
}
// ISBN
if isbns, err := book.MetadataByKey("identifier"); err == nil && len(isbns) > 0 {
for _, isbn := range isbns {
if strings.Contains(strings.ToLower(isbn), "isbn") {
// Extract ISBN number from identifier like "isbn:978-3-16-148410-0"
isbnParts := strings.SplitN(isbn, ":", 2)
if len(isbnParts) == 2 {
metadata.ISBN = isbnParts[1]
break
}
}
if strings.Contains(strings.ToLower(isbn), "asin") {
// Extract ASIN from identifier like "asin:B08XXXXX"
asinParts := strings.SplitN(isbn, ":", 2)
if len(asinParts) == 2 {
metadata.ASIN = asinParts[1]
}
}
}
}
// Tags
if tags, err := book.MetadataByKey("subject"); err == nil && len(tags) > 0 {
// Normalize tags for display
metadata.Tags = utils.NormalizeTags(tags)
}
return metadata, nil
}
// extractEPUBCover extracts the cover image from an EPUB file.
// It looks for:
// 1. An item with properties="cover-image" in the manifest
// 2. A meta tag with name="cover" pointing to an image
// 3. Common cover image paths like cover.jpg, cover.jpeg, cover.png
// Returns the path to the saved cover image, or empty string if no cover found.
func (s *MediaScanner) extractEPUBCover(epubPath string) (string, error) {
// Open the EPUB as a zip file to extract images
r, err := zip.OpenReader(epubPath)
if err != nil {
return "", fmt.Errorf("failed to open EPUB as zip: %v", err)
}
defer r.Close()
// Try to find cover image from OPF metadata
coverImageName := ""
// Attempt to read the OPF file to find cover reference
// First, find container.xml to locate the OPF
var opfPath string
for _, f := range r.File {
if f.Name == "META-INF/container.xml" {
rc, err := f.Open()
if err != nil {
continue
}
content, err := io.ReadAll(rc)
rc.Close()
if err != nil {
continue
}
// Parse container.xml to find OPF path
// Simple string search since we just need the path
opfStart := bytes.Index(content, []byte("<rootfile "))
if opfStart == -1 {
continue
}
opfStartAttr := bytes.Index(content[opfStart:], []byte("full-path="))
if opfStartAttr == -1 {
continue
}
opfStartAttr += len("full-path=")
quote := content[opfStart+opfStartAttr]
opfStartQuote := opfStart + opfStartAttr + 1
opfEndQuote := bytes.Index(content[opfStartQuote:], []byte{byte(quote)})
if opfEndQuote == -1 {
continue
}
opfPath = string(content[opfStartQuote : opfStartQuote+opfEndQuote])
break
}
}
if opfPath == "" {
// No OPF found, try common cover image paths
coverImageName = findCoverImageInZip(r.File)
} else {
// Read OPF to find cover reference
opfContent, err := readFileFromZip(r.File, opfPath)
if err != nil {
// Fall back to common paths
coverImageName = findCoverImageInZip(r.File)
} else {
coverImageName = findCoverInOPF(opfContent, r.File, opfPath)
}
}
if coverImageName == "" {
return "", nil // No cover found
}
// Extract the cover image
coverImage, err := extractImageFromZip(r.File, coverImageName, opfPath)
if err != nil {
return "", fmt.Errorf("failed to extract cover image: %v", err)
}
if len(coverImage) == 0 {
return "", nil
}
// Save cover to disk (same pattern as comics: {book_path}.cover.jpg)
coverPath := epubPath + ".cover.jpg"
if err := os.WriteFile(coverPath, coverImage, 0644); err != nil {
return "", fmt.Errorf("failed to write cover file: %v", err)
}
return coverPath, nil
}
// findCoverImageInZip searches for common cover image filenames in the zip
func findCoverImageInZip(files []*zip.File) string {
coverNames := []string{"cover.jpg", "cover.jpeg", "cover.png", "cover.webp",
"Cover.jpg", "Cover.jpeg", "Cover.png", "Cover.webp",
"images/cover.jpg", "Images/cover.jpg", "OEBPS/images/cover.jpg"}
for _, name := range coverNames {
for _, f := range files {
if strings.EqualFold(f.Name, name) {
return f.Name
}
}
}
return ""
}
// findCoverInOPF parses OPF content to find cover image reference
func findCoverInOPF(opfContent []byte, files []*zip.File, opfDir string) string {
contentStr := string(opfContent)
// Look for item with properties="cover-image"
coverImageRE := regexp.MustCompile(`<item[^>]*properties="[^"]*cover-image[^"]*"[^>]*id="([^"]+)"`)
matches := coverImageRE.FindStringSubmatch(contentStr)
if len(matches) > 1 {
coverID := matches[1]
// Find the href for this ID
hrefRE := regexp.MustCompile(fmt.Sprintf(`<item[^>]*id="%s"[^>]*href="([^"]+)"`, coverID))
hrefMatches := hrefRE.FindStringSubmatch(contentStr)
if len(hrefMatches) > 1 {
return resolveOPFPath(opfDir, hrefMatches[1])
}
}
// Look for meta name="cover"
metaCoverRE := regexp.MustCompile(`<meta[^>]*name="cover"[^>]*content="([^"]+)"`)
metaMatches := metaCoverRE.FindStringSubmatch(contentStr)
if len(metaMatches) > 1 {
coverContent := metaMatches[1]
// Could be "image-id" format
if strings.HasPrefix(coverContent, "image-") {
coverID := strings.TrimPrefix(coverContent, "image-")
hrefRE := regexp.MustCompile(fmt.Sprintf(`<item[^>]*id="%s"[^>]*href="([^"]+)"`, coverID))
hrefMatches := hrefRE.FindStringSubmatch(contentStr)
if len(hrefMatches) > 1 {
return resolveOPFPath(opfDir, hrefMatches[1])
}
}
}
// Fall back to searching common paths
return findCoverImageInZip(files)
}
// resolveOPFPath resolves a relative path against the OPF directory
func resolveOPFPath(opfDir, href string) string {
if opfDir == "" {
return href
}
// Handle ../ in href
if strings.HasPrefix(href, "../") {
// Simple case: just use the href as-is for now
return href
}
// Join the directory with the href
return filepath.Join(filepath.Dir(opfDir), href)
}
// readFileFromZip reads a file from the zip by name
func readFileFromZip(files []*zip.File, name string) ([]byte, error) {
// Normalize the name for comparison
name = filepath.ToSlash(name)
for _, f := range files {
fName := filepath.ToSlash(f.Name)
if fName == name {
rc, err := f.Open()
if err != nil {
return nil, err
}
defer rc.Close()
return io.ReadAll(rc)
}
}
return nil, fmt.Errorf("file not found: %s", name)
}
// extractImageFromZip extracts an image file and returns its contents
func extractImageFromZip(files []*zip.File, imagePath, opfDir string) ([]byte, error) {
// Try direct match first
for _, f := range files {
if strings.EqualFold(f.Name, imagePath) {
rc, err := f.Open()
if err != nil {
return nil, err
}
defer rc.Close()
return io.ReadAll(rc)
}
}
// Try resolved path
resolvedPath := resolveOPFPath(opfDir, imagePath)
for _, f := range files {
if strings.EqualFold(f.Name, resolvedPath) {
rc, err := f.Open()
if err != nil {
return nil, err
}
defer rc.Close()
return io.ReadAll(rc)
}
}
return nil, fmt.Errorf("image not found: %s", imagePath)
}
// findSidecarCover looks for cover images in the same directory as the media file.
// It checks for common cover filename patterns in priority order:
// 1. cover.jpg, cover.jpeg, cover.png, cover.webp
// 2. folder.jpg, folder.jpeg, folder.png, folder.webp
// 3. {basename}.jpg, {basename}.jpeg, etc. (same name as media file)
// 4. .folder.jpg (hidden file)
// Returns the full path to the cover file, or empty string if not found.
func findSidecarCover(mediaPath string) string {
dir := filepath.Dir(mediaPath)
baseName := strings.TrimSuffix(filepath.Base(mediaPath), filepath.Ext(mediaPath))
// Priority order for cover filenames
coverPatterns := []string{
"cover.jpg",
"cover.jpeg",
"cover.png",
"cover.webp",
"folder.jpg",
"folder.jpeg",
"folder.png",
"folder.webp",
".folder.jpg",
".folder.jpeg",
".folder.png",
}
// First, check exact match cover/folder names
for _, coverName := range coverPatterns {
coverPath := filepath.Join(dir, coverName)
if _, err := os.Stat(coverPath); err == nil {
return coverPath
}
}
// Second, check for {basename}.{ext} pattern
extensions := []string{".jpg", ".jpeg", ".png", ".webp"}
for _, ext := range extensions {
coverPath := filepath.Join(dir, baseName+ext)
if _, err := os.Stat(coverPath); err == nil {
return coverPath
}
// Also check uppercase extension
coverPathUpper := filepath.Join(dir, baseName+strings.ToUpper(ext))
if _, err := os.Stat(coverPathUpper); err == nil {
return coverPathUpper
}
}
return ""
}
func (s *MediaScanner) extractPDFMetadata(path string) (*MediaMetadata, error) {
metadata := &MediaMetadata{}
// Open PDF file for reading metadata
f, err := os.Open(path)
if err != nil {
fmt.Printf("Warning: failed to open PDF file %s: %v\n", path, err)
metadata.Title = strings.TrimSuffix(filepath.Base(path), ".pdf")
return metadata, nil
}
defer f.Close()
// Use pdfcpu API to read PDF metadata
// Configuration: nil = default (lenient mode)
pdfInfo, err := pdfcpuapi.PDFInfo(f, filepath.Base(path), nil, false, nil)
if err != nil {
fmt.Printf("Warning: failed to read PDF info from %s: %v\n", path, err)
// Fall back to filename as title
metadata.Title = strings.TrimSuffix(filepath.Base(path), ".pdf")
return metadata, nil
}
// Extract title
if pdfInfo.Title != "" {
metadata.Title = pdfInfo.Title
} else {
metadata.Title = strings.TrimSuffix(filepath.Base(path), ".pdf")
}
// Extract author
if pdfInfo.Author != "" {
metadata.Author = pdfInfo.Author
}
// Extract subject (use as description)
if pdfInfo.Subject != "" {
metadata.Description = pdfInfo.Subject
}
// Extract creator (use as author fallback)
if pdfInfo.Creator != "" && metadata.Author == "" {
metadata.Author = pdfInfo.Creator
}
// Extract producer (use as publisher)
if pdfInfo.Producer != "" {
metadata.Publisher = pdfInfo.Producer
}
// Try to extract cover image
coverPath, err := s.extractPDFCover(path)
if err != nil {
fmt.Printf("Warning: failed to extract PDF cover from %s: %v\n", path, err)
} else if coverPath != "" {
metadata.CoverPath = s.getRelativePath(coverPath)
}
// If no embedded cover, try sidecar
if metadata.CoverPath == "" {
sidecarCover := findSidecarCover(path)
if sidecarCover != "" {
metadata.CoverPath = s.getRelativePath(sidecarCover)
}
}
return metadata, nil
}
// extractPDFCover extracts a cover image from a PDF file.
// It uses pdfcpu API to extract images from the first page.
// Returns the path to the saved cover, or empty string if no cover found.
func (s *MediaScanner) extractPDFCover(pdfPath string) (string, error) {
// Create a temporary directory for extracted images
tmpDir, err := os.MkdirTemp("", "pdf-cover-")
if err != nil {
return "", fmt.Errorf("failed to create temp dir: %v", err)
}
defer os.RemoveAll(tmpDir)
// Use pdfcpu API to extract images from first page
// ExtractImagesFile(inFile, outDir string, selectedPages []string, conf *model.Configuration) error
err = pdfcpuapi.ExtractImagesFile(pdfPath, tmpDir, []string{"1"}, nil)
if err != nil {
// No images found or extraction failed - this is OK, just return empty
return "", nil
}
// Check for extracted images in the temp directory
entries, err := os.ReadDir(tmpDir)
if err != nil || len(entries) == 0 {
return "", nil
}
// Find the largest image (likely the cover)
var largestImage string
var largestSize int64
for _, entry := range entries {
if entry.IsDir() {
continue
}
info, err := entry.Info()
if err != nil {
continue
}
// Skip very small files (likely thumbnails or icons)
if info.Size() < 1000 {
continue
}
if info.Size() > largestSize {
largestImage = filepath.Join(tmpDir, entry.Name())
largestSize = info.Size()
}
}
if largestImage == "" {
return "", nil
}
// Read the image
imageData, err := os.ReadFile(largestImage)
if err != nil || len(imageData) == 0 {
return "", nil
}
// Save cover to disk (same pattern as comics: {pdf_path}.cover.jpg)
coverPath := pdfPath + ".cover.jpg"
if err := os.WriteFile(coverPath, imageData, 0644); err != nil {
return "", fmt.Errorf("failed to write cover file: %v", err)
}
return coverPath, nil
}
// ComicInfo represents metadata from ComicInfo.xml
type ComicInfo struct {
XMLName xml.Name `xml:"ComicInfo"`
Title string `xml:"Title"`
Series string `xml:"Series"`
Number int `xml:"Number"`
Volume int `xml:"Volume"`
Publisher string `xml:"Publisher"`
Year int `xml:"Year"`
Month int `xml:"Month"`
Day int `xml:"Day"`
Writer string `xml:"Writer"`
Penciller string `xml:"Penciller"`
Inker string `xml:"Inker"`
Colorist string `xml:"Colorist"`
Letterer string `xml:"Letterer"`
CoverArtist string `xml:"CoverArtist"`
Genre string `xml:"Genre"`
Tags string `xml:"Tags"`
Web string `xml:"Web"`
Notes string `xml:"Notes"`
}
// extractComicMetadata extracts metadata from comic archive (supports .cbz, .cbr, .cb7, .cbt)
func extractComicMetadata(filePath string) (*ComicInfo, []byte, error) {
ext := strings.ToLower(filepath.Ext(filePath))
switch ext {
case ".cbz":
return extractZipMetadata(filePath)
case ".cbr":
return extractRarMetadata(filePath)
case ".cb7":
return extract7ZipMetadata(filePath)
case ".cbt":
return extractTarMetadata(filePath)
default:
return nil, nil, fmt.Errorf("unsupported comic format: %s", ext)
}
}
// archiveFile represents a file in an archive for uniform handling
type archiveFile interface {
Name() string
Open() (io.ReadCloser, error)
}
// extractMetadataFromArchive extracts ComicInfo.xml and cover image from any archive format
func extractMetadataFromArchive(files []archiveFile) (*ComicInfo, []byte, error) {
var comicInfo *ComicInfo
var coverImage []byte
for _, f := range files {
if f.Name() == "ComicInfo.xml" {
rc, err := f.Open()
if err != nil {
return nil, nil, fmt.Errorf("failed to open ComicInfo.xml: %w", err)
}
data, err := io.ReadAll(rc)
rc.Close()
if err != nil {
return nil, nil, fmt.Errorf("failed to read ComicInfo.xml: %w", err)
}
comicInfo = &ComicInfo{}
if err := xml.Unmarshal(data, comicInfo); err != nil {
return nil, nil, fmt.Errorf("failed to parse ComicInfo.xml: %w", err)
}
}
if coverImage == nil && isImageFile(f.Name()) {
if !strings.Contains(filepath.Dir(f.Name()), string(filepath.Separator)) ||
filepath.Dir(f.Name()) == "." {
rc, err := f.Open()
if err != nil {
continue
}
coverImage, err = io.ReadAll(rc)
rc.Close()
if err == nil {
_, _, err = image.Decode(bytes.NewReader(coverImage))
if err != nil {
coverImage = nil
}
}
}
}
}
if comicInfo == nil {
comicInfo = &ComicInfo{}
}
return comicInfo, coverImage, nil
}
// extractZipMetadata extracts metadata from ZIP archives (.cbz)
func extractZipMetadata(filePath string) (*ComicInfo, []byte, error) {
r, err := zip.OpenReader(filePath)
if err != nil {
return nil, nil, fmt.Errorf("failed to open ZIP archive: %w", err)
}
defer r.Close()
files := make([]archiveFile, 0, len(r.File))
for _, f := range r.File {
files = append(files, &zipFileAdapter{f})
}
comicInfo, coverImage, err := extractMetadataFromArchive(files)
if err != nil {
return nil, nil, err
}
if comicInfo.Title == "" {
basename := filepath.Base(filePath)
comicInfo.Title = strings.TrimSuffix(basename, filepath.Ext(basename))
}
return comicInfo, coverImage, nil
}
// zipFileAdapter adapts zip.File to archiveFile interface
type zipFileAdapter struct {
*zip.File
}
func (z *zipFileAdapter) Name() string {
return z.File.Name
}
func (z *zipFileAdapter) Open() (io.ReadCloser, error) {
return z.File.Open()
}
// extractRarMetadata extracts metadata from RAR archives (.cbr)
func extractRarMetadata(filePath string) (*ComicInfo, []byte, error) {
r, err := rardecode.OpenReader(filePath, "")
if err != nil {
return nil, nil, fmt.Errorf("failed to open RAR archive: %w", err)
}
defer r.Close()
files := make([]archiveFile, 0, 100)
for {
header, err := r.Next()
if err == io.EOF {
break
}
if err != nil {
return nil, nil, fmt.Errorf("failed to read RAR entry: %w", err)
}
data, err := io.ReadAll(r)
if err != nil {
continue
}
files = append(files, &rarFileAdapter{
name: header.Name,
data: data,
})
}
comicInfo, coverImage, err := extractMetadataFromArchive(files)
if err != nil {
return nil, nil, err
}
if comicInfo.Title == "" {
basename := filepath.Base(filePath)
comicInfo.Title = strings.TrimSuffix(basename, filepath.Ext(basename))
}
return comicInfo, coverImage, nil
}
// rarFileAdapter stores RAR file data in memory
type rarFileAdapter struct {
name string
data []byte
}
func (r *rarFileAdapter) Name() string {
return r.name
}
func (r *rarFileAdapter) Open() (io.ReadCloser, error) {
return io.NopCloser(bytes.NewReader(r.data)), nil
}
// extract7ZipMetadata extracts metadata from 7-Zip archives (.cb7)
func extract7ZipMetadata(filePath string) (*ComicInfo, []byte, error) {
r, err := sevenzip.OpenReader(filePath)
if err != nil {
return nil, nil, fmt.Errorf("failed to open 7-Zip archive: %w", err)
}
defer r.Close()
files := make([]archiveFile, 0, len(r.File))
for _, f := range r.File {
files = append(files, &sevenZipFileAdapter{file: f})
}
comicInfo, coverImage, err := extractMetadataFromArchive(files)
if err != nil {
return nil, nil, err
}
if comicInfo.Title == "" {
basename := filepath.Base(filePath)
comicInfo.Title = strings.TrimSuffix(basename, filepath.Ext(basename))
}
return comicInfo, coverImage, nil
}
// sevenZipFileAdapter adapts sevenzip.File to archiveFile interface
type sevenZipFileAdapter struct {
file *sevenzip.File
}
func (s *sevenZipFileAdapter) Name() string {
return s.file.Name
}
func (s *sevenZipFileAdapter) Open() (io.ReadCloser, error) {
return s.file.Open()
}
// extractTarMetadata extracts metadata from TAR archives (.cbt)
func extractTarMetadata(filePath string) (*ComicInfo, []byte, error) {
f, err := os.Open(filePath)
if err != nil {
return nil, nil, fmt.Errorf("failed to open TAR archive: %w", err)
}
defer f.Close()
var tarReader *tar.Reader
if strings.HasSuffix(strings.ToLower(filePath), ".tar.gz") ||
strings.HasSuffix(strings.ToLower(filePath), ".tgz") {
gzReader, err := gzip.NewReader(f)
if err != nil {
return nil, nil, fmt.Errorf("failed to create gzip reader: %w", err)
}
defer gzReader.Close()
tarReader = tar.NewReader(gzReader)
} else if strings.HasSuffix(strings.ToLower(filePath), ".tar.bz2") ||
strings.HasSuffix(strings.ToLower(filePath), ".tbz2") {
bz2Reader := bzip2.NewReader(f)
tarReader = tar.NewReader(bz2Reader)
} else {
tarReader = tar.NewReader(f)
}
files := make([]archiveFile, 0, 100)
for {
header, err := tarReader.Next()
if err == io.EOF {
break
}
if err != nil {
return nil, nil, fmt.Errorf("failed to read TAR entry: %w", err)
}
if header.Typeflag == tar.TypeReg {
data, err := io.ReadAll(tarReader)
if err != nil {
continue
}
files = append(files, &tarFileAdapter{
name: header.Name,
data: data,
})
}
}
comicInfo, coverImage, err := extractMetadataFromArchive(files)
if err != nil {
return nil, nil, err
}
if comicInfo.Title == "" {
basename := filepath.Base(filePath)
comicInfo.Title = strings.TrimSuffix(basename, filepath.Ext(basename))
}
return comicInfo, coverImage, nil
}
// tarFileAdapter stores TAR file data in memory
type tarFileAdapter struct {
name string
data []byte
}
func (t *tarFileAdapter) Name() string {
return t.name
}
func (t *tarFileAdapter) Open() (io.ReadCloser, error) {
return io.NopCloser(bytes.NewReader(t.data)), nil
}
// isImageFile checks if a file is an image based on extension
func isImageFile(filename string) bool {
ext := strings.ToLower(filepath.Ext(filename))
return ext == ".jpg" || ext == ".jpeg" || ext == ".png" || ext == ".gif"
}
func (s *MediaScanner) updateMediaItem(ctx context.Context, mediaItemID pgtype.UUID, path string, info os.FileInfo) error {
// Re-extract metadata for the update
metadata, err := s.extractMetadata(path)
if err != nil {
fmt.Printf("Warning: failed to extract metadata for force rescan %s: %v\n", path, err)
metadata = &MediaMetadata{}
}
// Normalize metadata fields
metadata.Contributors = utils.NormalizeContributors(metadata.Contributors)
metadata.Tags = utils.NormalizeTags(metadata.Tags)
contributorsSearch := utils.NormalizeContributorsSearch(metadata.Contributors)
tagsSearch := utils.NormalizeTagsSearch(metadata.Tags)
// Call the database update - only update fields available in MediaMetadata
_, err = s.db.UpdateMediaItem(ctx, database.UpdateMediaItemParams{
ID: mediaItemID,
Title: metadata.Title,
Author: pgtype.Text{String: metadata.Author, Valid: metadata.Author != ""},
Isbn: pgtype.Text{String: utils.NormalizeISBNSafe(metadata.ISBN), Valid: metadata.ISBN != ""},
Description: pgtype.Text{String: metadata.Description, Valid: metadata.Description != ""},
CoverImagePath: pgtype.Text{String: s.getRelativePath(metadata.CoverPath), Valid: metadata.CoverPath != ""},
Series: pgtype.Text{String: metadata.Series, Valid: metadata.Series != ""},
SeriesNumber: pgtype.Int4{Int32: metadata.SeriesNumber, Valid: metadata.SeriesNumber > 0},
Tags: metadata.Tags,
TagsSearch: tagsSearch,
Asin: pgtype.Text{String: metadata.ASIN, Valid: metadata.ASIN != ""},
DatePublished: pgtype.Date{Time: metadata.PublishDate, Valid: !metadata.PublishDate.IsZero()},
Publisher: pgtype.Text{String: metadata.Publisher, Valid: metadata.Publisher != ""},
Contributors: metadata.Contributors,
ContributorsSearch: contributorsSearch,
})
return err
}
func (s *MediaScanner) getMediaItemByFilePath(ctx context.Context, filePath string, libraryID pgtype.UUID) (database.MediaItems, error) {
return s.db.GetMediaItemByFilePath(ctx, database.GetMediaItemByFilePathParams{
FilePath: s.getRelativePath(filePath),
LibraryID: libraryID,
})
}
func (s *MediaScanner) getMimeType(path string) string {
ext := strings.ToLower(filepath.Ext(path))
if mime, ok := MimeTypes[ext]; ok {
return mime
}
return "application/octet-stream"
}
func (s *MediaScanner) WatchChanges(ctx context.Context) error {
// Prevent duplicate calls
if !s.watching.CompareAndSwap(false, true) {
return fmt.Errorf("already watching")
}
// Reset flag when context is cancelled
go func() {
<-ctx.Done()
s.watching.Store(false)
}()
// Perform initial scan of all root folders
go s.performInitialScan(ctx)
// Start directory processor
go s.processDirtyDirectories(ctx)
// Start polling fallback
go s.StartPolling(ctx)
// Handle fsnotify events - queue them for debouncing
go func() {
for {
select {
case event, ok := <-s.watcher.Events:
if !ok {
return
}
// Handle new directories - add them to the watcher
if event.Has(fsnotify.Create) {
if info, err := os.Stat(event.Name); err == nil && info.IsDir() {
s.watcher.Add(event.Name)
}
}
// Mark directory dirty for ANY file change
if event.Has(fsnotify.Create | fsnotify.Write | fsnotify.Remove | fsnotify.Chmod | fsnotify.Rename) {
s.markDirectoryDirty(filepath.Dir(event.Name))
}
case err, ok := <-s.watcher.Errors:
if !ok {
return
}
fmt.Printf("Watcher error: %v\n", err)
case <-ctx.Done():
return
}
}
}()
return nil
}
func (s *MediaScanner) handleFileAdd(ctx context.Context, filePath string) {
fmt.Printf("New/modified media file detected: %s\n", filePath)
if _, err := s.processMediaFile(ctx, filePath); err != nil {
fmt.Printf("Error processing modified media file %s: %v\n", filePath, err)
}
}
func (s *MediaScanner) handleFileDelete(ctx context.Context, filePath string) {
s.logger.LogDelete(fmt.Sprintf("[WATCH-DELETE] File removed from filesystem: %s", filePath))
// Determine libraryID for this file
var libraryID pgtype.UUID
for _, folder := range s.folders {
if strings.HasPrefix(filePath, folder) {
lib, err := s.db.GetLibraryByFolder(ctx, folder)
if err == nil {
libraryID = lib.LibraryID
break
}
}
}
if !libraryID.Valid {
msg := fmt.Sprintf("[WATCH-DELETE] WARNING: could not determine library for deleted file: %s", filePath)
s.logger.LogDelete(msg)
s.logger.LogError(msg)
return
}
// Look up media item
existingItem, err := s.db.GetMediaItemByFilePath(ctx, database.GetMediaItemByFilePathParams{
FilePath: s.getRelativePath(filePath),
LibraryID: libraryID,
})
if err == nil {
msg := fmt.Sprintf("[WATCH-DELETE] Found media item to delete: ID=%s, Title=%s, Path=%s",
existingItem.ID, existingItem.Title, existingItem.FilePath)
s.logger.LogDelete(msg)
if err := s.db.DeleteMediaItem(ctx, existingItem.ID); err != nil {
errMsg := fmt.Sprintf("[WATCH-DELETE] ERROR: failed to delete media item %s: %v", existingItem.ID, err)
s.logger.LogDelete(errMsg)
s.logger.LogError(errMsg)
} else {
s.logger.LogDelete(fmt.Sprintf("[WATCH-DELETE] SUCCESS: deleted media item '%s' (was at %s)",
existingItem.Title, existingItem.FilePath))
}
} else if err != pgx.ErrNoRows {
errMsg := fmt.Sprintf("[WATCH-DELETE] ERROR: failed to look up media item for %s: %v", filePath, err)
s.logger.LogDelete(errMsg)
s.logger.LogError(errMsg)
} else {
s.logger.LogDelete(fmt.Sprintf("[WATCH-DELETE] No media item found in database for deleted file: %s", filePath))
}
}
func (s *MediaScanner) markDirectoryDirty(dirPath string) {
s.dirtyDirsMu.Lock()
defer s.dirtyDirsMu.Unlock()
// Only mark if within watched folders
var isWatched bool
for _, folder := range s.folders {
if strings.HasPrefix(dirPath, folder) {
isWatched = true
break
}
}
if !isWatched {
return
}
// Smart event merging (Jellyfin approach):
// 1. If parent dir exists, replace with parent (consolidate)
// 2. If sibling dirs exist, replace with common parent
// 3. Otherwise, add this dir
// Check if parent directory is already dirty
parentDir := filepath.Dir(dirPath)
if parentDir != dirPath { // Not at root
if _, parentExists := s.dirtyDirs[parentDir]; parentExists {
// Parent already being watched, reset its timestamp
s.dirtyDirs[parentDir] = time.Now()
return
}
}
// Check if any subdirectories are dirty, replace with parent
for existingDir := range s.dirtyDirs {
if strings.HasPrefix(existingDir, dirPath+"/") {
// This is a subdirectory, replace it with parent
delete(s.dirtyDirs, existingDir)
}
}
// NEW: Check for sibling directories and consolidate to parent
parentDir = filepath.Dir(dirPath)
for existingDir := range s.dirtyDirs {
existingParent := filepath.Dir(existingDir)
if existingParent == parentDir && existingParent != dirPath && existingParent != "." {
// Found a sibling! Both should be replaced with parent
delete(s.dirtyDirs, existingDir)
s.dirtyDirs[parentDir] = time.Now()
return
}
}
// Add/update this directory
s.dirtyDirs[dirPath] = time.Now()
}
func (s *MediaScanner) processDirtyDirectories(ctx context.Context) {
ticker := time.NewTicker(1 * time.Second)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
return
case <-ticker.C:
s.dirtyDirsMu.Lock()
now := time.Now()
readyDirs := make([]string, 0)
// Find directories that haven't been modified in 10 seconds
// This batches changes together (Audiobookshelf approach)
for dirPath, lastChange := range s.dirtyDirs {
if now.Sub(lastChange) >= 10*time.Second {
readyDirs = append(readyDirs, dirPath)
delete(s.dirtyDirs, dirPath)
}
}
s.dirtyDirsMu.Unlock()
// Process all ready directories in a batch via job queue
// Job queue serializes scans - prevents concurrent directory access
if len(readyDirs) > 0 {
for _, dirPath := range readyDirs {
// Create directory scan job with correct params for processDirectoryScanJob()
job := &Job{
ID: uuid.New().String(),
Type: JobTypeDirectoryScan,
Params: map[string]interface{}{
"directory": dirPath,
"db": s.db,
},
Status: JobStatusPending,
}
// Enqueue via global worker singleton
if WorkerInstance != nil {
WorkerInstance.Enqueue(job)
fmt.Printf("Enqueued directory scan job: %s\n", dirPath)
} else {
fmt.Printf("Warning: Worker not initialized, skipping directory scan: %s\n", dirPath)
}
}
}
}
}
}
// waitForFileStability checks if a file's mtime has stabilized
// Returns true when file is stable (not being modified)
// Polls every 3 seconds, times out after 60 seconds
// Uses atomic.Bool to prevent race conditions with concurrent checks
func (s *MediaScanner) waitForFileStability(filePath string) bool {
s.fileStabilityMu.Lock()
// Check if already being checked (atomic.Bool prevents race condition)
tracking, exists := s.fileStability[filePath]
if exists {
s.fileStabilityMu.Unlock()
// Another goroutine is already checking this file
if tracking.Load() {
return false // Still being checked
}
// Tracking exists but completed, remove stale entry
s.fileStabilityMu.Lock()
delete(s.fileStability, filePath)
}
// Start tracking with atomic.Bool set to true (checking in progress)
trackingFlag := &atomic.Bool{}
trackingFlag.Store(true)
s.fileStability[filePath] = trackingFlag
s.fileStabilityMu.Unlock()
// Get initial mtime
info, err := os.Stat(filePath)
if err != nil {
// Clean up tracking entry if file doesn't exist
s.fileStabilityMu.Lock()
delete(s.fileStability, filePath)
s.fileStabilityMu.Unlock()
return false
}
lastMtime := info.ModTime()
// Poll every 3 seconds for up to 60 seconds
timeout := time.After(60 * time.Second)
ticker := time.NewTicker(3 * time.Second)
defer ticker.Stop()
for {
select {
case <-timeout:
// Timeout - mark as done and clean up
trackingFlag.Store(false)
s.fileStabilityMu.Lock()
delete(s.fileStability, filePath)
s.fileStabilityMu.Unlock()
return false // File never stabilized
case <-ticker.C:
info, err := os.Stat(filePath)
if err != nil {
// File deleted - mark as done and clean up
trackingFlag.Store(false)
s.fileStabilityMu.Lock()
delete(s.fileStability, filePath)
s.fileStabilityMu.Unlock()
return false
}
currentMtime := info.ModTime()
if currentMtime.Equal(lastMtime) {
// File is stable! Mark as done and clean up
trackingFlag.Store(false)
s.fileStabilityMu.Lock()
delete(s.fileStability, filePath)
s.fileStabilityMu.Unlock()
return true
}
lastMtime = currentMtime
}
}
}
func (s *MediaScanner) scanDirectory(ctx context.Context, dirPath string) {
// Prevent concurrent scans of ANY directory
// Simple mutex is enough - job queue already serializes by directory
s.scan_mutex.Lock()
defer s.scan_mutex.Unlock()
// Find library for this directory
var libraryID pgtype.UUID
var rootFolder string
for _, folder := range s.folders {
if strings.HasPrefix(dirPath, folder) {
rootFolder = folder
if lib, err := s.db.GetLibraryByFolder(ctx, folder); err == nil {
libraryID = lib.LibraryID
break
}
}
}
// Check if libraryID is valid before proceeding
if !libraryID.Valid {
return
}
// Walk directory and process new files
filepath.WalkDir(dirPath, func(path string, d os.DirEntry, err error) error {
if err != nil {
return err
}
if d.IsDir() {
if path != dirPath {
return filepath.SkipDir
}
return nil
}
if !s.isScannableFile(path) {
return nil
}
// Check if file is stable before processing (Audiobookshelf approach)
if !s.waitForFileStability(path) {
return nil
}
relPath := strings.TrimPrefix(path, rootFolder+"/")
_, err = s.db.GetMediaItemByFilePath(ctx, database.GetMediaItemByFilePathParams{
FilePath: relPath,
LibraryID: libraryID,
})
if err == pgx.ErrNoRows {
if _, err := s.processMediaFile(ctx, path); err != nil {
s.errors++
} else {
s.newItems++
}
s.totalFiles++
}
return nil
})
}
// performInitialScan scans all root folders on startup
// This ensures existing files are detected before watching begins
func (s *MediaScanner) performInitialScan(ctx context.Context) {
fmt.Printf("Performing initial scan of root folders...\n")
for _, folder := range s.folders {
// Skip if folder doesn't exist
if _, err := os.Stat(folder); os.IsNotExist(err) {
fmt.Printf("Skipping nonexistent folder: %s\n", folder)
continue
}
// Submit scan job to worker (non-blocking)
job := &Job{
ID: uuid.New().String(),
Type: JobTypeDirectoryScan,
Params: map[string]interface{}{
"directory": folder,
"db": s.db,
},
Status: JobStatusPending,
}
if WorkerInstance != nil {
WorkerInstance.Enqueue(job)
fmt.Printf("Enqueued initial scan job: %s\n", folder)
} else {
fmt.Printf("Warning: Worker not initialized, skipping initial scan: %s\n", folder)
}
}
fmt.Printf("Initial scan jobs enqueued\n")
}
func (s *MediaScanner) Close() error {
fmt.Printf("Cleaning up scanner resources...\n")
// Stop watching
if s.watcher != nil {
s.watcher.Close()
}
// Clean up fileStability map to prevent memory leaks
s.fileStabilityMu.Lock()
s.fileStability = make(map[string]*atomic.Bool)
s.fileStabilityMu.Unlock()
// Clear dirty directories
s.dirtyDirsMu.Lock()
s.dirtyDirs = make(map[string]time.Time)
s.dirtyDirsMu.Unlock()
// Wait for in-progress scan to complete (with timeout)
timeout := time.After(5 * time.Second)
done := make(chan struct{})
go func() {
s.scan_mutex.Lock()
s.scan_mutex.Unlock()
close(done)
}()
select {
case <-done:
fmt.Printf("Scanner cleanup complete\n")
case <-timeout:
fmt.Printf("Timeout waiting for scan to complete\n")
}
return nil
}
func (s *MediaScanner) StartPolling(ctx context.Context) {
interval := s.GetPollInterval()
if interval <= 0 {
fmt.Println("Polling fallback disabled (interval = 0")
return
}
fmt.Printf("Polling fallback started with interval: %v\n", interval)
for {
ticker := time.NewTicker(interval)
defer ticker.Stop()
select {
case <-ctx.Done():
fmt.Println("Polling fallback stopped")
return
case <-ticker.C:
//Re-read interval each tick for dynamic updates
interval = s.GetPollInterval()
fmt.Printf("Running polling fallback sync (interval: %v)...\n", interval)
if err := s.SyncFilesystemWithDatabase(ctx); err != nil {
fmt.Printf("Polling sync error: %v\n", err)
}
}
}
}
func (s *MediaScanner) SyncFilesystemWithDatabase(ctx context.Context) error {
fmt.Println("[POLL-SYNC] Starting filesystem sync with database")
for _, folder := range s.folders {
lib, err := s.db.GetLibraryByFolder(ctx, folder)
if err != nil {
fmt.Printf("[POLL-SYNC] Warning: failed to get library for folder %s: %v\n", folder, err)
continue
}
libraryID := lib.LibraryID
// Get all media items from database for this library
dbItems, err := s.db.ListMediaItemsByLibrary(ctx, libraryID)
if err != nil {
fmt.Printf("[POLL-SYNC] Warning: failed to get library items: %v\n", err)
continue
}
// Build set of existing file paths from filesystem
existingPaths := make(map[string]bool)
filepath.WalkDir(folder, func(path string, d os.DirEntry, walkErr error) error {
if walkErr != nil {
return walkErr
}
if !d.IsDir() && s.isScannableFile(path) {
existingPaths[s.getRelativePath(path)] = true
}
return nil
})
// Check for orphaned items (in DB but not on filesystem)
for _, item := range dbItems {
if item.FilePath != "" && !existingPaths[item.FilePath] {
msg := fmt.Sprintf("[POLL-SYNC] Orphaned media item found: ID=%s, Title=%s, Path=%s",
item.ID, item.Title, item.FilePath)
s.logger.LogDelete(msg)
if err := s.db.DeleteMediaItem(ctx, item.ID); err != nil {
errMsg := fmt.Sprintf("[POLL-SYNC] ERROR: failed to delete orphaned item %s: %v", item.Title, err)
s.logger.LogDelete(errMsg)
s.logger.LogError(errMsg)
} else {
s.logger.LogDelete(fmt.Sprintf("[POLL-SYNC] SUCCESS: deleted orphaned item '%s'", item.Title))
}
}
}
// Check for new files (on filesystem but not in DB)
// This is expensive, so we just check a few representative files
// The fsnotify handler should catch most new files
for relPath := range existingPaths {
// Check if this file exists in DB
_, err := s.db.GetMediaItemByFilePath(ctx, database.GetMediaItemByFilePathParams{
FilePath: relPath,
LibraryID: libraryID,
})
if err == pgx.ErrNoRows {
// New file found - scan it
absPath := folder + "/" + relPath
if _, err := os.Stat(absPath); err == nil {
fmt.Printf("[POLL-SYNC] New file detected, scanning: %s\n", absPath)
if _, err := s.processMediaFile(ctx, absPath); err != nil {
fmt.Printf("[POLL-SYNC] Error scanning new file %s: %v\n", absPath, err)
}
}
}
}
}
fmt.Println("[POLL-SYNC] Filesystem sync completed")
return nil
}
// ============================================
// SCANNER ENHANCEMENTS
// ============================================
// calculateFileSHA256 calculates SHA-256 hash using streaming to avoid loading entire file into memory
func (s *MediaScanner) calculateFileSHA256(filePath string) (string, error) {
file, err := os.Open(filePath)
if err != nil {
return "", fmt.Errorf("failed to open file: %v", err)
}
defer file.Close()
hasher := sha256.New()
if _, err := io.Copy(hasher, file); err != nil {
return "", fmt.Errorf("failed to calculate hash: %v", err)
}
return hex.EncodeToString(hasher.Sum(nil)), nil
}
// OPFIdentifier represents an identifier from OPF metadata
type OPFIdentifier struct {
XMLName xml.Name `xml:"identifier"`
ID string `xml:"id,attr"`
Scheme string `xml:"scheme,attr"`
Content string `xml:",chardata"`
}
// OPFMetadata represents parsed OPF metadata
type OPFMetadata struct {
XMLName xml.Name `xml:"package"`
Version string `xml:"version,attr"`
Identifiers []OPFIdentifier `xml:"metadata>identifier"`
}
// extractOPFIdentifiers extracts identifiers from EPUB OPF file
func (s *MediaScanner) extractOPFIdentifiers(epubPath string) (opfIdentifier, opfUUID string, confidence string, err error) {
book, err := epub.ReadBook(epubPath)
if err != nil {
return "", "", "", fmt.Errorf("failed to open EPUB: %v", err)
}
// Extract OPF identifiers using go-epub library
identifiers, err := book.MetadataByKey("identifier")
if err != nil || len(identifiers) == 0 {
return "", "", "low", nil
}
var identifier, uuid string
for _, id := range identifiers {
id = strings.TrimSpace(id)
// Check for UUID format (urn:uuid:)
if strings.HasPrefix(strings.ToLower(id), "urn:uuid:") {
uuid = strings.TrimPrefix(strings.ToLower(id), "urn:uuid:")
continue
}
// Check if it's a plain UUID (8-4-4-4-12 format)
if isValidUUID(id) {
uuid = id
continue
}
// Check for ISBN
if strings.Contains(strings.ToLower(id), "isbn") {
isbn := s.extractISBNFromIdentifier(id)
if isbn != "" {
identifier = isbn
continue
}
}
// Use first identifier as fallback
if identifier == "" && id != "" {
identifier = id
}
}
confidence = s.determineHashConfidence(uuid, identifier)
return identifier, uuid, confidence, nil
}
// isValidUUID checks if string is a valid UUID (8-4-4-4-12 format)
func isValidUUID(idStr string) bool {
uuidRegex := regexp.MustCompile(`^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$`)
return uuidRegex.MatchString(idStr)
}
// extractISBNFromIdentifier extracts ISBN from identifier string
func (s *MediaScanner) extractISBNFromIdentifier(id string) string {
id = strings.TrimSpace(id)
// Remove "isbn:" prefix if present
if strings.HasPrefix(strings.ToLower(id), "isbn:") {
id = strings.TrimPrefix(strings.ToLower(id), "isbn:")
}
// Remove hyphens and spaces
isbn := regexp.MustCompile(`[\s-]`).ReplaceAllString(id, "")
// Check if it's a valid ISBN-10 or ISBN-13
if len(isbn) == 10 || len(isbn) == 13 {
return isbn
}
return ""
}
// determineHashConfidence determines confidence level based on available identifiers
func (s *MediaScanner) determineHashConfidence(uuid, identifier string) string {
if uuid != "" && isValidUUID(uuid) {
return "high"
}
if identifier != "" && (strings.Contains(strings.ToLower(identifier), "isbn") || len(identifier) >= 10) {
return "medium"
}
return "low"
}
// detectFormatType detects the format type based on file extension and content
func (s *MediaScanner) detectFormatType(filePath string) string {
base := strings.ToLower(filepath.Base(filePath))
// Check for compound extensions first (like .kepub.epub)
if strings.HasSuffix(base, ".kepub.epub") {
return "kepub"
}
ext := strings.ToLower(filepath.Ext(filePath))
switch ext {
case ".epub":
return "epub"
case ".kepub":
return "kepub"
case ".pdf":
return "pdf"
case ".cbz", ".cbr", ".cb7", ".cbt":
return "comic_archive"
case ".mobi":
return "mobi"
case ".azw", ".azw3":
return "kfx"
case ".txt":
return "txt"
case ".fb2":
return "fb2"
default:
return "unknown"
}
}
// extractHashInfo calculates hash and extracts OPF identifiers for a file
func (s *MediaScanner) extractHashInfo(filePath string) (*HashInfo, *FormatInfo, error) {
// Calculate SHA-256
fileSHA256, err := s.calculateFileSHA256(filePath)
if err != nil {
return nil, nil, fmt.Errorf("failed to calculate SHA-256: %v", err)
}
// Get file info
info, err := os.Stat(filePath)
if err != nil {
return nil, nil, fmt.Errorf("failed to get file info: %v", err)
}
// Extract OPF identifiers for EPUB files
var opfIdentifier, opfUUID, confidence string
if strings.HasSuffix(strings.ToLower(filePath), ".epub") {
opfIdentifier, opfUUID, confidence, err = s.extractOPFIdentifiers(filePath)
if err != nil {
// Non-fatal error, continue with low confidence
confidence = "low"
}
}
hashInfo := &HashInfo{
FileSHA256: fileSHA256,
OPFIdentifier: opfIdentifier,
OPFUUID: opfUUID,
HashConfidence: confidence,
}
formatInfo := &FormatInfo{
FormatType: s.detectFormatType(filePath),
FilePath: filePath,
FileSHA256: fileSHA256,
FileSizeBytes: info.Size(),
MimeType: s.getMimeType(filePath),
}
return hashInfo, formatInfo, nil
}
func (s *MediaScanner) getRelativePath(absolutePath string) string {
// Get the base folder paths from scanner
for _, baseFolder := range s.folders {
// Check if path is within this base folder
if relPath, ok := strings.CutPrefix(absolutePath, baseFolder); ok {
return strings.TrimPrefix(relPath, "/")
}
}
// Fallback: if no match, return as-is (shouldn't happen)
return absolutePath
}