john-okeefe
77990d0dc0
Force rescan was metadata-only: updateMediaItem never touched the hash identifiers, so a force scan could not backfill file_sha256 for items imported before hashing existed (or where extraction originally failed). Those items were invisible to content dedup and SHA-256 device matching with no way to fix short of delete + re-import. processMediaFile now refreshes hash identifiers in three cases: - force rescan (the admin Scan button becomes the backfill tool) - file size change (stored hash is stale - the bytes changed) - unchanged file with no stored hash (ordinary scans self-heal the legacy backlog incrementally, no admin action required) Each recompute runs recordHashConflictIfAny: when the freshly stored hash is now shared by more than one item in the library, the group is upserted into hash_conflicts for the admin Hash Conflicts page. The upsert is a no-op for already-tracked groups, so resolved 'keep both' decisions stick. Also extract a package-level computeFileSHA256 (the scanner method now delegates to it) so the startup backfill service can hash files without a scanner instance.
3324 lines
101 KiB
Go
3324 lines
101 KiB
Go
// Package services provides the core business logic layer for bookhoard,
|
|
// including media scanning, library management, search, analytics, and conversion services.
|
|
package services
|
|
|
|
import (
|
|
"archive/tar"
|
|
"archive/zip"
|
|
"bookhoard/internal/database"
|
|
"bookhoard/internal/utils"
|
|
"bytes"
|
|
"compress/bzip2"
|
|
"compress/gzip"
|
|
"context"
|
|
"crypto/sha256"
|
|
"encoding/hex"
|
|
"encoding/json"
|
|
"encoding/xml"
|
|
"errors"
|
|
"fmt"
|
|
"image"
|
|
_ "image/jpeg"
|
|
_ "image/png"
|
|
"io"
|
|
"io/fs"
|
|
"os"
|
|
"path/filepath"
|
|
"regexp"
|
|
"strconv"
|
|
"strings"
|
|
"sync"
|
|
"sync/atomic"
|
|
"time"
|
|
|
|
"bookhoard/internal/sevenzip"
|
|
|
|
epub "github.com/ArcadiaLin/go-epub"
|
|
"github.com/fsnotify/fsnotify"
|
|
"github.com/google/uuid"
|
|
"github.com/jackc/pgx/v5"
|
|
"github.com/jackc/pgx/v5/pgtype"
|
|
"github.com/nwaples/rardecode"
|
|
pdfcpuapi "github.com/pdfcpu/pdfcpu/pkg/api"
|
|
)
|
|
|
|
// MediaMetadata contains extracted metadata for media files (ebooks, comics, manga)
|
|
type MediaMetadata struct {
|
|
Title string
|
|
Author string
|
|
Description string
|
|
Series string
|
|
SeriesNumber int32
|
|
Publisher string
|
|
PublishDate time.Time
|
|
Contributors []string
|
|
CoverPath string
|
|
ISBN string
|
|
ASIN string
|
|
Tags []string
|
|
Genre string
|
|
|
|
FileHashInfo *HashInfo
|
|
FileFormats []*FormatInfo
|
|
|
|
// Reading direction fields (from ComicInfo.xml or computed)
|
|
MangaType string // Raw ComicInfo.xml Manga field
|
|
ReadingDirection string // Computed: auto, ltr, rtl, vertical
|
|
Language string // ISO 639-1 language code
|
|
|
|
// Additional metadata fields (from ComicInfo.xml or other metadata sources)
|
|
// Universal fields (apply to ebooks, audiobooks, comics)
|
|
SeriesCount int32 // Total items in series (Count field for comics, series count for books)
|
|
Volume int32 // Volume/omnibus number
|
|
Imprint string // Publisher imprint
|
|
AgeRating string // Age rating (Everyone, Teen, Mature, Adult)
|
|
WebURL string // URL to info page (Goodreads, ComicVine, etc.)
|
|
MetadataNotes string // Notes from metadata files (not user notes)
|
|
CommunityRating float64 // Pre-existing community rating (0-10)
|
|
PageCount int32 // Actual page count (images for comics, pages for PDF)
|
|
TotalCharacters int64 // Total text characters (for reflowable EPUBs)
|
|
ChapterCount int32 // Number of chapters detected
|
|
|
|
// Comic-specific fields
|
|
StoryArc string // Story arc name
|
|
IsBlackAndWhite bool // Black and white flag
|
|
AlternateInfo string // JSONB string of alternate series info
|
|
ScanInformation string // Scan information
|
|
Summary string // Summary from ComicInfo.xml
|
|
}
|
|
|
|
type HashInfo struct {
|
|
FileSHA256 string
|
|
OPFIdentifier string
|
|
OPFUUID string
|
|
HashConfidence string
|
|
}
|
|
|
|
type FormatInfo struct {
|
|
FormatType string
|
|
FilePath string
|
|
FileSHA256 string
|
|
FileSizeBytes int64
|
|
MimeType string
|
|
}
|
|
|
|
// MediaScanner scans library folders for media files (ebooks, comics, manga)
|
|
type MediaScanner struct {
|
|
db *database.Queries
|
|
watcher *fsnotify.Watcher
|
|
folders []string
|
|
adminID pgtype.UUID
|
|
defaultLibraryID pgtype.UUID
|
|
libraryTypes map[string][]string
|
|
forceRescan bool
|
|
logger *ScannerLogger
|
|
dirtyDirs map[string]time.Time
|
|
dirtyDirsMu sync.RWMutex
|
|
fileStability map[string]*atomic.Bool
|
|
fileStabilityMu sync.RWMutex
|
|
scanMutex sync.Mutex
|
|
scanInProgress atomic.Bool
|
|
watching atomic.Bool
|
|
settingsCache *SettingsCache
|
|
|
|
totalFiles int
|
|
newItems int
|
|
errors int
|
|
job *Job
|
|
}
|
|
|
|
// CalibreOPFMetadata represents intermediate parsed metadata from Calibre metadata.opf files
|
|
type CalibreOPFMetadata struct {
|
|
Title string
|
|
Authors []string
|
|
Tags []string
|
|
Description string
|
|
Publisher string
|
|
PublishDate *time.Time
|
|
Language string
|
|
ISBN string
|
|
ASIN string
|
|
UUID string
|
|
Contributors []string
|
|
Series string
|
|
SeriesIndex *float64
|
|
Rating *int32
|
|
Timestamp *time.Time
|
|
}
|
|
|
|
// NewMediaScanner creates a new media scanner instance.
|
|
//
|
|
// The fsnotify watcher is NOT created here. It is created lazily inside
|
|
// SetFolders only when watch=true (the long-lived watch-mode scanner).
|
|
// Ephemeral one-off scan jobs pass watch=false, so they never allocate a
|
|
// watcher (and thus can never panic on EMFILE/ENOSPC). This fixes the
|
|
// fd/inotify-watch leak where every scan job created a watcher that was
|
|
// never closed.
|
|
func NewMediaScanner(db *database.Queries) *MediaScanner {
|
|
return &MediaScanner{
|
|
db: db,
|
|
watcher: nil,
|
|
settingsCache: NewSettingsCache(30 * time.Second),
|
|
dirtyDirs: make(map[string]time.Time),
|
|
fileStability: make(map[string]*atomic.Bool),
|
|
watching: atomic.Bool{},
|
|
scanInProgress: atomic.Bool{},
|
|
folders: []string{},
|
|
adminID: pgtype.UUID{},
|
|
defaultLibraryID: pgtype.UUID{Valid: false},
|
|
libraryTypes: make(map[string][]string),
|
|
logger: NewScannerLogger(),
|
|
}
|
|
}
|
|
|
|
func (s *MediaScanner) GetPollInterval() time.Duration {
|
|
if cached, ok := s.settingsCache.Get("scan_poll_interval_seconds"); ok {
|
|
if seconds, err := strconv.Atoi(cached); err == nil {
|
|
return time.Duration(seconds) * time.Second
|
|
}
|
|
}
|
|
|
|
if s.db == nil {
|
|
return 5 * time.Minute
|
|
}
|
|
|
|
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
|
defer cancel()
|
|
|
|
setting, err := s.db.GetSystemSetting(ctx, "scan_poll_interval_seconds")
|
|
if err != nil || setting == "" {
|
|
return 5 * time.Minute
|
|
}
|
|
|
|
s.settingsCache.Set("scan_poll_interval_seconds", setting)
|
|
|
|
seconds, err := strconv.Atoi(setting)
|
|
if err != nil {
|
|
return 5 * time.Minute
|
|
}
|
|
return time.Duration(seconds) * time.Second
|
|
}
|
|
|
|
func (s *MediaScanner) GetAutoScanEnabled() bool {
|
|
// Check cache first
|
|
if cached, ok := s.settingsCache.Get("auto_scan_enabled"); ok {
|
|
return strings.ToLower(cached) == "true"
|
|
}
|
|
|
|
if s.db == nil {
|
|
return true
|
|
}
|
|
|
|
// Cache miss - query database
|
|
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
|
defer cancel()
|
|
|
|
setting, err := s.db.GetSystemSetting(ctx, "auto_scan_enabled")
|
|
if err != nil || setting == "" {
|
|
return true
|
|
}
|
|
|
|
// Store in cache
|
|
s.settingsCache.Set("auto_scan_enabled", setting)
|
|
|
|
return strings.ToLower(setting) == "true"
|
|
}
|
|
|
|
func (s *MediaScanner) SetAdminID(adminID pgtype.UUID) {
|
|
s.adminID = adminID
|
|
}
|
|
|
|
func (s *MediaScanner) SetLibraryID(libraryID pgtype.UUID) {
|
|
s.defaultLibraryID = libraryID
|
|
}
|
|
|
|
func (s *MediaScanner) SetForce(force bool) {
|
|
s.forceRescan = force
|
|
}
|
|
|
|
func (s *MediaScanner) GetStats() (int, int, int) {
|
|
return s.totalFiles, s.newItems, s.errors
|
|
}
|
|
|
|
// SetFolders configures the scanner's folders and (optionally) sets up an
|
|
// fsnotify watcher over the full directory tree.
|
|
//
|
|
// watch should be true only for the single long-lived watch-mode scanner that
|
|
// actually consumes watcher.Events. Ephemeral scan jobs must pass false so no
|
|
// watcher (and thus no fd/inotify watches) is allocated — the watcher is never
|
|
// read by scan jobs and previously leaked one watcher per job.
|
|
func (s *MediaScanner) SetFolders(folders []string, watch bool) error {
|
|
s.folders = folders
|
|
|
|
// Always close any previously-owned watcher so reconfiguration doesn't leak.
|
|
if s.watcher != nil {
|
|
if err := s.watcher.Close(); err != nil {
|
|
fmt.Printf("Warning: failed to close old watcher during folder reconfiguration: %v\n", err)
|
|
}
|
|
s.watcher = nil
|
|
}
|
|
|
|
// Create + populate a fresh watcher only when the caller intends to read events.
|
|
if watch {
|
|
watcher, err := fsnotify.NewWatcher()
|
|
if err != nil {
|
|
// Return an error instead of panicking so a failed watcher can't
|
|
// take down the whole process.
|
|
return fmt.Errorf("failed to create watcher: %w", err)
|
|
}
|
|
s.watcher = watcher
|
|
}
|
|
|
|
// Build cache of allowed extensions per folder
|
|
// Uses Go AllowedExtensions map as source of truth (not DB)
|
|
s.libraryTypes = make(map[string][]string)
|
|
ctx := context.Background()
|
|
|
|
for _, folder := range folders {
|
|
lib, err := s.db.GetLibraryByFolder(ctx, folder)
|
|
if err != nil {
|
|
fmt.Printf("Warning: failed to get library for folder %s: %v\n", folder, err)
|
|
continue
|
|
}
|
|
|
|
libType, err := s.db.GetLibraryType(ctx, lib.LibraryTypeID)
|
|
if err != nil {
|
|
fmt.Printf("Warning: failed to get library type for %s: %v\n", folder, err)
|
|
continue
|
|
}
|
|
|
|
if exts, ok := AllowedExtensions[libType.Name]; ok {
|
|
s.libraryTypes[folder] = exts
|
|
fmt.Printf("Scanner: Folder %s (type: %s) allows extensions: %v\n",
|
|
folder, libType.Name, exts)
|
|
} else {
|
|
s.libraryTypes[folder] = libType.AllowedExtensions
|
|
fmt.Printf("Scanner: Folder %s (type: %s) using DB extensions (no Go map entry): %v\n",
|
|
folder, libType.Name, libType.AllowedExtensions)
|
|
}
|
|
}
|
|
|
|
// Add all folders and their subdirectories to the watcher (like Audiobookshelf).
|
|
// Only when watching; scan jobs (watch=false) skip this entirely.
|
|
if s.watcher != nil {
|
|
watchCount := 0
|
|
for _, folder := range folders {
|
|
if err := s.watcher.Add(folder); err != nil {
|
|
fmt.Printf("[WATCHER] Warning: failed to watch root folder %s: %v\n", folder, err)
|
|
} else {
|
|
watchCount++
|
|
}
|
|
filepath.WalkDir(folder, func(path string, d fs.DirEntry, err error) error {
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if !d.IsDir() || path == folder {
|
|
return nil
|
|
}
|
|
if err := s.watcher.Add(path); err != nil {
|
|
fmt.Printf("[WATCHER] Warning: failed to watch subdirectory %s: %v\n", path, err)
|
|
} else {
|
|
watchCount++
|
|
}
|
|
return nil
|
|
})
|
|
}
|
|
|
|
fmt.Printf("[WATCHER] Now watching %d directories across %d root folders\n", watchCount, len(folders))
|
|
} else {
|
|
fmt.Printf("[SCANNER] Configured %d root folders (watch mode disabled, no inotify watcher)\n", len(folders))
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func (s *MediaScanner) enqueueLibraryScan(rootFolder string) {
|
|
if s.db == nil {
|
|
return
|
|
}
|
|
|
|
libRow, err := s.db.GetLibraryByFolderPathPrefix(context.Background(), rootFolder)
|
|
if err != nil {
|
|
fmt.Printf("[MTIME-POLL] Warning: could not find library for %s: %v\n", rootFolder, err)
|
|
return
|
|
}
|
|
|
|
folders, err := s.db.GetLibraryFolders(context.Background(), libRow.LibraryID)
|
|
if err != nil {
|
|
fmt.Printf("[MTIME-POLL] Warning: could not get folders for library: %v\n", err)
|
|
return
|
|
}
|
|
|
|
folderPaths := make([]string, len(folders))
|
|
for i, f := range folders {
|
|
folderPaths[i] = f.FolderPath
|
|
}
|
|
|
|
adminIDStr := ""
|
|
if libRow.CreatedByAdminID.Valid {
|
|
adminIDStr = uuid.UUID(libRow.CreatedByAdminID.Bytes).String()
|
|
}
|
|
if adminIDStr == "" {
|
|
fmt.Printf("[MTIME-POLL] Library has no owner, falling back to first admin\n")
|
|
fallbackAdmin, err := s.db.GetFirstAdmin(context.Background())
|
|
if err != nil {
|
|
fmt.Printf("[MTIME-POLL] Warning: no admin found in database, skipping scan\n")
|
|
return
|
|
}
|
|
adminIDStr = uuid.UUID(fallbackAdmin.Bytes).String()
|
|
}
|
|
|
|
libraryIDStr := uuid.UUID(libRow.LibraryID.Bytes).String()
|
|
|
|
job := &Job{
|
|
ID: uuid.New().String(),
|
|
Type: JobTypeScan,
|
|
Status: JobStatusPending,
|
|
UserID: adminIDStr,
|
|
Context: context.Background(),
|
|
Params: map[string]any{
|
|
"library_id": libraryIDStr,
|
|
"folders": folderPaths,
|
|
"admin_id": adminIDStr,
|
|
"db": s.db,
|
|
"force": false,
|
|
},
|
|
}
|
|
|
|
if WorkerInstance != nil {
|
|
WorkerInstance.Enqueue(job)
|
|
fmt.Printf("[MTIME-POLL] Enqueued library scan for %s (library: %s)\n", rootFolder, libraryIDStr)
|
|
}
|
|
}
|
|
|
|
func (s *MediaScanner) ScanFolders(ctx context.Context) error {
|
|
if len(s.folders) == 0 {
|
|
return fmt.Errorf("no folders set")
|
|
}
|
|
|
|
s.totalFiles = 0
|
|
s.newItems = 0
|
|
s.errors = 0
|
|
|
|
for _, folder := range s.folders {
|
|
if err := filepath.WalkDir(folder, func(path string, d fs.DirEntry, err error) error {
|
|
if !d.IsDir() && s.isScannableFile(path) {
|
|
s.totalFiles++
|
|
}
|
|
return nil
|
|
}); err != nil {
|
|
fmt.Printf("Warning: failed to count files in %s: %v\n", folder, err)
|
|
return nil
|
|
}
|
|
}
|
|
|
|
fmt.Printf("Starting scan of %d folders: %v (%d files to scan)\n", len(s.folders), s.folders, s.totalFiles)
|
|
|
|
processedFiles := 0
|
|
mediaFiles := 0
|
|
|
|
for _, folder := range s.folders {
|
|
fmt.Printf("Scanning folder: %s\n", folder)
|
|
|
|
if _, err := os.Stat(folder); os.IsNotExist(err) {
|
|
fmt.Printf("Folder does not exist: %s\n", folder)
|
|
s.errors++
|
|
continue
|
|
}
|
|
|
|
err := filepath.WalkDir(folder, func(path string, d fs.DirEntry, err error) error {
|
|
if err != nil {
|
|
fmt.Printf("Error accessing path %s: %v\n", path, err)
|
|
s.errors++
|
|
return err
|
|
}
|
|
|
|
if d.IsDir() {
|
|
if s.watcher != nil {
|
|
if err := s.watcher.Add(path); err != nil {
|
|
fmt.Printf("Warning: failed to watch subdirectory %s: %v\n", path, err)
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
if s.isScannableFile(path) {
|
|
mediaFiles++
|
|
processedFiles++
|
|
|
|
if processedFiles%10 == 0 && s.totalFiles > 0 {
|
|
progress := float64(processedFiles) / float64(s.totalFiles)
|
|
if s.job != nil {
|
|
s.job.UpdateProgress(progress, processedFiles, s.newItems, s.errors)
|
|
}
|
|
}
|
|
|
|
wasNew, err := s.processMediaFile(ctx, path)
|
|
if err != nil {
|
|
fmt.Printf("Error processing media file %s: %v\n", path, err)
|
|
s.errors++
|
|
} else if wasNew {
|
|
fmt.Printf("Successfully processed media file: %s\n", path)
|
|
} else {
|
|
fmt.Printf("Successfully processed media file: %s\n", path)
|
|
}
|
|
}
|
|
|
|
return nil
|
|
})
|
|
if err != nil {
|
|
s.errors++
|
|
return fmt.Errorf("failed to scan folder %s: %v", folder, err)
|
|
}
|
|
}
|
|
|
|
fmt.Printf("Scan completed: %d total files scanned, %d media files found, %d new items, %d errors\n",
|
|
processedFiles, mediaFiles, s.newItems, s.errors)
|
|
|
|
// Clean up: Find media items in DB that no longer exist on filesystem
|
|
for _, folder := range s.folders {
|
|
lib, err := s.db.GetLibraryByFolder(ctx, folder)
|
|
if err != nil {
|
|
continue
|
|
}
|
|
libraryID := lib.LibraryID
|
|
|
|
dbItems, err := s.db.ListMediaItemsByLibrary(ctx, libraryID)
|
|
if err != nil {
|
|
fmt.Printf("Warning: failed to get library items for cleanup: %v\n", err)
|
|
continue
|
|
}
|
|
|
|
// Build set of scanned file paths for this folder
|
|
scannedPaths := make(map[string]bool)
|
|
if err := filepath.WalkDir(folder, func(path string, d os.DirEntry, walkErr error) error {
|
|
if walkErr != nil {
|
|
return walkErr
|
|
}
|
|
if !d.IsDir() && s.isScannableFile(path) {
|
|
scannedPaths[s.getRelativePath(path)] = true
|
|
}
|
|
return nil
|
|
}); err != nil {
|
|
fmt.Printf("[RESCAN-CLEANUP] Warning: failed to walk directory %s, skipping orphan cleanup: %v\n", folder, err)
|
|
continue // Skip to next folder to avoid false deletions
|
|
}
|
|
|
|
// Delete items whose files no longer exist - with safety logging
|
|
for _, item := range dbItems {
|
|
filePath := item.FilePath
|
|
if filePath != "" && !scannedPaths[filePath] {
|
|
msg := fmt.Sprintf("[RESCAN-CLEANUP] Orphaned media item found: ID=%s, Title=%s, Path=%s",
|
|
item.ID, item.Title, filePath)
|
|
s.logger.LogDelete(msg)
|
|
|
|
delMsg := fmt.Sprintf("[RESCAN-CLEANUP] Deleting orphaned item '%s' (file no longer exists at %s)",
|
|
item.Title, filePath)
|
|
s.logger.LogDelete(delMsg)
|
|
|
|
if err := s.db.DeleteMediaItem(ctx, item.ID); err != nil {
|
|
errMsg := fmt.Sprintf("[RESCAN-CLEANUP] ERROR: failed to delete orphaned item %s: %v", item.Title, err)
|
|
s.logger.LogDelete(errMsg)
|
|
s.logger.LogError(errMsg)
|
|
} else {
|
|
s.logger.LogDelete(fmt.Sprintf("[RESCAN-CLEANUP] SUCCESS: deleted orphaned item '%s'", item.Title))
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
if s.job != nil && s.totalFiles > 0 {
|
|
s.job.UpdateProgress(1.0, processedFiles, s.newItems, s.errors)
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func (s *MediaScanner) isScannableFile(path string) bool {
|
|
ext := strings.ToLower(filepath.Ext(path))
|
|
|
|
// Find which folder this file belongs to
|
|
var folder string
|
|
for _, f := range s.folders {
|
|
if strings.HasPrefix(path, f) {
|
|
folder = f
|
|
break
|
|
}
|
|
}
|
|
|
|
// If no folder match, don't scan
|
|
if folder == "" {
|
|
return false
|
|
}
|
|
|
|
// Get allowed extensions for this folder's library
|
|
allowed, ok := s.libraryTypes[folder]
|
|
if !ok {
|
|
// No library type info, skip file
|
|
fmt.Printf("Warning: No library type info for folder %s, skipping %s\n", folder, path)
|
|
return false
|
|
}
|
|
|
|
// Check if file extension is allowed for this library type
|
|
for _, allowedExt := range allowed {
|
|
if ext == strings.ToLower(allowedExt) {
|
|
return true
|
|
}
|
|
}
|
|
|
|
return false
|
|
}
|
|
|
|
// extractFolderStructureMetadata extracts metadata from folder paths, prioritizing Calibre structure
|
|
func (s *MediaScanner) extractFolderStructureMetadata(path, rootFolder string) *MediaMetadata {
|
|
metadata := &MediaMetadata{}
|
|
|
|
// Get the relative path from root folder
|
|
relPath, err := filepath.Rel(rootFolder, path)
|
|
if err != nil {
|
|
return metadata
|
|
}
|
|
|
|
// Split into directory components
|
|
dir := filepath.Dir(relPath)
|
|
components := strings.Split(dir, string(filepath.Separator))
|
|
|
|
if len(components) < 2 {
|
|
return metadata // Not enough structure to extract
|
|
}
|
|
|
|
// Calibre structure detection
|
|
// Pattern 1: Author Name/Book Title/
|
|
// Pattern 2: Author Name/Series Name/Book Title/
|
|
// Pattern 3: Author Name/Series Name, Book #1 - Book Title/
|
|
|
|
author := strings.TrimSuffix(components[0], "_") // Remove trailing underscore if present
|
|
metadata.Author = strings.ReplaceAll(author, "_", " ")
|
|
|
|
if len(components) >= 3 {
|
|
// This might be a series structure
|
|
possibleSeries := components[1]
|
|
possibleTitle := components[2]
|
|
|
|
// Check for Calibre series format: "Series Name, Book #1 - Title"
|
|
seriesMatch := regexp.MustCompile(`^(.*),\s+Book\s+#(\d+)\s*-\s*(.*)$`).FindStringSubmatch(possibleSeries)
|
|
if len(seriesMatch) == 4 {
|
|
metadata.Series = strings.ReplaceAll(seriesMatch[1], "_", " ")
|
|
if seriesNum, err := strconv.ParseInt(seriesMatch[2], 10, 32); err == nil {
|
|
metadata.SeriesNumber = int32(seriesNum)
|
|
}
|
|
metadata.Title = strings.ReplaceAll(possibleTitle, "_", " ")
|
|
} else {
|
|
// Simple series structure: Author/Series/Title
|
|
metadata.Series = strings.ReplaceAll(possibleSeries, "_", " ")
|
|
metadata.Title = strings.ReplaceAll(possibleTitle, "_", " ")
|
|
|
|
// Try to extract series number from title
|
|
titleNumMatch := regexp.MustCompile(`^(.*)\s+(\d+)$`).FindStringSubmatch(metadata.Title)
|
|
if len(titleNumMatch) == 3 {
|
|
metadata.Title = titleNumMatch[1]
|
|
if seriesNum, err := strconv.ParseInt(titleNumMatch[2], 10, 32); err == nil {
|
|
metadata.SeriesNumber = int32(seriesNum)
|
|
}
|
|
}
|
|
}
|
|
} else {
|
|
// Simple structure: Author/Title
|
|
metadata.Title = strings.ReplaceAll(components[1], "_", " ")
|
|
}
|
|
|
|
return metadata
|
|
}
|
|
|
|
var bookExtensions = map[string]bool{
|
|
".epub": true, ".pdf": true, ".mobi": true, ".azw": true, ".azw3": true,
|
|
".fb2": true, ".txt": true, ".rtf": true, ".doc": true, ".docx": true,
|
|
".lit": true, ".pdb": true, ".djvu": true,
|
|
".cbz": true, ".cbr": true, ".cb7": true, ".cbt": true,
|
|
}
|
|
|
|
func hasSiblingBookFile(dir string) bool {
|
|
entries, err := os.ReadDir(dir)
|
|
if err != nil {
|
|
return false
|
|
}
|
|
for _, entry := range entries {
|
|
if !entry.IsDir() && bookExtensions[strings.ToLower(filepath.Ext(entry.Name()))] {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
func (s *MediaScanner) processMediaFile(ctx context.Context, path string) (bool, error) {
|
|
fmt.Printf("Processing media file: %s\n", path)
|
|
|
|
// Get file info
|
|
info, err := os.Stat(path)
|
|
if err != nil {
|
|
fmt.Printf("Failed to get file info for %s: %v\n", path, err)
|
|
return false, fmt.Errorf("failed to get file info: %v", err)
|
|
}
|
|
|
|
if isImageFile(path) && hasSiblingBookFile(filepath.Dir(path)) {
|
|
return false, nil
|
|
}
|
|
|
|
fileModTime := info.ModTime()
|
|
|
|
// Find library for this file's folder
|
|
var libraryID pgtype.UUID
|
|
if s.defaultLibraryID.Valid {
|
|
libraryID = s.defaultLibraryID
|
|
} else {
|
|
for _, folder := range s.folders {
|
|
if strings.HasPrefix(path, folder) {
|
|
lib, err := s.db.GetLibraryByFolder(ctx, folder)
|
|
if err != nil {
|
|
return false, fmt.Errorf("failed to find library for folder %s: %v", folder, err)
|
|
}
|
|
libraryID = lib.LibraryID
|
|
break
|
|
}
|
|
}
|
|
}
|
|
|
|
if !libraryID.Valid {
|
|
return false, fmt.Errorf("no library found for file path: %s", path)
|
|
}
|
|
|
|
// Check if media item already exists in database
|
|
existingItem, err := s.getMediaItemByFilePath(ctx, path, libraryID)
|
|
if err == nil {
|
|
fmt.Printf("Media item already exists in database: %s (size: %d vs %d)\n", path, existingItem.FileSize.Int64, info.Size())
|
|
|
|
// If force rescan is enabled, always re-process
|
|
if s.forceRescan {
|
|
fmt.Printf("Force rescan enabled, updating existing media item: %s\n", path)
|
|
// Use UPDATE instead of DELETE+INSERT to preserve created_at
|
|
if err := s.updateMediaItem(ctx, existingItem.ID, path, info); err != nil {
|
|
fmt.Printf("Warning: failed to update existing media item: %v\n", err)
|
|
}
|
|
// Recompute hash identifiers too - a force rescan is the admin's
|
|
// backfill tool and must refresh stale or missing hashes.
|
|
s.recomputeHashInfo(ctx, existingItem.ID, libraryID, path)
|
|
return false, nil
|
|
} else {
|
|
// Normal behavior: check if file has changed (by size)
|
|
if existingItem.FileSize.Int64 != info.Size() {
|
|
fmt.Printf("File size changed, updating media item: %s\n", path)
|
|
_ = s.updateMediaItem(ctx, existingItem.ID, path, info)
|
|
// The bytes changed, so any stored hash is stale.
|
|
s.recomputeHashInfo(ctx, existingItem.ID, libraryID, path)
|
|
return false, nil
|
|
}
|
|
// Self-heal items imported before hashing existed: even an unchanged
|
|
// file gets its hash computed if missing.
|
|
if !existingItem.FileSha256.Valid || existingItem.FileSha256.String == "" {
|
|
s.recomputeHashInfo(ctx, existingItem.ID, libraryID, path)
|
|
}
|
|
fmt.Printf("Media item already exists with same size, skipping: %s\n", path)
|
|
return false, nil
|
|
}
|
|
} else if !errors.Is(err, pgx.ErrNoRows) {
|
|
fmt.Printf("Database error checking media item existence: %v\n", err)
|
|
return false, fmt.Errorf("failed to check if media item exists: %v", err)
|
|
}
|
|
fmt.Printf("Media item does not exist in database, creating new entry: %s\n", path)
|
|
|
|
// Extract metadata from file first
|
|
metadata, err := s.extractMetadata(path)
|
|
if err != nil {
|
|
fmt.Printf("Warning: failed to extract metadata from %s: %v\n", path, err)
|
|
metadata = &MediaMetadata{}
|
|
}
|
|
|
|
// Extract hash information during metadata extraction
|
|
hashInfo, formatInfo, err := s.extractHashInfo(path)
|
|
if err != nil {
|
|
fmt.Printf("Warning: failed to extract hash info from %s: %v\n", path, err)
|
|
hashInfo = &HashInfo{}
|
|
} else {
|
|
metadata.FileHashInfo = hashInfo
|
|
metadata.FileFormats = []*FormatInfo{formatInfo}
|
|
fmt.Printf("Hash info for %s: SHA256=%s, OPF_ID=%s, OPF_UUID=%s, Confidence=%s\n",
|
|
path, hashInfo.FileSHA256, hashInfo.OPFIdentifier, hashInfo.OPFUUID, hashInfo.HashConfidence)
|
|
}
|
|
|
|
// Content dedup: if an item with the same SHA-256 already exists in this
|
|
// library (same file at a different path), treat it as existing rather than
|
|
// creating a duplicate. The file bytes are identical, so metadata matches.
|
|
if hashInfo.FileSHA256 != "" {
|
|
existingByHash, err := s.db.GetMediaItemBySHA256AndLibrary(ctx, database.GetMediaItemBySHA256AndLibraryParams{
|
|
FileSha256: pgtype.Text{String: hashInfo.FileSHA256, Valid: true},
|
|
LibraryID: libraryID,
|
|
})
|
|
if err == nil && existingByHash.ID.Valid {
|
|
fmt.Printf("Media item with same SHA-256 already exists in library (path %q), skipping duplicate: %s\n",
|
|
existingByHash.FilePath, path)
|
|
if s.forceRescan {
|
|
_ = s.updateMediaItem(ctx, existingByHash.ID, path, info)
|
|
}
|
|
return false, nil
|
|
} else if err != nil && !errors.Is(err, pgx.ErrNoRows) {
|
|
fmt.Printf("Warning: failed to check media item by SHA-256 for %s: %v\n", path, err)
|
|
}
|
|
}
|
|
|
|
// REMOVED: Comic metadata extraction now handled by mergeMetadata()
|
|
// This avoids duplicate extraction and ensures smart merging happens
|
|
|
|
// Try to get metadata from folder structure as fallback/enhancement
|
|
// Use the root folder that contains this file
|
|
var rootFolder string
|
|
for _, folder := range s.folders {
|
|
if strings.HasPrefix(path, folder) {
|
|
rootFolder = folder
|
|
break
|
|
}
|
|
}
|
|
|
|
if rootFolder != "" {
|
|
folderMetadata := s.extractFolderStructureMetadata(path, rootFolder)
|
|
|
|
// Use folder metadata as fallback for missing information
|
|
if metadata.Title == "" && folderMetadata.Title != "" {
|
|
metadata.Title = folderMetadata.Title
|
|
}
|
|
if metadata.Author == "" && folderMetadata.Author != "" {
|
|
metadata.Author = folderMetadata.Author
|
|
}
|
|
if metadata.Series == "" && folderMetadata.Series != "" {
|
|
metadata.Series = folderMetadata.Series
|
|
}
|
|
if metadata.SeriesNumber == 0 && folderMetadata.SeriesNumber > 0 {
|
|
metadata.SeriesNumber = folderMetadata.SeriesNumber
|
|
}
|
|
}
|
|
|
|
// Final fallback if still missing essential metadata
|
|
if metadata.Title == "" {
|
|
metadata.Title = strings.TrimSuffix(filepath.Base(path), filepath.Ext(path))
|
|
}
|
|
if metadata.Author == "" {
|
|
metadata.Author = "Unknown"
|
|
}
|
|
|
|
// libraryID already determined at start of function
|
|
|
|
// Normalize metadata fields for display
|
|
metadata.Contributors = utils.NormalizeContributors(metadata.Contributors)
|
|
metadata.Tags = utils.NormalizeTags(metadata.Tags)
|
|
|
|
// Normalize search fields
|
|
contributorsSearch := utils.NormalizeContributorsSearch(metadata.Contributors)
|
|
tagsSearch := utils.NormalizeTagsSearch(metadata.Tags)
|
|
|
|
// Create media item in database
|
|
relativePath := s.getRelativePath(path)
|
|
createdItem, err := s.db.CreateMediaItem(ctx, database.CreateMediaItemParams{
|
|
LibraryID: libraryID,
|
|
Title: metadata.Title,
|
|
Author: pgtype.Text{String: metadata.Author, Valid: metadata.Author != ""},
|
|
Isbn: pgtype.Text{String: utils.NormalizeISBNSafe(metadata.ISBN), Valid: metadata.ISBN != ""},
|
|
Asin: pgtype.Text{String: metadata.ASIN, Valid: metadata.ASIN != ""},
|
|
Description: pgtype.Text{String: metadata.Description, Valid: metadata.Description != ""},
|
|
FilePath: relativePath,
|
|
FileSize: pgtype.Int8{Int64: info.Size(), Valid: true},
|
|
MimeType: pgtype.Text{String: s.getMimeType(path), Valid: true},
|
|
CoverImagePath: pgtype.Text{String: metadata.CoverPath, Valid: metadata.CoverPath != ""},
|
|
Series: pgtype.Text{String: metadata.Series, Valid: metadata.Series != ""},
|
|
SeriesNumber: pgtype.Int4{Int32: metadata.SeriesNumber, Valid: metadata.SeriesNumber > 0},
|
|
Publisher: pgtype.Text{String: metadata.Publisher, Valid: metadata.Publisher != ""},
|
|
DatePublished: pgtype.Date{Time: metadata.PublishDate, Valid: !metadata.PublishDate.IsZero()},
|
|
Contributors: metadata.Contributors,
|
|
ContributorsSearch: contributorsSearch,
|
|
Tags: metadata.Tags,
|
|
TagsSearch: tagsSearch,
|
|
AddedByAdminID: s.adminID,
|
|
CreatedAt: pgtype.Timestamptz{Time: fileModTime, Valid: true},
|
|
ImportedAt: pgtype.Timestamptz{Time: time.Now(), Valid: true},
|
|
MangaType: pgtype.Text{String: metadata.MangaType, Valid: metadata.MangaType != ""},
|
|
ReadingDirection: pgtype.Text{String: metadata.ReadingDirection, Valid: metadata.ReadingDirection != ""},
|
|
SeriesCount: pgtype.Int4{Int32: metadata.SeriesCount, Valid: metadata.SeriesCount > 0},
|
|
Volume: pgtype.Int4{Int32: metadata.Volume, Valid: metadata.Volume > 0},
|
|
Imprint: pgtype.Text{String: metadata.Imprint, Valid: metadata.Imprint != ""},
|
|
AgeRating: pgtype.Text{String: metadata.AgeRating, Valid: metadata.AgeRating != ""},
|
|
WebUrl: pgtype.Text{String: metadata.WebURL, Valid: metadata.WebURL != ""},
|
|
StoryArc: pgtype.Text{String: metadata.StoryArc, Valid: metadata.StoryArc != ""},
|
|
IsBlackAndWhite: pgtype.Bool{Bool: metadata.IsBlackAndWhite, Valid: true},
|
|
MetadataNotes: pgtype.Text{String: metadata.MetadataNotes, Valid: metadata.MetadataNotes != ""},
|
|
AlternateInfo: func() []byte {
|
|
if metadata.AlternateInfo != "" {
|
|
return []byte(metadata.AlternateInfo)
|
|
}
|
|
return nil
|
|
}(),
|
|
ScanInformation: pgtype.Text{String: metadata.ScanInformation, Valid: metadata.ScanInformation != ""},
|
|
Summary: pgtype.Text{String: metadata.Summary, Valid: metadata.Summary != ""},
|
|
CommunityRating: pgtype.Float8{Float64: metadata.CommunityRating, Valid: metadata.CommunityRating > 0},
|
|
PageCount: pgtype.Int4{Int32: metadata.PageCount, Valid: metadata.PageCount > 0},
|
|
})
|
|
if err != nil {
|
|
return false, fmt.Errorf("failed to create media item: %v", err)
|
|
}
|
|
|
|
// Update hash information before database storage
|
|
if metadata.FileHashInfo != nil && metadata.FileHashInfo.FileSHA256 != "" {
|
|
_, err = s.db.UpdateMediaItemIdentifiers(ctx, database.UpdateMediaItemIdentifiersParams{
|
|
ID: createdItem.ID,
|
|
FileSha256: pgtype.Text{String: metadata.FileHashInfo.FileSHA256, Valid: true},
|
|
OpfIdentifier: pgtype.Text{String: metadata.FileHashInfo.OPFIdentifier, Valid: metadata.FileHashInfo.OPFIdentifier != ""},
|
|
OpfUuid: pgtype.Text{String: metadata.FileHashInfo.OPFUUID, Valid: metadata.FileHashInfo.OPFUUID != ""},
|
|
HashConfidence: pgtype.Text{String: metadata.FileHashInfo.HashConfidence, Valid: true},
|
|
})
|
|
if err != nil {
|
|
fmt.Printf("Warning: failed to update hash identifiers for %s: %v\n", path, err)
|
|
}
|
|
}
|
|
|
|
// Set format group, total characters, and chapter count
|
|
mimeType := s.getMimeType(path)
|
|
ext := strings.ToLower(filepath.Ext(path))
|
|
var formatGroup string
|
|
var isReflowable, hasFixedLayout bool
|
|
switch ext {
|
|
case ".epub":
|
|
isFixed, fixedErr := s.DetectFixedLayoutEPUB(path)
|
|
if fixedErr == nil && isFixed {
|
|
formatGroup = "fixed_layout"
|
|
hasFixedLayout = true
|
|
} else {
|
|
formatGroup = "reflowable"
|
|
isReflowable = true
|
|
}
|
|
case ".mobi", ".azw", ".azw3", ".fb2", ".txt":
|
|
formatGroup = "reflowable"
|
|
isReflowable = true
|
|
case ".pdf", ".djvu":
|
|
formatGroup = "fixed_layout"
|
|
hasFixedLayout = true
|
|
case ".cbz", ".cbr", ".cb7", ".cbt":
|
|
formatGroup = "comic_archive"
|
|
hasFixedLayout = true
|
|
default:
|
|
formatGroup = "unknown"
|
|
}
|
|
err = s.db.UpdateMediaItemFormatGroup(ctx, database.UpdateMediaItemFormatGroupParams{
|
|
ID: createdItem.ID,
|
|
FormatGroup: formatGroup,
|
|
FormatMimetype: pgtype.Text{String: mimeType, Valid: mimeType != ""},
|
|
IsReflowable: pgtype.Bool{Bool: isReflowable, Valid: true},
|
|
HasFixedLayout: pgtype.Bool{Bool: hasFixedLayout, Valid: true},
|
|
TotalCharacters: pgtype.Int8{Int64: metadata.TotalCharacters, Valid: metadata.TotalCharacters > 0},
|
|
ChapterCount: pgtype.Int4{Int32: metadata.ChapterCount, Valid: metadata.ChapterCount > 0},
|
|
})
|
|
if err != nil {
|
|
fmt.Printf("Warning: failed to update format info for %s: %v\n", path, err)
|
|
}
|
|
|
|
// Store format information in the database
|
|
for _, format := range metadata.FileFormats {
|
|
_, err = s.db.CreateMediaItemFormat(ctx, database.CreateMediaItemFormatParams{
|
|
MediaItemID: createdItem.ID,
|
|
FormatType: format.FormatType,
|
|
FilePath: pgtype.Text{String: s.getRelativePath(format.FilePath), Valid: true},
|
|
FileSha256: pgtype.Text{String: format.FileSHA256, Valid: true},
|
|
FileSizeBytes: pgtype.Int8{Int64: format.FileSizeBytes, Valid: true},
|
|
MimeType: pgtype.Text{String: format.MimeType, Valid: true},
|
|
})
|
|
if err != nil {
|
|
fmt.Printf("Warning: failed to create format entry for %s: %v\n", path, err)
|
|
}
|
|
}
|
|
|
|
s.newItems++
|
|
return true, nil
|
|
}
|
|
|
|
// extractCalibreSidecar checks for and parses a Calibre metadata.opf sidecar file
|
|
func (s *MediaScanner) extractCalibreSidecar(path string) *MediaMetadata {
|
|
// Get directory of media file
|
|
dir := filepath.Dir(path)
|
|
opfPath := filepath.Join(dir, "metadata.opf")
|
|
|
|
// Check if sidecar exists
|
|
if _, err := os.Stat(opfPath); os.IsNotExist(err) {
|
|
return nil // No sidecar, not an error
|
|
}
|
|
|
|
// Parse sidecar
|
|
metadata, err := s.parseCalibreMetadataOPF(opfPath)
|
|
if err != nil {
|
|
fmt.Printf("Warning: failed to parse Calibre metadata.opf: %v\n", err)
|
|
return nil // Parsing failed, fall back to embedded
|
|
}
|
|
|
|
return metadata
|
|
}
|
|
|
|
// mergeMetadata intelligently merges metadata from multiple sources
|
|
// Priority: metadata.opf (Calibre) → embedded metadata → folder structure → filename
|
|
// For comics: metadata.opf → ComicInfo.xml → folder structure → filename
|
|
func (s *MediaScanner) mergeMetadata(path string, calibreMetadata *MediaMetadata) (*MediaMetadata, error) {
|
|
metadata := calibreMetadata
|
|
if metadata == nil {
|
|
metadata = &MediaMetadata{}
|
|
}
|
|
|
|
ext := strings.ToLower(filepath.Ext(path))
|
|
|
|
// For EPUB files
|
|
if ext == ".epub" {
|
|
book, err := epub.ReadBook(path)
|
|
if err == nil {
|
|
genreTags := extractGenreTagsFromEPUB(book)
|
|
processGenresAndTags(metadata, genreTags)
|
|
if allText := book.AllChaptersText(); len(allText) > 0 {
|
|
metadata.TotalCharacters = int64(len(allText))
|
|
}
|
|
metadata.ChapterCount = int32(book.ChapterCount())
|
|
}
|
|
isFixed, fixedErr := s.DetectFixedLayoutEPUB(path)
|
|
if fixedErr == nil && isFixed {
|
|
if pageCount, imgErr := countArchiveImages(path); imgErr == nil && pageCount > 0 {
|
|
metadata.PageCount = int32(pageCount)
|
|
}
|
|
}
|
|
}
|
|
|
|
// For comic archives, try to extract ComicInfo.xml
|
|
if ext == ".cbz" || ext == ".cbr" || ext == ".cb7" || ext == ".cbt" {
|
|
comicInfo, cover, err := extractComicMetadata(path)
|
|
if err != nil {
|
|
fmt.Printf("Warning: failed to extract comic metadata from %s: %v\n", path, err)
|
|
} else if comicInfo != nil {
|
|
// Merge ComicInfo.xml fields (only if not already set from Calibre)
|
|
if metadata.Title == "" && comicInfo.Title != "" {
|
|
metadata.Title = comicInfo.Title
|
|
}
|
|
if metadata.Series == "" && comicInfo.Series != "" {
|
|
metadata.Series = comicInfo.Series
|
|
}
|
|
if metadata.SeriesNumber == 0 && comicInfo.Number > 0 {
|
|
metadata.SeriesNumber = int32(comicInfo.Number)
|
|
}
|
|
if metadata.Publisher == "" && comicInfo.Publisher != "" {
|
|
metadata.Publisher = comicInfo.Publisher
|
|
}
|
|
if metadata.Author == "" && comicInfo.Writer != "" {
|
|
metadata.Author = comicInfo.Writer
|
|
}
|
|
if metadata.Description == "" && comicInfo.Summary != "" {
|
|
metadata.Description = comicInfo.Summary
|
|
}
|
|
|
|
// NEW: Always extract reading direction from ComicInfo.xml
|
|
// (even if metadata.opf exists, since Calibre doesn't support this field)
|
|
metadata.MangaType = normalizeMangaType(comicInfo.Manga)
|
|
metadata.ReadingDirection = determineReadingDirection(comicInfo)
|
|
metadata.Language = comicInfo.LanguageISO
|
|
|
|
// NEW: Extract additional comic-specific fields
|
|
// Series information
|
|
if metadata.SeriesCount == 0 && comicInfo.Count > 0 {
|
|
metadata.SeriesCount = int32(comicInfo.Count)
|
|
}
|
|
if metadata.Volume == 0 && comicInfo.Volume > 0 {
|
|
metadata.Volume = int32(comicInfo.Volume)
|
|
}
|
|
|
|
// Publisher and classification
|
|
if metadata.Imprint == "" && comicInfo.Imprint != "" {
|
|
metadata.Imprint = comicInfo.Imprint
|
|
}
|
|
if metadata.StoryArc == "" && comicInfo.StoryArc != "" {
|
|
metadata.StoryArc = comicInfo.StoryArc
|
|
}
|
|
if metadata.AgeRating == "" && comicInfo.AgeRating != "" {
|
|
metadata.AgeRating = normalizeAgeRating(comicInfo.AgeRating)
|
|
}
|
|
|
|
// NEW: Process genres and tags (universal logic for all formats)
|
|
// Extract genre tags from ComicInfo.xml (Genre + Tags + Characters + Teams + Locations)
|
|
genreTags := extractGenreTagsFromComicInfo(comicInfo)
|
|
processGenresAndTags(metadata, genreTags)
|
|
|
|
// Additional metadata
|
|
if metadata.WebURL == "" && comicInfo.Web != "" {
|
|
metadata.WebURL = comicInfo.Web
|
|
}
|
|
if metadata.MetadataNotes == "" && comicInfo.Notes != "" {
|
|
metadata.MetadataNotes = comicInfo.Notes
|
|
}
|
|
if metadata.ScanInformation == "" && comicInfo.ScanInformation != "" {
|
|
metadata.ScanInformation = comicInfo.ScanInformation
|
|
}
|
|
if metadata.Summary == "" && comicInfo.Summary != "" {
|
|
metadata.Summary = comicInfo.Summary
|
|
}
|
|
|
|
// Boolean fields
|
|
if !metadata.IsBlackAndWhite && strings.ToLower(comicInfo.BlackAndWhite) == "yes" {
|
|
metadata.IsBlackAndWhite = true
|
|
}
|
|
if metadata.CommunityRating == 0 && comicInfo.CommunityRating > 0 {
|
|
metadata.CommunityRating = comicInfo.CommunityRating
|
|
}
|
|
|
|
// Alternate series information (store as JSONB string)
|
|
if metadata.AlternateInfo == "" && (comicInfo.AlternateSeries != "" || comicInfo.AlternateNumber > 0) {
|
|
alternateData := map[string]any{}
|
|
if comicInfo.AlternateSeries != "" {
|
|
alternateData["alternate_series"] = comicInfo.AlternateSeries
|
|
}
|
|
if comicInfo.AlternateNumber > 0 {
|
|
alternateData["alternate_number"] = comicInfo.AlternateNumber
|
|
}
|
|
if comicInfo.AlternateCount > 0 {
|
|
alternateData["alternate_count"] = comicInfo.AlternateCount
|
|
}
|
|
if len(alternateData) > 0 {
|
|
jsonBytes, err := json.Marshal(alternateData)
|
|
if err == nil {
|
|
metadata.AlternateInfo = string(jsonBytes)
|
|
}
|
|
}
|
|
}
|
|
|
|
// REMOVED: Tag enhancement now handled by processGenresAndTags()
|
|
// Characters, Teams, Locations are already processed via extractGenreTagsFromComicInfo()
|
|
|
|
// Extract cover if not already present
|
|
if len(cover) > 0 && metadata.CoverPath == "" {
|
|
coverPath := path + ".cover.jpg"
|
|
if err := os.WriteFile(coverPath, cover, 0644); err == nil {
|
|
metadata.CoverPath = s.getRelativePath(coverPath)
|
|
}
|
|
}
|
|
|
|
fmt.Printf("Merged comic metadata from %s: title=%s, series=%s, issue=%d, manga=%s, direction=%s\n",
|
|
path, comicInfo.Title, comicInfo.Series, comicInfo.Number, comicInfo.Manga, metadata.ReadingDirection)
|
|
}
|
|
|
|
if pageCount, err := countArchiveImages(path); err == nil && pageCount > 0 {
|
|
metadata.PageCount = int32(pageCount)
|
|
}
|
|
}
|
|
|
|
return metadata, nil
|
|
}
|
|
|
|
// containsTag checks if a tag already exists in the tags array
|
|
func containsTag(tags []string, tag string) bool {
|
|
tag = strings.ToLower(tag)
|
|
for _, t := range tags {
|
|
if strings.ToLower(t) == tag {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// normalizeMangaType normalizes ComicInfo.xml Manga field to database enum values
|
|
func normalizeMangaType(manga string) string {
|
|
switch strings.ToLower(strings.ReplaceAll(manga, " ", "")) {
|
|
case "unknown":
|
|
return "unknown"
|
|
case "no":
|
|
return "no"
|
|
case "yes":
|
|
return "yes"
|
|
case "yesandrighttoleft":
|
|
return "yes_and_right_to_left"
|
|
default:
|
|
return "unknown"
|
|
}
|
|
}
|
|
|
|
// determineReadingDirection computes reading direction from ComicInfo metadata
|
|
// Uses Manga field + language heuristics + genre tags
|
|
func determineReadingDirection(comicInfo *ComicInfo) string {
|
|
// 1. Check explicit Manga field
|
|
manga := normalizeMangaType(comicInfo.Manga)
|
|
switch manga {
|
|
case "yes_and_right_to_left":
|
|
return "rtl" // Traditional Japanese manga
|
|
case "yes", "no":
|
|
return "ltr" // Manga style but LTR, or Western comic
|
|
}
|
|
|
|
// 2. Language heuristic: Japanese → RTL
|
|
lang := strings.ToLower(comicInfo.LanguageISO)
|
|
if lang == "ja" || lang == "jpn" {
|
|
return "rtl"
|
|
}
|
|
|
|
// 3. Genre heuristic: webtoons/manhwa → vertical
|
|
tags := strings.ToLower(comicInfo.Tags + " " + comicInfo.Genre)
|
|
if strings.Contains(tags, "webtoon") || strings.Contains(tags, "manhwa") {
|
|
return "vertical" // Korean/Chinese webcomics
|
|
}
|
|
if strings.Contains(tags, "manga") {
|
|
return "rtl" // Japanese manga
|
|
}
|
|
|
|
// 4. Default: LTR (Western comics)
|
|
return "ltr"
|
|
}
|
|
|
|
// normalizeAgeRating normalizes age rating from ComicInfo.xml to standard values
|
|
func normalizeAgeRating(rating string) string {
|
|
rating = strings.ToLower(strings.TrimSpace(rating))
|
|
switch rating {
|
|
case "everyone", "e", "all ages":
|
|
return "Everyone"
|
|
case "teen", "t", "13+", "13+up":
|
|
return "Teen"
|
|
case "mature", "m", "17+", "17+up", "adults only":
|
|
return "Mature"
|
|
case "adult", "a", "18+":
|
|
return "Adult"
|
|
default:
|
|
return rating // Return original if unknown
|
|
}
|
|
}
|
|
|
|
// processGenresAndTags ensures ALL genres appear in the tags array without duplication
|
|
// This applies to ALL formats: EPUB, ComicInfo.xml, PDF metadata
|
|
// Strategy: Use existing `genre` column for primary genre, `tags` array for all genres
|
|
func processGenresAndTags(metadata *MediaMetadata, genreTags []string) {
|
|
if metadata.Tags == nil {
|
|
metadata.Tags = []string{}
|
|
}
|
|
|
|
// 1. Set primary genre (first genre tag wins if not already set)
|
|
if metadata.Genre == "" && len(genreTags) > 0 {
|
|
metadata.Genre = genreTags[0]
|
|
}
|
|
|
|
// 2. Ensure ALL genre tags appear in tags array (without duplication)
|
|
for _, genreTag := range genreTags {
|
|
genreTag = strings.TrimSpace(genreTag)
|
|
if genreTag != "" && !containsTag(metadata.Tags, genreTag) {
|
|
metadata.Tags = append(metadata.Tags, genreTag)
|
|
}
|
|
}
|
|
}
|
|
|
|
// extractGenreTagsFromEPUB extracts all <dc:subject> values from EPUB
|
|
// Returns array of genre tags
|
|
func extractGenreTagsFromEPUB(book *epub.Book) []string {
|
|
var genreTags []string
|
|
|
|
// EPUB stores genres in <dc:subject> metadata
|
|
if subjects, err := book.MetadataByKey("subject"); err == nil && len(subjects) > 0 {
|
|
for _, subject := range subjects {
|
|
subject = strings.TrimSpace(subject)
|
|
if subject != "" {
|
|
genreTags = append(genreTags, subject)
|
|
}
|
|
}
|
|
}
|
|
|
|
return genreTags
|
|
}
|
|
|
|
// extractGenreTagsFromComicInfo extracts genres from ComicInfo.xml
|
|
// Genre field + Tags field + Characters + Teams + Locations
|
|
// Returns array of genre tags
|
|
func extractGenreTagsFromComicInfo(comicInfo *ComicInfo) []string {
|
|
var genreTags []string
|
|
|
|
// 1. Add Genre field
|
|
if comicInfo.Genre != "" {
|
|
genreTags = append(genreTags, strings.Split(comicInfo.Genre, ",")...)
|
|
}
|
|
|
|
// 2. Add Tags field (comma-separated)
|
|
if comicInfo.Tags != "" {
|
|
genreTags = append(genreTags, strings.Split(comicInfo.Tags, ",")...)
|
|
}
|
|
|
|
// 3. Add Characters (comma-separated)
|
|
if comicInfo.Characters != "" {
|
|
genreTags = append(genreTags, strings.Split(comicInfo.Characters, ",")...)
|
|
}
|
|
|
|
// 4. Add Teams (comma-separated)
|
|
if comicInfo.Teams != "" {
|
|
genreTags = append(genreTags, strings.Split(comicInfo.Teams, ",")...)
|
|
}
|
|
|
|
// 5. Add Locations (comma-separated)
|
|
if comicInfo.Locations != "" {
|
|
genreTags = append(genreTags, strings.Split(comicInfo.Locations, ",")...)
|
|
}
|
|
|
|
// Trim whitespace from all tags
|
|
for i := range genreTags {
|
|
genreTags[i] = strings.TrimSpace(genreTags[i])
|
|
}
|
|
|
|
return genreTags
|
|
}
|
|
|
|
func (s *MediaScanner) extractMetadata(path string) (*MediaMetadata, error) {
|
|
// Try Calibre sidecar first
|
|
calibreMetadata := s.extractCalibreSidecar(path)
|
|
if calibreMetadata != nil {
|
|
fmt.Printf("Using Calibre metadata.opf for %s\n", path)
|
|
|
|
// Try to find cover image for sidecar metadata
|
|
coverPath := findSidecarCover(path)
|
|
if coverPath != "" {
|
|
calibreMetadata.CoverPath = s.getRelativePath(coverPath)
|
|
}
|
|
|
|
return s.mergeMetadata(path, calibreMetadata)
|
|
}
|
|
|
|
// EXISTING: Fallback to embedded metadata
|
|
ext := strings.ToLower(filepath.Ext(path))
|
|
|
|
switch ext {
|
|
case ".epub", ".kepub":
|
|
metadata := &MediaMetadata{}
|
|
result, err := s.extractEPUBMetadata(path)
|
|
if err == nil {
|
|
return result, nil
|
|
}
|
|
if result != nil {
|
|
metadata = result
|
|
}
|
|
// Enhanced format detection for EPUBs
|
|
isFixedLayout, detectErr := s.DetectFixedLayoutEPUB(path)
|
|
if detectErr == nil && isFixedLayout {
|
|
metadata.FileFormats = []*FormatInfo{{
|
|
FormatType: "fixed_layout",
|
|
FilePath: path,
|
|
MimeType: s.getMimeType(path),
|
|
}}
|
|
if pageCount, imgErr := countArchiveImages(path); imgErr == nil && pageCount > 0 {
|
|
metadata.PageCount = int32(pageCount)
|
|
}
|
|
}
|
|
// Try to extract embedded cover
|
|
coverPath, err := s.extractEPUBCover(path)
|
|
if err != nil {
|
|
fmt.Printf("Warning: failed to extract EPUB cover from %s: %v\n", path, err)
|
|
} else if coverPath != "" {
|
|
metadata.CoverPath = s.getRelativePath(coverPath)
|
|
}
|
|
// If no embedded cover, try sidecar
|
|
if metadata.CoverPath == "" {
|
|
sidecarCover := findSidecarCover(path)
|
|
if sidecarCover != "" {
|
|
metadata.CoverPath = s.getRelativePath(sidecarCover)
|
|
}
|
|
}
|
|
return metadata, nil
|
|
case ".pdf":
|
|
return s.extractPDFMetadata(path)
|
|
case ".cbz", ".cbr", ".cb7", ".cbt":
|
|
metadata, err := s.mergeMetadata(path, nil)
|
|
if err != nil {
|
|
return &MediaMetadata{
|
|
Title: strings.TrimSuffix(filepath.Base(path), ext),
|
|
}, nil
|
|
}
|
|
if metadata.Title == "" {
|
|
metadata.Title = strings.TrimSuffix(filepath.Base(path), ext)
|
|
}
|
|
// If no cover from archive, try sidecar
|
|
if metadata.CoverPath == "" {
|
|
sidecarCover := findSidecarCover(path)
|
|
if sidecarCover != "" {
|
|
metadata.CoverPath = s.getRelativePath(sidecarCover)
|
|
}
|
|
}
|
|
return metadata, nil
|
|
default:
|
|
// For other formats, return basic metadata
|
|
return &MediaMetadata{
|
|
Title: strings.TrimSuffix(filepath.Base(path), ext),
|
|
}, nil
|
|
}
|
|
}
|
|
|
|
func (s *MediaScanner) extractEPUBMetadata(path string) (*MediaMetadata, error) {
|
|
book, err := epub.ReadBook(path)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to open EPUB: %v", err)
|
|
}
|
|
|
|
metadata := &MediaMetadata{}
|
|
|
|
// Title
|
|
if title, err := book.Title(); err == nil && title != "" {
|
|
metadata.Title = title
|
|
}
|
|
|
|
// Author
|
|
if authors, err := book.MetadataByKey("creator"); err == nil && len(authors) > 0 {
|
|
metadata.Author = authors[0]
|
|
}
|
|
|
|
// Description
|
|
if descriptions, err := book.MetadataByKey("description"); err == nil && len(descriptions) > 0 {
|
|
metadata.Description = descriptions[0]
|
|
}
|
|
|
|
// Publisher
|
|
if publishers, err := book.MetadataByKey("publisher"); err == nil && len(publishers) > 0 {
|
|
metadata.Publisher = publishers[0]
|
|
}
|
|
|
|
// Series and series number (Calibre specific metadata)
|
|
if series, err := book.MetadataByKey("calibre:series"); err == nil && len(series) > 0 {
|
|
metadata.Series = series[0]
|
|
}
|
|
if seriesIndex, err := book.MetadataByKey("calibre:series_index"); err == nil && len(seriesIndex) > 0 {
|
|
if index, err := strconv.ParseFloat(seriesIndex[0], 32); err == nil {
|
|
metadata.SeriesNumber = int32(index)
|
|
}
|
|
}
|
|
|
|
// Publish date
|
|
if dates, err := book.MetadataByKey("date"); err == nil && len(dates) > 0 {
|
|
if date, err := time.Parse("2006-01-02", dates[0]); err == nil {
|
|
metadata.PublishDate = date
|
|
} else {
|
|
// Try alternative date formats
|
|
if date, err := time.Parse("2006", dates[0]); err == nil {
|
|
metadata.PublishDate = date
|
|
}
|
|
}
|
|
}
|
|
|
|
// Contributors
|
|
if contributors, err := book.MetadataByKey("contributor"); err == nil && len(contributors) > 0 {
|
|
// Normalize contributors for display
|
|
metadata.Contributors = utils.NormalizeContributors(contributors)
|
|
}
|
|
|
|
// ISBN
|
|
if isbns, err := book.MetadataByKey("identifier"); err == nil && len(isbns) > 0 {
|
|
for _, isbn := range isbns {
|
|
if strings.Contains(strings.ToLower(isbn), "isbn") {
|
|
// Extract ISBN number from identifier like "isbn:978-3-16-148410-0"
|
|
isbnParts := strings.SplitN(isbn, ":", 2)
|
|
if len(isbnParts) == 2 {
|
|
metadata.ISBN = isbnParts[1]
|
|
break
|
|
}
|
|
}
|
|
if strings.Contains(strings.ToLower(isbn), "asin") {
|
|
// Extract ASIN from identifier like "asin:B08XXXXX"
|
|
asinParts := strings.SplitN(isbn, ":", 2)
|
|
if len(asinParts) == 2 {
|
|
metadata.ASIN = asinParts[1]
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// Tags
|
|
if tags, err := book.MetadataByKey("subject"); err == nil && len(tags) > 0 {
|
|
// Normalize tags for display
|
|
metadata.Tags = utils.NormalizeTags(tags)
|
|
}
|
|
|
|
return metadata, nil
|
|
}
|
|
|
|
// DetectFixedLayoutEPUB checks if EPUB has fixed-layout (manga) characteristics
|
|
// by examining the OPF file for rendition metadata and content indicators
|
|
func (s *MediaScanner) DetectFixedLayoutEPUB(epubPath string) (bool, error) {
|
|
// Open EPUB ZIP file
|
|
r, err := zip.OpenReader(epubPath)
|
|
if err != nil {
|
|
return false, fmt.Errorf("failed to open EPUB: %w", err)
|
|
}
|
|
defer func() {
|
|
if err := r.Close(); err != nil {
|
|
fmt.Printf("Warning: failed to close EPUB file %s: %v\n", epubPath, err)
|
|
}
|
|
}()
|
|
|
|
// Find and read OPF file
|
|
var opfFile *zip.File
|
|
for _, f := range r.File {
|
|
if strings.HasSuffix(f.Name, ".opf") {
|
|
opfFile = f
|
|
break
|
|
}
|
|
// Also check in META-INF directory
|
|
if strings.Contains(f.Name, "META-INF/") && strings.HasSuffix(f.Name, ".opf") {
|
|
opfFile = f
|
|
break
|
|
}
|
|
}
|
|
|
|
if opfFile == nil {
|
|
return false, fmt.Errorf("OPF file not found in EPUB")
|
|
}
|
|
|
|
// Read OPF content
|
|
rc, err := opfFile.Open()
|
|
if err != nil {
|
|
return false, fmt.Errorf("failed to open OPF: %w", err)
|
|
}
|
|
defer func() {
|
|
if err := rc.Close(); err != nil {
|
|
fmt.Printf("Warning: failed to close OPF file reader from %s: %v\n", epubPath, err)
|
|
}
|
|
}()
|
|
|
|
opfContent, err := io.ReadAll(rc)
|
|
if err != nil {
|
|
return false, fmt.Errorf("failed to read OPF: %w", err)
|
|
}
|
|
|
|
// Check for fixed-layout indicators
|
|
opfString := string(opfContent)
|
|
|
|
// Check 1: rendition:layout = pre-paginated (EPUB 3 fixed layout)
|
|
if strings.Contains(opfString, `rendition:layout">pre-paginated<`) ||
|
|
strings.Contains(opfString, `rendition:layout="pre-paginated"`) {
|
|
return true, nil
|
|
}
|
|
|
|
// Check 2: RTL page progression (manga indicator)
|
|
if strings.Contains(opfString, `page-progression-direction="rtl"`) {
|
|
return true, nil
|
|
}
|
|
|
|
// Check 3: Image-heavy content (count <img> tags)
|
|
// Threshold of 50 images suggests manga/comic vs novel
|
|
imgCount := strings.Count(opfString, `<img`)
|
|
if imgCount > 50 {
|
|
return true, nil
|
|
}
|
|
|
|
// Check 4: Manga subject tag
|
|
lowerOPF := strings.ToLower(opfString)
|
|
if strings.Contains(lowerOPF, `<dc:subject`) &&
|
|
(strings.Contains(lowerOPF, `manga`) ||
|
|
strings.Contains(lowerOPF, `comic`)) {
|
|
return true, nil
|
|
}
|
|
|
|
return false, nil
|
|
}
|
|
|
|
// ValidateMediaItemForLibrary checks if item matches library type expectations
|
|
// and returns issue description if validation fails, nil if valid
|
|
func (s *MediaScanner) ValidateMediaItemForLibrary(
|
|
ctx context.Context,
|
|
mediaItem database.MediaItems,
|
|
library database.GetLibraryWithTypeRow,
|
|
) *string {
|
|
// Get library type from the joined query result
|
|
libraryTypeName := library.TypeName
|
|
|
|
// Manga library validation
|
|
if libraryTypeName == "manga" {
|
|
// Must be fixed-layout or comic archive
|
|
if mediaItem.FormatGroup != "fixed_layout" &&
|
|
mediaItem.FormatGroup != "comic_archive" {
|
|
str := fmt.Sprintf(
|
|
"EPUB file '%s' is reflowable (text-based), not fixed-layout (image-based). "+
|
|
"Manga library only accepts fixed-layout EPUBs, CBZ, CBR, or image files. "+
|
|
"Consider moving this file to an ebooks library.",
|
|
mediaItem.Title,
|
|
)
|
|
return &str
|
|
}
|
|
|
|
// Set manga-specific flags for fixed-layout EPUBs
|
|
if mediaItem.FormatGroup == "fixed_layout" {
|
|
// Default manga type if not set
|
|
if !mediaItem.MangaType.Valid ||
|
|
mediaItem.MangaType.String == "unknown" ||
|
|
mediaItem.MangaType.String == "" {
|
|
mediaItem.MangaType = pgtype.Text{String: "yes", Valid: true}
|
|
}
|
|
// Default reading direction for manga
|
|
if !mediaItem.ReadingDirection.Valid ||
|
|
mediaItem.ReadingDirection.String == "auto" ||
|
|
mediaItem.ReadingDirection.String == "" {
|
|
mediaItem.ReadingDirection = pgtype.Text{String: "rtl", Valid: true}
|
|
}
|
|
}
|
|
}
|
|
|
|
// Comics library validation
|
|
if libraryTypeName == "comics" {
|
|
// Accept comic archives and fixed-layout
|
|
if mediaItem.FormatGroup != "comic_archive" &&
|
|
mediaItem.FormatGroup != "fixed_layout" {
|
|
str := fmt.Sprintf(
|
|
"File '%s' is not a comic archive format. "+
|
|
"Comics library only accepts CBZ, CBR, CB7, CBT, PDF, or fixed-layout EPUBs.",
|
|
mediaItem.Title,
|
|
)
|
|
return &str
|
|
}
|
|
}
|
|
|
|
// Ebooks library validation
|
|
if libraryTypeName == "ebooks" {
|
|
// Flag manga for potential reorganization (info level)
|
|
if mediaItem.FormatGroup == "fixed_layout" &&
|
|
(!mediaItem.MangaType.Valid || mediaItem.MangaType.String == "yes" || mediaItem.MangaType.String == "yes_and_right_to_left") {
|
|
str := fmt.Sprintf(
|
|
"File '%s' appears to be manga (fixed-layout with images). "+
|
|
"Consider moving to a manga or comics library for better organization.",
|
|
mediaItem.Title,
|
|
)
|
|
return &str
|
|
}
|
|
}
|
|
|
|
return nil // No issues
|
|
}
|
|
|
|
// LogProcessingIssue records items that can't be processed properly
|
|
func (s *MediaScanner) LogProcessingIssue(
|
|
ctx context.Context,
|
|
mediaItemID uuid.UUID,
|
|
libraryID uuid.UUID,
|
|
issueType string,
|
|
description string,
|
|
severity string,
|
|
) error {
|
|
_, err := s.db.CreateProcessingIssue(ctx, database.CreateProcessingIssueParams{
|
|
MediaItemID: pgtype.UUID{Bytes: mediaItemID, Valid: true},
|
|
LibraryID: pgtype.UUID{Bytes: libraryID, Valid: true},
|
|
IssueType: issueType,
|
|
IssueDescription: description,
|
|
Severity: severity,
|
|
})
|
|
return err
|
|
}
|
|
|
|
// parseCalibreMetadataOPF parses a Calibre metadata.opf file and extracts metadata
|
|
func (s *MediaScanner) parseCalibreMetadataOPF(opfPath string) (*MediaMetadata, error) {
|
|
// Open file
|
|
file, err := os.Open(opfPath)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to open metadata.opf: %v", err)
|
|
}
|
|
defer func() {
|
|
if err := file.Close(); err != nil {
|
|
fmt.Printf("Warning: failed to close metadata.opf: %v\n", err)
|
|
}
|
|
}()
|
|
// Define XML structure for parsing with full Dublin Core namespace URLs
|
|
var opf struct {
|
|
XMLName xml.Name `xml:"package"`
|
|
Metadata struct {
|
|
XMLName xml.Name `xml:"metadata"`
|
|
Titles []string `xml:"http://purl.org/dc/elements/1.1/ title"`
|
|
Creators []string `xml:"http://purl.org/dc/elements/1.1/ creator"`
|
|
Subjects []string `xml:"http://purl.org/dc/elements/1.1/ subject"`
|
|
Desc []string `xml:"http://purl.org/dc/elements/1.1/ description"`
|
|
Publisher []string `xml:"http://purl.org/dc/elements/1.1/ publisher"`
|
|
Dates []string `xml:"http://purl.org/dc/elements/1.1/ date"`
|
|
Language []string `xml:"http://purl.org/dc/elements/1.1/ language"`
|
|
Identifiers []struct {
|
|
Scheme string `xml:"http://www.idpf.org/2007/opf scheme,attr"`
|
|
Value string `xml:",chardata"`
|
|
} `xml:"http://purl.org/dc/elements/1.1/ identifier"`
|
|
Contributors []string `xml:"http://purl.org/dc/elements/1.1/ contributor"`
|
|
// Calibre-specific meta tags - capture all, filter later
|
|
MetaTags []struct {
|
|
Name string `xml:"name,attr"`
|
|
Value string `xml:"content,attr"`
|
|
} `xml:"meta"`
|
|
} `xml:"metadata"`
|
|
}
|
|
// Parse XML
|
|
if err := xml.NewDecoder(file).Decode(&opf); err != nil {
|
|
return nil, fmt.Errorf("failed to parse metadata.opf XML: %v", err)
|
|
}
|
|
// Map to MediaMetadata struct
|
|
metadata := &MediaMetadata{}
|
|
// Title (required)
|
|
if len(opf.Metadata.Titles) > 0 {
|
|
metadata.Title = opf.Metadata.Titles[0]
|
|
}
|
|
// Author (first creator)
|
|
if len(opf.Metadata.Creators) > 0 {
|
|
metadata.Author = opf.Metadata.Creators[0]
|
|
}
|
|
// Tags (all subjects)
|
|
if len(opf.Metadata.Subjects) > 0 {
|
|
metadata.Tags = utils.NormalizeTags(opf.Metadata.Subjects)
|
|
}
|
|
// Description
|
|
if len(opf.Metadata.Desc) > 0 {
|
|
metadata.Description = opf.Metadata.Desc[0]
|
|
}
|
|
// Publisher
|
|
if len(opf.Metadata.Publisher) > 0 {
|
|
metadata.Publisher = opf.Metadata.Publisher[0]
|
|
}
|
|
// Publish date
|
|
if len(opf.Metadata.Dates) > 0 {
|
|
if date, err := time.Parse("2006-01-02T15:04:05Z07:00", opf.Metadata.Dates[0]); err == nil {
|
|
metadata.PublishDate = date
|
|
} else if date, err := time.Parse("2006-01-02", opf.Metadata.Dates[0]); err == nil {
|
|
metadata.PublishDate = date
|
|
} else {
|
|
// Try alternative date formats
|
|
if date, err := time.Parse("2006", opf.Metadata.Dates[0]); err == nil {
|
|
metadata.PublishDate = date
|
|
}
|
|
}
|
|
}
|
|
// Identifiers (ISBN, ASIN)
|
|
for _, id := range opf.Metadata.Identifiers {
|
|
switch strings.ToUpper(id.Scheme) {
|
|
case "ISBN":
|
|
metadata.ISBN = utils.NormalizeISBNSafe(id.Value)
|
|
case "ASIN":
|
|
metadata.ASIN = id.Value
|
|
case "UUID", "CALIBRE":
|
|
// Store UUID in hash info, not metadata
|
|
// Will be extracted by extractHashInfo()
|
|
}
|
|
}
|
|
// Contributors
|
|
if len(opf.Metadata.Contributors) > 0 {
|
|
metadata.Contributors = utils.NormalizeContributors(opf.Metadata.Contributors)
|
|
}
|
|
// Calibre-specific meta tags (filter by name attribute)
|
|
for _, meta := range opf.Metadata.MetaTags {
|
|
switch meta.Name {
|
|
case "calibre:series":
|
|
metadata.Series = meta.Value
|
|
case "calibre:series_index":
|
|
if index, err := strconv.ParseFloat(meta.Value, 32); err == nil {
|
|
metadata.SeriesNumber = int32(index)
|
|
}
|
|
case "calibre:rating":
|
|
// Not imported (ratings are per-user in Bookhoard)
|
|
case "calibre:title_sort":
|
|
// Not imported (Bookhoard has its own sorting logic)
|
|
case "calibre:timestamp":
|
|
// Could be used for created_at, but skipping for now
|
|
}
|
|
}
|
|
return metadata, nil
|
|
}
|
|
|
|
// extractEPUBCover extracts the cover image from an EPUB file.
|
|
// It looks for:
|
|
// 1. An item with properties="cover-image" in the manifest
|
|
// 2. A meta tag with name="cover" pointing to an image
|
|
// 3. Common cover image paths like cover.jpg, cover.jpeg, cover.png
|
|
// Returns the path to the saved cover image, or empty string if no cover found.
|
|
func (s *MediaScanner) extractEPUBCover(epubPath string) (string, error) {
|
|
// Open the EPUB as a zip file to extract images
|
|
r, err := zip.OpenReader(epubPath)
|
|
if err != nil {
|
|
return "", fmt.Errorf("failed to open EPUB as zip: %v", err)
|
|
}
|
|
defer func() {
|
|
if err := r.Close(); err != nil {
|
|
fmt.Printf("Warning: failed to close EPUB zip reader for %s: %v\n", epubPath, err)
|
|
}
|
|
}()
|
|
|
|
// Try to find cover image from OPF metadata
|
|
coverImageName := ""
|
|
|
|
// Attempt to read the OPF file to find cover reference
|
|
// First, find container.xml to locate the OPF
|
|
var opfPath string
|
|
for _, f := range r.File {
|
|
if f.Name == "META-INF/container.xml" {
|
|
rc, err := f.Open()
|
|
if err != nil {
|
|
continue
|
|
}
|
|
content, readErr := io.ReadAll(rc)
|
|
if closeErr := rc.Close(); closeErr != nil {
|
|
fmt.Printf("Warning: failed to close META-INF/container.xml reader in %s: %v\n", epubPath, closeErr)
|
|
}
|
|
if readErr != nil {
|
|
continue
|
|
}
|
|
// Parse container.xml to find OPF path
|
|
// Simple string search since we just need the path
|
|
opfStart := bytes.Index(content, []byte("<rootfile "))
|
|
if opfStart == -1 {
|
|
continue
|
|
}
|
|
opfStartAttr := bytes.Index(content[opfStart:], []byte("full-path="))
|
|
if opfStartAttr == -1 {
|
|
continue
|
|
}
|
|
opfStartAttr += len("full-path=")
|
|
quote := content[opfStart+opfStartAttr]
|
|
opfStartQuote := opfStart + opfStartAttr + 1
|
|
opfEndQuote := bytes.Index(content[opfStartQuote:], []byte{quote})
|
|
if opfEndQuote == -1 {
|
|
continue
|
|
}
|
|
opfPath = string(content[opfStartQuote : opfStartQuote+opfEndQuote])
|
|
break
|
|
}
|
|
}
|
|
|
|
if opfPath == "" {
|
|
// No OPF found, try common cover image paths
|
|
coverImageName = findCoverImageInZip(r.File)
|
|
} else {
|
|
// Read OPF to find cover reference
|
|
opfContent, err := readFileFromZip(r.File, opfPath)
|
|
if err != nil {
|
|
// Fall back to common paths
|
|
coverImageName = findCoverImageInZip(r.File)
|
|
} else {
|
|
coverImageName = findCoverInOPF(opfContent, r.File, opfPath)
|
|
}
|
|
}
|
|
|
|
if coverImageName == "" {
|
|
return "", nil // No cover found
|
|
}
|
|
|
|
// Extract the cover image
|
|
coverImage, err := extractImageFromZip(r.File, coverImageName, opfPath)
|
|
if err != nil {
|
|
return "", fmt.Errorf("failed to extract cover image: %v", err)
|
|
}
|
|
if len(coverImage) == 0 {
|
|
return "", nil
|
|
}
|
|
|
|
// Save cover to disk (same pattern as comics: {book_path}.cover.jpg)
|
|
coverPath := epubPath + ".cover.jpg"
|
|
if err := os.WriteFile(coverPath, coverImage, 0644); err != nil {
|
|
return "", fmt.Errorf("failed to write cover file: %v", err)
|
|
}
|
|
|
|
return coverPath, nil
|
|
}
|
|
|
|
// findCoverImageInZip searches for common cover image filenames in the zip
|
|
func findCoverImageInZip(files []*zip.File) string {
|
|
coverNames := []string{"cover.jpg", "cover.jpeg", "cover.png", "cover.webp",
|
|
"Cover.jpg", "Cover.jpeg", "Cover.png", "Cover.webp",
|
|
"images/cover.jpg", "Images/cover.jpg", "OEBPS/images/cover.jpg"}
|
|
|
|
for _, name := range coverNames {
|
|
for _, f := range files {
|
|
if strings.EqualFold(f.Name, name) {
|
|
return f.Name
|
|
}
|
|
}
|
|
}
|
|
return ""
|
|
}
|
|
|
|
// findCoverInOPF parses OPF content to find cover image reference
|
|
func findCoverInOPF(opfContent []byte, files []*zip.File, opfDir string) string {
|
|
contentStr := string(opfContent)
|
|
|
|
// Look for item with properties="cover-image"
|
|
coverImageRE := regexp.MustCompile(`<item[^>]*properties="[^"]*cover-image[^"]*"[^>]*id="([^"]+)"`)
|
|
matches := coverImageRE.FindStringSubmatch(contentStr)
|
|
if len(matches) > 1 {
|
|
coverID := matches[1]
|
|
// Find the href for this ID
|
|
hrefRE := regexp.MustCompile(fmt.Sprintf(`<item[^>]*id="%s"[^>]*href="([^"]+)"`, coverID))
|
|
hrefMatches := hrefRE.FindStringSubmatch(contentStr)
|
|
if len(hrefMatches) > 1 {
|
|
return resolveOPFPath(opfDir, hrefMatches[1])
|
|
}
|
|
}
|
|
|
|
// Look for meta name="cover"
|
|
metaCoverRE := regexp.MustCompile(`<meta[^>]*name="cover"[^>]*content="([^"]+)"`)
|
|
metaMatches := metaCoverRE.FindStringSubmatch(contentStr)
|
|
if len(metaMatches) > 1 {
|
|
coverContent := metaMatches[1]
|
|
// Could be "image-id" format
|
|
if coverID, found := strings.CutPrefix(coverContent, "image-"); found {
|
|
hrefRE := regexp.MustCompile(fmt.Sprintf(`<item[^>]*id="%s"[^>]*href="([^"]+)"`, coverID))
|
|
hrefMatches := hrefRE.FindStringSubmatch(contentStr)
|
|
if len(hrefMatches) > 1 {
|
|
return resolveOPFPath(opfDir, hrefMatches[1])
|
|
}
|
|
}
|
|
}
|
|
|
|
// Fall back to searching common paths
|
|
return findCoverImageInZip(files)
|
|
}
|
|
|
|
// resolveOPFPath resolves a relative path against the OPF directory
|
|
func resolveOPFPath(opfDir, href string) string {
|
|
if opfDir == "" {
|
|
return href
|
|
}
|
|
// Handle ../ in href
|
|
if strings.HasPrefix(href, "../") {
|
|
// Simple case: just use the href as-is for now
|
|
return href
|
|
}
|
|
// Join the directory with the href
|
|
return filepath.Join(filepath.Dir(opfDir), href)
|
|
}
|
|
|
|
// readFileFromZip reads a file from the zip by name
|
|
func readFileFromZip(files []*zip.File, name string) ([]byte, error) {
|
|
// Normalize the name for comparison
|
|
name = filepath.ToSlash(name)
|
|
for _, f := range files {
|
|
fName := filepath.ToSlash(f.Name)
|
|
if fName == name {
|
|
rc, err := f.Open()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer func() {
|
|
if err := rc.Close(); err != nil {
|
|
fmt.Printf("Warning: failed to close zip file reader for %s: %v\n", name, err)
|
|
}
|
|
}()
|
|
return io.ReadAll(rc)
|
|
}
|
|
}
|
|
return nil, fmt.Errorf("file not found: %s", name)
|
|
}
|
|
|
|
// extractImageFromZip extracts an image file and returns its contents
|
|
func extractImageFromZip(files []*zip.File, imagePath, opfDir string) ([]byte, error) {
|
|
// Try direct match first
|
|
for _, f := range files {
|
|
if strings.EqualFold(f.Name, imagePath) {
|
|
rc, err := f.Open()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer func() {
|
|
if err := rc.Close(); err != nil {
|
|
fmt.Printf("Warning: failed to close zip image reader for %s: %v\n", imagePath, err)
|
|
}
|
|
}()
|
|
return io.ReadAll(rc)
|
|
}
|
|
}
|
|
|
|
// Try resolved path
|
|
resolvedPath := resolveOPFPath(opfDir, imagePath)
|
|
for _, f := range files {
|
|
if strings.EqualFold(f.Name, resolvedPath) {
|
|
rc, err := f.Open()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer func() {
|
|
if err := rc.Close(); err != nil {
|
|
fmt.Printf("Warning: failed to close zip image reader for resolved path %s: %v\n", resolvedPath, err)
|
|
}
|
|
}()
|
|
return io.ReadAll(rc)
|
|
}
|
|
}
|
|
|
|
return nil, fmt.Errorf("image not found: %s", imagePath)
|
|
}
|
|
|
|
// findSidecarCover looks for cover images in the same directory as the media file.
|
|
// It checks for common cover filename patterns in priority order:
|
|
// 1. cover.jpg, cover.jpeg, cover.png, cover.webp
|
|
// 2. folder.jpg, folder.jpeg, folder.png, folder.webp
|
|
// 3. {basename}.jpg, {basename}.jpeg, etc. (same name as media file)
|
|
// 4. .folder.jpg (hidden file)
|
|
// Returns the full path to the cover file, or empty string if not found.
|
|
func findSidecarCover(mediaPath string) string {
|
|
dir := filepath.Dir(mediaPath)
|
|
baseName := strings.TrimSuffix(filepath.Base(mediaPath), filepath.Ext(mediaPath))
|
|
|
|
// Priority order for cover filenames
|
|
coverPatterns := []string{
|
|
"cover.jpg",
|
|
"cover.jpeg",
|
|
"cover.png",
|
|
"cover.webp",
|
|
"folder.jpg",
|
|
"folder.jpeg",
|
|
"folder.png",
|
|
"folder.webp",
|
|
".folder.jpg",
|
|
".folder.jpeg",
|
|
".folder.png",
|
|
}
|
|
|
|
// First, check exact match cover/folder names
|
|
for _, coverName := range coverPatterns {
|
|
coverPath := filepath.Join(dir, coverName)
|
|
if _, err := os.Stat(coverPath); err == nil {
|
|
return coverPath
|
|
}
|
|
}
|
|
|
|
// Second, check for {basename}.{ext} pattern
|
|
extensions := []string{".jpg", ".jpeg", ".png", ".webp"}
|
|
for _, ext := range extensions {
|
|
coverPath := filepath.Join(dir, baseName+ext)
|
|
if _, err := os.Stat(coverPath); err == nil {
|
|
return coverPath
|
|
}
|
|
// Also check uppercase extension
|
|
coverPathUpper := filepath.Join(dir, baseName+strings.ToUpper(ext))
|
|
if _, err := os.Stat(coverPathUpper); err == nil {
|
|
return coverPathUpper
|
|
}
|
|
}
|
|
|
|
return ""
|
|
}
|
|
|
|
func (s *MediaScanner) extractPDFMetadata(path string) (*MediaMetadata, error) {
|
|
metadata := &MediaMetadata{}
|
|
|
|
// Open PDF file for reading metadata
|
|
f, err := os.Open(path)
|
|
if err != nil {
|
|
fmt.Printf("Warning: failed to open PDF file %s: %v\n", path, err)
|
|
metadata.Title = strings.TrimSuffix(filepath.Base(path), ".pdf")
|
|
return metadata, nil
|
|
}
|
|
defer func() {
|
|
if err := f.Close(); err != nil {
|
|
fmt.Printf("Warning: failed to close PDF file %s: %v\n", path, err)
|
|
}
|
|
}()
|
|
|
|
// Use pdfcpu API to read PDF metadata
|
|
// Configuration: nil = default (lenient mode)
|
|
pdfInfo, err := pdfcpuapi.PDFInfo(f, filepath.Base(path), nil, false, nil)
|
|
if err != nil {
|
|
fmt.Printf("Warning: failed to read PDF info from %s: %v\n", path, err)
|
|
// Fall back to filename as title
|
|
metadata.Title = strings.TrimSuffix(filepath.Base(path), ".pdf")
|
|
return metadata, nil
|
|
}
|
|
|
|
// Extract title
|
|
if pdfInfo.Title != "" {
|
|
metadata.Title = pdfInfo.Title
|
|
} else {
|
|
metadata.Title = strings.TrimSuffix(filepath.Base(path), ".pdf")
|
|
}
|
|
|
|
// Extract author
|
|
if pdfInfo.Author != "" {
|
|
metadata.Author = pdfInfo.Author
|
|
}
|
|
|
|
// Extract subject (use as description)
|
|
if pdfInfo.Subject != "" {
|
|
metadata.Description = pdfInfo.Subject
|
|
}
|
|
|
|
// Extract creator (use as author fallback)
|
|
if pdfInfo.Creator != "" && metadata.Author == "" {
|
|
metadata.Author = pdfInfo.Creator
|
|
}
|
|
|
|
// Extract producer (use as publisher)
|
|
if pdfInfo.Producer != "" {
|
|
metadata.Publisher = pdfInfo.Producer
|
|
}
|
|
|
|
metadata.PageCount = int32(pdfInfo.PageCount)
|
|
|
|
// Try to extract cover image
|
|
coverPath, err := s.extractPDFCover(path)
|
|
if err != nil {
|
|
fmt.Printf("Warning: failed to extract PDF cover from %s: %v\n", path, err)
|
|
} else if coverPath != "" {
|
|
metadata.CoverPath = s.getRelativePath(coverPath)
|
|
}
|
|
|
|
// If no embedded cover, try sidecar
|
|
if metadata.CoverPath == "" {
|
|
sidecarCover := findSidecarCover(path)
|
|
if sidecarCover != "" {
|
|
metadata.CoverPath = s.getRelativePath(sidecarCover)
|
|
}
|
|
}
|
|
|
|
return metadata, nil
|
|
}
|
|
|
|
// extractPDFCover extracts a cover image from a PDF file.
|
|
// It uses pdfcpu API to extract images from the first page.
|
|
// Returns the path to the saved cover, or empty string if no cover found.
|
|
func (s *MediaScanner) extractPDFCover(pdfPath string) (string, error) {
|
|
// Create a temporary directory for extracted images
|
|
tmpDir, err := os.MkdirTemp("", "pdf-cover-")
|
|
if err != nil {
|
|
return "", fmt.Errorf("failed to create temp dir: %v", err)
|
|
}
|
|
defer func() {
|
|
if err := os.RemoveAll(tmpDir); err != nil {
|
|
fmt.Printf("Warning: failed to remove temp directory %s: %v\n", tmpDir, err)
|
|
}
|
|
}()
|
|
|
|
// Use pdfcpu API to extract images from first page
|
|
// ExtractImagesFile(inFile, outDir string, selectedPages []string, conf *model.Configuration) error
|
|
err = pdfcpuapi.ExtractImagesFile(pdfPath, tmpDir, []string{"1"}, nil)
|
|
if err != nil {
|
|
// No images found or extraction failed - this is OK, just return empty
|
|
return "", nil
|
|
}
|
|
|
|
// Check for extracted images in the temp directory
|
|
entries, err := os.ReadDir(tmpDir)
|
|
if err != nil || len(entries) == 0 {
|
|
return "", nil
|
|
}
|
|
|
|
// Find the largest image (likely the cover)
|
|
var largestImage string
|
|
var largestSize int64
|
|
|
|
for _, entry := range entries {
|
|
if entry.IsDir() {
|
|
continue
|
|
}
|
|
info, err := entry.Info()
|
|
if err != nil {
|
|
continue
|
|
}
|
|
// Skip very small files (likely thumbnails or icons)
|
|
if info.Size() < 1000 {
|
|
continue
|
|
}
|
|
if info.Size() > largestSize {
|
|
largestImage = filepath.Join(tmpDir, entry.Name())
|
|
largestSize = info.Size()
|
|
}
|
|
}
|
|
|
|
if largestImage == "" {
|
|
return "", nil
|
|
}
|
|
|
|
// Read the image
|
|
imageData, err := os.ReadFile(largestImage)
|
|
if err != nil || len(imageData) == 0 {
|
|
return "", nil
|
|
}
|
|
|
|
// Save cover to disk (same pattern as comics: {pdf_path}.cover.jpg)
|
|
coverPath := pdfPath + ".cover.jpg"
|
|
if err := os.WriteFile(coverPath, imageData, 0644); err != nil {
|
|
return "", fmt.Errorf("failed to write cover file: %v", err)
|
|
}
|
|
|
|
return coverPath, nil
|
|
}
|
|
|
|
// ComicInfo represents metadata from ComicInfo.xml
|
|
type ComicInfo struct {
|
|
XMLName xml.Name `xml:"ComicInfo"`
|
|
Title string `xml:"Title"`
|
|
Series string `xml:"Series"`
|
|
Number int `xml:"Number"`
|
|
Volume int `xml:"Volume"`
|
|
Publisher string `xml:"Publisher"`
|
|
Year int `xml:"Year"`
|
|
Month int `xml:"Month"`
|
|
Day int `xml:"Day"`
|
|
Writer string `xml:"Writer"`
|
|
Penciller string `xml:"Penciller"`
|
|
Inker string `xml:"Inker"`
|
|
Colorist string `xml:"Colorist"`
|
|
Letterer string `xml:"Letterer"`
|
|
CoverArtist string `xml:"CoverArtist"`
|
|
Genre string `xml:"Genre"`
|
|
Tags string `xml:"Tags"`
|
|
Web string `xml:"Web"`
|
|
Notes string `xml:"Notes"`
|
|
|
|
// NEW: Reading direction fields from ComicInfo.xml v2.0
|
|
Manga string `xml:"Manga"` // Unknown, No, Yes, YesAndRightToLeft
|
|
LanguageISO string `xml:"LanguageISO"` // ISO 639-1 language code for heuristics
|
|
|
|
// NEW: Additional comic-specific fields (19 total fields from ComicInfo.xml)
|
|
Count int `xml:"Count"` // Total issues in series
|
|
AlternateSeries string `xml:"AlternateSeries"`
|
|
AlternateNumber int `xml:"AlternateNumber"`
|
|
AlternateCount int `xml:"AlternateCount"`
|
|
Summary string `xml:"Summary"`
|
|
Imprint string `xml:"Imprint"`
|
|
StoryArc string `xml:"StoryArc"`
|
|
SeriesGroup string `xml:"SeriesGroup"`
|
|
AgeRating string `xml:"AgeRating"`
|
|
CommunityRating float64 `xml:"CommunityRating"`
|
|
MainCharacterOrTeam string `xml:"MainCharacterOrTeam"`
|
|
Review string `xml:"Review"`
|
|
BlackAndWhite string `xml:"BlackAndWhite"` // "Yes" or "No"
|
|
ScanInformation string `xml:"ScanInformation"`
|
|
Characters string `xml:"Characters"`
|
|
Teams string `xml:"Teams"`
|
|
Locations string `xml:"Locations"`
|
|
}
|
|
|
|
// extractComicMetadata extracts metadata from comic archive (supports .cbz, .cbr, .cb7, .cbt)
|
|
func extractComicMetadata(filePath string) (*ComicInfo, []byte, error) {
|
|
ext := strings.ToLower(filepath.Ext(filePath))
|
|
|
|
switch ext {
|
|
case ".cbz":
|
|
return extractZipMetadata(filePath)
|
|
case ".cbr":
|
|
return extractRarMetadata(filePath)
|
|
case ".cb7":
|
|
return extract7ZipMetadata(filePath)
|
|
case ".cbt":
|
|
return extractTarMetadata(filePath)
|
|
default:
|
|
return nil, nil, fmt.Errorf("unsupported comic format: %s", ext)
|
|
}
|
|
}
|
|
|
|
// archiveFile represents a file in an archive for uniform handling
|
|
type archiveFile interface {
|
|
Name() string
|
|
Open() (io.ReadCloser, error)
|
|
}
|
|
|
|
// extractMetadataFromArchive extracts ComicInfo.xml and cover image from any archive format
|
|
func extractMetadataFromArchive(files []archiveFile) (*ComicInfo, []byte, error) {
|
|
var comicInfo *ComicInfo
|
|
var coverImage []byte
|
|
|
|
for _, f := range files {
|
|
if f.Name() == "ComicInfo.xml" {
|
|
rc, err := f.Open()
|
|
if err != nil {
|
|
return nil, nil, fmt.Errorf("failed to open ComicInfo.xml: %w", err)
|
|
}
|
|
|
|
data, err := io.ReadAll(rc)
|
|
if closeErr := rc.Close(); closeErr != nil {
|
|
fmt.Printf("Warning: failed to close ComicInfo.xml reader: %v\n", closeErr)
|
|
}
|
|
if err != nil {
|
|
return nil, nil, fmt.Errorf("failed to read ComicInfo.xml: %w", err)
|
|
}
|
|
|
|
comicInfo = &ComicInfo{}
|
|
if err := xml.Unmarshal(data, comicInfo); err != nil {
|
|
return nil, nil, fmt.Errorf("failed to parse ComicInfo.xml: %w", err)
|
|
}
|
|
}
|
|
|
|
if coverImage == nil && isImageFile(f.Name()) {
|
|
if !strings.Contains(filepath.Dir(f.Name()), string(filepath.Separator)) ||
|
|
filepath.Dir(f.Name()) == "." {
|
|
rc, err := f.Open()
|
|
if err != nil {
|
|
continue
|
|
}
|
|
|
|
coverImage, err = io.ReadAll(rc)
|
|
if closeErr := rc.Close(); closeErr != nil {
|
|
fmt.Printf("Warning: failed to close cover image reader for %s: %v\n", f.Name(), closeErr)
|
|
}
|
|
if err == nil {
|
|
_, _, err = image.Decode(bytes.NewReader(coverImage))
|
|
if err != nil {
|
|
coverImage = nil
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
if comicInfo == nil {
|
|
comicInfo = &ComicInfo{}
|
|
}
|
|
|
|
return comicInfo, coverImage, nil
|
|
}
|
|
|
|
// extractZipMetadata extracts metadata from ZIP archives (.cbz)
|
|
func extractZipMetadata(filePath string) (*ComicInfo, []byte, error) {
|
|
r, err := zip.OpenReader(filePath)
|
|
if err != nil {
|
|
return nil, nil, fmt.Errorf("failed to open ZIP archive: %w", err)
|
|
}
|
|
defer func() {
|
|
if err := r.Close(); err != nil {
|
|
fmt.Printf("Warning: failed to close ZIP archive reader for %s: %v\n", filePath, err)
|
|
}
|
|
}()
|
|
|
|
files := make([]archiveFile, 0, len(r.File))
|
|
for _, f := range r.File {
|
|
files = append(files, &zipFileAdapter{f})
|
|
}
|
|
|
|
comicInfo, coverImage, err := extractMetadataFromArchive(files)
|
|
if err != nil {
|
|
return nil, nil, err
|
|
}
|
|
|
|
if comicInfo.Title == "" {
|
|
basename := filepath.Base(filePath)
|
|
comicInfo.Title = strings.TrimSuffix(basename, filepath.Ext(basename))
|
|
}
|
|
|
|
return comicInfo, coverImage, nil
|
|
}
|
|
|
|
// zipFileAdapter adapts zip.File to archiveFile interface
|
|
type zipFileAdapter struct {
|
|
*zip.File
|
|
}
|
|
|
|
func (z *zipFileAdapter) Name() string {
|
|
return z.File.Name
|
|
}
|
|
|
|
func (z *zipFileAdapter) Open() (io.ReadCloser, error) {
|
|
return z.File.Open()
|
|
}
|
|
|
|
// extractRarMetadata extracts metadata from RAR archives (.cbr)
|
|
func extractRarMetadata(filePath string) (*ComicInfo, []byte, error) {
|
|
r, err := rardecode.OpenReader(filePath, "")
|
|
if err != nil {
|
|
return nil, nil, fmt.Errorf("failed to open RAR archive: %w", err)
|
|
}
|
|
defer func() {
|
|
if err := r.Close(); err != nil {
|
|
fmt.Printf("Warning: failed to close RAR archive reader for %s: %v\n", filePath, err)
|
|
}
|
|
}()
|
|
|
|
files := make([]archiveFile, 0, 100)
|
|
for {
|
|
header, err := r.Next()
|
|
if err == io.EOF {
|
|
break
|
|
}
|
|
if err != nil {
|
|
return nil, nil, fmt.Errorf("failed to read RAR entry: %w", err)
|
|
}
|
|
|
|
data, err := io.ReadAll(r)
|
|
if err != nil {
|
|
continue
|
|
}
|
|
files = append(files, &rarFileAdapter{
|
|
name: header.Name,
|
|
data: data,
|
|
})
|
|
}
|
|
|
|
comicInfo, coverImage, err := extractMetadataFromArchive(files)
|
|
if err != nil {
|
|
return nil, nil, err
|
|
}
|
|
|
|
if comicInfo.Title == "" {
|
|
basename := filepath.Base(filePath)
|
|
comicInfo.Title = strings.TrimSuffix(basename, filepath.Ext(basename))
|
|
}
|
|
|
|
return comicInfo, coverImage, nil
|
|
}
|
|
|
|
// rarFileAdapter stores RAR file data in memory
|
|
type rarFileAdapter struct {
|
|
name string
|
|
data []byte
|
|
}
|
|
|
|
func (r *rarFileAdapter) Name() string {
|
|
return r.name
|
|
}
|
|
|
|
func (r *rarFileAdapter) Open() (io.ReadCloser, error) {
|
|
return io.NopCloser(bytes.NewReader(r.data)), nil
|
|
}
|
|
|
|
// extract7ZipMetadata extracts metadata from 7-Zip archives (.cb7)
|
|
func extract7ZipMetadata(filePath string) (*ComicInfo, []byte, error) {
|
|
r, err := sevenzip.OpenReader(filePath)
|
|
if err != nil {
|
|
return nil, nil, fmt.Errorf("failed to open 7-Zip archive: %w", err)
|
|
}
|
|
defer func() {
|
|
if err := r.Close(); err != nil {
|
|
fmt.Printf("Warning: failed to close 7-Zip archive reader for %s: %v\n", filePath, err)
|
|
}
|
|
}()
|
|
|
|
files := make([]archiveFile, 0, len(r.File))
|
|
for _, f := range r.File {
|
|
files = append(files, &sevenZipFileAdapter{file: f})
|
|
}
|
|
|
|
comicInfo, coverImage, err := extractMetadataFromArchive(files)
|
|
if err != nil {
|
|
return nil, nil, err
|
|
}
|
|
|
|
if comicInfo.Title == "" {
|
|
basename := filepath.Base(filePath)
|
|
comicInfo.Title = strings.TrimSuffix(basename, filepath.Ext(basename))
|
|
}
|
|
|
|
return comicInfo, coverImage, nil
|
|
}
|
|
|
|
// sevenZipFileAdapter adapts sevenzip.File to archiveFile interface
|
|
type sevenZipFileAdapter struct {
|
|
file *sevenzip.File
|
|
}
|
|
|
|
func (s *sevenZipFileAdapter) Name() string {
|
|
return s.file.Name
|
|
}
|
|
|
|
func (s *sevenZipFileAdapter) Open() (io.ReadCloser, error) {
|
|
return s.file.Open()
|
|
}
|
|
|
|
// extractTarMetadata extracts metadata from TAR archives (.cbt)
|
|
func extractTarMetadata(filePath string) (*ComicInfo, []byte, error) {
|
|
f, err := os.Open(filePath)
|
|
if err != nil {
|
|
return nil, nil, fmt.Errorf("failed to open TAR archive: %w", err)
|
|
}
|
|
defer func() {
|
|
if err := f.Close(); err != nil {
|
|
fmt.Printf("Warning: failed to close TAR archive file %s: %v\n", filePath, err)
|
|
}
|
|
}()
|
|
|
|
var tarReader *tar.Reader
|
|
if strings.HasSuffix(strings.ToLower(filePath), ".tar.gz") ||
|
|
strings.HasSuffix(strings.ToLower(filePath), ".tgz") {
|
|
gzReader, err := gzip.NewReader(f)
|
|
if err != nil {
|
|
return nil, nil, fmt.Errorf("failed to create gzip reader: %w", err)
|
|
}
|
|
defer func() {
|
|
if err := gzReader.Close(); err != nil {
|
|
fmt.Printf("Warning: failed to close gzip reader for %s: %v\n", filePath, err)
|
|
}
|
|
}()
|
|
tarReader = tar.NewReader(gzReader)
|
|
} else if strings.HasSuffix(strings.ToLower(filePath), ".tar.bz2") ||
|
|
strings.HasSuffix(strings.ToLower(filePath), ".tbz2") {
|
|
bz2Reader := bzip2.NewReader(f)
|
|
tarReader = tar.NewReader(bz2Reader)
|
|
} else {
|
|
tarReader = tar.NewReader(f)
|
|
}
|
|
|
|
files := make([]archiveFile, 0, 100)
|
|
for {
|
|
header, err := tarReader.Next()
|
|
if err == io.EOF {
|
|
break
|
|
}
|
|
if err != nil {
|
|
return nil, nil, fmt.Errorf("failed to read TAR entry: %w", err)
|
|
}
|
|
|
|
if header.Typeflag == tar.TypeReg {
|
|
data, err := io.ReadAll(tarReader)
|
|
if err != nil {
|
|
continue
|
|
}
|
|
files = append(files, &tarFileAdapter{
|
|
name: header.Name,
|
|
data: data,
|
|
})
|
|
}
|
|
}
|
|
|
|
comicInfo, coverImage, err := extractMetadataFromArchive(files)
|
|
if err != nil {
|
|
return nil, nil, err
|
|
}
|
|
|
|
if comicInfo.Title == "" {
|
|
basename := filepath.Base(filePath)
|
|
comicInfo.Title = strings.TrimSuffix(basename, filepath.Ext(basename))
|
|
}
|
|
|
|
return comicInfo, coverImage, nil
|
|
}
|
|
|
|
// tarFileAdapter stores TAR file data in memory
|
|
type tarFileAdapter struct {
|
|
name string
|
|
data []byte
|
|
}
|
|
|
|
func (t *tarFileAdapter) Name() string {
|
|
return t.name
|
|
}
|
|
|
|
func (t *tarFileAdapter) Open() (io.ReadCloser, error) {
|
|
return io.NopCloser(bytes.NewReader(t.data)), nil
|
|
}
|
|
|
|
// isImageFile checks if a file is an image based on extension
|
|
func isImageFile(filename string) bool {
|
|
ext := strings.ToLower(filepath.Ext(filename))
|
|
switch ext {
|
|
case ".jpg", ".jpeg", ".png", ".gif", ".webp", ".bmp", ".avif", ".tiff", ".tif":
|
|
return true
|
|
}
|
|
return false
|
|
}
|
|
|
|
// countArchiveImages counts image files in a comic archive
|
|
func countArchiveImages(filePath string) (int, error) {
|
|
ext := strings.ToLower(filepath.Ext(filePath))
|
|
count := 0
|
|
|
|
switch ext {
|
|
case ".cbz", ".epub":
|
|
r, err := zip.OpenReader(filePath)
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
defer r.Close()
|
|
for _, f := range r.File {
|
|
if !f.FileInfo().IsDir() && isImageFile(f.Name) {
|
|
count++
|
|
}
|
|
}
|
|
case ".cbr":
|
|
r, err := rardecode.OpenReader(filePath, "")
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
defer r.Close()
|
|
for {
|
|
header, err := r.Next()
|
|
if err == io.EOF {
|
|
break
|
|
}
|
|
if err != nil {
|
|
break
|
|
}
|
|
if !header.IsDir && isImageFile(header.Name) {
|
|
count++
|
|
}
|
|
}
|
|
case ".cb7":
|
|
sz, err := sevenzip.OpenReader(filePath)
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
defer sz.Close()
|
|
for _, f := range sz.File {
|
|
if !f.FileInfo().IsDir() && isImageFile(f.Name) {
|
|
count++
|
|
}
|
|
}
|
|
case ".cbt":
|
|
f, err := os.Open(filePath)
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
defer f.Close()
|
|
tr := tar.NewReader(f)
|
|
for {
|
|
header, err := tr.Next()
|
|
if err == io.EOF {
|
|
break
|
|
}
|
|
if err != nil {
|
|
break
|
|
}
|
|
if !header.FileInfo().IsDir() && isImageFile(header.Name) {
|
|
count++
|
|
}
|
|
}
|
|
}
|
|
|
|
return count, nil
|
|
}
|
|
|
|
func (s *MediaScanner) updateMediaItem(ctx context.Context, mediaItemID pgtype.UUID, path string, _ os.FileInfo) error {
|
|
// Re-extract metadata for the update
|
|
metadata, err := s.extractMetadata(path)
|
|
if err != nil {
|
|
fmt.Printf("Warning: failed to extract metadata for force rescan %s: %v\n", path, err)
|
|
metadata = &MediaMetadata{}
|
|
}
|
|
|
|
// Normalize metadata fields
|
|
metadata.Contributors = utils.NormalizeContributors(metadata.Contributors)
|
|
metadata.Tags = utils.NormalizeTags(metadata.Tags)
|
|
contributorsSearch := utils.NormalizeContributorsSearch(metadata.Contributors)
|
|
tagsSearch := utils.NormalizeTagsSearch(metadata.Tags)
|
|
|
|
// Call the database update - only update fields available in MediaMetadata
|
|
var alternateInfoBytes []byte
|
|
if metadata.AlternateInfo != "" {
|
|
alternateInfoBytes = []byte(metadata.AlternateInfo)
|
|
}
|
|
_, err = s.db.UpdateMediaItem(ctx, database.UpdateMediaItemParams{
|
|
ID: mediaItemID,
|
|
Title: metadata.Title,
|
|
Author: pgtype.Text{String: metadata.Author, Valid: metadata.Author != ""},
|
|
Isbn: pgtype.Text{String: utils.NormalizeISBNSafe(metadata.ISBN), Valid: metadata.ISBN != ""},
|
|
Description: pgtype.Text{String: metadata.Description, Valid: metadata.Description != ""},
|
|
CoverImagePath: pgtype.Text{String: s.getRelativePath(metadata.CoverPath), Valid: metadata.CoverPath != ""},
|
|
Series: pgtype.Text{String: metadata.Series, Valid: metadata.Series != ""},
|
|
SeriesNumber: pgtype.Int4{Int32: metadata.SeriesNumber, Valid: metadata.SeriesNumber > 0},
|
|
Tags: metadata.Tags,
|
|
TagsSearch: tagsSearch,
|
|
Asin: pgtype.Text{String: metadata.ASIN, Valid: metadata.ASIN != ""},
|
|
DatePublished: pgtype.Date{Time: metadata.PublishDate, Valid: !metadata.PublishDate.IsZero()},
|
|
Publisher: pgtype.Text{String: metadata.Publisher, Valid: metadata.Publisher != ""},
|
|
Contributors: metadata.Contributors,
|
|
ContributorsSearch: contributorsSearch,
|
|
Language: pgtype.Text{String: metadata.Language, Valid: metadata.Language != ""},
|
|
Genre: pgtype.Text{String: metadata.Genre, Valid: metadata.Genre != ""},
|
|
PageCount: pgtype.Int4{Int32: metadata.PageCount, Valid: metadata.PageCount > 0},
|
|
MangaType: pgtype.Text{String: metadata.MangaType, Valid: metadata.MangaType != ""},
|
|
ReadingDirection: pgtype.Text{String: metadata.ReadingDirection, Valid: metadata.ReadingDirection != ""},
|
|
SeriesCount: pgtype.Int4{Int32: metadata.SeriesCount, Valid: metadata.SeriesCount > 0},
|
|
Volume: pgtype.Int4{Int32: metadata.Volume, Valid: metadata.Volume > 0},
|
|
Imprint: pgtype.Text{String: metadata.Imprint, Valid: metadata.Imprint != ""},
|
|
AgeRating: pgtype.Text{String: metadata.AgeRating, Valid: metadata.AgeRating != ""},
|
|
WebUrl: pgtype.Text{String: metadata.WebURL, Valid: metadata.WebURL != ""},
|
|
MetadataNotes: pgtype.Text{String: metadata.MetadataNotes, Valid: metadata.MetadataNotes != ""},
|
|
CommunityRating: pgtype.Float8{Float64: metadata.CommunityRating, Valid: metadata.CommunityRating > 0},
|
|
StoryArc: pgtype.Text{String: metadata.StoryArc, Valid: metadata.StoryArc != ""},
|
|
IsBlackAndWhite: pgtype.Bool{Bool: metadata.IsBlackAndWhite, Valid: metadata.IsBlackAndWhite},
|
|
AlternateInfo: alternateInfoBytes,
|
|
ScanInformation: pgtype.Text{String: metadata.ScanInformation, Valid: metadata.ScanInformation != ""},
|
|
Summary: pgtype.Text{String: metadata.Summary, Valid: metadata.Summary != ""},
|
|
})
|
|
return err
|
|
}
|
|
|
|
func (s *MediaScanner) getMediaItemByFilePath(ctx context.Context, filePath string, libraryID pgtype.UUID) (database.MediaItems, error) {
|
|
return s.db.GetMediaItemByFilePath(ctx, database.GetMediaItemByFilePathParams{
|
|
FilePath: s.getRelativePath(filePath),
|
|
LibraryID: libraryID,
|
|
})
|
|
}
|
|
|
|
// recomputeHashInfo recomputes the file's hash identifiers and stores them on
|
|
// the media item (plus its per-format row). Called on force rescan, on file
|
|
// size change, and when an unchanged item is found with no stored hash, so
|
|
// items imported before hashing existed are backfilled by ordinary scans.
|
|
// After storing, it records a hash conflict if the same content now exists at
|
|
// more than one path in the library.
|
|
func (s *MediaScanner) recomputeHashInfo(ctx context.Context, mediaItemID pgtype.UUID, libraryID pgtype.UUID, path string) {
|
|
hashInfo, formatInfo, err := s.extractHashInfo(path)
|
|
if err != nil {
|
|
fmt.Printf("Warning: failed to extract hash info from %s: %v\n", path, err)
|
|
return
|
|
}
|
|
if hashInfo == nil || hashInfo.FileSHA256 == "" {
|
|
return
|
|
}
|
|
|
|
_, err = s.db.UpdateMediaItemIdentifiers(ctx, database.UpdateMediaItemIdentifiersParams{
|
|
ID: mediaItemID,
|
|
FileSha256: pgtype.Text{String: hashInfo.FileSHA256, Valid: true},
|
|
OpfIdentifier: pgtype.Text{String: hashInfo.OPFIdentifier, Valid: hashInfo.OPFIdentifier != ""},
|
|
OpfUuid: pgtype.Text{String: hashInfo.OPFUUID, Valid: hashInfo.OPFUUID != ""},
|
|
HashConfidence: pgtype.Text{String: hashInfo.HashConfidence, Valid: hashInfo.HashConfidence != ""},
|
|
})
|
|
if err != nil {
|
|
fmt.Printf("Warning: failed to update hash identifiers for %s: %v\n", path, err)
|
|
return
|
|
}
|
|
|
|
if formatInfo != nil {
|
|
_, _ = s.db.CreateMediaItemFormat(ctx, database.CreateMediaItemFormatParams{
|
|
MediaItemID: mediaItemID,
|
|
FormatType: formatInfo.FormatType,
|
|
FilePath: pgtype.Text{String: s.getRelativePath(formatInfo.FilePath), Valid: true},
|
|
FileSha256: pgtype.Text{String: formatInfo.FileSHA256, Valid: true},
|
|
FileSizeBytes: pgtype.Int8{Int64: formatInfo.FileSizeBytes, Valid: true},
|
|
MimeType: pgtype.Text{String: formatInfo.MimeType, Valid: true},
|
|
})
|
|
}
|
|
|
|
s.recordHashConflictIfAny(ctx, libraryID, hashInfo.FileSHA256)
|
|
}
|
|
|
|
// recordHashConflictIfAny flags a pending hash conflict when the given content
|
|
// hash is now shared by more than one media item in the same library. The
|
|
// upsert is a no-op for already-tracked (including resolved) groups.
|
|
func (s *MediaScanner) recordHashConflictIfAny(ctx context.Context, libraryID pgtype.UUID, fileSHA256 string) {
|
|
items, err := s.db.ListMediaItemsBySHA256AndLibrary(ctx, database.ListMediaItemsBySHA256AndLibraryParams{
|
|
FileSha256: pgtype.Text{String: fileSHA256, Valid: true},
|
|
LibraryID: libraryID,
|
|
})
|
|
if err != nil {
|
|
return
|
|
}
|
|
if len(items) > 1 {
|
|
fmt.Printf("Hash conflict: %d media items share SHA-256 %s in one library\n", len(items), fileSHA256)
|
|
_ = s.db.CreateHashConflict(ctx, database.CreateHashConflictParams{
|
|
LibraryID: libraryID,
|
|
FileSha256: fileSHA256,
|
|
})
|
|
}
|
|
}
|
|
|
|
func (s *MediaScanner) getMimeType(path string) string {
|
|
ext := strings.ToLower(filepath.Ext(path))
|
|
if mime, ok := MimeTypes[ext]; ok {
|
|
return mime
|
|
}
|
|
return "application/octet-stream"
|
|
}
|
|
|
|
func (s *MediaScanner) WatchChanges(ctx context.Context) error {
|
|
if !s.watching.CompareAndSwap(false, true) {
|
|
return fmt.Errorf("already watching")
|
|
}
|
|
|
|
go func() {
|
|
<-ctx.Done()
|
|
s.watching.Store(false)
|
|
}()
|
|
|
|
go s.performInitialScan(ctx)
|
|
|
|
go s.processDirtyDirectories(ctx)
|
|
|
|
go s.startBackupScan(ctx)
|
|
|
|
go func() {
|
|
// The event loop only runs if a real watcher was set up (watch=true).
|
|
// If watching with no watcher (e.g. inotify unavailable through a Docker
|
|
// bind mount), polling via startBackupScan above still handles detection.
|
|
if s.watcher == nil {
|
|
fmt.Printf("[WATCHER] No inotify watcher configured; relying on periodic polling for change detection\n")
|
|
return
|
|
}
|
|
fmt.Printf("[WATCHER] Event loop started for %d folders\n", len(s.folders))
|
|
for {
|
|
select {
|
|
case event, ok := <-s.watcher.Events:
|
|
if !ok {
|
|
fmt.Printf("[WATCHER] Event channel closed\n")
|
|
return
|
|
}
|
|
|
|
if event.Has(fsnotify.Create) {
|
|
if info, err := os.Stat(event.Name); err == nil && info.IsDir() {
|
|
if err := s.watcher.Add(event.Name); err != nil {
|
|
fmt.Printf("[WATCHER] Warning: failed to watch new directory %s: %v\n", event.Name, err)
|
|
} else {
|
|
fmt.Printf("[WATCHER] Now watching new directory: %s\n", event.Name)
|
|
}
|
|
}
|
|
}
|
|
|
|
if event.Has(fsnotify.Create | fsnotify.Write | fsnotify.Remove | fsnotify.Rename) {
|
|
fmt.Printf("[WATCHER] Event: %s on %s\n", event.Op, event.Name)
|
|
s.markDirectoryDirty(filepath.Dir(event.Name))
|
|
}
|
|
|
|
case err, ok := <-s.watcher.Errors:
|
|
if !ok {
|
|
fmt.Printf("[WATCHER] Error channel closed\n")
|
|
return
|
|
}
|
|
fmt.Printf("[WATCHER] Error: %v\n", err)
|
|
|
|
case <-ctx.Done():
|
|
fmt.Printf("[WATCHER] Event loop stopped\n")
|
|
return
|
|
}
|
|
}
|
|
}()
|
|
return nil
|
|
}
|
|
|
|
func (s *MediaScanner) markDirectoryDirty(dirPath string) {
|
|
s.dirtyDirsMu.Lock()
|
|
defer s.dirtyDirsMu.Unlock()
|
|
|
|
// Only mark if within watched folders
|
|
var isWatched bool
|
|
for _, folder := range s.folders {
|
|
if strings.HasPrefix(dirPath, folder) {
|
|
isWatched = true
|
|
break
|
|
}
|
|
}
|
|
if !isWatched {
|
|
return
|
|
}
|
|
|
|
// Smart event merging (Jellyfin approach):
|
|
// 1. If parent dir exists, replace with parent (consolidate)
|
|
// 2. If sibling dirs exist, replace with common parent
|
|
// 3. Otherwise, add this dir
|
|
|
|
// Check if parent directory is already dirty
|
|
parentDir := filepath.Dir(dirPath)
|
|
if parentDir != dirPath { // Not at root
|
|
if _, parentExists := s.dirtyDirs[parentDir]; parentExists {
|
|
// Parent already being watched, reset its timestamp
|
|
s.dirtyDirs[parentDir] = time.Now()
|
|
return
|
|
}
|
|
}
|
|
|
|
// Check if any subdirectories are dirty, replace with parent
|
|
for existingDir := range s.dirtyDirs {
|
|
if strings.HasPrefix(existingDir, dirPath+"/") {
|
|
// This is a subdirectory, replace it with parent
|
|
delete(s.dirtyDirs, existingDir)
|
|
}
|
|
}
|
|
|
|
// NEW: Check for sibling directories and consolidate to parent
|
|
parentDir = filepath.Dir(dirPath)
|
|
for existingDir := range s.dirtyDirs {
|
|
existingParent := filepath.Dir(existingDir)
|
|
if existingParent == parentDir && existingParent != dirPath && existingParent != "." {
|
|
// Found a sibling! Both should be replaced with parent
|
|
delete(s.dirtyDirs, existingDir)
|
|
s.dirtyDirs[parentDir] = time.Now()
|
|
return
|
|
}
|
|
}
|
|
|
|
// Add/update this directory
|
|
s.dirtyDirs[dirPath] = time.Now()
|
|
}
|
|
|
|
func (s *MediaScanner) processDirtyDirectories(ctx context.Context) {
|
|
ticker := time.NewTicker(1 * time.Second)
|
|
defer ticker.Stop()
|
|
|
|
for {
|
|
select {
|
|
case <-ctx.Done():
|
|
return
|
|
|
|
case <-ticker.C:
|
|
s.dirtyDirsMu.Lock()
|
|
|
|
now := time.Now()
|
|
readyDirs := make([]string, 0)
|
|
|
|
for dirPath, lastChange := range s.dirtyDirs {
|
|
if now.Sub(lastChange) >= 10*time.Second {
|
|
readyDirs = append(readyDirs, dirPath)
|
|
delete(s.dirtyDirs, dirPath)
|
|
}
|
|
}
|
|
|
|
s.dirtyDirsMu.Unlock()
|
|
|
|
if len(readyDirs) == 0 {
|
|
continue
|
|
}
|
|
|
|
affectedRoots := make(map[string]bool)
|
|
for _, dirPath := range readyDirs {
|
|
for _, folder := range s.folders {
|
|
if strings.HasPrefix(dirPath, folder) {
|
|
affectedRoots[folder] = true
|
|
break
|
|
}
|
|
}
|
|
}
|
|
|
|
for rootFolder := range affectedRoots {
|
|
s.enqueueLibraryScan(rootFolder)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// waitForFileStability checks if a file's mtime has stabilized
|
|
// Returns true when file is stable (not being modified)
|
|
// Polls every 3 seconds, times out after 60 seconds
|
|
// Uses atomic.Bool to prevent race conditions with concurrent checks
|
|
func (s *MediaScanner) waitForFileStability(filePath string) bool {
|
|
s.fileStabilityMu.Lock()
|
|
|
|
// Check if already being checked (atomic.Bool prevents race condition)
|
|
tracking, exists := s.fileStability[filePath]
|
|
if exists {
|
|
s.fileStabilityMu.Unlock()
|
|
// Another goroutine is already checking this file
|
|
if tracking.Load() {
|
|
return false // Still being checked
|
|
}
|
|
// Tracking exists but completed, remove stale entry
|
|
s.fileStabilityMu.Lock()
|
|
delete(s.fileStability, filePath)
|
|
}
|
|
|
|
// Start tracking with atomic.Bool set to true (checking in progress)
|
|
trackingFlag := &atomic.Bool{}
|
|
trackingFlag.Store(true)
|
|
s.fileStability[filePath] = trackingFlag
|
|
s.fileStabilityMu.Unlock()
|
|
|
|
// Get initial mtime
|
|
info, err := os.Stat(filePath)
|
|
if err != nil {
|
|
// Clean up tracking entry if file doesn't exist
|
|
s.fileStabilityMu.Lock()
|
|
delete(s.fileStability, filePath)
|
|
s.fileStabilityMu.Unlock()
|
|
return false
|
|
}
|
|
lastMtime := info.ModTime()
|
|
|
|
// Poll every 3 seconds for up to 60 seconds
|
|
timeout := time.After(60 * time.Second)
|
|
ticker := time.NewTicker(3 * time.Second)
|
|
defer ticker.Stop()
|
|
|
|
for {
|
|
select {
|
|
case <-timeout:
|
|
// Timeout - mark as done and clean up
|
|
trackingFlag.Store(false)
|
|
s.fileStabilityMu.Lock()
|
|
delete(s.fileStability, filePath)
|
|
s.fileStabilityMu.Unlock()
|
|
return false // File never stabilized
|
|
|
|
case <-ticker.C:
|
|
info, err := os.Stat(filePath)
|
|
if err != nil {
|
|
// File deleted - mark as done and clean up
|
|
trackingFlag.Store(false)
|
|
s.fileStabilityMu.Lock()
|
|
delete(s.fileStability, filePath)
|
|
s.fileStabilityMu.Unlock()
|
|
return false
|
|
}
|
|
|
|
currentMtime := info.ModTime()
|
|
if currentMtime.Equal(lastMtime) {
|
|
// File is stable! Mark as done and clean up
|
|
trackingFlag.Store(false)
|
|
s.fileStabilityMu.Lock()
|
|
delete(s.fileStability, filePath)
|
|
s.fileStabilityMu.Unlock()
|
|
return true
|
|
}
|
|
|
|
lastMtime = currentMtime
|
|
}
|
|
}
|
|
}
|
|
|
|
func (s *MediaScanner) scanDirectory(ctx context.Context, dirPath string) {
|
|
// Prevent concurrent scans of ANY directory
|
|
// Simple mutex is enough - job queue already serializes by directory
|
|
s.scanMutex.Lock()
|
|
defer s.scanMutex.Unlock()
|
|
|
|
s.scanInProgress.Store(true)
|
|
defer s.scanInProgress.Store(false)
|
|
|
|
// Find library for this directory
|
|
var libraryID pgtype.UUID
|
|
var rootFolder string
|
|
|
|
for _, folder := range s.folders {
|
|
if strings.HasPrefix(dirPath, folder) {
|
|
rootFolder = folder
|
|
if lib, err := s.db.GetLibraryByFolderPathPrefix(ctx, dirPath); err == nil {
|
|
libraryID = lib.LibraryID
|
|
break
|
|
}
|
|
}
|
|
}
|
|
|
|
// Check if libraryID is valid before proceeding
|
|
if !libraryID.Valid {
|
|
return
|
|
}
|
|
|
|
// Walk directory and process new files (recurses into subdirectories)
|
|
if err := filepath.WalkDir(dirPath, func(path string, d os.DirEntry, err error) error {
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if d.IsDir() {
|
|
return nil
|
|
}
|
|
if !s.isScannableFile(path) {
|
|
return nil
|
|
}
|
|
|
|
// Check if file is stable before processing (Audiobookshelf approach)
|
|
if !s.waitForFileStability(path) {
|
|
return nil
|
|
}
|
|
relPath := strings.TrimPrefix(path, rootFolder+"/")
|
|
_, err = s.db.GetMediaItemByFilePath(ctx, database.GetMediaItemByFilePathParams{
|
|
FilePath: relPath,
|
|
LibraryID: libraryID,
|
|
})
|
|
|
|
if errors.Is(err, pgx.ErrNoRows) {
|
|
if _, err := s.processMediaFile(ctx, path); err != nil {
|
|
s.errors++
|
|
} else {
|
|
s.newItems++
|
|
}
|
|
s.totalFiles++
|
|
}
|
|
|
|
return nil
|
|
}); err != nil {
|
|
fmt.Printf("Warning: failed to walk directory %s: %v\n", dirPath, err)
|
|
}
|
|
}
|
|
|
|
// performInitialScan scans all root folders on startup
|
|
// This ensures existing files are detected before watching begins
|
|
func (s *MediaScanner) performInitialScan(ctx context.Context) {
|
|
fmt.Printf("Performing initial scan of root folders...\n")
|
|
|
|
if s.defaultLibraryID.Valid && s.adminID.Valid {
|
|
folderPaths := s.folders
|
|
libraryIDStr := uuid.UUID(s.defaultLibraryID.Bytes).String()
|
|
adminIDStr := uuid.UUID(s.adminID.Bytes).String()
|
|
|
|
job := &Job{
|
|
ID: uuid.New().String(),
|
|
Type: JobTypeScan,
|
|
Status: JobStatusPending,
|
|
UserID: adminIDStr,
|
|
Context: context.Background(),
|
|
Params: map[string]any{
|
|
"library_id": libraryIDStr,
|
|
"folders": folderPaths,
|
|
"admin_id": adminIDStr,
|
|
"db": s.db,
|
|
"force": false,
|
|
},
|
|
}
|
|
|
|
if WorkerInstance != nil {
|
|
WorkerInstance.Enqueue(job)
|
|
fmt.Printf("Enqueued initial library scan job\n")
|
|
} else {
|
|
fmt.Printf("Warning: Worker not initialized, skipping initial scan\n")
|
|
}
|
|
} else {
|
|
fmt.Printf("Warning: no library/admin ID set, skipping initial scan\n")
|
|
}
|
|
|
|
fmt.Printf("Initial scan jobs enqueued\n")
|
|
}
|
|
|
|
func (s *MediaScanner) Close() error {
|
|
fmt.Printf("Cleaning up scanner resources...\n")
|
|
|
|
// Stop watching
|
|
if s.watcher != nil {
|
|
if err := s.watcher.Close(); err != nil {
|
|
fmt.Printf("Warning: failed to close fsnotify watcher: %v\n", err)
|
|
}
|
|
}
|
|
|
|
// Clean up fileStability map to prevent memory leaks
|
|
s.fileStabilityMu.Lock()
|
|
s.fileStability = make(map[string]*atomic.Bool)
|
|
s.fileStabilityMu.Unlock()
|
|
|
|
// Clear dirty directories
|
|
s.dirtyDirsMu.Lock()
|
|
s.dirtyDirs = make(map[string]time.Time)
|
|
s.dirtyDirsMu.Unlock()
|
|
|
|
// Wait for in-progress scan to complete (with timeout)
|
|
timeout := time.After(5 * time.Second)
|
|
done := make(chan struct{})
|
|
|
|
go func() {
|
|
for s.scanInProgress.Load() {
|
|
time.Sleep(100 * time.Millisecond)
|
|
}
|
|
close(done)
|
|
}()
|
|
|
|
select {
|
|
case <-done:
|
|
fmt.Printf("Scanner cleanup complete\n")
|
|
case <-timeout:
|
|
fmt.Printf("Timeout waiting for scan to complete\n")
|
|
}
|
|
|
|
return nil
|
|
}
|
|
func (s *MediaScanner) startBackupScan(ctx context.Context) {
|
|
interval := s.GetPollInterval()
|
|
if interval <= 0 {
|
|
fmt.Println("[BACKUP-SCAN] Periodic scan disabled (interval = 0)")
|
|
return
|
|
}
|
|
fmt.Printf("[BACKUP-SCAN] Periodic scan started with interval: %v\n", interval)
|
|
|
|
for {
|
|
ticker := time.NewTicker(interval)
|
|
defer ticker.Stop()
|
|
|
|
select {
|
|
case <-ctx.Done():
|
|
fmt.Println("[BACKUP-SCAN] Periodic scan stopped")
|
|
return
|
|
case <-ticker.C:
|
|
interval = s.GetPollInterval()
|
|
if !s.GetAutoScanEnabled() {
|
|
continue
|
|
}
|
|
fmt.Printf("[BACKUP-SCAN] Running periodic full scan (interval: %v)...\n", interval)
|
|
for _, folder := range s.folders {
|
|
s.enqueueLibraryScan(folder)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
func (s *MediaScanner) calculateFileSHA256(filePath string) (string, error) {
|
|
return computeFileSHA256(filePath)
|
|
}
|
|
|
|
// computeFileSHA256 is the package-level full-file SHA-256 used by the hash
|
|
// backfill service; the MediaScanner method delegates to it.
|
|
func computeFileSHA256(filePath string) (string, error) {
|
|
file, err := os.Open(filePath)
|
|
if err != nil {
|
|
return "", fmt.Errorf("failed to open file: %v", err)
|
|
}
|
|
defer func() {
|
|
if err := file.Close(); err != nil {
|
|
fmt.Printf("Warning: failed to close file %s: %v\n", filePath, err)
|
|
}
|
|
}()
|
|
|
|
hasher := sha256.New()
|
|
if _, err := io.Copy(hasher, file); err != nil {
|
|
return "", fmt.Errorf("failed to calculate hash: %v", err)
|
|
}
|
|
|
|
return hex.EncodeToString(hasher.Sum(nil)), nil
|
|
}
|
|
|
|
// OPFIdentifier represents an identifier from OPF metadata
|
|
type OPFIdentifier struct {
|
|
XMLName xml.Name `xml:"identifier"`
|
|
ID string `xml:"id,attr"`
|
|
Scheme string `xml:"scheme,attr"`
|
|
Content string `xml:",chardata"`
|
|
}
|
|
|
|
// OPFMetadata represents parsed OPF metadata
|
|
type OPFMetadata struct {
|
|
XMLName xml.Name `xml:"package"`
|
|
Version string `xml:"version,attr"`
|
|
Identifiers []OPFIdentifier `xml:"metadata>identifier"`
|
|
}
|
|
|
|
// extractOPFIdentifiers extracts identifiers from EPUB OPF file
|
|
func (s *MediaScanner) extractOPFIdentifiers(epubPath string) (opfIdentifier, opfUUID string, confidence string, err error) {
|
|
book, err := epub.ReadBook(epubPath)
|
|
if err != nil {
|
|
return "", "", "", fmt.Errorf("failed to open EPUB: %v", err)
|
|
}
|
|
|
|
// Extract OPF identifiers using go-epub library
|
|
identifiers, err := book.MetadataByKey("identifier")
|
|
if err != nil || len(identifiers) == 0 {
|
|
return "", "", "low", nil
|
|
}
|
|
|
|
var identifier, uuidString string
|
|
for _, id := range identifiers {
|
|
id = strings.TrimSpace(id)
|
|
|
|
// Check for UUID format (urn:uuid:)
|
|
if trimmed, found := strings.CutPrefix(strings.ToLower(id), "urn:uuid:"); found {
|
|
uuidString = trimmed
|
|
continue
|
|
}
|
|
|
|
// Check if it's a plain UUID (8-4-4-4-12 format)
|
|
if isValidUUID(id) {
|
|
uuidString = id
|
|
continue
|
|
}
|
|
|
|
// Check for ISBN
|
|
if strings.Contains(strings.ToLower(id), "isbn") {
|
|
isbn := s.extractISBNFromIdentifier(id)
|
|
if isbn != "" {
|
|
identifier = isbn
|
|
continue
|
|
}
|
|
}
|
|
|
|
// Use first identifier as fallback
|
|
if identifier == "" && id != "" {
|
|
identifier = id
|
|
}
|
|
}
|
|
|
|
confidence = s.determineHashConfidence(uuidString, identifier)
|
|
|
|
return identifier, uuidString, confidence, nil
|
|
}
|
|
|
|
// isValidUUID checks if string is a valid UUID (8-4-4-4-12 format)
|
|
func isValidUUID(idStr string) bool {
|
|
uuidRegex := regexp.MustCompile(`^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$`)
|
|
return uuidRegex.MatchString(idStr)
|
|
}
|
|
|
|
// extractISBNFromIdentifier extracts ISBN from identifier string
|
|
func (s *MediaScanner) extractISBNFromIdentifier(id string) string {
|
|
id = strings.TrimSpace(id)
|
|
|
|
// Remove "isbn:" prefix if present
|
|
if trimmed, found := strings.CutPrefix(strings.ToLower(id), "isbn:"); found {
|
|
id = trimmed
|
|
}
|
|
|
|
// Remove hyphens and spaces
|
|
isbn := regexp.MustCompile(`[\s-]`).ReplaceAllString(id, "")
|
|
|
|
// Check if it's a valid ISBN-10 or ISBN-13
|
|
if len(isbn) == 10 || len(isbn) == 13 {
|
|
return isbn
|
|
}
|
|
|
|
return ""
|
|
}
|
|
|
|
// determineHashConfidence determines confidence level based on available identifiers
|
|
func (s *MediaScanner) determineHashConfidence(uuid, identifier string) string {
|
|
if uuid != "" && isValidUUID(uuid) {
|
|
return "high"
|
|
}
|
|
if identifier != "" && (strings.Contains(strings.ToLower(identifier), "isbn") || len(identifier) >= 10) {
|
|
return "medium"
|
|
}
|
|
return "low"
|
|
}
|
|
|
|
// detectFormatType detects the format type based on file extension and content
|
|
func (s *MediaScanner) detectFormatType(filePath string) string {
|
|
base := strings.ToLower(filepath.Base(filePath))
|
|
|
|
// Check for compound extensions first (like .kepub.epub)
|
|
if strings.HasSuffix(base, ".kepub.epub") {
|
|
return "kepub"
|
|
}
|
|
|
|
ext := strings.ToLower(filepath.Ext(filePath))
|
|
|
|
switch ext {
|
|
case ".epub":
|
|
return "epub"
|
|
case ".kepub":
|
|
return "kepub"
|
|
case ".pdf":
|
|
return "pdf"
|
|
case ".cbz", ".cbr", ".cb7", ".cbt":
|
|
return "comic_archive"
|
|
case ".mobi":
|
|
return "mobi"
|
|
case ".azw", ".azw3":
|
|
return "kfx"
|
|
case ".txt":
|
|
return "txt"
|
|
case ".fb2":
|
|
return "fb2"
|
|
default:
|
|
return "unknown"
|
|
}
|
|
}
|
|
|
|
// extractHashInfo calculates hash and extracts OPF identifiers for a file
|
|
func (s *MediaScanner) extractHashInfo(filePath string) (*HashInfo, *FormatInfo, error) {
|
|
// Calculate SHA-256
|
|
fileSHA256, err := s.calculateFileSHA256(filePath)
|
|
if err != nil {
|
|
return nil, nil, fmt.Errorf("failed to calculate SHA-256: %v", err)
|
|
}
|
|
|
|
// Get file info
|
|
info, err := os.Stat(filePath)
|
|
if err != nil {
|
|
return nil, nil, fmt.Errorf("failed to get file info: %v", err)
|
|
}
|
|
|
|
// Extract OPF identifiers for EPUB files
|
|
var opfIdentifier, opfUUID, confidence string
|
|
if strings.HasSuffix(strings.ToLower(filePath), ".epub") {
|
|
opfIdentifier, opfUUID, confidence, err = s.extractOPFIdentifiers(filePath)
|
|
if err != nil {
|
|
// Non-fatal error, continue with low confidence
|
|
confidence = "low"
|
|
}
|
|
}
|
|
|
|
hashInfo := &HashInfo{
|
|
FileSHA256: fileSHA256,
|
|
OPFIdentifier: opfIdentifier,
|
|
OPFUUID: opfUUID,
|
|
HashConfidence: confidence,
|
|
}
|
|
|
|
formatInfo := &FormatInfo{
|
|
FormatType: s.detectFormatType(filePath),
|
|
FilePath: filePath,
|
|
FileSHA256: fileSHA256,
|
|
FileSizeBytes: info.Size(),
|
|
MimeType: s.getMimeType(filePath),
|
|
}
|
|
|
|
return hashInfo, formatInfo, nil
|
|
}
|
|
|
|
func (s *MediaScanner) getRelativePath(absolutePath string) string {
|
|
// Get the base folder paths from scanner
|
|
for _, baseFolder := range s.folders {
|
|
// Check if path is within this base folder
|
|
if relPath, ok := strings.CutPrefix(absolutePath, baseFolder); ok {
|
|
return strings.TrimPrefix(relPath, "/")
|
|
}
|
|
}
|
|
// Fallback: if no match, return as-is (shouldn't happen)
|
|
return absolutePath
|
|
}
|