- Add SettingsCache with TTL-based invalidation (30 seconds) - Cache scan_poll_interval_seconds and auto_scan_enabled settings - Reduce database queries from every poll/check to once per TTL period - Improve error handling with proper fallback values - Simplify boolean parsing with strings.ToLower for consistency This optimization reduces database load when checking scan settings, which occurs frequently during media scanning operations.
2297 lines
66 KiB
Go
2297 lines
66 KiB
Go
package services
|
|
|
|
import (
|
|
"archive/tar"
|
|
"archive/zip"
|
|
"bookhoard/internal/database"
|
|
"bookhoard/internal/utils"
|
|
"bytes"
|
|
"compress/bzip2"
|
|
"compress/gzip"
|
|
"context"
|
|
"crypto/sha256"
|
|
"encoding/hex"
|
|
"encoding/xml"
|
|
"fmt"
|
|
"image"
|
|
_ "image/jpeg"
|
|
_ "image/png"
|
|
"io"
|
|
"io/fs"
|
|
"os"
|
|
"path/filepath"
|
|
"regexp"
|
|
"strconv"
|
|
"strings"
|
|
"sync"
|
|
"sync/atomic"
|
|
"time"
|
|
|
|
"bookhoard/internal/sevenzip"
|
|
|
|
epub "github.com/ArcadiaLin/go-epub"
|
|
"github.com/fsnotify/fsnotify"
|
|
"github.com/google/uuid"
|
|
"github.com/jackc/pgx/v5"
|
|
"github.com/jackc/pgx/v5/pgtype"
|
|
"github.com/nwaples/rardecode"
|
|
pdfcpuapi "github.com/pdfcpu/pdfcpu/pkg/api"
|
|
)
|
|
|
|
// MediaMetadata contains extracted metadata for media files (ebooks, comics, manga)
|
|
type MediaMetadata struct {
|
|
Title string
|
|
Author string
|
|
Description string
|
|
Series string
|
|
SeriesNumber int32
|
|
Publisher string
|
|
PublishDate time.Time
|
|
Contributors []string
|
|
CoverPath string
|
|
ISBN string
|
|
ASIN string
|
|
Tags []string
|
|
|
|
FileHashInfo *HashInfo
|
|
FileFormats []*FormatInfo
|
|
}
|
|
|
|
type HashInfo struct {
|
|
FileSHA256 string
|
|
OPFIdentifier string
|
|
OPFUUID string
|
|
HashConfidence string
|
|
}
|
|
|
|
type FormatInfo struct {
|
|
FormatType string
|
|
FilePath string
|
|
FileSHA256 string
|
|
FileSizeBytes int64
|
|
MimeType string
|
|
}
|
|
|
|
// MediaScanner scans library folders for media files (ebooks, comics, manga)
|
|
type MediaScanner struct {
|
|
db *database.Queries
|
|
watcher *fsnotify.Watcher
|
|
folders []string
|
|
adminID pgtype.UUID
|
|
defaultLibraryID pgtype.UUID
|
|
libraryTypes map[string][]string
|
|
forceRescan bool
|
|
logger *ScannerLogger
|
|
dirtyDirs map[string]time.Time
|
|
dirtyDirsMu sync.RWMutex
|
|
fileStability map[string]*atomic.Bool
|
|
fileStabilityMu sync.RWMutex
|
|
scan_mutex sync.Mutex
|
|
pollInterval time.Duration
|
|
watching atomic.Bool
|
|
settingsCache *SettingsCache
|
|
|
|
totalFiles int
|
|
newItems int
|
|
errors int
|
|
job *Job
|
|
}
|
|
|
|
// NewMediaScanner creates a new media scanner instance
|
|
func NewMediaScanner(db *database.Queries) *MediaScanner {
|
|
watcher, err := fsnotify.NewWatcher()
|
|
if err != nil {
|
|
panic(fmt.Sprintf("Failed to create file watcher: %v", err))
|
|
}
|
|
|
|
return &MediaScanner{
|
|
db: db,
|
|
watcher: watcher,
|
|
settingsCache: NewSettingsCache(30 * time.Second),
|
|
dirtyDirs: make(map[string]time.Time),
|
|
fileStability: make(map[string]*atomic.Bool),
|
|
pollInterval: 60 * time.Second,
|
|
watching: atomic.Bool{},
|
|
folders: []string{},
|
|
adminID: pgtype.UUID{},
|
|
defaultLibraryID: pgtype.UUID{Valid: false},
|
|
libraryTypes: make(map[string][]string),
|
|
logger: NewScannerLogger(),
|
|
}
|
|
}
|
|
|
|
func (s *MediaScanner) GetPollInterval() time.Duration {
|
|
// Check cache first
|
|
if cached, ok := s.settingsCache.Get("scan_poll_interval_seconds"); ok {
|
|
if seconds, err := strconv.Atoi(cached); err == nil {
|
|
return time.Duration(seconds) * time.Second
|
|
}
|
|
}
|
|
|
|
// Cache miss - query database
|
|
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
|
defer cancel()
|
|
|
|
setting, err := s.db.GetSystemSetting(ctx, "scan_poll_interval_seconds")
|
|
if err != nil || setting == "" {
|
|
return 60 * time.Second
|
|
}
|
|
|
|
// Store in cache
|
|
s.settingsCache.Set("scan_poll_interval_seconds", setting)
|
|
|
|
// Convert to duration
|
|
seconds, err := strconv.Atoi(setting)
|
|
if err != nil {
|
|
return 60 * time.Second
|
|
}
|
|
return time.Duration(seconds) * time.Second
|
|
}
|
|
|
|
func (s *MediaScanner) GetAutoScanEnabled() bool {
|
|
// Check cache first
|
|
if cached, ok := s.settingsCache.Get("auto_scan_enabled"); ok {
|
|
return strings.ToLower(cached) == "true"
|
|
}
|
|
|
|
// Cache miss - query database
|
|
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
|
defer cancel()
|
|
|
|
setting, err := s.db.GetSystemSetting(ctx, "auto_scan_enabled")
|
|
if err != nil || setting == "" {
|
|
return true
|
|
}
|
|
|
|
// Store in cache
|
|
s.settingsCache.Set("auto_scan_enabled", setting)
|
|
|
|
return strings.ToLower(setting) == "true"
|
|
}
|
|
|
|
func (s *MediaScanner) SetAdminID(adminID pgtype.UUID) {
|
|
s.adminID = adminID
|
|
}
|
|
|
|
func (s *MediaScanner) SetLibraryID(libraryID pgtype.UUID) {
|
|
s.defaultLibraryID = libraryID
|
|
}
|
|
|
|
func (s *MediaScanner) SetForce(force bool) {
|
|
s.forceRescan = force
|
|
}
|
|
|
|
func (s *MediaScanner) GetStats() (int, int, int) {
|
|
return s.totalFiles, s.newItems, s.errors
|
|
}
|
|
|
|
func (s *MediaScanner) SetFolders(folders []string) error {
|
|
s.folders = folders
|
|
|
|
// Remove old watch if exists
|
|
if s.watcher != nil {
|
|
s.watcher.Close()
|
|
}
|
|
|
|
// Create new watcher
|
|
watcher, err := fsnotify.NewWatcher()
|
|
if err != nil {
|
|
return fmt.Errorf("failed to create watcher: %v", err)
|
|
}
|
|
s.watcher = watcher
|
|
|
|
// Build cache of allowed extensions per folder
|
|
s.libraryTypes = make(map[string][]string)
|
|
ctx := context.Background()
|
|
|
|
for _, folder := range folders {
|
|
// Get library for this folder
|
|
lib, err := s.db.GetLibraryByFolder(ctx, folder)
|
|
if err != nil {
|
|
fmt.Printf("Warning: failed to get library for folder %s: %v\n", folder, err)
|
|
continue
|
|
}
|
|
|
|
// Get library type with allowed extensions
|
|
libType, err := s.db.GetLibraryType(ctx, lib.LibraryTypeID)
|
|
if err != nil {
|
|
fmt.Printf("Warning: failed to get library type for %s: %v\n", folder, err)
|
|
continue
|
|
}
|
|
|
|
// Cache allowed extensions for this folder
|
|
s.libraryTypes[folder] = libType.AllowedExtensions
|
|
fmt.Printf("Scanner: Folder %s (type: %s) allows extensions: %v\n",
|
|
folder, libType.Name, libType.AllowedExtensions)
|
|
}
|
|
|
|
// Add all folders to watch
|
|
for _, folder := range folders {
|
|
if err := s.watcher.Add(folder); err != nil {
|
|
fmt.Printf("Warning: failed to watch folder %s: %v\n", folder, err)
|
|
}
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func (s *MediaScanner) ScanFolders(ctx context.Context) error {
|
|
if len(s.folders) == 0 {
|
|
return fmt.Errorf("no folders set")
|
|
}
|
|
|
|
s.totalFiles = 0
|
|
s.newItems = 0
|
|
s.errors = 0
|
|
|
|
for _, folder := range s.folders {
|
|
filepath.WalkDir(folder, func(path string, d fs.DirEntry, err error) error {
|
|
if !d.IsDir() && s.isScannableFile(path) {
|
|
s.totalFiles++
|
|
}
|
|
return nil
|
|
})
|
|
}
|
|
|
|
fmt.Printf("Starting scan of %d folders: %v (%d files to scan)\n", len(s.folders), s.folders, s.totalFiles)
|
|
|
|
processedFiles := 0
|
|
mediaFiles := 0
|
|
|
|
for _, folder := range s.folders {
|
|
fmt.Printf("Scanning folder: %s\n", folder)
|
|
|
|
if _, err := os.Stat(folder); os.IsNotExist(err) {
|
|
fmt.Printf("Folder does not exist: %s\n", folder)
|
|
s.errors++
|
|
continue
|
|
}
|
|
|
|
err := filepath.WalkDir(folder, func(path string, d fs.DirEntry, err error) error {
|
|
if err != nil {
|
|
fmt.Printf("Error accessing path %s: %v\n", path, err)
|
|
s.errors++
|
|
return err
|
|
}
|
|
|
|
if d.IsDir() {
|
|
if err := s.watcher.Add(path); err != nil {
|
|
fmt.Printf("Warning: failed to watch subdirectory %s: %v\n", path, err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
if s.isScannableFile(path) {
|
|
mediaFiles++
|
|
processedFiles++
|
|
|
|
if processedFiles%10 == 0 && s.totalFiles > 0 {
|
|
progress := float64(processedFiles) / float64(s.totalFiles)
|
|
if s.job != nil {
|
|
s.job.UpdateProgress(progress, processedFiles, s.newItems, s.errors)
|
|
}
|
|
}
|
|
|
|
wasNew, err := s.processMediaFile(ctx, path)
|
|
if err != nil {
|
|
fmt.Printf("Error processing media file %s: %v\n", path, err)
|
|
s.errors++
|
|
} else if wasNew {
|
|
fmt.Printf("Successfully processed media file: %s\n", path)
|
|
} else {
|
|
fmt.Printf("Successfully processed media file: %s\n", path)
|
|
}
|
|
}
|
|
|
|
return nil
|
|
})
|
|
if err != nil {
|
|
s.errors++
|
|
return fmt.Errorf("failed to scan folder %s: %v", folder, err)
|
|
}
|
|
}
|
|
|
|
fmt.Printf("Scan completed: %d total files scanned, %d media files found, %d new items, %d errors\n",
|
|
processedFiles, mediaFiles, s.newItems, s.errors)
|
|
|
|
// Clean up: Find media items in DB that no longer exist on filesystem
|
|
for _, folder := range s.folders {
|
|
lib, err := s.db.GetLibraryByFolder(ctx, folder)
|
|
if err != nil {
|
|
continue
|
|
}
|
|
libraryID := lib.LibraryID
|
|
|
|
dbItems, err := s.db.ListMediaItemsByLibrary(ctx, libraryID)
|
|
if err != nil {
|
|
fmt.Printf("Warning: failed to get library items for cleanup: %v\n", err)
|
|
continue
|
|
}
|
|
|
|
// Build set of scanned file paths for this folder
|
|
scannedPaths := make(map[string]bool)
|
|
filepath.WalkDir(folder, func(path string, d os.DirEntry, walkErr error) error {
|
|
if walkErr != nil {
|
|
return walkErr
|
|
}
|
|
if !d.IsDir() && s.isScannableFile(path) {
|
|
scannedPaths[s.getRelativePath(path)] = true
|
|
}
|
|
return nil
|
|
})
|
|
|
|
// Delete items whose files no longer exist - with safety logging
|
|
for _, item := range dbItems {
|
|
filePath := item.FilePath
|
|
if filePath != "" && !scannedPaths[filePath] {
|
|
msg := fmt.Sprintf("[RESCAN-CLEANUP] Orphaned media item found: ID=%s, Title=%s, Path=%s",
|
|
item.ID, item.Title, filePath)
|
|
s.logger.LogDelete(msg)
|
|
|
|
delMsg := fmt.Sprintf("[RESCAN-CLEANUP] Deleting orphaned item '%s' (file no longer exists at %s)",
|
|
item.Title, filePath)
|
|
s.logger.LogDelete(delMsg)
|
|
|
|
if err := s.db.DeleteMediaItem(ctx, item.ID); err != nil {
|
|
errMsg := fmt.Sprintf("[RESCAN-CLEANUP] ERROR: failed to delete orphaned item %s: %v", item.Title, err)
|
|
s.logger.LogDelete(errMsg)
|
|
s.logger.LogError(errMsg)
|
|
} else {
|
|
s.logger.LogDelete(fmt.Sprintf("[RESCAN-CLEANUP] SUCCESS: deleted orphaned item '%s'", item.Title))
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
if s.job != nil && s.totalFiles > 0 {
|
|
s.job.UpdateProgress(1.0, processedFiles, s.newItems, s.errors)
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func (s *MediaScanner) isScannableFile(path string) bool {
|
|
ext := strings.ToLower(filepath.Ext(path))
|
|
|
|
// Find which folder this file belongs to
|
|
var folder string
|
|
for _, f := range s.folders {
|
|
if strings.HasPrefix(path, f) {
|
|
folder = f
|
|
break
|
|
}
|
|
}
|
|
|
|
// If no folder match, don't scan
|
|
if folder == "" {
|
|
return false
|
|
}
|
|
|
|
// Get allowed extensions for this folder's library
|
|
allowed, ok := s.libraryTypes[folder]
|
|
if !ok {
|
|
// No library type info, skip file
|
|
fmt.Printf("Warning: No library type info for folder %s, skipping %s\n", folder, path)
|
|
return false
|
|
}
|
|
|
|
// Check if file extension is allowed for this library type
|
|
for _, allowedExt := range allowed {
|
|
if ext == strings.ToLower(allowedExt) {
|
|
return true
|
|
}
|
|
}
|
|
|
|
return false
|
|
}
|
|
|
|
// extractFolderStructureMetadata extracts metadata from folder paths, prioritizing Calibre structure
|
|
func (s *MediaScanner) extractFolderStructureMetadata(path, rootFolder string) *MediaMetadata {
|
|
metadata := &MediaMetadata{}
|
|
|
|
// Get the relative path from root folder
|
|
relPath, err := filepath.Rel(rootFolder, path)
|
|
if err != nil {
|
|
return metadata
|
|
}
|
|
|
|
// Split into directory components
|
|
dir := filepath.Dir(relPath)
|
|
components := strings.Split(dir, string(filepath.Separator))
|
|
|
|
if len(components) < 2 {
|
|
return metadata // Not enough structure to extract
|
|
}
|
|
|
|
// Calibre structure detection
|
|
// Pattern 1: Author Name/Book Title/
|
|
// Pattern 2: Author Name/Series Name/Book Title/
|
|
// Pattern 3: Author Name/Series Name, Book #1 - Book Title/
|
|
|
|
author := strings.TrimSuffix(components[0], "_") // Remove trailing underscore if present
|
|
metadata.Author = strings.ReplaceAll(author, "_", " ")
|
|
|
|
if len(components) >= 3 {
|
|
// This might be a series structure
|
|
possibleSeries := components[1]
|
|
possibleTitle := components[2]
|
|
|
|
// Check for Calibre series format: "Series Name, Book #1 - Title"
|
|
seriesMatch := regexp.MustCompile(`^(.*),\s+Book\s+#(\d+)\s*-\s*(.*)$`).FindStringSubmatch(possibleSeries)
|
|
if len(seriesMatch) == 4 {
|
|
metadata.Series = strings.ReplaceAll(seriesMatch[1], "_", " ")
|
|
if seriesNum, err := strconv.ParseInt(seriesMatch[2], 10, 32); err == nil {
|
|
metadata.SeriesNumber = int32(seriesNum)
|
|
}
|
|
metadata.Title = strings.ReplaceAll(possibleTitle, "_", " ")
|
|
} else {
|
|
// Simple series structure: Author/Series/Title
|
|
metadata.Series = strings.ReplaceAll(possibleSeries, "_", " ")
|
|
metadata.Title = strings.ReplaceAll(possibleTitle, "_", " ")
|
|
|
|
// Try to extract series number from title
|
|
titleNumMatch := regexp.MustCompile(`^(.*)\s+(\d+)$`).FindStringSubmatch(metadata.Title)
|
|
if len(titleNumMatch) == 3 {
|
|
metadata.Title = titleNumMatch[1]
|
|
if seriesNum, err := strconv.ParseInt(titleNumMatch[2], 10, 32); err == nil {
|
|
metadata.SeriesNumber = int32(seriesNum)
|
|
}
|
|
}
|
|
}
|
|
} else {
|
|
// Simple structure: Author/Title
|
|
metadata.Title = strings.ReplaceAll(components[1], "_", " ")
|
|
}
|
|
|
|
return metadata
|
|
}
|
|
|
|
func (s *MediaScanner) processMediaFile(ctx context.Context, path string) (bool, error) {
|
|
fmt.Printf("Processing media file: %s\n", path)
|
|
|
|
// Get file info
|
|
info, err := os.Stat(path)
|
|
if err != nil {
|
|
fmt.Printf("Failed to get file info for %s: %v\n", path, err)
|
|
return false, fmt.Errorf("failed to get file info: %v", err)
|
|
}
|
|
|
|
fmt.Printf("File info for %s: size=%d\n", path, info.Size())
|
|
|
|
// Get file modification time for created_at
|
|
fileModTime := info.ModTime()
|
|
|
|
// Find library for this file's folder
|
|
var libraryID pgtype.UUID
|
|
if s.defaultLibraryID.Valid {
|
|
libraryID = s.defaultLibraryID
|
|
} else {
|
|
for _, folder := range s.folders {
|
|
if strings.HasPrefix(path, folder) {
|
|
lib, err := s.db.GetLibraryByFolder(ctx, folder)
|
|
if err != nil {
|
|
return false, fmt.Errorf("failed to find library for folder %s: %v", folder, err)
|
|
}
|
|
libraryID = lib.LibraryID
|
|
break
|
|
}
|
|
}
|
|
}
|
|
|
|
if !libraryID.Valid {
|
|
return false, fmt.Errorf("no library found for file path: %s", path)
|
|
}
|
|
|
|
// Check if media item already exists in database
|
|
existingItem, err := s.getMediaItemByFilePath(ctx, path, libraryID)
|
|
if err == nil {
|
|
fmt.Printf("Media item already exists in database: %s (size: %d vs %d)\n", path, existingItem.FileSize.Int64, info.Size())
|
|
|
|
// If force rescan is enabled, always re-process
|
|
if s.forceRescan {
|
|
fmt.Printf("Force rescan enabled, updating existing media item: %s\n", path)
|
|
// Use UPDATE instead of DELETE+INSERT to preserve created_at
|
|
if err := s.updateMediaItem(ctx, existingItem.ID, path, info); err != nil {
|
|
fmt.Printf("Warning: failed to update existing media item: %v\n", err)
|
|
}
|
|
return false, nil
|
|
} else {
|
|
// Normal behavior: check if file has changed (by size)
|
|
if existingItem.FileSize.Int64 != info.Size() {
|
|
fmt.Printf("File size changed, updating media item: %s\n", path)
|
|
_ = s.updateMediaItem(ctx, existingItem.ID, path, info)
|
|
return false, nil
|
|
}
|
|
fmt.Printf("Media item already exists with same size, skipping: %s\n", path)
|
|
return false, nil
|
|
}
|
|
} else if err != pgx.ErrNoRows {
|
|
fmt.Printf("Database error checking media item existence: %v\n", err)
|
|
return false, fmt.Errorf("failed to check if media item exists: %v", err)
|
|
}
|
|
fmt.Printf("Media item does not exist in database, creating new entry: %s\n", path)
|
|
|
|
// Extract metadata from file first
|
|
metadata, err := s.extractMetadata(path)
|
|
if err != nil {
|
|
fmt.Printf("Warning: failed to extract metadata from %s: %v\n", path, err)
|
|
metadata = &MediaMetadata{}
|
|
}
|
|
|
|
// Extract hash information during metadata extraction
|
|
hashInfo, formatInfo, err := s.extractHashInfo(path)
|
|
if err != nil {
|
|
fmt.Printf("Warning: failed to extract hash info from %s: %v\n", path, err)
|
|
hashInfo = &HashInfo{}
|
|
formatInfo = &FormatInfo{}
|
|
} else {
|
|
metadata.FileHashInfo = hashInfo
|
|
metadata.FileFormats = []*FormatInfo{formatInfo}
|
|
fmt.Printf("Hash info for %s: SHA256=%s, OPF_ID=%s, OPF_UUID=%s, Confidence=%s\n",
|
|
path, hashInfo.FileSHA256, hashInfo.OPFIdentifier, hashInfo.OPFUUID, hashInfo.HashConfidence)
|
|
}
|
|
|
|
var comicInfo *ComicInfo
|
|
var coverImage []byte
|
|
|
|
ext := strings.ToLower(filepath.Ext(path))
|
|
if ext == ".cbz" || ext == ".cbr" || ext == ".cb7" || ext == ".cbt" ||
|
|
strings.HasSuffix(strings.ToLower(path), ".tar.gz") ||
|
|
strings.HasSuffix(strings.ToLower(path), ".tar.bz2") ||
|
|
strings.HasSuffix(strings.ToLower(path), ".tgz") ||
|
|
strings.HasSuffix(strings.ToLower(path), ".tbz2") {
|
|
info, cover, err := extractComicMetadata(path)
|
|
if err != nil {
|
|
fmt.Printf("Warning: failed to extract comic metadata from %s: %v\n", path, err)
|
|
} else {
|
|
comicInfo = info
|
|
coverImage = cover
|
|
if comicInfo.Title != "" && metadata.Title == "" {
|
|
metadata.Title = comicInfo.Title
|
|
}
|
|
if comicInfo.Series != "" && metadata.Series == "" {
|
|
metadata.Series = comicInfo.Series
|
|
}
|
|
if comicInfo.Number > 0 && metadata.SeriesNumber == 0 {
|
|
metadata.SeriesNumber = int32(comicInfo.Number)
|
|
}
|
|
if comicInfo.Publisher != "" && metadata.Publisher == "" {
|
|
metadata.Publisher = comicInfo.Publisher
|
|
}
|
|
if comicInfo.Writer != "" && metadata.Author == "" {
|
|
metadata.Author = comicInfo.Writer
|
|
}
|
|
if len(coverImage) > 0 && metadata.CoverPath == "" {
|
|
coverPath := path + ".cover.jpg"
|
|
if err := os.WriteFile(coverPath, coverImage, 0644); err == nil {
|
|
metadata.CoverPath = s.getRelativePath(coverPath)
|
|
}
|
|
}
|
|
fmt.Printf("Extracted comic metadata from %s: title=%s, series=%s, issue=%d\n",
|
|
path, comicInfo.Title, comicInfo.Series, comicInfo.Number)
|
|
}
|
|
}
|
|
|
|
// Try to get metadata from folder structure as fallback/enhancement
|
|
// Use the root folder that contains this file
|
|
var rootFolder string
|
|
for _, folder := range s.folders {
|
|
if strings.HasPrefix(path, folder) {
|
|
rootFolder = folder
|
|
break
|
|
}
|
|
}
|
|
|
|
if rootFolder != "" {
|
|
folderMetadata := s.extractFolderStructureMetadata(path, rootFolder)
|
|
|
|
// Use folder metadata as fallback for missing information
|
|
if metadata.Title == "" && folderMetadata.Title != "" {
|
|
metadata.Title = folderMetadata.Title
|
|
}
|
|
if metadata.Author == "" && folderMetadata.Author != "" {
|
|
metadata.Author = folderMetadata.Author
|
|
}
|
|
if metadata.Series == "" && folderMetadata.Series != "" {
|
|
metadata.Series = folderMetadata.Series
|
|
}
|
|
if metadata.SeriesNumber == 0 && folderMetadata.SeriesNumber > 0 {
|
|
metadata.SeriesNumber = folderMetadata.SeriesNumber
|
|
}
|
|
}
|
|
|
|
// Final fallback if still missing essential metadata
|
|
if metadata.Title == "" {
|
|
metadata.Title = strings.TrimSuffix(filepath.Base(path), filepath.Ext(path))
|
|
}
|
|
if metadata.Author == "" {
|
|
metadata.Author = "Unknown"
|
|
}
|
|
|
|
// libraryID already determined at start of function
|
|
|
|
// Normalize metadata fields for display
|
|
metadata.Contributors = utils.NormalizeContributors(metadata.Contributors)
|
|
metadata.Tags = utils.NormalizeTags(metadata.Tags)
|
|
|
|
// Normalize search fields
|
|
contributorsSearch := utils.NormalizeContributorsSearch(metadata.Contributors)
|
|
tagsSearch := utils.NormalizeTagsSearch(metadata.Tags)
|
|
|
|
// Create media item in database
|
|
relativePath := s.getRelativePath(path)
|
|
createdItem, err := s.db.CreateMediaItem(ctx, database.CreateMediaItemParams{
|
|
LibraryID: libraryID,
|
|
Title: metadata.Title,
|
|
Author: pgtype.Text{String: metadata.Author, Valid: metadata.Author != ""},
|
|
Isbn: pgtype.Text{String: utils.NormalizeISBNSafe(metadata.ISBN), Valid: metadata.ISBN != ""},
|
|
Asin: pgtype.Text{String: metadata.ASIN, Valid: metadata.ASIN != ""},
|
|
Description: pgtype.Text{String: metadata.Description, Valid: metadata.Description != ""},
|
|
FilePath: relativePath,
|
|
FileSize: pgtype.Int8{Int64: info.Size(), Valid: true},
|
|
MimeType: pgtype.Text{String: s.getMimeType(path), Valid: true},
|
|
CoverImagePath: pgtype.Text{String: metadata.CoverPath, Valid: metadata.CoverPath != ""},
|
|
Series: pgtype.Text{String: metadata.Series, Valid: metadata.Series != ""},
|
|
SeriesNumber: pgtype.Int4{Int32: metadata.SeriesNumber, Valid: metadata.SeriesNumber > 0},
|
|
Publisher: pgtype.Text{String: metadata.Publisher, Valid: metadata.Publisher != ""},
|
|
DatePublished: pgtype.Date{Time: metadata.PublishDate, Valid: !metadata.PublishDate.IsZero()},
|
|
Contributors: metadata.Contributors,
|
|
ContributorsSearch: contributorsSearch,
|
|
Tags: metadata.Tags,
|
|
TagsSearch: tagsSearch,
|
|
AddedByAdminID: s.adminID,
|
|
CreatedAt: pgtype.Timestamptz{Time: fileModTime, Valid: true},
|
|
})
|
|
if err != nil {
|
|
return false, fmt.Errorf("failed to create media item: %v", err)
|
|
}
|
|
|
|
// Update hash information before database storage
|
|
if metadata.FileHashInfo != nil && metadata.FileHashInfo.FileSHA256 != "" {
|
|
_, err = s.db.UpdateMediaItemIdentifiers(ctx, database.UpdateMediaItemIdentifiersParams{
|
|
ID: createdItem.ID,
|
|
FileSha256: pgtype.Text{String: metadata.FileHashInfo.FileSHA256, Valid: true},
|
|
OpfIdentifier: pgtype.Text{String: metadata.FileHashInfo.OPFIdentifier, Valid: metadata.FileHashInfo.OPFIdentifier != ""},
|
|
OpfUuid: pgtype.Text{String: metadata.FileHashInfo.OPFUUID, Valid: metadata.FileHashInfo.OPFUUID != ""},
|
|
HashConfidence: pgtype.Text{String: metadata.FileHashInfo.HashConfidence, Valid: true},
|
|
})
|
|
if err != nil {
|
|
fmt.Printf("Warning: failed to update hash identifiers for %s: %v\n", path, err)
|
|
}
|
|
}
|
|
|
|
// Store format information in the database
|
|
for _, format := range metadata.FileFormats {
|
|
_, err = s.db.CreateMediaItemFormat(ctx, database.CreateMediaItemFormatParams{
|
|
MediaItemID: createdItem.ID,
|
|
FormatType: format.FormatType,
|
|
FilePath: pgtype.Text{String: s.getRelativePath(format.FilePath), Valid: true},
|
|
FileSha256: pgtype.Text{String: format.FileSHA256, Valid: true},
|
|
FileSizeBytes: pgtype.Int8{Int64: format.FileSizeBytes, Valid: true},
|
|
MimeType: pgtype.Text{String: format.MimeType, Valid: true},
|
|
})
|
|
if err != nil {
|
|
fmt.Printf("Warning: failed to create format entry for %s: %v\n", path, err)
|
|
}
|
|
}
|
|
|
|
s.newItems++
|
|
return true, nil
|
|
}
|
|
|
|
func (s *MediaScanner) extractMetadata(path string) (*MediaMetadata, error) {
|
|
ext := strings.ToLower(filepath.Ext(path))
|
|
|
|
switch ext {
|
|
case ".epub":
|
|
metadata, err := s.extractEPUBMetadata(path)
|
|
if err != nil {
|
|
return metadata, err
|
|
}
|
|
// Try to extract embedded cover
|
|
coverPath, err := s.extractEPUBCover(path)
|
|
if err != nil {
|
|
fmt.Printf("Warning: failed to extract EPUB cover from %s: %v\n", path, err)
|
|
} else if coverPath != "" {
|
|
metadata.CoverPath = s.getRelativePath(coverPath)
|
|
}
|
|
// If no embedded cover, try sidecar
|
|
if metadata.CoverPath == "" {
|
|
sidecarCover := findSidecarCover(path)
|
|
if sidecarCover != "" {
|
|
metadata.CoverPath = s.getRelativePath(sidecarCover)
|
|
}
|
|
}
|
|
return metadata, nil
|
|
case ".pdf":
|
|
return s.extractPDFMetadata(path)
|
|
default:
|
|
// For other formats, return basic metadata
|
|
return &MediaMetadata{
|
|
Title: strings.TrimSuffix(filepath.Base(path), ext),
|
|
}, nil
|
|
}
|
|
}
|
|
|
|
func (s *MediaScanner) extractEPUBMetadata(path string) (*MediaMetadata, error) {
|
|
book, err := epub.ReadBook(path)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to open EPUB: %v", err)
|
|
}
|
|
|
|
metadata := &MediaMetadata{}
|
|
|
|
// Title
|
|
if title, err := book.Title(); err == nil && title != "" {
|
|
metadata.Title = title
|
|
}
|
|
|
|
// Author
|
|
if authors, err := book.MetadataByKey("creator"); err == nil && len(authors) > 0 {
|
|
metadata.Author = authors[0]
|
|
}
|
|
|
|
// Description
|
|
if descriptions, err := book.MetadataByKey("description"); err == nil && len(descriptions) > 0 {
|
|
metadata.Description = descriptions[0]
|
|
}
|
|
|
|
// Publisher
|
|
if publishers, err := book.MetadataByKey("publisher"); err == nil && len(publishers) > 0 {
|
|
metadata.Publisher = publishers[0]
|
|
}
|
|
|
|
// Series and series number (Calibre specific metadata)
|
|
if series, err := book.MetadataByKey("calibre:series"); err == nil && len(series) > 0 {
|
|
metadata.Series = series[0]
|
|
}
|
|
if seriesIndex, err := book.MetadataByKey("calibre:series_index"); err == nil && len(seriesIndex) > 0 {
|
|
if index, err := strconv.ParseFloat(seriesIndex[0], 32); err == nil {
|
|
metadata.SeriesNumber = int32(index)
|
|
}
|
|
}
|
|
|
|
// Publish date
|
|
if dates, err := book.MetadataByKey("date"); err == nil && len(dates) > 0 {
|
|
if date, err := time.Parse("2006-01-02", dates[0]); err == nil {
|
|
metadata.PublishDate = date
|
|
} else {
|
|
// Try alternative date formats
|
|
if date, err := time.Parse("2006", dates[0]); err == nil {
|
|
metadata.PublishDate = date
|
|
}
|
|
}
|
|
}
|
|
|
|
// Contributors
|
|
if contributors, err := book.MetadataByKey("contributor"); err == nil && len(contributors) > 0 {
|
|
// Normalize contributors for display
|
|
metadata.Contributors = utils.NormalizeContributors(contributors)
|
|
}
|
|
|
|
// ISBN
|
|
if isbns, err := book.MetadataByKey("identifier"); err == nil && len(isbns) > 0 {
|
|
for _, isbn := range isbns {
|
|
if strings.Contains(strings.ToLower(isbn), "isbn") {
|
|
// Extract ISBN number from identifier like "isbn:978-3-16-148410-0"
|
|
isbnParts := strings.SplitN(isbn, ":", 2)
|
|
if len(isbnParts) == 2 {
|
|
metadata.ISBN = isbnParts[1]
|
|
break
|
|
}
|
|
}
|
|
if strings.Contains(strings.ToLower(isbn), "asin") {
|
|
// Extract ASIN from identifier like "asin:B08XXXXX"
|
|
asinParts := strings.SplitN(isbn, ":", 2)
|
|
if len(asinParts) == 2 {
|
|
metadata.ASIN = asinParts[1]
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// Tags
|
|
if tags, err := book.MetadataByKey("subject"); err == nil && len(tags) > 0 {
|
|
// Normalize tags for display
|
|
metadata.Tags = utils.NormalizeTags(tags)
|
|
}
|
|
|
|
return metadata, nil
|
|
}
|
|
|
|
// extractEPUBCover extracts the cover image from an EPUB file.
|
|
// It looks for:
|
|
// 1. An item with properties="cover-image" in the manifest
|
|
// 2. A meta tag with name="cover" pointing to an image
|
|
// 3. Common cover image paths like cover.jpg, cover.jpeg, cover.png
|
|
// Returns the path to the saved cover image, or empty string if no cover found.
|
|
func (s *MediaScanner) extractEPUBCover(epubPath string) (string, error) {
|
|
// Open the EPUB as a zip file to extract images
|
|
r, err := zip.OpenReader(epubPath)
|
|
if err != nil {
|
|
return "", fmt.Errorf("failed to open EPUB as zip: %v", err)
|
|
}
|
|
defer r.Close()
|
|
|
|
// Try to find cover image from OPF metadata
|
|
coverImageName := ""
|
|
|
|
// Attempt to read the OPF file to find cover reference
|
|
// First, find container.xml to locate the OPF
|
|
var opfPath string
|
|
for _, f := range r.File {
|
|
if f.Name == "META-INF/container.xml" {
|
|
rc, err := f.Open()
|
|
if err != nil {
|
|
continue
|
|
}
|
|
content, err := io.ReadAll(rc)
|
|
rc.Close()
|
|
if err != nil {
|
|
continue
|
|
}
|
|
// Parse container.xml to find OPF path
|
|
// Simple string search since we just need the path
|
|
opfStart := bytes.Index(content, []byte("<rootfile "))
|
|
if opfStart == -1 {
|
|
continue
|
|
}
|
|
opfStartAttr := bytes.Index(content[opfStart:], []byte("full-path="))
|
|
if opfStartAttr == -1 {
|
|
continue
|
|
}
|
|
opfStartAttr += len("full-path=")
|
|
quote := content[opfStart+opfStartAttr]
|
|
opfStartQuote := opfStart + opfStartAttr + 1
|
|
opfEndQuote := bytes.Index(content[opfStartQuote:], []byte{byte(quote)})
|
|
if opfEndQuote == -1 {
|
|
continue
|
|
}
|
|
opfPath = string(content[opfStartQuote : opfStartQuote+opfEndQuote])
|
|
break
|
|
}
|
|
}
|
|
|
|
if opfPath == "" {
|
|
// No OPF found, try common cover image paths
|
|
coverImageName = findCoverImageInZip(r.File)
|
|
} else {
|
|
// Read OPF to find cover reference
|
|
opfContent, err := readFileFromZip(r.File, opfPath)
|
|
if err != nil {
|
|
// Fall back to common paths
|
|
coverImageName = findCoverImageInZip(r.File)
|
|
} else {
|
|
coverImageName = findCoverInOPF(opfContent, r.File, opfPath)
|
|
}
|
|
}
|
|
|
|
if coverImageName == "" {
|
|
return "", nil // No cover found
|
|
}
|
|
|
|
// Extract the cover image
|
|
coverImage, err := extractImageFromZip(r.File, coverImageName, opfPath)
|
|
if err != nil {
|
|
return "", fmt.Errorf("failed to extract cover image: %v", err)
|
|
}
|
|
if len(coverImage) == 0 {
|
|
return "", nil
|
|
}
|
|
|
|
// Save cover to disk (same pattern as comics: {book_path}.cover.jpg)
|
|
coverPath := epubPath + ".cover.jpg"
|
|
if err := os.WriteFile(coverPath, coverImage, 0644); err != nil {
|
|
return "", fmt.Errorf("failed to write cover file: %v", err)
|
|
}
|
|
|
|
return coverPath, nil
|
|
}
|
|
|
|
// findCoverImageInZip searches for common cover image filenames in the zip
|
|
func findCoverImageInZip(files []*zip.File) string {
|
|
coverNames := []string{"cover.jpg", "cover.jpeg", "cover.png", "cover.webp",
|
|
"Cover.jpg", "Cover.jpeg", "Cover.png", "Cover.webp",
|
|
"images/cover.jpg", "Images/cover.jpg", "OEBPS/images/cover.jpg"}
|
|
|
|
for _, name := range coverNames {
|
|
for _, f := range files {
|
|
if strings.EqualFold(f.Name, name) {
|
|
return f.Name
|
|
}
|
|
}
|
|
}
|
|
return ""
|
|
}
|
|
|
|
// findCoverInOPF parses OPF content to find cover image reference
|
|
func findCoverInOPF(opfContent []byte, files []*zip.File, opfDir string) string {
|
|
contentStr := string(opfContent)
|
|
|
|
// Look for item with properties="cover-image"
|
|
coverImageRE := regexp.MustCompile(`<item[^>]*properties="[^"]*cover-image[^"]*"[^>]*id="([^"]+)"`)
|
|
matches := coverImageRE.FindStringSubmatch(contentStr)
|
|
if len(matches) > 1 {
|
|
coverID := matches[1]
|
|
// Find the href for this ID
|
|
hrefRE := regexp.MustCompile(fmt.Sprintf(`<item[^>]*id="%s"[^>]*href="([^"]+)"`, coverID))
|
|
hrefMatches := hrefRE.FindStringSubmatch(contentStr)
|
|
if len(hrefMatches) > 1 {
|
|
return resolveOPFPath(opfDir, hrefMatches[1])
|
|
}
|
|
}
|
|
|
|
// Look for meta name="cover"
|
|
metaCoverRE := regexp.MustCompile(`<meta[^>]*name="cover"[^>]*content="([^"]+)"`)
|
|
metaMatches := metaCoverRE.FindStringSubmatch(contentStr)
|
|
if len(metaMatches) > 1 {
|
|
coverContent := metaMatches[1]
|
|
// Could be "image-id" format
|
|
if strings.HasPrefix(coverContent, "image-") {
|
|
coverID := strings.TrimPrefix(coverContent, "image-")
|
|
hrefRE := regexp.MustCompile(fmt.Sprintf(`<item[^>]*id="%s"[^>]*href="([^"]+)"`, coverID))
|
|
hrefMatches := hrefRE.FindStringSubmatch(contentStr)
|
|
if len(hrefMatches) > 1 {
|
|
return resolveOPFPath(opfDir, hrefMatches[1])
|
|
}
|
|
}
|
|
}
|
|
|
|
// Fall back to searching common paths
|
|
return findCoverImageInZip(files)
|
|
}
|
|
|
|
// resolveOPFPath resolves a relative path against the OPF directory
|
|
func resolveOPFPath(opfDir, href string) string {
|
|
if opfDir == "" {
|
|
return href
|
|
}
|
|
// Handle ../ in href
|
|
if strings.HasPrefix(href, "../") {
|
|
// Simple case: just use the href as-is for now
|
|
return href
|
|
}
|
|
// Join the directory with the href
|
|
return filepath.Join(filepath.Dir(opfDir), href)
|
|
}
|
|
|
|
// readFileFromZip reads a file from the zip by name
|
|
func readFileFromZip(files []*zip.File, name string) ([]byte, error) {
|
|
// Normalize the name for comparison
|
|
name = filepath.ToSlash(name)
|
|
for _, f := range files {
|
|
fName := filepath.ToSlash(f.Name)
|
|
if fName == name {
|
|
rc, err := f.Open()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer rc.Close()
|
|
return io.ReadAll(rc)
|
|
}
|
|
}
|
|
return nil, fmt.Errorf("file not found: %s", name)
|
|
}
|
|
|
|
// extractImageFromZip extracts an image file and returns its contents
|
|
func extractImageFromZip(files []*zip.File, imagePath, opfDir string) ([]byte, error) {
|
|
// Try direct match first
|
|
for _, f := range files {
|
|
if strings.EqualFold(f.Name, imagePath) {
|
|
rc, err := f.Open()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer rc.Close()
|
|
return io.ReadAll(rc)
|
|
}
|
|
}
|
|
|
|
// Try resolved path
|
|
resolvedPath := resolveOPFPath(opfDir, imagePath)
|
|
for _, f := range files {
|
|
if strings.EqualFold(f.Name, resolvedPath) {
|
|
rc, err := f.Open()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer rc.Close()
|
|
return io.ReadAll(rc)
|
|
}
|
|
}
|
|
|
|
return nil, fmt.Errorf("image not found: %s", imagePath)
|
|
}
|
|
|
|
// findSidecarCover looks for cover images in the same directory as the media file.
|
|
// It checks for common cover filename patterns in priority order:
|
|
// 1. cover.jpg, cover.jpeg, cover.png, cover.webp
|
|
// 2. folder.jpg, folder.jpeg, folder.png, folder.webp
|
|
// 3. {basename}.jpg, {basename}.jpeg, etc. (same name as media file)
|
|
// 4. .folder.jpg (hidden file)
|
|
// Returns the full path to the cover file, or empty string if not found.
|
|
func findSidecarCover(mediaPath string) string {
|
|
dir := filepath.Dir(mediaPath)
|
|
baseName := strings.TrimSuffix(filepath.Base(mediaPath), filepath.Ext(mediaPath))
|
|
|
|
// Priority order for cover filenames
|
|
coverPatterns := []string{
|
|
"cover.jpg",
|
|
"cover.jpeg",
|
|
"cover.png",
|
|
"cover.webp",
|
|
"folder.jpg",
|
|
"folder.jpeg",
|
|
"folder.png",
|
|
"folder.webp",
|
|
".folder.jpg",
|
|
".folder.jpeg",
|
|
".folder.png",
|
|
}
|
|
|
|
// First, check exact match cover/folder names
|
|
for _, coverName := range coverPatterns {
|
|
coverPath := filepath.Join(dir, coverName)
|
|
if _, err := os.Stat(coverPath); err == nil {
|
|
return coverPath
|
|
}
|
|
}
|
|
|
|
// Second, check for {basename}.{ext} pattern
|
|
extensions := []string{".jpg", ".jpeg", ".png", ".webp"}
|
|
for _, ext := range extensions {
|
|
coverPath := filepath.Join(dir, baseName+ext)
|
|
if _, err := os.Stat(coverPath); err == nil {
|
|
return coverPath
|
|
}
|
|
// Also check uppercase extension
|
|
coverPathUpper := filepath.Join(dir, baseName+strings.ToUpper(ext))
|
|
if _, err := os.Stat(coverPathUpper); err == nil {
|
|
return coverPathUpper
|
|
}
|
|
}
|
|
|
|
return ""
|
|
}
|
|
|
|
func (s *MediaScanner) extractPDFMetadata(path string) (*MediaMetadata, error) {
|
|
metadata := &MediaMetadata{}
|
|
|
|
// Open PDF file for reading metadata
|
|
f, err := os.Open(path)
|
|
if err != nil {
|
|
fmt.Printf("Warning: failed to open PDF file %s: %v\n", path, err)
|
|
metadata.Title = strings.TrimSuffix(filepath.Base(path), ".pdf")
|
|
return metadata, nil
|
|
}
|
|
defer f.Close()
|
|
|
|
// Use pdfcpu API to read PDF metadata
|
|
// Configuration: nil = default (lenient mode)
|
|
pdfInfo, err := pdfcpuapi.PDFInfo(f, filepath.Base(path), nil, nil)
|
|
if err != nil {
|
|
fmt.Printf("Warning: failed to read PDF info from %s: %v\n", path, err)
|
|
// Fall back to filename as title
|
|
metadata.Title = strings.TrimSuffix(filepath.Base(path), ".pdf")
|
|
return metadata, nil
|
|
}
|
|
|
|
// Extract title
|
|
if pdfInfo.Title != "" {
|
|
metadata.Title = pdfInfo.Title
|
|
} else {
|
|
metadata.Title = strings.TrimSuffix(filepath.Base(path), ".pdf")
|
|
}
|
|
|
|
// Extract author
|
|
if pdfInfo.Author != "" {
|
|
metadata.Author = pdfInfo.Author
|
|
}
|
|
|
|
// Extract subject (use as description)
|
|
if pdfInfo.Subject != "" {
|
|
metadata.Description = pdfInfo.Subject
|
|
}
|
|
|
|
// Extract creator (use as author fallback)
|
|
if pdfInfo.Creator != "" && metadata.Author == "" {
|
|
metadata.Author = pdfInfo.Creator
|
|
}
|
|
|
|
// Extract producer (use as publisher)
|
|
if pdfInfo.Producer != "" {
|
|
metadata.Publisher = pdfInfo.Producer
|
|
}
|
|
|
|
// Try to extract cover image
|
|
coverPath, err := s.extractPDFCover(path)
|
|
if err != nil {
|
|
fmt.Printf("Warning: failed to extract PDF cover from %s: %v\n", path, err)
|
|
} else if coverPath != "" {
|
|
metadata.CoverPath = s.getRelativePath(coverPath)
|
|
}
|
|
|
|
// If no embedded cover, try sidecar
|
|
if metadata.CoverPath == "" {
|
|
sidecarCover := findSidecarCover(path)
|
|
if sidecarCover != "" {
|
|
metadata.CoverPath = s.getRelativePath(sidecarCover)
|
|
}
|
|
}
|
|
|
|
return metadata, nil
|
|
}
|
|
|
|
// extractPDFCover extracts a cover image from a PDF file.
|
|
// It uses pdfcpu API to extract images from the first page.
|
|
// Returns the path to the saved cover, or empty string if no cover found.
|
|
func (s *MediaScanner) extractPDFCover(pdfPath string) (string, error) {
|
|
// Create a temporary directory for extracted images
|
|
tmpDir, err := os.MkdirTemp("", "pdf-cover-")
|
|
if err != nil {
|
|
return "", fmt.Errorf("failed to create temp dir: %v", err)
|
|
}
|
|
defer os.RemoveAll(tmpDir)
|
|
|
|
// Use pdfcpu API to extract images from first page
|
|
// ExtractImagesFile(inFile, outDir string, selectedPages []string, conf *model.Configuration) error
|
|
err = pdfcpuapi.ExtractImagesFile(pdfPath, tmpDir, []string{"1"}, nil)
|
|
if err != nil {
|
|
// No images found or extraction failed - this is OK, just return empty
|
|
return "", nil
|
|
}
|
|
|
|
// Check for extracted images in the temp directory
|
|
entries, err := os.ReadDir(tmpDir)
|
|
if err != nil || len(entries) == 0 {
|
|
return "", nil
|
|
}
|
|
|
|
// Find the largest image (likely the cover)
|
|
var largestImage string
|
|
var largestSize int64
|
|
|
|
for _, entry := range entries {
|
|
if entry.IsDir() {
|
|
continue
|
|
}
|
|
info, err := entry.Info()
|
|
if err != nil {
|
|
continue
|
|
}
|
|
// Skip very small files (likely thumbnails or icons)
|
|
if info.Size() < 1000 {
|
|
continue
|
|
}
|
|
if info.Size() > largestSize {
|
|
largestImage = filepath.Join(tmpDir, entry.Name())
|
|
largestSize = info.Size()
|
|
}
|
|
}
|
|
|
|
if largestImage == "" {
|
|
return "", nil
|
|
}
|
|
|
|
// Read the image
|
|
imageData, err := os.ReadFile(largestImage)
|
|
if err != nil || len(imageData) == 0 {
|
|
return "", nil
|
|
}
|
|
|
|
// Save cover to disk (same pattern as comics: {pdf_path}.cover.jpg)
|
|
coverPath := pdfPath + ".cover.jpg"
|
|
if err := os.WriteFile(coverPath, imageData, 0644); err != nil {
|
|
return "", fmt.Errorf("failed to write cover file: %v", err)
|
|
}
|
|
|
|
return coverPath, nil
|
|
}
|
|
|
|
// ComicInfo represents metadata from ComicInfo.xml
|
|
type ComicInfo struct {
|
|
XMLName xml.Name `xml:"ComicInfo"`
|
|
Title string `xml:"Title"`
|
|
Series string `xml:"Series"`
|
|
Number int `xml:"Number"`
|
|
Volume int `xml:"Volume"`
|
|
Publisher string `xml:"Publisher"`
|
|
Year int `xml:"Year"`
|
|
Month int `xml:"Month"`
|
|
Day int `xml:"Day"`
|
|
Writer string `xml:"Writer"`
|
|
Penciller string `xml:"Penciller"`
|
|
Inker string `xml:"Inker"`
|
|
Colorist string `xml:"Colorist"`
|
|
Letterer string `xml:"Letterer"`
|
|
CoverArtist string `xml:"CoverArtist"`
|
|
Genre string `xml:"Genre"`
|
|
Tags string `xml:"Tags"`
|
|
Web string `xml:"Web"`
|
|
Notes string `xml:"Notes"`
|
|
}
|
|
|
|
// extractComicMetadata extracts metadata from comic archive (supports .cbz, .cbr, .cb7, .cbt)
|
|
func extractComicMetadata(filePath string) (*ComicInfo, []byte, error) {
|
|
ext := strings.ToLower(filepath.Ext(filePath))
|
|
|
|
switch ext {
|
|
case ".cbz":
|
|
return extractZipMetadata(filePath)
|
|
case ".cbr":
|
|
return extractRarMetadata(filePath)
|
|
case ".cb7":
|
|
return extract7ZipMetadata(filePath)
|
|
case ".cbt":
|
|
return extractTarMetadata(filePath)
|
|
default:
|
|
return nil, nil, fmt.Errorf("unsupported comic format: %s", ext)
|
|
}
|
|
}
|
|
|
|
// archiveFile represents a file in an archive for uniform handling
|
|
type archiveFile interface {
|
|
Name() string
|
|
Open() (io.ReadCloser, error)
|
|
}
|
|
|
|
// extractMetadataFromArchive extracts ComicInfo.xml and cover image from any archive format
|
|
func extractMetadataFromArchive(files []archiveFile) (*ComicInfo, []byte, error) {
|
|
var comicInfo *ComicInfo
|
|
var coverImage []byte
|
|
|
|
for _, f := range files {
|
|
if f.Name() == "ComicInfo.xml" {
|
|
rc, err := f.Open()
|
|
if err != nil {
|
|
return nil, nil, fmt.Errorf("failed to open ComicInfo.xml: %w", err)
|
|
}
|
|
|
|
data, err := io.ReadAll(rc)
|
|
rc.Close()
|
|
if err != nil {
|
|
return nil, nil, fmt.Errorf("failed to read ComicInfo.xml: %w", err)
|
|
}
|
|
|
|
comicInfo = &ComicInfo{}
|
|
if err := xml.Unmarshal(data, comicInfo); err != nil {
|
|
return nil, nil, fmt.Errorf("failed to parse ComicInfo.xml: %w", err)
|
|
}
|
|
}
|
|
|
|
if coverImage == nil && isImageFile(f.Name()) {
|
|
if !strings.Contains(filepath.Dir(f.Name()), string(filepath.Separator)) ||
|
|
filepath.Dir(f.Name()) == "." {
|
|
rc, err := f.Open()
|
|
if err != nil {
|
|
continue
|
|
}
|
|
|
|
coverImage, err = io.ReadAll(rc)
|
|
rc.Close()
|
|
if err == nil {
|
|
_, _, err = image.Decode(bytes.NewReader(coverImage))
|
|
if err != nil {
|
|
coverImage = nil
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
if comicInfo == nil {
|
|
comicInfo = &ComicInfo{}
|
|
}
|
|
|
|
return comicInfo, coverImage, nil
|
|
}
|
|
|
|
// extractZipMetadata extracts metadata from ZIP archives (.cbz)
|
|
func extractZipMetadata(filePath string) (*ComicInfo, []byte, error) {
|
|
r, err := zip.OpenReader(filePath)
|
|
if err != nil {
|
|
return nil, nil, fmt.Errorf("failed to open ZIP archive: %w", err)
|
|
}
|
|
defer r.Close()
|
|
|
|
files := make([]archiveFile, 0, len(r.File))
|
|
for _, f := range r.File {
|
|
files = append(files, &zipFileAdapter{f})
|
|
}
|
|
|
|
comicInfo, coverImage, err := extractMetadataFromArchive(files)
|
|
if err != nil {
|
|
return nil, nil, err
|
|
}
|
|
|
|
if comicInfo.Title == "" {
|
|
basename := filepath.Base(filePath)
|
|
comicInfo.Title = strings.TrimSuffix(basename, filepath.Ext(basename))
|
|
}
|
|
|
|
return comicInfo, coverImage, nil
|
|
}
|
|
|
|
// zipFileAdapter adapts zip.File to archiveFile interface
|
|
type zipFileAdapter struct {
|
|
*zip.File
|
|
}
|
|
|
|
func (z *zipFileAdapter) Name() string {
|
|
return z.File.Name
|
|
}
|
|
|
|
func (z *zipFileAdapter) Open() (io.ReadCloser, error) {
|
|
return z.File.Open()
|
|
}
|
|
|
|
// extractRarMetadata extracts metadata from RAR archives (.cbr)
|
|
func extractRarMetadata(filePath string) (*ComicInfo, []byte, error) {
|
|
r, err := rardecode.OpenReader(filePath, "")
|
|
if err != nil {
|
|
return nil, nil, fmt.Errorf("failed to open RAR archive: %w", err)
|
|
}
|
|
defer r.Close()
|
|
|
|
files := make([]archiveFile, 0, 100)
|
|
for {
|
|
header, err := r.Next()
|
|
if err == io.EOF {
|
|
break
|
|
}
|
|
if err != nil {
|
|
return nil, nil, fmt.Errorf("failed to read RAR entry: %w", err)
|
|
}
|
|
|
|
data, err := io.ReadAll(r)
|
|
if err != nil {
|
|
continue
|
|
}
|
|
files = append(files, &rarFileAdapter{
|
|
name: header.Name,
|
|
data: data,
|
|
})
|
|
}
|
|
|
|
comicInfo, coverImage, err := extractMetadataFromArchive(files)
|
|
if err != nil {
|
|
return nil, nil, err
|
|
}
|
|
|
|
if comicInfo.Title == "" {
|
|
basename := filepath.Base(filePath)
|
|
comicInfo.Title = strings.TrimSuffix(basename, filepath.Ext(basename))
|
|
}
|
|
|
|
return comicInfo, coverImage, nil
|
|
}
|
|
|
|
// rarFileAdapter stores RAR file data in memory
|
|
type rarFileAdapter struct {
|
|
name string
|
|
data []byte
|
|
}
|
|
|
|
func (r *rarFileAdapter) Name() string {
|
|
return r.name
|
|
}
|
|
|
|
func (r *rarFileAdapter) Open() (io.ReadCloser, error) {
|
|
return io.NopCloser(bytes.NewReader(r.data)), nil
|
|
}
|
|
|
|
// extract7ZipMetadata extracts metadata from 7-Zip archives (.cb7)
|
|
func extract7ZipMetadata(filePath string) (*ComicInfo, []byte, error) {
|
|
r, err := sevenzip.OpenReader(filePath)
|
|
if err != nil {
|
|
return nil, nil, fmt.Errorf("failed to open 7-Zip archive: %w", err)
|
|
}
|
|
defer r.Close()
|
|
|
|
files := make([]archiveFile, 0, len(r.File))
|
|
for _, f := range r.File {
|
|
files = append(files, &sevenZipFileAdapter{file: f})
|
|
}
|
|
|
|
comicInfo, coverImage, err := extractMetadataFromArchive(files)
|
|
if err != nil {
|
|
return nil, nil, err
|
|
}
|
|
|
|
if comicInfo.Title == "" {
|
|
basename := filepath.Base(filePath)
|
|
comicInfo.Title = strings.TrimSuffix(basename, filepath.Ext(basename))
|
|
}
|
|
|
|
return comicInfo, coverImage, nil
|
|
}
|
|
|
|
// sevenZipFileAdapter adapts sevenzip.File to archiveFile interface
|
|
type sevenZipFileAdapter struct {
|
|
file *sevenzip.File
|
|
}
|
|
|
|
func (s *sevenZipFileAdapter) Name() string {
|
|
return s.file.Name
|
|
}
|
|
|
|
func (s *sevenZipFileAdapter) Open() (io.ReadCloser, error) {
|
|
return s.file.Open()
|
|
}
|
|
|
|
// extractTarMetadata extracts metadata from TAR archives (.cbt)
|
|
func extractTarMetadata(filePath string) (*ComicInfo, []byte, error) {
|
|
f, err := os.Open(filePath)
|
|
if err != nil {
|
|
return nil, nil, fmt.Errorf("failed to open TAR archive: %w", err)
|
|
}
|
|
defer f.Close()
|
|
|
|
var tarReader *tar.Reader
|
|
if strings.HasSuffix(strings.ToLower(filePath), ".tar.gz") ||
|
|
strings.HasSuffix(strings.ToLower(filePath), ".tgz") {
|
|
gzReader, err := gzip.NewReader(f)
|
|
if err != nil {
|
|
return nil, nil, fmt.Errorf("failed to create gzip reader: %w", err)
|
|
}
|
|
defer gzReader.Close()
|
|
tarReader = tar.NewReader(gzReader)
|
|
} else if strings.HasSuffix(strings.ToLower(filePath), ".tar.bz2") ||
|
|
strings.HasSuffix(strings.ToLower(filePath), ".tbz2") {
|
|
bz2Reader := bzip2.NewReader(f)
|
|
tarReader = tar.NewReader(bz2Reader)
|
|
} else {
|
|
tarReader = tar.NewReader(f)
|
|
}
|
|
|
|
files := make([]archiveFile, 0, 100)
|
|
for {
|
|
header, err := tarReader.Next()
|
|
if err == io.EOF {
|
|
break
|
|
}
|
|
if err != nil {
|
|
return nil, nil, fmt.Errorf("failed to read TAR entry: %w", err)
|
|
}
|
|
|
|
if header.Typeflag == tar.TypeReg {
|
|
data, err := io.ReadAll(tarReader)
|
|
if err != nil {
|
|
continue
|
|
}
|
|
files = append(files, &tarFileAdapter{
|
|
name: header.Name,
|
|
data: data,
|
|
})
|
|
}
|
|
}
|
|
|
|
comicInfo, coverImage, err := extractMetadataFromArchive(files)
|
|
if err != nil {
|
|
return nil, nil, err
|
|
}
|
|
|
|
if comicInfo.Title == "" {
|
|
basename := filepath.Base(filePath)
|
|
comicInfo.Title = strings.TrimSuffix(basename, filepath.Ext(basename))
|
|
}
|
|
|
|
return comicInfo, coverImage, nil
|
|
}
|
|
|
|
// tarFileAdapter stores TAR file data in memory
|
|
type tarFileAdapter struct {
|
|
name string
|
|
data []byte
|
|
}
|
|
|
|
func (t *tarFileAdapter) Name() string {
|
|
return t.name
|
|
}
|
|
|
|
func (t *tarFileAdapter) Open() (io.ReadCloser, error) {
|
|
return io.NopCloser(bytes.NewReader(t.data)), nil
|
|
}
|
|
|
|
// isImageFile checks if a file is an image based on extension
|
|
func isImageFile(filename string) bool {
|
|
ext := strings.ToLower(filepath.Ext(filename))
|
|
return ext == ".jpg" || ext == ".jpeg" || ext == ".png" || ext == ".gif"
|
|
}
|
|
|
|
func (s *MediaScanner) updateMediaItem(ctx context.Context, mediaItemID pgtype.UUID, path string, info os.FileInfo) error {
|
|
// Re-extract metadata for the update
|
|
metadata, err := s.extractMetadata(path)
|
|
if err != nil {
|
|
fmt.Printf("Warning: failed to extract metadata for force rescan %s: %v\n", path, err)
|
|
metadata = &MediaMetadata{}
|
|
}
|
|
|
|
// Normalize metadata fields
|
|
metadata.Contributors = utils.NormalizeContributors(metadata.Contributors)
|
|
metadata.Tags = utils.NormalizeTags(metadata.Tags)
|
|
contributorsSearch := utils.NormalizeContributorsSearch(metadata.Contributors)
|
|
tagsSearch := utils.NormalizeTagsSearch(metadata.Tags)
|
|
|
|
// Call the database update - only update fields available in MediaMetadata
|
|
_, err = s.db.UpdateMediaItem(ctx, database.UpdateMediaItemParams{
|
|
ID: mediaItemID,
|
|
Title: metadata.Title,
|
|
Author: pgtype.Text{String: metadata.Author, Valid: metadata.Author != ""},
|
|
Isbn: pgtype.Text{String: utils.NormalizeISBNSafe(metadata.ISBN), Valid: metadata.ISBN != ""},
|
|
Description: pgtype.Text{String: metadata.Description, Valid: metadata.Description != ""},
|
|
CoverImagePath: pgtype.Text{String: s.getRelativePath(metadata.CoverPath), Valid: metadata.CoverPath != ""},
|
|
Series: pgtype.Text{String: metadata.Series, Valid: metadata.Series != ""},
|
|
SeriesNumber: pgtype.Int4{Int32: metadata.SeriesNumber, Valid: metadata.SeriesNumber > 0},
|
|
Tags: metadata.Tags,
|
|
TagsSearch: tagsSearch,
|
|
Asin: pgtype.Text{String: metadata.ASIN, Valid: metadata.ASIN != ""},
|
|
DatePublished: pgtype.Date{Time: metadata.PublishDate, Valid: !metadata.PublishDate.IsZero()},
|
|
Publisher: pgtype.Text{String: metadata.Publisher, Valid: metadata.Publisher != ""},
|
|
Contributors: metadata.Contributors,
|
|
ContributorsSearch: contributorsSearch,
|
|
})
|
|
return err
|
|
}
|
|
|
|
func (s *MediaScanner) getMediaItemByFilePath(ctx context.Context, filePath string, libraryID pgtype.UUID) (database.MediaItems, error) {
|
|
return s.db.GetMediaItemByFilePath(ctx, database.GetMediaItemByFilePathParams{
|
|
FilePath: s.getRelativePath(filePath),
|
|
LibraryID: libraryID,
|
|
})
|
|
}
|
|
|
|
func (s *MediaScanner) getMimeType(path string) string {
|
|
ext := strings.ToLower(filepath.Ext(path))
|
|
if mime, ok := MimeTypes[ext]; ok {
|
|
return mime
|
|
}
|
|
return "application/octet-stream"
|
|
}
|
|
|
|
func (s *MediaScanner) WatchChanges(ctx context.Context) error {
|
|
// Prevent duplicate calls
|
|
if !s.watching.CompareAndSwap(false, true) {
|
|
return fmt.Errorf("already watching")
|
|
}
|
|
|
|
// Reset flag when context is cancelled
|
|
go func() {
|
|
<-ctx.Done()
|
|
s.watching.Store(false)
|
|
}()
|
|
|
|
// Perform initial scan of all root folders
|
|
go s.performInitialScan(ctx)
|
|
|
|
// Start directory processor
|
|
go s.processDirtyDirectories(ctx)
|
|
|
|
// Start polling fallback
|
|
go s.StartPolling(ctx)
|
|
|
|
// Handle fsnotify events - queue them for debouncing
|
|
go func() {
|
|
for {
|
|
select {
|
|
case event, ok := <-s.watcher.Events:
|
|
if !ok {
|
|
return
|
|
}
|
|
|
|
// Handle new directories - add them to the watcher
|
|
if event.Has(fsnotify.Create) {
|
|
if info, err := os.Stat(event.Name); err == nil && info.IsDir() {
|
|
s.watcher.Add(event.Name)
|
|
}
|
|
}
|
|
|
|
// Mark directory dirty for ANY file change
|
|
if event.Has(fsnotify.Create | fsnotify.Write | fsnotify.Remove | fsnotify.Chmod | fsnotify.Rename) {
|
|
s.markDirectoryDirty(filepath.Dir(event.Name))
|
|
}
|
|
|
|
case err, ok := <-s.watcher.Errors:
|
|
if !ok {
|
|
return
|
|
}
|
|
fmt.Printf("Watcher error: %v\n", err)
|
|
|
|
case <-ctx.Done():
|
|
return
|
|
}
|
|
}
|
|
}()
|
|
return nil
|
|
}
|
|
func (s *MediaScanner) handleFileAdd(ctx context.Context, filePath string) {
|
|
fmt.Printf("New/modified media file detected: %s\n", filePath)
|
|
if _, err := s.processMediaFile(ctx, filePath); err != nil {
|
|
fmt.Printf("Error processing modified media file %s: %v\n", filePath, err)
|
|
}
|
|
}
|
|
func (s *MediaScanner) handleFileDelete(ctx context.Context, filePath string) {
|
|
s.logger.LogDelete(fmt.Sprintf("[WATCH-DELETE] File removed from filesystem: %s", filePath))
|
|
// Determine libraryID for this file
|
|
var libraryID pgtype.UUID
|
|
for _, folder := range s.folders {
|
|
if strings.HasPrefix(filePath, folder) {
|
|
lib, err := s.db.GetLibraryByFolder(ctx, folder)
|
|
if err == nil {
|
|
libraryID = lib.LibraryID
|
|
break
|
|
}
|
|
}
|
|
}
|
|
if !libraryID.Valid {
|
|
msg := fmt.Sprintf("[WATCH-DELETE] WARNING: could not determine library for deleted file: %s", filePath)
|
|
s.logger.LogDelete(msg)
|
|
s.logger.LogError(msg)
|
|
return
|
|
}
|
|
// Look up media item
|
|
existingItem, err := s.db.GetMediaItemByFilePath(ctx, database.GetMediaItemByFilePathParams{
|
|
FilePath: s.getRelativePath(filePath),
|
|
LibraryID: libraryID,
|
|
})
|
|
if err == nil {
|
|
msg := fmt.Sprintf("[WATCH-DELETE] Found media item to delete: ID=%s, Title=%s, Path=%s",
|
|
existingItem.ID, existingItem.Title, existingItem.FilePath)
|
|
s.logger.LogDelete(msg)
|
|
if err := s.db.DeleteMediaItem(ctx, existingItem.ID); err != nil {
|
|
errMsg := fmt.Sprintf("[WATCH-DELETE] ERROR: failed to delete media item %s: %v", existingItem.ID, err)
|
|
s.logger.LogDelete(errMsg)
|
|
s.logger.LogError(errMsg)
|
|
} else {
|
|
s.logger.LogDelete(fmt.Sprintf("[WATCH-DELETE] SUCCESS: deleted media item '%s' (was at %s)",
|
|
existingItem.Title, existingItem.FilePath))
|
|
}
|
|
} else if err != pgx.ErrNoRows {
|
|
errMsg := fmt.Sprintf("[WATCH-DELETE] ERROR: failed to look up media item for %s: %v", filePath, err)
|
|
s.logger.LogDelete(errMsg)
|
|
s.logger.LogError(errMsg)
|
|
} else {
|
|
s.logger.LogDelete(fmt.Sprintf("[WATCH-DELETE] No media item found in database for deleted file: %s", filePath))
|
|
}
|
|
}
|
|
|
|
func (s *MediaScanner) markDirectoryDirty(dirPath string) {
|
|
s.dirtyDirsMu.Lock()
|
|
defer s.dirtyDirsMu.Unlock()
|
|
|
|
// Only mark if within watched folders
|
|
var isWatched bool
|
|
for _, folder := range s.folders {
|
|
if strings.HasPrefix(dirPath, folder) {
|
|
isWatched = true
|
|
break
|
|
}
|
|
}
|
|
if !isWatched {
|
|
return
|
|
}
|
|
|
|
// Smart event merging (Jellyfin approach):
|
|
// 1. If parent dir exists, replace with parent (consolidate)
|
|
// 2. If sibling dirs exist, replace with common parent
|
|
// 3. Otherwise, add this dir
|
|
|
|
// Check if parent directory is already dirty
|
|
parentDir := filepath.Dir(dirPath)
|
|
if parentDir != dirPath { // Not at root
|
|
if _, parentExists := s.dirtyDirs[parentDir]; parentExists {
|
|
// Parent already being watched, reset its timestamp
|
|
s.dirtyDirs[parentDir] = time.Now()
|
|
return
|
|
}
|
|
}
|
|
|
|
// Check if any subdirectories are dirty, replace with parent
|
|
for existingDir := range s.dirtyDirs {
|
|
if strings.HasPrefix(existingDir, dirPath+"/") {
|
|
// This is a subdirectory, replace it with parent
|
|
delete(s.dirtyDirs, existingDir)
|
|
}
|
|
}
|
|
|
|
// NEW: Check for sibling directories and consolidate to parent
|
|
parentDir = filepath.Dir(dirPath)
|
|
for existingDir := range s.dirtyDirs {
|
|
existingParent := filepath.Dir(existingDir)
|
|
if existingParent == parentDir && existingParent != dirPath && existingParent != "." {
|
|
// Found a sibling! Both should be replaced with parent
|
|
delete(s.dirtyDirs, existingDir)
|
|
s.dirtyDirs[parentDir] = time.Now()
|
|
return
|
|
}
|
|
}
|
|
|
|
// Add/update this directory
|
|
s.dirtyDirs[dirPath] = time.Now()
|
|
}
|
|
|
|
func (s *MediaScanner) processDirtyDirectories(ctx context.Context) {
|
|
ticker := time.NewTicker(1 * time.Second)
|
|
defer ticker.Stop()
|
|
|
|
for {
|
|
select {
|
|
case <-ctx.Done():
|
|
return
|
|
|
|
case <-ticker.C:
|
|
s.dirtyDirsMu.Lock()
|
|
|
|
now := time.Now()
|
|
readyDirs := make([]string, 0)
|
|
|
|
// Find directories that haven't been modified in 10 seconds
|
|
// This batches changes together (Audiobookshelf approach)
|
|
for dirPath, lastChange := range s.dirtyDirs {
|
|
if now.Sub(lastChange) >= 10*time.Second {
|
|
readyDirs = append(readyDirs, dirPath)
|
|
delete(s.dirtyDirs, dirPath)
|
|
}
|
|
}
|
|
|
|
s.dirtyDirsMu.Unlock()
|
|
|
|
// Process all ready directories in a batch via job queue
|
|
// Job queue serializes scans - prevents concurrent directory access
|
|
if len(readyDirs) > 0 {
|
|
for _, dirPath := range readyDirs {
|
|
// Create directory scan job with correct params for processDirectoryScanJob()
|
|
job := &Job{
|
|
ID: uuid.New().String(),
|
|
Type: JobTypeDirectoryScan,
|
|
Params: map[string]interface{}{
|
|
"directory": dirPath,
|
|
"db": s.db,
|
|
},
|
|
Status: JobStatusPending,
|
|
}
|
|
|
|
// Enqueue via global worker singleton
|
|
if WorkerInstance != nil {
|
|
WorkerInstance.Enqueue(job)
|
|
fmt.Printf("Enqueued directory scan job: %s\n", dirPath)
|
|
} else {
|
|
fmt.Printf("Warning: Worker not initialized, skipping directory scan: %s\n", dirPath)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// waitForFileStability checks if a file's mtime has stabilized
|
|
// Returns true when file is stable (not being modified)
|
|
// Polls every 3 seconds, times out after 60 seconds
|
|
// Uses atomic.Bool to prevent race conditions with concurrent checks
|
|
func (s *MediaScanner) waitForFileStability(filePath string) bool {
|
|
s.fileStabilityMu.Lock()
|
|
|
|
// Check if already being checked (atomic.Bool prevents race condition)
|
|
tracking, exists := s.fileStability[filePath]
|
|
if exists {
|
|
s.fileStabilityMu.Unlock()
|
|
// Another goroutine is already checking this file
|
|
if tracking.Load() {
|
|
return false // Still being checked
|
|
}
|
|
// Tracking exists but completed, remove stale entry
|
|
s.fileStabilityMu.Lock()
|
|
delete(s.fileStability, filePath)
|
|
}
|
|
|
|
// Start tracking with atomic.Bool set to true (checking in progress)
|
|
trackingFlag := &atomic.Bool{}
|
|
trackingFlag.Store(true)
|
|
s.fileStability[filePath] = trackingFlag
|
|
s.fileStabilityMu.Unlock()
|
|
|
|
// Get initial mtime
|
|
info, err := os.Stat(filePath)
|
|
if err != nil {
|
|
// Clean up tracking entry if file doesn't exist
|
|
s.fileStabilityMu.Lock()
|
|
delete(s.fileStability, filePath)
|
|
s.fileStabilityMu.Unlock()
|
|
return false
|
|
}
|
|
lastMtime := info.ModTime()
|
|
|
|
// Poll every 3 seconds for up to 60 seconds
|
|
timeout := time.After(60 * time.Second)
|
|
ticker := time.NewTicker(3 * time.Second)
|
|
defer ticker.Stop()
|
|
|
|
for {
|
|
select {
|
|
case <-timeout:
|
|
// Timeout - mark as done and clean up
|
|
trackingFlag.Store(false)
|
|
s.fileStabilityMu.Lock()
|
|
delete(s.fileStability, filePath)
|
|
s.fileStabilityMu.Unlock()
|
|
return false // File never stabilized
|
|
|
|
case <-ticker.C:
|
|
info, err := os.Stat(filePath)
|
|
if err != nil {
|
|
// File deleted - mark as done and clean up
|
|
trackingFlag.Store(false)
|
|
s.fileStabilityMu.Lock()
|
|
delete(s.fileStability, filePath)
|
|
s.fileStabilityMu.Unlock()
|
|
return false
|
|
}
|
|
|
|
currentMtime := info.ModTime()
|
|
if currentMtime.Equal(lastMtime) {
|
|
// File is stable! Mark as done and clean up
|
|
trackingFlag.Store(false)
|
|
s.fileStabilityMu.Lock()
|
|
delete(s.fileStability, filePath)
|
|
s.fileStabilityMu.Unlock()
|
|
return true
|
|
}
|
|
|
|
lastMtime = currentMtime
|
|
}
|
|
}
|
|
}
|
|
|
|
func (s *MediaScanner) scanDirectory(ctx context.Context, dirPath string) {
|
|
// Prevent concurrent scans of ANY directory
|
|
// Simple mutex is enough - job queue already serializes by directory
|
|
s.scan_mutex.Lock()
|
|
defer s.scan_mutex.Unlock()
|
|
|
|
// Find library for this directory
|
|
var libraryID pgtype.UUID
|
|
var rootFolder string
|
|
|
|
for _, folder := range s.folders {
|
|
if strings.HasPrefix(dirPath, folder) {
|
|
rootFolder = folder
|
|
if lib, err := s.db.GetLibraryByFolder(ctx, folder); err == nil {
|
|
libraryID = lib.LibraryID
|
|
break
|
|
}
|
|
}
|
|
}
|
|
|
|
// Check if libraryID is valid before proceeding
|
|
if !libraryID.Valid {
|
|
return
|
|
}
|
|
|
|
// Walk directory and process new files
|
|
filepath.WalkDir(dirPath, func(path string, d os.DirEntry, err error) error {
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if d.IsDir() {
|
|
return filepath.SkipDir
|
|
}
|
|
if !s.isScannableFile(path) {
|
|
return nil
|
|
}
|
|
|
|
// Check if file is stable before processing (Audiobookshelf approach)
|
|
if !s.waitForFileStability(path) {
|
|
return nil
|
|
}
|
|
|
|
relPath := strings.TrimPrefix(path, rootFolder+"/")
|
|
_, err = s.db.GetMediaItemByFilePath(ctx, database.GetMediaItemByFilePathParams{
|
|
FilePath: relPath,
|
|
LibraryID: libraryID,
|
|
})
|
|
|
|
if err == pgx.ErrNoRows {
|
|
if _, err := s.processMediaFile(ctx, path); err != nil {
|
|
s.errors++
|
|
} else {
|
|
s.newItems++
|
|
}
|
|
s.totalFiles++
|
|
}
|
|
|
|
return nil
|
|
})
|
|
}
|
|
|
|
// performInitialScan scans all root folders on startup
|
|
// This ensures existing files are detected before watching begins
|
|
func (s *MediaScanner) performInitialScan(ctx context.Context) {
|
|
fmt.Printf("Performing initial scan of root folders...\n")
|
|
|
|
for _, folder := range s.folders {
|
|
// Skip if folder doesn't exist
|
|
if _, err := os.Stat(folder); os.IsNotExist(err) {
|
|
fmt.Printf("Skipping nonexistent folder: %s\n", folder)
|
|
continue
|
|
}
|
|
|
|
// Submit scan job to worker (non-blocking)
|
|
job := &Job{
|
|
ID: uuid.New().String(),
|
|
Type: JobTypeDirectoryScan,
|
|
Params: map[string]interface{}{
|
|
"directory": folder,
|
|
"db": s.db,
|
|
},
|
|
Status: JobStatusPending,
|
|
}
|
|
|
|
if WorkerInstance != nil {
|
|
WorkerInstance.Enqueue(job)
|
|
fmt.Printf("Enqueued initial scan job: %s\n", folder)
|
|
} else {
|
|
fmt.Printf("Warning: Worker not initialized, skipping initial scan: %s\n", folder)
|
|
}
|
|
}
|
|
|
|
fmt.Printf("Initial scan jobs enqueued\n")
|
|
}
|
|
|
|
func (s *MediaScanner) Close() error {
|
|
fmt.Printf("Cleaning up scanner resources...\n")
|
|
|
|
// Stop watching
|
|
if s.watcher != nil {
|
|
s.watcher.Close()
|
|
}
|
|
|
|
// Clean up fileStability map to prevent memory leaks
|
|
s.fileStabilityMu.Lock()
|
|
s.fileStability = make(map[string]*atomic.Bool)
|
|
s.fileStabilityMu.Unlock()
|
|
|
|
// Clear dirty directories
|
|
s.dirtyDirsMu.Lock()
|
|
s.dirtyDirs = make(map[string]time.Time)
|
|
s.dirtyDirsMu.Unlock()
|
|
|
|
// Wait for in-progress scan to complete (with timeout)
|
|
timeout := time.After(5 * time.Second)
|
|
done := make(chan struct{})
|
|
|
|
go func() {
|
|
s.scan_mutex.Lock()
|
|
s.scan_mutex.Unlock()
|
|
close(done)
|
|
}()
|
|
|
|
select {
|
|
case <-done:
|
|
fmt.Printf("Scanner cleanup complete\n")
|
|
case <-timeout:
|
|
fmt.Printf("Timeout waiting for scan to complete\n")
|
|
}
|
|
|
|
return nil
|
|
}
|
|
func (s *MediaScanner) StartPolling(ctx context.Context) {
|
|
interval := s.GetPollInterval()
|
|
if interval <= 0 {
|
|
fmt.Println("Polling fallback disabled (interval = 0")
|
|
return
|
|
}
|
|
fmt.Printf("Polling fallback started with interval: %v\n", interval)
|
|
|
|
for {
|
|
ticker := time.NewTicker(interval)
|
|
defer ticker.Stop()
|
|
|
|
select {
|
|
case <-ctx.Done():
|
|
fmt.Println("Polling fallback stopped")
|
|
return
|
|
case <-ticker.C:
|
|
//Re-read interval each tick for dynamic updates
|
|
interval = s.GetPollInterval()
|
|
fmt.Printf("Running polling fallback sync (interval: %v)...\n", interval)
|
|
if err := s.SyncFilesystemWithDatabase(ctx); err != nil {
|
|
fmt.Printf("Polling sync error: %v\n", err)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
func (s *MediaScanner) SyncFilesystemWithDatabase(ctx context.Context) error {
|
|
fmt.Println("[POLL-SYNC] Starting filesystem sync with database")
|
|
for _, folder := range s.folders {
|
|
lib, err := s.db.GetLibraryByFolder(ctx, folder)
|
|
if err != nil {
|
|
fmt.Printf("[POLL-SYNC] Warning: failed to get library for folder %s: %v\n", folder, err)
|
|
continue
|
|
}
|
|
libraryID := lib.LibraryID
|
|
// Get all media items from database for this library
|
|
dbItems, err := s.db.ListMediaItemsByLibrary(ctx, libraryID)
|
|
if err != nil {
|
|
fmt.Printf("[POLL-SYNC] Warning: failed to get library items: %v\n", err)
|
|
continue
|
|
}
|
|
// Build set of existing file paths from filesystem
|
|
existingPaths := make(map[string]bool)
|
|
filepath.WalkDir(folder, func(path string, d os.DirEntry, walkErr error) error {
|
|
if walkErr != nil {
|
|
return walkErr
|
|
}
|
|
if !d.IsDir() && s.isScannableFile(path) {
|
|
existingPaths[s.getRelativePath(path)] = true
|
|
}
|
|
return nil
|
|
})
|
|
// Check for orphaned items (in DB but not on filesystem)
|
|
for _, item := range dbItems {
|
|
if item.FilePath != "" && !existingPaths[item.FilePath] {
|
|
msg := fmt.Sprintf("[POLL-SYNC] Orphaned media item found: ID=%s, Title=%s, Path=%s",
|
|
item.ID, item.Title, item.FilePath)
|
|
s.logger.LogDelete(msg)
|
|
if err := s.db.DeleteMediaItem(ctx, item.ID); err != nil {
|
|
errMsg := fmt.Sprintf("[POLL-SYNC] ERROR: failed to delete orphaned item %s: %v", item.Title, err)
|
|
s.logger.LogDelete(errMsg)
|
|
s.logger.LogError(errMsg)
|
|
} else {
|
|
s.logger.LogDelete(fmt.Sprintf("[POLL-SYNC] SUCCESS: deleted orphaned item '%s'", item.Title))
|
|
}
|
|
}
|
|
}
|
|
// Check for new files (on filesystem but not in DB)
|
|
// This is expensive, so we just check a few representative files
|
|
// The fsnotify handler should catch most new files
|
|
for relPath := range existingPaths {
|
|
// Check if this file exists in DB
|
|
_, err := s.db.GetMediaItemByFilePath(ctx, database.GetMediaItemByFilePathParams{
|
|
FilePath: relPath,
|
|
LibraryID: libraryID,
|
|
})
|
|
if err == pgx.ErrNoRows {
|
|
// New file found - scan it
|
|
absPath := folder + "/" + relPath
|
|
if _, err := os.Stat(absPath); err == nil {
|
|
fmt.Printf("[POLL-SYNC] New file detected, scanning: %s\n", absPath)
|
|
if _, err := s.processMediaFile(ctx, absPath); err != nil {
|
|
fmt.Printf("[POLL-SYNC] Error scanning new file %s: %v\n", absPath, err)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
fmt.Println("[POLL-SYNC] Filesystem sync completed")
|
|
return nil
|
|
}
|
|
|
|
// ============================================
|
|
// SCANNER ENHANCEMENTS
|
|
// ============================================
|
|
|
|
// calculateFileSHA256 calculates SHA-256 hash using streaming to avoid loading entire file into memory
|
|
func (s *MediaScanner) calculateFileSHA256(filePath string) (string, error) {
|
|
file, err := os.Open(filePath)
|
|
if err != nil {
|
|
return "", fmt.Errorf("failed to open file: %v", err)
|
|
}
|
|
defer file.Close()
|
|
|
|
hasher := sha256.New()
|
|
if _, err := io.Copy(hasher, file); err != nil {
|
|
return "", fmt.Errorf("failed to calculate hash: %v", err)
|
|
}
|
|
|
|
return hex.EncodeToString(hasher.Sum(nil)), nil
|
|
}
|
|
|
|
// OPFIdentifier represents an identifier from OPF metadata
|
|
type OPFIdentifier struct {
|
|
XMLName xml.Name `xml:"identifier"`
|
|
ID string `xml:"id,attr"`
|
|
Scheme string `xml:"scheme,attr"`
|
|
Content string `xml:",chardata"`
|
|
}
|
|
|
|
// OPFMetadata represents parsed OPF metadata
|
|
type OPFMetadata struct {
|
|
XMLName xml.Name `xml:"package"`
|
|
Version string `xml:"version,attr"`
|
|
Identifiers []OPFIdentifier `xml:"metadata>identifier"`
|
|
}
|
|
|
|
// extractOPFIdentifiers extracts identifiers from EPUB OPF file
|
|
func (s *MediaScanner) extractOPFIdentifiers(epubPath string) (opfIdentifier, opfUUID string, confidence string, err error) {
|
|
book, err := epub.ReadBook(epubPath)
|
|
if err != nil {
|
|
return "", "", "", fmt.Errorf("failed to open EPUB: %v", err)
|
|
}
|
|
|
|
// Extract OPF identifiers using go-epub library
|
|
identifiers, err := book.MetadataByKey("identifier")
|
|
if err != nil || len(identifiers) == 0 {
|
|
return "", "", "low", nil
|
|
}
|
|
|
|
var identifier, uuid string
|
|
for _, id := range identifiers {
|
|
id = strings.TrimSpace(id)
|
|
|
|
// Check for UUID format (urn:uuid:)
|
|
if strings.HasPrefix(strings.ToLower(id), "urn:uuid:") {
|
|
uuid = strings.TrimPrefix(strings.ToLower(id), "urn:uuid:")
|
|
continue
|
|
}
|
|
|
|
// Check if it's a plain UUID (8-4-4-4-12 format)
|
|
if isValidUUID(id) {
|
|
uuid = id
|
|
continue
|
|
}
|
|
|
|
// Check for ISBN
|
|
if strings.Contains(strings.ToLower(id), "isbn") {
|
|
isbn := s.extractISBNFromIdentifier(id)
|
|
if isbn != "" {
|
|
identifier = isbn
|
|
continue
|
|
}
|
|
}
|
|
|
|
// Use first identifier as fallback
|
|
if identifier == "" && id != "" {
|
|
identifier = id
|
|
}
|
|
}
|
|
|
|
confidence = s.determineHashConfidence(uuid, identifier)
|
|
|
|
return identifier, uuid, confidence, nil
|
|
}
|
|
|
|
// isValidUUID checks if string is a valid UUID (8-4-4-4-12 format)
|
|
func isValidUUID(idStr string) bool {
|
|
uuidRegex := regexp.MustCompile(`^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$`)
|
|
return uuidRegex.MatchString(idStr)
|
|
}
|
|
|
|
// extractISBNFromIdentifier extracts ISBN from identifier string
|
|
func (s *MediaScanner) extractISBNFromIdentifier(id string) string {
|
|
id = strings.TrimSpace(id)
|
|
|
|
// Remove "isbn:" prefix if present
|
|
if strings.HasPrefix(strings.ToLower(id), "isbn:") {
|
|
id = strings.TrimPrefix(strings.ToLower(id), "isbn:")
|
|
}
|
|
|
|
// Remove hyphens and spaces
|
|
isbn := regexp.MustCompile(`[\s-]`).ReplaceAllString(id, "")
|
|
|
|
// Check if it's a valid ISBN-10 or ISBN-13
|
|
if len(isbn) == 10 || len(isbn) == 13 {
|
|
return isbn
|
|
}
|
|
|
|
return ""
|
|
}
|
|
|
|
// determineHashConfidence determines confidence level based on available identifiers
|
|
func (s *MediaScanner) determineHashConfidence(uuid, identifier string) string {
|
|
if uuid != "" && isValidUUID(uuid) {
|
|
return "high"
|
|
}
|
|
if identifier != "" && (strings.Contains(strings.ToLower(identifier), "isbn") || len(identifier) >= 10) {
|
|
return "medium"
|
|
}
|
|
return "low"
|
|
}
|
|
|
|
// detectFormatType detects the format type based on file extension and content
|
|
func (s *MediaScanner) detectFormatType(filePath string) string {
|
|
base := strings.ToLower(filepath.Base(filePath))
|
|
|
|
// Check for compound extensions first (like .kepub.epub)
|
|
if strings.HasSuffix(base, ".kepub.epub") {
|
|
return "kepub"
|
|
}
|
|
|
|
ext := strings.ToLower(filepath.Ext(filePath))
|
|
|
|
switch ext {
|
|
case ".epub":
|
|
return "epub"
|
|
case ".kepub":
|
|
return "kepub"
|
|
case ".pdf":
|
|
return "pdf"
|
|
case ".cbz", ".cbr", ".cb7", ".cbt":
|
|
return "comic_archive"
|
|
case ".mobi":
|
|
return "mobi"
|
|
case ".azw", ".azw3":
|
|
return "kfx"
|
|
case ".txt":
|
|
return "txt"
|
|
case ".fb2":
|
|
return "fb2"
|
|
default:
|
|
return "unknown"
|
|
}
|
|
}
|
|
|
|
// extractHashInfo calculates hash and extracts OPF identifiers for a file
|
|
func (s *MediaScanner) extractHashInfo(filePath string) (*HashInfo, *FormatInfo, error) {
|
|
// Calculate SHA-256
|
|
fileSHA256, err := s.calculateFileSHA256(filePath)
|
|
if err != nil {
|
|
return nil, nil, fmt.Errorf("failed to calculate SHA-256: %v", err)
|
|
}
|
|
|
|
// Get file info
|
|
info, err := os.Stat(filePath)
|
|
if err != nil {
|
|
return nil, nil, fmt.Errorf("failed to get file info: %v", err)
|
|
}
|
|
|
|
// Extract OPF identifiers for EPUB files
|
|
var opfIdentifier, opfUUID, confidence string
|
|
if strings.HasSuffix(strings.ToLower(filePath), ".epub") {
|
|
opfIdentifier, opfUUID, confidence, err = s.extractOPFIdentifiers(filePath)
|
|
if err != nil {
|
|
// Non-fatal error, continue with low confidence
|
|
confidence = "low"
|
|
}
|
|
}
|
|
|
|
hashInfo := &HashInfo{
|
|
FileSHA256: fileSHA256,
|
|
OPFIdentifier: opfIdentifier,
|
|
OPFUUID: opfUUID,
|
|
HashConfidence: confidence,
|
|
}
|
|
|
|
formatInfo := &FormatInfo{
|
|
FormatType: s.detectFormatType(filePath),
|
|
FilePath: filePath,
|
|
FileSHA256: fileSHA256,
|
|
FileSizeBytes: info.Size(),
|
|
MimeType: s.getMimeType(filePath),
|
|
}
|
|
|
|
return hashInfo, formatInfo, nil
|
|
}
|
|
|
|
func (s *MediaScanner) getRelativePath(absolutePath string) string {
|
|
// Get the base folder paths from scanner
|
|
for _, baseFolder := range s.folders {
|
|
// Check if path is within this base folder
|
|
if relPath, ok := strings.CutPrefix(absolutePath, baseFolder); ok {
|
|
return strings.TrimPrefix(relPath, "/")
|
|
}
|
|
}
|
|
// Fallback: if no match, return as-is (shouldn't happen)
|
|
return absolutePath
|
|
}
|