// Package services provides the core business logic layer for bookhoard, // including media scanning, library management, search, analytics, and conversion services. package services import ( "archive/tar" "archive/zip" "bookhoard/internal/database" "bookhoard/internal/utils" "bytes" "compress/bzip2" "compress/gzip" "context" "crypto/sha256" "encoding/hex" "encoding/json" "encoding/xml" "errors" "fmt" "image" _ "image/jpeg" _ "image/png" "io" "io/fs" "os" "path/filepath" "regexp" "strconv" "strings" "sync" "sync/atomic" "time" "bookhoard/internal/sevenzip" epub "github.com/ArcadiaLin/go-epub" "github.com/fsnotify/fsnotify" "github.com/google/uuid" "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgtype" "github.com/nwaples/rardecode" pdfcpuapi "github.com/pdfcpu/pdfcpu/pkg/api" ) // MediaMetadata contains extracted metadata for media files (ebooks, comics, manga) type MediaMetadata struct { Title string Author string Description string Series string SeriesNumber int32 Publisher string PublishDate time.Time Contributors []string CoverPath string ISBN string ASIN string Tags []string Genre string FileHashInfo *HashInfo FileFormats []*FormatInfo // Reading direction fields (from ComicInfo.xml or computed) MangaType string // Raw ComicInfo.xml Manga field ReadingDirection string // Computed: auto, ltr, rtl, vertical Language string // ISO 639-1 language code // Additional metadata fields (from ComicInfo.xml or other metadata sources) // Universal fields (apply to ebooks, audiobooks, comics) SeriesCount int32 // Total items in series (Count field for comics, series count for books) Volume int32 // Volume/omnibus number Imprint string // Publisher imprint AgeRating string // Age rating (Everyone, Teen, Mature, Adult) WebURL string // URL to info page (Goodreads, ComicVine, etc.) MetadataNotes string // Notes from metadata files (not user notes) CommunityRating float64 // Pre-existing community rating (0-10) PageCount int32 // Actual page count (images for comics, pages for PDF) TotalCharacters int64 // Total text characters (for reflowable EPUBs) ChapterCount int32 // Number of chapters detected // Comic-specific fields StoryArc string // Story arc name IsBlackAndWhite bool // Black and white flag AlternateInfo string // JSONB string of alternate series info ScanInformation string // Scan information Summary string // Summary from ComicInfo.xml } type HashInfo struct { FileSHA256 string OPFIdentifier string OPFUUID string HashConfidence string } type FormatInfo struct { FormatType string FilePath string FileSHA256 string FileSizeBytes int64 MimeType string } // MediaScanner scans library folders for media files (ebooks, comics, manga) type MediaScanner struct { db *database.Queries watcher *fsnotify.Watcher folders []string adminID pgtype.UUID defaultLibraryID pgtype.UUID libraryTypes map[string][]string forceRescan bool logger *ScannerLogger dirtyDirs map[string]time.Time dirtyDirsMu sync.RWMutex fileStability map[string]*atomic.Bool fileStabilityMu sync.RWMutex scanMutex sync.Mutex scanInProgress atomic.Bool watching atomic.Bool settingsCache *SettingsCache totalFiles int newItems int errors int job *Job } // CalibreOPFMetadata represents intermediate parsed metadata from Calibre metadata.opf files type CalibreOPFMetadata struct { Title string Authors []string Tags []string Description string Publisher string PublishDate *time.Time Language string ISBN string ASIN string UUID string Contributors []string Series string SeriesIndex *float64 Rating *int32 Timestamp *time.Time } // NewMediaScanner creates a new media scanner instance func NewMediaScanner(db *database.Queries) *MediaScanner { watcher, err := fsnotify.NewWatcher() if err != nil { panic(fmt.Sprintf("Failed to create file watcher: %v", err)) } return &MediaScanner{ db: db, watcher: watcher, settingsCache: NewSettingsCache(30 * time.Second), dirtyDirs: make(map[string]time.Time), fileStability: make(map[string]*atomic.Bool), watching: atomic.Bool{}, scanInProgress: atomic.Bool{}, folders: []string{}, adminID: pgtype.UUID{}, defaultLibraryID: pgtype.UUID{Valid: false}, libraryTypes: make(map[string][]string), logger: NewScannerLogger(), } } func (s *MediaScanner) GetPollInterval() time.Duration { if cached, ok := s.settingsCache.Get("scan_poll_interval_seconds"); ok { if seconds, err := strconv.Atoi(cached); err == nil { return time.Duration(seconds) * time.Second } } if s.db == nil { return 5 * time.Minute } ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() setting, err := s.db.GetSystemSetting(ctx, "scan_poll_interval_seconds") if err != nil || setting == "" { return 5 * time.Minute } s.settingsCache.Set("scan_poll_interval_seconds", setting) seconds, err := strconv.Atoi(setting) if err != nil { return 5 * time.Minute } return time.Duration(seconds) * time.Second } func (s *MediaScanner) GetAutoScanEnabled() bool { // Check cache first if cached, ok := s.settingsCache.Get("auto_scan_enabled"); ok { return strings.ToLower(cached) == "true" } if s.db == nil { return true } // Cache miss - query database ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() setting, err := s.db.GetSystemSetting(ctx, "auto_scan_enabled") if err != nil || setting == "" { return true } // Store in cache s.settingsCache.Set("auto_scan_enabled", setting) return strings.ToLower(setting) == "true" } func (s *MediaScanner) SetAdminID(adminID pgtype.UUID) { s.adminID = adminID } func (s *MediaScanner) SetLibraryID(libraryID pgtype.UUID) { s.defaultLibraryID = libraryID } func (s *MediaScanner) SetForce(force bool) { s.forceRescan = force } func (s *MediaScanner) GetStats() (int, int, int) { return s.totalFiles, s.newItems, s.errors } func (s *MediaScanner) SetFolders(folders []string) error { s.folders = folders // Remove old watch if exists if s.watcher != nil { if s.watcher != nil { if err := s.watcher.Close(); err != nil { fmt.Printf("Warning: failed to close old watcher during folder reconfiguration: %v\n", err) } } } // Create new watcher watcher, err := fsnotify.NewWatcher() if err != nil { return fmt.Errorf("failed to create watcher: %v", err) } s.watcher = watcher // Build cache of allowed extensions per folder // Uses Go AllowedExtensions map as source of truth (not DB) s.libraryTypes = make(map[string][]string) ctx := context.Background() for _, folder := range folders { lib, err := s.db.GetLibraryByFolder(ctx, folder) if err != nil { fmt.Printf("Warning: failed to get library for folder %s: %v\n", folder, err) continue } libType, err := s.db.GetLibraryType(ctx, lib.LibraryTypeID) if err != nil { fmt.Printf("Warning: failed to get library type for %s: %v\n", folder, err) continue } if exts, ok := AllowedExtensions[libType.Name]; ok { s.libraryTypes[folder] = exts fmt.Printf("Scanner: Folder %s (type: %s) allows extensions: %v\n", folder, libType.Name, exts) } else { s.libraryTypes[folder] = libType.AllowedExtensions fmt.Printf("Scanner: Folder %s (type: %s) using DB extensions (no Go map entry): %v\n", folder, libType.Name, libType.AllowedExtensions) } } // Add all folders and their subdirectories to the watcher (like Audiobookshelf) watchCount := 0 for _, folder := range folders { if err := s.watcher.Add(folder); err != nil { fmt.Printf("[WATCHER] Warning: failed to watch root folder %s: %v\n", folder, err) } else { watchCount++ } filepath.WalkDir(folder, func(path string, d fs.DirEntry, err error) error { if err != nil { return err } if !d.IsDir() || path == folder { return nil } if err := s.watcher.Add(path); err != nil { fmt.Printf("[WATCHER] Warning: failed to watch subdirectory %s: %v\n", path, err) } else { watchCount++ } return nil }) } fmt.Printf("[WATCHER] Now watching %d directories across %d root folders\n", watchCount, len(folders)) return nil } func (s *MediaScanner) enqueueLibraryScan(rootFolder string) { if s.db == nil { return } libRow, err := s.db.GetLibraryByFolderPathPrefix(context.Background(), rootFolder) if err != nil { fmt.Printf("[MTIME-POLL] Warning: could not find library for %s: %v\n", rootFolder, err) return } folders, err := s.db.GetLibraryFolders(context.Background(), libRow.LibraryID) if err != nil { fmt.Printf("[MTIME-POLL] Warning: could not get folders for library: %v\n", err) return } folderPaths := make([]string, len(folders)) for i, f := range folders { folderPaths[i] = f.FolderPath } adminIDStr := "" if libRow.CreatedByAdminID.Valid { adminIDStr = uuid.UUID(libRow.CreatedByAdminID.Bytes).String() } if adminIDStr == "" { fmt.Printf("[MTIME-POLL] Library has no owner, falling back to first admin\n") fallbackAdmin, err := s.db.GetFirstAdmin(context.Background()) if err != nil { fmt.Printf("[MTIME-POLL] Warning: no admin found in database, skipping scan\n") return } adminIDStr = uuid.UUID(fallbackAdmin.Bytes).String() } libraryIDStr := uuid.UUID(libRow.LibraryID.Bytes).String() job := &Job{ ID: uuid.New().String(), Type: JobTypeScan, Status: JobStatusPending, UserID: adminIDStr, Context: context.Background(), Params: map[string]any{ "library_id": libraryIDStr, "folders": folderPaths, "admin_id": adminIDStr, "db": s.db, "force": false, }, } if WorkerInstance != nil { WorkerInstance.Enqueue(job) fmt.Printf("[MTIME-POLL] Enqueued library scan for %s (library: %s)\n", rootFolder, libraryIDStr) } } func (s *MediaScanner) ScanFolders(ctx context.Context) error { if len(s.folders) == 0 { return fmt.Errorf("no folders set") } s.totalFiles = 0 s.newItems = 0 s.errors = 0 for _, folder := range s.folders { if err := filepath.WalkDir(folder, func(path string, d fs.DirEntry, err error) error { if !d.IsDir() && s.isScannableFile(path) { s.totalFiles++ } return nil }); err != nil { fmt.Printf("Warning: failed to count files in %s: %v\n", folder, err) return nil } } fmt.Printf("Starting scan of %d folders: %v (%d files to scan)\n", len(s.folders), s.folders, s.totalFiles) processedFiles := 0 mediaFiles := 0 for _, folder := range s.folders { fmt.Printf("Scanning folder: %s\n", folder) if _, err := os.Stat(folder); os.IsNotExist(err) { fmt.Printf("Folder does not exist: %s\n", folder) s.errors++ continue } err := filepath.WalkDir(folder, func(path string, d fs.DirEntry, err error) error { if err != nil { fmt.Printf("Error accessing path %s: %v\n", path, err) s.errors++ return err } if d.IsDir() { if err := s.watcher.Add(path); err != nil { fmt.Printf("Warning: failed to watch subdirectory %s: %v\n", path, err) } return nil } if s.isScannableFile(path) { mediaFiles++ processedFiles++ if processedFiles%10 == 0 && s.totalFiles > 0 { progress := float64(processedFiles) / float64(s.totalFiles) if s.job != nil { s.job.UpdateProgress(progress, processedFiles, s.newItems, s.errors) } } wasNew, err := s.processMediaFile(ctx, path) if err != nil { fmt.Printf("Error processing media file %s: %v\n", path, err) s.errors++ } else if wasNew { fmt.Printf("Successfully processed media file: %s\n", path) } else { fmt.Printf("Successfully processed media file: %s\n", path) } } return nil }) if err != nil { s.errors++ return fmt.Errorf("failed to scan folder %s: %v", folder, err) } } fmt.Printf("Scan completed: %d total files scanned, %d media files found, %d new items, %d errors\n", processedFiles, mediaFiles, s.newItems, s.errors) // Clean up: Find media items in DB that no longer exist on filesystem for _, folder := range s.folders { lib, err := s.db.GetLibraryByFolder(ctx, folder) if err != nil { continue } libraryID := lib.LibraryID dbItems, err := s.db.ListMediaItemsByLibrary(ctx, libraryID) if err != nil { fmt.Printf("Warning: failed to get library items for cleanup: %v\n", err) continue } // Build set of scanned file paths for this folder scannedPaths := make(map[string]bool) if err := filepath.WalkDir(folder, func(path string, d os.DirEntry, walkErr error) error { if walkErr != nil { return walkErr } if !d.IsDir() && s.isScannableFile(path) { scannedPaths[s.getRelativePath(path)] = true } return nil }); err != nil { fmt.Printf("[RESCAN-CLEANUP] Warning: failed to walk directory %s, skipping orphan cleanup: %v\n", folder, err) continue // Skip to next folder to avoid false deletions } // Delete items whose files no longer exist - with safety logging for _, item := range dbItems { filePath := item.FilePath if filePath != "" && !scannedPaths[filePath] { msg := fmt.Sprintf("[RESCAN-CLEANUP] Orphaned media item found: ID=%s, Title=%s, Path=%s", item.ID, item.Title, filePath) s.logger.LogDelete(msg) delMsg := fmt.Sprintf("[RESCAN-CLEANUP] Deleting orphaned item '%s' (file no longer exists at %s)", item.Title, filePath) s.logger.LogDelete(delMsg) if err := s.db.DeleteMediaItem(ctx, item.ID); err != nil { errMsg := fmt.Sprintf("[RESCAN-CLEANUP] ERROR: failed to delete orphaned item %s: %v", item.Title, err) s.logger.LogDelete(errMsg) s.logger.LogError(errMsg) } else { s.logger.LogDelete(fmt.Sprintf("[RESCAN-CLEANUP] SUCCESS: deleted orphaned item '%s'", item.Title)) } } } } if s.job != nil && s.totalFiles > 0 { s.job.UpdateProgress(1.0, processedFiles, s.newItems, s.errors) } return nil } func (s *MediaScanner) isScannableFile(path string) bool { ext := strings.ToLower(filepath.Ext(path)) // Find which folder this file belongs to var folder string for _, f := range s.folders { if strings.HasPrefix(path, f) { folder = f break } } // If no folder match, don't scan if folder == "" { return false } // Get allowed extensions for this folder's library allowed, ok := s.libraryTypes[folder] if !ok { // No library type info, skip file fmt.Printf("Warning: No library type info for folder %s, skipping %s\n", folder, path) return false } // Check if file extension is allowed for this library type for _, allowedExt := range allowed { if ext == strings.ToLower(allowedExt) { return true } } return false } // extractFolderStructureMetadata extracts metadata from folder paths, prioritizing Calibre structure func (s *MediaScanner) extractFolderStructureMetadata(path, rootFolder string) *MediaMetadata { metadata := &MediaMetadata{} // Get the relative path from root folder relPath, err := filepath.Rel(rootFolder, path) if err != nil { return metadata } // Split into directory components dir := filepath.Dir(relPath) components := strings.Split(dir, string(filepath.Separator)) if len(components) < 2 { return metadata // Not enough structure to extract } // Calibre structure detection // Pattern 1: Author Name/Book Title/ // Pattern 2: Author Name/Series Name/Book Title/ // Pattern 3: Author Name/Series Name, Book #1 - Book Title/ author := strings.TrimSuffix(components[0], "_") // Remove trailing underscore if present metadata.Author = strings.ReplaceAll(author, "_", " ") if len(components) >= 3 { // This might be a series structure possibleSeries := components[1] possibleTitle := components[2] // Check for Calibre series format: "Series Name, Book #1 - Title" seriesMatch := regexp.MustCompile(`^(.*),\s+Book\s+#(\d+)\s*-\s*(.*)$`).FindStringSubmatch(possibleSeries) if len(seriesMatch) == 4 { metadata.Series = strings.ReplaceAll(seriesMatch[1], "_", " ") if seriesNum, err := strconv.ParseInt(seriesMatch[2], 10, 32); err == nil { metadata.SeriesNumber = int32(seriesNum) } metadata.Title = strings.ReplaceAll(possibleTitle, "_", " ") } else { // Simple series structure: Author/Series/Title metadata.Series = strings.ReplaceAll(possibleSeries, "_", " ") metadata.Title = strings.ReplaceAll(possibleTitle, "_", " ") // Try to extract series number from title titleNumMatch := regexp.MustCompile(`^(.*)\s+(\d+)$`).FindStringSubmatch(metadata.Title) if len(titleNumMatch) == 3 { metadata.Title = titleNumMatch[1] if seriesNum, err := strconv.ParseInt(titleNumMatch[2], 10, 32); err == nil { metadata.SeriesNumber = int32(seriesNum) } } } } else { // Simple structure: Author/Title metadata.Title = strings.ReplaceAll(components[1], "_", " ") } return metadata } var bookExtensions = map[string]bool{ ".epub": true, ".pdf": true, ".mobi": true, ".azw": true, ".azw3": true, ".fb2": true, ".txt": true, ".rtf": true, ".doc": true, ".docx": true, ".lit": true, ".pdb": true, ".djvu": true, ".cbz": true, ".cbr": true, ".cb7": true, ".cbt": true, } func hasSiblingBookFile(dir string) bool { entries, err := os.ReadDir(dir) if err != nil { return false } for _, entry := range entries { if !entry.IsDir() && bookExtensions[strings.ToLower(filepath.Ext(entry.Name()))] { return true } } return false } func (s *MediaScanner) processMediaFile(ctx context.Context, path string) (bool, error) { fmt.Printf("Processing media file: %s\n", path) // Get file info info, err := os.Stat(path) if err != nil { fmt.Printf("Failed to get file info for %s: %v\n", path, err) return false, fmt.Errorf("failed to get file info: %v", err) } if isImageFile(path) && hasSiblingBookFile(filepath.Dir(path)) { return false, nil } fileModTime := info.ModTime() // Find library for this file's folder var libraryID pgtype.UUID if s.defaultLibraryID.Valid { libraryID = s.defaultLibraryID } else { for _, folder := range s.folders { if strings.HasPrefix(path, folder) { lib, err := s.db.GetLibraryByFolder(ctx, folder) if err != nil { return false, fmt.Errorf("failed to find library for folder %s: %v", folder, err) } libraryID = lib.LibraryID break } } } if !libraryID.Valid { return false, fmt.Errorf("no library found for file path: %s", path) } // Check if media item already exists in database existingItem, err := s.getMediaItemByFilePath(ctx, path, libraryID) if err == nil { fmt.Printf("Media item already exists in database: %s (size: %d vs %d)\n", path, existingItem.FileSize.Int64, info.Size()) // If force rescan is enabled, always re-process if s.forceRescan { fmt.Printf("Force rescan enabled, updating existing media item: %s\n", path) // Use UPDATE instead of DELETE+INSERT to preserve created_at if err := s.updateMediaItem(ctx, existingItem.ID, path, info); err != nil { fmt.Printf("Warning: failed to update existing media item: %v\n", err) } return false, nil } else { // Normal behavior: check if file has changed (by size) if existingItem.FileSize.Int64 != info.Size() { fmt.Printf("File size changed, updating media item: %s\n", path) _ = s.updateMediaItem(ctx, existingItem.ID, path, info) return false, nil } fmt.Printf("Media item already exists with same size, skipping: %s\n", path) return false, nil } } else if !errors.Is(err, pgx.ErrNoRows) { fmt.Printf("Database error checking media item existence: %v\n", err) return false, fmt.Errorf("failed to check if media item exists: %v", err) } fmt.Printf("Media item does not exist in database, creating new entry: %s\n", path) // Extract metadata from file first metadata, err := s.extractMetadata(path) if err != nil { fmt.Printf("Warning: failed to extract metadata from %s: %v\n", path, err) metadata = &MediaMetadata{} } // Extract hash information during metadata extraction hashInfo, formatInfo, err := s.extractHashInfo(path) if err != nil { fmt.Printf("Warning: failed to extract hash info from %s: %v\n", path, err) hashInfo = &HashInfo{} } else { metadata.FileHashInfo = hashInfo metadata.FileFormats = []*FormatInfo{formatInfo} fmt.Printf("Hash info for %s: SHA256=%s, OPF_ID=%s, OPF_UUID=%s, Confidence=%s\n", path, hashInfo.FileSHA256, hashInfo.OPFIdentifier, hashInfo.OPFUUID, hashInfo.HashConfidence) } // REMOVED: Comic metadata extraction now handled by mergeMetadata() // This avoids duplicate extraction and ensures smart merging happens // Try to get metadata from folder structure as fallback/enhancement // Use the root folder that contains this file var rootFolder string for _, folder := range s.folders { if strings.HasPrefix(path, folder) { rootFolder = folder break } } if rootFolder != "" { folderMetadata := s.extractFolderStructureMetadata(path, rootFolder) // Use folder metadata as fallback for missing information if metadata.Title == "" && folderMetadata.Title != "" { metadata.Title = folderMetadata.Title } if metadata.Author == "" && folderMetadata.Author != "" { metadata.Author = folderMetadata.Author } if metadata.Series == "" && folderMetadata.Series != "" { metadata.Series = folderMetadata.Series } if metadata.SeriesNumber == 0 && folderMetadata.SeriesNumber > 0 { metadata.SeriesNumber = folderMetadata.SeriesNumber } } // Final fallback if still missing essential metadata if metadata.Title == "" { metadata.Title = strings.TrimSuffix(filepath.Base(path), filepath.Ext(path)) } if metadata.Author == "" { metadata.Author = "Unknown" } // libraryID already determined at start of function // Normalize metadata fields for display metadata.Contributors = utils.NormalizeContributors(metadata.Contributors) metadata.Tags = utils.NormalizeTags(metadata.Tags) // Normalize search fields contributorsSearch := utils.NormalizeContributorsSearch(metadata.Contributors) tagsSearch := utils.NormalizeTagsSearch(metadata.Tags) // Create media item in database relativePath := s.getRelativePath(path) createdItem, err := s.db.CreateMediaItem(ctx, database.CreateMediaItemParams{ LibraryID: libraryID, Title: metadata.Title, Author: pgtype.Text{String: metadata.Author, Valid: metadata.Author != ""}, Isbn: pgtype.Text{String: utils.NormalizeISBNSafe(metadata.ISBN), Valid: metadata.ISBN != ""}, Asin: pgtype.Text{String: metadata.ASIN, Valid: metadata.ASIN != ""}, Description: pgtype.Text{String: metadata.Description, Valid: metadata.Description != ""}, FilePath: relativePath, FileSize: pgtype.Int8{Int64: info.Size(), Valid: true}, MimeType: pgtype.Text{String: s.getMimeType(path), Valid: true}, CoverImagePath: pgtype.Text{String: metadata.CoverPath, Valid: metadata.CoverPath != ""}, Series: pgtype.Text{String: metadata.Series, Valid: metadata.Series != ""}, SeriesNumber: pgtype.Int4{Int32: metadata.SeriesNumber, Valid: metadata.SeriesNumber > 0}, Publisher: pgtype.Text{String: metadata.Publisher, Valid: metadata.Publisher != ""}, DatePublished: pgtype.Date{Time: metadata.PublishDate, Valid: !metadata.PublishDate.IsZero()}, Contributors: metadata.Contributors, ContributorsSearch: contributorsSearch, Tags: metadata.Tags, TagsSearch: tagsSearch, AddedByAdminID: s.adminID, CreatedAt: pgtype.Timestamptz{Time: fileModTime, Valid: true}, ImportedAt: pgtype.Timestamptz{Time: time.Now(), Valid: true}, MangaType: pgtype.Text{String: metadata.MangaType, Valid: metadata.MangaType != ""}, ReadingDirection: pgtype.Text{String: metadata.ReadingDirection, Valid: metadata.ReadingDirection != ""}, SeriesCount: pgtype.Int4{Int32: metadata.SeriesCount, Valid: metadata.SeriesCount > 0}, Volume: pgtype.Int4{Int32: metadata.Volume, Valid: metadata.Volume > 0}, Imprint: pgtype.Text{String: metadata.Imprint, Valid: metadata.Imprint != ""}, AgeRating: pgtype.Text{String: metadata.AgeRating, Valid: metadata.AgeRating != ""}, WebUrl: pgtype.Text{String: metadata.WebURL, Valid: metadata.WebURL != ""}, StoryArc: pgtype.Text{String: metadata.StoryArc, Valid: metadata.StoryArc != ""}, IsBlackAndWhite: pgtype.Bool{Bool: metadata.IsBlackAndWhite, Valid: true}, MetadataNotes: pgtype.Text{String: metadata.MetadataNotes, Valid: metadata.MetadataNotes != ""}, AlternateInfo: func() []byte { if metadata.AlternateInfo != "" { return []byte(metadata.AlternateInfo) } return nil }(), ScanInformation: pgtype.Text{String: metadata.ScanInformation, Valid: metadata.ScanInformation != ""}, Summary: pgtype.Text{String: metadata.Summary, Valid: metadata.Summary != ""}, CommunityRating: pgtype.Float8{Float64: metadata.CommunityRating, Valid: metadata.CommunityRating > 0}, PageCount: pgtype.Int4{Int32: metadata.PageCount, Valid: metadata.PageCount > 0}, }) if err != nil { return false, fmt.Errorf("failed to create media item: %v", err) } // Update hash information before database storage if metadata.FileHashInfo != nil && metadata.FileHashInfo.FileSHA256 != "" { _, err = s.db.UpdateMediaItemIdentifiers(ctx, database.UpdateMediaItemIdentifiersParams{ ID: createdItem.ID, FileSha256: pgtype.Text{String: metadata.FileHashInfo.FileSHA256, Valid: true}, OpfIdentifier: pgtype.Text{String: metadata.FileHashInfo.OPFIdentifier, Valid: metadata.FileHashInfo.OPFIdentifier != ""}, OpfUuid: pgtype.Text{String: metadata.FileHashInfo.OPFUUID, Valid: metadata.FileHashInfo.OPFUUID != ""}, HashConfidence: pgtype.Text{String: metadata.FileHashInfo.HashConfidence, Valid: true}, }) if err != nil { fmt.Printf("Warning: failed to update hash identifiers for %s: %v\n", path, err) } } // Set format group, total characters, and chapter count mimeType := s.getMimeType(path) ext := strings.ToLower(filepath.Ext(path)) var formatGroup string var isReflowable, hasFixedLayout bool switch ext { case ".epub": isFixed, fixedErr := s.DetectFixedLayoutEPUB(path) if fixedErr == nil && isFixed { formatGroup = "fixed_layout" hasFixedLayout = true } else { formatGroup = "reflowable" isReflowable = true } case ".mobi", ".azw", ".azw3", ".fb2", ".txt": formatGroup = "reflowable" isReflowable = true case ".pdf", ".djvu": formatGroup = "fixed_layout" hasFixedLayout = true case ".cbz", ".cbr", ".cb7", ".cbt": formatGroup = "comic_archive" hasFixedLayout = true default: formatGroup = "unknown" } err = s.db.UpdateMediaItemFormatGroup(ctx, database.UpdateMediaItemFormatGroupParams{ ID: createdItem.ID, FormatGroup: formatGroup, FormatMimetype: pgtype.Text{String: mimeType, Valid: mimeType != ""}, IsReflowable: pgtype.Bool{Bool: isReflowable, Valid: true}, HasFixedLayout: pgtype.Bool{Bool: hasFixedLayout, Valid: true}, TotalCharacters: pgtype.Int8{Int64: metadata.TotalCharacters, Valid: metadata.TotalCharacters > 0}, ChapterCount: pgtype.Int4{Int32: metadata.ChapterCount, Valid: metadata.ChapterCount > 0}, }) if err != nil { fmt.Printf("Warning: failed to update format info for %s: %v\n", path, err) } // Store format information in the database for _, format := range metadata.FileFormats { _, err = s.db.CreateMediaItemFormat(ctx, database.CreateMediaItemFormatParams{ MediaItemID: createdItem.ID, FormatType: format.FormatType, FilePath: pgtype.Text{String: s.getRelativePath(format.FilePath), Valid: true}, FileSha256: pgtype.Text{String: format.FileSHA256, Valid: true}, FileSizeBytes: pgtype.Int8{Int64: format.FileSizeBytes, Valid: true}, MimeType: pgtype.Text{String: format.MimeType, Valid: true}, }) if err != nil { fmt.Printf("Warning: failed to create format entry for %s: %v\n", path, err) } } s.newItems++ return true, nil } // extractCalibreSidecar checks for and parses a Calibre metadata.opf sidecar file func (s *MediaScanner) extractCalibreSidecar(path string) *MediaMetadata { // Get directory of media file dir := filepath.Dir(path) opfPath := filepath.Join(dir, "metadata.opf") // Check if sidecar exists if _, err := os.Stat(opfPath); os.IsNotExist(err) { return nil // No sidecar, not an error } // Parse sidecar metadata, err := s.parseCalibreMetadataOPF(opfPath) if err != nil { fmt.Printf("Warning: failed to parse Calibre metadata.opf: %v\n", err) return nil // Parsing failed, fall back to embedded } return metadata } // mergeMetadata intelligently merges metadata from multiple sources // Priority: metadata.opf (Calibre) → embedded metadata → folder structure → filename // For comics: metadata.opf → ComicInfo.xml → folder structure → filename func (s *MediaScanner) mergeMetadata(path string, calibreMetadata *MediaMetadata) (*MediaMetadata, error) { metadata := calibreMetadata if metadata == nil { metadata = &MediaMetadata{} } ext := strings.ToLower(filepath.Ext(path)) // For EPUB files if ext == ".epub" { book, err := epub.ReadBook(path) if err == nil { genreTags := extractGenreTagsFromEPUB(book) processGenresAndTags(metadata, genreTags) if allText := book.AllChaptersText(); len(allText) > 0 { metadata.TotalCharacters = int64(len(allText)) } metadata.ChapterCount = int32(book.ChapterCount()) } isFixed, fixedErr := s.DetectFixedLayoutEPUB(path) if fixedErr == nil && isFixed { if pageCount, imgErr := countArchiveImages(path); imgErr == nil && pageCount > 0 { metadata.PageCount = int32(pageCount) } } } // For comic archives, try to extract ComicInfo.xml if ext == ".cbz" || ext == ".cbr" || ext == ".cb7" || ext == ".cbt" { comicInfo, cover, err := extractComicMetadata(path) if err != nil { fmt.Printf("Warning: failed to extract comic metadata from %s: %v\n", path, err) } else if comicInfo != nil { // Merge ComicInfo.xml fields (only if not already set from Calibre) if metadata.Title == "" && comicInfo.Title != "" { metadata.Title = comicInfo.Title } if metadata.Series == "" && comicInfo.Series != "" { metadata.Series = comicInfo.Series } if metadata.SeriesNumber == 0 && comicInfo.Number > 0 { metadata.SeriesNumber = int32(comicInfo.Number) } if metadata.Publisher == "" && comicInfo.Publisher != "" { metadata.Publisher = comicInfo.Publisher } if metadata.Author == "" && comicInfo.Writer != "" { metadata.Author = comicInfo.Writer } if metadata.Description == "" && comicInfo.Summary != "" { metadata.Description = comicInfo.Summary } // NEW: Always extract reading direction from ComicInfo.xml // (even if metadata.opf exists, since Calibre doesn't support this field) metadata.MangaType = normalizeMangaType(comicInfo.Manga) metadata.ReadingDirection = determineReadingDirection(comicInfo) metadata.Language = comicInfo.LanguageISO // NEW: Extract additional comic-specific fields // Series information if metadata.SeriesCount == 0 && comicInfo.Count > 0 { metadata.SeriesCount = int32(comicInfo.Count) } if metadata.Volume == 0 && comicInfo.Volume > 0 { metadata.Volume = int32(comicInfo.Volume) } // Publisher and classification if metadata.Imprint == "" && comicInfo.Imprint != "" { metadata.Imprint = comicInfo.Imprint } if metadata.StoryArc == "" && comicInfo.StoryArc != "" { metadata.StoryArc = comicInfo.StoryArc } if metadata.AgeRating == "" && comicInfo.AgeRating != "" { metadata.AgeRating = normalizeAgeRating(comicInfo.AgeRating) } // NEW: Process genres and tags (universal logic for all formats) // Extract genre tags from ComicInfo.xml (Genre + Tags + Characters + Teams + Locations) genreTags := extractGenreTagsFromComicInfo(comicInfo) processGenresAndTags(metadata, genreTags) // Additional metadata if metadata.WebURL == "" && comicInfo.Web != "" { metadata.WebURL = comicInfo.Web } if metadata.MetadataNotes == "" && comicInfo.Notes != "" { metadata.MetadataNotes = comicInfo.Notes } if metadata.ScanInformation == "" && comicInfo.ScanInformation != "" { metadata.ScanInformation = comicInfo.ScanInformation } if metadata.Summary == "" && comicInfo.Summary != "" { metadata.Summary = comicInfo.Summary } // Boolean fields if !metadata.IsBlackAndWhite && strings.ToLower(comicInfo.BlackAndWhite) == "yes" { metadata.IsBlackAndWhite = true } if metadata.CommunityRating == 0 && comicInfo.CommunityRating > 0 { metadata.CommunityRating = comicInfo.CommunityRating } // Alternate series information (store as JSONB string) if metadata.AlternateInfo == "" && (comicInfo.AlternateSeries != "" || comicInfo.AlternateNumber > 0) { alternateData := map[string]any{} if comicInfo.AlternateSeries != "" { alternateData["alternate_series"] = comicInfo.AlternateSeries } if comicInfo.AlternateNumber > 0 { alternateData["alternate_number"] = comicInfo.AlternateNumber } if comicInfo.AlternateCount > 0 { alternateData["alternate_count"] = comicInfo.AlternateCount } if len(alternateData) > 0 { jsonBytes, err := json.Marshal(alternateData) if err == nil { metadata.AlternateInfo = string(jsonBytes) } } } // REMOVED: Tag enhancement now handled by processGenresAndTags() // Characters, Teams, Locations are already processed via extractGenreTagsFromComicInfo() // Extract cover if not already present if len(cover) > 0 && metadata.CoverPath == "" { coverPath := path + ".cover.jpg" if err := os.WriteFile(coverPath, cover, 0644); err == nil { metadata.CoverPath = s.getRelativePath(coverPath) } } fmt.Printf("Merged comic metadata from %s: title=%s, series=%s, issue=%d, manga=%s, direction=%s\n", path, comicInfo.Title, comicInfo.Series, comicInfo.Number, comicInfo.Manga, metadata.ReadingDirection) } if pageCount, err := countArchiveImages(path); err == nil && pageCount > 0 { metadata.PageCount = int32(pageCount) } } return metadata, nil } // containsTag checks if a tag already exists in the tags array func containsTag(tags []string, tag string) bool { tag = strings.ToLower(tag) for _, t := range tags { if strings.ToLower(t) == tag { return true } } return false } // normalizeMangaType normalizes ComicInfo.xml Manga field to database enum values func normalizeMangaType(manga string) string { switch strings.ToLower(strings.ReplaceAll(manga, " ", "")) { case "unknown": return "unknown" case "no": return "no" case "yes": return "yes" case "yesandrighttoleft": return "yes_and_right_to_left" default: return "unknown" } } // determineReadingDirection computes reading direction from ComicInfo metadata // Uses Manga field + language heuristics + genre tags func determineReadingDirection(comicInfo *ComicInfo) string { // 1. Check explicit Manga field manga := normalizeMangaType(comicInfo.Manga) switch manga { case "yes_and_right_to_left": return "rtl" // Traditional Japanese manga case "yes", "no": return "ltr" // Manga style but LTR, or Western comic } // 2. Language heuristic: Japanese → RTL lang := strings.ToLower(comicInfo.LanguageISO) if lang == "ja" || lang == "jpn" { return "rtl" } // 3. Genre heuristic: webtoons/manhwa → vertical tags := strings.ToLower(comicInfo.Tags + " " + comicInfo.Genre) if strings.Contains(tags, "webtoon") || strings.Contains(tags, "manhwa") { return "vertical" // Korean/Chinese webcomics } if strings.Contains(tags, "manga") { return "rtl" // Japanese manga } // 4. Default: LTR (Western comics) return "ltr" } // normalizeAgeRating normalizes age rating from ComicInfo.xml to standard values func normalizeAgeRating(rating string) string { rating = strings.ToLower(strings.TrimSpace(rating)) switch rating { case "everyone", "e", "all ages": return "Everyone" case "teen", "t", "13+", "13+up": return "Teen" case "mature", "m", "17+", "17+up", "adults only": return "Mature" case "adult", "a", "18+": return "Adult" default: return rating // Return original if unknown } } // processGenresAndTags ensures ALL genres appear in the tags array without duplication // This applies to ALL formats: EPUB, ComicInfo.xml, PDF metadata // Strategy: Use existing `genre` column for primary genre, `tags` array for all genres func processGenresAndTags(metadata *MediaMetadata, genreTags []string) { if metadata.Tags == nil { metadata.Tags = []string{} } // 1. Set primary genre (first genre tag wins if not already set) if metadata.Genre == "" && len(genreTags) > 0 { metadata.Genre = genreTags[0] } // 2. Ensure ALL genre tags appear in tags array (without duplication) for _, genreTag := range genreTags { genreTag = strings.TrimSpace(genreTag) if genreTag != "" && !containsTag(metadata.Tags, genreTag) { metadata.Tags = append(metadata.Tags, genreTag) } } } // extractGenreTagsFromEPUB extracts all values from EPUB // Returns array of genre tags func extractGenreTagsFromEPUB(book *epub.Book) []string { var genreTags []string // EPUB stores genres in metadata if subjects, err := book.MetadataByKey("subject"); err == nil && len(subjects) > 0 { for _, subject := range subjects { subject = strings.TrimSpace(subject) if subject != "" { genreTags = append(genreTags, subject) } } } return genreTags } // extractGenreTagsFromComicInfo extracts genres from ComicInfo.xml // Genre field + Tags field + Characters + Teams + Locations // Returns array of genre tags func extractGenreTagsFromComicInfo(comicInfo *ComicInfo) []string { var genreTags []string // 1. Add Genre field if comicInfo.Genre != "" { genreTags = append(genreTags, strings.Split(comicInfo.Genre, ",")...) } // 2. Add Tags field (comma-separated) if comicInfo.Tags != "" { genreTags = append(genreTags, strings.Split(comicInfo.Tags, ",")...) } // 3. Add Characters (comma-separated) if comicInfo.Characters != "" { genreTags = append(genreTags, strings.Split(comicInfo.Characters, ",")...) } // 4. Add Teams (comma-separated) if comicInfo.Teams != "" { genreTags = append(genreTags, strings.Split(comicInfo.Teams, ",")...) } // 5. Add Locations (comma-separated) if comicInfo.Locations != "" { genreTags = append(genreTags, strings.Split(comicInfo.Locations, ",")...) } // Trim whitespace from all tags for i := range genreTags { genreTags[i] = strings.TrimSpace(genreTags[i]) } return genreTags } func (s *MediaScanner) extractMetadata(path string) (*MediaMetadata, error) { // Try Calibre sidecar first calibreMetadata := s.extractCalibreSidecar(path) if calibreMetadata != nil { fmt.Printf("Using Calibre metadata.opf for %s\n", path) // Try to find cover image for sidecar metadata coverPath := findSidecarCover(path) if coverPath != "" { calibreMetadata.CoverPath = s.getRelativePath(coverPath) } return s.mergeMetadata(path, calibreMetadata) } // EXISTING: Fallback to embedded metadata ext := strings.ToLower(filepath.Ext(path)) switch ext { case ".epub", ".kepub": metadata := &MediaMetadata{} result, err := s.extractEPUBMetadata(path) if err == nil { return result, nil } if result != nil { metadata = result } // Enhanced format detection for EPUBs isFixedLayout, detectErr := s.DetectFixedLayoutEPUB(path) if detectErr == nil && isFixedLayout { metadata.FileFormats = []*FormatInfo{{ FormatType: "fixed_layout", FilePath: path, MimeType: s.getMimeType(path), }} if pageCount, imgErr := countArchiveImages(path); imgErr == nil && pageCount > 0 { metadata.PageCount = int32(pageCount) } } // Try to extract embedded cover coverPath, err := s.extractEPUBCover(path) if err != nil { fmt.Printf("Warning: failed to extract EPUB cover from %s: %v\n", path, err) } else if coverPath != "" { metadata.CoverPath = s.getRelativePath(coverPath) } // If no embedded cover, try sidecar if metadata.CoverPath == "" { sidecarCover := findSidecarCover(path) if sidecarCover != "" { metadata.CoverPath = s.getRelativePath(sidecarCover) } } return metadata, nil case ".pdf": return s.extractPDFMetadata(path) case ".cbz", ".cbr", ".cb7", ".cbt": metadata, err := s.mergeMetadata(path, nil) if err != nil { return &MediaMetadata{ Title: strings.TrimSuffix(filepath.Base(path), ext), }, nil } if metadata.Title == "" { metadata.Title = strings.TrimSuffix(filepath.Base(path), ext) } // If no cover from archive, try sidecar if metadata.CoverPath == "" { sidecarCover := findSidecarCover(path) if sidecarCover != "" { metadata.CoverPath = s.getRelativePath(sidecarCover) } } return metadata, nil default: // For other formats, return basic metadata return &MediaMetadata{ Title: strings.TrimSuffix(filepath.Base(path), ext), }, nil } } func (s *MediaScanner) extractEPUBMetadata(path string) (*MediaMetadata, error) { book, err := epub.ReadBook(path) if err != nil { return nil, fmt.Errorf("failed to open EPUB: %v", err) } metadata := &MediaMetadata{} // Title if title, err := book.Title(); err == nil && title != "" { metadata.Title = title } // Author if authors, err := book.MetadataByKey("creator"); err == nil && len(authors) > 0 { metadata.Author = authors[0] } // Description if descriptions, err := book.MetadataByKey("description"); err == nil && len(descriptions) > 0 { metadata.Description = descriptions[0] } // Publisher if publishers, err := book.MetadataByKey("publisher"); err == nil && len(publishers) > 0 { metadata.Publisher = publishers[0] } // Series and series number (Calibre specific metadata) if series, err := book.MetadataByKey("calibre:series"); err == nil && len(series) > 0 { metadata.Series = series[0] } if seriesIndex, err := book.MetadataByKey("calibre:series_index"); err == nil && len(seriesIndex) > 0 { if index, err := strconv.ParseFloat(seriesIndex[0], 32); err == nil { metadata.SeriesNumber = int32(index) } } // Publish date if dates, err := book.MetadataByKey("date"); err == nil && len(dates) > 0 { if date, err := time.Parse("2006-01-02", dates[0]); err == nil { metadata.PublishDate = date } else { // Try alternative date formats if date, err := time.Parse("2006", dates[0]); err == nil { metadata.PublishDate = date } } } // Contributors if contributors, err := book.MetadataByKey("contributor"); err == nil && len(contributors) > 0 { // Normalize contributors for display metadata.Contributors = utils.NormalizeContributors(contributors) } // ISBN if isbns, err := book.MetadataByKey("identifier"); err == nil && len(isbns) > 0 { for _, isbn := range isbns { if strings.Contains(strings.ToLower(isbn), "isbn") { // Extract ISBN number from identifier like "isbn:978-3-16-148410-0" isbnParts := strings.SplitN(isbn, ":", 2) if len(isbnParts) == 2 { metadata.ISBN = isbnParts[1] break } } if strings.Contains(strings.ToLower(isbn), "asin") { // Extract ASIN from identifier like "asin:B08XXXXX" asinParts := strings.SplitN(isbn, ":", 2) if len(asinParts) == 2 { metadata.ASIN = asinParts[1] } } } } // Tags if tags, err := book.MetadataByKey("subject"); err == nil && len(tags) > 0 { // Normalize tags for display metadata.Tags = utils.NormalizeTags(tags) } return metadata, nil } // DetectFixedLayoutEPUB checks if EPUB has fixed-layout (manga) characteristics // by examining the OPF file for rendition metadata and content indicators func (s *MediaScanner) DetectFixedLayoutEPUB(epubPath string) (bool, error) { // Open EPUB ZIP file r, err := zip.OpenReader(epubPath) if err != nil { return false, fmt.Errorf("failed to open EPUB: %w", err) } defer func() { if err := r.Close(); err != nil { fmt.Printf("Warning: failed to close EPUB file %s: %v\n", epubPath, err) } }() // Find and read OPF file var opfFile *zip.File for _, f := range r.File { if strings.HasSuffix(f.Name, ".opf") { opfFile = f break } // Also check in META-INF directory if strings.Contains(f.Name, "META-INF/") && strings.HasSuffix(f.Name, ".opf") { opfFile = f break } } if opfFile == nil { return false, fmt.Errorf("OPF file not found in EPUB") } // Read OPF content rc, err := opfFile.Open() if err != nil { return false, fmt.Errorf("failed to open OPF: %w", err) } defer func() { if err := rc.Close(); err != nil { fmt.Printf("Warning: failed to close OPF file reader from %s: %v\n", epubPath, err) } }() opfContent, err := io.ReadAll(rc) if err != nil { return false, fmt.Errorf("failed to read OPF: %w", err) } // Check for fixed-layout indicators opfString := string(opfContent) // Check 1: rendition:layout = pre-paginated (EPUB 3 fixed layout) if strings.Contains(opfString, `rendition:layout">pre-paginated<`) || strings.Contains(opfString, `rendition:layout="pre-paginated"`) { return true, nil } // Check 2: RTL page progression (manga indicator) if strings.Contains(opfString, `page-progression-direction="rtl"`) { return true, nil } // Check 3: Image-heavy content (count tags) // Threshold of 50 images suggests manga/comic vs novel imgCount := strings.Count(opfString, ` 50 { return true, nil } // Check 4: Manga subject tag lowerOPF := strings.ToLower(opfString) if strings.Contains(lowerOPF, ` 0 { metadata.Title = opf.Metadata.Titles[0] } // Author (first creator) if len(opf.Metadata.Creators) > 0 { metadata.Author = opf.Metadata.Creators[0] } // Tags (all subjects) if len(opf.Metadata.Subjects) > 0 { metadata.Tags = utils.NormalizeTags(opf.Metadata.Subjects) } // Description if len(opf.Metadata.Desc) > 0 { metadata.Description = opf.Metadata.Desc[0] } // Publisher if len(opf.Metadata.Publisher) > 0 { metadata.Publisher = opf.Metadata.Publisher[0] } // Publish date if len(opf.Metadata.Dates) > 0 { if date, err := time.Parse("2006-01-02T15:04:05Z07:00", opf.Metadata.Dates[0]); err == nil { metadata.PublishDate = date } else if date, err := time.Parse("2006-01-02", opf.Metadata.Dates[0]); err == nil { metadata.PublishDate = date } else { // Try alternative date formats if date, err := time.Parse("2006", opf.Metadata.Dates[0]); err == nil { metadata.PublishDate = date } } } // Identifiers (ISBN, ASIN) for _, id := range opf.Metadata.Identifiers { switch strings.ToUpper(id.Scheme) { case "ISBN": metadata.ISBN = utils.NormalizeISBNSafe(id.Value) case "ASIN": metadata.ASIN = id.Value case "UUID", "CALIBRE": // Store UUID in hash info, not metadata // Will be extracted by extractHashInfo() } } // Contributors if len(opf.Metadata.Contributors) > 0 { metadata.Contributors = utils.NormalizeContributors(opf.Metadata.Contributors) } // Calibre-specific meta tags (filter by name attribute) for _, meta := range opf.Metadata.MetaTags { switch meta.Name { case "calibre:series": metadata.Series = meta.Value case "calibre:series_index": if index, err := strconv.ParseFloat(meta.Value, 32); err == nil { metadata.SeriesNumber = int32(index) } case "calibre:rating": // Not imported (ratings are per-user in Bookhoard) case "calibre:title_sort": // Not imported (Bookhoard has its own sorting logic) case "calibre:timestamp": // Could be used for created_at, but skipping for now } } return metadata, nil } // extractEPUBCover extracts the cover image from an EPUB file. // It looks for: // 1. An item with properties="cover-image" in the manifest // 2. A meta tag with name="cover" pointing to an image // 3. Common cover image paths like cover.jpg, cover.jpeg, cover.png // Returns the path to the saved cover image, or empty string if no cover found. func (s *MediaScanner) extractEPUBCover(epubPath string) (string, error) { // Open the EPUB as a zip file to extract images r, err := zip.OpenReader(epubPath) if err != nil { return "", fmt.Errorf("failed to open EPUB as zip: %v", err) } defer func() { if err := r.Close(); err != nil { fmt.Printf("Warning: failed to close EPUB zip reader for %s: %v\n", epubPath, err) } }() // Try to find cover image from OPF metadata coverImageName := "" // Attempt to read the OPF file to find cover reference // First, find container.xml to locate the OPF var opfPath string for _, f := range r.File { if f.Name == "META-INF/container.xml" { rc, err := f.Open() if err != nil { continue } content, readErr := io.ReadAll(rc) if closeErr := rc.Close(); closeErr != nil { fmt.Printf("Warning: failed to close META-INF/container.xml reader in %s: %v\n", epubPath, closeErr) } if readErr != nil { continue } // Parse container.xml to find OPF path // Simple string search since we just need the path opfStart := bytes.Index(content, []byte("]*properties="[^"]*cover-image[^"]*"[^>]*id="([^"]+)"`) matches := coverImageRE.FindStringSubmatch(contentStr) if len(matches) > 1 { coverID := matches[1] // Find the href for this ID hrefRE := regexp.MustCompile(fmt.Sprintf(`]*id="%s"[^>]*href="([^"]+)"`, coverID)) hrefMatches := hrefRE.FindStringSubmatch(contentStr) if len(hrefMatches) > 1 { return resolveOPFPath(opfDir, hrefMatches[1]) } } // Look for meta name="cover" metaCoverRE := regexp.MustCompile(`]*name="cover"[^>]*content="([^"]+)"`) metaMatches := metaCoverRE.FindStringSubmatch(contentStr) if len(metaMatches) > 1 { coverContent := metaMatches[1] // Could be "image-id" format if coverID, found := strings.CutPrefix(coverContent, "image-"); found { hrefRE := regexp.MustCompile(fmt.Sprintf(`]*id="%s"[^>]*href="([^"]+)"`, coverID)) hrefMatches := hrefRE.FindStringSubmatch(contentStr) if len(hrefMatches) > 1 { return resolveOPFPath(opfDir, hrefMatches[1]) } } } // Fall back to searching common paths return findCoverImageInZip(files) } // resolveOPFPath resolves a relative path against the OPF directory func resolveOPFPath(opfDir, href string) string { if opfDir == "" { return href } // Handle ../ in href if strings.HasPrefix(href, "../") { // Simple case: just use the href as-is for now return href } // Join the directory with the href return filepath.Join(filepath.Dir(opfDir), href) } // readFileFromZip reads a file from the zip by name func readFileFromZip(files []*zip.File, name string) ([]byte, error) { // Normalize the name for comparison name = filepath.ToSlash(name) for _, f := range files { fName := filepath.ToSlash(f.Name) if fName == name { rc, err := f.Open() if err != nil { return nil, err } defer func() { if err := rc.Close(); err != nil { fmt.Printf("Warning: failed to close zip file reader for %s: %v\n", name, err) } }() return io.ReadAll(rc) } } return nil, fmt.Errorf("file not found: %s", name) } // extractImageFromZip extracts an image file and returns its contents func extractImageFromZip(files []*zip.File, imagePath, opfDir string) ([]byte, error) { // Try direct match first for _, f := range files { if strings.EqualFold(f.Name, imagePath) { rc, err := f.Open() if err != nil { return nil, err } defer func() { if err := rc.Close(); err != nil { fmt.Printf("Warning: failed to close zip image reader for %s: %v\n", imagePath, err) } }() return io.ReadAll(rc) } } // Try resolved path resolvedPath := resolveOPFPath(opfDir, imagePath) for _, f := range files { if strings.EqualFold(f.Name, resolvedPath) { rc, err := f.Open() if err != nil { return nil, err } defer func() { if err := rc.Close(); err != nil { fmt.Printf("Warning: failed to close zip image reader for resolved path %s: %v\n", resolvedPath, err) } }() return io.ReadAll(rc) } } return nil, fmt.Errorf("image not found: %s", imagePath) } // findSidecarCover looks for cover images in the same directory as the media file. // It checks for common cover filename patterns in priority order: // 1. cover.jpg, cover.jpeg, cover.png, cover.webp // 2. folder.jpg, folder.jpeg, folder.png, folder.webp // 3. {basename}.jpg, {basename}.jpeg, etc. (same name as media file) // 4. .folder.jpg (hidden file) // Returns the full path to the cover file, or empty string if not found. func findSidecarCover(mediaPath string) string { dir := filepath.Dir(mediaPath) baseName := strings.TrimSuffix(filepath.Base(mediaPath), filepath.Ext(mediaPath)) // Priority order for cover filenames coverPatterns := []string{ "cover.jpg", "cover.jpeg", "cover.png", "cover.webp", "folder.jpg", "folder.jpeg", "folder.png", "folder.webp", ".folder.jpg", ".folder.jpeg", ".folder.png", } // First, check exact match cover/folder names for _, coverName := range coverPatterns { coverPath := filepath.Join(dir, coverName) if _, err := os.Stat(coverPath); err == nil { return coverPath } } // Second, check for {basename}.{ext} pattern extensions := []string{".jpg", ".jpeg", ".png", ".webp"} for _, ext := range extensions { coverPath := filepath.Join(dir, baseName+ext) if _, err := os.Stat(coverPath); err == nil { return coverPath } // Also check uppercase extension coverPathUpper := filepath.Join(dir, baseName+strings.ToUpper(ext)) if _, err := os.Stat(coverPathUpper); err == nil { return coverPathUpper } } return "" } func (s *MediaScanner) extractPDFMetadata(path string) (*MediaMetadata, error) { metadata := &MediaMetadata{} // Open PDF file for reading metadata f, err := os.Open(path) if err != nil { fmt.Printf("Warning: failed to open PDF file %s: %v\n", path, err) metadata.Title = strings.TrimSuffix(filepath.Base(path), ".pdf") return metadata, nil } defer func() { if err := f.Close(); err != nil { fmt.Printf("Warning: failed to close PDF file %s: %v\n", path, err) } }() // Use pdfcpu API to read PDF metadata // Configuration: nil = default (lenient mode) pdfInfo, err := pdfcpuapi.PDFInfo(f, filepath.Base(path), nil, false, nil) if err != nil { fmt.Printf("Warning: failed to read PDF info from %s: %v\n", path, err) // Fall back to filename as title metadata.Title = strings.TrimSuffix(filepath.Base(path), ".pdf") return metadata, nil } // Extract title if pdfInfo.Title != "" { metadata.Title = pdfInfo.Title } else { metadata.Title = strings.TrimSuffix(filepath.Base(path), ".pdf") } // Extract author if pdfInfo.Author != "" { metadata.Author = pdfInfo.Author } // Extract subject (use as description) if pdfInfo.Subject != "" { metadata.Description = pdfInfo.Subject } // Extract creator (use as author fallback) if pdfInfo.Creator != "" && metadata.Author == "" { metadata.Author = pdfInfo.Creator } // Extract producer (use as publisher) if pdfInfo.Producer != "" { metadata.Publisher = pdfInfo.Producer } metadata.PageCount = int32(pdfInfo.PageCount) // Try to extract cover image coverPath, err := s.extractPDFCover(path) if err != nil { fmt.Printf("Warning: failed to extract PDF cover from %s: %v\n", path, err) } else if coverPath != "" { metadata.CoverPath = s.getRelativePath(coverPath) } // If no embedded cover, try sidecar if metadata.CoverPath == "" { sidecarCover := findSidecarCover(path) if sidecarCover != "" { metadata.CoverPath = s.getRelativePath(sidecarCover) } } return metadata, nil } // extractPDFCover extracts a cover image from a PDF file. // It uses pdfcpu API to extract images from the first page. // Returns the path to the saved cover, or empty string if no cover found. func (s *MediaScanner) extractPDFCover(pdfPath string) (string, error) { // Create a temporary directory for extracted images tmpDir, err := os.MkdirTemp("", "pdf-cover-") if err != nil { return "", fmt.Errorf("failed to create temp dir: %v", err) } defer func() { if err := os.RemoveAll(tmpDir); err != nil { fmt.Printf("Warning: failed to remove temp directory %s: %v\n", tmpDir, err) } }() // Use pdfcpu API to extract images from first page // ExtractImagesFile(inFile, outDir string, selectedPages []string, conf *model.Configuration) error err = pdfcpuapi.ExtractImagesFile(pdfPath, tmpDir, []string{"1"}, nil) if err != nil { // No images found or extraction failed - this is OK, just return empty return "", nil } // Check for extracted images in the temp directory entries, err := os.ReadDir(tmpDir) if err != nil || len(entries) == 0 { return "", nil } // Find the largest image (likely the cover) var largestImage string var largestSize int64 for _, entry := range entries { if entry.IsDir() { continue } info, err := entry.Info() if err != nil { continue } // Skip very small files (likely thumbnails or icons) if info.Size() < 1000 { continue } if info.Size() > largestSize { largestImage = filepath.Join(tmpDir, entry.Name()) largestSize = info.Size() } } if largestImage == "" { return "", nil } // Read the image imageData, err := os.ReadFile(largestImage) if err != nil || len(imageData) == 0 { return "", nil } // Save cover to disk (same pattern as comics: {pdf_path}.cover.jpg) coverPath := pdfPath + ".cover.jpg" if err := os.WriteFile(coverPath, imageData, 0644); err != nil { return "", fmt.Errorf("failed to write cover file: %v", err) } return coverPath, nil } // ComicInfo represents metadata from ComicInfo.xml type ComicInfo struct { XMLName xml.Name `xml:"ComicInfo"` Title string `xml:"Title"` Series string `xml:"Series"` Number int `xml:"Number"` Volume int `xml:"Volume"` Publisher string `xml:"Publisher"` Year int `xml:"Year"` Month int `xml:"Month"` Day int `xml:"Day"` Writer string `xml:"Writer"` Penciller string `xml:"Penciller"` Inker string `xml:"Inker"` Colorist string `xml:"Colorist"` Letterer string `xml:"Letterer"` CoverArtist string `xml:"CoverArtist"` Genre string `xml:"Genre"` Tags string `xml:"Tags"` Web string `xml:"Web"` Notes string `xml:"Notes"` // NEW: Reading direction fields from ComicInfo.xml v2.0 Manga string `xml:"Manga"` // Unknown, No, Yes, YesAndRightToLeft LanguageISO string `xml:"LanguageISO"` // ISO 639-1 language code for heuristics // NEW: Additional comic-specific fields (19 total fields from ComicInfo.xml) Count int `xml:"Count"` // Total issues in series AlternateSeries string `xml:"AlternateSeries"` AlternateNumber int `xml:"AlternateNumber"` AlternateCount int `xml:"AlternateCount"` Summary string `xml:"Summary"` Imprint string `xml:"Imprint"` StoryArc string `xml:"StoryArc"` SeriesGroup string `xml:"SeriesGroup"` AgeRating string `xml:"AgeRating"` CommunityRating float64 `xml:"CommunityRating"` MainCharacterOrTeam string `xml:"MainCharacterOrTeam"` Review string `xml:"Review"` BlackAndWhite string `xml:"BlackAndWhite"` // "Yes" or "No" ScanInformation string `xml:"ScanInformation"` Characters string `xml:"Characters"` Teams string `xml:"Teams"` Locations string `xml:"Locations"` } // extractComicMetadata extracts metadata from comic archive (supports .cbz, .cbr, .cb7, .cbt) func extractComicMetadata(filePath string) (*ComicInfo, []byte, error) { ext := strings.ToLower(filepath.Ext(filePath)) switch ext { case ".cbz": return extractZipMetadata(filePath) case ".cbr": return extractRarMetadata(filePath) case ".cb7": return extract7ZipMetadata(filePath) case ".cbt": return extractTarMetadata(filePath) default: return nil, nil, fmt.Errorf("unsupported comic format: %s", ext) } } // archiveFile represents a file in an archive for uniform handling type archiveFile interface { Name() string Open() (io.ReadCloser, error) } // extractMetadataFromArchive extracts ComicInfo.xml and cover image from any archive format func extractMetadataFromArchive(files []archiveFile) (*ComicInfo, []byte, error) { var comicInfo *ComicInfo var coverImage []byte for _, f := range files { if f.Name() == "ComicInfo.xml" { rc, err := f.Open() if err != nil { return nil, nil, fmt.Errorf("failed to open ComicInfo.xml: %w", err) } data, err := io.ReadAll(rc) if closeErr := rc.Close(); closeErr != nil { fmt.Printf("Warning: failed to close ComicInfo.xml reader: %v\n", closeErr) } if err != nil { return nil, nil, fmt.Errorf("failed to read ComicInfo.xml: %w", err) } comicInfo = &ComicInfo{} if err := xml.Unmarshal(data, comicInfo); err != nil { return nil, nil, fmt.Errorf("failed to parse ComicInfo.xml: %w", err) } } if coverImage == nil && isImageFile(f.Name()) { if !strings.Contains(filepath.Dir(f.Name()), string(filepath.Separator)) || filepath.Dir(f.Name()) == "." { rc, err := f.Open() if err != nil { continue } coverImage, err = io.ReadAll(rc) if closeErr := rc.Close(); closeErr != nil { fmt.Printf("Warning: failed to close cover image reader for %s: %v\n", f.Name(), closeErr) } if err == nil { _, _, err = image.Decode(bytes.NewReader(coverImage)) if err != nil { coverImage = nil } } } } } if comicInfo == nil { comicInfo = &ComicInfo{} } return comicInfo, coverImage, nil } // extractZipMetadata extracts metadata from ZIP archives (.cbz) func extractZipMetadata(filePath string) (*ComicInfo, []byte, error) { r, err := zip.OpenReader(filePath) if err != nil { return nil, nil, fmt.Errorf("failed to open ZIP archive: %w", err) } defer func() { if err := r.Close(); err != nil { fmt.Printf("Warning: failed to close ZIP archive reader for %s: %v\n", filePath, err) } }() files := make([]archiveFile, 0, len(r.File)) for _, f := range r.File { files = append(files, &zipFileAdapter{f}) } comicInfo, coverImage, err := extractMetadataFromArchive(files) if err != nil { return nil, nil, err } if comicInfo.Title == "" { basename := filepath.Base(filePath) comicInfo.Title = strings.TrimSuffix(basename, filepath.Ext(basename)) } return comicInfo, coverImage, nil } // zipFileAdapter adapts zip.File to archiveFile interface type zipFileAdapter struct { *zip.File } func (z *zipFileAdapter) Name() string { return z.File.Name } func (z *zipFileAdapter) Open() (io.ReadCloser, error) { return z.File.Open() } // extractRarMetadata extracts metadata from RAR archives (.cbr) func extractRarMetadata(filePath string) (*ComicInfo, []byte, error) { r, err := rardecode.OpenReader(filePath, "") if err != nil { return nil, nil, fmt.Errorf("failed to open RAR archive: %w", err) } defer func() { if err := r.Close(); err != nil { fmt.Printf("Warning: failed to close RAR archive reader for %s: %v\n", filePath, err) } }() files := make([]archiveFile, 0, 100) for { header, err := r.Next() if err == io.EOF { break } if err != nil { return nil, nil, fmt.Errorf("failed to read RAR entry: %w", err) } data, err := io.ReadAll(r) if err != nil { continue } files = append(files, &rarFileAdapter{ name: header.Name, data: data, }) } comicInfo, coverImage, err := extractMetadataFromArchive(files) if err != nil { return nil, nil, err } if comicInfo.Title == "" { basename := filepath.Base(filePath) comicInfo.Title = strings.TrimSuffix(basename, filepath.Ext(basename)) } return comicInfo, coverImage, nil } // rarFileAdapter stores RAR file data in memory type rarFileAdapter struct { name string data []byte } func (r *rarFileAdapter) Name() string { return r.name } func (r *rarFileAdapter) Open() (io.ReadCloser, error) { return io.NopCloser(bytes.NewReader(r.data)), nil } // extract7ZipMetadata extracts metadata from 7-Zip archives (.cb7) func extract7ZipMetadata(filePath string) (*ComicInfo, []byte, error) { r, err := sevenzip.OpenReader(filePath) if err != nil { return nil, nil, fmt.Errorf("failed to open 7-Zip archive: %w", err) } defer func() { if err := r.Close(); err != nil { fmt.Printf("Warning: failed to close 7-Zip archive reader for %s: %v\n", filePath, err) } }() files := make([]archiveFile, 0, len(r.File)) for _, f := range r.File { files = append(files, &sevenZipFileAdapter{file: f}) } comicInfo, coverImage, err := extractMetadataFromArchive(files) if err != nil { return nil, nil, err } if comicInfo.Title == "" { basename := filepath.Base(filePath) comicInfo.Title = strings.TrimSuffix(basename, filepath.Ext(basename)) } return comicInfo, coverImage, nil } // sevenZipFileAdapter adapts sevenzip.File to archiveFile interface type sevenZipFileAdapter struct { file *sevenzip.File } func (s *sevenZipFileAdapter) Name() string { return s.file.Name } func (s *sevenZipFileAdapter) Open() (io.ReadCloser, error) { return s.file.Open() } // extractTarMetadata extracts metadata from TAR archives (.cbt) func extractTarMetadata(filePath string) (*ComicInfo, []byte, error) { f, err := os.Open(filePath) if err != nil { return nil, nil, fmt.Errorf("failed to open TAR archive: %w", err) } defer func() { if err := f.Close(); err != nil { fmt.Printf("Warning: failed to close TAR archive file %s: %v\n", filePath, err) } }() var tarReader *tar.Reader if strings.HasSuffix(strings.ToLower(filePath), ".tar.gz") || strings.HasSuffix(strings.ToLower(filePath), ".tgz") { gzReader, err := gzip.NewReader(f) if err != nil { return nil, nil, fmt.Errorf("failed to create gzip reader: %w", err) } defer func() { if err := gzReader.Close(); err != nil { fmt.Printf("Warning: failed to close gzip reader for %s: %v\n", filePath, err) } }() tarReader = tar.NewReader(gzReader) } else if strings.HasSuffix(strings.ToLower(filePath), ".tar.bz2") || strings.HasSuffix(strings.ToLower(filePath), ".tbz2") { bz2Reader := bzip2.NewReader(f) tarReader = tar.NewReader(bz2Reader) } else { tarReader = tar.NewReader(f) } files := make([]archiveFile, 0, 100) for { header, err := tarReader.Next() if err == io.EOF { break } if err != nil { return nil, nil, fmt.Errorf("failed to read TAR entry: %w", err) } if header.Typeflag == tar.TypeReg { data, err := io.ReadAll(tarReader) if err != nil { continue } files = append(files, &tarFileAdapter{ name: header.Name, data: data, }) } } comicInfo, coverImage, err := extractMetadataFromArchive(files) if err != nil { return nil, nil, err } if comicInfo.Title == "" { basename := filepath.Base(filePath) comicInfo.Title = strings.TrimSuffix(basename, filepath.Ext(basename)) } return comicInfo, coverImage, nil } // tarFileAdapter stores TAR file data in memory type tarFileAdapter struct { name string data []byte } func (t *tarFileAdapter) Name() string { return t.name } func (t *tarFileAdapter) Open() (io.ReadCloser, error) { return io.NopCloser(bytes.NewReader(t.data)), nil } // isImageFile checks if a file is an image based on extension func isImageFile(filename string) bool { ext := strings.ToLower(filepath.Ext(filename)) switch ext { case ".jpg", ".jpeg", ".png", ".gif", ".webp", ".bmp", ".avif", ".tiff", ".tif": return true } return false } // countArchiveImages counts image files in a comic archive func countArchiveImages(filePath string) (int, error) { ext := strings.ToLower(filepath.Ext(filePath)) count := 0 switch ext { case ".cbz", ".epub": r, err := zip.OpenReader(filePath) if err != nil { return 0, err } defer r.Close() for _, f := range r.File { if !f.FileInfo().IsDir() && isImageFile(f.Name) { count++ } } case ".cbr": r, err := rardecode.OpenReader(filePath, "") if err != nil { return 0, err } defer r.Close() for { header, err := r.Next() if err == io.EOF { break } if err != nil { break } if !header.IsDir && isImageFile(header.Name) { count++ } } case ".cb7": sz, err := sevenzip.OpenReader(filePath) if err != nil { return 0, err } defer sz.Close() for _, f := range sz.File { if !f.FileInfo().IsDir() && isImageFile(f.Name) { count++ } } case ".cbt": f, err := os.Open(filePath) if err != nil { return 0, err } defer f.Close() tr := tar.NewReader(f) for { header, err := tr.Next() if err == io.EOF { break } if err != nil { break } if !header.FileInfo().IsDir() && isImageFile(header.Name) { count++ } } } return count, nil } func (s *MediaScanner) updateMediaItem(ctx context.Context, mediaItemID pgtype.UUID, path string, _ os.FileInfo) error { // Re-extract metadata for the update metadata, err := s.extractMetadata(path) if err != nil { fmt.Printf("Warning: failed to extract metadata for force rescan %s: %v\n", path, err) metadata = &MediaMetadata{} } // Normalize metadata fields metadata.Contributors = utils.NormalizeContributors(metadata.Contributors) metadata.Tags = utils.NormalizeTags(metadata.Tags) contributorsSearch := utils.NormalizeContributorsSearch(metadata.Contributors) tagsSearch := utils.NormalizeTagsSearch(metadata.Tags) // Call the database update - only update fields available in MediaMetadata var alternateInfoBytes []byte if metadata.AlternateInfo != "" { alternateInfoBytes = []byte(metadata.AlternateInfo) } _, err = s.db.UpdateMediaItem(ctx, database.UpdateMediaItemParams{ ID: mediaItemID, Title: metadata.Title, Author: pgtype.Text{String: metadata.Author, Valid: metadata.Author != ""}, Isbn: pgtype.Text{String: utils.NormalizeISBNSafe(metadata.ISBN), Valid: metadata.ISBN != ""}, Description: pgtype.Text{String: metadata.Description, Valid: metadata.Description != ""}, CoverImagePath: pgtype.Text{String: s.getRelativePath(metadata.CoverPath), Valid: metadata.CoverPath != ""}, Series: pgtype.Text{String: metadata.Series, Valid: metadata.Series != ""}, SeriesNumber: pgtype.Int4{Int32: metadata.SeriesNumber, Valid: metadata.SeriesNumber > 0}, Tags: metadata.Tags, TagsSearch: tagsSearch, Asin: pgtype.Text{String: metadata.ASIN, Valid: metadata.ASIN != ""}, DatePublished: pgtype.Date{Time: metadata.PublishDate, Valid: !metadata.PublishDate.IsZero()}, Publisher: pgtype.Text{String: metadata.Publisher, Valid: metadata.Publisher != ""}, Contributors: metadata.Contributors, ContributorsSearch: contributorsSearch, Language: pgtype.Text{String: metadata.Language, Valid: metadata.Language != ""}, Genre: pgtype.Text{String: metadata.Genre, Valid: metadata.Genre != ""}, PageCount: pgtype.Int4{Int32: metadata.PageCount, Valid: metadata.PageCount > 0}, MangaType: pgtype.Text{String: metadata.MangaType, Valid: metadata.MangaType != ""}, ReadingDirection: pgtype.Text{String: metadata.ReadingDirection, Valid: metadata.ReadingDirection != ""}, SeriesCount: pgtype.Int4{Int32: metadata.SeriesCount, Valid: metadata.SeriesCount > 0}, Volume: pgtype.Int4{Int32: metadata.Volume, Valid: metadata.Volume > 0}, Imprint: pgtype.Text{String: metadata.Imprint, Valid: metadata.Imprint != ""}, AgeRating: pgtype.Text{String: metadata.AgeRating, Valid: metadata.AgeRating != ""}, WebUrl: pgtype.Text{String: metadata.WebURL, Valid: metadata.WebURL != ""}, MetadataNotes: pgtype.Text{String: metadata.MetadataNotes, Valid: metadata.MetadataNotes != ""}, CommunityRating: pgtype.Float8{Float64: metadata.CommunityRating, Valid: metadata.CommunityRating > 0}, StoryArc: pgtype.Text{String: metadata.StoryArc, Valid: metadata.StoryArc != ""}, IsBlackAndWhite: pgtype.Bool{Bool: metadata.IsBlackAndWhite, Valid: metadata.IsBlackAndWhite}, AlternateInfo: alternateInfoBytes, ScanInformation: pgtype.Text{String: metadata.ScanInformation, Valid: metadata.ScanInformation != ""}, Summary: pgtype.Text{String: metadata.Summary, Valid: metadata.Summary != ""}, }) return err } func (s *MediaScanner) getMediaItemByFilePath(ctx context.Context, filePath string, libraryID pgtype.UUID) (database.MediaItems, error) { return s.db.GetMediaItemByFilePath(ctx, database.GetMediaItemByFilePathParams{ FilePath: s.getRelativePath(filePath), LibraryID: libraryID, }) } func (s *MediaScanner) getMimeType(path string) string { ext := strings.ToLower(filepath.Ext(path)) if mime, ok := MimeTypes[ext]; ok { return mime } return "application/octet-stream" } func (s *MediaScanner) WatchChanges(ctx context.Context) error { if !s.watching.CompareAndSwap(false, true) { return fmt.Errorf("already watching") } go func() { <-ctx.Done() s.watching.Store(false) }() go s.performInitialScan(ctx) go s.processDirtyDirectories(ctx) go s.startBackupScan(ctx) go func() { fmt.Printf("[WATCHER] Event loop started for %d folders\n", len(s.folders)) for { select { case event, ok := <-s.watcher.Events: if !ok { fmt.Printf("[WATCHER] Event channel closed\n") return } if event.Has(fsnotify.Create) { if info, err := os.Stat(event.Name); err == nil && info.IsDir() { if err := s.watcher.Add(event.Name); err != nil { fmt.Printf("[WATCHER] Warning: failed to watch new directory %s: %v\n", event.Name, err) } else { fmt.Printf("[WATCHER] Now watching new directory: %s\n", event.Name) } } } if event.Has(fsnotify.Create | fsnotify.Write | fsnotify.Remove | fsnotify.Rename) { fmt.Printf("[WATCHER] Event: %s on %s\n", event.Op, event.Name) s.markDirectoryDirty(filepath.Dir(event.Name)) } case err, ok := <-s.watcher.Errors: if !ok { fmt.Printf("[WATCHER] Error channel closed\n") return } fmt.Printf("[WATCHER] Error: %v\n", err) case <-ctx.Done(): fmt.Printf("[WATCHER] Event loop stopped\n") return } } }() return nil } func (s *MediaScanner) markDirectoryDirty(dirPath string) { s.dirtyDirsMu.Lock() defer s.dirtyDirsMu.Unlock() // Only mark if within watched folders var isWatched bool for _, folder := range s.folders { if strings.HasPrefix(dirPath, folder) { isWatched = true break } } if !isWatched { return } // Smart event merging (Jellyfin approach): // 1. If parent dir exists, replace with parent (consolidate) // 2. If sibling dirs exist, replace with common parent // 3. Otherwise, add this dir // Check if parent directory is already dirty parentDir := filepath.Dir(dirPath) if parentDir != dirPath { // Not at root if _, parentExists := s.dirtyDirs[parentDir]; parentExists { // Parent already being watched, reset its timestamp s.dirtyDirs[parentDir] = time.Now() return } } // Check if any subdirectories are dirty, replace with parent for existingDir := range s.dirtyDirs { if strings.HasPrefix(existingDir, dirPath+"/") { // This is a subdirectory, replace it with parent delete(s.dirtyDirs, existingDir) } } // NEW: Check for sibling directories and consolidate to parent parentDir = filepath.Dir(dirPath) for existingDir := range s.dirtyDirs { existingParent := filepath.Dir(existingDir) if existingParent == parentDir && existingParent != dirPath && existingParent != "." { // Found a sibling! Both should be replaced with parent delete(s.dirtyDirs, existingDir) s.dirtyDirs[parentDir] = time.Now() return } } // Add/update this directory s.dirtyDirs[dirPath] = time.Now() } func (s *MediaScanner) processDirtyDirectories(ctx context.Context) { ticker := time.NewTicker(1 * time.Second) defer ticker.Stop() for { select { case <-ctx.Done(): return case <-ticker.C: s.dirtyDirsMu.Lock() now := time.Now() readyDirs := make([]string, 0) for dirPath, lastChange := range s.dirtyDirs { if now.Sub(lastChange) >= 10*time.Second { readyDirs = append(readyDirs, dirPath) delete(s.dirtyDirs, dirPath) } } s.dirtyDirsMu.Unlock() if len(readyDirs) == 0 { continue } affectedRoots := make(map[string]bool) for _, dirPath := range readyDirs { for _, folder := range s.folders { if strings.HasPrefix(dirPath, folder) { affectedRoots[folder] = true break } } } for rootFolder := range affectedRoots { s.enqueueLibraryScan(rootFolder) } } } } // waitForFileStability checks if a file's mtime has stabilized // Returns true when file is stable (not being modified) // Polls every 3 seconds, times out after 60 seconds // Uses atomic.Bool to prevent race conditions with concurrent checks func (s *MediaScanner) waitForFileStability(filePath string) bool { s.fileStabilityMu.Lock() // Check if already being checked (atomic.Bool prevents race condition) tracking, exists := s.fileStability[filePath] if exists { s.fileStabilityMu.Unlock() // Another goroutine is already checking this file if tracking.Load() { return false // Still being checked } // Tracking exists but completed, remove stale entry s.fileStabilityMu.Lock() delete(s.fileStability, filePath) } // Start tracking with atomic.Bool set to true (checking in progress) trackingFlag := &atomic.Bool{} trackingFlag.Store(true) s.fileStability[filePath] = trackingFlag s.fileStabilityMu.Unlock() // Get initial mtime info, err := os.Stat(filePath) if err != nil { // Clean up tracking entry if file doesn't exist s.fileStabilityMu.Lock() delete(s.fileStability, filePath) s.fileStabilityMu.Unlock() return false } lastMtime := info.ModTime() // Poll every 3 seconds for up to 60 seconds timeout := time.After(60 * time.Second) ticker := time.NewTicker(3 * time.Second) defer ticker.Stop() for { select { case <-timeout: // Timeout - mark as done and clean up trackingFlag.Store(false) s.fileStabilityMu.Lock() delete(s.fileStability, filePath) s.fileStabilityMu.Unlock() return false // File never stabilized case <-ticker.C: info, err := os.Stat(filePath) if err != nil { // File deleted - mark as done and clean up trackingFlag.Store(false) s.fileStabilityMu.Lock() delete(s.fileStability, filePath) s.fileStabilityMu.Unlock() return false } currentMtime := info.ModTime() if currentMtime.Equal(lastMtime) { // File is stable! Mark as done and clean up trackingFlag.Store(false) s.fileStabilityMu.Lock() delete(s.fileStability, filePath) s.fileStabilityMu.Unlock() return true } lastMtime = currentMtime } } } func (s *MediaScanner) scanDirectory(ctx context.Context, dirPath string) { // Prevent concurrent scans of ANY directory // Simple mutex is enough - job queue already serializes by directory s.scanMutex.Lock() defer s.scanMutex.Unlock() s.scanInProgress.Store(true) defer s.scanInProgress.Store(false) // Find library for this directory var libraryID pgtype.UUID var rootFolder string for _, folder := range s.folders { if strings.HasPrefix(dirPath, folder) { rootFolder = folder if lib, err := s.db.GetLibraryByFolderPathPrefix(ctx, dirPath); err == nil { libraryID = lib.LibraryID break } } } // Check if libraryID is valid before proceeding if !libraryID.Valid { return } // Walk directory and process new files (recurses into subdirectories) if err := filepath.WalkDir(dirPath, func(path string, d os.DirEntry, err error) error { if err != nil { return err } if d.IsDir() { return nil } if !s.isScannableFile(path) { return nil } // Check if file is stable before processing (Audiobookshelf approach) if !s.waitForFileStability(path) { return nil } relPath := strings.TrimPrefix(path, rootFolder+"/") _, err = s.db.GetMediaItemByFilePath(ctx, database.GetMediaItemByFilePathParams{ FilePath: relPath, LibraryID: libraryID, }) if errors.Is(err, pgx.ErrNoRows) { if _, err := s.processMediaFile(ctx, path); err != nil { s.errors++ } else { s.newItems++ } s.totalFiles++ } return nil }); err != nil { fmt.Printf("Warning: failed to walk directory %s: %v\n", dirPath, err) } } // performInitialScan scans all root folders on startup // This ensures existing files are detected before watching begins func (s *MediaScanner) performInitialScan(ctx context.Context) { fmt.Printf("Performing initial scan of root folders...\n") if s.defaultLibraryID.Valid && s.adminID.Valid { folderPaths := s.folders libraryIDStr := uuid.UUID(s.defaultLibraryID.Bytes).String() adminIDStr := uuid.UUID(s.adminID.Bytes).String() job := &Job{ ID: uuid.New().String(), Type: JobTypeScan, Status: JobStatusPending, UserID: adminIDStr, Context: context.Background(), Params: map[string]any{ "library_id": libraryIDStr, "folders": folderPaths, "admin_id": adminIDStr, "db": s.db, "force": false, }, } if WorkerInstance != nil { WorkerInstance.Enqueue(job) fmt.Printf("Enqueued initial library scan job\n") } else { fmt.Printf("Warning: Worker not initialized, skipping initial scan\n") } } else { fmt.Printf("Warning: no library/admin ID set, skipping initial scan\n") } fmt.Printf("Initial scan jobs enqueued\n") } func (s *MediaScanner) Close() error { fmt.Printf("Cleaning up scanner resources...\n") // Stop watching if s.watcher != nil { if err := s.watcher.Close(); err != nil { fmt.Printf("Warning: failed to close fsnotify watcher: %v\n", err) } } // Clean up fileStability map to prevent memory leaks s.fileStabilityMu.Lock() s.fileStability = make(map[string]*atomic.Bool) s.fileStabilityMu.Unlock() // Clear dirty directories s.dirtyDirsMu.Lock() s.dirtyDirs = make(map[string]time.Time) s.dirtyDirsMu.Unlock() // Wait for in-progress scan to complete (with timeout) timeout := time.After(5 * time.Second) done := make(chan struct{}) go func() { for s.scanInProgress.Load() { time.Sleep(100 * time.Millisecond) } close(done) }() select { case <-done: fmt.Printf("Scanner cleanup complete\n") case <-timeout: fmt.Printf("Timeout waiting for scan to complete\n") } return nil } func (s *MediaScanner) startBackupScan(ctx context.Context) { interval := s.GetPollInterval() if interval <= 0 { fmt.Println("[BACKUP-SCAN] Periodic scan disabled (interval = 0)") return } fmt.Printf("[BACKUP-SCAN] Periodic scan started with interval: %v\n", interval) for { ticker := time.NewTicker(interval) defer ticker.Stop() select { case <-ctx.Done(): fmt.Println("[BACKUP-SCAN] Periodic scan stopped") return case <-ticker.C: interval = s.GetPollInterval() if !s.GetAutoScanEnabled() { continue } fmt.Printf("[BACKUP-SCAN] Running periodic full scan (interval: %v)...\n", interval) for _, folder := range s.folders { s.enqueueLibraryScan(folder) } } } } func (s *MediaScanner) calculateFileSHA256(filePath string) (string, error) { file, err := os.Open(filePath) if err != nil { return "", fmt.Errorf("failed to open file: %v", err) } defer func() { if err := file.Close(); err != nil { fmt.Printf("Warning: failed to close file %s: %v\n", filePath, err) } }() hasher := sha256.New() if _, err := io.Copy(hasher, file); err != nil { return "", fmt.Errorf("failed to calculate hash: %v", err) } return hex.EncodeToString(hasher.Sum(nil)), nil } // OPFIdentifier represents an identifier from OPF metadata type OPFIdentifier struct { XMLName xml.Name `xml:"identifier"` ID string `xml:"id,attr"` Scheme string `xml:"scheme,attr"` Content string `xml:",chardata"` } // OPFMetadata represents parsed OPF metadata type OPFMetadata struct { XMLName xml.Name `xml:"package"` Version string `xml:"version,attr"` Identifiers []OPFIdentifier `xml:"metadata>identifier"` } // extractOPFIdentifiers extracts identifiers from EPUB OPF file func (s *MediaScanner) extractOPFIdentifiers(epubPath string) (opfIdentifier, opfUUID string, confidence string, err error) { book, err := epub.ReadBook(epubPath) if err != nil { return "", "", "", fmt.Errorf("failed to open EPUB: %v", err) } // Extract OPF identifiers using go-epub library identifiers, err := book.MetadataByKey("identifier") if err != nil || len(identifiers) == 0 { return "", "", "low", nil } var identifier, uuidString string for _, id := range identifiers { id = strings.TrimSpace(id) // Check for UUID format (urn:uuid:) if trimmed, found := strings.CutPrefix(strings.ToLower(id), "urn:uuid:"); found { uuidString = trimmed continue } // Check if it's a plain UUID (8-4-4-4-12 format) if isValidUUID(id) { uuidString = id continue } // Check for ISBN if strings.Contains(strings.ToLower(id), "isbn") { isbn := s.extractISBNFromIdentifier(id) if isbn != "" { identifier = isbn continue } } // Use first identifier as fallback if identifier == "" && id != "" { identifier = id } } confidence = s.determineHashConfidence(uuidString, identifier) return identifier, uuidString, confidence, nil } // isValidUUID checks if string is a valid UUID (8-4-4-4-12 format) func isValidUUID(idStr string) bool { uuidRegex := regexp.MustCompile(`^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$`) return uuidRegex.MatchString(idStr) } // extractISBNFromIdentifier extracts ISBN from identifier string func (s *MediaScanner) extractISBNFromIdentifier(id string) string { id = strings.TrimSpace(id) // Remove "isbn:" prefix if present if trimmed, found := strings.CutPrefix(strings.ToLower(id), "isbn:"); found { id = trimmed } // Remove hyphens and spaces isbn := regexp.MustCompile(`[\s-]`).ReplaceAllString(id, "") // Check if it's a valid ISBN-10 or ISBN-13 if len(isbn) == 10 || len(isbn) == 13 { return isbn } return "" } // determineHashConfidence determines confidence level based on available identifiers func (s *MediaScanner) determineHashConfidence(uuid, identifier string) string { if uuid != "" && isValidUUID(uuid) { return "high" } if identifier != "" && (strings.Contains(strings.ToLower(identifier), "isbn") || len(identifier) >= 10) { return "medium" } return "low" } // detectFormatType detects the format type based on file extension and content func (s *MediaScanner) detectFormatType(filePath string) string { base := strings.ToLower(filepath.Base(filePath)) // Check for compound extensions first (like .kepub.epub) if strings.HasSuffix(base, ".kepub.epub") { return "kepub" } ext := strings.ToLower(filepath.Ext(filePath)) switch ext { case ".epub": return "epub" case ".kepub": return "kepub" case ".pdf": return "pdf" case ".cbz", ".cbr", ".cb7", ".cbt": return "comic_archive" case ".mobi": return "mobi" case ".azw", ".azw3": return "kfx" case ".txt": return "txt" case ".fb2": return "fb2" default: return "unknown" } } // extractHashInfo calculates hash and extracts OPF identifiers for a file func (s *MediaScanner) extractHashInfo(filePath string) (*HashInfo, *FormatInfo, error) { // Calculate SHA-256 fileSHA256, err := s.calculateFileSHA256(filePath) if err != nil { return nil, nil, fmt.Errorf("failed to calculate SHA-256: %v", err) } // Get file info info, err := os.Stat(filePath) if err != nil { return nil, nil, fmt.Errorf("failed to get file info: %v", err) } // Extract OPF identifiers for EPUB files var opfIdentifier, opfUUID, confidence string if strings.HasSuffix(strings.ToLower(filePath), ".epub") { opfIdentifier, opfUUID, confidence, err = s.extractOPFIdentifiers(filePath) if err != nil { // Non-fatal error, continue with low confidence confidence = "low" } } hashInfo := &HashInfo{ FileSHA256: fileSHA256, OPFIdentifier: opfIdentifier, OPFUUID: opfUUID, HashConfidence: confidence, } formatInfo := &FormatInfo{ FormatType: s.detectFormatType(filePath), FilePath: filePath, FileSHA256: fileSHA256, FileSizeBytes: info.Size(), MimeType: s.getMimeType(filePath), } return hashInfo, formatInfo, nil } func (s *MediaScanner) getRelativePath(absolutePath string) string { // Get the base folder paths from scanner for _, baseFolder := range s.folders { // Check if path is within this base folder if relPath, ok := strings.CutPrefix(absolutePath, baseFolder); ok { return strings.TrimPrefix(relPath, "/") } } // Fallback: if no match, return as-is (shouldn't happen) return absolutePath }