package services import ( "archive/tar" "archive/zip" "bookhoard/internal/database" "bookhoard/internal/utils" "bytes" "compress/bzip2" "compress/gzip" "context" "crypto/sha256" "encoding/hex" "encoding/xml" "fmt" "image" _ "image/jpeg" _ "image/png" "io" "io/fs" "os" "path/filepath" "regexp" "strconv" "strings" "sync" "sync/atomic" "time" "bookhoard/internal/sevenzip" epub "github.com/ArcadiaLin/go-epub" "github.com/fsnotify/fsnotify" "github.com/google/uuid" "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgtype" "github.com/nwaples/rardecode" pdfcpuapi "github.com/pdfcpu/pdfcpu/pkg/api" ) // MediaMetadata contains extracted metadata for media files (ebooks, comics, manga) type MediaMetadata struct { Title string Author string Description string Series string SeriesNumber int32 Publisher string PublishDate time.Time Contributors []string CoverPath string ISBN string ASIN string Tags []string FileHashInfo *HashInfo FileFormats []*FormatInfo } type HashInfo struct { FileSHA256 string OPFIdentifier string OPFUUID string HashConfidence string } type FormatInfo struct { FormatType string FilePath string FileSHA256 string FileSizeBytes int64 MimeType string } // MediaScanner scans library folders for media files (ebooks, comics, manga) type MediaScanner struct { db *database.Queries watcher *fsnotify.Watcher folders []string adminID pgtype.UUID defaultLibraryID pgtype.UUID libraryTypes map[string][]string forceRescan bool logger *ScannerLogger dirtyDirs map[string]time.Time dirtyDirsMu sync.RWMutex fileStability map[string]*atomic.Bool fileStabilityMu sync.RWMutex scan_mutex sync.Mutex pollInterval time.Duration watching atomic.Bool settingsCache *SettingsCache totalFiles int newItems int errors int job *Job } // NewMediaScanner creates a new media scanner instance func NewMediaScanner(db *database.Queries) *MediaScanner { watcher, err := fsnotify.NewWatcher() if err != nil { panic(fmt.Sprintf("Failed to create file watcher: %v", err)) } return &MediaScanner{ db: db, watcher: watcher, settingsCache: NewSettingsCache(30 * time.Second), dirtyDirs: make(map[string]time.Time), fileStability: make(map[string]*atomic.Bool), pollInterval: 60 * time.Second, watching: atomic.Bool{}, folders: []string{}, adminID: pgtype.UUID{}, defaultLibraryID: pgtype.UUID{Valid: false}, libraryTypes: make(map[string][]string), logger: NewScannerLogger(), } } func (s *MediaScanner) GetPollInterval() time.Duration { // Check cache first if cached, ok := s.settingsCache.Get("scan_poll_interval_seconds"); ok { if seconds, err := strconv.Atoi(cached); err == nil { return time.Duration(seconds) * time.Second } } if s.db == nil { return 60 * time.Second } // Cache miss - query database ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() setting, err := s.db.GetSystemSetting(ctx, "scan_poll_interval_seconds") if err != nil || setting == "" { return 60 * time.Second } // Store in cache s.settingsCache.Set("scan_poll_interval_seconds", setting) // Convert to duration seconds, err := strconv.Atoi(setting) if err != nil { return 60 * time.Second } return time.Duration(seconds) * time.Second } func (s *MediaScanner) GetAutoScanEnabled() bool { // Check cache first if cached, ok := s.settingsCache.Get("auto_scan_enabled"); ok { return strings.ToLower(cached) == "true" } if s.db == nil { return true } // Cache miss - query database ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() setting, err := s.db.GetSystemSetting(ctx, "auto_scan_enabled") if err != nil || setting == "" { return true } // Store in cache s.settingsCache.Set("auto_scan_enabled", setting) return strings.ToLower(setting) == "true" } func (s *MediaScanner) SetAdminID(adminID pgtype.UUID) { s.adminID = adminID } func (s *MediaScanner) SetLibraryID(libraryID pgtype.UUID) { s.defaultLibraryID = libraryID } func (s *MediaScanner) SetForce(force bool) { s.forceRescan = force } func (s *MediaScanner) GetStats() (int, int, int) { return s.totalFiles, s.newItems, s.errors } func (s *MediaScanner) SetFolders(folders []string) error { s.folders = folders // Remove old watch if exists if s.watcher != nil { s.watcher.Close() } // Create new watcher watcher, err := fsnotify.NewWatcher() if err != nil { return fmt.Errorf("failed to create watcher: %v", err) } s.watcher = watcher // Build cache of allowed extensions per folder s.libraryTypes = make(map[string][]string) ctx := context.Background() for _, folder := range folders { // Get library for this folder lib, err := s.db.GetLibraryByFolder(ctx, folder) if err != nil { fmt.Printf("Warning: failed to get library for folder %s: %v\n", folder, err) continue } // Get library type with allowed extensions libType, err := s.db.GetLibraryType(ctx, lib.LibraryTypeID) if err != nil { fmt.Printf("Warning: failed to get library type for %s: %v\n", folder, err) continue } // Cache allowed extensions for this folder s.libraryTypes[folder] = libType.AllowedExtensions fmt.Printf("Scanner: Folder %s (type: %s) allows extensions: %v\n", folder, libType.Name, libType.AllowedExtensions) } // Add all folders to watch for _, folder := range folders { if err := s.watcher.Add(folder); err != nil { fmt.Printf("Warning: failed to watch folder %s: %v\n", folder, err) } } return nil } func (s *MediaScanner) ScanFolders(ctx context.Context) error { if len(s.folders) == 0 { return fmt.Errorf("no folders set") } s.totalFiles = 0 s.newItems = 0 s.errors = 0 for _, folder := range s.folders { filepath.WalkDir(folder, func(path string, d fs.DirEntry, err error) error { if !d.IsDir() && s.isScannableFile(path) { s.totalFiles++ } return nil }) } fmt.Printf("Starting scan of %d folders: %v (%d files to scan)\n", len(s.folders), s.folders, s.totalFiles) processedFiles := 0 mediaFiles := 0 for _, folder := range s.folders { fmt.Printf("Scanning folder: %s\n", folder) if _, err := os.Stat(folder); os.IsNotExist(err) { fmt.Printf("Folder does not exist: %s\n", folder) s.errors++ continue } err := filepath.WalkDir(folder, func(path string, d fs.DirEntry, err error) error { if err != nil { fmt.Printf("Error accessing path %s: %v\n", path, err) s.errors++ return err } if d.IsDir() { if err := s.watcher.Add(path); err != nil { fmt.Printf("Warning: failed to watch subdirectory %s: %v\n", path, err) } return nil } if s.isScannableFile(path) { mediaFiles++ processedFiles++ if processedFiles%10 == 0 && s.totalFiles > 0 { progress := float64(processedFiles) / float64(s.totalFiles) if s.job != nil { s.job.UpdateProgress(progress, processedFiles, s.newItems, s.errors) } } wasNew, err := s.processMediaFile(ctx, path) if err != nil { fmt.Printf("Error processing media file %s: %v\n", path, err) s.errors++ } else if wasNew { fmt.Printf("Successfully processed media file: %s\n", path) } else { fmt.Printf("Successfully processed media file: %s\n", path) } } return nil }) if err != nil { s.errors++ return fmt.Errorf("failed to scan folder %s: %v", folder, err) } } fmt.Printf("Scan completed: %d total files scanned, %d media files found, %d new items, %d errors\n", processedFiles, mediaFiles, s.newItems, s.errors) // Clean up: Find media items in DB that no longer exist on filesystem for _, folder := range s.folders { lib, err := s.db.GetLibraryByFolder(ctx, folder) if err != nil { continue } libraryID := lib.LibraryID dbItems, err := s.db.ListMediaItemsByLibrary(ctx, libraryID) if err != nil { fmt.Printf("Warning: failed to get library items for cleanup: %v\n", err) continue } // Build set of scanned file paths for this folder scannedPaths := make(map[string]bool) filepath.WalkDir(folder, func(path string, d os.DirEntry, walkErr error) error { if walkErr != nil { return walkErr } if !d.IsDir() && s.isScannableFile(path) { scannedPaths[s.getRelativePath(path)] = true } return nil }) // Delete items whose files no longer exist - with safety logging for _, item := range dbItems { filePath := item.FilePath if filePath != "" && !scannedPaths[filePath] { msg := fmt.Sprintf("[RESCAN-CLEANUP] Orphaned media item found: ID=%s, Title=%s, Path=%s", item.ID, item.Title, filePath) s.logger.LogDelete(msg) delMsg := fmt.Sprintf("[RESCAN-CLEANUP] Deleting orphaned item '%s' (file no longer exists at %s)", item.Title, filePath) s.logger.LogDelete(delMsg) if err := s.db.DeleteMediaItem(ctx, item.ID); err != nil { errMsg := fmt.Sprintf("[RESCAN-CLEANUP] ERROR: failed to delete orphaned item %s: %v", item.Title, err) s.logger.LogDelete(errMsg) s.logger.LogError(errMsg) } else { s.logger.LogDelete(fmt.Sprintf("[RESCAN-CLEANUP] SUCCESS: deleted orphaned item '%s'", item.Title)) } } } } if s.job != nil && s.totalFiles > 0 { s.job.UpdateProgress(1.0, processedFiles, s.newItems, s.errors) } return nil } func (s *MediaScanner) isScannableFile(path string) bool { ext := strings.ToLower(filepath.Ext(path)) // Find which folder this file belongs to var folder string for _, f := range s.folders { if strings.HasPrefix(path, f) { folder = f break } } // If no folder match, don't scan if folder == "" { return false } // Get allowed extensions for this folder's library allowed, ok := s.libraryTypes[folder] if !ok { // No library type info, skip file fmt.Printf("Warning: No library type info for folder %s, skipping %s\n", folder, path) return false } // Check if file extension is allowed for this library type for _, allowedExt := range allowed { if ext == strings.ToLower(allowedExt) { return true } } return false } // extractFolderStructureMetadata extracts metadata from folder paths, prioritizing Calibre structure func (s *MediaScanner) extractFolderStructureMetadata(path, rootFolder string) *MediaMetadata { metadata := &MediaMetadata{} // Get the relative path from root folder relPath, err := filepath.Rel(rootFolder, path) if err != nil { return metadata } // Split into directory components dir := filepath.Dir(relPath) components := strings.Split(dir, string(filepath.Separator)) if len(components) < 2 { return metadata // Not enough structure to extract } // Calibre structure detection // Pattern 1: Author Name/Book Title/ // Pattern 2: Author Name/Series Name/Book Title/ // Pattern 3: Author Name/Series Name, Book #1 - Book Title/ author := strings.TrimSuffix(components[0], "_") // Remove trailing underscore if present metadata.Author = strings.ReplaceAll(author, "_", " ") if len(components) >= 3 { // This might be a series structure possibleSeries := components[1] possibleTitle := components[2] // Check for Calibre series format: "Series Name, Book #1 - Title" seriesMatch := regexp.MustCompile(`^(.*),\s+Book\s+#(\d+)\s*-\s*(.*)$`).FindStringSubmatch(possibleSeries) if len(seriesMatch) == 4 { metadata.Series = strings.ReplaceAll(seriesMatch[1], "_", " ") if seriesNum, err := strconv.ParseInt(seriesMatch[2], 10, 32); err == nil { metadata.SeriesNumber = int32(seriesNum) } metadata.Title = strings.ReplaceAll(possibleTitle, "_", " ") } else { // Simple series structure: Author/Series/Title metadata.Series = strings.ReplaceAll(possibleSeries, "_", " ") metadata.Title = strings.ReplaceAll(possibleTitle, "_", " ") // Try to extract series number from title titleNumMatch := regexp.MustCompile(`^(.*)\s+(\d+)$`).FindStringSubmatch(metadata.Title) if len(titleNumMatch) == 3 { metadata.Title = titleNumMatch[1] if seriesNum, err := strconv.ParseInt(titleNumMatch[2], 10, 32); err == nil { metadata.SeriesNumber = int32(seriesNum) } } } } else { // Simple structure: Author/Title metadata.Title = strings.ReplaceAll(components[1], "_", " ") } return metadata } func (s *MediaScanner) processMediaFile(ctx context.Context, path string) (bool, error) { fmt.Printf("Processing media file: %s\n", path) // Get file info info, err := os.Stat(path) if err != nil { fmt.Printf("Failed to get file info for %s: %v\n", path, err) return false, fmt.Errorf("failed to get file info: %v", err) } fmt.Printf("File info for %s: size=%d\n", path, info.Size()) // Get file modification time for created_at fileModTime := info.ModTime() // Find library for this file's folder var libraryID pgtype.UUID if s.defaultLibraryID.Valid { libraryID = s.defaultLibraryID } else { for _, folder := range s.folders { if strings.HasPrefix(path, folder) { lib, err := s.db.GetLibraryByFolder(ctx, folder) if err != nil { return false, fmt.Errorf("failed to find library for folder %s: %v", folder, err) } libraryID = lib.LibraryID break } } } if !libraryID.Valid { return false, fmt.Errorf("no library found for file path: %s", path) } // Check if media item already exists in database existingItem, err := s.getMediaItemByFilePath(ctx, path, libraryID) if err == nil { fmt.Printf("Media item already exists in database: %s (size: %d vs %d)\n", path, existingItem.FileSize.Int64, info.Size()) // If force rescan is enabled, always re-process if s.forceRescan { fmt.Printf("Force rescan enabled, updating existing media item: %s\n", path) // Use UPDATE instead of DELETE+INSERT to preserve created_at if err := s.updateMediaItem(ctx, existingItem.ID, path, info); err != nil { fmt.Printf("Warning: failed to update existing media item: %v\n", err) } return false, nil } else { // Normal behavior: check if file has changed (by size) if existingItem.FileSize.Int64 != info.Size() { fmt.Printf("File size changed, updating media item: %s\n", path) _ = s.updateMediaItem(ctx, existingItem.ID, path, info) return false, nil } fmt.Printf("Media item already exists with same size, skipping: %s\n", path) return false, nil } } else if err != pgx.ErrNoRows { fmt.Printf("Database error checking media item existence: %v\n", err) return false, fmt.Errorf("failed to check if media item exists: %v", err) } fmt.Printf("Media item does not exist in database, creating new entry: %s\n", path) // Extract metadata from file first metadata, err := s.extractMetadata(path) if err != nil { fmt.Printf("Warning: failed to extract metadata from %s: %v\n", path, err) metadata = &MediaMetadata{} } // Extract hash information during metadata extraction hashInfo, formatInfo, err := s.extractHashInfo(path) if err != nil { fmt.Printf("Warning: failed to extract hash info from %s: %v\n", path, err) hashInfo = &HashInfo{} formatInfo = &FormatInfo{} } else { metadata.FileHashInfo = hashInfo metadata.FileFormats = []*FormatInfo{formatInfo} fmt.Printf("Hash info for %s: SHA256=%s, OPF_ID=%s, OPF_UUID=%s, Confidence=%s\n", path, hashInfo.FileSHA256, hashInfo.OPFIdentifier, hashInfo.OPFUUID, hashInfo.HashConfidence) } var comicInfo *ComicInfo var coverImage []byte ext := strings.ToLower(filepath.Ext(path)) if ext == ".cbz" || ext == ".cbr" || ext == ".cb7" || ext == ".cbt" || strings.HasSuffix(strings.ToLower(path), ".tar.gz") || strings.HasSuffix(strings.ToLower(path), ".tar.bz2") || strings.HasSuffix(strings.ToLower(path), ".tgz") || strings.HasSuffix(strings.ToLower(path), ".tbz2") { info, cover, err := extractComicMetadata(path) if err != nil { fmt.Printf("Warning: failed to extract comic metadata from %s: %v\n", path, err) } else { comicInfo = info coverImage = cover if comicInfo.Title != "" && metadata.Title == "" { metadata.Title = comicInfo.Title } if comicInfo.Series != "" && metadata.Series == "" { metadata.Series = comicInfo.Series } if comicInfo.Number > 0 && metadata.SeriesNumber == 0 { metadata.SeriesNumber = int32(comicInfo.Number) } if comicInfo.Publisher != "" && metadata.Publisher == "" { metadata.Publisher = comicInfo.Publisher } if comicInfo.Writer != "" && metadata.Author == "" { metadata.Author = comicInfo.Writer } if len(coverImage) > 0 && metadata.CoverPath == "" { coverPath := path + ".cover.jpg" if err := os.WriteFile(coverPath, coverImage, 0644); err == nil { metadata.CoverPath = s.getRelativePath(coverPath) } } fmt.Printf("Extracted comic metadata from %s: title=%s, series=%s, issue=%d\n", path, comicInfo.Title, comicInfo.Series, comicInfo.Number) } } // Try to get metadata from folder structure as fallback/enhancement // Use the root folder that contains this file var rootFolder string for _, folder := range s.folders { if strings.HasPrefix(path, folder) { rootFolder = folder break } } if rootFolder != "" { folderMetadata := s.extractFolderStructureMetadata(path, rootFolder) // Use folder metadata as fallback for missing information if metadata.Title == "" && folderMetadata.Title != "" { metadata.Title = folderMetadata.Title } if metadata.Author == "" && folderMetadata.Author != "" { metadata.Author = folderMetadata.Author } if metadata.Series == "" && folderMetadata.Series != "" { metadata.Series = folderMetadata.Series } if metadata.SeriesNumber == 0 && folderMetadata.SeriesNumber > 0 { metadata.SeriesNumber = folderMetadata.SeriesNumber } } // Final fallback if still missing essential metadata if metadata.Title == "" { metadata.Title = strings.TrimSuffix(filepath.Base(path), filepath.Ext(path)) } if metadata.Author == "" { metadata.Author = "Unknown" } // libraryID already determined at start of function // Normalize metadata fields for display metadata.Contributors = utils.NormalizeContributors(metadata.Contributors) metadata.Tags = utils.NormalizeTags(metadata.Tags) // Normalize search fields contributorsSearch := utils.NormalizeContributorsSearch(metadata.Contributors) tagsSearch := utils.NormalizeTagsSearch(metadata.Tags) // Create media item in database relativePath := s.getRelativePath(path) createdItem, err := s.db.CreateMediaItem(ctx, database.CreateMediaItemParams{ LibraryID: libraryID, Title: metadata.Title, Author: pgtype.Text{String: metadata.Author, Valid: metadata.Author != ""}, Isbn: pgtype.Text{String: utils.NormalizeISBNSafe(metadata.ISBN), Valid: metadata.ISBN != ""}, Asin: pgtype.Text{String: metadata.ASIN, Valid: metadata.ASIN != ""}, Description: pgtype.Text{String: metadata.Description, Valid: metadata.Description != ""}, FilePath: relativePath, FileSize: pgtype.Int8{Int64: info.Size(), Valid: true}, MimeType: pgtype.Text{String: s.getMimeType(path), Valid: true}, CoverImagePath: pgtype.Text{String: metadata.CoverPath, Valid: metadata.CoverPath != ""}, Series: pgtype.Text{String: metadata.Series, Valid: metadata.Series != ""}, SeriesNumber: pgtype.Int4{Int32: metadata.SeriesNumber, Valid: metadata.SeriesNumber > 0}, Publisher: pgtype.Text{String: metadata.Publisher, Valid: metadata.Publisher != ""}, DatePublished: pgtype.Date{Time: metadata.PublishDate, Valid: !metadata.PublishDate.IsZero()}, Contributors: metadata.Contributors, ContributorsSearch: contributorsSearch, Tags: metadata.Tags, TagsSearch: tagsSearch, AddedByAdminID: s.adminID, CreatedAt: pgtype.Timestamptz{Time: fileModTime, Valid: true}, }) if err != nil { return false, fmt.Errorf("failed to create media item: %v", err) } // Update hash information before database storage if metadata.FileHashInfo != nil && metadata.FileHashInfo.FileSHA256 != "" { _, err = s.db.UpdateMediaItemIdentifiers(ctx, database.UpdateMediaItemIdentifiersParams{ ID: createdItem.ID, FileSha256: pgtype.Text{String: metadata.FileHashInfo.FileSHA256, Valid: true}, OpfIdentifier: pgtype.Text{String: metadata.FileHashInfo.OPFIdentifier, Valid: metadata.FileHashInfo.OPFIdentifier != ""}, OpfUuid: pgtype.Text{String: metadata.FileHashInfo.OPFUUID, Valid: metadata.FileHashInfo.OPFUUID != ""}, HashConfidence: pgtype.Text{String: metadata.FileHashInfo.HashConfidence, Valid: true}, }) if err != nil { fmt.Printf("Warning: failed to update hash identifiers for %s: %v\n", path, err) } } // Store format information in the database for _, format := range metadata.FileFormats { _, err = s.db.CreateMediaItemFormat(ctx, database.CreateMediaItemFormatParams{ MediaItemID: createdItem.ID, FormatType: format.FormatType, FilePath: pgtype.Text{String: s.getRelativePath(format.FilePath), Valid: true}, FileSha256: pgtype.Text{String: format.FileSHA256, Valid: true}, FileSizeBytes: pgtype.Int8{Int64: format.FileSizeBytes, Valid: true}, MimeType: pgtype.Text{String: format.MimeType, Valid: true}, }) if err != nil { fmt.Printf("Warning: failed to create format entry for %s: %v\n", path, err) } } s.newItems++ return true, nil } func (s *MediaScanner) extractMetadata(path string) (*MediaMetadata, error) { ext := strings.ToLower(filepath.Ext(path)) switch ext { case ".epub": metadata, err := s.extractEPUBMetadata(path) if err != nil { return metadata, err } // Try to extract embedded cover coverPath, err := s.extractEPUBCover(path) if err != nil { fmt.Printf("Warning: failed to extract EPUB cover from %s: %v\n", path, err) } else if coverPath != "" { metadata.CoverPath = s.getRelativePath(coverPath) } // If no embedded cover, try sidecar if metadata.CoverPath == "" { sidecarCover := findSidecarCover(path) if sidecarCover != "" { metadata.CoverPath = s.getRelativePath(sidecarCover) } } return metadata, nil case ".pdf": return s.extractPDFMetadata(path) default: // For other formats, return basic metadata return &MediaMetadata{ Title: strings.TrimSuffix(filepath.Base(path), ext), }, nil } } func (s *MediaScanner) extractEPUBMetadata(path string) (*MediaMetadata, error) { book, err := epub.ReadBook(path) if err != nil { return nil, fmt.Errorf("failed to open EPUB: %v", err) } metadata := &MediaMetadata{} // Title if title, err := book.Title(); err == nil && title != "" { metadata.Title = title } // Author if authors, err := book.MetadataByKey("creator"); err == nil && len(authors) > 0 { metadata.Author = authors[0] } // Description if descriptions, err := book.MetadataByKey("description"); err == nil && len(descriptions) > 0 { metadata.Description = descriptions[0] } // Publisher if publishers, err := book.MetadataByKey("publisher"); err == nil && len(publishers) > 0 { metadata.Publisher = publishers[0] } // Series and series number (Calibre specific metadata) if series, err := book.MetadataByKey("calibre:series"); err == nil && len(series) > 0 { metadata.Series = series[0] } if seriesIndex, err := book.MetadataByKey("calibre:series_index"); err == nil && len(seriesIndex) > 0 { if index, err := strconv.ParseFloat(seriesIndex[0], 32); err == nil { metadata.SeriesNumber = int32(index) } } // Publish date if dates, err := book.MetadataByKey("date"); err == nil && len(dates) > 0 { if date, err := time.Parse("2006-01-02", dates[0]); err == nil { metadata.PublishDate = date } else { // Try alternative date formats if date, err := time.Parse("2006", dates[0]); err == nil { metadata.PublishDate = date } } } // Contributors if contributors, err := book.MetadataByKey("contributor"); err == nil && len(contributors) > 0 { // Normalize contributors for display metadata.Contributors = utils.NormalizeContributors(contributors) } // ISBN if isbns, err := book.MetadataByKey("identifier"); err == nil && len(isbns) > 0 { for _, isbn := range isbns { if strings.Contains(strings.ToLower(isbn), "isbn") { // Extract ISBN number from identifier like "isbn:978-3-16-148410-0" isbnParts := strings.SplitN(isbn, ":", 2) if len(isbnParts) == 2 { metadata.ISBN = isbnParts[1] break } } if strings.Contains(strings.ToLower(isbn), "asin") { // Extract ASIN from identifier like "asin:B08XXXXX" asinParts := strings.SplitN(isbn, ":", 2) if len(asinParts) == 2 { metadata.ASIN = asinParts[1] } } } } // Tags if tags, err := book.MetadataByKey("subject"); err == nil && len(tags) > 0 { // Normalize tags for display metadata.Tags = utils.NormalizeTags(tags) } return metadata, nil } // extractEPUBCover extracts the cover image from an EPUB file. // It looks for: // 1. An item with properties="cover-image" in the manifest // 2. A meta tag with name="cover" pointing to an image // 3. Common cover image paths like cover.jpg, cover.jpeg, cover.png // Returns the path to the saved cover image, or empty string if no cover found. func (s *MediaScanner) extractEPUBCover(epubPath string) (string, error) { // Open the EPUB as a zip file to extract images r, err := zip.OpenReader(epubPath) if err != nil { return "", fmt.Errorf("failed to open EPUB as zip: %v", err) } defer r.Close() // Try to find cover image from OPF metadata coverImageName := "" // Attempt to read the OPF file to find cover reference // First, find container.xml to locate the OPF var opfPath string for _, f := range r.File { if f.Name == "META-INF/container.xml" { rc, err := f.Open() if err != nil { continue } content, err := io.ReadAll(rc) rc.Close() if err != nil { continue } // Parse container.xml to find OPF path // Simple string search since we just need the path opfStart := bytes.Index(content, []byte("]*properties="[^"]*cover-image[^"]*"[^>]*id="([^"]+)"`) matches := coverImageRE.FindStringSubmatch(contentStr) if len(matches) > 1 { coverID := matches[1] // Find the href for this ID hrefRE := regexp.MustCompile(fmt.Sprintf(`]*id="%s"[^>]*href="([^"]+)"`, coverID)) hrefMatches := hrefRE.FindStringSubmatch(contentStr) if len(hrefMatches) > 1 { return resolveOPFPath(opfDir, hrefMatches[1]) } } // Look for meta name="cover" metaCoverRE := regexp.MustCompile(`]*name="cover"[^>]*content="([^"]+)"`) metaMatches := metaCoverRE.FindStringSubmatch(contentStr) if len(metaMatches) > 1 { coverContent := metaMatches[1] // Could be "image-id" format if strings.HasPrefix(coverContent, "image-") { coverID := strings.TrimPrefix(coverContent, "image-") hrefRE := regexp.MustCompile(fmt.Sprintf(`]*id="%s"[^>]*href="([^"]+)"`, coverID)) hrefMatches := hrefRE.FindStringSubmatch(contentStr) if len(hrefMatches) > 1 { return resolveOPFPath(opfDir, hrefMatches[1]) } } } // Fall back to searching common paths return findCoverImageInZip(files) } // resolveOPFPath resolves a relative path against the OPF directory func resolveOPFPath(opfDir, href string) string { if opfDir == "" { return href } // Handle ../ in href if strings.HasPrefix(href, "../") { // Simple case: just use the href as-is for now return href } // Join the directory with the href return filepath.Join(filepath.Dir(opfDir), href) } // readFileFromZip reads a file from the zip by name func readFileFromZip(files []*zip.File, name string) ([]byte, error) { // Normalize the name for comparison name = filepath.ToSlash(name) for _, f := range files { fName := filepath.ToSlash(f.Name) if fName == name { rc, err := f.Open() if err != nil { return nil, err } defer rc.Close() return io.ReadAll(rc) } } return nil, fmt.Errorf("file not found: %s", name) } // extractImageFromZip extracts an image file and returns its contents func extractImageFromZip(files []*zip.File, imagePath, opfDir string) ([]byte, error) { // Try direct match first for _, f := range files { if strings.EqualFold(f.Name, imagePath) { rc, err := f.Open() if err != nil { return nil, err } defer rc.Close() return io.ReadAll(rc) } } // Try resolved path resolvedPath := resolveOPFPath(opfDir, imagePath) for _, f := range files { if strings.EqualFold(f.Name, resolvedPath) { rc, err := f.Open() if err != nil { return nil, err } defer rc.Close() return io.ReadAll(rc) } } return nil, fmt.Errorf("image not found: %s", imagePath) } // findSidecarCover looks for cover images in the same directory as the media file. // It checks for common cover filename patterns in priority order: // 1. cover.jpg, cover.jpeg, cover.png, cover.webp // 2. folder.jpg, folder.jpeg, folder.png, folder.webp // 3. {basename}.jpg, {basename}.jpeg, etc. (same name as media file) // 4. .folder.jpg (hidden file) // Returns the full path to the cover file, or empty string if not found. func findSidecarCover(mediaPath string) string { dir := filepath.Dir(mediaPath) baseName := strings.TrimSuffix(filepath.Base(mediaPath), filepath.Ext(mediaPath)) // Priority order for cover filenames coverPatterns := []string{ "cover.jpg", "cover.jpeg", "cover.png", "cover.webp", "folder.jpg", "folder.jpeg", "folder.png", "folder.webp", ".folder.jpg", ".folder.jpeg", ".folder.png", } // First, check exact match cover/folder names for _, coverName := range coverPatterns { coverPath := filepath.Join(dir, coverName) if _, err := os.Stat(coverPath); err == nil { return coverPath } } // Second, check for {basename}.{ext} pattern extensions := []string{".jpg", ".jpeg", ".png", ".webp"} for _, ext := range extensions { coverPath := filepath.Join(dir, baseName+ext) if _, err := os.Stat(coverPath); err == nil { return coverPath } // Also check uppercase extension coverPathUpper := filepath.Join(dir, baseName+strings.ToUpper(ext)) if _, err := os.Stat(coverPathUpper); err == nil { return coverPathUpper } } return "" } func (s *MediaScanner) extractPDFMetadata(path string) (*MediaMetadata, error) { metadata := &MediaMetadata{} // Open PDF file for reading metadata f, err := os.Open(path) if err != nil { fmt.Printf("Warning: failed to open PDF file %s: %v\n", path, err) metadata.Title = strings.TrimSuffix(filepath.Base(path), ".pdf") return metadata, nil } defer f.Close() // Use pdfcpu API to read PDF metadata // Configuration: nil = default (lenient mode) pdfInfo, err := pdfcpuapi.PDFInfo(f, filepath.Base(path), nil, false, nil) if err != nil { fmt.Printf("Warning: failed to read PDF info from %s: %v\n", path, err) // Fall back to filename as title metadata.Title = strings.TrimSuffix(filepath.Base(path), ".pdf") return metadata, nil } // Extract title if pdfInfo.Title != "" { metadata.Title = pdfInfo.Title } else { metadata.Title = strings.TrimSuffix(filepath.Base(path), ".pdf") } // Extract author if pdfInfo.Author != "" { metadata.Author = pdfInfo.Author } // Extract subject (use as description) if pdfInfo.Subject != "" { metadata.Description = pdfInfo.Subject } // Extract creator (use as author fallback) if pdfInfo.Creator != "" && metadata.Author == "" { metadata.Author = pdfInfo.Creator } // Extract producer (use as publisher) if pdfInfo.Producer != "" { metadata.Publisher = pdfInfo.Producer } // Try to extract cover image coverPath, err := s.extractPDFCover(path) if err != nil { fmt.Printf("Warning: failed to extract PDF cover from %s: %v\n", path, err) } else if coverPath != "" { metadata.CoverPath = s.getRelativePath(coverPath) } // If no embedded cover, try sidecar if metadata.CoverPath == "" { sidecarCover := findSidecarCover(path) if sidecarCover != "" { metadata.CoverPath = s.getRelativePath(sidecarCover) } } return metadata, nil } // extractPDFCover extracts a cover image from a PDF file. // It uses pdfcpu API to extract images from the first page. // Returns the path to the saved cover, or empty string if no cover found. func (s *MediaScanner) extractPDFCover(pdfPath string) (string, error) { // Create a temporary directory for extracted images tmpDir, err := os.MkdirTemp("", "pdf-cover-") if err != nil { return "", fmt.Errorf("failed to create temp dir: %v", err) } defer os.RemoveAll(tmpDir) // Use pdfcpu API to extract images from first page // ExtractImagesFile(inFile, outDir string, selectedPages []string, conf *model.Configuration) error err = pdfcpuapi.ExtractImagesFile(pdfPath, tmpDir, []string{"1"}, nil) if err != nil { // No images found or extraction failed - this is OK, just return empty return "", nil } // Check for extracted images in the temp directory entries, err := os.ReadDir(tmpDir) if err != nil || len(entries) == 0 { return "", nil } // Find the largest image (likely the cover) var largestImage string var largestSize int64 for _, entry := range entries { if entry.IsDir() { continue } info, err := entry.Info() if err != nil { continue } // Skip very small files (likely thumbnails or icons) if info.Size() < 1000 { continue } if info.Size() > largestSize { largestImage = filepath.Join(tmpDir, entry.Name()) largestSize = info.Size() } } if largestImage == "" { return "", nil } // Read the image imageData, err := os.ReadFile(largestImage) if err != nil || len(imageData) == 0 { return "", nil } // Save cover to disk (same pattern as comics: {pdf_path}.cover.jpg) coverPath := pdfPath + ".cover.jpg" if err := os.WriteFile(coverPath, imageData, 0644); err != nil { return "", fmt.Errorf("failed to write cover file: %v", err) } return coverPath, nil } // ComicInfo represents metadata from ComicInfo.xml type ComicInfo struct { XMLName xml.Name `xml:"ComicInfo"` Title string `xml:"Title"` Series string `xml:"Series"` Number int `xml:"Number"` Volume int `xml:"Volume"` Publisher string `xml:"Publisher"` Year int `xml:"Year"` Month int `xml:"Month"` Day int `xml:"Day"` Writer string `xml:"Writer"` Penciller string `xml:"Penciller"` Inker string `xml:"Inker"` Colorist string `xml:"Colorist"` Letterer string `xml:"Letterer"` CoverArtist string `xml:"CoverArtist"` Genre string `xml:"Genre"` Tags string `xml:"Tags"` Web string `xml:"Web"` Notes string `xml:"Notes"` } // extractComicMetadata extracts metadata from comic archive (supports .cbz, .cbr, .cb7, .cbt) func extractComicMetadata(filePath string) (*ComicInfo, []byte, error) { ext := strings.ToLower(filepath.Ext(filePath)) switch ext { case ".cbz": return extractZipMetadata(filePath) case ".cbr": return extractRarMetadata(filePath) case ".cb7": return extract7ZipMetadata(filePath) case ".cbt": return extractTarMetadata(filePath) default: return nil, nil, fmt.Errorf("unsupported comic format: %s", ext) } } // archiveFile represents a file in an archive for uniform handling type archiveFile interface { Name() string Open() (io.ReadCloser, error) } // extractMetadataFromArchive extracts ComicInfo.xml and cover image from any archive format func extractMetadataFromArchive(files []archiveFile) (*ComicInfo, []byte, error) { var comicInfo *ComicInfo var coverImage []byte for _, f := range files { if f.Name() == "ComicInfo.xml" { rc, err := f.Open() if err != nil { return nil, nil, fmt.Errorf("failed to open ComicInfo.xml: %w", err) } data, err := io.ReadAll(rc) rc.Close() if err != nil { return nil, nil, fmt.Errorf("failed to read ComicInfo.xml: %w", err) } comicInfo = &ComicInfo{} if err := xml.Unmarshal(data, comicInfo); err != nil { return nil, nil, fmt.Errorf("failed to parse ComicInfo.xml: %w", err) } } if coverImage == nil && isImageFile(f.Name()) { if !strings.Contains(filepath.Dir(f.Name()), string(filepath.Separator)) || filepath.Dir(f.Name()) == "." { rc, err := f.Open() if err != nil { continue } coverImage, err = io.ReadAll(rc) rc.Close() if err == nil { _, _, err = image.Decode(bytes.NewReader(coverImage)) if err != nil { coverImage = nil } } } } } if comicInfo == nil { comicInfo = &ComicInfo{} } return comicInfo, coverImage, nil } // extractZipMetadata extracts metadata from ZIP archives (.cbz) func extractZipMetadata(filePath string) (*ComicInfo, []byte, error) { r, err := zip.OpenReader(filePath) if err != nil { return nil, nil, fmt.Errorf("failed to open ZIP archive: %w", err) } defer r.Close() files := make([]archiveFile, 0, len(r.File)) for _, f := range r.File { files = append(files, &zipFileAdapter{f}) } comicInfo, coverImage, err := extractMetadataFromArchive(files) if err != nil { return nil, nil, err } if comicInfo.Title == "" { basename := filepath.Base(filePath) comicInfo.Title = strings.TrimSuffix(basename, filepath.Ext(basename)) } return comicInfo, coverImage, nil } // zipFileAdapter adapts zip.File to archiveFile interface type zipFileAdapter struct { *zip.File } func (z *zipFileAdapter) Name() string { return z.File.Name } func (z *zipFileAdapter) Open() (io.ReadCloser, error) { return z.File.Open() } // extractRarMetadata extracts metadata from RAR archives (.cbr) func extractRarMetadata(filePath string) (*ComicInfo, []byte, error) { r, err := rardecode.OpenReader(filePath, "") if err != nil { return nil, nil, fmt.Errorf("failed to open RAR archive: %w", err) } defer r.Close() files := make([]archiveFile, 0, 100) for { header, err := r.Next() if err == io.EOF { break } if err != nil { return nil, nil, fmt.Errorf("failed to read RAR entry: %w", err) } data, err := io.ReadAll(r) if err != nil { continue } files = append(files, &rarFileAdapter{ name: header.Name, data: data, }) } comicInfo, coverImage, err := extractMetadataFromArchive(files) if err != nil { return nil, nil, err } if comicInfo.Title == "" { basename := filepath.Base(filePath) comicInfo.Title = strings.TrimSuffix(basename, filepath.Ext(basename)) } return comicInfo, coverImage, nil } // rarFileAdapter stores RAR file data in memory type rarFileAdapter struct { name string data []byte } func (r *rarFileAdapter) Name() string { return r.name } func (r *rarFileAdapter) Open() (io.ReadCloser, error) { return io.NopCloser(bytes.NewReader(r.data)), nil } // extract7ZipMetadata extracts metadata from 7-Zip archives (.cb7) func extract7ZipMetadata(filePath string) (*ComicInfo, []byte, error) { r, err := sevenzip.OpenReader(filePath) if err != nil { return nil, nil, fmt.Errorf("failed to open 7-Zip archive: %w", err) } defer r.Close() files := make([]archiveFile, 0, len(r.File)) for _, f := range r.File { files = append(files, &sevenZipFileAdapter{file: f}) } comicInfo, coverImage, err := extractMetadataFromArchive(files) if err != nil { return nil, nil, err } if comicInfo.Title == "" { basename := filepath.Base(filePath) comicInfo.Title = strings.TrimSuffix(basename, filepath.Ext(basename)) } return comicInfo, coverImage, nil } // sevenZipFileAdapter adapts sevenzip.File to archiveFile interface type sevenZipFileAdapter struct { file *sevenzip.File } func (s *sevenZipFileAdapter) Name() string { return s.file.Name } func (s *sevenZipFileAdapter) Open() (io.ReadCloser, error) { return s.file.Open() } // extractTarMetadata extracts metadata from TAR archives (.cbt) func extractTarMetadata(filePath string) (*ComicInfo, []byte, error) { f, err := os.Open(filePath) if err != nil { return nil, nil, fmt.Errorf("failed to open TAR archive: %w", err) } defer f.Close() var tarReader *tar.Reader if strings.HasSuffix(strings.ToLower(filePath), ".tar.gz") || strings.HasSuffix(strings.ToLower(filePath), ".tgz") { gzReader, err := gzip.NewReader(f) if err != nil { return nil, nil, fmt.Errorf("failed to create gzip reader: %w", err) } defer gzReader.Close() tarReader = tar.NewReader(gzReader) } else if strings.HasSuffix(strings.ToLower(filePath), ".tar.bz2") || strings.HasSuffix(strings.ToLower(filePath), ".tbz2") { bz2Reader := bzip2.NewReader(f) tarReader = tar.NewReader(bz2Reader) } else { tarReader = tar.NewReader(f) } files := make([]archiveFile, 0, 100) for { header, err := tarReader.Next() if err == io.EOF { break } if err != nil { return nil, nil, fmt.Errorf("failed to read TAR entry: %w", err) } if header.Typeflag == tar.TypeReg { data, err := io.ReadAll(tarReader) if err != nil { continue } files = append(files, &tarFileAdapter{ name: header.Name, data: data, }) } } comicInfo, coverImage, err := extractMetadataFromArchive(files) if err != nil { return nil, nil, err } if comicInfo.Title == "" { basename := filepath.Base(filePath) comicInfo.Title = strings.TrimSuffix(basename, filepath.Ext(basename)) } return comicInfo, coverImage, nil } // tarFileAdapter stores TAR file data in memory type tarFileAdapter struct { name string data []byte } func (t *tarFileAdapter) Name() string { return t.name } func (t *tarFileAdapter) Open() (io.ReadCloser, error) { return io.NopCloser(bytes.NewReader(t.data)), nil } // isImageFile checks if a file is an image based on extension func isImageFile(filename string) bool { ext := strings.ToLower(filepath.Ext(filename)) return ext == ".jpg" || ext == ".jpeg" || ext == ".png" || ext == ".gif" } func (s *MediaScanner) updateMediaItem(ctx context.Context, mediaItemID pgtype.UUID, path string, info os.FileInfo) error { // Re-extract metadata for the update metadata, err := s.extractMetadata(path) if err != nil { fmt.Printf("Warning: failed to extract metadata for force rescan %s: %v\n", path, err) metadata = &MediaMetadata{} } // Normalize metadata fields metadata.Contributors = utils.NormalizeContributors(metadata.Contributors) metadata.Tags = utils.NormalizeTags(metadata.Tags) contributorsSearch := utils.NormalizeContributorsSearch(metadata.Contributors) tagsSearch := utils.NormalizeTagsSearch(metadata.Tags) // Call the database update - only update fields available in MediaMetadata _, err = s.db.UpdateMediaItem(ctx, database.UpdateMediaItemParams{ ID: mediaItemID, Title: metadata.Title, Author: pgtype.Text{String: metadata.Author, Valid: metadata.Author != ""}, Isbn: pgtype.Text{String: utils.NormalizeISBNSafe(metadata.ISBN), Valid: metadata.ISBN != ""}, Description: pgtype.Text{String: metadata.Description, Valid: metadata.Description != ""}, CoverImagePath: pgtype.Text{String: s.getRelativePath(metadata.CoverPath), Valid: metadata.CoverPath != ""}, Series: pgtype.Text{String: metadata.Series, Valid: metadata.Series != ""}, SeriesNumber: pgtype.Int4{Int32: metadata.SeriesNumber, Valid: metadata.SeriesNumber > 0}, Tags: metadata.Tags, TagsSearch: tagsSearch, Asin: pgtype.Text{String: metadata.ASIN, Valid: metadata.ASIN != ""}, DatePublished: pgtype.Date{Time: metadata.PublishDate, Valid: !metadata.PublishDate.IsZero()}, Publisher: pgtype.Text{String: metadata.Publisher, Valid: metadata.Publisher != ""}, Contributors: metadata.Contributors, ContributorsSearch: contributorsSearch, }) return err } func (s *MediaScanner) getMediaItemByFilePath(ctx context.Context, filePath string, libraryID pgtype.UUID) (database.MediaItems, error) { return s.db.GetMediaItemByFilePath(ctx, database.GetMediaItemByFilePathParams{ FilePath: s.getRelativePath(filePath), LibraryID: libraryID, }) } func (s *MediaScanner) getMimeType(path string) string { ext := strings.ToLower(filepath.Ext(path)) if mime, ok := MimeTypes[ext]; ok { return mime } return "application/octet-stream" } func (s *MediaScanner) WatchChanges(ctx context.Context) error { // Prevent duplicate calls if !s.watching.CompareAndSwap(false, true) { return fmt.Errorf("already watching") } // Reset flag when context is cancelled go func() { <-ctx.Done() s.watching.Store(false) }() // Perform initial scan of all root folders go s.performInitialScan(ctx) // Start directory processor go s.processDirtyDirectories(ctx) // Start polling fallback go s.StartPolling(ctx) // Handle fsnotify events - queue them for debouncing go func() { for { select { case event, ok := <-s.watcher.Events: if !ok { return } // Handle new directories - add them to the watcher if event.Has(fsnotify.Create) { if info, err := os.Stat(event.Name); err == nil && info.IsDir() { s.watcher.Add(event.Name) } } // Mark directory dirty for ANY file change if event.Has(fsnotify.Create | fsnotify.Write | fsnotify.Remove | fsnotify.Chmod | fsnotify.Rename) { s.markDirectoryDirty(filepath.Dir(event.Name)) } case err, ok := <-s.watcher.Errors: if !ok { return } fmt.Printf("Watcher error: %v\n", err) case <-ctx.Done(): return } } }() return nil } func (s *MediaScanner) handleFileAdd(ctx context.Context, filePath string) { fmt.Printf("New/modified media file detected: %s\n", filePath) if _, err := s.processMediaFile(ctx, filePath); err != nil { fmt.Printf("Error processing modified media file %s: %v\n", filePath, err) } } func (s *MediaScanner) handleFileDelete(ctx context.Context, filePath string) { s.logger.LogDelete(fmt.Sprintf("[WATCH-DELETE] File removed from filesystem: %s", filePath)) // Determine libraryID for this file var libraryID pgtype.UUID for _, folder := range s.folders { if strings.HasPrefix(filePath, folder) { lib, err := s.db.GetLibraryByFolder(ctx, folder) if err == nil { libraryID = lib.LibraryID break } } } if !libraryID.Valid { msg := fmt.Sprintf("[WATCH-DELETE] WARNING: could not determine library for deleted file: %s", filePath) s.logger.LogDelete(msg) s.logger.LogError(msg) return } // Look up media item existingItem, err := s.db.GetMediaItemByFilePath(ctx, database.GetMediaItemByFilePathParams{ FilePath: s.getRelativePath(filePath), LibraryID: libraryID, }) if err == nil { msg := fmt.Sprintf("[WATCH-DELETE] Found media item to delete: ID=%s, Title=%s, Path=%s", existingItem.ID, existingItem.Title, existingItem.FilePath) s.logger.LogDelete(msg) if err := s.db.DeleteMediaItem(ctx, existingItem.ID); err != nil { errMsg := fmt.Sprintf("[WATCH-DELETE] ERROR: failed to delete media item %s: %v", existingItem.ID, err) s.logger.LogDelete(errMsg) s.logger.LogError(errMsg) } else { s.logger.LogDelete(fmt.Sprintf("[WATCH-DELETE] SUCCESS: deleted media item '%s' (was at %s)", existingItem.Title, existingItem.FilePath)) } } else if err != pgx.ErrNoRows { errMsg := fmt.Sprintf("[WATCH-DELETE] ERROR: failed to look up media item for %s: %v", filePath, err) s.logger.LogDelete(errMsg) s.logger.LogError(errMsg) } else { s.logger.LogDelete(fmt.Sprintf("[WATCH-DELETE] No media item found in database for deleted file: %s", filePath)) } } func (s *MediaScanner) markDirectoryDirty(dirPath string) { s.dirtyDirsMu.Lock() defer s.dirtyDirsMu.Unlock() // Only mark if within watched folders var isWatched bool for _, folder := range s.folders { if strings.HasPrefix(dirPath, folder) { isWatched = true break } } if !isWatched { return } // Smart event merging (Jellyfin approach): // 1. If parent dir exists, replace with parent (consolidate) // 2. If sibling dirs exist, replace with common parent // 3. Otherwise, add this dir // Check if parent directory is already dirty parentDir := filepath.Dir(dirPath) if parentDir != dirPath { // Not at root if _, parentExists := s.dirtyDirs[parentDir]; parentExists { // Parent already being watched, reset its timestamp s.dirtyDirs[parentDir] = time.Now() return } } // Check if any subdirectories are dirty, replace with parent for existingDir := range s.dirtyDirs { if strings.HasPrefix(existingDir, dirPath+"/") { // This is a subdirectory, replace it with parent delete(s.dirtyDirs, existingDir) } } // NEW: Check for sibling directories and consolidate to parent parentDir = filepath.Dir(dirPath) for existingDir := range s.dirtyDirs { existingParent := filepath.Dir(existingDir) if existingParent == parentDir && existingParent != dirPath && existingParent != "." { // Found a sibling! Both should be replaced with parent delete(s.dirtyDirs, existingDir) s.dirtyDirs[parentDir] = time.Now() return } } // Add/update this directory s.dirtyDirs[dirPath] = time.Now() } func (s *MediaScanner) processDirtyDirectories(ctx context.Context) { ticker := time.NewTicker(1 * time.Second) defer ticker.Stop() for { select { case <-ctx.Done(): return case <-ticker.C: s.dirtyDirsMu.Lock() now := time.Now() readyDirs := make([]string, 0) // Find directories that haven't been modified in 10 seconds // This batches changes together (Audiobookshelf approach) for dirPath, lastChange := range s.dirtyDirs { if now.Sub(lastChange) >= 10*time.Second { readyDirs = append(readyDirs, dirPath) delete(s.dirtyDirs, dirPath) } } s.dirtyDirsMu.Unlock() // Process all ready directories in a batch via job queue // Job queue serializes scans - prevents concurrent directory access if len(readyDirs) > 0 { for _, dirPath := range readyDirs { // Create directory scan job with correct params for processDirectoryScanJob() job := &Job{ ID: uuid.New().String(), Type: JobTypeDirectoryScan, Params: map[string]interface{}{ "directory": dirPath, "db": s.db, }, Status: JobStatusPending, } // Enqueue via global worker singleton if WorkerInstance != nil { WorkerInstance.Enqueue(job) fmt.Printf("Enqueued directory scan job: %s\n", dirPath) } else { fmt.Printf("Warning: Worker not initialized, skipping directory scan: %s\n", dirPath) } } } } } } // waitForFileStability checks if a file's mtime has stabilized // Returns true when file is stable (not being modified) // Polls every 3 seconds, times out after 60 seconds // Uses atomic.Bool to prevent race conditions with concurrent checks func (s *MediaScanner) waitForFileStability(filePath string) bool { s.fileStabilityMu.Lock() // Check if already being checked (atomic.Bool prevents race condition) tracking, exists := s.fileStability[filePath] if exists { s.fileStabilityMu.Unlock() // Another goroutine is already checking this file if tracking.Load() { return false // Still being checked } // Tracking exists but completed, remove stale entry s.fileStabilityMu.Lock() delete(s.fileStability, filePath) } // Start tracking with atomic.Bool set to true (checking in progress) trackingFlag := &atomic.Bool{} trackingFlag.Store(true) s.fileStability[filePath] = trackingFlag s.fileStabilityMu.Unlock() // Get initial mtime info, err := os.Stat(filePath) if err != nil { // Clean up tracking entry if file doesn't exist s.fileStabilityMu.Lock() delete(s.fileStability, filePath) s.fileStabilityMu.Unlock() return false } lastMtime := info.ModTime() // Poll every 3 seconds for up to 60 seconds timeout := time.After(60 * time.Second) ticker := time.NewTicker(3 * time.Second) defer ticker.Stop() for { select { case <-timeout: // Timeout - mark as done and clean up trackingFlag.Store(false) s.fileStabilityMu.Lock() delete(s.fileStability, filePath) s.fileStabilityMu.Unlock() return false // File never stabilized case <-ticker.C: info, err := os.Stat(filePath) if err != nil { // File deleted - mark as done and clean up trackingFlag.Store(false) s.fileStabilityMu.Lock() delete(s.fileStability, filePath) s.fileStabilityMu.Unlock() return false } currentMtime := info.ModTime() if currentMtime.Equal(lastMtime) { // File is stable! Mark as done and clean up trackingFlag.Store(false) s.fileStabilityMu.Lock() delete(s.fileStability, filePath) s.fileStabilityMu.Unlock() return true } lastMtime = currentMtime } } } func (s *MediaScanner) scanDirectory(ctx context.Context, dirPath string) { // Debug: Track file processing filesSeen := 0 filesProcessed := 0 // Prevent concurrent scans of ANY directory // Simple mutex is enough - job queue already serializes by directory s.scan_mutex.Lock() defer s.scan_mutex.Unlock() // Find library for this directory var libraryID pgtype.UUID var rootFolder string for _, folder := range s.folders { if strings.HasPrefix(dirPath, folder) { rootFolder = folder if lib, err := s.db.GetLibraryByFolder(ctx, folder); err == nil { libraryID = lib.LibraryID break } } } // Check if libraryID is valid before proceeding if !libraryID.Valid { return } // Walk directory and process new files filepath.WalkDir(dirPath, func(path string, d os.DirEntry, err error) error { if err != nil { return err } if d.IsDir() { return filepath.SkipDir } if !s.isScannableFile(path) { return nil } // Check if file is stable before processing (Audiobookshelf approach) if !s.waitForFileStability(path) { return nil } filesSeen++ relPath := strings.TrimPrefix(path, rootFolder+"/") _, err = s.db.GetMediaItemByFilePath(ctx, database.GetMediaItemByFilePathParams{ FilePath: relPath, LibraryID: libraryID, }) if err == pgx.ErrNoRows { if _, err := s.processMediaFile(ctx, path); err != nil { s.errors++ } else { s.newItems++ } s.totalFiles++ filesProcessed++ } return nil }) // Debug: Report scan results fmt.Printf("DEBUG: scanDirectory of %s - filesSeen: %d, filesProcessed: %d, totalFiles: %d, newItems: %d, errors: %d\n", dirPath, filesSeen, filesProcessed, s.totalFiles, s.newItems, s.errors) } // performInitialScan scans all root folders on startup // This ensures existing files are detected before watching begins func (s *MediaScanner) performInitialScan(ctx context.Context) { fmt.Printf("Performing initial scan of root folders...\n") for _, folder := range s.folders { // Skip if folder doesn't exist if _, err := os.Stat(folder); os.IsNotExist(err) { fmt.Printf("Skipping nonexistent folder: %s\n", folder) continue } // Submit scan job to worker (non-blocking) job := &Job{ ID: uuid.New().String(), Type: JobTypeDirectoryScan, Params: map[string]interface{}{ "directory": folder, "db": s.db, }, Status: JobStatusPending, } if WorkerInstance != nil { WorkerInstance.Enqueue(job) fmt.Printf("Enqueued initial scan job: %s\n", folder) } else { fmt.Printf("Warning: Worker not initialized, skipping initial scan: %s\n", folder) } } fmt.Printf("Initial scan jobs enqueued\n") } func (s *MediaScanner) Close() error { fmt.Printf("Cleaning up scanner resources...\n") // Stop watching if s.watcher != nil { s.watcher.Close() } // Clean up fileStability map to prevent memory leaks s.fileStabilityMu.Lock() s.fileStability = make(map[string]*atomic.Bool) s.fileStabilityMu.Unlock() // Clear dirty directories s.dirtyDirsMu.Lock() s.dirtyDirs = make(map[string]time.Time) s.dirtyDirsMu.Unlock() // Wait for in-progress scan to complete (with timeout) timeout := time.After(5 * time.Second) done := make(chan struct{}) go func() { s.scan_mutex.Lock() s.scan_mutex.Unlock() close(done) }() select { case <-done: fmt.Printf("Scanner cleanup complete\n") case <-timeout: fmt.Printf("Timeout waiting for scan to complete\n") } return nil } func (s *MediaScanner) StartPolling(ctx context.Context) { interval := s.GetPollInterval() if interval <= 0 { fmt.Println("Polling fallback disabled (interval = 0") return } fmt.Printf("Polling fallback started with interval: %v\n", interval) for { ticker := time.NewTicker(interval) defer ticker.Stop() select { case <-ctx.Done(): fmt.Println("Polling fallback stopped") return case <-ticker.C: //Re-read interval each tick for dynamic updates interval = s.GetPollInterval() fmt.Printf("Running polling fallback sync (interval: %v)...\n", interval) if err := s.SyncFilesystemWithDatabase(ctx); err != nil { fmt.Printf("Polling sync error: %v\n", err) } } } } func (s *MediaScanner) SyncFilesystemWithDatabase(ctx context.Context) error { fmt.Println("[POLL-SYNC] Starting filesystem sync with database") for _, folder := range s.folders { lib, err := s.db.GetLibraryByFolder(ctx, folder) if err != nil { fmt.Printf("[POLL-SYNC] Warning: failed to get library for folder %s: %v\n", folder, err) continue } libraryID := lib.LibraryID // Get all media items from database for this library dbItems, err := s.db.ListMediaItemsByLibrary(ctx, libraryID) if err != nil { fmt.Printf("[POLL-SYNC] Warning: failed to get library items: %v\n", err) continue } // Build set of existing file paths from filesystem existingPaths := make(map[string]bool) filepath.WalkDir(folder, func(path string, d os.DirEntry, walkErr error) error { if walkErr != nil { return walkErr } if !d.IsDir() && s.isScannableFile(path) { existingPaths[s.getRelativePath(path)] = true } return nil }) // Check for orphaned items (in DB but not on filesystem) for _, item := range dbItems { if item.FilePath != "" && !existingPaths[item.FilePath] { msg := fmt.Sprintf("[POLL-SYNC] Orphaned media item found: ID=%s, Title=%s, Path=%s", item.ID, item.Title, item.FilePath) s.logger.LogDelete(msg) if err := s.db.DeleteMediaItem(ctx, item.ID); err != nil { errMsg := fmt.Sprintf("[POLL-SYNC] ERROR: failed to delete orphaned item %s: %v", item.Title, err) s.logger.LogDelete(errMsg) s.logger.LogError(errMsg) } else { s.logger.LogDelete(fmt.Sprintf("[POLL-SYNC] SUCCESS: deleted orphaned item '%s'", item.Title)) } } } // Check for new files (on filesystem but not in DB) // This is expensive, so we just check a few representative files // The fsnotify handler should catch most new files for relPath := range existingPaths { // Check if this file exists in DB _, err := s.db.GetMediaItemByFilePath(ctx, database.GetMediaItemByFilePathParams{ FilePath: relPath, LibraryID: libraryID, }) if err == pgx.ErrNoRows { // New file found - scan it absPath := folder + "/" + relPath if _, err := os.Stat(absPath); err == nil { fmt.Printf("[POLL-SYNC] New file detected, scanning: %s\n", absPath) if _, err := s.processMediaFile(ctx, absPath); err != nil { fmt.Printf("[POLL-SYNC] Error scanning new file %s: %v\n", absPath, err) } } } } } fmt.Println("[POLL-SYNC] Filesystem sync completed") return nil } // ============================================ // SCANNER ENHANCEMENTS // ============================================ // calculateFileSHA256 calculates SHA-256 hash using streaming to avoid loading entire file into memory func (s *MediaScanner) calculateFileSHA256(filePath string) (string, error) { file, err := os.Open(filePath) if err != nil { return "", fmt.Errorf("failed to open file: %v", err) } defer file.Close() hasher := sha256.New() if _, err := io.Copy(hasher, file); err != nil { return "", fmt.Errorf("failed to calculate hash: %v", err) } return hex.EncodeToString(hasher.Sum(nil)), nil } // OPFIdentifier represents an identifier from OPF metadata type OPFIdentifier struct { XMLName xml.Name `xml:"identifier"` ID string `xml:"id,attr"` Scheme string `xml:"scheme,attr"` Content string `xml:",chardata"` } // OPFMetadata represents parsed OPF metadata type OPFMetadata struct { XMLName xml.Name `xml:"package"` Version string `xml:"version,attr"` Identifiers []OPFIdentifier `xml:"metadata>identifier"` } // extractOPFIdentifiers extracts identifiers from EPUB OPF file func (s *MediaScanner) extractOPFIdentifiers(epubPath string) (opfIdentifier, opfUUID string, confidence string, err error) { book, err := epub.ReadBook(epubPath) if err != nil { return "", "", "", fmt.Errorf("failed to open EPUB: %v", err) } // Extract OPF identifiers using go-epub library identifiers, err := book.MetadataByKey("identifier") if err != nil || len(identifiers) == 0 { return "", "", "low", nil } var identifier, uuid string for _, id := range identifiers { id = strings.TrimSpace(id) // Check for UUID format (urn:uuid:) if strings.HasPrefix(strings.ToLower(id), "urn:uuid:") { uuid = strings.TrimPrefix(strings.ToLower(id), "urn:uuid:") continue } // Check if it's a plain UUID (8-4-4-4-12 format) if isValidUUID(id) { uuid = id continue } // Check for ISBN if strings.Contains(strings.ToLower(id), "isbn") { isbn := s.extractISBNFromIdentifier(id) if isbn != "" { identifier = isbn continue } } // Use first identifier as fallback if identifier == "" && id != "" { identifier = id } } confidence = s.determineHashConfidence(uuid, identifier) return identifier, uuid, confidence, nil } // isValidUUID checks if string is a valid UUID (8-4-4-4-12 format) func isValidUUID(idStr string) bool { uuidRegex := regexp.MustCompile(`^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$`) return uuidRegex.MatchString(idStr) } // extractISBNFromIdentifier extracts ISBN from identifier string func (s *MediaScanner) extractISBNFromIdentifier(id string) string { id = strings.TrimSpace(id) // Remove "isbn:" prefix if present if strings.HasPrefix(strings.ToLower(id), "isbn:") { id = strings.TrimPrefix(strings.ToLower(id), "isbn:") } // Remove hyphens and spaces isbn := regexp.MustCompile(`[\s-]`).ReplaceAllString(id, "") // Check if it's a valid ISBN-10 or ISBN-13 if len(isbn) == 10 || len(isbn) == 13 { return isbn } return "" } // determineHashConfidence determines confidence level based on available identifiers func (s *MediaScanner) determineHashConfidence(uuid, identifier string) string { if uuid != "" && isValidUUID(uuid) { return "high" } if identifier != "" && (strings.Contains(strings.ToLower(identifier), "isbn") || len(identifier) >= 10) { return "medium" } return "low" } // detectFormatType detects the format type based on file extension and content func (s *MediaScanner) detectFormatType(filePath string) string { base := strings.ToLower(filepath.Base(filePath)) // Check for compound extensions first (like .kepub.epub) if strings.HasSuffix(base, ".kepub.epub") { return "kepub" } ext := strings.ToLower(filepath.Ext(filePath)) switch ext { case ".epub": return "epub" case ".kepub": return "kepub" case ".pdf": return "pdf" case ".cbz", ".cbr", ".cb7", ".cbt": return "comic_archive" case ".mobi": return "mobi" case ".azw", ".azw3": return "kfx" case ".txt": return "txt" case ".fb2": return "fb2" default: return "unknown" } } // extractHashInfo calculates hash and extracts OPF identifiers for a file func (s *MediaScanner) extractHashInfo(filePath string) (*HashInfo, *FormatInfo, error) { // Calculate SHA-256 fileSHA256, err := s.calculateFileSHA256(filePath) if err != nil { return nil, nil, fmt.Errorf("failed to calculate SHA-256: %v", err) } // Get file info info, err := os.Stat(filePath) if err != nil { return nil, nil, fmt.Errorf("failed to get file info: %v", err) } // Extract OPF identifiers for EPUB files var opfIdentifier, opfUUID, confidence string if strings.HasSuffix(strings.ToLower(filePath), ".epub") { opfIdentifier, opfUUID, confidence, err = s.extractOPFIdentifiers(filePath) if err != nil { // Non-fatal error, continue with low confidence confidence = "low" } } hashInfo := &HashInfo{ FileSHA256: fileSHA256, OPFIdentifier: opfIdentifier, OPFUUID: opfUUID, HashConfidence: confidence, } formatInfo := &FormatInfo{ FormatType: s.detectFormatType(filePath), FilePath: filePath, FileSHA256: fileSHA256, FileSizeBytes: info.Size(), MimeType: s.getMimeType(filePath), } return hashInfo, formatInfo, nil } func (s *MediaScanner) getRelativePath(absolutePath string) string { // Get the base folder paths from scanner for _, baseFolder := range s.folders { // Check if path is within this base folder if relPath, ok := strings.CutPrefix(absolutePath, baseFolder); ok { return strings.TrimPrefix(relPath, "/") } } // Fallback: if no match, return as-is (shouldn't happen) return absolutePath }