package services import ( "archive/tar" "archive/zip" "bookhoard/internal/database" "bookhoard/internal/utils" "bytes" "compress/bzip2" "compress/gzip" "context" "crypto/sha256" "encoding/hex" "encoding/xml" "fmt" "image" _ "image/jpeg" _ "image/png" "io" "io/fs" "os" "path/filepath" "regexp" "strconv" "strings" "time" epub "github.com/ArcadiaLin/go-epub" "github.com/bodgit/sevenzip" "github.com/fsnotify/fsnotify" "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgtype" "github.com/nwaples/rardecode" ) // MediaMetadata contains extracted metadata for media files (ebooks, comics, manga) type MediaMetadata struct { Title string Author string Description string Series string SeriesNumber int32 Publisher string PublishDate time.Time Contributors []string CoverPath string ISBN string ASIN string Tags []string Phase1HashInfo *HashInfo Phase1FormatFormats []*FormatInfo } type HashInfo struct { FileSHA256 string OPFIdentifier string OPFUUID string HashConfidence string } type FormatInfo struct { FormatType string FilePath string FileSHA256 string FileSizeBytes int64 MimeType string } // MediaScanner scans library folders for media files (ebooks, comics, manga) type MediaScanner struct { db *database.Queries watcher *fsnotify.Watcher folders []string adminID pgtype.UUID defaultLibraryID pgtype.UUID libraryTypes map[string][]string } // NewMediaScanner creates a new media scanner instance func NewMediaScanner(db *database.Queries) *MediaScanner { watcher, err := fsnotify.NewWatcher() if err != nil { panic(fmt.Sprintf("Failed to create file watcher: %v", err)) } return &MediaScanner{ db: db, watcher: watcher, folders: []string{}, adminID: pgtype.UUID{}, defaultLibraryID: pgtype.UUID{Valid: false}, libraryTypes: make(map[string][]string), } } func (s *MediaScanner) SetAdminID(adminID pgtype.UUID) { s.adminID = adminID } func (s *MediaScanner) SetFolders(folders []string) error { s.folders = folders // Remove old watch if exists if s.watcher != nil { s.watcher.Close() } // Create new watcher watcher, err := fsnotify.NewWatcher() if err != nil { return fmt.Errorf("failed to create watcher: %v", err) } s.watcher = watcher // Build cache of allowed extensions per folder s.libraryTypes = make(map[string][]string) ctx := context.Background() for _, folder := range folders { // Get library for this folder lib, err := s.db.GetLibraryByFolder(ctx, folder) if err != nil { fmt.Printf("Warning: failed to get library for folder %s: %v\n", folder, err) continue } // Get library type with allowed extensions libType, err := s.db.GetLibraryType(ctx, lib.LibraryTypeID) if err != nil { fmt.Printf("Warning: failed to get library type for %s: %v\n", folder, err) continue } // Cache allowed extensions for this folder s.libraryTypes[folder] = libType.AllowedExtensions fmt.Printf("Scanner: Folder %s (type: %s) allows extensions: %v\n", folder, libType.Name, libType.AllowedExtensions) } // Add all folders to watch for _, folder := range folders { if err := s.watcher.Add(folder); err != nil { fmt.Printf("Warning: failed to watch folder %s: %v\n", folder, err) } } return nil } func (s *MediaScanner) ScanFolders(ctx context.Context) error { if len(s.folders) == 0 { return fmt.Errorf("no folders set") } fmt.Printf("Starting scan of %d folders: %v\n", len(s.folders), s.folders) totalFiles := 0 mediaFiles := 0 for _, folder := range s.folders { fmt.Printf("Scanning folder: %s\n", folder) // Check if folder exists if _, err := os.Stat(folder); os.IsNotExist(err) { fmt.Printf("Folder does not exist: %s\n", folder) continue } err := filepath.WalkDir(folder, func(path string, d fs.DirEntry, err error) error { if err != nil { fmt.Printf("Error accessing path %s: %v\n", path, err) return err } totalFiles++ if d.IsDir() { // Also watch subdirectories if err := s.watcher.Add(path); err != nil { fmt.Printf("Warning: failed to watch subdirectory %s: %v\n", path, err) } return nil } // Check if file should be scanned based on library type if s.isScannableFile(path) { mediaFiles++ fmt.Printf("Found media file: %s\n", path) if err := s.processMediaFile(ctx, path); err != nil { fmt.Printf("Error processing media file %s: %v\n", path, err) } else { fmt.Printf("Successfully processed media file: %s\n", path) } } return nil }) if err != nil { return fmt.Errorf("failed to scan folder %s: %v", folder, err) } } fmt.Printf("Scan completed: %d total files scanned, %d media files found\n", totalFiles, mediaFiles) return nil } func (s *MediaScanner) isScannableFile(path string) bool { ext := strings.ToLower(filepath.Ext(path)) // Find which folder this file belongs to var folder string for _, f := range s.folders { if strings.HasPrefix(path, f) { folder = f break } } // If no folder match, don't scan if folder == "" { return false } // Get allowed extensions for this folder's library allowed, ok := s.libraryTypes[folder] if !ok { // No library type info, skip file fmt.Printf("Warning: No library type info for folder %s, skipping %s\n", folder, path) return false } // Check if file extension is allowed for this library type for _, allowedExt := range allowed { if ext == strings.ToLower(allowedExt) { return true } } return false } // extractFolderStructureMetadata extracts metadata from folder paths, prioritizing Calibre structure func (s *MediaScanner) extractFolderStructureMetadata(path, rootFolder string) *MediaMetadata { metadata := &MediaMetadata{} // Get the relative path from root folder relPath, err := filepath.Rel(rootFolder, path) if err != nil { return metadata } // Split into directory components dir := filepath.Dir(relPath) components := strings.Split(dir, string(filepath.Separator)) if len(components) < 2 { return metadata // Not enough structure to extract } // Calibre structure detection // Pattern 1: Author Name/Book Title/ // Pattern 2: Author Name/Series Name/Book Title/ // Pattern 3: Author Name/Series Name, Book #1 - Book Title/ author := strings.TrimSuffix(components[0], "_") // Remove trailing underscore if present metadata.Author = strings.ReplaceAll(author, "_", " ") if len(components) >= 3 { // This might be a series structure possibleSeries := components[1] possibleTitle := components[2] // Check for Calibre series format: "Series Name, Book #1 - Title" seriesMatch := regexp.MustCompile(`^(.*),\s+Book\s+#(\d+)\s*-\s*(.*)$`).FindStringSubmatch(possibleSeries) if len(seriesMatch) == 4 { metadata.Series = strings.ReplaceAll(seriesMatch[1], "_", " ") if seriesNum, err := strconv.ParseInt(seriesMatch[2], 10, 32); err == nil { metadata.SeriesNumber = int32(seriesNum) } metadata.Title = strings.ReplaceAll(possibleTitle, "_", " ") } else { // Simple series structure: Author/Series/Title metadata.Series = strings.ReplaceAll(possibleSeries, "_", " ") metadata.Title = strings.ReplaceAll(possibleTitle, "_", " ") // Try to extract series number from title titleNumMatch := regexp.MustCompile(`^(.*)\s+(\d+)$`).FindStringSubmatch(metadata.Title) if len(titleNumMatch) == 3 { metadata.Title = titleNumMatch[1] if seriesNum, err := strconv.ParseInt(titleNumMatch[2], 10, 32); err == nil { metadata.SeriesNumber = int32(seriesNum) } } } } else { // Simple structure: Author/Title metadata.Title = strings.ReplaceAll(components[1], "_", " ") } return metadata } func (s *MediaScanner) processMediaFile(ctx context.Context, path string) error { fmt.Printf("Processing media file: %s\n", path) // Get file info info, err := os.Stat(path) if err != nil { fmt.Printf("Failed to get file info for %s: %v\n", path, err) return fmt.Errorf("failed to get file info: %v", err) } fmt.Printf("File info for %s: size=%d\n", path, info.Size()) // Check if media item already exists in database existingItem, err := s.getMediaItemByFilePath(ctx, path) if err == nil { fmt.Printf("Media item already exists in database: %s (size: %d vs %d)\n", path, existingItem.FileSize.Int64, info.Size()) // Media item exists, check if file has changed (by size) if existingItem.FileSize.Int64 != info.Size() { fmt.Printf("File size changed, updating media item: %s\n", path) return s.updateMediaItem(ctx, existingItem.ID, path, info) } fmt.Printf("Media item already exists with same size, skipping: %s\n", path) return nil // Skip if already exists and size matches } else if err != pgx.ErrNoRows { fmt.Printf("Database error checking media item existence: %v\n", err) // Some other error occurred return fmt.Errorf("failed to check if media item exists: %v", err) } fmt.Printf("Media item does not exist in database, creating new entry: %s\n", path) // Extract metadata from file first metadata, err := s.extractMetadata(path) if err != nil { fmt.Printf("Warning: failed to extract metadata from %s: %v\n", path, err) metadata = &MediaMetadata{} } // Extract hash information (Phase 2) hashInfo, formatInfo, err := s.extractHashInfo(path) if err != nil { fmt.Printf("Warning: failed to extract hash info from %s: %v\n", path, err) hashInfo = &HashInfo{} formatInfo = &FormatInfo{} } else { metadata.Phase1HashInfo = hashInfo metadata.Phase1FormatFormats = []*FormatInfo{formatInfo} fmt.Printf("Hash info for %s: SHA256=%s, OPF_ID=%s, OPF_UUID=%s, Confidence=%s\n", path, hashInfo.FileSHA256, hashInfo.OPFIdentifier, hashInfo.OPFUUID, hashInfo.HashConfidence) } var comicInfo *ComicInfo var coverImage []byte ext := strings.ToLower(filepath.Ext(path)) if ext == ".cbz" || ext == ".cbr" || ext == ".cb7" || ext == ".cbt" || strings.HasSuffix(strings.ToLower(path), ".tar.gz") || strings.HasSuffix(strings.ToLower(path), ".tar.bz2") || strings.HasSuffix(strings.ToLower(path), ".tgz") || strings.HasSuffix(strings.ToLower(path), ".tbz2") { info, cover, err := extractComicMetadata(path) if err != nil { fmt.Printf("Warning: failed to extract comic metadata from %s: %v\n", path, err) } else { comicInfo = info coverImage = cover if comicInfo.Title != "" && metadata.Title == "" { metadata.Title = comicInfo.Title } if comicInfo.Series != "" && metadata.Series == "" { metadata.Series = comicInfo.Series } if comicInfo.Number > 0 && metadata.SeriesNumber == 0 { metadata.SeriesNumber = int32(comicInfo.Number) } if comicInfo.Publisher != "" && metadata.Publisher == "" { metadata.Publisher = comicInfo.Publisher } if comicInfo.Writer != "" && metadata.Author == "" { metadata.Author = comicInfo.Writer } if len(coverImage) > 0 && metadata.CoverPath == "" { coverPath := path + ".cover.jpg" if err := os.WriteFile(coverPath, coverImage, 0644); err == nil { metadata.CoverPath = coverPath } } fmt.Printf("Extracted comic metadata from %s: title=%s, series=%s, issue=%d\n", path, comicInfo.Title, comicInfo.Series, comicInfo.Number) } } // Try to get metadata from folder structure as fallback/enhancement // Use the root folder that contains this file var rootFolder string for _, folder := range s.folders { if strings.HasPrefix(path, folder) { rootFolder = folder break } } if rootFolder != "" { folderMetadata := s.extractFolderStructureMetadata(path, rootFolder) // Use folder metadata as fallback for missing information if metadata.Title == "" && folderMetadata.Title != "" { metadata.Title = folderMetadata.Title } if metadata.Author == "" && folderMetadata.Author != "" { metadata.Author = folderMetadata.Author } if metadata.Series == "" && folderMetadata.Series != "" { metadata.Series = folderMetadata.Series } if metadata.SeriesNumber == 0 && folderMetadata.SeriesNumber > 0 { metadata.SeriesNumber = folderMetadata.SeriesNumber } } // Final fallback if still missing essential metadata if metadata.Title == "" { metadata.Title = strings.TrimSuffix(filepath.Base(path), filepath.Ext(path)) } if metadata.Author == "" { metadata.Author = "Unknown" } // Find library for this folder var libraryID pgtype.UUID for _, folder := range s.folders { if strings.HasPrefix(path, folder) { lib, err := s.db.GetLibraryByFolder(ctx, folder) if err != nil { return fmt.Errorf("failed to find library for folder %s: %v", folder, err) } libraryID = lib.LibraryID break } } if !libraryID.Valid { return fmt.Errorf("no library found for file path: %s", path) } // Normalize metadata fields for display metadata.Contributors = utils.NormalizeContributors(metadata.Contributors) metadata.Tags = utils.NormalizeTags(metadata.Tags) // Normalize search fields contributorsSearch := utils.NormalizeContributorsSearch(metadata.Contributors) tagsSearch := utils.NormalizeTagsSearch(metadata.Tags) // Create media item in database createdItem, err := s.db.CreateMediaItem(ctx, database.CreateMediaItemParams{ LibraryID: libraryID, Title: metadata.Title, Author: pgtype.Text{String: metadata.Author, Valid: metadata.Author != ""}, Isbn: pgtype.Text{String: utils.NormalizeISBN(metadata.ISBN), Valid: metadata.ISBN != ""}, Asin: pgtype.Text{String: metadata.ASIN, Valid: metadata.ASIN != ""}, Description: pgtype.Text{String: metadata.Description, Valid: metadata.Description != ""}, FilePath: path, FileSize: pgtype.Int8{Int64: info.Size(), Valid: true}, MimeType: pgtype.Text{String: s.getMimeType(path), Valid: true}, CoverImagePath: pgtype.Text{String: metadata.CoverPath, Valid: metadata.CoverPath != ""}, Series: pgtype.Text{String: metadata.Series, Valid: metadata.Series != ""}, SeriesNumber: pgtype.Int4{Int32: metadata.SeriesNumber, Valid: metadata.SeriesNumber > 0}, Publisher: pgtype.Text{String: metadata.Publisher, Valid: metadata.Publisher != ""}, DatePublished: pgtype.Date{Time: metadata.PublishDate, Valid: !metadata.PublishDate.IsZero()}, Contributors: metadata.Contributors, ContributorsSearch: contributorsSearch, Tags: metadata.Tags, TagsSearch: tagsSearch, AddedByAdminID: s.adminID, }) if err != nil { return fmt.Errorf("failed to create media item: %v", err) } // Update hash information (Phase 2) if metadata.Phase1HashInfo != nil && metadata.Phase1HashInfo.FileSHA256 != "" { _, err = s.db.UpdateMediaItemIdentifiers(ctx, database.UpdateMediaItemIdentifiersParams{ ID: createdItem.ID, FileSha256: pgtype.Text{String: metadata.Phase1HashInfo.FileSHA256, Valid: true}, OpfIdentifier: pgtype.Text{String: metadata.Phase1HashInfo.OPFIdentifier, Valid: metadata.Phase1HashInfo.OPFIdentifier != ""}, OpfUuid: pgtype.Text{String: metadata.Phase1HashInfo.OPFUUID, Valid: metadata.Phase1HashInfo.OPFUUID != ""}, HashConfidence: pgtype.Text{String: metadata.Phase1HashInfo.HashConfidence, Valid: true}, }) if err != nil { fmt.Printf("Warning: failed to update hash identifiers for %s: %v\n", path, err) } } // Store format information (Phase 2) for _, format := range metadata.Phase1FormatFormats { _, err = s.db.CreateMediaItemFormat(ctx, database.CreateMediaItemFormatParams{ MediaItemID: createdItem.ID, FormatType: format.FormatType, FilePath: pgtype.Text{String: format.FilePath, Valid: true}, FileSha256: pgtype.Text{String: format.FileSHA256, Valid: true}, FileSizeBytes: pgtype.Int8{Int64: format.FileSizeBytes, Valid: true}, MimeType: pgtype.Text{String: format.MimeType, Valid: true}, }) if err != nil { fmt.Printf("Warning: failed to create format entry for %s: %v\n", path, err) } } return err } func (s *MediaScanner) extractMetadata(path string) (*MediaMetadata, error) { ext := strings.ToLower(filepath.Ext(path)) switch ext { case ".epub": return s.extractEPUBMetadata(path) case ".pdf": return s.extractPDFMetadata(path) default: // For other formats, return basic metadata return &MediaMetadata{ Title: strings.TrimSuffix(filepath.Base(path), ext), }, nil } } func (s *MediaScanner) extractEPUBMetadata(path string) (*MediaMetadata, error) { book, err := epub.ReadBook(path) if err != nil { return nil, fmt.Errorf("failed to open EPUB: %v", err) } metadata := &MediaMetadata{} // Title if title, err := book.Title(); err == nil && title != "" { metadata.Title = title } // Author if authors, err := book.MetadataByKey("creator"); err == nil && len(authors) > 0 { metadata.Author = authors[0] } // Description if descriptions, err := book.MetadataByKey("description"); err == nil && len(descriptions) > 0 { metadata.Description = descriptions[0] } // Publisher if publishers, err := book.MetadataByKey("publisher"); err == nil && len(publishers) > 0 { metadata.Publisher = publishers[0] } // Series and series number (Calibre specific metadata) if series, err := book.MetadataByKey("calibre:series"); err == nil && len(series) > 0 { metadata.Series = series[0] } if seriesIndex, err := book.MetadataByKey("calibre:series_index"); err == nil && len(seriesIndex) > 0 { if index, err := strconv.ParseFloat(seriesIndex[0], 32); err == nil { metadata.SeriesNumber = int32(index) } } // Publish date if dates, err := book.MetadataByKey("date"); err == nil && len(dates) > 0 { if date, err := time.Parse("2006-01-02", dates[0]); err == nil { metadata.PublishDate = date } else { // Try alternative date formats if date, err := time.Parse("2006", dates[0]); err == nil { metadata.PublishDate = date } } } // Contributors if contributors, err := book.MetadataByKey("contributor"); err == nil && len(contributors) > 0 { // Normalize contributors for display metadata.Contributors = utils.NormalizeContributors(contributors) } // ISBN if isbns, err := book.MetadataByKey("identifier"); err == nil && len(isbns) > 0 { for _, isbn := range isbns { if strings.Contains(strings.ToLower(isbn), "isbn") { // Extract ISBN number from identifier like "isbn:978-3-16-148410-0" isbnParts := strings.SplitN(isbn, ":", 2) if len(isbnParts) == 2 { metadata.ISBN = isbnParts[1] break } } if strings.Contains(strings.ToLower(isbn), "asin") { // Extract ASIN from identifier like "asin:B08XXXXX" asinParts := strings.SplitN(isbn, ":", 2) if len(asinParts) == 2 { metadata.ASIN = asinParts[1] } } } } // Tags if tags, err := book.MetadataByKey("subject"); err == nil && len(tags) > 0 { // Normalize tags for display metadata.Tags = utils.NormalizeTags(tags) } return metadata, nil } func (s *MediaScanner) extractPDFMetadata(path string) (*MediaMetadata, error) { // For now, return basic metadata since PDF extraction requires additional libraries // In a future enhancement, you could use libraries like github.com/ledongthuc/pdf filename := strings.TrimSuffix(filepath.Base(path), ".pdf") return &MediaMetadata{ Title: filename, }, nil } // ComicInfo represents metadata from ComicInfo.xml type ComicInfo struct { XMLName xml.Name `xml:"ComicInfo"` Title string `xml:"Title"` Series string `xml:"Series"` Number int `xml:"Number"` Volume int `xml:"Volume"` Publisher string `xml:"Publisher"` Year int `xml:"Year"` Month int `xml:"Month"` Day int `xml:"Day"` Writer string `xml:"Writer"` Penciller string `xml:"Penciller"` Inker string `xml:"Inker"` Colorist string `xml:"Colorist"` Letterer string `xml:"Letterer"` CoverArtist string `xml:"CoverArtist"` Genre string `xml:"Genre"` Tags string `xml:"Tags"` Web string `xml:"Web"` Notes string `xml:"Notes"` } // extractComicMetadata extracts metadata from comic archive (supports .cbz, .cbr, .cb7, .cbt) func extractComicMetadata(filePath string) (*ComicInfo, []byte, error) { ext := strings.ToLower(filepath.Ext(filePath)) switch ext { case ".cbz": return extractZipMetadata(filePath) case ".cbr": return extractRarMetadata(filePath) case ".cb7": return extract7ZipMetadata(filePath) case ".cbt": return extractTarMetadata(filePath) default: return nil, nil, fmt.Errorf("unsupported comic format: %s", ext) } } // archiveFile represents a file in an archive for uniform handling type archiveFile interface { Name() string Open() (io.ReadCloser, error) } // extractMetadataFromArchive extracts ComicInfo.xml and cover image from any archive format func extractMetadataFromArchive(files []archiveFile) (*ComicInfo, []byte, error) { var comicInfo *ComicInfo var coverImage []byte for _, f := range files { if f.Name() == "ComicInfo.xml" { rc, err := f.Open() if err != nil { return nil, nil, fmt.Errorf("failed to open ComicInfo.xml: %w", err) } data, err := io.ReadAll(rc) rc.Close() if err != nil { return nil, nil, fmt.Errorf("failed to read ComicInfo.xml: %w", err) } comicInfo = &ComicInfo{} if err := xml.Unmarshal(data, comicInfo); err != nil { return nil, nil, fmt.Errorf("failed to parse ComicInfo.xml: %w", err) } } if coverImage == nil && isImageFile(f.Name()) { if !strings.Contains(filepath.Dir(f.Name()), string(filepath.Separator)) || filepath.Dir(f.Name()) == "." { rc, err := f.Open() if err != nil { continue } coverImage, err = io.ReadAll(rc) rc.Close() if err == nil { _, _, err = image.Decode(bytes.NewReader(coverImage)) if err != nil { coverImage = nil } } } } } if comicInfo == nil { comicInfo = &ComicInfo{} } return comicInfo, coverImage, nil } // extractZipMetadata extracts metadata from ZIP archives (.cbz) func extractZipMetadata(filePath string) (*ComicInfo, []byte, error) { r, err := zip.OpenReader(filePath) if err != nil { return nil, nil, fmt.Errorf("failed to open ZIP archive: %w", err) } defer r.Close() files := make([]archiveFile, 0, len(r.File)) for _, f := range r.File { files = append(files, &zipFileAdapter{f}) } comicInfo, coverImage, err := extractMetadataFromArchive(files) if err != nil { return nil, nil, err } if comicInfo.Title == "" { basename := filepath.Base(filePath) comicInfo.Title = strings.TrimSuffix(basename, filepath.Ext(basename)) } return comicInfo, coverImage, nil } // zipFileAdapter adapts zip.File to archiveFile interface type zipFileAdapter struct { *zip.File } func (z *zipFileAdapter) Name() string { return z.File.Name } func (z *zipFileAdapter) Open() (io.ReadCloser, error) { return z.File.Open() } // extractRarMetadata extracts metadata from RAR archives (.cbr) func extractRarMetadata(filePath string) (*ComicInfo, []byte, error) { r, err := rardecode.OpenReader(filePath, "") if err != nil { return nil, nil, fmt.Errorf("failed to open RAR archive: %w", err) } defer r.Close() files := make([]archiveFile, 0, 100) for { header, err := r.Next() if err == io.EOF { break } if err != nil { return nil, nil, fmt.Errorf("failed to read RAR entry: %w", err) } data, err := io.ReadAll(r) if err != nil { continue } files = append(files, &rarFileAdapter{ name: header.Name, data: data, }) } comicInfo, coverImage, err := extractMetadataFromArchive(files) if err != nil { return nil, nil, err } if comicInfo.Title == "" { basename := filepath.Base(filePath) comicInfo.Title = strings.TrimSuffix(basename, filepath.Ext(basename)) } return comicInfo, coverImage, nil } // rarFileAdapter stores RAR file data in memory type rarFileAdapter struct { name string data []byte } func (r *rarFileAdapter) Name() string { return r.name } func (r *rarFileAdapter) Open() (io.ReadCloser, error) { return io.NopCloser(bytes.NewReader(r.data)), nil } // extract7ZipMetadata extracts metadata from 7-Zip archives (.cb7) func extract7ZipMetadata(filePath string) (*ComicInfo, []byte, error) { r, err := sevenzip.OpenReader(filePath) if err != nil { return nil, nil, fmt.Errorf("failed to open 7-Zip archive: %w", err) } defer r.Close() files := make([]archiveFile, 0, len(r.File)) for _, f := range r.File { files = append(files, &sevenZipFileAdapter{file: f}) } comicInfo, coverImage, err := extractMetadataFromArchive(files) if err != nil { return nil, nil, err } if comicInfo.Title == "" { basename := filepath.Base(filePath) comicInfo.Title = strings.TrimSuffix(basename, filepath.Ext(basename)) } return comicInfo, coverImage, nil } // sevenZipFileAdapter adapts sevenzip.File to archiveFile interface type sevenZipFileAdapter struct { file *sevenzip.File } func (s *sevenZipFileAdapter) Name() string { return s.file.Name } func (s *sevenZipFileAdapter) Open() (io.ReadCloser, error) { return s.file.Open() } // extractTarMetadata extracts metadata from TAR archives (.cbt) func extractTarMetadata(filePath string) (*ComicInfo, []byte, error) { f, err := os.Open(filePath) if err != nil { return nil, nil, fmt.Errorf("failed to open TAR archive: %w", err) } defer f.Close() var tarReader *tar.Reader if strings.HasSuffix(strings.ToLower(filePath), ".tar.gz") || strings.HasSuffix(strings.ToLower(filePath), ".tgz") { gzReader, err := gzip.NewReader(f) if err != nil { return nil, nil, fmt.Errorf("failed to create gzip reader: %w", err) } defer gzReader.Close() tarReader = tar.NewReader(gzReader) } else if strings.HasSuffix(strings.ToLower(filePath), ".tar.bz2") || strings.HasSuffix(strings.ToLower(filePath), ".tbz2") { bz2Reader := bzip2.NewReader(f) tarReader = tar.NewReader(bz2Reader) } else { tarReader = tar.NewReader(f) } files := make([]archiveFile, 0, 100) for { header, err := tarReader.Next() if err == io.EOF { break } if err != nil { return nil, nil, fmt.Errorf("failed to read TAR entry: %w", err) } if header.Typeflag == tar.TypeReg { data, err := io.ReadAll(tarReader) if err != nil { continue } files = append(files, &tarFileAdapter{ name: header.Name, data: data, }) } } comicInfo, coverImage, err := extractMetadataFromArchive(files) if err != nil { return nil, nil, err } if comicInfo.Title == "" { basename := filepath.Base(filePath) comicInfo.Title = strings.TrimSuffix(basename, filepath.Ext(basename)) } return comicInfo, coverImage, nil } // tarFileAdapter stores TAR file data in memory type tarFileAdapter struct { name string data []byte } func (t *tarFileAdapter) Name() string { return t.name } func (t *tarFileAdapter) Open() (io.ReadCloser, error) { return io.NopCloser(bytes.NewReader(t.data)), nil } // isImageFile checks if a file is an image based on extension func isImageFile(filename string) bool { ext := strings.ToLower(filepath.Ext(filename)) return ext == ".jpg" || ext == ".jpeg" || ext == ".png" || ext == ".gif" } func (s *MediaScanner) updateMediaItem(ctx context.Context, mediaItemID pgtype.UUID, filePath string, info os.FileInfo) error { // Update disabled - scanner creates items but doesn't update return nil } func (s *MediaScanner) getMediaItemByFilePath(ctx context.Context, filePath string) (database.MediaItems, error) { return s.db.GetMediaItemByFilePath(ctx, filePath) } func (s *MediaScanner) getMimeType(path string) string { ext := strings.ToLower(filepath.Ext(path)) switch ext { case ".epub": return "application/epub+zip" case ".pdf": return "application/pdf" case ".mobi": return "application/x-mobipocket-ebook" case ".azw3": return "application/vnd.amazon.ebook" case ".fb2": return "application/x-fictionbook+xml" case ".txt": return "text/plain" case ".cbz": return "application/vnd.comicbook+zip" case ".cbr": return "application/vnd.comicbook-rar" case ".cb7": return "application/x-7z-compressed" case ".cbt": return "application/x-tar" default: return "application/octet-stream" } } func (s *MediaScanner) WatchChanges(ctx context.Context) { go func() { for { select { case event, ok := <-s.watcher.Events: if !ok { return } // Handle new directories - add them to the watcher if event.Has(fsnotify.Create) { info, err := os.Stat(event.Name) if err == nil && info.IsDir() { // Add the new directory to the watcher if err := s.watcher.Add(event.Name); err != nil { fmt.Printf("Warning: failed to watch new directory %s: %v\n", event.Name, err) } else { fmt.Printf("Now watching new directory: %s\n", event.Name) } } } // Handle file modifications and creations if (event.Has(fsnotify.Create) || event.Has(fsnotify.Write)) && s.isScannableFile(event.Name) { fmt.Printf("New/modified media file detected: %s\n", event.Name) if err := s.processMediaFile(ctx, event.Name); err != nil { fmt.Printf("Error processing modified media file %s: %v\n", event.Name, err) } } case err, ok := <-s.watcher.Errors: if !ok { return } fmt.Printf("Watcher error: %v\n", err) case <-ctx.Done(): return } } }() } func (s *MediaScanner) Close() error { if s.watcher != nil { return s.watcher.Close() } return nil } // ============================================ // PHASE 2: SCANNER ENHANCEMENTS (Week 1-2) // ============================================ // calculateFileSHA256 calculates SHA-256 hash using streaming to avoid loading entire file into memory func (s *MediaScanner) calculateFileSHA256(filePath string) (string, error) { file, err := os.Open(filePath) if err != nil { return "", fmt.Errorf("failed to open file: %v", err) } defer file.Close() hasher := sha256.New() if _, err := io.Copy(hasher, file); err != nil { return "", fmt.Errorf("failed to calculate hash: %v", err) } return hex.EncodeToString(hasher.Sum(nil)), nil } // OPFIdentifier represents an identifier from OPF metadata type OPFIdentifier struct { XMLName xml.Name `xml:"identifier"` ID string `xml:"id,attr"` Scheme string `xml:"scheme,attr"` Content string `xml:",chardata"` } // OPFMetadata represents parsed OPF metadata type OPFMetadata struct { XMLName xml.Name `xml:"package"` Version string `xml:"version,attr"` Identifiers []OPFIdentifier `xml:"metadata>identifier"` } // extractOPFIdentifiers extracts identifiers from EPUB OPF file func (s *MediaScanner) extractOPFIdentifiers(epubPath string) (opfIdentifier, opfUUID string, confidence string, err error) { book, err := epub.ReadBook(epubPath) if err != nil { return "", "", "", fmt.Errorf("failed to open EPUB: %v", err) } // Extract OPF identifiers using go-epub library identifiers, err := book.MetadataByKey("identifier") if err != nil || len(identifiers) == 0 { return "", "", "low", nil } var identifier, uuid string for _, id := range identifiers { id = strings.TrimSpace(id) // Check for UUID format (urn:uuid:) if strings.HasPrefix(strings.ToLower(id), "urn:uuid:") { uuid = strings.TrimPrefix(strings.ToLower(id), "urn:uuid:") continue } // Check if it's a plain UUID (8-4-4-4-12 format) if isValidUUID(id) { uuid = id continue } // Check for ISBN if strings.Contains(strings.ToLower(id), "isbn") { isbn := s.extractISBNFromIdentifier(id) if isbn != "" { identifier = isbn continue } } // Use first identifier as fallback if identifier == "" && id != "" { identifier = id } } confidence = s.determineHashConfidence(uuid, identifier) return identifier, uuid, confidence, nil } // isValidUUID checks if string is a valid UUID (8-4-4-4-12 format) func isValidUUID(idStr string) bool { uuidRegex := regexp.MustCompile(`^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$`) return uuidRegex.MatchString(idStr) } // extractISBNFromIdentifier extracts ISBN from identifier string func (s *MediaScanner) extractISBNFromIdentifier(id string) string { id = strings.TrimSpace(id) // Remove "isbn:" prefix if present if strings.HasPrefix(strings.ToLower(id), "isbn:") { id = strings.TrimPrefix(strings.ToLower(id), "isbn:") } // Remove hyphens and spaces isbn := regexp.MustCompile(`[\s-]`).ReplaceAllString(id, "") // Check if it's a valid ISBN-10 or ISBN-13 if len(isbn) == 10 || len(isbn) == 13 { return isbn } return "" } // determineHashConfidence determines confidence level based on available identifiers func (s *MediaScanner) determineHashConfidence(uuid, identifier string) string { if uuid != "" && isValidUUID(uuid) { return "high" } if identifier != "" && (strings.Contains(strings.ToLower(identifier), "isbn") || len(identifier) >= 10) { return "medium" } return "low" } // detectFormatType detects the format type based on file extension and content func (s *MediaScanner) detectFormatType(filePath string) string { base := strings.ToLower(filepath.Base(filePath)) // Check for compound extensions first (like .kepub.epub) if strings.HasSuffix(base, ".kepub.epub") { return "kepub" } ext := strings.ToLower(filepath.Ext(filePath)) switch ext { case ".epub": return "epub" case ".kepub": return "kepub" case ".pdf": return "pdf" case ".cbz", ".cbr", ".cb7", ".cbt": return "comic_archive" case ".mobi": return "mobi" case ".azw", ".azw3": return "kfx" case ".txt": return "txt" case ".fb2": return "fb2" default: return "unknown" } } // extractHashInfo calculates hash and extracts OPF identifiers for a file func (s *MediaScanner) extractHashInfo(filePath string) (*HashInfo, *FormatInfo, error) { // Calculate SHA-256 fileSHA256, err := s.calculateFileSHA256(filePath) if err != nil { return nil, nil, fmt.Errorf("failed to calculate SHA-256: %v", err) } // Get file info info, err := os.Stat(filePath) if err != nil { return nil, nil, fmt.Errorf("failed to get file info: %v", err) } // Extract OPF identifiers for EPUB files var opfIdentifier, opfUUID, confidence string if strings.HasSuffix(strings.ToLower(filePath), ".epub") { opfIdentifier, opfUUID, confidence, err = s.extractOPFIdentifiers(filePath) if err != nil { // Non-fatal error, continue with low confidence confidence = "low" } } hashInfo := &HashInfo{ FileSHA256: fileSHA256, OPFIdentifier: opfIdentifier, OPFUUID: opfUUID, HashConfidence: confidence, } formatInfo := &FormatInfo{ FormatType: s.detectFormatType(filePath), FilePath: filePath, FileSHA256: fileSHA256, FileSizeBytes: info.Size(), MimeType: s.getMimeType(filePath), } return hashInfo, formatInfo, nil }