Phase 2 of scanner enhancement plan Changes: - Add libraryTypes map[string][]string field to EbookScanner - Initialize libraryTypes cache in NewEbookScanner - Build library types cache in SetFolders by querying database - Replace isEbookFile with isScannableFile for library-aware filtering - Update ScanFolders and WatchChanges to use isScannableFile This prevents cross-contamination between library types: - Epub libraries only scan .epub files - Comic libraries only scan .cbz/.cbr files - Manga libraries only scan appropriate formats - Each library type has configurable allowed extensions Files are now filtered based on their library's allowed extensions, ensuring only supported formats are scanned for each library type.
1213 lines
34 KiB
Go
1213 lines
34 KiB
Go
package services
|
|
|
|
import (
|
|
"archive/tar"
|
|
"archive/zip"
|
|
"bookhoard/internal/database"
|
|
"bookhoard/internal/utils"
|
|
"bytes"
|
|
"compress/bzip2"
|
|
"compress/gzip"
|
|
"context"
|
|
"crypto/sha256"
|
|
"encoding/hex"
|
|
"encoding/xml"
|
|
"fmt"
|
|
"image"
|
|
_ "image/jpeg"
|
|
_ "image/png"
|
|
"io"
|
|
"io/fs"
|
|
"os"
|
|
"path/filepath"
|
|
"regexp"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
|
|
epub "github.com/ArcadiaLin/go-epub"
|
|
"github.com/bodgit/sevenzip"
|
|
"github.com/fsnotify/fsnotify"
|
|
"github.com/jackc/pgx/v5"
|
|
"github.com/jackc/pgx/v5/pgtype"
|
|
"github.com/nwaples/rardecode"
|
|
)
|
|
|
|
type EbookMetadata struct {
|
|
Title string
|
|
Author string
|
|
Description string
|
|
Series string
|
|
SeriesNumber int32
|
|
Publisher string
|
|
PublishDate time.Time
|
|
Contributors string
|
|
CoverPath string
|
|
ISBN string
|
|
ASIN string
|
|
Tags string
|
|
|
|
Phase1HashInfo *HashInfo
|
|
Phase1FormatFormats []*FormatInfo
|
|
}
|
|
|
|
type HashInfo struct {
|
|
FileSHA256 string
|
|
OPFIdentifier string
|
|
OPFUUID string
|
|
HashConfidence string
|
|
}
|
|
|
|
type FormatInfo struct {
|
|
FormatType string
|
|
FilePath string
|
|
FileSHA256 string
|
|
FileSizeBytes int64
|
|
MimeType string
|
|
}
|
|
|
|
type EbookScanner struct {
|
|
db *database.Queries
|
|
watcher *fsnotify.Watcher
|
|
folders []string
|
|
adminID pgtype.UUID
|
|
defaultLibraryID pgtype.UUID
|
|
libraryTypes map[string][]string
|
|
}
|
|
|
|
func NewEbookScanner(db *database.Queries) *EbookScanner {
|
|
watcher, err := fsnotify.NewWatcher()
|
|
if err != nil {
|
|
panic(fmt.Sprintf("Failed to create file watcher: %v", err))
|
|
}
|
|
|
|
return &EbookScanner{
|
|
db: db,
|
|
watcher: watcher,
|
|
folders: []string{},
|
|
adminID: pgtype.UUID{},
|
|
defaultLibraryID: pgtype.UUID{Valid: false},
|
|
libraryTypes: make(map[string][]string),
|
|
}
|
|
}
|
|
|
|
func (s *EbookScanner) SetAdminID(adminID pgtype.UUID) {
|
|
s.adminID = adminID
|
|
}
|
|
|
|
func (s *EbookScanner) SetFolders(folders []string) error {
|
|
s.folders = folders
|
|
|
|
// Remove old watch if exists
|
|
if s.watcher != nil {
|
|
s.watcher.Close()
|
|
}
|
|
|
|
// Create new watcher
|
|
watcher, err := fsnotify.NewWatcher()
|
|
if err != nil {
|
|
return fmt.Errorf("failed to create watcher: %v", err)
|
|
}
|
|
s.watcher = watcher
|
|
|
|
// Build cache of allowed extensions per folder
|
|
s.libraryTypes = make(map[string][]string)
|
|
ctx := context.Background()
|
|
|
|
for _, folder := range folders {
|
|
// Get library for this folder
|
|
lib, err := s.db.GetLibraryByFolder(ctx, folder)
|
|
if err != nil {
|
|
fmt.Printf("Warning: failed to get library for folder %s: %v\n", folder, err)
|
|
continue
|
|
}
|
|
|
|
// Get library type with allowed extensions
|
|
libType, err := s.db.GetLibraryType(ctx, lib.LibraryTypeID)
|
|
if err != nil {
|
|
fmt.Printf("Warning: failed to get library type for %s: %v\n", folder, err)
|
|
continue
|
|
}
|
|
|
|
// Cache allowed extensions for this folder
|
|
s.libraryTypes[folder] = libType.AllowedExtensions
|
|
fmt.Printf("Scanner: Folder %s (type: %s) allows extensions: %v\n",
|
|
folder, libType.Name, libType.AllowedExtensions)
|
|
}
|
|
|
|
// Add all folders to watch
|
|
for _, folder := range folders {
|
|
if err := s.watcher.Add(folder); err != nil {
|
|
fmt.Printf("Warning: failed to watch folder %s: %v\n", folder, err)
|
|
}
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func (s *EbookScanner) ScanFolders(ctx context.Context) error {
|
|
if len(s.folders) == 0 {
|
|
return fmt.Errorf("no folders set")
|
|
}
|
|
|
|
fmt.Printf("Starting scan of %d folders: %v\n", len(s.folders), s.folders)
|
|
|
|
totalFiles := 0
|
|
ebookFiles := 0
|
|
|
|
for _, folder := range s.folders {
|
|
fmt.Printf("Scanning folder: %s\n", folder)
|
|
|
|
// Check if folder exists
|
|
if _, err := os.Stat(folder); os.IsNotExist(err) {
|
|
fmt.Printf("Folder does not exist: %s\n", folder)
|
|
continue
|
|
}
|
|
|
|
err := filepath.WalkDir(folder, func(path string, d fs.DirEntry, err error) error {
|
|
if err != nil {
|
|
fmt.Printf("Error accessing path %s: %v\n", path, err)
|
|
return err
|
|
}
|
|
|
|
totalFiles++
|
|
|
|
if d.IsDir() {
|
|
// Also watch subdirectories
|
|
if err := s.watcher.Add(path); err != nil {
|
|
fmt.Printf("Warning: failed to watch subdirectory %s: %v\n", path, err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// Check if file should be scanned based on library type
|
|
if s.isScannableFile(path) {
|
|
ebookFiles++
|
|
fmt.Printf("Found ebook file: %s\n", path)
|
|
if err := s.processEbookFile(ctx, path); err != nil {
|
|
fmt.Printf("Error processing ebook %s: %v\n", path, err)
|
|
} else {
|
|
fmt.Printf("Successfully processed ebook: %s\n", path)
|
|
}
|
|
}
|
|
|
|
return nil
|
|
})
|
|
if err != nil {
|
|
return fmt.Errorf("failed to scan folder %s: %v", folder, err)
|
|
}
|
|
}
|
|
|
|
fmt.Printf("Scan completed: %d total files scanned, %d ebook files found\n", totalFiles, ebookFiles)
|
|
return nil
|
|
}
|
|
|
|
func (s *EbookScanner) isEbookFile(path string) bool {
|
|
ext := strings.ToLower(filepath.Ext(path))
|
|
switch ext {
|
|
case ".epub", ".pdf", ".mobi", ".azw3", ".fb2", ".txt":
|
|
return true
|
|
default:
|
|
return false
|
|
}
|
|
}
|
|
|
|
func (s *EbookScanner) isScannableFile(path string) bool {
|
|
ext := strings.ToLower(filepath.Ext(path))
|
|
|
|
// Find which folder this file belongs to
|
|
var folder string
|
|
for _, f := range s.folders {
|
|
if strings.HasPrefix(path, f) {
|
|
folder = f
|
|
break
|
|
}
|
|
}
|
|
|
|
// If no folder match, don't scan
|
|
if folder == "" {
|
|
return false
|
|
}
|
|
|
|
// Get allowed extensions for this folder's library
|
|
allowed, ok := s.libraryTypes[folder]
|
|
if !ok {
|
|
// No library type info, skip file
|
|
fmt.Printf("Warning: No library type info for folder %s, skipping %s\n", folder, path)
|
|
return false
|
|
}
|
|
|
|
// Check if file extension is allowed for this library type
|
|
for _, allowedExt := range allowed {
|
|
if ext == strings.ToLower(allowedExt) {
|
|
return true
|
|
}
|
|
}
|
|
|
|
return false
|
|
}
|
|
|
|
// extractFolderStructureMetadata extracts metadata from folder paths, prioritizing Calibre structure
|
|
func (s *EbookScanner) extractFolderStructureMetadata(path, rootFolder string) *EbookMetadata {
|
|
metadata := &EbookMetadata{}
|
|
|
|
// Get the relative path from root folder
|
|
relPath, err := filepath.Rel(rootFolder, path)
|
|
if err != nil {
|
|
return metadata
|
|
}
|
|
|
|
// Split into directory components
|
|
dir := filepath.Dir(relPath)
|
|
components := strings.Split(dir, string(filepath.Separator))
|
|
|
|
if len(components) < 2 {
|
|
return metadata // Not enough structure to extract
|
|
}
|
|
|
|
// Calibre structure detection
|
|
// Pattern 1: Author Name/Book Title/
|
|
// Pattern 2: Author Name/Series Name/Book Title/
|
|
// Pattern 3: Author Name/Series Name, Book #1 - Book Title/
|
|
|
|
author := strings.TrimSuffix(components[0], "_") // Remove trailing underscore if present
|
|
metadata.Author = strings.ReplaceAll(author, "_", " ")
|
|
|
|
if len(components) >= 3 {
|
|
// This might be a series structure
|
|
possibleSeries := components[1]
|
|
possibleTitle := components[2]
|
|
|
|
// Check for Calibre series format: "Series Name, Book #1 - Title"
|
|
seriesMatch := regexp.MustCompile(`^(.*),\s+Book\s+#(\d+)\s*-\s*(.*)$`).FindStringSubmatch(possibleSeries)
|
|
if len(seriesMatch) == 4 {
|
|
metadata.Series = strings.ReplaceAll(seriesMatch[1], "_", " ")
|
|
if seriesNum, err := strconv.ParseInt(seriesMatch[2], 10, 32); err == nil {
|
|
metadata.SeriesNumber = int32(seriesNum)
|
|
}
|
|
metadata.Title = strings.ReplaceAll(possibleTitle, "_", " ")
|
|
} else {
|
|
// Simple series structure: Author/Series/Title
|
|
metadata.Series = strings.ReplaceAll(possibleSeries, "_", " ")
|
|
metadata.Title = strings.ReplaceAll(possibleTitle, "_", " ")
|
|
|
|
// Try to extract series number from title
|
|
titleNumMatch := regexp.MustCompile(`^(.*)\s+(\d+)$`).FindStringSubmatch(metadata.Title)
|
|
if len(titleNumMatch) == 3 {
|
|
metadata.Title = titleNumMatch[1]
|
|
if seriesNum, err := strconv.ParseInt(titleNumMatch[2], 10, 32); err == nil {
|
|
metadata.SeriesNumber = int32(seriesNum)
|
|
}
|
|
}
|
|
}
|
|
} else {
|
|
// Simple structure: Author/Title
|
|
metadata.Title = strings.ReplaceAll(components[1], "_", " ")
|
|
}
|
|
|
|
return metadata
|
|
}
|
|
|
|
func (s *EbookScanner) processEbookFile(ctx context.Context, path string) error {
|
|
fmt.Printf("Processing ebook file: %s\n", path)
|
|
|
|
// Get file info
|
|
info, err := os.Stat(path)
|
|
if err != nil {
|
|
fmt.Printf("Failed to get file info for %s: %v\n", path, err)
|
|
return fmt.Errorf("failed to get file info: %v", err)
|
|
}
|
|
|
|
fmt.Printf("File info for %s: size=%d\n", path, info.Size())
|
|
|
|
// Check if ebook already exists in database
|
|
existingEbook, err := s.getEbookByFilePath(ctx, path)
|
|
if err == nil {
|
|
fmt.Printf("Ebook already exists in database: %s (size: %d vs %d)\n", path, existingEbook.FileSize.Int64, info.Size())
|
|
// Ebook exists, check if file has changed (by size)
|
|
if existingEbook.FileSize.Int64 != info.Size() {
|
|
fmt.Printf("File size changed, updating ebook: %s\n", path)
|
|
return s.updateEbook(ctx, existingEbook.ID, path, info)
|
|
}
|
|
fmt.Printf("Ebook already exists with same size, skipping: %s\n", path)
|
|
return nil // Skip if already exists and size matches
|
|
} else if err != pgx.ErrNoRows {
|
|
fmt.Printf("Database error checking ebook existence: %v\n", err)
|
|
// Some other error occurred
|
|
return fmt.Errorf("failed to check if ebook exists: %v", err)
|
|
}
|
|
fmt.Printf("Ebook does not exist in database, creating new entry: %s\n", path)
|
|
|
|
// Extract metadata from file first
|
|
metadata, err := s.extractMetadata(path)
|
|
if err != nil {
|
|
fmt.Printf("Warning: failed to extract metadata from %s: %v\n", path, err)
|
|
metadata = &EbookMetadata{}
|
|
}
|
|
|
|
// Extract hash information (Phase 2)
|
|
hashInfo, formatInfo, err := s.extractHashInfo(path)
|
|
if err != nil {
|
|
fmt.Printf("Warning: failed to extract hash info from %s: %v\n", path, err)
|
|
hashInfo = &HashInfo{}
|
|
formatInfo = &FormatInfo{}
|
|
} else {
|
|
metadata.Phase1HashInfo = hashInfo
|
|
metadata.Phase1FormatFormats = []*FormatInfo{formatInfo}
|
|
fmt.Printf("Hash info for %s: SHA256=%s, OPF_ID=%s, OPF_UUID=%s, Confidence=%s\n",
|
|
path, hashInfo.FileSHA256, hashInfo.OPFIdentifier, hashInfo.OPFUUID, hashInfo.HashConfidence)
|
|
}
|
|
|
|
var comicInfo *ComicInfo
|
|
var coverImage []byte
|
|
|
|
ext := strings.ToLower(filepath.Ext(path))
|
|
if ext == ".cbz" || ext == ".cbr" || ext == ".cb7" || ext == ".cbt" ||
|
|
strings.HasSuffix(strings.ToLower(path), ".tar.gz") ||
|
|
strings.HasSuffix(strings.ToLower(path), ".tar.bz2") ||
|
|
strings.HasSuffix(strings.ToLower(path), ".tgz") ||
|
|
strings.HasSuffix(strings.ToLower(path), ".tbz2") {
|
|
info, cover, err := extractComicMetadata(path)
|
|
if err != nil {
|
|
fmt.Printf("Warning: failed to extract comic metadata from %s: %v\n", path, err)
|
|
} else {
|
|
comicInfo = info
|
|
coverImage = cover
|
|
if comicInfo.Title != "" && metadata.Title == "" {
|
|
metadata.Title = comicInfo.Title
|
|
}
|
|
if comicInfo.Series != "" && metadata.Series == "" {
|
|
metadata.Series = comicInfo.Series
|
|
}
|
|
if comicInfo.Number > 0 && metadata.SeriesNumber == 0 {
|
|
metadata.SeriesNumber = int32(comicInfo.Number)
|
|
}
|
|
if comicInfo.Publisher != "" && metadata.Publisher == "" {
|
|
metadata.Publisher = comicInfo.Publisher
|
|
}
|
|
if comicInfo.Writer != "" && metadata.Author == "" {
|
|
metadata.Author = comicInfo.Writer
|
|
}
|
|
if len(coverImage) > 0 && metadata.CoverPath == "" {
|
|
coverPath := path + ".cover.jpg"
|
|
if err := os.WriteFile(coverPath, coverImage, 0644); err == nil {
|
|
metadata.CoverPath = coverPath
|
|
}
|
|
}
|
|
fmt.Printf("Extracted comic metadata from %s: title=%s, series=%s, issue=%d\n",
|
|
path, comicInfo.Title, comicInfo.Series, comicInfo.Number)
|
|
}
|
|
}
|
|
|
|
// Try to get metadata from folder structure as fallback/enhancement
|
|
// Use the root folder that contains this file
|
|
var rootFolder string
|
|
for _, folder := range s.folders {
|
|
if strings.HasPrefix(path, folder) {
|
|
rootFolder = folder
|
|
break
|
|
}
|
|
}
|
|
|
|
if rootFolder != "" {
|
|
folderMetadata := s.extractFolderStructureMetadata(path, rootFolder)
|
|
|
|
// Use folder metadata as fallback for missing information
|
|
if metadata.Title == "" && folderMetadata.Title != "" {
|
|
metadata.Title = folderMetadata.Title
|
|
}
|
|
if metadata.Author == "" && folderMetadata.Author != "" {
|
|
metadata.Author = folderMetadata.Author
|
|
}
|
|
if metadata.Series == "" && folderMetadata.Series != "" {
|
|
metadata.Series = folderMetadata.Series
|
|
}
|
|
if metadata.SeriesNumber == 0 && folderMetadata.SeriesNumber > 0 {
|
|
metadata.SeriesNumber = folderMetadata.SeriesNumber
|
|
}
|
|
}
|
|
|
|
// Final fallback if still missing essential metadata
|
|
if metadata.Title == "" {
|
|
metadata.Title = strings.TrimSuffix(filepath.Base(path), filepath.Ext(path))
|
|
}
|
|
if metadata.Author == "" {
|
|
metadata.Author = "Unknown"
|
|
}
|
|
|
|
// Find library for this folder
|
|
var libraryID pgtype.UUID
|
|
for _, folder := range s.folders {
|
|
if strings.HasPrefix(path, folder) {
|
|
lib, err := s.db.GetLibraryByFolder(ctx, folder)
|
|
if err != nil {
|
|
return fmt.Errorf("failed to find library for folder %s: %v", folder, err)
|
|
}
|
|
libraryID = lib.LibraryID
|
|
break
|
|
}
|
|
}
|
|
|
|
if !libraryID.Valid {
|
|
return fmt.Errorf("no library found for file path: %s", path)
|
|
}
|
|
|
|
// Create media item in database
|
|
createdItem, err := s.db.CreateMediaItem(ctx, database.CreateMediaItemParams{
|
|
LibraryID: libraryID,
|
|
Title: metadata.Title,
|
|
Author: pgtype.Text{String: metadata.Author, Valid: metadata.Author != ""},
|
|
Isbn: pgtype.Text{String: utils.NormalizeISBN(metadata.ISBN), Valid: metadata.ISBN != ""},
|
|
Asin: pgtype.Text{String: metadata.ASIN, Valid: metadata.ASIN != ""},
|
|
Description: pgtype.Text{String: metadata.Description, Valid: metadata.Description != ""},
|
|
FilePath: path,
|
|
FileSize: pgtype.Int8{Int64: info.Size(), Valid: true},
|
|
MimeType: pgtype.Text{String: s.getMimeType(path), Valid: true},
|
|
CoverImagePath: pgtype.Text{String: metadata.CoverPath, Valid: metadata.CoverPath != ""},
|
|
Series: pgtype.Text{String: metadata.Series, Valid: metadata.Series != ""},
|
|
SeriesNumber: pgtype.Int4{Int32: metadata.SeriesNumber, Valid: metadata.SeriesNumber > 0},
|
|
Publisher: pgtype.Text{String: metadata.Publisher, Valid: metadata.Publisher != ""},
|
|
DatePublished: pgtype.Date{Time: metadata.PublishDate, Valid: !metadata.PublishDate.IsZero()},
|
|
Contributors: pgtype.Text{String: metadata.Contributors, Valid: metadata.Contributors != ""},
|
|
Tags: pgtype.Text{String: metadata.Tags, Valid: metadata.Tags != ""},
|
|
AddedByAdminID: s.adminID,
|
|
})
|
|
if err != nil {
|
|
return fmt.Errorf("failed to create media item: %v", err)
|
|
}
|
|
|
|
// Update hash information (Phase 2)
|
|
if metadata.Phase1HashInfo != nil && metadata.Phase1HashInfo.FileSHA256 != "" {
|
|
_, err = s.db.UpdateMediaItemIdentifiers(ctx, database.UpdateMediaItemIdentifiersParams{
|
|
ID: createdItem.ID,
|
|
FileSha256: pgtype.Text{String: metadata.Phase1HashInfo.FileSHA256, Valid: true},
|
|
OpfIdentifier: pgtype.Text{String: metadata.Phase1HashInfo.OPFIdentifier, Valid: metadata.Phase1HashInfo.OPFIdentifier != ""},
|
|
OpfUuid: pgtype.Text{String: metadata.Phase1HashInfo.OPFUUID, Valid: metadata.Phase1HashInfo.OPFUUID != ""},
|
|
HashConfidence: pgtype.Text{String: metadata.Phase1HashInfo.HashConfidence, Valid: true},
|
|
})
|
|
if err != nil {
|
|
fmt.Printf("Warning: failed to update hash identifiers for %s: %v\n", path, err)
|
|
}
|
|
}
|
|
|
|
// Store format information (Phase 2)
|
|
for _, format := range metadata.Phase1FormatFormats {
|
|
_, err = s.db.CreateMediaItemFormat(ctx, database.CreateMediaItemFormatParams{
|
|
MediaItemID: createdItem.ID,
|
|
FormatType: format.FormatType,
|
|
FilePath: pgtype.Text{String: format.FilePath, Valid: true},
|
|
FileSha256: pgtype.Text{String: format.FileSHA256, Valid: true},
|
|
FileSizeBytes: pgtype.Int8{Int64: format.FileSizeBytes, Valid: true},
|
|
MimeType: pgtype.Text{String: format.MimeType, Valid: true},
|
|
})
|
|
if err != nil {
|
|
fmt.Printf("Warning: failed to create format entry for %s: %v\n", path, err)
|
|
}
|
|
}
|
|
|
|
return err
|
|
}
|
|
|
|
func (s *EbookScanner) extractMetadata(path string) (*EbookMetadata, error) {
|
|
ext := strings.ToLower(filepath.Ext(path))
|
|
|
|
switch ext {
|
|
case ".epub":
|
|
return s.extractEPUBMetadata(path)
|
|
case ".pdf":
|
|
return s.extractPDFMetadata(path)
|
|
default:
|
|
// For other formats, return basic metadata
|
|
return &EbookMetadata{
|
|
Title: strings.TrimSuffix(filepath.Base(path), ext),
|
|
}, nil
|
|
}
|
|
}
|
|
|
|
func (s *EbookScanner) extractEPUBMetadata(path string) (*EbookMetadata, error) {
|
|
book, err := epub.ReadBook(path)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to open EPUB: %v", err)
|
|
}
|
|
|
|
metadata := &EbookMetadata{}
|
|
|
|
// Title
|
|
if title, err := book.Title(); err == nil && title != "" {
|
|
metadata.Title = title
|
|
}
|
|
|
|
// Author
|
|
if authors, err := book.MetadataByKey("creator"); err == nil && len(authors) > 0 {
|
|
metadata.Author = authors[0]
|
|
}
|
|
|
|
// Description
|
|
if descriptions, err := book.MetadataByKey("description"); err == nil && len(descriptions) > 0 {
|
|
metadata.Description = descriptions[0]
|
|
}
|
|
|
|
// Publisher
|
|
if publishers, err := book.MetadataByKey("publisher"); err == nil && len(publishers) > 0 {
|
|
metadata.Publisher = publishers[0]
|
|
}
|
|
|
|
// Series and series number (Calibre specific metadata)
|
|
if series, err := book.MetadataByKey("calibre:series"); err == nil && len(series) > 0 {
|
|
metadata.Series = series[0]
|
|
}
|
|
if seriesIndex, err := book.MetadataByKey("calibre:series_index"); err == nil && len(seriesIndex) > 0 {
|
|
if index, err := strconv.ParseFloat(seriesIndex[0], 32); err == nil {
|
|
metadata.SeriesNumber = int32(index)
|
|
}
|
|
}
|
|
|
|
// Publish date
|
|
if dates, err := book.MetadataByKey("date"); err == nil && len(dates) > 0 {
|
|
if date, err := time.Parse("2006-01-02", dates[0]); err == nil {
|
|
metadata.PublishDate = date
|
|
} else {
|
|
// Try alternative date formats
|
|
if date, err := time.Parse("2006", dates[0]); err == nil {
|
|
metadata.PublishDate = date
|
|
}
|
|
}
|
|
}
|
|
|
|
// Contributors
|
|
if contributors, err := book.MetadataByKey("contributor"); err == nil && len(contributors) > 0 {
|
|
metadata.Contributors = strings.Join(contributors, ", ")
|
|
}
|
|
|
|
// ISBN
|
|
if isbns, err := book.MetadataByKey("identifier"); err == nil && len(isbns) > 0 {
|
|
for _, isbn := range isbns {
|
|
if strings.Contains(strings.ToLower(isbn), "isbn") {
|
|
// Extract ISBN number from identifier like "isbn:978-3-16-148410-0"
|
|
isbnParts := strings.SplitN(isbn, ":", 2)
|
|
if len(isbnParts) == 2 {
|
|
metadata.ISBN = isbnParts[1]
|
|
break
|
|
}
|
|
}
|
|
if strings.Contains(strings.ToLower(isbn), "asin") {
|
|
// Extract ASIN from identifier like "asin:B08XXXXX"
|
|
asinParts := strings.SplitN(isbn, ":", 2)
|
|
if len(asinParts) == 2 {
|
|
metadata.ASIN = asinParts[1]
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// Tags
|
|
if tags, err := book.MetadataByKey("subject"); err == nil && len(tags) > 0 {
|
|
metadata.Tags = strings.Join(tags, ", ")
|
|
}
|
|
|
|
return metadata, nil
|
|
}
|
|
|
|
func (s *EbookScanner) extractPDFMetadata(path string) (*EbookMetadata, error) {
|
|
// For now, return basic metadata since PDF extraction requires additional libraries
|
|
// In a future enhancement, you could use libraries like github.com/ledongthuc/pdf
|
|
filename := strings.TrimSuffix(filepath.Base(path), ".pdf")
|
|
|
|
return &EbookMetadata{
|
|
Title: filename,
|
|
}, nil
|
|
}
|
|
|
|
// ComicInfo represents metadata from ComicInfo.xml
|
|
type ComicInfo struct {
|
|
XMLName xml.Name `xml:"ComicInfo"`
|
|
Title string `xml:"Title"`
|
|
Series string `xml:"Series"`
|
|
Number int `xml:"Number"`
|
|
Volume int `xml:"Volume"`
|
|
Publisher string `xml:"Publisher"`
|
|
Year int `xml:"Year"`
|
|
Month int `xml:"Month"`
|
|
Day int `xml:"Day"`
|
|
Writer string `xml:"Writer"`
|
|
Penciller string `xml:"Penciller"`
|
|
Inker string `xml:"Inker"`
|
|
Colorist string `xml:"Colorist"`
|
|
Letterer string `xml:"Letterer"`
|
|
CoverArtist string `xml:"CoverArtist"`
|
|
Genre string `xml:"Genre"`
|
|
Tags string `xml:"Tags"`
|
|
Web string `xml:"Web"`
|
|
Notes string `xml:"Notes"`
|
|
}
|
|
|
|
// extractComicMetadata extracts metadata from comic archive (supports .cbz, .cbr, .cb7, .cbt)
|
|
func extractComicMetadata(filePath string) (*ComicInfo, []byte, error) {
|
|
ext := strings.ToLower(filepath.Ext(filePath))
|
|
|
|
switch ext {
|
|
case ".cbz":
|
|
return extractZipMetadata(filePath)
|
|
case ".cbr":
|
|
return extractRarMetadata(filePath)
|
|
case ".cb7":
|
|
return extract7ZipMetadata(filePath)
|
|
case ".cbt":
|
|
return extractTarMetadata(filePath)
|
|
default:
|
|
return nil, nil, fmt.Errorf("unsupported comic format: %s", ext)
|
|
}
|
|
}
|
|
|
|
// archiveFile represents a file in an archive for uniform handling
|
|
type archiveFile interface {
|
|
Name() string
|
|
Open() (io.ReadCloser, error)
|
|
}
|
|
|
|
// extractMetadataFromArchive extracts ComicInfo.xml and cover image from any archive format
|
|
func extractMetadataFromArchive(files []archiveFile) (*ComicInfo, []byte, error) {
|
|
var comicInfo *ComicInfo
|
|
var coverImage []byte
|
|
|
|
for _, f := range files {
|
|
if f.Name() == "ComicInfo.xml" {
|
|
rc, err := f.Open()
|
|
if err != nil {
|
|
return nil, nil, fmt.Errorf("failed to open ComicInfo.xml: %w", err)
|
|
}
|
|
|
|
data, err := io.ReadAll(rc)
|
|
rc.Close()
|
|
if err != nil {
|
|
return nil, nil, fmt.Errorf("failed to read ComicInfo.xml: %w", err)
|
|
}
|
|
|
|
comicInfo = &ComicInfo{}
|
|
if err := xml.Unmarshal(data, comicInfo); err != nil {
|
|
return nil, nil, fmt.Errorf("failed to parse ComicInfo.xml: %w", err)
|
|
}
|
|
}
|
|
|
|
if coverImage == nil && isImageFile(f.Name()) {
|
|
if !strings.Contains(filepath.Dir(f.Name()), string(filepath.Separator)) ||
|
|
filepath.Dir(f.Name()) == "." {
|
|
rc, err := f.Open()
|
|
if err != nil {
|
|
continue
|
|
}
|
|
|
|
coverImage, err = io.ReadAll(rc)
|
|
rc.Close()
|
|
if err == nil {
|
|
_, _, err = image.Decode(bytes.NewReader(coverImage))
|
|
if err != nil {
|
|
coverImage = nil
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
if comicInfo == nil {
|
|
comicInfo = &ComicInfo{}
|
|
}
|
|
|
|
return comicInfo, coverImage, nil
|
|
}
|
|
|
|
// extractZipMetadata extracts metadata from ZIP archives (.cbz)
|
|
func extractZipMetadata(filePath string) (*ComicInfo, []byte, error) {
|
|
r, err := zip.OpenReader(filePath)
|
|
if err != nil {
|
|
return nil, nil, fmt.Errorf("failed to open ZIP archive: %w", err)
|
|
}
|
|
defer r.Close()
|
|
|
|
files := make([]archiveFile, 0, len(r.File))
|
|
for _, f := range r.File {
|
|
files = append(files, &zipFileAdapter{f})
|
|
}
|
|
|
|
comicInfo, coverImage, err := extractMetadataFromArchive(files)
|
|
if err != nil {
|
|
return nil, nil, err
|
|
}
|
|
|
|
if comicInfo.Title == "" {
|
|
basename := filepath.Base(filePath)
|
|
comicInfo.Title = strings.TrimSuffix(basename, filepath.Ext(basename))
|
|
}
|
|
|
|
return comicInfo, coverImage, nil
|
|
}
|
|
|
|
// zipFileAdapter adapts zip.File to archiveFile interface
|
|
type zipFileAdapter struct {
|
|
*zip.File
|
|
}
|
|
|
|
func (z *zipFileAdapter) Name() string {
|
|
return z.File.Name
|
|
}
|
|
|
|
func (z *zipFileAdapter) Open() (io.ReadCloser, error) {
|
|
return z.File.Open()
|
|
}
|
|
|
|
// extractRarMetadata extracts metadata from RAR archives (.cbr)
|
|
func extractRarMetadata(filePath string) (*ComicInfo, []byte, error) {
|
|
r, err := rardecode.OpenReader(filePath, "")
|
|
if err != nil {
|
|
return nil, nil, fmt.Errorf("failed to open RAR archive: %w", err)
|
|
}
|
|
defer r.Close()
|
|
|
|
files := make([]archiveFile, 0, 100)
|
|
for {
|
|
header, err := r.Next()
|
|
if err == io.EOF {
|
|
break
|
|
}
|
|
if err != nil {
|
|
return nil, nil, fmt.Errorf("failed to read RAR entry: %w", err)
|
|
}
|
|
|
|
data, err := io.ReadAll(r)
|
|
if err != nil {
|
|
continue
|
|
}
|
|
files = append(files, &rarFileAdapter{
|
|
name: header.Name,
|
|
data: data,
|
|
})
|
|
}
|
|
|
|
comicInfo, coverImage, err := extractMetadataFromArchive(files)
|
|
if err != nil {
|
|
return nil, nil, err
|
|
}
|
|
|
|
if comicInfo.Title == "" {
|
|
basename := filepath.Base(filePath)
|
|
comicInfo.Title = strings.TrimSuffix(basename, filepath.Ext(basename))
|
|
}
|
|
|
|
return comicInfo, coverImage, nil
|
|
}
|
|
|
|
// rarFileAdapter stores RAR file data in memory
|
|
type rarFileAdapter struct {
|
|
name string
|
|
data []byte
|
|
}
|
|
|
|
func (r *rarFileAdapter) Name() string {
|
|
return r.name
|
|
}
|
|
|
|
func (r *rarFileAdapter) Open() (io.ReadCloser, error) {
|
|
return io.NopCloser(bytes.NewReader(r.data)), nil
|
|
}
|
|
|
|
// extract7ZipMetadata extracts metadata from 7-Zip archives (.cb7)
|
|
func extract7ZipMetadata(filePath string) (*ComicInfo, []byte, error) {
|
|
r, err := sevenzip.OpenReader(filePath)
|
|
if err != nil {
|
|
return nil, nil, fmt.Errorf("failed to open 7-Zip archive: %w", err)
|
|
}
|
|
defer r.Close()
|
|
|
|
files := make([]archiveFile, 0, len(r.File))
|
|
for _, f := range r.File {
|
|
files = append(files, &sevenZipFileAdapter{file: f})
|
|
}
|
|
|
|
comicInfo, coverImage, err := extractMetadataFromArchive(files)
|
|
if err != nil {
|
|
return nil, nil, err
|
|
}
|
|
|
|
if comicInfo.Title == "" {
|
|
basename := filepath.Base(filePath)
|
|
comicInfo.Title = strings.TrimSuffix(basename, filepath.Ext(basename))
|
|
}
|
|
|
|
return comicInfo, coverImage, nil
|
|
}
|
|
|
|
// sevenZipFileAdapter adapts sevenzip.File to archiveFile interface
|
|
type sevenZipFileAdapter struct {
|
|
file *sevenzip.File
|
|
}
|
|
|
|
func (s *sevenZipFileAdapter) Name() string {
|
|
return s.file.Name
|
|
}
|
|
|
|
func (s *sevenZipFileAdapter) Open() (io.ReadCloser, error) {
|
|
return s.file.Open()
|
|
}
|
|
|
|
// extractTarMetadata extracts metadata from TAR archives (.cbt)
|
|
func extractTarMetadata(filePath string) (*ComicInfo, []byte, error) {
|
|
f, err := os.Open(filePath)
|
|
if err != nil {
|
|
return nil, nil, fmt.Errorf("failed to open TAR archive: %w", err)
|
|
}
|
|
defer f.Close()
|
|
|
|
var tarReader *tar.Reader
|
|
if strings.HasSuffix(strings.ToLower(filePath), ".tar.gz") ||
|
|
strings.HasSuffix(strings.ToLower(filePath), ".tgz") {
|
|
gzReader, err := gzip.NewReader(f)
|
|
if err != nil {
|
|
return nil, nil, fmt.Errorf("failed to create gzip reader: %w", err)
|
|
}
|
|
defer gzReader.Close()
|
|
tarReader = tar.NewReader(gzReader)
|
|
} else if strings.HasSuffix(strings.ToLower(filePath), ".tar.bz2") ||
|
|
strings.HasSuffix(strings.ToLower(filePath), ".tbz2") {
|
|
bz2Reader := bzip2.NewReader(f)
|
|
tarReader = tar.NewReader(bz2Reader)
|
|
} else {
|
|
tarReader = tar.NewReader(f)
|
|
}
|
|
|
|
files := make([]archiveFile, 0, 100)
|
|
for {
|
|
header, err := tarReader.Next()
|
|
if err == io.EOF {
|
|
break
|
|
}
|
|
if err != nil {
|
|
return nil, nil, fmt.Errorf("failed to read TAR entry: %w", err)
|
|
}
|
|
|
|
if header.Typeflag == tar.TypeReg {
|
|
data, err := io.ReadAll(tarReader)
|
|
if err != nil {
|
|
continue
|
|
}
|
|
files = append(files, &tarFileAdapter{
|
|
name: header.Name,
|
|
data: data,
|
|
})
|
|
}
|
|
}
|
|
|
|
comicInfo, coverImage, err := extractMetadataFromArchive(files)
|
|
if err != nil {
|
|
return nil, nil, err
|
|
}
|
|
|
|
if comicInfo.Title == "" {
|
|
basename := filepath.Base(filePath)
|
|
comicInfo.Title = strings.TrimSuffix(basename, filepath.Ext(basename))
|
|
}
|
|
|
|
return comicInfo, coverImage, nil
|
|
}
|
|
|
|
// tarFileAdapter stores TAR file data in memory
|
|
type tarFileAdapter struct {
|
|
name string
|
|
data []byte
|
|
}
|
|
|
|
func (t *tarFileAdapter) Name() string {
|
|
return t.name
|
|
}
|
|
|
|
func (t *tarFileAdapter) Open() (io.ReadCloser, error) {
|
|
return io.NopCloser(bytes.NewReader(t.data)), nil
|
|
}
|
|
|
|
// isImageFile checks if a file is an image based on extension
|
|
func isImageFile(filename string) bool {
|
|
ext := strings.ToLower(filepath.Ext(filename))
|
|
return ext == ".jpg" || ext == ".jpeg" || ext == ".png" || ext == ".gif"
|
|
}
|
|
|
|
func (s *EbookScanner) updateEbook(ctx context.Context, ebookID pgtype.UUID, filePath string, info os.FileInfo) error {
|
|
// Update disabled - scanner creates items but doesn't update
|
|
return nil
|
|
}
|
|
|
|
func (s *EbookScanner) getEbookByFilePath(ctx context.Context, filePath string) (database.MediaItems, error) {
|
|
return s.db.GetMediaItemByFilePath(ctx, filePath)
|
|
}
|
|
|
|
func (s *EbookScanner) getMimeType(path string) string {
|
|
ext := strings.ToLower(filepath.Ext(path))
|
|
switch ext {
|
|
case ".epub":
|
|
return "application/epub+zip"
|
|
case ".pdf":
|
|
return "application/pdf"
|
|
case ".mobi":
|
|
return "application/x-mobipocket-ebook"
|
|
case ".azw3":
|
|
return "application/vnd.amazon.ebook"
|
|
case ".fb2":
|
|
return "application/x-fictionbook+xml"
|
|
case ".txt":
|
|
return "text/plain"
|
|
case ".cbz":
|
|
return "application/vnd.comicbook+zip"
|
|
case ".cbr":
|
|
return "application/vnd.comicbook-rar"
|
|
case ".cb7":
|
|
return "application/x-7z-compressed"
|
|
case ".cbt":
|
|
return "application/x-tar"
|
|
default:
|
|
return "application/octet-stream"
|
|
}
|
|
}
|
|
|
|
func (s *EbookScanner) WatchChanges(ctx context.Context) {
|
|
go func() {
|
|
for {
|
|
select {
|
|
case event, ok := <-s.watcher.Events:
|
|
if !ok {
|
|
return
|
|
}
|
|
|
|
// Handle new directories - add them to the watcher
|
|
if event.Has(fsnotify.Create) {
|
|
info, err := os.Stat(event.Name)
|
|
if err == nil && info.IsDir() {
|
|
// Add the new directory to the watcher
|
|
if err := s.watcher.Add(event.Name); err != nil {
|
|
fmt.Printf("Warning: failed to watch new directory %s: %v\n", event.Name, err)
|
|
} else {
|
|
fmt.Printf("Now watching new directory: %s\n", event.Name)
|
|
}
|
|
}
|
|
}
|
|
|
|
// Handle file modifications and creations
|
|
if (event.Has(fsnotify.Create) || event.Has(fsnotify.Write)) && s.isScannableFile(event.Name) {
|
|
fmt.Printf("New/modified ebook detected: %s\n", event.Name)
|
|
if err := s.processEbookFile(ctx, event.Name); err != nil {
|
|
fmt.Printf("Error processing modified ebook %s: %v\n", event.Name, err)
|
|
}
|
|
}
|
|
|
|
case err, ok := <-s.watcher.Errors:
|
|
if !ok {
|
|
return
|
|
}
|
|
fmt.Printf("Watcher error: %v\n", err)
|
|
case <-ctx.Done():
|
|
return
|
|
}
|
|
}
|
|
}()
|
|
}
|
|
|
|
func (s *EbookScanner) Close() error {
|
|
if s.watcher != nil {
|
|
return s.watcher.Close()
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// ============================================
|
|
// PHASE 2: SCANNER ENHANCEMENTS (Week 1-2)
|
|
// ============================================
|
|
|
|
// calculateFileSHA256 calculates SHA-256 hash using streaming to avoid loading entire file into memory
|
|
func (s *EbookScanner) calculateFileSHA256(filePath string) (string, error) {
|
|
file, err := os.Open(filePath)
|
|
if err != nil {
|
|
return "", fmt.Errorf("failed to open file: %v", err)
|
|
}
|
|
defer file.Close()
|
|
|
|
hasher := sha256.New()
|
|
if _, err := io.Copy(hasher, file); err != nil {
|
|
return "", fmt.Errorf("failed to calculate hash: %v", err)
|
|
}
|
|
|
|
return hex.EncodeToString(hasher.Sum(nil)), nil
|
|
}
|
|
|
|
// OPFIdentifier represents an identifier from OPF metadata
|
|
type OPFIdentifier struct {
|
|
XMLName xml.Name `xml:"identifier"`
|
|
ID string `xml:"id,attr"`
|
|
Scheme string `xml:"scheme,attr"`
|
|
Content string `xml:",chardata"`
|
|
}
|
|
|
|
// OPFMetadata represents parsed OPF metadata
|
|
type OPFMetadata struct {
|
|
XMLName xml.Name `xml:"package"`
|
|
Version string `xml:"version,attr"`
|
|
Identifiers []OPFIdentifier `xml:"metadata>identifier"`
|
|
}
|
|
|
|
// extractOPFIdentifiers extracts identifiers from EPUB OPF file
|
|
func (s *EbookScanner) extractOPFIdentifiers(epubPath string) (opfIdentifier, opfUUID string, confidence string, err error) {
|
|
book, err := epub.ReadBook(epubPath)
|
|
if err != nil {
|
|
return "", "", "", fmt.Errorf("failed to open EPUB: %v", err)
|
|
}
|
|
|
|
// Extract OPF identifiers using go-epub library
|
|
identifiers, err := book.MetadataByKey("identifier")
|
|
if err != nil || len(identifiers) == 0 {
|
|
return "", "", "low", nil
|
|
}
|
|
|
|
var identifier, uuid string
|
|
for _, id := range identifiers {
|
|
id = strings.TrimSpace(id)
|
|
|
|
// Check for UUID format (urn:uuid:)
|
|
if strings.HasPrefix(strings.ToLower(id), "urn:uuid:") {
|
|
uuid = strings.TrimPrefix(strings.ToLower(id), "urn:uuid:")
|
|
continue
|
|
}
|
|
|
|
// Check if it's a plain UUID (8-4-4-4-12 format)
|
|
if isValidUUID(id) {
|
|
uuid = id
|
|
continue
|
|
}
|
|
|
|
// Check for ISBN
|
|
if strings.Contains(strings.ToLower(id), "isbn") {
|
|
isbn := s.extractISBNFromIdentifier(id)
|
|
if isbn != "" {
|
|
identifier = isbn
|
|
continue
|
|
}
|
|
}
|
|
|
|
// Use first identifier as fallback
|
|
if identifier == "" && id != "" {
|
|
identifier = id
|
|
}
|
|
}
|
|
|
|
confidence = s.determineHashConfidence(uuid, identifier)
|
|
|
|
return identifier, uuid, confidence, nil
|
|
}
|
|
|
|
// isValidUUID checks if string is a valid UUID (8-4-4-4-12 format)
|
|
func isValidUUID(idStr string) bool {
|
|
uuidRegex := regexp.MustCompile(`^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$`)
|
|
return uuidRegex.MatchString(idStr)
|
|
}
|
|
|
|
// extractISBNFromIdentifier extracts ISBN from identifier string
|
|
func (s *EbookScanner) extractISBNFromIdentifier(id string) string {
|
|
id = strings.TrimSpace(id)
|
|
|
|
// Remove "isbn:" prefix if present
|
|
if strings.HasPrefix(strings.ToLower(id), "isbn:") {
|
|
id = strings.TrimPrefix(strings.ToLower(id), "isbn:")
|
|
}
|
|
|
|
// Remove hyphens and spaces
|
|
isbn := regexp.MustCompile(`[\s-]`).ReplaceAllString(id, "")
|
|
|
|
// Check if it's a valid ISBN-10 or ISBN-13
|
|
if len(isbn) == 10 || len(isbn) == 13 {
|
|
return isbn
|
|
}
|
|
|
|
return ""
|
|
}
|
|
|
|
// determineHashConfidence determines confidence level based on available identifiers
|
|
func (s *EbookScanner) determineHashConfidence(uuid, identifier string) string {
|
|
if uuid != "" && isValidUUID(uuid) {
|
|
return "high"
|
|
}
|
|
if identifier != "" && (strings.Contains(strings.ToLower(identifier), "isbn") || len(identifier) >= 10) {
|
|
return "medium"
|
|
}
|
|
return "low"
|
|
}
|
|
|
|
// detectFormatType detects the format type based on file extension and content
|
|
func (s *EbookScanner) detectFormatType(filePath string) string {
|
|
base := strings.ToLower(filepath.Base(filePath))
|
|
|
|
// Check for compound extensions first (like .kepub.epub)
|
|
if strings.HasSuffix(base, ".kepub.epub") {
|
|
return "kepub"
|
|
}
|
|
|
|
ext := strings.ToLower(filepath.Ext(filePath))
|
|
|
|
switch ext {
|
|
case ".epub":
|
|
return "epub"
|
|
case ".kepub":
|
|
return "kepub"
|
|
case ".pdf":
|
|
return "pdf"
|
|
case ".cbz", ".cbr", ".cb7", ".cbt":
|
|
return "comic_archive"
|
|
case ".mobi":
|
|
return "mobi"
|
|
case ".azw", ".azw3":
|
|
return "kfx"
|
|
case ".txt":
|
|
return "txt"
|
|
case ".fb2":
|
|
return "fb2"
|
|
default:
|
|
return "unknown"
|
|
}
|
|
}
|
|
|
|
// extractHashInfo calculates hash and extracts OPF identifiers for a file
|
|
func (s *EbookScanner) extractHashInfo(filePath string) (*HashInfo, *FormatInfo, error) {
|
|
// Calculate SHA-256
|
|
fileSHA256, err := s.calculateFileSHA256(filePath)
|
|
if err != nil {
|
|
return nil, nil, fmt.Errorf("failed to calculate SHA-256: %v", err)
|
|
}
|
|
|
|
// Get file info
|
|
info, err := os.Stat(filePath)
|
|
if err != nil {
|
|
return nil, nil, fmt.Errorf("failed to get file info: %v", err)
|
|
}
|
|
|
|
// Extract OPF identifiers for EPUB files
|
|
var opfIdentifier, opfUUID, confidence string
|
|
if strings.HasSuffix(strings.ToLower(filePath), ".epub") {
|
|
opfIdentifier, opfUUID, confidence, err = s.extractOPFIdentifiers(filePath)
|
|
if err != nil {
|
|
// Non-fatal error, continue with low confidence
|
|
confidence = "low"
|
|
}
|
|
}
|
|
|
|
hashInfo := &HashInfo{
|
|
FileSHA256: fileSHA256,
|
|
OPFIdentifier: opfIdentifier,
|
|
OPFUUID: opfUUID,
|
|
HashConfidence: confidence,
|
|
}
|
|
|
|
formatInfo := &FormatInfo{
|
|
FormatType: s.detectFormatType(filePath),
|
|
FilePath: filePath,
|
|
FileSHA256: fileSHA256,
|
|
FileSizeBytes: info.Size(),
|
|
MimeType: s.getMimeType(filePath),
|
|
}
|
|
|
|
return hashInfo, formatInfo, nil
|
|
}
|