Updated import path from github.com/bodgit/sevenzip to bookhoard/internal/sevenzip to use the vendored package.
1218 lines
34 KiB
Go
1218 lines
34 KiB
Go
package services
|
|
|
|
import (
|
|
"archive/tar"
|
|
"archive/zip"
|
|
"bookhoard/internal/database"
|
|
"bookhoard/internal/utils"
|
|
"bytes"
|
|
"compress/bzip2"
|
|
"compress/gzip"
|
|
"context"
|
|
"crypto/sha256"
|
|
"encoding/hex"
|
|
"encoding/xml"
|
|
"fmt"
|
|
"image"
|
|
_ "image/jpeg"
|
|
_ "image/png"
|
|
"io"
|
|
"io/fs"
|
|
"os"
|
|
"path/filepath"
|
|
"regexp"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
|
|
"bookhoard/internal/sevenzip"
|
|
epub "github.com/ArcadiaLin/go-epub"
|
|
"github.com/fsnotify/fsnotify"
|
|
"github.com/jackc/pgx/v5"
|
|
"github.com/jackc/pgx/v5/pgtype"
|
|
"github.com/nwaples/rardecode"
|
|
)
|
|
|
|
// MediaMetadata contains extracted metadata for media files (ebooks, comics, manga)
|
|
type MediaMetadata struct {
|
|
Title string
|
|
Author string
|
|
Description string
|
|
Series string
|
|
SeriesNumber int32
|
|
Publisher string
|
|
PublishDate time.Time
|
|
Contributors []string
|
|
CoverPath string
|
|
ISBN string
|
|
ASIN string
|
|
Tags []string
|
|
|
|
FileHashInfo *HashInfo
|
|
FileFormats []*FormatInfo
|
|
}
|
|
|
|
type HashInfo struct {
|
|
FileSHA256 string
|
|
OPFIdentifier string
|
|
OPFUUID string
|
|
HashConfidence string
|
|
}
|
|
|
|
type FormatInfo struct {
|
|
FormatType string
|
|
FilePath string
|
|
FileSHA256 string
|
|
FileSizeBytes int64
|
|
MimeType string
|
|
}
|
|
|
|
// MediaScanner scans library folders for media files (ebooks, comics, manga)
|
|
type MediaScanner struct {
|
|
db *database.Queries
|
|
watcher *fsnotify.Watcher
|
|
folders []string
|
|
adminID pgtype.UUID
|
|
defaultLibraryID pgtype.UUID
|
|
libraryTypes map[string][]string
|
|
}
|
|
|
|
// NewMediaScanner creates a new media scanner instance
|
|
func NewMediaScanner(db *database.Queries) *MediaScanner {
|
|
watcher, err := fsnotify.NewWatcher()
|
|
if err != nil {
|
|
panic(fmt.Sprintf("Failed to create file watcher: %v", err))
|
|
}
|
|
|
|
return &MediaScanner{
|
|
db: db,
|
|
watcher: watcher,
|
|
folders: []string{},
|
|
adminID: pgtype.UUID{},
|
|
defaultLibraryID: pgtype.UUID{Valid: false},
|
|
libraryTypes: make(map[string][]string),
|
|
}
|
|
}
|
|
|
|
func (s *MediaScanner) SetAdminID(adminID pgtype.UUID) {
|
|
s.adminID = adminID
|
|
}
|
|
|
|
func (s *MediaScanner) SetFolders(folders []string) error {
|
|
s.folders = folders
|
|
|
|
// Remove old watch if exists
|
|
if s.watcher != nil {
|
|
s.watcher.Close()
|
|
}
|
|
|
|
// Create new watcher
|
|
watcher, err := fsnotify.NewWatcher()
|
|
if err != nil {
|
|
return fmt.Errorf("failed to create watcher: %v", err)
|
|
}
|
|
s.watcher = watcher
|
|
|
|
// Build cache of allowed extensions per folder
|
|
s.libraryTypes = make(map[string][]string)
|
|
ctx := context.Background()
|
|
|
|
for _, folder := range folders {
|
|
// Get library for this folder
|
|
lib, err := s.db.GetLibraryByFolder(ctx, folder)
|
|
if err != nil {
|
|
fmt.Printf("Warning: failed to get library for folder %s: %v\n", folder, err)
|
|
continue
|
|
}
|
|
|
|
// Get library type with allowed extensions
|
|
libType, err := s.db.GetLibraryType(ctx, lib.LibraryTypeID)
|
|
if err != nil {
|
|
fmt.Printf("Warning: failed to get library type for %s: %v\n", folder, err)
|
|
continue
|
|
}
|
|
|
|
// Cache allowed extensions for this folder
|
|
s.libraryTypes[folder] = libType.AllowedExtensions
|
|
fmt.Printf("Scanner: Folder %s (type: %s) allows extensions: %v\n",
|
|
folder, libType.Name, libType.AllowedExtensions)
|
|
}
|
|
|
|
// Add all folders to watch
|
|
for _, folder := range folders {
|
|
if err := s.watcher.Add(folder); err != nil {
|
|
fmt.Printf("Warning: failed to watch folder %s: %v\n", folder, err)
|
|
}
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func (s *MediaScanner) ScanFolders(ctx context.Context) error {
|
|
if len(s.folders) == 0 {
|
|
return fmt.Errorf("no folders set")
|
|
}
|
|
|
|
fmt.Printf("Starting scan of %d folders: %v\n", len(s.folders), s.folders)
|
|
|
|
totalFiles := 0
|
|
mediaFiles := 0
|
|
|
|
for _, folder := range s.folders {
|
|
fmt.Printf("Scanning folder: %s\n", folder)
|
|
|
|
// Check if folder exists
|
|
if _, err := os.Stat(folder); os.IsNotExist(err) {
|
|
fmt.Printf("Folder does not exist: %s\n", folder)
|
|
continue
|
|
}
|
|
|
|
err := filepath.WalkDir(folder, func(path string, d fs.DirEntry, err error) error {
|
|
if err != nil {
|
|
fmt.Printf("Error accessing path %s: %v\n", path, err)
|
|
return err
|
|
}
|
|
|
|
totalFiles++
|
|
|
|
if d.IsDir() {
|
|
// Also watch subdirectories
|
|
if err := s.watcher.Add(path); err != nil {
|
|
fmt.Printf("Warning: failed to watch subdirectory %s: %v\n", path, err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// Check if file should be scanned based on library type
|
|
if s.isScannableFile(path) {
|
|
mediaFiles++
|
|
fmt.Printf("Found media file: %s\n", path)
|
|
if err := s.processMediaFile(ctx, path); err != nil {
|
|
fmt.Printf("Error processing media file %s: %v\n", path, err)
|
|
} else {
|
|
fmt.Printf("Successfully processed media file: %s\n", path)
|
|
}
|
|
}
|
|
|
|
return nil
|
|
})
|
|
if err != nil {
|
|
return fmt.Errorf("failed to scan folder %s: %v", folder, err)
|
|
}
|
|
}
|
|
|
|
fmt.Printf("Scan completed: %d total files scanned, %d media files found\n", totalFiles, mediaFiles)
|
|
return nil
|
|
}
|
|
|
|
func (s *MediaScanner) isScannableFile(path string) bool {
|
|
ext := strings.ToLower(filepath.Ext(path))
|
|
|
|
// Find which folder this file belongs to
|
|
var folder string
|
|
for _, f := range s.folders {
|
|
if strings.HasPrefix(path, f) {
|
|
folder = f
|
|
break
|
|
}
|
|
}
|
|
|
|
// If no folder match, don't scan
|
|
if folder == "" {
|
|
return false
|
|
}
|
|
|
|
// Get allowed extensions for this folder's library
|
|
allowed, ok := s.libraryTypes[folder]
|
|
if !ok {
|
|
// No library type info, skip file
|
|
fmt.Printf("Warning: No library type info for folder %s, skipping %s\n", folder, path)
|
|
return false
|
|
}
|
|
|
|
// Check if file extension is allowed for this library type
|
|
for _, allowedExt := range allowed {
|
|
if ext == strings.ToLower(allowedExt) {
|
|
return true
|
|
}
|
|
}
|
|
|
|
return false
|
|
}
|
|
|
|
// extractFolderStructureMetadata extracts metadata from folder paths, prioritizing Calibre structure
|
|
func (s *MediaScanner) extractFolderStructureMetadata(path, rootFolder string) *MediaMetadata {
|
|
metadata := &MediaMetadata{}
|
|
|
|
// Get the relative path from root folder
|
|
relPath, err := filepath.Rel(rootFolder, path)
|
|
if err != nil {
|
|
return metadata
|
|
}
|
|
|
|
// Split into directory components
|
|
dir := filepath.Dir(relPath)
|
|
components := strings.Split(dir, string(filepath.Separator))
|
|
|
|
if len(components) < 2 {
|
|
return metadata // Not enough structure to extract
|
|
}
|
|
|
|
// Calibre structure detection
|
|
// Pattern 1: Author Name/Book Title/
|
|
// Pattern 2: Author Name/Series Name/Book Title/
|
|
// Pattern 3: Author Name/Series Name, Book #1 - Book Title/
|
|
|
|
author := strings.TrimSuffix(components[0], "_") // Remove trailing underscore if present
|
|
metadata.Author = strings.ReplaceAll(author, "_", " ")
|
|
|
|
if len(components) >= 3 {
|
|
// This might be a series structure
|
|
possibleSeries := components[1]
|
|
possibleTitle := components[2]
|
|
|
|
// Check for Calibre series format: "Series Name, Book #1 - Title"
|
|
seriesMatch := regexp.MustCompile(`^(.*),\s+Book\s+#(\d+)\s*-\s*(.*)$`).FindStringSubmatch(possibleSeries)
|
|
if len(seriesMatch) == 4 {
|
|
metadata.Series = strings.ReplaceAll(seriesMatch[1], "_", " ")
|
|
if seriesNum, err := strconv.ParseInt(seriesMatch[2], 10, 32); err == nil {
|
|
metadata.SeriesNumber = int32(seriesNum)
|
|
}
|
|
metadata.Title = strings.ReplaceAll(possibleTitle, "_", " ")
|
|
} else {
|
|
// Simple series structure: Author/Series/Title
|
|
metadata.Series = strings.ReplaceAll(possibleSeries, "_", " ")
|
|
metadata.Title = strings.ReplaceAll(possibleTitle, "_", " ")
|
|
|
|
// Try to extract series number from title
|
|
titleNumMatch := regexp.MustCompile(`^(.*)\s+(\d+)$`).FindStringSubmatch(metadata.Title)
|
|
if len(titleNumMatch) == 3 {
|
|
metadata.Title = titleNumMatch[1]
|
|
if seriesNum, err := strconv.ParseInt(titleNumMatch[2], 10, 32); err == nil {
|
|
metadata.SeriesNumber = int32(seriesNum)
|
|
}
|
|
}
|
|
}
|
|
} else {
|
|
// Simple structure: Author/Title
|
|
metadata.Title = strings.ReplaceAll(components[1], "_", " ")
|
|
}
|
|
|
|
return metadata
|
|
}
|
|
|
|
func (s *MediaScanner) processMediaFile(ctx context.Context, path string) error {
|
|
fmt.Printf("Processing media file: %s\n", path)
|
|
|
|
// Get file info
|
|
info, err := os.Stat(path)
|
|
if err != nil {
|
|
fmt.Printf("Failed to get file info for %s: %v\n", path, err)
|
|
return fmt.Errorf("failed to get file info: %v", err)
|
|
}
|
|
|
|
fmt.Printf("File info for %s: size=%d\n", path, info.Size())
|
|
|
|
// Check if media item already exists in database
|
|
existingItem, err := s.getMediaItemByFilePath(ctx, path)
|
|
if err == nil {
|
|
fmt.Printf("Media item already exists in database: %s (size: %d vs %d)\n", path, existingItem.FileSize.Int64, info.Size())
|
|
// Media item exists, check if file has changed (by size)
|
|
if existingItem.FileSize.Int64 != info.Size() {
|
|
fmt.Printf("File size changed, updating media item: %s\n", path)
|
|
return s.updateMediaItem(ctx, existingItem.ID, path, info)
|
|
}
|
|
fmt.Printf("Media item already exists with same size, skipping: %s\n", path)
|
|
return nil // Skip if already exists and size matches
|
|
} else if err != pgx.ErrNoRows {
|
|
fmt.Printf("Database error checking media item existence: %v\n", err)
|
|
// Some other error occurred
|
|
return fmt.Errorf("failed to check if media item exists: %v", err)
|
|
}
|
|
fmt.Printf("Media item does not exist in database, creating new entry: %s\n", path)
|
|
|
|
// Extract metadata from file first
|
|
metadata, err := s.extractMetadata(path)
|
|
if err != nil {
|
|
fmt.Printf("Warning: failed to extract metadata from %s: %v\n", path, err)
|
|
metadata = &MediaMetadata{}
|
|
}
|
|
|
|
// Extract hash information during metadata extraction
|
|
hashInfo, formatInfo, err := s.extractHashInfo(path)
|
|
if err != nil {
|
|
fmt.Printf("Warning: failed to extract hash info from %s: %v\n", path, err)
|
|
hashInfo = &HashInfo{}
|
|
formatInfo = &FormatInfo{}
|
|
} else {
|
|
metadata.FileHashInfo = hashInfo
|
|
metadata.FileFormats = []*FormatInfo{formatInfo}
|
|
fmt.Printf("Hash info for %s: SHA256=%s, OPF_ID=%s, OPF_UUID=%s, Confidence=%s\n",
|
|
path, hashInfo.FileSHA256, hashInfo.OPFIdentifier, hashInfo.OPFUUID, hashInfo.HashConfidence)
|
|
}
|
|
|
|
var comicInfo *ComicInfo
|
|
var coverImage []byte
|
|
|
|
ext := strings.ToLower(filepath.Ext(path))
|
|
if ext == ".cbz" || ext == ".cbr" || ext == ".cb7" || ext == ".cbt" ||
|
|
strings.HasSuffix(strings.ToLower(path), ".tar.gz") ||
|
|
strings.HasSuffix(strings.ToLower(path), ".tar.bz2") ||
|
|
strings.HasSuffix(strings.ToLower(path), ".tgz") ||
|
|
strings.HasSuffix(strings.ToLower(path), ".tbz2") {
|
|
info, cover, err := extractComicMetadata(path)
|
|
if err != nil {
|
|
fmt.Printf("Warning: failed to extract comic metadata from %s: %v\n", path, err)
|
|
} else {
|
|
comicInfo = info
|
|
coverImage = cover
|
|
if comicInfo.Title != "" && metadata.Title == "" {
|
|
metadata.Title = comicInfo.Title
|
|
}
|
|
if comicInfo.Series != "" && metadata.Series == "" {
|
|
metadata.Series = comicInfo.Series
|
|
}
|
|
if comicInfo.Number > 0 && metadata.SeriesNumber == 0 {
|
|
metadata.SeriesNumber = int32(comicInfo.Number)
|
|
}
|
|
if comicInfo.Publisher != "" && metadata.Publisher == "" {
|
|
metadata.Publisher = comicInfo.Publisher
|
|
}
|
|
if comicInfo.Writer != "" && metadata.Author == "" {
|
|
metadata.Author = comicInfo.Writer
|
|
}
|
|
if len(coverImage) > 0 && metadata.CoverPath == "" {
|
|
coverPath := path + ".cover.jpg"
|
|
if err := os.WriteFile(coverPath, coverImage, 0644); err == nil {
|
|
metadata.CoverPath = coverPath
|
|
}
|
|
}
|
|
fmt.Printf("Extracted comic metadata from %s: title=%s, series=%s, issue=%d\n",
|
|
path, comicInfo.Title, comicInfo.Series, comicInfo.Number)
|
|
}
|
|
}
|
|
|
|
// Try to get metadata from folder structure as fallback/enhancement
|
|
// Use the root folder that contains this file
|
|
var rootFolder string
|
|
for _, folder := range s.folders {
|
|
if strings.HasPrefix(path, folder) {
|
|
rootFolder = folder
|
|
break
|
|
}
|
|
}
|
|
|
|
if rootFolder != "" {
|
|
folderMetadata := s.extractFolderStructureMetadata(path, rootFolder)
|
|
|
|
// Use folder metadata as fallback for missing information
|
|
if metadata.Title == "" && folderMetadata.Title != "" {
|
|
metadata.Title = folderMetadata.Title
|
|
}
|
|
if metadata.Author == "" && folderMetadata.Author != "" {
|
|
metadata.Author = folderMetadata.Author
|
|
}
|
|
if metadata.Series == "" && folderMetadata.Series != "" {
|
|
metadata.Series = folderMetadata.Series
|
|
}
|
|
if metadata.SeriesNumber == 0 && folderMetadata.SeriesNumber > 0 {
|
|
metadata.SeriesNumber = folderMetadata.SeriesNumber
|
|
}
|
|
}
|
|
|
|
// Final fallback if still missing essential metadata
|
|
if metadata.Title == "" {
|
|
metadata.Title = strings.TrimSuffix(filepath.Base(path), filepath.Ext(path))
|
|
}
|
|
if metadata.Author == "" {
|
|
metadata.Author = "Unknown"
|
|
}
|
|
|
|
// Find library for this folder
|
|
var libraryID pgtype.UUID
|
|
for _, folder := range s.folders {
|
|
if strings.HasPrefix(path, folder) {
|
|
lib, err := s.db.GetLibraryByFolder(ctx, folder)
|
|
if err != nil {
|
|
return fmt.Errorf("failed to find library for folder %s: %v", folder, err)
|
|
}
|
|
libraryID = lib.LibraryID
|
|
break
|
|
}
|
|
}
|
|
|
|
if !libraryID.Valid {
|
|
return fmt.Errorf("no library found for file path: %s", path)
|
|
}
|
|
|
|
// Normalize metadata fields for display
|
|
metadata.Contributors = utils.NormalizeContributors(metadata.Contributors)
|
|
metadata.Tags = utils.NormalizeTags(metadata.Tags)
|
|
|
|
// Normalize search fields
|
|
contributorsSearch := utils.NormalizeContributorsSearch(metadata.Contributors)
|
|
tagsSearch := utils.NormalizeTagsSearch(metadata.Tags)
|
|
|
|
// Create media item in database
|
|
createdItem, err := s.db.CreateMediaItem(ctx, database.CreateMediaItemParams{
|
|
LibraryID: libraryID,
|
|
Title: metadata.Title,
|
|
Author: pgtype.Text{String: metadata.Author, Valid: metadata.Author != ""},
|
|
Isbn: pgtype.Text{String: utils.NormalizeISBNSafe(metadata.ISBN), Valid: metadata.ISBN != ""},
|
|
Asin: pgtype.Text{String: metadata.ASIN, Valid: metadata.ASIN != ""},
|
|
Description: pgtype.Text{String: metadata.Description, Valid: metadata.Description != ""},
|
|
FilePath: path,
|
|
FileSize: pgtype.Int8{Int64: info.Size(), Valid: true},
|
|
MimeType: pgtype.Text{String: s.getMimeType(path), Valid: true},
|
|
CoverImagePath: pgtype.Text{String: metadata.CoverPath, Valid: metadata.CoverPath != ""},
|
|
Series: pgtype.Text{String: metadata.Series, Valid: metadata.Series != ""},
|
|
SeriesNumber: pgtype.Int4{Int32: metadata.SeriesNumber, Valid: metadata.SeriesNumber > 0},
|
|
Publisher: pgtype.Text{String: metadata.Publisher, Valid: metadata.Publisher != ""},
|
|
DatePublished: pgtype.Date{Time: metadata.PublishDate, Valid: !metadata.PublishDate.IsZero()},
|
|
Contributors: metadata.Contributors,
|
|
ContributorsSearch: contributorsSearch,
|
|
Tags: metadata.Tags,
|
|
TagsSearch: tagsSearch,
|
|
AddedByAdminID: s.adminID,
|
|
})
|
|
if err != nil {
|
|
return fmt.Errorf("failed to create media item: %v", err)
|
|
}
|
|
|
|
// Update hash information before database storage
|
|
if metadata.FileHashInfo != nil && metadata.FileHashInfo.FileSHA256 != "" {
|
|
_, err = s.db.UpdateMediaItemIdentifiers(ctx, database.UpdateMediaItemIdentifiersParams{
|
|
ID: createdItem.ID,
|
|
FileSha256: pgtype.Text{String: metadata.FileHashInfo.FileSHA256, Valid: true},
|
|
OpfIdentifier: pgtype.Text{String: metadata.FileHashInfo.OPFIdentifier, Valid: metadata.FileHashInfo.OPFIdentifier != ""},
|
|
OpfUuid: pgtype.Text{String: metadata.FileHashInfo.OPFUUID, Valid: metadata.FileHashInfo.OPFUUID != ""},
|
|
HashConfidence: pgtype.Text{String: metadata.FileHashInfo.HashConfidence, Valid: true},
|
|
})
|
|
if err != nil {
|
|
fmt.Printf("Warning: failed to update hash identifiers for %s: %v\n", path, err)
|
|
}
|
|
}
|
|
|
|
// Store format information in the database
|
|
for _, format := range metadata.FileFormats {
|
|
_, err = s.db.CreateMediaItemFormat(ctx, database.CreateMediaItemFormatParams{
|
|
MediaItemID: createdItem.ID,
|
|
FormatType: format.FormatType,
|
|
FilePath: pgtype.Text{String: format.FilePath, Valid: true},
|
|
FileSha256: pgtype.Text{String: format.FileSHA256, Valid: true},
|
|
FileSizeBytes: pgtype.Int8{Int64: format.FileSizeBytes, Valid: true},
|
|
MimeType: pgtype.Text{String: format.MimeType, Valid: true},
|
|
})
|
|
if err != nil {
|
|
fmt.Printf("Warning: failed to create format entry for %s: %v\n", path, err)
|
|
}
|
|
}
|
|
|
|
return err
|
|
}
|
|
|
|
func (s *MediaScanner) extractMetadata(path string) (*MediaMetadata, error) {
|
|
ext := strings.ToLower(filepath.Ext(path))
|
|
|
|
switch ext {
|
|
case ".epub":
|
|
return s.extractEPUBMetadata(path)
|
|
case ".pdf":
|
|
return s.extractPDFMetadata(path)
|
|
default:
|
|
// For other formats, return basic metadata
|
|
return &MediaMetadata{
|
|
Title: strings.TrimSuffix(filepath.Base(path), ext),
|
|
}, nil
|
|
}
|
|
}
|
|
|
|
func (s *MediaScanner) extractEPUBMetadata(path string) (*MediaMetadata, error) {
|
|
book, err := epub.ReadBook(path)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to open EPUB: %v", err)
|
|
}
|
|
|
|
metadata := &MediaMetadata{}
|
|
|
|
// Title
|
|
if title, err := book.Title(); err == nil && title != "" {
|
|
metadata.Title = title
|
|
}
|
|
|
|
// Author
|
|
if authors, err := book.MetadataByKey("creator"); err == nil && len(authors) > 0 {
|
|
metadata.Author = authors[0]
|
|
}
|
|
|
|
// Description
|
|
if descriptions, err := book.MetadataByKey("description"); err == nil && len(descriptions) > 0 {
|
|
metadata.Description = descriptions[0]
|
|
}
|
|
|
|
// Publisher
|
|
if publishers, err := book.MetadataByKey("publisher"); err == nil && len(publishers) > 0 {
|
|
metadata.Publisher = publishers[0]
|
|
}
|
|
|
|
// Series and series number (Calibre specific metadata)
|
|
if series, err := book.MetadataByKey("calibre:series"); err == nil && len(series) > 0 {
|
|
metadata.Series = series[0]
|
|
}
|
|
if seriesIndex, err := book.MetadataByKey("calibre:series_index"); err == nil && len(seriesIndex) > 0 {
|
|
if index, err := strconv.ParseFloat(seriesIndex[0], 32); err == nil {
|
|
metadata.SeriesNumber = int32(index)
|
|
}
|
|
}
|
|
|
|
// Publish date
|
|
if dates, err := book.MetadataByKey("date"); err == nil && len(dates) > 0 {
|
|
if date, err := time.Parse("2006-01-02", dates[0]); err == nil {
|
|
metadata.PublishDate = date
|
|
} else {
|
|
// Try alternative date formats
|
|
if date, err := time.Parse("2006", dates[0]); err == nil {
|
|
metadata.PublishDate = date
|
|
}
|
|
}
|
|
}
|
|
|
|
// Contributors
|
|
if contributors, err := book.MetadataByKey("contributor"); err == nil && len(contributors) > 0 {
|
|
// Normalize contributors for display
|
|
metadata.Contributors = utils.NormalizeContributors(contributors)
|
|
}
|
|
|
|
// ISBN
|
|
if isbns, err := book.MetadataByKey("identifier"); err == nil && len(isbns) > 0 {
|
|
for _, isbn := range isbns {
|
|
if strings.Contains(strings.ToLower(isbn), "isbn") {
|
|
// Extract ISBN number from identifier like "isbn:978-3-16-148410-0"
|
|
isbnParts := strings.SplitN(isbn, ":", 2)
|
|
if len(isbnParts) == 2 {
|
|
metadata.ISBN = isbnParts[1]
|
|
break
|
|
}
|
|
}
|
|
if strings.Contains(strings.ToLower(isbn), "asin") {
|
|
// Extract ASIN from identifier like "asin:B08XXXXX"
|
|
asinParts := strings.SplitN(isbn, ":", 2)
|
|
if len(asinParts) == 2 {
|
|
metadata.ASIN = asinParts[1]
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// Tags
|
|
if tags, err := book.MetadataByKey("subject"); err == nil && len(tags) > 0 {
|
|
// Normalize tags for display
|
|
metadata.Tags = utils.NormalizeTags(tags)
|
|
}
|
|
|
|
return metadata, nil
|
|
}
|
|
|
|
func (s *MediaScanner) extractPDFMetadata(path string) (*MediaMetadata, error) {
|
|
// For now, return basic metadata since PDF extraction requires additional libraries
|
|
// In a future enhancement, you could use libraries like github.com/ledongthuc/pdf
|
|
filename := strings.TrimSuffix(filepath.Base(path), ".pdf")
|
|
|
|
return &MediaMetadata{
|
|
Title: filename,
|
|
}, nil
|
|
}
|
|
|
|
// ComicInfo represents metadata from ComicInfo.xml
|
|
type ComicInfo struct {
|
|
XMLName xml.Name `xml:"ComicInfo"`
|
|
Title string `xml:"Title"`
|
|
Series string `xml:"Series"`
|
|
Number int `xml:"Number"`
|
|
Volume int `xml:"Volume"`
|
|
Publisher string `xml:"Publisher"`
|
|
Year int `xml:"Year"`
|
|
Month int `xml:"Month"`
|
|
Day int `xml:"Day"`
|
|
Writer string `xml:"Writer"`
|
|
Penciller string `xml:"Penciller"`
|
|
Inker string `xml:"Inker"`
|
|
Colorist string `xml:"Colorist"`
|
|
Letterer string `xml:"Letterer"`
|
|
CoverArtist string `xml:"CoverArtist"`
|
|
Genre string `xml:"Genre"`
|
|
Tags string `xml:"Tags"`
|
|
Web string `xml:"Web"`
|
|
Notes string `xml:"Notes"`
|
|
}
|
|
|
|
// extractComicMetadata extracts metadata from comic archive (supports .cbz, .cbr, .cb7, .cbt)
|
|
func extractComicMetadata(filePath string) (*ComicInfo, []byte, error) {
|
|
ext := strings.ToLower(filepath.Ext(filePath))
|
|
|
|
switch ext {
|
|
case ".cbz":
|
|
return extractZipMetadata(filePath)
|
|
case ".cbr":
|
|
return extractRarMetadata(filePath)
|
|
case ".cb7":
|
|
return extract7ZipMetadata(filePath)
|
|
case ".cbt":
|
|
return extractTarMetadata(filePath)
|
|
default:
|
|
return nil, nil, fmt.Errorf("unsupported comic format: %s", ext)
|
|
}
|
|
}
|
|
|
|
// archiveFile represents a file in an archive for uniform handling
|
|
type archiveFile interface {
|
|
Name() string
|
|
Open() (io.ReadCloser, error)
|
|
}
|
|
|
|
// extractMetadataFromArchive extracts ComicInfo.xml and cover image from any archive format
|
|
func extractMetadataFromArchive(files []archiveFile) (*ComicInfo, []byte, error) {
|
|
var comicInfo *ComicInfo
|
|
var coverImage []byte
|
|
|
|
for _, f := range files {
|
|
if f.Name() == "ComicInfo.xml" {
|
|
rc, err := f.Open()
|
|
if err != nil {
|
|
return nil, nil, fmt.Errorf("failed to open ComicInfo.xml: %w", err)
|
|
}
|
|
|
|
data, err := io.ReadAll(rc)
|
|
rc.Close()
|
|
if err != nil {
|
|
return nil, nil, fmt.Errorf("failed to read ComicInfo.xml: %w", err)
|
|
}
|
|
|
|
comicInfo = &ComicInfo{}
|
|
if err := xml.Unmarshal(data, comicInfo); err != nil {
|
|
return nil, nil, fmt.Errorf("failed to parse ComicInfo.xml: %w", err)
|
|
}
|
|
}
|
|
|
|
if coverImage == nil && isImageFile(f.Name()) {
|
|
if !strings.Contains(filepath.Dir(f.Name()), string(filepath.Separator)) ||
|
|
filepath.Dir(f.Name()) == "." {
|
|
rc, err := f.Open()
|
|
if err != nil {
|
|
continue
|
|
}
|
|
|
|
coverImage, err = io.ReadAll(rc)
|
|
rc.Close()
|
|
if err == nil {
|
|
_, _, err = image.Decode(bytes.NewReader(coverImage))
|
|
if err != nil {
|
|
coverImage = nil
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
if comicInfo == nil {
|
|
comicInfo = &ComicInfo{}
|
|
}
|
|
|
|
return comicInfo, coverImage, nil
|
|
}
|
|
|
|
// extractZipMetadata extracts metadata from ZIP archives (.cbz)
|
|
func extractZipMetadata(filePath string) (*ComicInfo, []byte, error) {
|
|
r, err := zip.OpenReader(filePath)
|
|
if err != nil {
|
|
return nil, nil, fmt.Errorf("failed to open ZIP archive: %w", err)
|
|
}
|
|
defer r.Close()
|
|
|
|
files := make([]archiveFile, 0, len(r.File))
|
|
for _, f := range r.File {
|
|
files = append(files, &zipFileAdapter{f})
|
|
}
|
|
|
|
comicInfo, coverImage, err := extractMetadataFromArchive(files)
|
|
if err != nil {
|
|
return nil, nil, err
|
|
}
|
|
|
|
if comicInfo.Title == "" {
|
|
basename := filepath.Base(filePath)
|
|
comicInfo.Title = strings.TrimSuffix(basename, filepath.Ext(basename))
|
|
}
|
|
|
|
return comicInfo, coverImage, nil
|
|
}
|
|
|
|
// zipFileAdapter adapts zip.File to archiveFile interface
|
|
type zipFileAdapter struct {
|
|
*zip.File
|
|
}
|
|
|
|
func (z *zipFileAdapter) Name() string {
|
|
return z.File.Name
|
|
}
|
|
|
|
func (z *zipFileAdapter) Open() (io.ReadCloser, error) {
|
|
return z.File.Open()
|
|
}
|
|
|
|
// extractRarMetadata extracts metadata from RAR archives (.cbr)
|
|
func extractRarMetadata(filePath string) (*ComicInfo, []byte, error) {
|
|
r, err := rardecode.OpenReader(filePath, "")
|
|
if err != nil {
|
|
return nil, nil, fmt.Errorf("failed to open RAR archive: %w", err)
|
|
}
|
|
defer r.Close()
|
|
|
|
files := make([]archiveFile, 0, 100)
|
|
for {
|
|
header, err := r.Next()
|
|
if err == io.EOF {
|
|
break
|
|
}
|
|
if err != nil {
|
|
return nil, nil, fmt.Errorf("failed to read RAR entry: %w", err)
|
|
}
|
|
|
|
data, err := io.ReadAll(r)
|
|
if err != nil {
|
|
continue
|
|
}
|
|
files = append(files, &rarFileAdapter{
|
|
name: header.Name,
|
|
data: data,
|
|
})
|
|
}
|
|
|
|
comicInfo, coverImage, err := extractMetadataFromArchive(files)
|
|
if err != nil {
|
|
return nil, nil, err
|
|
}
|
|
|
|
if comicInfo.Title == "" {
|
|
basename := filepath.Base(filePath)
|
|
comicInfo.Title = strings.TrimSuffix(basename, filepath.Ext(basename))
|
|
}
|
|
|
|
return comicInfo, coverImage, nil
|
|
}
|
|
|
|
// rarFileAdapter stores RAR file data in memory
|
|
type rarFileAdapter struct {
|
|
name string
|
|
data []byte
|
|
}
|
|
|
|
func (r *rarFileAdapter) Name() string {
|
|
return r.name
|
|
}
|
|
|
|
func (r *rarFileAdapter) Open() (io.ReadCloser, error) {
|
|
return io.NopCloser(bytes.NewReader(r.data)), nil
|
|
}
|
|
|
|
// extract7ZipMetadata extracts metadata from 7-Zip archives (.cb7)
|
|
func extract7ZipMetadata(filePath string) (*ComicInfo, []byte, error) {
|
|
r, err := sevenzip.OpenReader(filePath)
|
|
if err != nil {
|
|
return nil, nil, fmt.Errorf("failed to open 7-Zip archive: %w", err)
|
|
}
|
|
defer r.Close()
|
|
|
|
files := make([]archiveFile, 0, len(r.File))
|
|
for _, f := range r.File {
|
|
files = append(files, &sevenZipFileAdapter{file: f})
|
|
}
|
|
|
|
comicInfo, coverImage, err := extractMetadataFromArchive(files)
|
|
if err != nil {
|
|
return nil, nil, err
|
|
}
|
|
|
|
if comicInfo.Title == "" {
|
|
basename := filepath.Base(filePath)
|
|
comicInfo.Title = strings.TrimSuffix(basename, filepath.Ext(basename))
|
|
}
|
|
|
|
return comicInfo, coverImage, nil
|
|
}
|
|
|
|
// sevenZipFileAdapter adapts sevenzip.File to archiveFile interface
|
|
type sevenZipFileAdapter struct {
|
|
file *sevenzip.File
|
|
}
|
|
|
|
func (s *sevenZipFileAdapter) Name() string {
|
|
return s.file.Name
|
|
}
|
|
|
|
func (s *sevenZipFileAdapter) Open() (io.ReadCloser, error) {
|
|
return s.file.Open()
|
|
}
|
|
|
|
// extractTarMetadata extracts metadata from TAR archives (.cbt)
|
|
func extractTarMetadata(filePath string) (*ComicInfo, []byte, error) {
|
|
f, err := os.Open(filePath)
|
|
if err != nil {
|
|
return nil, nil, fmt.Errorf("failed to open TAR archive: %w", err)
|
|
}
|
|
defer f.Close()
|
|
|
|
var tarReader *tar.Reader
|
|
if strings.HasSuffix(strings.ToLower(filePath), ".tar.gz") ||
|
|
strings.HasSuffix(strings.ToLower(filePath), ".tgz") {
|
|
gzReader, err := gzip.NewReader(f)
|
|
if err != nil {
|
|
return nil, nil, fmt.Errorf("failed to create gzip reader: %w", err)
|
|
}
|
|
defer gzReader.Close()
|
|
tarReader = tar.NewReader(gzReader)
|
|
} else if strings.HasSuffix(strings.ToLower(filePath), ".tar.bz2") ||
|
|
strings.HasSuffix(strings.ToLower(filePath), ".tbz2") {
|
|
bz2Reader := bzip2.NewReader(f)
|
|
tarReader = tar.NewReader(bz2Reader)
|
|
} else {
|
|
tarReader = tar.NewReader(f)
|
|
}
|
|
|
|
files := make([]archiveFile, 0, 100)
|
|
for {
|
|
header, err := tarReader.Next()
|
|
if err == io.EOF {
|
|
break
|
|
}
|
|
if err != nil {
|
|
return nil, nil, fmt.Errorf("failed to read TAR entry: %w", err)
|
|
}
|
|
|
|
if header.Typeflag == tar.TypeReg {
|
|
data, err := io.ReadAll(tarReader)
|
|
if err != nil {
|
|
continue
|
|
}
|
|
files = append(files, &tarFileAdapter{
|
|
name: header.Name,
|
|
data: data,
|
|
})
|
|
}
|
|
}
|
|
|
|
comicInfo, coverImage, err := extractMetadataFromArchive(files)
|
|
if err != nil {
|
|
return nil, nil, err
|
|
}
|
|
|
|
if comicInfo.Title == "" {
|
|
basename := filepath.Base(filePath)
|
|
comicInfo.Title = strings.TrimSuffix(basename, filepath.Ext(basename))
|
|
}
|
|
|
|
return comicInfo, coverImage, nil
|
|
}
|
|
|
|
// tarFileAdapter stores TAR file data in memory
|
|
type tarFileAdapter struct {
|
|
name string
|
|
data []byte
|
|
}
|
|
|
|
func (t *tarFileAdapter) Name() string {
|
|
return t.name
|
|
}
|
|
|
|
func (t *tarFileAdapter) Open() (io.ReadCloser, error) {
|
|
return io.NopCloser(bytes.NewReader(t.data)), nil
|
|
}
|
|
|
|
// isImageFile checks if a file is an image based on extension
|
|
func isImageFile(filename string) bool {
|
|
ext := strings.ToLower(filepath.Ext(filename))
|
|
return ext == ".jpg" || ext == ".jpeg" || ext == ".png" || ext == ".gif"
|
|
}
|
|
|
|
func (s *MediaScanner) updateMediaItem(ctx context.Context, mediaItemID pgtype.UUID, filePath string, info os.FileInfo) error {
|
|
// Update disabled - scanner creates items but doesn't update
|
|
return nil
|
|
}
|
|
|
|
func (s *MediaScanner) getMediaItemByFilePath(ctx context.Context, filePath string) (database.MediaItems, error) {
|
|
return s.db.GetMediaItemByFilePath(ctx, filePath)
|
|
}
|
|
|
|
func (s *MediaScanner) getMimeType(path string) string {
|
|
ext := strings.ToLower(filepath.Ext(path))
|
|
switch ext {
|
|
case ".epub":
|
|
return "application/epub+zip"
|
|
case ".pdf":
|
|
return "application/pdf"
|
|
case ".mobi":
|
|
return "application/x-mobipocket-ebook"
|
|
case ".azw3":
|
|
return "application/vnd.amazon.ebook"
|
|
case ".fb2":
|
|
return "application/x-fictionbook+xml"
|
|
case ".txt":
|
|
return "text/plain"
|
|
case ".cbz":
|
|
return "application/vnd.comicbook+zip"
|
|
case ".cbr":
|
|
return "application/vnd.comicbook-rar"
|
|
case ".cb7":
|
|
return "application/x-7z-compressed"
|
|
case ".cbt":
|
|
return "application/x-tar"
|
|
default:
|
|
return "application/octet-stream"
|
|
}
|
|
}
|
|
|
|
func (s *MediaScanner) WatchChanges(ctx context.Context) {
|
|
go func() {
|
|
for {
|
|
select {
|
|
case event, ok := <-s.watcher.Events:
|
|
if !ok {
|
|
return
|
|
}
|
|
|
|
// Handle new directories - add them to the watcher
|
|
if event.Has(fsnotify.Create) {
|
|
info, err := os.Stat(event.Name)
|
|
if err == nil && info.IsDir() {
|
|
// Add the new directory to the watcher
|
|
if err := s.watcher.Add(event.Name); err != nil {
|
|
fmt.Printf("Warning: failed to watch new directory %s: %v\n", event.Name, err)
|
|
} else {
|
|
fmt.Printf("Now watching new directory: %s\n", event.Name)
|
|
}
|
|
}
|
|
}
|
|
|
|
// Handle file modifications and creations
|
|
if (event.Has(fsnotify.Create) || event.Has(fsnotify.Write)) && s.isScannableFile(event.Name) {
|
|
fmt.Printf("New/modified media file detected: %s\n", event.Name)
|
|
if err := s.processMediaFile(ctx, event.Name); err != nil {
|
|
fmt.Printf("Error processing modified media file %s: %v\n", event.Name, err)
|
|
}
|
|
}
|
|
|
|
case err, ok := <-s.watcher.Errors:
|
|
if !ok {
|
|
return
|
|
}
|
|
fmt.Printf("Watcher error: %v\n", err)
|
|
case <-ctx.Done():
|
|
return
|
|
}
|
|
}
|
|
}()
|
|
}
|
|
|
|
func (s *MediaScanner) Close() error {
|
|
if s.watcher != nil {
|
|
return s.watcher.Close()
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// ============================================
|
|
// SCANNER ENHANCEMENTS
|
|
// ============================================
|
|
|
|
// calculateFileSHA256 calculates SHA-256 hash using streaming to avoid loading entire file into memory
|
|
func (s *MediaScanner) calculateFileSHA256(filePath string) (string, error) {
|
|
file, err := os.Open(filePath)
|
|
if err != nil {
|
|
return "", fmt.Errorf("failed to open file: %v", err)
|
|
}
|
|
defer file.Close()
|
|
|
|
hasher := sha256.New()
|
|
if _, err := io.Copy(hasher, file); err != nil {
|
|
return "", fmt.Errorf("failed to calculate hash: %v", err)
|
|
}
|
|
|
|
return hex.EncodeToString(hasher.Sum(nil)), nil
|
|
}
|
|
|
|
// OPFIdentifier represents an identifier from OPF metadata
|
|
type OPFIdentifier struct {
|
|
XMLName xml.Name `xml:"identifier"`
|
|
ID string `xml:"id,attr"`
|
|
Scheme string `xml:"scheme,attr"`
|
|
Content string `xml:",chardata"`
|
|
}
|
|
|
|
// OPFMetadata represents parsed OPF metadata
|
|
type OPFMetadata struct {
|
|
XMLName xml.Name `xml:"package"`
|
|
Version string `xml:"version,attr"`
|
|
Identifiers []OPFIdentifier `xml:"metadata>identifier"`
|
|
}
|
|
|
|
// extractOPFIdentifiers extracts identifiers from EPUB OPF file
|
|
func (s *MediaScanner) extractOPFIdentifiers(epubPath string) (opfIdentifier, opfUUID string, confidence string, err error) {
|
|
book, err := epub.ReadBook(epubPath)
|
|
if err != nil {
|
|
return "", "", "", fmt.Errorf("failed to open EPUB: %v", err)
|
|
}
|
|
|
|
// Extract OPF identifiers using go-epub library
|
|
identifiers, err := book.MetadataByKey("identifier")
|
|
if err != nil || len(identifiers) == 0 {
|
|
return "", "", "low", nil
|
|
}
|
|
|
|
var identifier, uuid string
|
|
for _, id := range identifiers {
|
|
id = strings.TrimSpace(id)
|
|
|
|
// Check for UUID format (urn:uuid:)
|
|
if strings.HasPrefix(strings.ToLower(id), "urn:uuid:") {
|
|
uuid = strings.TrimPrefix(strings.ToLower(id), "urn:uuid:")
|
|
continue
|
|
}
|
|
|
|
// Check if it's a plain UUID (8-4-4-4-12 format)
|
|
if isValidUUID(id) {
|
|
uuid = id
|
|
continue
|
|
}
|
|
|
|
// Check for ISBN
|
|
if strings.Contains(strings.ToLower(id), "isbn") {
|
|
isbn := s.extractISBNFromIdentifier(id)
|
|
if isbn != "" {
|
|
identifier = isbn
|
|
continue
|
|
}
|
|
}
|
|
|
|
// Use first identifier as fallback
|
|
if identifier == "" && id != "" {
|
|
identifier = id
|
|
}
|
|
}
|
|
|
|
confidence = s.determineHashConfidence(uuid, identifier)
|
|
|
|
return identifier, uuid, confidence, nil
|
|
}
|
|
|
|
// isValidUUID checks if string is a valid UUID (8-4-4-4-12 format)
|
|
func isValidUUID(idStr string) bool {
|
|
uuidRegex := regexp.MustCompile(`^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$`)
|
|
return uuidRegex.MatchString(idStr)
|
|
}
|
|
|
|
// extractISBNFromIdentifier extracts ISBN from identifier string
|
|
func (s *MediaScanner) extractISBNFromIdentifier(id string) string {
|
|
id = strings.TrimSpace(id)
|
|
|
|
// Remove "isbn:" prefix if present
|
|
if strings.HasPrefix(strings.ToLower(id), "isbn:") {
|
|
id = strings.TrimPrefix(strings.ToLower(id), "isbn:")
|
|
}
|
|
|
|
// Remove hyphens and spaces
|
|
isbn := regexp.MustCompile(`[\s-]`).ReplaceAllString(id, "")
|
|
|
|
// Check if it's a valid ISBN-10 or ISBN-13
|
|
if len(isbn) == 10 || len(isbn) == 13 {
|
|
return isbn
|
|
}
|
|
|
|
return ""
|
|
}
|
|
|
|
// determineHashConfidence determines confidence level based on available identifiers
|
|
func (s *MediaScanner) determineHashConfidence(uuid, identifier string) string {
|
|
if uuid != "" && isValidUUID(uuid) {
|
|
return "high"
|
|
}
|
|
if identifier != "" && (strings.Contains(strings.ToLower(identifier), "isbn") || len(identifier) >= 10) {
|
|
return "medium"
|
|
}
|
|
return "low"
|
|
}
|
|
|
|
// detectFormatType detects the format type based on file extension and content
|
|
func (s *MediaScanner) detectFormatType(filePath string) string {
|
|
base := strings.ToLower(filepath.Base(filePath))
|
|
|
|
// Check for compound extensions first (like .kepub.epub)
|
|
if strings.HasSuffix(base, ".kepub.epub") {
|
|
return "kepub"
|
|
}
|
|
|
|
ext := strings.ToLower(filepath.Ext(filePath))
|
|
|
|
switch ext {
|
|
case ".epub":
|
|
return "epub"
|
|
case ".kepub":
|
|
return "kepub"
|
|
case ".pdf":
|
|
return "pdf"
|
|
case ".cbz", ".cbr", ".cb7", ".cbt":
|
|
return "comic_archive"
|
|
case ".mobi":
|
|
return "mobi"
|
|
case ".azw", ".azw3":
|
|
return "kfx"
|
|
case ".txt":
|
|
return "txt"
|
|
case ".fb2":
|
|
return "fb2"
|
|
default:
|
|
return "unknown"
|
|
}
|
|
}
|
|
|
|
// extractHashInfo calculates hash and extracts OPF identifiers for a file
|
|
func (s *MediaScanner) extractHashInfo(filePath string) (*HashInfo, *FormatInfo, error) {
|
|
// Calculate SHA-256
|
|
fileSHA256, err := s.calculateFileSHA256(filePath)
|
|
if err != nil {
|
|
return nil, nil, fmt.Errorf("failed to calculate SHA-256: %v", err)
|
|
}
|
|
|
|
// Get file info
|
|
info, err := os.Stat(filePath)
|
|
if err != nil {
|
|
return nil, nil, fmt.Errorf("failed to get file info: %v", err)
|
|
}
|
|
|
|
// Extract OPF identifiers for EPUB files
|
|
var opfIdentifier, opfUUID, confidence string
|
|
if strings.HasSuffix(strings.ToLower(filePath), ".epub") {
|
|
opfIdentifier, opfUUID, confidence, err = s.extractOPFIdentifiers(filePath)
|
|
if err != nil {
|
|
// Non-fatal error, continue with low confidence
|
|
confidence = "low"
|
|
}
|
|
}
|
|
|
|
hashInfo := &HashInfo{
|
|
FileSHA256: fileSHA256,
|
|
OPFIdentifier: opfIdentifier,
|
|
OPFUUID: opfUUID,
|
|
HashConfidence: confidence,
|
|
}
|
|
|
|
formatInfo := &FormatInfo{
|
|
FormatType: s.detectFormatType(filePath),
|
|
FilePath: filePath,
|
|
FileSHA256: fileSHA256,
|
|
FileSizeBytes: info.Size(),
|
|
MimeType: s.getMimeType(filePath),
|
|
}
|
|
|
|
return hashInfo, formatInfo, nil
|
|
}
|