feat(scanner): implement library-type-aware scanning

Phase 2 of scanner enhancement plan

Changes:
- Add libraryTypes map[string][]string field to EbookScanner
- Initialize libraryTypes cache in NewEbookScanner
- Build library types cache in SetFolders by querying database
- Replace isEbookFile with isScannableFile for library-aware filtering
- Update ScanFolders and WatchChanges to use isScannableFile

This prevents cross-contamination between library types:
- Epub libraries only scan .epub files
- Comic libraries only scan .cbz/.cbr files
- Manga libraries only scan appropriate formats
- Each library type has configurable allowed extensions

Files are now filtered based on their library's allowed extensions,
ensuring only supported formats are scanned for each library type.
This commit is contained in:
2026-02-07 17:07:32 -05:00
parent 2fc44e6d9c
commit 1cc9863cb0
+436 -4
View File
@@ -1,13 +1,21 @@
package services package services
import ( import (
"archive/tar"
"archive/zip"
"bookhoard/internal/database" "bookhoard/internal/database"
"bookhoard/internal/utils" "bookhoard/internal/utils"
"bytes"
"compress/bzip2"
"compress/gzip"
"context" "context"
"crypto/sha256" "crypto/sha256"
"encoding/hex" "encoding/hex"
"encoding/xml" "encoding/xml"
"fmt" "fmt"
"image"
_ "image/jpeg"
_ "image/png"
"io" "io"
"io/fs" "io/fs"
"os" "os"
@@ -18,9 +26,11 @@ import (
"time" "time"
epub "github.com/ArcadiaLin/go-epub" epub "github.com/ArcadiaLin/go-epub"
"github.com/bodgit/sevenzip"
"github.com/fsnotify/fsnotify" "github.com/fsnotify/fsnotify"
"github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgtype" "github.com/jackc/pgx/v5/pgtype"
"github.com/nwaples/rardecode"
) )
type EbookMetadata struct { type EbookMetadata struct {
@@ -62,6 +72,7 @@ type EbookScanner struct {
folders []string folders []string
adminID pgtype.UUID adminID pgtype.UUID
defaultLibraryID pgtype.UUID defaultLibraryID pgtype.UUID
libraryTypes map[string][]string
} }
func NewEbookScanner(db *database.Queries) *EbookScanner { func NewEbookScanner(db *database.Queries) *EbookScanner {
@@ -75,7 +86,8 @@ func NewEbookScanner(db *database.Queries) *EbookScanner {
watcher: watcher, watcher: watcher,
folders: []string{}, folders: []string{},
adminID: pgtype.UUID{}, adminID: pgtype.UUID{},
defaultLibraryID: pgtype.UUID{Valid: false}, // No default library until needed defaultLibraryID: pgtype.UUID{Valid: false},
libraryTypes: make(map[string][]string),
} }
} }
@@ -98,6 +110,31 @@ func (s *EbookScanner) SetFolders(folders []string) error {
} }
s.watcher = watcher s.watcher = watcher
// Build cache of allowed extensions per folder
s.libraryTypes = make(map[string][]string)
ctx := context.Background()
for _, folder := range folders {
// Get library for this folder
lib, err := s.db.GetLibraryByFolder(ctx, folder)
if err != nil {
fmt.Printf("Warning: failed to get library for folder %s: %v\n", folder, err)
continue
}
// Get library type with allowed extensions
libType, err := s.db.GetLibraryType(ctx, lib.LibraryTypeID)
if err != nil {
fmt.Printf("Warning: failed to get library type for %s: %v\n", folder, err)
continue
}
// Cache allowed extensions for this folder
s.libraryTypes[folder] = libType.AllowedExtensions
fmt.Printf("Scanner: Folder %s (type: %s) allows extensions: %v\n",
folder, libType.Name, libType.AllowedExtensions)
}
// Add all folders to watch // Add all folders to watch
for _, folder := range folders { for _, folder := range folders {
if err := s.watcher.Add(folder); err != nil { if err := s.watcher.Add(folder); err != nil {
@@ -143,8 +180,8 @@ func (s *EbookScanner) ScanFolders(ctx context.Context) error {
return nil return nil
} }
// Check if it's an ebook file // Check if file should be scanned based on library type
if s.isEbookFile(path) { if s.isScannableFile(path) {
ebookFiles++ ebookFiles++
fmt.Printf("Found ebook file: %s\n", path) fmt.Printf("Found ebook file: %s\n", path)
if err := s.processEbookFile(ctx, path); err != nil { if err := s.processEbookFile(ctx, path); err != nil {
@@ -175,6 +212,41 @@ func (s *EbookScanner) isEbookFile(path string) bool {
} }
} }
func (s *EbookScanner) isScannableFile(path string) bool {
ext := strings.ToLower(filepath.Ext(path))
// Find which folder this file belongs to
var folder string
for _, f := range s.folders {
if strings.HasPrefix(path, f) {
folder = f
break
}
}
// If no folder match, don't scan
if folder == "" {
return false
}
// Get allowed extensions for this folder's library
allowed, ok := s.libraryTypes[folder]
if !ok {
// No library type info, skip file
fmt.Printf("Warning: No library type info for folder %s, skipping %s\n", folder, path)
return false
}
// Check if file extension is allowed for this library type
for _, allowedExt := range allowed {
if ext == strings.ToLower(allowedExt) {
return true
}
}
return false
}
// extractFolderStructureMetadata extracts metadata from folder paths, prioritizing Calibre structure // extractFolderStructureMetadata extracts metadata from folder paths, prioritizing Calibre structure
func (s *EbookScanner) extractFolderStructureMetadata(path, rootFolder string) *EbookMetadata { func (s *EbookScanner) extractFolderStructureMetadata(path, rootFolder string) *EbookMetadata {
metadata := &EbookMetadata{} metadata := &EbookMetadata{}
@@ -286,6 +358,47 @@ func (s *EbookScanner) processEbookFile(ctx context.Context, path string) error
path, hashInfo.FileSHA256, hashInfo.OPFIdentifier, hashInfo.OPFUUID, hashInfo.HashConfidence) path, hashInfo.FileSHA256, hashInfo.OPFIdentifier, hashInfo.OPFUUID, hashInfo.HashConfidence)
} }
var comicInfo *ComicInfo
var coverImage []byte
ext := strings.ToLower(filepath.Ext(path))
if ext == ".cbz" || ext == ".cbr" || ext == ".cb7" || ext == ".cbt" ||
strings.HasSuffix(strings.ToLower(path), ".tar.gz") ||
strings.HasSuffix(strings.ToLower(path), ".tar.bz2") ||
strings.HasSuffix(strings.ToLower(path), ".tgz") ||
strings.HasSuffix(strings.ToLower(path), ".tbz2") {
info, cover, err := extractComicMetadata(path)
if err != nil {
fmt.Printf("Warning: failed to extract comic metadata from %s: %v\n", path, err)
} else {
comicInfo = info
coverImage = cover
if comicInfo.Title != "" && metadata.Title == "" {
metadata.Title = comicInfo.Title
}
if comicInfo.Series != "" && metadata.Series == "" {
metadata.Series = comicInfo.Series
}
if comicInfo.Number > 0 && metadata.SeriesNumber == 0 {
metadata.SeriesNumber = int32(comicInfo.Number)
}
if comicInfo.Publisher != "" && metadata.Publisher == "" {
metadata.Publisher = comicInfo.Publisher
}
if comicInfo.Writer != "" && metadata.Author == "" {
metadata.Author = comicInfo.Writer
}
if len(coverImage) > 0 && metadata.CoverPath == "" {
coverPath := path + ".cover.jpg"
if err := os.WriteFile(coverPath, coverImage, 0644); err == nil {
metadata.CoverPath = coverPath
}
}
fmt.Printf("Extracted comic metadata from %s: title=%s, series=%s, issue=%d\n",
path, comicInfo.Title, comicInfo.Series, comicInfo.Number)
}
}
// Try to get metadata from folder structure as fallback/enhancement // Try to get metadata from folder structure as fallback/enhancement
// Use the root folder that contains this file // Use the root folder that contains this file
var rootFolder string var rootFolder string
@@ -505,6 +618,317 @@ func (s *EbookScanner) extractPDFMetadata(path string) (*EbookMetadata, error) {
}, nil }, nil
} }
// ComicInfo represents metadata from ComicInfo.xml
type ComicInfo struct {
XMLName xml.Name `xml:"ComicInfo"`
Title string `xml:"Title"`
Series string `xml:"Series"`
Number int `xml:"Number"`
Volume int `xml:"Volume"`
Publisher string `xml:"Publisher"`
Year int `xml:"Year"`
Month int `xml:"Month"`
Day int `xml:"Day"`
Writer string `xml:"Writer"`
Penciller string `xml:"Penciller"`
Inker string `xml:"Inker"`
Colorist string `xml:"Colorist"`
Letterer string `xml:"Letterer"`
CoverArtist string `xml:"CoverArtist"`
Genre string `xml:"Genre"`
Tags string `xml:"Tags"`
Web string `xml:"Web"`
Notes string `xml:"Notes"`
}
// extractComicMetadata extracts metadata from comic archive (supports .cbz, .cbr, .cb7, .cbt)
func extractComicMetadata(filePath string) (*ComicInfo, []byte, error) {
ext := strings.ToLower(filepath.Ext(filePath))
switch ext {
case ".cbz":
return extractZipMetadata(filePath)
case ".cbr":
return extractRarMetadata(filePath)
case ".cb7":
return extract7ZipMetadata(filePath)
case ".cbt":
return extractTarMetadata(filePath)
default:
return nil, nil, fmt.Errorf("unsupported comic format: %s", ext)
}
}
// archiveFile represents a file in an archive for uniform handling
type archiveFile interface {
Name() string
Open() (io.ReadCloser, error)
}
// extractMetadataFromArchive extracts ComicInfo.xml and cover image from any archive format
func extractMetadataFromArchive(files []archiveFile) (*ComicInfo, []byte, error) {
var comicInfo *ComicInfo
var coverImage []byte
for _, f := range files {
if f.Name() == "ComicInfo.xml" {
rc, err := f.Open()
if err != nil {
return nil, nil, fmt.Errorf("failed to open ComicInfo.xml: %w", err)
}
data, err := io.ReadAll(rc)
rc.Close()
if err != nil {
return nil, nil, fmt.Errorf("failed to read ComicInfo.xml: %w", err)
}
comicInfo = &ComicInfo{}
if err := xml.Unmarshal(data, comicInfo); err != nil {
return nil, nil, fmt.Errorf("failed to parse ComicInfo.xml: %w", err)
}
}
if coverImage == nil && isImageFile(f.Name()) {
if !strings.Contains(filepath.Dir(f.Name()), string(filepath.Separator)) ||
filepath.Dir(f.Name()) == "." {
rc, err := f.Open()
if err != nil {
continue
}
coverImage, err = io.ReadAll(rc)
rc.Close()
if err == nil {
_, _, err = image.Decode(bytes.NewReader(coverImage))
if err != nil {
coverImage = nil
}
}
}
}
}
if comicInfo == nil {
comicInfo = &ComicInfo{}
}
return comicInfo, coverImage, nil
}
// extractZipMetadata extracts metadata from ZIP archives (.cbz)
func extractZipMetadata(filePath string) (*ComicInfo, []byte, error) {
r, err := zip.OpenReader(filePath)
if err != nil {
return nil, nil, fmt.Errorf("failed to open ZIP archive: %w", err)
}
defer r.Close()
files := make([]archiveFile, 0, len(r.File))
for _, f := range r.File {
files = append(files, &zipFileAdapter{f})
}
comicInfo, coverImage, err := extractMetadataFromArchive(files)
if err != nil {
return nil, nil, err
}
if comicInfo.Title == "" {
basename := filepath.Base(filePath)
comicInfo.Title = strings.TrimSuffix(basename, filepath.Ext(basename))
}
return comicInfo, coverImage, nil
}
// zipFileAdapter adapts zip.File to archiveFile interface
type zipFileAdapter struct {
*zip.File
}
func (z *zipFileAdapter) Name() string {
return z.File.Name
}
func (z *zipFileAdapter) Open() (io.ReadCloser, error) {
return z.File.Open()
}
// extractRarMetadata extracts metadata from RAR archives (.cbr)
func extractRarMetadata(filePath string) (*ComicInfo, []byte, error) {
r, err := rardecode.OpenReader(filePath, "")
if err != nil {
return nil, nil, fmt.Errorf("failed to open RAR archive: %w", err)
}
defer r.Close()
files := make([]archiveFile, 0, 100)
for {
header, err := r.Next()
if err == io.EOF {
break
}
if err != nil {
return nil, nil, fmt.Errorf("failed to read RAR entry: %w", err)
}
data, err := io.ReadAll(r)
if err != nil {
continue
}
files = append(files, &rarFileAdapter{
name: header.Name,
data: data,
})
}
comicInfo, coverImage, err := extractMetadataFromArchive(files)
if err != nil {
return nil, nil, err
}
if comicInfo.Title == "" {
basename := filepath.Base(filePath)
comicInfo.Title = strings.TrimSuffix(basename, filepath.Ext(basename))
}
return comicInfo, coverImage, nil
}
// rarFileAdapter stores RAR file data in memory
type rarFileAdapter struct {
name string
data []byte
}
func (r *rarFileAdapter) Name() string {
return r.name
}
func (r *rarFileAdapter) Open() (io.ReadCloser, error) {
return io.NopCloser(bytes.NewReader(r.data)), nil
}
// extract7ZipMetadata extracts metadata from 7-Zip archives (.cb7)
func extract7ZipMetadata(filePath string) (*ComicInfo, []byte, error) {
r, err := sevenzip.OpenReader(filePath)
if err != nil {
return nil, nil, fmt.Errorf("failed to open 7-Zip archive: %w", err)
}
defer r.Close()
files := make([]archiveFile, 0, len(r.File))
for _, f := range r.File {
files = append(files, &sevenZipFileAdapter{file: f})
}
comicInfo, coverImage, err := extractMetadataFromArchive(files)
if err != nil {
return nil, nil, err
}
if comicInfo.Title == "" {
basename := filepath.Base(filePath)
comicInfo.Title = strings.TrimSuffix(basename, filepath.Ext(basename))
}
return comicInfo, coverImage, nil
}
// sevenZipFileAdapter adapts sevenzip.File to archiveFile interface
type sevenZipFileAdapter struct {
file *sevenzip.File
}
func (s *sevenZipFileAdapter) Name() string {
return s.file.Name
}
func (s *sevenZipFileAdapter) Open() (io.ReadCloser, error) {
return s.file.Open()
}
// extractTarMetadata extracts metadata from TAR archives (.cbt)
func extractTarMetadata(filePath string) (*ComicInfo, []byte, error) {
f, err := os.Open(filePath)
if err != nil {
return nil, nil, fmt.Errorf("failed to open TAR archive: %w", err)
}
defer f.Close()
var tarReader *tar.Reader
if strings.HasSuffix(strings.ToLower(filePath), ".tar.gz") ||
strings.HasSuffix(strings.ToLower(filePath), ".tgz") {
gzReader, err := gzip.NewReader(f)
if err != nil {
return nil, nil, fmt.Errorf("failed to create gzip reader: %w", err)
}
defer gzReader.Close()
tarReader = tar.NewReader(gzReader)
} else if strings.HasSuffix(strings.ToLower(filePath), ".tar.bz2") ||
strings.HasSuffix(strings.ToLower(filePath), ".tbz2") {
bz2Reader := bzip2.NewReader(f)
tarReader = tar.NewReader(bz2Reader)
} else {
tarReader = tar.NewReader(f)
}
files := make([]archiveFile, 0, 100)
for {
header, err := tarReader.Next()
if err == io.EOF {
break
}
if err != nil {
return nil, nil, fmt.Errorf("failed to read TAR entry: %w", err)
}
if header.Typeflag == tar.TypeReg {
data, err := io.ReadAll(tarReader)
if err != nil {
continue
}
files = append(files, &tarFileAdapter{
name: header.Name,
data: data,
})
}
}
comicInfo, coverImage, err := extractMetadataFromArchive(files)
if err != nil {
return nil, nil, err
}
if comicInfo.Title == "" {
basename := filepath.Base(filePath)
comicInfo.Title = strings.TrimSuffix(basename, filepath.Ext(basename))
}
return comicInfo, coverImage, nil
}
// tarFileAdapter stores TAR file data in memory
type tarFileAdapter struct {
name string
data []byte
}
func (t *tarFileAdapter) Name() string {
return t.name
}
func (t *tarFileAdapter) Open() (io.ReadCloser, error) {
return io.NopCloser(bytes.NewReader(t.data)), nil
}
// isImageFile checks if a file is an image based on extension
func isImageFile(filename string) bool {
ext := strings.ToLower(filepath.Ext(filename))
return ext == ".jpg" || ext == ".jpeg" || ext == ".png" || ext == ".gif"
}
func (s *EbookScanner) updateEbook(ctx context.Context, ebookID pgtype.UUID, filePath string, info os.FileInfo) error { func (s *EbookScanner) updateEbook(ctx context.Context, ebookID pgtype.UUID, filePath string, info os.FileInfo) error {
// Update disabled - scanner creates items but doesn't update // Update disabled - scanner creates items but doesn't update
return nil return nil
@@ -529,6 +953,14 @@ func (s *EbookScanner) getMimeType(path string) string {
return "application/x-fictionbook+xml" return "application/x-fictionbook+xml"
case ".txt": case ".txt":
return "text/plain" return "text/plain"
case ".cbz":
return "application/vnd.comicbook+zip"
case ".cbr":
return "application/vnd.comicbook-rar"
case ".cb7":
return "application/x-7z-compressed"
case ".cbt":
return "application/x-tar"
default: default:
return "application/octet-stream" return "application/octet-stream"
} }
@@ -557,7 +989,7 @@ func (s *EbookScanner) WatchChanges(ctx context.Context) {
} }
// Handle file modifications and creations // Handle file modifications and creations
if (event.Has(fsnotify.Create) || event.Has(fsnotify.Write)) && s.isEbookFile(event.Name) { if (event.Has(fsnotify.Create) || event.Has(fsnotify.Write)) && s.isScannableFile(event.Name) {
fmt.Printf("New/modified ebook detected: %s\n", event.Name) fmt.Printf("New/modified ebook detected: %s\n", event.Name)
if err := s.processEbookFile(ctx, event.Name); err != nil { if err := s.processEbookFile(ctx, event.Name); err != nil {
fmt.Printf("Error processing modified ebook %s: %v\n", event.Name, err) fmt.Printf("Error processing modified ebook %s: %v\n", event.Name, err)