fix(scanner): populate page count and total characters during media scanning
The media scanner never populated page_count or total_characters in media_items, leaving progress display and reading position calculations with no reliable data. This commit fixes data population for all formats: Comics (CBZ/CBR/CB7/CBT): - Add countArchiveImages() helper that walks archive entries and counts image files (.jpg, .jpeg, .png, .gif, .webp) - Call it during comic metadata merge to set metadata.PageCount PDFs: - Extract pdfInfo.PageCount from the pdfcpu library (already available from PDFInfo call, just never used) and set metadata.PageCount Reflowable EPUBs: - Use book.AllChaptersText() to compute metadata.TotalCharacters - Use book.ChapterCount() to set metadata.ChapterCount Fixed-layout EPUBs (manga/comics in EPUB format): - Merge .epub into the .cbz case in countArchiveImages since both are ZIP archives with images - Detect fixed-layout EPUBs via DetectFixedLayoutEPUB() in both the Calibre sidecar path (mergeMetadata) and the no-sidecar path (extractMetadata), counting images when fixed-layout is detected Format group on creation: - Remove the guard condition on UpdateMediaItemFormatGroup so that format_group, is_reflowable, and has_fixed_layout are set immediately for every new item (not just items with text data) - Use DetectFixedLayoutEPUB() instead of hardcoding all .epub as reflowable, correctly classifying fixed-layout EPUBs Also pass PageCount to CreateMediaItem and add PageCount, TotalCharacters, and ChapterCount fields to the MediaMetadata struct.
This commit is contained in:
@@ -75,6 +75,9 @@ type MediaMetadata struct {
|
|||||||
WebURL string // URL to info page (Goodreads, ComicVine, etc.)
|
WebURL string // URL to info page (Goodreads, ComicVine, etc.)
|
||||||
MetadataNotes string // Notes from metadata files (not user notes)
|
MetadataNotes string // Notes from metadata files (not user notes)
|
||||||
CommunityRating float64 // Pre-existing community rating (0-10)
|
CommunityRating float64 // Pre-existing community rating (0-10)
|
||||||
|
PageCount int32 // Actual page count (images for comics, pages for PDF)
|
||||||
|
TotalCharacters int64 // Total text characters (for reflowable EPUBs)
|
||||||
|
ChapterCount int32 // Number of chapters detected
|
||||||
|
|
||||||
// Comic-specific fields
|
// Comic-specific fields
|
||||||
StoryArc string // Story arc name
|
StoryArc string // Story arc name
|
||||||
@@ -707,6 +710,7 @@ func (s *MediaScanner) processMediaFile(ctx context.Context, path string) (bool,
|
|||||||
ScanInformation: pgtype.Text{String: metadata.ScanInformation, Valid: metadata.ScanInformation != ""},
|
ScanInformation: pgtype.Text{String: metadata.ScanInformation, Valid: metadata.ScanInformation != ""},
|
||||||
Summary: pgtype.Text{String: metadata.Summary, Valid: metadata.Summary != ""},
|
Summary: pgtype.Text{String: metadata.Summary, Valid: metadata.Summary != ""},
|
||||||
CommunityRating: pgtype.Float8{Float64: metadata.CommunityRating, Valid: metadata.CommunityRating > 0},
|
CommunityRating: pgtype.Float8{Float64: metadata.CommunityRating, Valid: metadata.CommunityRating > 0},
|
||||||
|
PageCount: pgtype.Int4{Int32: metadata.PageCount, Valid: metadata.PageCount > 0},
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return false, fmt.Errorf("failed to create media item: %v", err)
|
return false, fmt.Errorf("failed to create media item: %v", err)
|
||||||
@@ -726,6 +730,46 @@ func (s *MediaScanner) processMediaFile(ctx context.Context, path string) (bool,
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Set format group, total characters, and chapter count
|
||||||
|
mimeType := s.getMimeType(path)
|
||||||
|
ext := strings.ToLower(filepath.Ext(path))
|
||||||
|
var formatGroup string
|
||||||
|
var isReflowable, hasFixedLayout bool
|
||||||
|
switch ext {
|
||||||
|
case ".epub":
|
||||||
|
isFixed, fixedErr := s.DetectFixedLayoutEPUB(path)
|
||||||
|
if fixedErr == nil && isFixed {
|
||||||
|
formatGroup = "fixed_layout"
|
||||||
|
hasFixedLayout = true
|
||||||
|
} else {
|
||||||
|
formatGroup = "reflowable"
|
||||||
|
isReflowable = true
|
||||||
|
}
|
||||||
|
case ".mobi", ".azw", ".azw3", ".fb2", ".txt":
|
||||||
|
formatGroup = "reflowable"
|
||||||
|
isReflowable = true
|
||||||
|
case ".pdf", ".djvu":
|
||||||
|
formatGroup = "fixed_layout"
|
||||||
|
hasFixedLayout = true
|
||||||
|
case ".cbz", ".cbr", ".cb7", ".cbt":
|
||||||
|
formatGroup = "comic_archive"
|
||||||
|
hasFixedLayout = true
|
||||||
|
default:
|
||||||
|
formatGroup = "unknown"
|
||||||
|
}
|
||||||
|
err = s.db.UpdateMediaItemFormatGroup(ctx, database.UpdateMediaItemFormatGroupParams{
|
||||||
|
ID: createdItem.ID,
|
||||||
|
FormatGroup: formatGroup,
|
||||||
|
FormatMimetype: pgtype.Text{String: mimeType, Valid: mimeType != ""},
|
||||||
|
IsReflowable: pgtype.Bool{Bool: isReflowable, Valid: true},
|
||||||
|
HasFixedLayout: pgtype.Bool{Bool: hasFixedLayout, Valid: true},
|
||||||
|
TotalCharacters: pgtype.Int8{Int64: metadata.TotalCharacters, Valid: metadata.TotalCharacters > 0},
|
||||||
|
ChapterCount: pgtype.Int4{Int32: metadata.ChapterCount, Valid: metadata.ChapterCount > 0},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
fmt.Printf("Warning: failed to update format info for %s: %v\n", path, err)
|
||||||
|
}
|
||||||
|
|
||||||
// Store format information in the database
|
// Store format information in the database
|
||||||
for _, format := range metadata.FileFormats {
|
for _, format := range metadata.FileFormats {
|
||||||
_, err = s.db.CreateMediaItemFormat(ctx, database.CreateMediaItemFormatParams{
|
_, err = s.db.CreateMediaItemFormat(ctx, database.CreateMediaItemFormatParams{
|
||||||
@@ -783,6 +827,16 @@ func (s *MediaScanner) mergeMetadata(path string, calibreMetadata *MediaMetadata
|
|||||||
if err == nil {
|
if err == nil {
|
||||||
genreTags := extractGenreTagsFromEPUB(book)
|
genreTags := extractGenreTagsFromEPUB(book)
|
||||||
processGenresAndTags(metadata, genreTags)
|
processGenresAndTags(metadata, genreTags)
|
||||||
|
if allText := book.AllChaptersText(); len(allText) > 0 {
|
||||||
|
metadata.TotalCharacters = int64(len(allText))
|
||||||
|
}
|
||||||
|
metadata.ChapterCount = int32(book.ChapterCount())
|
||||||
|
}
|
||||||
|
isFixed, fixedErr := s.DetectFixedLayoutEPUB(path)
|
||||||
|
if fixedErr == nil && isFixed {
|
||||||
|
if pageCount, imgErr := countArchiveImages(path); imgErr == nil && pageCount > 0 {
|
||||||
|
metadata.PageCount = int32(pageCount)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -899,6 +953,10 @@ func (s *MediaScanner) mergeMetadata(path string, calibreMetadata *MediaMetadata
|
|||||||
fmt.Printf("Merged comic metadata from %s: title=%s, series=%s, issue=%d, manga=%s, direction=%s\n",
|
fmt.Printf("Merged comic metadata from %s: title=%s, series=%s, issue=%d, manga=%s, direction=%s\n",
|
||||||
path, comicInfo.Title, comicInfo.Series, comicInfo.Number, comicInfo.Manga, metadata.ReadingDirection)
|
path, comicInfo.Title, comicInfo.Series, comicInfo.Number, comicInfo.Manga, metadata.ReadingDirection)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if pageCount, err := countArchiveImages(path); err == nil && pageCount > 0 {
|
||||||
|
metadata.PageCount = int32(pageCount)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return metadata, nil
|
return metadata, nil
|
||||||
@@ -1089,12 +1147,14 @@ func (s *MediaScanner) extractMetadata(path string) (*MediaMetadata, error) {
|
|||||||
// Enhanced format detection for EPUBs
|
// Enhanced format detection for EPUBs
|
||||||
isFixedLayout, detectErr := s.DetectFixedLayoutEPUB(path)
|
isFixedLayout, detectErr := s.DetectFixedLayoutEPUB(path)
|
||||||
if detectErr == nil && isFixedLayout {
|
if detectErr == nil && isFixedLayout {
|
||||||
// Override format group for manga EPUBs
|
|
||||||
metadata.FileFormats = []*FormatInfo{{
|
metadata.FileFormats = []*FormatInfo{{
|
||||||
FormatType: "fixed_layout",
|
FormatType: "fixed_layout",
|
||||||
FilePath: path,
|
FilePath: path,
|
||||||
MimeType: s.getMimeType(path),
|
MimeType: s.getMimeType(path),
|
||||||
}}
|
}}
|
||||||
|
if pageCount, imgErr := countArchiveImages(path); imgErr == nil && pageCount > 0 {
|
||||||
|
metadata.PageCount = int32(pageCount)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
// Try to extract embedded cover
|
// Try to extract embedded cover
|
||||||
coverPath, err := s.extractEPUBCover(path)
|
coverPath, err := s.extractEPUBCover(path)
|
||||||
@@ -1832,6 +1892,8 @@ func (s *MediaScanner) extractPDFMetadata(path string) (*MediaMetadata, error) {
|
|||||||
metadata.Publisher = pdfInfo.Producer
|
metadata.Publisher = pdfInfo.Producer
|
||||||
}
|
}
|
||||||
|
|
||||||
|
metadata.PageCount = int32(pdfInfo.PageCount)
|
||||||
|
|
||||||
// Try to extract cover image
|
// Try to extract cover image
|
||||||
coverPath, err := s.extractPDFCover(path)
|
coverPath, err := s.extractPDFCover(path)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -2276,7 +2338,77 @@ func (t *tarFileAdapter) Open() (io.ReadCloser, error) {
|
|||||||
// isImageFile checks if a file is an image based on extension
|
// isImageFile checks if a file is an image based on extension
|
||||||
func isImageFile(filename string) bool {
|
func isImageFile(filename string) bool {
|
||||||
ext := strings.ToLower(filepath.Ext(filename))
|
ext := strings.ToLower(filepath.Ext(filename))
|
||||||
return ext == ".jpg" || ext == ".jpeg" || ext == ".png" || ext == ".gif"
|
return ext == ".jpg" || ext == ".jpeg" || ext == ".png" || ext == ".gif" || ext == ".webp"
|
||||||
|
}
|
||||||
|
|
||||||
|
// countArchiveImages counts image files in a comic archive
|
||||||
|
func countArchiveImages(filePath string) (int, error) {
|
||||||
|
ext := strings.ToLower(filepath.Ext(filePath))
|
||||||
|
count := 0
|
||||||
|
|
||||||
|
switch ext {
|
||||||
|
case ".cbz", ".epub":
|
||||||
|
r, err := zip.OpenReader(filePath)
|
||||||
|
if err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
defer r.Close()
|
||||||
|
for _, f := range r.File {
|
||||||
|
if !f.FileInfo().IsDir() && isImageFile(f.Name) {
|
||||||
|
count++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
case ".cbr":
|
||||||
|
r, err := rardecode.OpenReader(filePath, "")
|
||||||
|
if err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
defer r.Close()
|
||||||
|
for {
|
||||||
|
header, err := r.Next()
|
||||||
|
if err == io.EOF {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
if !header.IsDir && isImageFile(header.Name) {
|
||||||
|
count++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
case ".cb7":
|
||||||
|
sz, err := sevenzip.OpenReader(filePath)
|
||||||
|
if err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
defer sz.Close()
|
||||||
|
for _, f := range sz.File {
|
||||||
|
if !f.FileInfo().IsDir() && isImageFile(f.Name) {
|
||||||
|
count++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
case ".cbt":
|
||||||
|
f, err := os.Open(filePath)
|
||||||
|
if err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
defer f.Close()
|
||||||
|
tr := tar.NewReader(f)
|
||||||
|
for {
|
||||||
|
header, err := tr.Next()
|
||||||
|
if err == io.EOF {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
if !header.FileInfo().IsDir() && isImageFile(header.Name) {
|
||||||
|
count++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return count, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *MediaScanner) updateMediaItem(ctx context.Context, mediaItemID pgtype.UUID, path string, _ os.FileInfo) error {
|
func (s *MediaScanner) updateMediaItem(ctx context.Context, mediaItemID pgtype.UUID, path string, _ os.FileInfo) error {
|
||||||
|
|||||||
Reference in New Issue
Block a user