From 831668a07dbcbcf9b0494d2db11d4fc4a007652d Mon Sep 17 00:00:00 2001 From: John O'Keefe Date: Sun, 29 Mar 2026 21:12:23 -0400 Subject: [PATCH] feat(media_scanner): implement smart metadata merging and ComicInfo.xml v2.0 support MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Phase 2-3 implementation: Complete ComicInfo.xml parsing with intelligent Calibre merging. Data Structure Updates: - ComicInfo struct: Add 19 ComicInfo.xml v2.0 fields (Manga, LanguageISO, Count, AlternateSeries, AlternateNumber, AlternateCount, Summary, Imprint, StoryArc, SeriesGroup, AgeRating, CommunityRating, MainCharacterOrTeam, Review, BlackAndWhite, ScanInformation, Characters, Teams, Locations) - MediaMetadata struct: Add 14 fields for reading direction and universal/comic metadata New Functions: - mergeMetadata(): Smart merging with priority: Calibre metadata.opf → ComicInfo.xml → embedded metadata. Extracts reading direction even when metadata.opf exists. - normalizeMangaType(): Normalize ComicInfo.xml Manga field to database enum - determineReadingDirection(): Compute reading direction from Manga + language + genre heuristics - normalizeAgeRating(): Standardize age rating values (Everyone, Teen, Mature, Adult) - processGenresAndTags(): Universal genre/tag processing for all formats - extractGenreTagsFromEPUB(): Extract all values from EPUB - extractGenreTagsFromComicInfo(): Extract genres from Genre + Tags + Characters + Teams + Locations - containsTag(): Helper to prevent duplicate tags Logic Changes: - extractMetadata(): Now calls mergeMetadata() for smart metadata combination - processMediaFile(): Updated CreateMediaItem call with all 14 new fields - Removed duplicate comic metadata extraction (now handled by mergeMetadata) - CommunityRating uses simple pgtype.Float8 (DOUBLE PRECISION) instead of pgtype.Numeric This enables complete ComicInfo.xml v2.0 support with 19 fields plus 5 universal fields that apply to all media formats (ebooks, audiobooks, comics). Relates to: Phase 2 (data structures), Phase 3 (smart merging), Phase 4 (media item creation) --- internal/services/media_scanner.go | 408 +++++++++++++++++++++++++---- 1 file changed, 364 insertions(+), 44 deletions(-) diff --git a/internal/services/media_scanner.go b/internal/services/media_scanner.go index ed6cdc5..64af7f6 100644 --- a/internal/services/media_scanner.go +++ b/internal/services/media_scanner.go @@ -11,6 +11,7 @@ import ( "context" "crypto/sha256" "encoding/hex" + "encoding/json" "encoding/xml" "fmt" "image" @@ -55,6 +56,28 @@ type MediaMetadata struct { FileHashInfo *HashInfo FileFormats []*FormatInfo + + // NEW: Reading direction fields (from ComicInfo.xml or computed) + MangaType string // Raw ComicInfo.xml Manga field + ReadingDirection string // Computed: auto, ltr, rtl, vertical + Language string // ISO 639-1 language code + + // NEW: Additional metadata fields (from ComicInfo.xml or other metadata sources) + // Universal fields (apply to ebooks, audiobooks, comics) + SeriesCount int32 // Total items in series (Count field for comics, series count for books) + Volume int32 // Volume/omnibus number + Imprint string // Publisher imprint + AgeRating string // Age rating (Everyone, Teen, Mature, Adult) + WebURL string // URL to info page (Goodreads, ComicVine, etc.) + MetadataNotes string // Notes from metadata files (not user notes) + CommunityRating float64 // Pre-existing community rating (0-10) + + // Comic-specific fields + StoryArc string // Story arc name + IsBlackAndWhite bool // Black and white flag + AlternateInfo string // JSONB string of alternate series info + ScanInformation string // Scan information + Summary string // Summary from ComicInfo.xml } type HashInfo struct { @@ -580,46 +603,8 @@ func (s *MediaScanner) processMediaFile(ctx context.Context, path string) (bool, path, hashInfo.FileSHA256, hashInfo.OPFIdentifier, hashInfo.OPFUUID, hashInfo.HashConfidence) } - var comicInfo *ComicInfo - var coverImage []byte - - ext := strings.ToLower(filepath.Ext(path)) - if ext == ".cbz" || ext == ".cbr" || ext == ".cb7" || ext == ".cbt" || - strings.HasSuffix(strings.ToLower(path), ".tar.gz") || - strings.HasSuffix(strings.ToLower(path), ".tar.bz2") || - strings.HasSuffix(strings.ToLower(path), ".tgz") || - strings.HasSuffix(strings.ToLower(path), ".tbz2") { - info, cover, err := extractComicMetadata(path) - if err != nil { - fmt.Printf("Warning: failed to extract comic metadata from %s: %v\n", path, err) - } else { - comicInfo = info - coverImage = cover - if comicInfo.Title != "" && metadata.Title == "" { - metadata.Title = comicInfo.Title - } - if comicInfo.Series != "" && metadata.Series == "" { - metadata.Series = comicInfo.Series - } - if comicInfo.Number > 0 && metadata.SeriesNumber == 0 { - metadata.SeriesNumber = int32(comicInfo.Number) - } - if comicInfo.Publisher != "" && metadata.Publisher == "" { - metadata.Publisher = comicInfo.Publisher - } - if comicInfo.Writer != "" && metadata.Author == "" { - metadata.Author = comicInfo.Writer - } - if len(coverImage) > 0 && metadata.CoverPath == "" { - coverPath := path + ".cover.jpg" - if err := os.WriteFile(coverPath, coverImage, 0644); err == nil { - metadata.CoverPath = s.getRelativePath(coverPath) - } - } - fmt.Printf("Extracted comic metadata from %s: title=%s, series=%s, issue=%d\n", - path, comicInfo.Title, comicInfo.Series, comicInfo.Number) - } - } + // REMOVED: Comic metadata extraction now handled by mergeMetadata() + // This avoids duplicate extraction and ensures smart merging happens // Try to get metadata from folder structure as fallback/enhancement // Use the root folder that contains this file @@ -690,6 +675,25 @@ func (s *MediaScanner) processMediaFile(ctx context.Context, path string) (bool, TagsSearch: tagsSearch, AddedByAdminID: s.adminID, CreatedAt: pgtype.Timestamptz{Time: fileModTime, Valid: true}, + MangaType: pgtype.Text{String: metadata.MangaType, Valid: metadata.MangaType != ""}, + ReadingDirection: pgtype.Text{String: metadata.ReadingDirection, Valid: metadata.ReadingDirection != ""}, + SeriesCount: pgtype.Int4{Int32: metadata.SeriesCount, Valid: metadata.SeriesCount > 0}, + Volume: pgtype.Int4{Int32: metadata.Volume, Valid: metadata.Volume > 0}, + Imprint: pgtype.Text{String: metadata.Imprint, Valid: metadata.Imprint != ""}, + AgeRating: pgtype.Text{String: metadata.AgeRating, Valid: metadata.AgeRating != ""}, + WebUrl: pgtype.Text{String: metadata.WebURL, Valid: metadata.WebURL != ""}, + StoryArc: pgtype.Text{String: metadata.StoryArc, Valid: metadata.StoryArc != ""}, + IsBlackAndWhite: pgtype.Bool{Bool: metadata.IsBlackAndWhite, Valid: true}, + MetadataNotes: pgtype.Text{String: metadata.MetadataNotes, Valid: metadata.MetadataNotes != ""}, + AlternateInfo: func() []byte { + if metadata.AlternateInfo != "" { + return []byte(metadata.AlternateInfo) + } + return nil + }(), + ScanInformation: pgtype.Text{String: metadata.ScanInformation, Valid: metadata.ScanInformation != ""}, + Summary: pgtype.Text{String: metadata.Summary, Valid: metadata.Summary != ""}, + CommunityRating: pgtype.Float8{Float64: metadata.CommunityRating, Valid: metadata.CommunityRating > 0}, }) if err != nil { return false, fmt.Errorf("failed to create media item: %v", err) @@ -749,18 +753,311 @@ func (s *MediaScanner) extractCalibreSidecar(path string) *MediaMetadata { return metadata } +// mergeMetadata intelligently merges metadata from multiple sources +// Priority: metadata.opf (Calibre) → embedded metadata → folder structure → filename +// For comics: metadata.opf → ComicInfo.xml → folder structure → filename +func (s *MediaScanner) mergeMetadata(path string, calibreMetadata *MediaMetadata) (*MediaMetadata, error) { + metadata := calibreMetadata + if metadata == nil { + metadata = &MediaMetadata{} + } + + ext := strings.ToLower(filepath.Ext(path)) + + // For EPUB files + if ext == ".epub" { + book, err := epub.ReadBook(path) + if err == nil { + genreTags := extractGenreTagsFromEPUB(book) + processGenresAndTags(metadata, genreTags) + } + } + + // For comic archives, try to extract ComicInfo.xml + if ext == ".cbz" || ext == ".cbr" || ext == ".cb7" || ext == ".cbt" { + comicInfo, cover, err := extractComicMetadata(path) + if err != nil { + fmt.Printf("Warning: failed to extract comic metadata from %s: %v\n", path, err) + } else if comicInfo != nil { + // Merge ComicInfo.xml fields (only if not already set from Calibre) + if metadata.Title == "" && comicInfo.Title != "" { + metadata.Title = comicInfo.Title + } + if metadata.Series == "" && comicInfo.Series != "" { + metadata.Series = comicInfo.Series + } + if metadata.SeriesNumber == 0 && comicInfo.Number > 0 { + metadata.SeriesNumber = int32(comicInfo.Number) + } + if metadata.Publisher == "" && comicInfo.Publisher != "" { + metadata.Publisher = comicInfo.Publisher + } + if metadata.Author == "" && comicInfo.Writer != "" { + metadata.Author = comicInfo.Writer + } + if metadata.Description == "" && comicInfo.Summary != "" { + metadata.Description = comicInfo.Summary + } + + // NEW: Always extract reading direction from ComicInfo.xml + // (even if metadata.opf exists, since Calibre doesn't support this field) + metadata.MangaType = normalizeMangaType(comicInfo.Manga) + metadata.ReadingDirection = determineReadingDirection(comicInfo) + metadata.Language = comicInfo.LanguageISO + + // NEW: Extract additional comic-specific fields + // Series information + if metadata.SeriesCount == 0 && comicInfo.Count > 0 { + metadata.SeriesCount = int32(comicInfo.Count) + } + if metadata.Volume == 0 && comicInfo.Volume > 0 { + metadata.Volume = int32(comicInfo.Volume) + } + + // Publisher and classification + if metadata.Imprint == "" && comicInfo.Imprint != "" { + metadata.Imprint = comicInfo.Imprint + } + if metadata.StoryArc == "" && comicInfo.StoryArc != "" { + metadata.StoryArc = comicInfo.StoryArc + } + if metadata.AgeRating == "" && comicInfo.AgeRating != "" { + metadata.AgeRating = normalizeAgeRating(comicInfo.AgeRating) + } + + // NEW: Process genres and tags (universal logic for all formats) + // Extract genre tags from ComicInfo.xml (Genre + Tags + Characters + Teams + Locations) + genreTags := extractGenreTagsFromComicInfo(comicInfo) + processGenresAndTags(metadata, genreTags) + + // Additional metadata + if metadata.WebURL == "" && comicInfo.Web != "" { + metadata.WebURL = comicInfo.Web + } + if metadata.MetadataNotes == "" && comicInfo.Notes != "" { + metadata.MetadataNotes = comicInfo.Notes + } + if metadata.ScanInformation == "" && comicInfo.ScanInformation != "" { + metadata.ScanInformation = comicInfo.ScanInformation + } + if metadata.Summary == "" && comicInfo.Summary != "" { + metadata.Summary = comicInfo.Summary + } + + // Boolean fields + if !metadata.IsBlackAndWhite && strings.ToLower(comicInfo.BlackAndWhite) == "yes" { + metadata.IsBlackAndWhite = true + } + if metadata.CommunityRating == 0 && comicInfo.CommunityRating > 0 { + metadata.CommunityRating = comicInfo.CommunityRating + } + + // Alternate series information (store as JSONB string) + if metadata.AlternateInfo == "" && (comicInfo.AlternateSeries != "" || comicInfo.AlternateNumber > 0) { + alternateData := map[string]interface{}{} + if comicInfo.AlternateSeries != "" { + alternateData["alternate_series"] = comicInfo.AlternateSeries + } + if comicInfo.AlternateNumber > 0 { + alternateData["alternate_number"] = comicInfo.AlternateNumber + } + if comicInfo.AlternateCount > 0 { + alternateData["alternate_count"] = comicInfo.AlternateCount + } + if len(alternateData) > 0 { + jsonBytes, err := json.Marshal(alternateData) + if err == nil { + metadata.AlternateInfo = string(jsonBytes) + } + } + } + + // REMOVED: Tag enhancement now handled by processGenresAndTags() + // Characters, Teams, Locations are already processed via extractGenreTagsFromComicInfo() + + // Extract cover if not already present + if len(cover) > 0 && metadata.CoverPath == "" { + coverPath := path + ".cover.jpg" + if err := os.WriteFile(coverPath, cover, 0644); err == nil { + metadata.CoverPath = s.getRelativePath(coverPath) + } + } + + fmt.Printf("Merged comic metadata from %s: title=%s, series=%s, issue=%d, manga=%s, direction=%s\n", + path, comicInfo.Title, comicInfo.Series, comicInfo.Number, comicInfo.Manga, metadata.ReadingDirection) + } + } + + return metadata, nil +} + +// containsTag checks if a tag already exists in the tags array +func containsTag(tags []string, tag string) bool { + tag = strings.ToLower(tag) + for _, t := range tags { + if strings.ToLower(t) == tag { + return true + } + } + return false +} + +// normalizeMangaType normalizes ComicInfo.xml Manga field to database enum values +func normalizeMangaType(manga string) string { + switch strings.ToLower(strings.ReplaceAll(manga, " ", "")) { + case "unknown": + return "unknown" + case "no": + return "no" + case "yes": + return "yes" + case "yesandrighttoleft": + return "yes_and_right_to_left" + default: + return "unknown" + } +} + +// determineReadingDirection computes reading direction from ComicInfo metadata +// Uses Manga field + language heuristics + genre tags +func determineReadingDirection(comicInfo *ComicInfo) string { + // 1. Check explicit Manga field + manga := normalizeMangaType(comicInfo.Manga) + switch manga { + case "yes_and_right_to_left": + return "rtl" // Traditional Japanese manga + case "yes", "no": + return "ltr" // Manga style but LTR, or Western comic + } + + // 2. Language heuristic: Japanese → RTL + lang := strings.ToLower(comicInfo.LanguageISO) + if lang == "ja" || lang == "jpn" { + return "rtl" + } + + // 3. Genre heuristic: webtoons/manhwa → vertical + tags := strings.ToLower(comicInfo.Tags + " " + comicInfo.Genre) + if strings.Contains(tags, "webtoon") || strings.Contains(tags, "manhwa") { + return "vertical" // Korean/Chinese webcomics + } + if strings.Contains(tags, "manga") && (lang == "ja" || lang == "jpn") { + return "rtl" // Japanese manga + } + + // 4. Default: LTR (Western comics) + return "ltr" +} + +// normalizeAgeRating normalizes age rating from ComicInfo.xml to standard values +func normalizeAgeRating(rating string) string { + rating = strings.ToLower(strings.TrimSpace(rating)) + switch rating { + case "everyone", "e", "all ages": + return "Everyone" + case "teen", "t", "13+", "13+up": + return "Teen" + case "mature", "m", "17+", "17+up", "adults only": + return "Mature" + case "adult", "a", "18+": + return "Adult" + default: + return rating // Return original if unknown + } +} + +// processGenresAndTags ensures ALL genres appear in the tags array without duplication +// This applies to ALL formats: EPUB, ComicInfo.xml, PDF metadata +// Strategy: Use existing `genre` column for primary genre, `tags` array for all genres +func processGenresAndTags(metadata *MediaMetadata, genreTags []string) { + if metadata.Tags == nil { + metadata.Tags = []string{} + } + + // 1. Set primary genre (first genre tag wins if not already set) + if metadata.Genre == "" && len(genreTags) > 0 { + metadata.Genre = genreTags[0] + } + + // 2. Ensure ALL genre tags appear in tags array (without duplication) + for _, genreTag := range genreTags { + genreTag = strings.TrimSpace(genreTag) + if genreTag != "" && !containsTag(metadata.Tags, genreTag) { + metadata.Tags = append(metadata.Tags, genreTag) + } + } +} + +// extractGenreTagsFromEPUB extracts all values from EPUB +// Returns array of genre tags +func extractGenreTagsFromEPUB(book *epub.Book) []string { + var genreTags []string + + // EPUB stores genres in metadata + if subjects, err := book.MetadataByKey("subject"); err == nil && len(subjects) > 0 { + for _, subject := range subjects { + subject = strings.TrimSpace(subject) + if subject != "" { + genreTags = append(genreTags, subject) + } + } + } + + return genreTags +} + +// extractGenreTagsFromComicInfo extracts genres from ComicInfo.xml +// Genre field + Tags field + Characters + Teams + Locations +// Returns array of genre tags +func extractGenreTagsFromComicInfo(comicInfo *ComicInfo) []string { + var genreTags []string + + // 1. Add Genre field + if comicInfo.Genre != "" { + genreTags = append(genreTags, strings.Split(comicInfo.Genre, ",")...) + } + + // 2. Add Tags field (comma-separated) + if comicInfo.Tags != "" { + genreTags = append(genreTags, strings.Split(comicInfo.Tags, ",")...) + } + + // 3. Add Characters (comma-separated) + if comicInfo.Characters != "" { + genreTags = append(genreTags, strings.Split(comicInfo.Characters, ",")...) + } + + // 4. Add Teams (comma-separated) + if comicInfo.Teams != "" { + genreTags = append(genreTags, strings.Split(comicInfo.Teams, ",")...) + } + + // 5. Add Locations (comma-separated) + if comicInfo.Locations != "" { + genreTags = append(genreTags, strings.Split(comicInfo.Locations, ",")...) + } + + // Trim whitespace from all tags + for i := range genreTags { + genreTags[i] = strings.TrimSpace(genreTags[i]) + } + + return genreTags +} + func (s *MediaScanner) extractMetadata(path string) (*MediaMetadata, error) { - // NEW: Try Calibre sidecar first - if metadata := s.extractCalibreSidecar(path); metadata != nil { + // Try Calibre sidecar first + calibreMetadata := s.extractCalibreSidecar(path) + if calibreMetadata != nil { fmt.Printf("Using Calibre metadata.opf for %s\n", path) // Try to find cover image for sidecar metadata coverPath := findSidecarCover(path) if coverPath != "" { - metadata.CoverPath = s.getRelativePath(coverPath) + calibreMetadata.CoverPath = s.getRelativePath(coverPath) } - return metadata, nil + return s.mergeMetadata(path, calibreMetadata) } // EXISTING: Fallback to embedded metadata @@ -1400,6 +1697,29 @@ type ComicInfo struct { Tags string `xml:"Tags"` Web string `xml:"Web"` Notes string `xml:"Notes"` + + // NEW: Reading direction fields from ComicInfo.xml v2.0 + Manga string `xml:"Manga"` // Unknown, No, Yes, YesAndRightToLeft + LanguageISO string `xml:"LanguageISO"` // ISO 639-1 language code for heuristics + + // NEW: Additional comic-specific fields (19 total fields from ComicInfo.xml) + Count int `xml:"Count"` // Total issues in series + AlternateSeries string `xml:"AlternateSeries"` + AlternateNumber int `xml:"AlternateNumber"` + AlternateCount int `xml:"AlternateCount"` + Summary string `xml:"Summary"` + Imprint string `xml:"Imprint"` + StoryArc string `xml:"StoryArc"` + SeriesGroup string `xml:"SeriesGroup"` + AgeRating string `xml:"AgeRating"` + CommunityRating float64 `xml:"CommunityRating"` + MainCharacterOrTeam string `xml:"MainCharacterOrTeam"` + Review string `xml:"Review"` + BlackAndWhite string `xml:"BlackAndWhite"` // "Yes" or "No" + ScanInformation string `xml:"ScanInformation"` + Characters string `xml:"Characters"` + Teams string `xml:"Teams"` + Locations string `xml:"Locations"` } // extractComicMetadata extracts metadata from comic archive (supports .cbz, .cbr, .cb7, .cbt)