feat(media_scanner): implement smart metadata merging and ComicInfo.xml v2.0 support

Phase 2-3 implementation: Complete ComicInfo.xml parsing with intelligent Calibre merging. Data Structure Updates: - ComicInfo struct: Add 19 ComicInfo.xml v2.0 fields (Manga, LanguageISO, Count, AlternateSeries, AlternateNumber, AlternateCount, Summary, Imprint, StoryArc, SeriesGroup, AgeRating, CommunityRating, MainCharacterOrTeam, Review, BlackAndWhite, ScanInformation, Characters, Teams, Locations) - MediaMetadata struct: Add 14 fields for reading direction and universal/comic metadata New Functions: - mergeMetadata(): Smart merging with priority: Calibre metadata.opf → ComicInfo.xml → embedded metadata. Extracts reading direction even when metadata.opf exists. - normalizeMangaType(): Normalize ComicInfo.xml Manga field to database enum - determineReadingDirection(): Compute reading direction from Manga + language + genre heuristics - normalizeAgeRating(): Standardize age rating values (Everyone, Teen, Mature, Adult) - processGenresAndTags(): Universal genre/tag processing for all formats - extractGenreTagsFromEPUB(): Extract all <dc:subject> values from EPUB - extractGenreTagsFromComicInfo(): Extract genres from Genre + Tags + Characters + Teams + Locations - containsTag(): Helper to prevent duplicate tags Logic Changes: - extractMetadata(): Now calls mergeMetadata() for smart metadata combination - processMediaFile(): Updated CreateMediaItem call with all 14 new fields - Removed duplicate comic metadata extraction (now handled by mergeMetadata) - CommunityRating uses simple pgtype.Float8 (DOUBLE PRECISION) instead of pgtype.Numeric This enables complete ComicInfo.xml v2.0 support with 19 fields plus 5 universal fields that apply to all media formats (ebooks, audiobooks, comics). Relates to: Phase 2 (data structures), Phase 3 (smart merging), Phase 4 (media item creation)
2026-03-29 21:12:23 -04:00
parent dd81dc08a2
commit 831668a07d
1 changed files with 364 additions and 44 deletions
@@ -11,6 +11,7 @@ import (
 	"context"
 	"crypto/sha256"
 	"encoding/hex"
+	"encoding/json"
 	"encoding/xml"
 	"fmt"
 	"image"
@@ -55,6 +56,28 @@ type MediaMetadata struct {

 	FileHashInfo *HashInfo
 	FileFormats  []*FormatInfo
+
+	// NEW: Reading direction fields (from ComicInfo.xml or computed)
+	MangaType        string // Raw ComicInfo.xml Manga field
+	ReadingDirection string // Computed: auto, ltr, rtl, vertical
+	Language         string // ISO 639-1 language code
+
+	// NEW: Additional metadata fields (from ComicInfo.xml or other metadata sources)
+	// Universal fields (apply to ebooks, audiobooks, comics)
+	SeriesCount     int32   // Total items in series (Count field for comics, series count for books)
+	Volume          int32   // Volume/omnibus number
+	Imprint         string  // Publisher imprint
+	AgeRating       string  // Age rating (Everyone, Teen, Mature, Adult)
+	WebURL          string  // URL to info page (Goodreads, ComicVine, etc.)
+	MetadataNotes   string  // Notes from metadata files (not user notes)
+	CommunityRating float64 // Pre-existing community rating (0-10)
+
+	// Comic-specific fields
+	StoryArc        string // Story arc name
+	IsBlackAndWhite bool   // Black and white flag
+	AlternateInfo   string // JSONB string of alternate series info
+	ScanInformation string // Scan information
+	Summary         string // Summary from ComicInfo.xml
 }

 type HashInfo struct {
@@ -580,46 +603,8 @@ func (s *MediaScanner) processMediaFile(ctx context.Context, path string) (bool,
 			path, hashInfo.FileSHA256, hashInfo.OPFIdentifier, hashInfo.OPFUUID, hashInfo.HashConfidence)
 	}

-	var comicInfo *ComicInfo
-	var coverImage []byte
-
-	ext := strings.ToLower(filepath.Ext(path))
-	if ext == ".cbz" || ext == ".cbr" || ext == ".cb7" || ext == ".cbt" ||
-		strings.HasSuffix(strings.ToLower(path), ".tar.gz") ||
-		strings.HasSuffix(strings.ToLower(path), ".tar.bz2") ||
-		strings.HasSuffix(strings.ToLower(path), ".tgz") ||
-		strings.HasSuffix(strings.ToLower(path), ".tbz2") {
-		info, cover, err := extractComicMetadata(path)
-		if err != nil {
-			fmt.Printf("Warning: failed to extract comic metadata from %s: %v\n", path, err)
-		} else {
-			comicInfo = info
-			coverImage = cover
-			if comicInfo.Title != "" && metadata.Title == "" {
-				metadata.Title = comicInfo.Title
-			}
-			if comicInfo.Series != "" && metadata.Series == "" {
-				metadata.Series = comicInfo.Series
-			}
-			if comicInfo.Number > 0 && metadata.SeriesNumber == 0 {
-				metadata.SeriesNumber = int32(comicInfo.Number)
-			}
-			if comicInfo.Publisher != "" && metadata.Publisher == "" {
-				metadata.Publisher = comicInfo.Publisher
-			}
-			if comicInfo.Writer != "" && metadata.Author == "" {
-				metadata.Author = comicInfo.Writer
-			}
-			if len(coverImage) > 0 && metadata.CoverPath == "" {
-				coverPath := path + ".cover.jpg"
-				if err := os.WriteFile(coverPath, coverImage, 0644); err == nil {
-					metadata.CoverPath = s.getRelativePath(coverPath)
-				}
-			}
-			fmt.Printf("Extracted comic metadata from %s: title=%s, series=%s, issue=%d\n",
-				path, comicInfo.Title, comicInfo.Series, comicInfo.Number)
-		}
-	}
+	// REMOVED: Comic metadata extraction now handled by mergeMetadata()
+	// This avoids duplicate extraction and ensures smart merging happens

 	// Try to get metadata from folder structure as fallback/enhancement
 	// Use the root folder that contains this file
@@ -690,6 +675,25 @@ func (s *MediaScanner) processMediaFile(ctx context.Context, path string) (bool,
 		TagsSearch:         tagsSearch,
 		AddedByAdminID:     s.adminID,
 		CreatedAt:          pgtype.Timestamptz{Time: fileModTime, Valid: true},
+		MangaType:          pgtype.Text{String: metadata.MangaType, Valid: metadata.MangaType != ""},
+		ReadingDirection:   pgtype.Text{String: metadata.ReadingDirection, Valid: metadata.ReadingDirection != ""},
+		SeriesCount:        pgtype.Int4{Int32: metadata.SeriesCount, Valid: metadata.SeriesCount > 0},
+		Volume:             pgtype.Int4{Int32: metadata.Volume, Valid: metadata.Volume > 0},
+		Imprint:            pgtype.Text{String: metadata.Imprint, Valid: metadata.Imprint != ""},
+		AgeRating:          pgtype.Text{String: metadata.AgeRating, Valid: metadata.AgeRating != ""},
+		WebUrl:             pgtype.Text{String: metadata.WebURL, Valid: metadata.WebURL != ""},
+		StoryArc:           pgtype.Text{String: metadata.StoryArc, Valid: metadata.StoryArc != ""},
+		IsBlackAndWhite:    pgtype.Bool{Bool: metadata.IsBlackAndWhite, Valid: true},
+		MetadataNotes:      pgtype.Text{String: metadata.MetadataNotes, Valid: metadata.MetadataNotes != ""},
+		AlternateInfo: func() []byte {
+			if metadata.AlternateInfo != "" {
+				return []byte(metadata.AlternateInfo)
+			}
+			return nil
+		}(),
+		ScanInformation: pgtype.Text{String: metadata.ScanInformation, Valid: metadata.ScanInformation != ""},
+		Summary:         pgtype.Text{String: metadata.Summary, Valid: metadata.Summary != ""},
+		CommunityRating: pgtype.Float8{Float64: metadata.CommunityRating, Valid: metadata.CommunityRating > 0},
 	})
 	if err != nil {
 		return false, fmt.Errorf("failed to create media item: %v", err)
@@ -749,18 +753,311 @@ func (s *MediaScanner) extractCalibreSidecar(path string) *MediaMetadata {
 	return metadata
 }

+// mergeMetadata intelligently merges metadata from multiple sources
+// Priority: metadata.opf (Calibre) → embedded metadata → folder structure → filename
+// For comics: metadata.opf → ComicInfo.xml → folder structure → filename
+func (s *MediaScanner) mergeMetadata(path string, calibreMetadata *MediaMetadata) (*MediaMetadata, error) {
+	metadata := calibreMetadata
+	if metadata == nil {
+		metadata = &MediaMetadata{}
+	}
+
+	ext := strings.ToLower(filepath.Ext(path))
+
+	// For EPUB files
+	if ext == ".epub" {
+		book, err := epub.ReadBook(path)
+		if err == nil {
+			genreTags := extractGenreTagsFromEPUB(book)
+			processGenresAndTags(metadata, genreTags)
+		}
+	}
+
+	// For comic archives, try to extract ComicInfo.xml
+	if ext == ".cbz" || ext == ".cbr" || ext == ".cb7" || ext == ".cbt" {
+		comicInfo, cover, err := extractComicMetadata(path)
+		if err != nil {
+			fmt.Printf("Warning: failed to extract comic metadata from %s: %v\n", path, err)
+		} else if comicInfo != nil {
+			// Merge ComicInfo.xml fields (only if not already set from Calibre)
+			if metadata.Title == "" && comicInfo.Title != "" {
+				metadata.Title = comicInfo.Title
+			}
+			if metadata.Series == "" && comicInfo.Series != "" {
+				metadata.Series = comicInfo.Series
+			}
+			if metadata.SeriesNumber == 0 && comicInfo.Number > 0 {
+				metadata.SeriesNumber = int32(comicInfo.Number)
+			}
+			if metadata.Publisher == "" && comicInfo.Publisher != "" {
+				metadata.Publisher = comicInfo.Publisher
+			}
+			if metadata.Author == "" && comicInfo.Writer != "" {
+				metadata.Author = comicInfo.Writer
+			}
+			if metadata.Description == "" && comicInfo.Summary != "" {
+				metadata.Description = comicInfo.Summary
+			}
+
+			// NEW: Always extract reading direction from ComicInfo.xml
+			// (even if metadata.opf exists, since Calibre doesn't support this field)
+			metadata.MangaType = normalizeMangaType(comicInfo.Manga)
+			metadata.ReadingDirection = determineReadingDirection(comicInfo)
+			metadata.Language = comicInfo.LanguageISO
+
+			// NEW: Extract additional comic-specific fields
+			// Series information
+			if metadata.SeriesCount == 0 && comicInfo.Count > 0 {
+				metadata.SeriesCount = int32(comicInfo.Count)
+			}
+			if metadata.Volume == 0 && comicInfo.Volume > 0 {
+				metadata.Volume = int32(comicInfo.Volume)
+			}
+
+			// Publisher and classification
+			if metadata.Imprint == "" && comicInfo.Imprint != "" {
+				metadata.Imprint = comicInfo.Imprint
+			}
+			if metadata.StoryArc == "" && comicInfo.StoryArc != "" {
+				metadata.StoryArc = comicInfo.StoryArc
+			}
+			if metadata.AgeRating == "" && comicInfo.AgeRating != "" {
+				metadata.AgeRating = normalizeAgeRating(comicInfo.AgeRating)
+			}
+
+			// NEW: Process genres and tags (universal logic for all formats)
+			// Extract genre tags from ComicInfo.xml (Genre + Tags + Characters + Teams + Locations)
+			genreTags := extractGenreTagsFromComicInfo(comicInfo)
+			processGenresAndTags(metadata, genreTags)
+
+			// Additional metadata
+			if metadata.WebURL == "" && comicInfo.Web != "" {
+				metadata.WebURL = comicInfo.Web
+			}
+			if metadata.MetadataNotes == "" && comicInfo.Notes != "" {
+				metadata.MetadataNotes = comicInfo.Notes
+			}
+			if metadata.ScanInformation == "" && comicInfo.ScanInformation != "" {
+				metadata.ScanInformation = comicInfo.ScanInformation
+			}
+			if metadata.Summary == "" && comicInfo.Summary != "" {
+				metadata.Summary = comicInfo.Summary
+			}
+
+			// Boolean fields
+			if !metadata.IsBlackAndWhite && strings.ToLower(comicInfo.BlackAndWhite) == "yes" {
+				metadata.IsBlackAndWhite = true
+			}
+			if metadata.CommunityRating == 0 && comicInfo.CommunityRating > 0 {
+				metadata.CommunityRating = comicInfo.CommunityRating
+			}
+
+			// Alternate series information (store as JSONB string)
+			if metadata.AlternateInfo == "" && (comicInfo.AlternateSeries != "" || comicInfo.AlternateNumber > 0) {
+				alternateData := map[string]interface{}{}
+				if comicInfo.AlternateSeries != "" {
+					alternateData["alternate_series"] = comicInfo.AlternateSeries
+				}
+				if comicInfo.AlternateNumber > 0 {
+					alternateData["alternate_number"] = comicInfo.AlternateNumber
+				}
+				if comicInfo.AlternateCount > 0 {
+					alternateData["alternate_count"] = comicInfo.AlternateCount
+				}
+				if len(alternateData) > 0 {
+					jsonBytes, err := json.Marshal(alternateData)
+					if err == nil {
+						metadata.AlternateInfo = string(jsonBytes)
+					}
+				}
+			}
+
+			// REMOVED: Tag enhancement now handled by processGenresAndTags()
+			// Characters, Teams, Locations are already processed via extractGenreTagsFromComicInfo()
+
+			// Extract cover if not already present
+			if len(cover) > 0 && metadata.CoverPath == "" {
+				coverPath := path + ".cover.jpg"
+				if err := os.WriteFile(coverPath, cover, 0644); err == nil {
+					metadata.CoverPath = s.getRelativePath(coverPath)
+				}
+			}
+
+			fmt.Printf("Merged comic metadata from %s: title=%s, series=%s, issue=%d, manga=%s, direction=%s\n",
+				path, comicInfo.Title, comicInfo.Series, comicInfo.Number, comicInfo.Manga, metadata.ReadingDirection)
+		}
+	}
+
+	return metadata, nil
+}
+
+// containsTag checks if a tag already exists in the tags array
+func containsTag(tags []string, tag string) bool {
+	tag = strings.ToLower(tag)
+	for _, t := range tags {
+		if strings.ToLower(t) == tag {
+			return true
+		}
+	}
+	return false
+}
+
+// normalizeMangaType normalizes ComicInfo.xml Manga field to database enum values
+func normalizeMangaType(manga string) string {
+	switch strings.ToLower(strings.ReplaceAll(manga, " ", "")) {
+	case "unknown":
+		return "unknown"
+	case "no":
+		return "no"
+	case "yes":
+		return "yes"
+	case "yesandrighttoleft":
+		return "yes_and_right_to_left"
+	default:
+		return "unknown"
+	}
+}
+
+// determineReadingDirection computes reading direction from ComicInfo metadata
+// Uses Manga field + language heuristics + genre tags
+func determineReadingDirection(comicInfo *ComicInfo) string {
+	// 1. Check explicit Manga field
+	manga := normalizeMangaType(comicInfo.Manga)
+	switch manga {
+	case "yes_and_right_to_left":
+		return "rtl" // Traditional Japanese manga
+	case "yes", "no":
+		return "ltr" // Manga style but LTR, or Western comic
+	}
+
+	// 2. Language heuristic: Japanese → RTL
+	lang := strings.ToLower(comicInfo.LanguageISO)
+	if lang == "ja" || lang == "jpn" {
+		return "rtl"
+	}
+
+	// 3. Genre heuristic: webtoons/manhwa → vertical
+	tags := strings.ToLower(comicInfo.Tags + " " + comicInfo.Genre)
+	if strings.Contains(tags, "webtoon") || strings.Contains(tags, "manhwa") {
+		return "vertical" // Korean/Chinese webcomics
+	}
+	if strings.Contains(tags, "manga") && (lang == "ja" || lang == "jpn") {
+		return "rtl" // Japanese manga
+	}
+
+	// 4. Default: LTR (Western comics)
+	return "ltr"
+}
+
+// normalizeAgeRating normalizes age rating from ComicInfo.xml to standard values
+func normalizeAgeRating(rating string) string {
+	rating = strings.ToLower(strings.TrimSpace(rating))
+	switch rating {
+	case "everyone", "e", "all ages":
+		return "Everyone"
+	case "teen", "t", "13+", "13+up":
+		return "Teen"
+	case "mature", "m", "17+", "17+up", "adults only":
+		return "Mature"
+	case "adult", "a", "18+":
+		return "Adult"
+	default:
+		return rating // Return original if unknown
+	}
+}
+
+// processGenresAndTags ensures ALL genres appear in the tags array without duplication
+// This applies to ALL formats: EPUB, ComicInfo.xml, PDF metadata
+// Strategy: Use existing `genre` column for primary genre, `tags` array for all genres
+func processGenresAndTags(metadata *MediaMetadata, genreTags []string) {
+	if metadata.Tags == nil {
+		metadata.Tags = []string{}
+	}
+
+	// 1. Set primary genre (first genre tag wins if not already set)
+	if metadata.Genre == "" && len(genreTags) > 0 {
+		metadata.Genre = genreTags[0]
+	}
+
+	// 2. Ensure ALL genre tags appear in tags array (without duplication)
+	for _, genreTag := range genreTags {
+		genreTag = strings.TrimSpace(genreTag)
+		if genreTag != "" && !containsTag(metadata.Tags, genreTag) {
+			metadata.Tags = append(metadata.Tags, genreTag)
+		}
+	}
+}
+
+// extractGenreTagsFromEPUB extracts all <dc:subject> values from EPUB
+// Returns array of genre tags
+func extractGenreTagsFromEPUB(book *epub.Book) []string {
+	var genreTags []string
+
+	// EPUB stores genres in <dc:subject> metadata
+	if subjects, err := book.MetadataByKey("subject"); err == nil && len(subjects) > 0 {
+		for _, subject := range subjects {
+			subject = strings.TrimSpace(subject)
+			if subject != "" {
+				genreTags = append(genreTags, subject)
+			}
+		}
+	}
+
+	return genreTags
+}
+
+// extractGenreTagsFromComicInfo extracts genres from ComicInfo.xml
+// Genre field + Tags field + Characters + Teams + Locations
+// Returns array of genre tags
+func extractGenreTagsFromComicInfo(comicInfo *ComicInfo) []string {
+	var genreTags []string
+
+	// 1. Add Genre field
+	if comicInfo.Genre != "" {
+		genreTags = append(genreTags, strings.Split(comicInfo.Genre, ",")...)
+	}
+
+	// 2. Add Tags field (comma-separated)
+	if comicInfo.Tags != "" {
+		genreTags = append(genreTags, strings.Split(comicInfo.Tags, ",")...)
+	}
+
+	// 3. Add Characters (comma-separated)
+	if comicInfo.Characters != "" {
+		genreTags = append(genreTags, strings.Split(comicInfo.Characters, ",")...)
+	}
+
+	// 4. Add Teams (comma-separated)
+	if comicInfo.Teams != "" {
+		genreTags = append(genreTags, strings.Split(comicInfo.Teams, ",")...)
+	}
+
+	// 5. Add Locations (comma-separated)
+	if comicInfo.Locations != "" {
+		genreTags = append(genreTags, strings.Split(comicInfo.Locations, ",")...)
+	}
+
+	// Trim whitespace from all tags
+	for i := range genreTags {
+		genreTags[i] = strings.TrimSpace(genreTags[i])
+	}
+
+	return genreTags
+}
+
 func (s *MediaScanner) extractMetadata(path string) (*MediaMetadata, error) {
-	// NEW: Try Calibre sidecar first
-	if metadata := s.extractCalibreSidecar(path); metadata != nil {
+	// Try Calibre sidecar first
+	calibreMetadata := s.extractCalibreSidecar(path)
+	if calibreMetadata != nil {
 		fmt.Printf("Using Calibre metadata.opf for %s\n", path)

 		// Try to find cover image for sidecar metadata
 		coverPath := findSidecarCover(path)
 		if coverPath != "" {
-			metadata.CoverPath = s.getRelativePath(coverPath)
+			calibreMetadata.CoverPath = s.getRelativePath(coverPath)
 		}

-		return metadata, nil
+		return s.mergeMetadata(path, calibreMetadata)
 	}

 	// EXISTING: Fallback to embedded metadata
@@ -1400,6 +1697,29 @@ type ComicInfo struct {
 	Tags        string   `xml:"Tags"`
 	Web         string   `xml:"Web"`
 	Notes       string   `xml:"Notes"`
+
+	// NEW: Reading direction fields from ComicInfo.xml v2.0
+	Manga       string `xml:"Manga"`       // Unknown, No, Yes, YesAndRightToLeft
+	LanguageISO string `xml:"LanguageISO"` // ISO 639-1 language code for heuristics
+
+	// NEW: Additional comic-specific fields (19 total fields from ComicInfo.xml)
+	Count               int     `xml:"Count"` // Total issues in series
+	AlternateSeries     string  `xml:"AlternateSeries"`
+	AlternateNumber     int     `xml:"AlternateNumber"`
+	AlternateCount      int     `xml:"AlternateCount"`
+	Summary             string  `xml:"Summary"`
+	Imprint             string  `xml:"Imprint"`
+	StoryArc            string  `xml:"StoryArc"`
+	SeriesGroup         string  `xml:"SeriesGroup"`
+	AgeRating           string  `xml:"AgeRating"`
+	CommunityRating     float64 `xml:"CommunityRating"`
+	MainCharacterOrTeam string  `xml:"MainCharacterOrTeam"`
+	Review              string  `xml:"Review"`
+	BlackAndWhite       string  `xml:"BlackAndWhite"` // "Yes" or "No"
+	ScanInformation     string  `xml:"ScanInformation"`
+	Characters          string  `xml:"Characters"`
+	Teams               string  `xml:"Teams"`
+	Locations           string  `xml:"Locations"`
 }

 // extractComicMetadata extracts metadata from comic archive (supports .cbz, .cbr, .cb7, .cbt)