diff --git a/internal/services/genre_lexicon.go b/internal/services/genre_lexicon.go new file mode 100644 index 00000000..e905c5d3 --- /dev/null +++ b/internal/services/genre_lexicon.go @@ -0,0 +1,142 @@ +package services + +import "strings" + +// Smart genre selection for sidecar/embedded metadata (A2c plan §2.5). +// +// Sources arrive as comma-blobs ("Epic, Action & Adventure, Military, +// Sword & Sorcery" — one Audible genre string), media-format noise +// ("Audiobook"), or maximally generic tags ("Fiction"). This is a +// deterministic ranked rule set — no network, no guessing service: +// +// 1. split candidate strings on , ; / +// 2. alias-map to canonical genres +// 3. discard demoted (format/edition) tokens — they never win +// 4. most specific (longest) canonical hit wins; "Fiction" is last resort +// 5. every candidate (split parts included) still lands in tags, so +// filtering keeps full coverage even when the genre column holds one + +var genreAliases = map[string]string{ + "sci-fi": "science fiction", "scifi": "science fiction", "sf": "science fiction", + "sff": "science fiction", "sci fi": "science fiction", + "speculative fiction": "science fiction", + " detectives": "mystery", "detective": "mystery", + "whodunnit": "mystery", "whodunit": "mystery", + "adventure fiction": "action & adventure", + "action and adventure": "action & adventure", + "ya": "young adult", + "mg": "middle grade", + "graphic novel": "graphic novels", + "coming of age": "coming of age", + "coming-of-age": "coming of age", + "histfiction": "historical fiction", + "epic": "epic fantasy", + "high fantasy": "epic fantasy", + "military scifi": "military science fiction", + "military sf": "military science fiction", + "space opera": "space opera", + "post apocalyptic": "post-apocalyptic", + "self help": "self-help", + "sci_fi": "science fiction", +} + +// canonicalGenres is the known-genre set (lowercase). A candidate that +// normalizes to one of these wins the `genre` column; longest match beats +// shorter (Sword & Sorcery > Fiction). +var canonicalGenres = map[string]bool{ + "fiction": true, "nonfiction": true, "literary fiction": true, + "classics": true, "poetry": true, "short stories": true, + "anthology": true, "coming of age": true, "satire": true, "humor": true, + "fantasy": true, "epic fantasy": true, "sword & sorcery": true, + "dark fantasy": true, "urban fantasy": true, "paranormal": true, + "paranormal romance": true, "high fantasy": true, "fairy tales": true, + "mythology": true, "folklore": true, + "science fiction": true, "space opera": true, "cyberpunk": true, + "military science fiction": true, "dystopian": true, + "post-apocalyptic": true, "apocalyptic": true, "time travel": true, + "alternate history": true, "first contact": true, "steampunk": true, + "litrpg": true, "progression fantasy": true, + "historical fiction": true, "alternate world": true, + "romance": true, "contemporary romance": true, "historical romance": true, + "mystery": true, "cozy mystery": true, "crime": true, "true crime": true, + "police procedural": true, "noir": true, "hardboiled": true, + "thriller": true, "suspense": true, "espionage": true, "horror": true, + "gothic": true, "supernatural": true, + "action & adventure": true, "adventure": true, "western": true, + "military": true, "war": true, + "biography": true, "autobiography": true, "memoir": true, + "history": true, "science": true, "philosophy": true, + "psychology": true, "self-help": true, "business": true, + "economics": true, "politics": true, "religion": true, + "spirituality": true, "travel": true, "cooking": true, "food": true, + "health": true, "fitness": true, "sports": true, "music": true, + "art": true, "photography": true, "education": true, "reference": true, + "young adult": true, "middle grade": true, "children's": true, + "comics": true, "graphic novels": true, "manga": true, + "legal": true, "medical": true, +} + +// genreDemoted never wins the genre column — media formats and editions, +// not genres. They still pass through to tags. +var genreDemoted = map[string]bool{ + "audiobook": true, "unabridged": true, "abridged": true, + "dramatization": true, "dramatized": true, "audio drama": true, +} + +// splitGenreCandidates splits compound genre strings ("Epic, Action & +// Adventure" — one Audible blob) into individual candidates. +func splitGenreCandidates(genres []string) []string { + var out []string + for _, g := range genres { + for _, part := range strings.Split(g, ",") { + for _, p := range strings.Split(part, ";") { + for _, piece := range strings.Split(p, "/") { + t := strings.TrimSpace(piece) + if t != "" { + out = append(out, t) + } + } + } + } + } + return out +} + +// pickSmartGenre chooses the best genre from candidate strings. Returns the +// winner ("" when there are no candidates) and the full candidate list for +// the tags array. +func pickSmartGenre(genres []string) (genre string, all []string) { + all = splitGenreCandidates(genres) + best := "" + bestLen := -1 + for _, c := range all { + lower := strings.ToLower(strings.TrimSpace(c)) + if lower == "" || genreDemoted[lower] { + continue + } + if genreAliases[lower] != "" { + lower = genreAliases[lower] + } + if !canonicalGenres[lower] { + continue + } + // Specificity: the longest canonical form wins; earlier candidates + // win ties (source order is provider significance). + if len(lower) > bestLen { + best = lower + bestLen = len(lower) + } + } + if best != "" { + return best, all + } + // No lexicon hit: fall back to the first non-demoted candidate verbatim + // (display honesty over false taxonomy), "" when only noise existed. + for _, c := range all { + lower := strings.ToLower(strings.TrimSpace(c)) + if lower != "" && !genreDemoted[lower] { + return strings.TrimSpace(c), all + } + } + return "", all +} diff --git a/internal/services/media_scanner.go b/internal/services/media_scanner.go index b24932fc..53f0d69b 100644 --- a/internal/services/media_scanner.go +++ b/internal/services/media_scanner.go @@ -31,6 +31,8 @@ import ( "regexp" "strconv" "strings" + + "github.com/microcosm-cc/bluemonday" "sync" "sync/atomic" "time" @@ -94,6 +96,11 @@ type MediaMetadata struct { ScanInformation string // Scan information Summary string // Summary from ComicInfo.xml + // A2c: ordering-construct series entries parsed from the sidecar + // ("The Witcher (Reading Order) #5"). They never touch the series + // columns — the collections layer curates them into playlists. + PlaylistSeries []SeriesCandidate + // Audiobook fields (media_items.audio_metadata JSONB — only the // audio paths read it; see media_scanner_audio.go) AudioMeta *AudioMetadataPayload @@ -738,8 +745,15 @@ func (s *MediaScanner) extractFolderStructureMetadata(path, rootFolder string) * } } } else { - // Simple structure: Author/Title - metadata.Title = strings.ReplaceAll(components[1], "_", " ") + // Simple structure: Author/Title — with the A2c conservative + // "Title: , Book N" split when the post-colon tail is + // unambiguous (arbitrary colons never split). + if title, subtitle, split := splitTitleOnSubtitle(strings.ReplaceAll(components[1], "_", " ")); split { + metadata.Title = title + metadata.Subtitle = subtitle + } else { + metadata.Title = strings.ReplaceAll(components[1], "_", " ") + } } return metadata @@ -990,6 +1004,12 @@ func (s *MediaScanner) processMediaFile(ctx context.Context, path string) (bool, } } + // A2c §2.7: title/subtitle-derived series fallback. Lowest precedence — + // fills blanks only. "The Witcher, Book 3" as subtitle (or the post-colon + // segment of the title) yields series "The Witcher" + number 3, so + // Audible-style filenames carry working series data with no sidecar. + applyTitleDerivedSeries(metadata) + // Final fallback if still missing essential metadata if metadata.Title == "" { metadata.Title = strings.TrimSuffix(filepath.Base(path), filepath.Ext(path)) @@ -1027,6 +1047,7 @@ func (s *MediaScanner) processMediaFile(ctx context.Context, path string) (bool, SeriesNumber: pgtype.Float8{Float64: metadata.SeriesNumber, Valid: metadata.SeriesNumber > 0}, Publishers: nilToEmpty(utils.SingleOrSlice(metadata.Publisher, metadata.Publishers)), PublishersSearch: utils.NormalizeTagsSearch(utils.SingleOrSlice(metadata.Publisher, metadata.Publishers)), + Language: pgtype.Text{String: metadata.Language, Valid: metadata.Language != ""}, DatePublished: pgtype.Date{Time: metadata.PublishDate, Valid: !metadata.PublishDate.IsZero()}, Genre: pgtype.Text{String: metadata.Genre, Valid: metadata.Genre != ""}, Contributors: metadata.Contributors, @@ -1121,6 +1142,55 @@ func (s *MediaScanner) processMediaFile(ctx context.Context, path string) (bool, return true, nil } +// applyTitleDerivedSeries fills a missing series and/or ordinal from the +// item's own subtitle or title text when they follow the unambiguous +// Audible/Amazon ", Book N" convention. Lowest-precedence source: +// anything any other extractor set stays untouched. No display mutation — +// title/subtitle render exactly as written; only the series fields fill. +func applyTitleDerivedSeries(metadata *MediaMetadata) { + if metadata == nil { + return + } + needName := metadata.Series == "" + needNum := metadata.SeriesNumber == 0 + if !needName && !needNum { + return + } + + // Prefer the subtitle, then the post-colon segment of the title. + tryTexts := []string{metadata.Subtitle} + if _, tail, split := splitTitleOnSubtitle(metadata.Title); split { + tryTexts = append(tryTexts, tail) + } + // A number parsed alongside a DIFFERENT series name belongs to that + // name, not to the series we already have — accept it only when the + // derived name matches the existing series (case-insensitive). + for _, text := range tryTexts { + name, num, hasNum, ok := deriveSeriesFromText(text) + if !ok { + continue + } + if name == metadata.Title { + continue // degenerate: "the series" is the whole title + } + if needName { + metadata.Series = name + metadata.SeriesNumber = num + needName = false + needNum = false + if !needName && !needNum { + return + } + continue + } + if needNum && hasNum && strings.EqualFold(name, metadata.Series) { + metadata.SeriesNumber = num + needNum = false + return + } + } +} + // extractCalibreSidecar checks for and parses a Calibre metadata.opf sidecar file func (s *MediaScanner) extractCalibreSidecar(path string) *MediaMetadata { // Get directory of media file @@ -1190,17 +1260,18 @@ func extractAudiobookshelfSidecar(path string) *MediaMetadata { return nil } - // Series is tolerant: ABS/Calibre sidecars write an ARRAY of - // {series, sequence} objects; Libation writes a plain STRING (or - // null). An array-typed field made Libation's whole sidecar fail to - // parse and silently fall to folder-structure metadata (found by - // the audiobook ingest). + // Series is tolerant across ALL on-disk shapes: ABS abmetadata writes + // an array of STRINGS ("The Witcher #3"), some exports write an array + // of {series, sequence} OBJECTS, Libation writes a plain STRING. The + // string-array case used to fall through silently, dropping series for + // every ABS-matched book (the A2c finding). var sidecar struct { Title string `json:"title"` Subtitle string `json:"subtitle"` Authors []string `json:"authors"` Narrators []string `json:"narrators"` Abridged *bool `json:"abridged"` + Explicit *bool `json:"explicit"` Series json.RawMessage `json:"series"` Genres []string `json:"genres"` Tags []string `json:"tags"` @@ -1217,44 +1288,88 @@ func extractAudiobookshelfSidecar(path string) *MediaMetadata { return nil } - var seriesName, seriesSequence string - var seriesArr []struct { - Series string `json:"series"` - Sequence string `json:"sequence"` - } - if len(sidecar.Series) > 0 { - trimmed := strings.TrimSpace(string(sidecar.Series)) - if strings.HasPrefix(trimmed, "[") || strings.HasPrefix(trimmed, "{") { - if err := json.Unmarshal(sidecar.Series, &seriesArr); err == nil && len(seriesArr) > 0 { - seriesName = seriesArr[0].Series - seriesSequence = seriesArr[0].Sequence - } - } else if trimmed != "null" && trimmed != "" { - // Plain string form (Libation) - seriesName = strings.Trim(trimmed, "\"") - } - } + candidates := parseSeriesCandidates(sidecar.Series) metadata := &MediaMetadata{ Title: strings.TrimSpace(sidecar.Title), } - // Subtitle joins the title Calibre-style ("Main: Subtitle") - there is no - // separate subtitle column, and Calibre-sidecar books arrive pre-joined. - if subtitle := strings.TrimSpace(sidecar.Subtitle); subtitle != "" && metadata.Title != "" { - metadata.Title = metadata.Title + ": " + subtitle + // Subtitle is a first-class column now (A2b) — never joined into the + // title, never trimmed away. + if subtitle := strings.TrimSpace(sidecar.Subtitle); subtitle != "" { + metadata.Subtitle = subtitle } if len(sidecar.Authors) > 0 { + // Full array is the write truth; keep the legacy single fill for + // paths that still read metadata.Author. + metadata.Authors = sidecar.Authors metadata.Author = strings.TrimSpace(sidecar.Authors[0]) } - if seriesName != "" { - metadata.Series = strings.TrimSpace(seriesName) - if index, err := strconv.ParseFloat(strings.TrimSpace(seriesSequence), 64); err == nil { - metadata.SeriesNumber = index + if sidecar.Publisher != nil && strings.TrimSpace(*sidecar.Publisher) != "" { + metadata.Publisher = strings.TrimSpace(*sidecar.Publisher) + metadata.Publishers = []string{metadata.Publisher} + } + if sidecar.Explicit != nil { + metadata.Explicit = *sidecar.Explicit + } + + // Three-way series assignment (A2c §2.2): + // primary bibliographic series -> series / series_number columns + // additional bibliographic ones -> alternate_info JSONB (comic shape) + // ordering constructs -> PlaylistSeries (curated collections; + // never the series column) + var extras, playlists []SeriesCandidate + var primary *SeriesCandidate + for i, c := range candidates { + if isPlaylistSeriesName(c.Name) { + playlists = append(playlists, c) + continue } + if primary == nil { + primary = &candidates[i] + continue + } + extras = append(extras, c) + } + if primary != nil { + metadata.Series = primary.Name + metadata.SeriesNumber = primary.Number + if primary.HasNum && primary.Sequence != "" { + seq := strings.TrimSpace(primary.Sequence) + if _, err := strconv.ParseFloat(seq, 64); err != nil { + // Lossy extraction ("1a", "Book 2"): keep the raw text so + // nothing the source said is lost. + metadata.AlternateInfo = preserveSeriesSequenceRaw(metadata.AlternateInfo, seq) + } + } + } + if len(extras) > 0 && metadata.AlternateInfo == "" { + e := extras[0] + alt := map[string]interface{}{"alternate_series": e.Name} + if e.HasNum { + alt["alternate_number"] = e.Number + } + if b, err := json.Marshal(alt); err == nil { + metadata.AlternateInfo = string(b) + } + } + if len(playlists) > 0 { + metadata.PlaylistSeries = playlists + fmt.Printf("Series: %d playlist-style series entries on %q route to curated collections, not the series column\n", len(playlists), metadata.Title) + } + + // Smart genre (A2c §2.5): most-specific canonical genre wins the genre + // column; every candidate (split parts included) stays in tags. + if genre, _ := pickSmartGenre(sidecar.Genres); genre != "" { + metadata.Genre = genre } if tags := append(append([]string{}, sidecar.Genres...), sidecar.Tags...); len(tags) > 0 { metadata.Tags = utils.NormalizeTags(tags) } + if sidecar.Language != nil { + // Display names ("English") normalize to ISO 639-1 ("en") — the + // media_items.language convention. + metadata.Language = normalizeLanguageCode(*sidecar.Language) + } if sidecar.PublishedDate != nil { if date, err := time.Parse("2006-01-02", strings.TrimSpace(*sidecar.PublishedDate)); err == nil { metadata.PublishDate = date @@ -1271,7 +1386,10 @@ func extractAudiobookshelfSidecar(path string) *MediaMetadata { metadata.Publisher = strings.TrimSpace(*sidecar.Publisher) } if sidecar.Description != nil { - metadata.Description = strings.TrimSpace(*sidecar.Description) + // ABS descriptions are Audible HTML (already sanitized upstream, but + // untrusted). bluemonday UGC policy keeps the basic formatting tags + // the UI renders and strips everything else. + metadata.Description = strings.TrimSpace(sanitizeSidecarDescription(*sidecar.Description)) } if sidecar.ISBN != nil { metadata.ISBN = utils.NormalizeISBNSafe(strings.TrimSpace(*sidecar.ISBN)) @@ -1287,13 +1405,32 @@ func extractAudiobookshelfSidecar(path string) *MediaMetadata { if sidecar.Abridged != nil { metadata.Abridged = *sidecar.Abridged } - if sidecar.Language != nil { - metadata.Language = strings.TrimSpace(*sidecar.Language) - } return metadata } +// sanitizeSidecarDescription normalizes untrusted sidecar HTML to a safe +// subset. Plain text passes through unchanged. +func sanitizeSidecarDescription(raw string) string { + return bluemonday.UGCPolicy().Sanitize(raw) +} + +// preserveSeriesSequenceRaw records the un-parseable original sequence text +// ("1a", "Book 2") inside the alternate_info JSONB so the float column can +// stay an honest ordering key while no source information is lost. +func preserveSeriesSequenceRaw(existingJSON, raw string) string { + entry := map[string]interface{}{} + if existingJSON != "" { + _ = json.Unmarshal([]byte(existingJSON), &entry) + } + entry["series_sequence_raw"] = raw + b, err := json.Marshal(entry) + if err != nil { + return existingJSON + } + return string(b) +} + // mergeMetadata intelligently merges metadata from multiple sources // Priority: metadata.opf (Calibre) → embedded metadata → folder structure → filename // For comics: metadata.opf → ComicInfo.xml → folder structure → filename diff --git a/internal/services/media_scanner_audio.go b/internal/services/media_scanner_audio.go index 98cce1c2..77ff7a0e 100644 --- a/internal/services/media_scanner_audio.go +++ b/internal/services/media_scanner_audio.go @@ -408,12 +408,10 @@ func applyAudioTagFallbacks(metadata *MediaMetadata, report *ffprobeReport) { metadata.Title = album } } - // Subtitle joins the title Calibre-style, mirroring the ABS sidecar - // path — there is no separate subtitle column. Skip when the title - // already contains it (tools sometimes pre-join). - if subtitle := report.tag("----:com.apple.itunes:subtitle", "subtitle"); subtitle != "" && metadata.Title != "" && - !strings.Contains(strings.ToLower(metadata.Title), strings.ToLower(subtitle)) { - metadata.Title = metadata.Title + ": " + subtitle + // Subtitle is a first-class column (A2b) — no more Calibre-style title + // joining; the tool-provided subtitle lands in its own column. + if subtitle := report.tag("----:com.apple.itunes:subtitle", "subtitle"); subtitle != "" { + metadata.Subtitle = subtitle } if metadata.Author == "" { if artist := report.tag("album_artist"); artist != "" { diff --git a/internal/services/media_scanner_audio_metadata_test.go b/internal/services/media_scanner_audio_metadata_test.go index e2adf018..adc41486 100644 --- a/internal/services/media_scanner_audio_metadata_test.go +++ b/internal/services/media_scanner_audio_metadata_test.go @@ -148,9 +148,13 @@ func TestApplyAudioTagFallbacks(t *testing.T) { "----:com.apple.iTunes:ASIN": "B0ASIN1234", "----:com.apple.iTunes:ABRIDGED": "1", })) - if meta.Title != "The Tagged Album: Or, A Life" { + // A2b: subtitle is a first-class column — no title joining. + if meta.Title != "The Tagged Album" { t.Fatalf("title = %q", meta.Title) } + if meta.Subtitle != "Or, A Life" { + t.Fatalf("subtitle = %q", meta.Subtitle) + } if meta.Author != "Tag Author" { t.Fatalf("author = %q", meta.Author) } diff --git a/internal/services/media_scanner_audio_multifile.go b/internal/services/media_scanner_audio_multifile.go index 6dab408e..44aa9b17 100644 --- a/internal/services/media_scanner_audio_multifile.go +++ b/internal/services/media_scanner_audio_multifile.go @@ -533,6 +533,9 @@ func (s *MediaScanner) extractAudioBookMetadata(representative, rootFolder strin // zip books. The hash columns stay NULL — content dedup is path-based for // multi-file books (there is no single byte stream to hash). func (s *MediaScanner) createAudioBookItem(ctx context.Context, libraryID pgtype.UUID, metadata *MediaMetadata, relPath string, totalSize int64, mimeType string, modTime pgtype.Timestamptz) (database.MediaItems, error) { + // A2c §2.7: same title/subtitle-derived series fallback as the + // single-file path — folder-of-tracks books carry the same naming. + applyTitleDerivedSeries(metadata) return s.db.CreateMediaItem(ctx, database.CreateMediaItemParams{ LibraryID: libraryID, Title: metadata.Title, diff --git a/internal/services/series_parsing.go b/internal/services/series_parsing.go new file mode 100644 index 00000000..ebfe1af3 --- /dev/null +++ b/internal/services/series_parsing.go @@ -0,0 +1,235 @@ +package services + +import ( + "encoding/json" + "regexp" + "strconv" + "strings" +) + +// Series-metadata parsing helpers for the scanner (A2c overhaul). +// +// Sources disagree wildly on how a series and its ordinal are spelled: +// ABS sidecars write ["Name #3"] string arrays or [{series, sequence}] +// objects, Libation writes a bare string, Audible prefixes ordinals with +// "Book ", Calibre uses decimal indices. These helpers normalize all of +// them into candidates the scanner can rank and assign. + +// SeriesCandidate is one parsed series membership from any source. +type SeriesCandidate struct { + Name string + Sequence string // raw sequence text ("3", "1.5", "1a", "Book 2") + Number float64 // leading number of Sequence (0 = none parsed) + HasNum bool + FromSidecar bool +} + +// Playlist markers that identify ordering constructs rather than +// bibliographic series. Deliberately explicit: this classification routes +// entries away from the series column into curated collections, so a false +// positive creates user-visible objects. Prefer misses over false hits. +var playlistMarkers = []string{ + "reading order", + "chronological order", + "publication order", + "in order", + "timeline", +} + +var ( + seriesSeqSuffixRe = regexp.MustCompile(`\s+#([^#\s]+)$`) + // Leading ordinal inside a sequence string: "3", "1.5", "Book 2", + // "2, Dramatized Adaptation" → 2. Mirrors ABS cleanSeriesSequence. + seriesLeadingNumRe = regexp.MustCompile(`\d+(?:\.\d+)?`) + // ", Book N" (also ; – : separators), Audible/Amazon style. + deriveSeriesRe = regexp.MustCompile(`(?i)^\s*(.+?)\s*[,;:–-]\s*book\s+(\d+(?:\.\d+)?)\s*$`) + // Conservative "Title: , Book N" split — only when the post-colon + // text is itself a series-and-number tail. + titleSubtitleSplitRe = regexp.MustCompile(`(?i)^(.{1,200}?)\s*:\s*(.+,\s*book\s+\d+(?:\.\d+)?)$`) + whitespaceRe = regexp.MustCompile(`\s+`) +) + +// splitSeriesString splits "Name #1a" into name + sequence, ABS +// parseSeriesString-compatible: the sequence is anything after the last +// " #" up to end-of-string, containing no whitespace or further '#'. +func splitSeriesString(s string) (name, sequence string) { + s = strings.TrimSpace(s) + loc := seriesSeqSuffixRe.FindStringSubmatchIndex(s) + if loc == nil { + return s, "" + } + return strings.TrimSpace(s[:loc[0]]), s[loc[2]:loc[3]] +} + +// seriesSequenceNumber extracts the ordinal from a raw sequence string. +// Returns the number, whether one was found, and whether information was +// lost in the extraction (caller should preserve the raw text, e.g. "1a"). +func seriesSequenceNumber(sequence string) (num float64, found, lossy bool) { + trimmed := strings.TrimSpace(sequence) + if trimmed == "" { + return 0, false, false + } + token := seriesLeadingNumRe.FindString(trimmed) + if token == "" { + return 0, false, false + } + num, err := strconv.ParseFloat(token, 64) + if err != nil { + return 0, false, false + } + // "1.5" tokenizes exactly (not lossy); "1a"/"Book 2" keep a raw tail. + return num, true, trimmed != token +} + +// isPlaylistSeriesName reports whether the series name is an ordering +// construct ("The Witcher (Reading Order)") rather than a bibliography. +// Match discipline: the marker must be a parenthetical qualifier or a +// trailing phrase — a series merely CONTAINING the word "timeline" mid-name +// ("The Timeline of...") is a real series, not a playlist. +func isPlaylistSeriesName(name string) bool { + lower := strings.ToLower(strings.TrimSpace(name)) + for _, m := range playlistMarkers { + if strings.HasSuffix(lower, m) { + return true + } + if open := strings.Index(lower, "("); open >= 0 { + if strings.Contains(lower[open:], m) { + return true + } + } + } + return false +} + +// parseSeriesCandidates normalizes the sidecar `series` field, which is +// written as an array of STRINGS ("Name #3" — ABS abmetadata), an array of +// OBJECTS ({series, sequence} — some exports), or a plain JSON string +// (Libation), into candidates. +func parseSeriesCandidates(raw json.RawMessage) []SeriesCandidate { + trimmed := strings.TrimSpace(string(raw)) + if trimmed == "" || trimmed == "null" { + return nil + } + var out []SeriesCandidate + + add := func(name, sequence string, fromSidecar bool) { + name = strings.TrimSpace(name) + if name == "" { + return + } + num, found, _ := seriesSequenceNumber(sequence) + out = append(out, SeriesCandidate{ + Name: name, Sequence: sequence, Number: num, HasNum: found, + FromSidecar: fromSidecar, + }) + } + + if strings.HasPrefix(trimmed, "[") { + // Array of objects first ({"series": ..., "sequence": ...}). + var objs []struct { + Series string `json:"series"` + Name string `json:"name"` + Sequence string `json:"sequence"` + } + if err := json.Unmarshal(raw, &objs); err == nil && len(objs) > 0 { + for _, o := range objs { + name := o.Series + if name == "" { + name = o.Name + } + add(name, o.Sequence, true) + } + return out + } + // Array of strings: ABS abmetadata form "Name #3". + var strs []string + if err := json.Unmarshal(raw, &strs); err == nil { + for _, s := range strs { + name, seq := splitSeriesString(s) + add(name, seq, true) + } + return out + } + return nil + } + + // Plain JSON string (Libation): "Name" or "Name #3". + var single string + if err := json.Unmarshal(raw, &single); err == nil { + name, seq := splitSeriesString(single) + add(name, seq, true) + } + return out +} + +// deriveSeriesFromText extracts ", Book N" from a title/subtitle +// string (Audible/Amazon convention). Returns ok=false when the text is not +// obviously a series tail — this fallback must not guess. +func deriveSeriesFromText(text string) (name string, num float64, hasNum bool, ok bool) { + if strings.TrimSpace(text) == "" { + return "", 0, false, false + } + m := deriveSeriesRe.FindStringSubmatch(text) + if m == nil { + return "", 0, false, false + } + name = whitespaceRe.ReplaceAllString(strings.TrimSpace(m[1]), " ") + if name == "" || len(name) > 200 || strings.HasPrefix(strings.ToLower(name), "book ") { + return "", 0, false, false + } + num, err := strconv.ParseFloat(m[2], 64) + if err != nil { + return "", 0, false, false + } + return name, num, true, true +} + +// splitTitleOnSubtitle splits "Title: , Book N" into title + +// subtitle, but ONLY when the post-colon text is itself a series-and-number +// tail — arbitrary colons in titles are never touched. +func splitTitleOnSubtitle(folderTitle string) (title, subtitle string, split bool) { + m := titleSubtitleSplitRe.FindStringSubmatch(strings.TrimSpace(folderTitle)) + if m == nil { + return folderTitle, "", false + } + return strings.TrimSpace(m[1]), strings.TrimSpace(m[2]), true +} + +// normalizeLanguageCode maps sidecar/display language names to ISO 639-1 +// codes (the media_items.language convention). Values already in code form +// pass through; unknown display names are kept verbatim when they fit the +// VARCHAR(10) column, else dropped (empty) — never truncate mid-word. +func normalizeLanguageCode(v string) string { + v = strings.TrimSpace(v) + if v == "" { + return "" + } + if code, ok := languageNameToCode[strings.ToLower(v)]; ok { + return code + } + lower := strings.ToLower(v) + if len(lower) == 2 { + return lower // already a code (or close enough to be harmless) + } + if len(v) <= 10 { + return v + } + return "" +} + +var languageNameToCode = map[string]string{ + "english": "en", "spanish": "es", "español": "es", "french": "fr", + "français": "fr", "german": "de", "deutsch": "de", "italian": "it", + "italiano": "it", "portuguese": "pt", "dutch": "nl", "russian": "ru", + "ukrainian": "uk", "polish": "pl", "czech": "cs", "slovak": "sk", + "swedish": "sv", "norwegian": "no", "danish": "da", "finnish": "fi", + "icelandic": "is", "hungarian": "hu", "romanian": "ro", "bulgarian": "bg", + "greek": "el", "turkish": "tr", "arabic": "ar", "hebrew": "he", + "persian": "fa", "hindi": "hi", "bengali": "bn", "urdu": "ur", + "thai": "th", "vietnamese": "vi", "indonesian": "id", "malay": "ms", + "japanese": "ja", "korean": "ko", "chinese": "zh", "mandarin": "zh", + "cantonese": "zh", "swahili": "sw", "afrikaans": "af", "catalan": "ca", + "croatian": "hr", "serbian": "sr", "slovenian": "sl", "lithuanian": "lt", + "latvian": "lv", "estonian": "et", "georgian": "ka", "armenian": "hy", + "welsh": "cy", "irish": "ga", "basque": "eu", "galician": "gl", +} diff --git a/internal/services/series_parsing_test.go b/internal/services/series_parsing_test.go new file mode 100644 index 00000000..6c20237b --- /dev/null +++ b/internal/services/series_parsing_test.go @@ -0,0 +1,300 @@ +package services + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "github.com/jackc/pgx/v5/pgtype" +) + +func TestSplitSeriesString(t *testing.T) { + cases := []struct{ in, name, seq string }{ + {"The Witcher #3", "The Witcher", "3"}, + {"Name #1a", "Name", "1a"}, + {"No Sequence", "No Sequence", ""}, + {"Name #1#a", "Name #1#a", ""}, // embedded # is not a sequence + } + for _, c := range cases { + name, seq := splitSeriesString(c.in) + if name != c.name || seq != c.seq { + t.Errorf("splitSeriesString(%q) = (%q, %q), want (%q, %q)", c.in, name, seq, c.name, c.seq) + } + } +} + +func TestSeriesSequenceNumber(t *testing.T) { + cases := []struct { + in string + num float64 + found bool + lossy bool + }{ + {"3", 3, true, false}, + {"1.5", 1.5, true, false}, + {"1a", 1, true, true}, + {"Book 2", 2, true, true}, + {"2, Dramatized Adaptation", 2, true, true}, + {"", 0, false, false}, + {"novella", 0, false, false}, + } + for _, c := range cases { + num, found, lossy := seriesSequenceNumber(c.in) + if num != c.num || found != c.found || lossy != c.lossy { + t.Errorf("seriesSequenceNumber(%q) = (%v,%v,%v), want (%v,%v,%v)", + c.in, num, found, lossy, c.num, c.found, c.lossy) + } + } +} + +// The A2c headline bug: ABS abmetadata writes series as an array of +// "Name #seq" STRINGS. The old parser only accepted object arrays and +// silently dropped every ABS-matched series. +func TestParseSeriesCandidatesStringArray(t *testing.T) { + raw := []byte(`["The Witcher #3", "The Witcher (Reading Order) #5"]`) + candidates := parseSeriesCandidates(raw) + if len(candidates) != 2 { + t.Fatalf("got %d candidates, want 2", len(candidates)) + } + if candidates[0].Name != "The Witcher" || candidates[0].Number != 3 || !candidates[0].HasNum { + t.Errorf("candidate[0] = %+v", candidates[0]) + } + if candidates[1].Name != "The Witcher (Reading Order)" || candidates[1].Number != 5 { + t.Errorf("candidate[1] = %+v", candidates[1]) + } + if !isPlaylistSeriesName(candidates[1].Name) { + t.Errorf("reading-order entry must classify as playlist-ish") + } + if isPlaylistSeriesName(candidates[0].Name) { + t.Errorf("publication series must not classify as playlist-ish") + } +} + +func TestParseSeriesCandidatesOtherShapes(t *testing.T) { + objs := parseSeriesCandidates([]byte(`[{"series":"Great Series","sequence":"2.5"}]`)) + if len(objs) != 1 || objs[0].Name != "Great Series" || objs[0].Number != 2.5 { + t.Errorf("object form: %+v", objs) + } + libation := parseSeriesCandidates([]byte(`"Libation Series #4"`)) + if len(libation) != 1 || libation[0].Name != "Libation Series" || libation[0].Number != 4 { + t.Errorf("plain string form: %+v", libation) + } + if parseSeriesCandidates([]byte(`null`)) != nil { + t.Errorf("null must yield no candidates") + } +} + +func TestIsPlaylistSeriesName(t *testing.T) { + positives := []string{ + "The Witcher (Reading Order)", + "Discworld Reading Order", + "Narnia Chronological Order", + "Robin Hobb in order", + "Series Publication Order", + "Middle Earth Timeline", + } + for _, p := range positives { + if !isPlaylistSeriesName(p) { + t.Errorf("%q should be playlist-ish", p) + } + } + negatives := []string{"The Witcher", "Witcher Saga", "The Timeline of..."} + for _, n := range negatives { + if isPlaylistSeriesName(n) { + t.Errorf("%q must not be playlist-ish", n) + } + } +} + +func TestDeriveSeriesFromText(t *testing.T) { + name, num, hasNum, ok := deriveSeriesFromText("The Witcher, Book 3") + if !ok || name != "The Witcher" || num != 3 || !hasNum { + t.Errorf("got (%q,%v,%v,%v)", name, num, hasNum, ok) + } + name, num, hasNum, ok = deriveSeriesFromText("The Witcher, Book 1.5") + if !ok || name != "The Witcher" || num != 1.5 || !hasNum { + t.Errorf("decimal: got (%q,%v,%v,%v)", name, num, hasNum, ok) + } + // Not obviously a series tail → must refuse rather than guess. + for _, negative := range []string{"1984", "The Fellowship", "Book Ends"} { + if _, _, _, ok := deriveSeriesFromText(negative); ok { + t.Errorf("%q must not derive a series", negative) + } + } +} + +func TestSplitTitleOnSubtitle(t *testing.T) { + title, subtitle, split := splitTitleOnSubtitle("Baptism of Fire: The Witcher, Book 3") + if !split || title != "Baptism of Fire" || subtitle != "The Witcher, Book 3" { + t.Errorf("got (%q,%q,%v)", title, subtitle, split) + } + // Arbitrary colons never split. + _, _, split = splitTitleOnSubtitle("Star Wars: A New Hope") + if split { + t.Errorf("non-series colon must not split") + } +} + +func TestApplyTitleDerivedSeries(t *testing.T) { + // The user's actual naming, two-level folder, no sidecar anywhere. + meta := &MediaMetadata{Title: "Baptism of Fire: The Witcher, Book 3"} + applyTitleDerivedSeries(meta) + if meta.Series != "The Witcher" || meta.SeriesNumber != 3 { + t.Errorf("series = %q #%v, want The Witcher #3", meta.Series, meta.SeriesNumber) + } + if meta.Title != "Baptism of Fire: The Witcher, Book 3" { + t.Errorf("fallback must never mutate the display title, got %q", meta.Title) + } + + // Subtitle already split by the folder parser: number + name both land. + meta = &MediaMetadata{Title: "Blood of Elves", Subtitle: "The Witcher, Book 1"} + applyTitleDerivedSeries(meta) + if meta.Series != "The Witcher" || meta.SeriesNumber != 1 { + t.Errorf("series = %q #%v, want The Witcher #1", meta.Series, meta.SeriesNumber) + } + + // Existing series data is never overwritten (blanks-only fallback)... + meta = &MediaMetadata{Title: "X", Subtitle: "Derived, Book 9", Series: "Real Series"} + applyTitleDerivedSeries(meta) + if meta.Series != "Real Series" || meta.SeriesNumber != 0 { + t.Errorf("a number from a different series name must not apply, got %q #%v", meta.Series, meta.SeriesNumber) + } + // ...but a number whose derived name MATCHES the existing series fills it. + meta = &MediaMetadata{Title: "X", Subtitle: "Real Series, Book 9", Series: "Real Series"} + applyTitleDerivedSeries(meta) + if meta.Series != "Real Series" || meta.SeriesNumber != 9 { + t.Errorf("matching-name number must fill, got %q #%v", meta.Series, meta.SeriesNumber) + } +} + +func TestPickSmartGenre(t *testing.T) { + // The real Malazan Audible blob: one string, four genres. + genre, all := pickSmartGenre([]string{"Epic, Action & Adventure, Military, Sword & Sorcery"}) + if genre != "action & adventure" { + t.Errorf("genre = %q, want action & adventure", genre) + } + if len(all) != 4 { + t.Errorf("candidates = %v, want the 4 split parts", all) + } + // Format noise never wins. + if genre, _ := pickSmartGenre([]string{"Audiobook"}); genre != "" { + t.Errorf("audiobook is not a genre, got %q", genre) + } + // Fiction is the last resort, not the winner. + if genre, _ := pickSmartGenre([]string{"Fiction"}); genre != "fiction" { + t.Errorf("plain Fiction should still resolve, got %q", genre) + } + if genre, _ := pickSmartGenre([]string{"Fiction", "Epic Fantasy"}); genre != "epic fantasy" { + t.Errorf("specificity must beat generic, got %q", genre) + } +} + +func TestNormalizeLanguageCode(t *testing.T) { + cases := []struct{ in, want string }{ + {"English", "en"}, + {"en", "en"}, + {"German", "de"}, + {"", ""}, + {"Norwegian (Bokmål)", ""}, // too long for VARCHAR(10); dropped, never truncated + {"Pig Latin", "Pig Latin"}, // unknown display name that fits: kept + } + for _, c := range cases { + if got := normalizeLanguageCode(c.in); got != c.want { + t.Errorf("normalizeLanguageCode(%q) = %q, want %q", c.in, got, c.want) + } + } +} + +// End-to-end: a REAL ABS-shaped sidecar (string-array series, the shape +// every ABS-matched book writes) must land series, language, explicit and +// the full author array. +func TestExtractAudiobookshelfSidecarStringArraySeries(t *testing.T) { + dir := t.TempDir() + sidecar := `{ + "title": "Blood of Elves", + "authors": ["Andrzej Sapkowski"], + "narrators": ["Peter Kenny"], + "series": ["The Witcher #1", "The Witcher (Reading Order) #3"], + "genres": ["Fiction"], + "language": "English", + "explicit": true, + "publisher": "Orbit" + }` + if err := os.WriteFile(filepath.Join(dir, "metadata.json"), []byte(sidecar), 0644); err != nil { + t.Fatal(err) + } + meta := extractAudiobookshelfSidecar(filepath.Join(dir, "book.m4b")) + if meta == nil { + t.Fatal("sidecar not extracted") + } + if meta.Series != "The Witcher" || meta.SeriesNumber != 1 { + t.Errorf("series = %q #%v", meta.Series, meta.SeriesNumber) + } + if len(meta.Authors) != 1 || meta.Authors[0] != "Andrzej Sapkowski" { + t.Errorf("authors array = %v", meta.Authors) + } + if meta.Language != "en" { + t.Errorf("language = %q, want en", meta.Language) + } + if !meta.Explicit { + t.Errorf("explicit flag lost") + } + if len(meta.PlaylistSeries) != 1 || meta.PlaylistSeries[0].Name != "The Witcher (Reading Order)" { + t.Errorf("playlist series = %+v", meta.PlaylistSeries) + } + if meta.AlternateInfo != "" { + t.Errorf("playlist entry must not leak into alternate_info, got %q", meta.AlternateInfo) + } +} + +// Additional real series (the Drizzt case: Icewind Dale #1 = Legend of +// Drizzt #4) land in alternate_info with the comic-established shape. +func TestExtractAudiobookshelfSidecarExtraSeriesToAlternateInfo(t *testing.T) { + dir := t.TempDir() + sidecar := `{ + "title": "The Crystal Shard", + "authors": ["R.A. Salvatore"], + "series": [ + {"series": "The Icewind Dale Trilogy", "sequence": "1"}, + {"series": "The Legend of Drizzt", "sequence": "4"} + ] + }` + if err := os.WriteFile(filepath.Join(dir, "metadata.json"), []byte(sidecar), 0644); err != nil { + t.Fatal(err) + } + meta := extractAudiobookshelfSidecar(filepath.Join(dir, "book.m4b")) + if meta.Series != "The Icewind Dale Trilogy" || meta.SeriesNumber != 1 { + t.Errorf("series = %q #%v", meta.Series, meta.SeriesNumber) + } + if meta.AlternateInfo == "" { + t.Fatal("second series must land in alternate_info") + } + // Lossy sequence text is preserved, never silently dropped. + meta2 := extractAudiobookshelfSidecarPath(t, dir, `{"title":"X","series":["Saga #1a"]}`) + if meta2 == nil || meta2.Series != "Saga" || meta2.SeriesNumber != 1 { + t.Fatalf("lossy series = %+v", meta2) + } + if meta2.AlternateInfo == "" || !strings.Contains(meta2.AlternateInfo, "series_sequence_raw") || !strings.Contains(meta2.AlternateInfo, "1a") { + t.Errorf("raw sequence not preserved: %q", meta2.AlternateInfo) + } +} + +func extractAudiobookshelfSidecarPath(t *testing.T, dir, json string) *MediaMetadata { + t.Helper() + // extractAudiobookshelfSidecar looks for exactly "metadata.json"; use a + // fresh dir per call instead. + fresh := t.TempDir() + if err := os.WriteFile(filepath.Join(fresh, "metadata.json"), []byte(json), 0644); err != nil { + t.Fatal(err) + } + return extractAudiobookshelfSidecar(filepath.Join(fresh, "book.m4b")) +} + +// The series page tiebreaker rides on the same ordering contract the +// scanner feeds: numbers sort numerically, NULLs last (§3.6). +func TestSeriesNumberFloatOrderingValues(t *testing.T) { + if !(pgtype.Float8{Float64: 1}.Float64 < pgtype.Float8{Float64: 1.5}.Float64) { + t.Fatal("1 must sort before 1.5") + } +}