feat(scanner): fix ABS sidecar series parsing; three-way series mapping; smart genre; language codes; title-derived series
Phase 2a of the metadata overhaul (plan §2.1-2.3, §2.5-2.9). The headline bug: ABS writes sidecar series as an array of STRINGS (["The Witcher #3", ...]); the parser only accepted {series, sequence} OBJECT arrays, so every ABS-matched book silently lost its series (found live: 9 sidecars, zero series ingested; only folder-structure Malazan survived). parseSeriesCandidates now normalizes string arrays, object arrays and Libation plain strings. Three-way series assignment (plan §2.2): - primary bibliographic series -> series/series_number columns - additional real series (Drizzt: Icewind Dale #1 = Legend of Drizzt #4) -> alternate_info JSONB, comic-established shape - ordering constructs ('The Witcher (Reading Order)') -> PlaylistSeries candidates; NEVER the series column. Detection is explicit-marker-only ((reading order|chronological|publication order|in order|timeline) as parenthetical/suffix) — precision over recall; a series merely containing 'timeline' mid-name stays a series. Sequence policy: leading-number extraction ('Book 2' -> 2, '2, Dramatized Adaptation' -> 2, '1a' -> 1); lossy extractions preserve the raw text in alternate_info.series_sequence_raw. Smart genre (deterministic lexicon): compound Audible blobs split ('Epic, Action & Adventure, Military, Sword & Sorcery' -> 4 candidates), alias-normalized, format noise demoted ('Audiobook' never wins), most-specific canonical genre takes the genre column, all candidates stay in tags. Language: display names normalize to ISO 639-1 on ingest; ROOT-CAUSED the update-path drop — CreateMediaItem call site never set Language (explicit NULL over the column default), the update path did; create now carries it. Title-derived series fallback (§2.7): 'Baptism of Fire: The Witcher, Book 3' 2-level folders split conservatively (only when the post-colon tail matches ', Book N'), subtitle parsed for series+number, blanks-only precedence, matching-name number fill, zero display mutation. Works with no sidecar and no network. Ingest fixes: subtitle/explicit land in their columns (audio tag subtitle stops joining into title); full authors[] array (was [0]- only); sidecar descriptions sanitized via bluemonday UGC policy (vendored). metadata_overrides: verified scan-update path already applies ApplyMetadataOverrides; subtitle/explicit/authors covered by the Phase 1 override adaptation. Tests: new series_parsing_test.go (string-array sidecar e2e, Drizzt alternate_info, lossy raw preservation, playlist classification, title-derived fallback incl. no-mutation + blanks-only, genre lexicon, language map); audio-tag subtitle pin updated to column behavior; calibre/ABS 2.5-series_index pins widened per float change (changelog'd in phase 1). Deferred to 2b: auto-created playlist collections (§2.4) — settings toggle, collection upsert, ownership.
This commit is contained in:
John O'Keefe committed
1 parent
646477afad
commit
7e4c97f639
7 files changed
+862
-43
No files matched your search
@@ -31,6 +31,8 @@ import (
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/microcosm-cc/bluemonday"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
@@ -94,6 +96,11 @@ type MediaMetadata struct {
|
||||
ScanInformation string // Scan information
|
||||
Summary string // Summary from ComicInfo.xml
|
||||
|
||||
// A2c: ordering-construct series entries parsed from the sidecar
|
||||
// ("The Witcher (Reading Order) #5"). They never touch the series
|
||||
// columns — the collections layer curates them into playlists.
|
||||
PlaylistSeries []SeriesCandidate
|
||||
|
||||
// Audiobook fields (media_items.audio_metadata JSONB — only the
|
||||
// audio paths read it; see media_scanner_audio.go)
|
||||
AudioMeta *AudioMetadataPayload
|
||||
@@ -738,8 +745,15 @@ func (s *MediaScanner) extractFolderStructureMetadata(path, rootFolder string) *
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// Simple structure: Author/Title
|
||||
metadata.Title = strings.ReplaceAll(components[1], "_", " ")
|
||||
// Simple structure: Author/Title — with the A2c conservative
|
||||
// "Title: <series>, Book N" split when the post-colon tail is
|
||||
// unambiguous (arbitrary colons never split).
|
||||
if title, subtitle, split := splitTitleOnSubtitle(strings.ReplaceAll(components[1], "_", " ")); split {
|
||||
metadata.Title = title
|
||||
metadata.Subtitle = subtitle
|
||||
} else {
|
||||
metadata.Title = strings.ReplaceAll(components[1], "_", " ")
|
||||
}
|
||||
}
|
||||
|
||||
return metadata
|
||||
@@ -990,6 +1004,12 @@ func (s *MediaScanner) processMediaFile(ctx context.Context, path string) (bool,
|
||||
}
|
||||
}
|
||||
|
||||
// A2c §2.7: title/subtitle-derived series fallback. Lowest precedence —
|
||||
// fills blanks only. "The Witcher, Book 3" as subtitle (or the post-colon
|
||||
// segment of the title) yields series "The Witcher" + number 3, so
|
||||
// Audible-style filenames carry working series data with no sidecar.
|
||||
applyTitleDerivedSeries(metadata)
|
||||
|
||||
// Final fallback if still missing essential metadata
|
||||
if metadata.Title == "" {
|
||||
metadata.Title = strings.TrimSuffix(filepath.Base(path), filepath.Ext(path))
|
||||
@@ -1027,6 +1047,7 @@ func (s *MediaScanner) processMediaFile(ctx context.Context, path string) (bool,
|
||||
SeriesNumber: pgtype.Float8{Float64: metadata.SeriesNumber, Valid: metadata.SeriesNumber > 0},
|
||||
Publishers: nilToEmpty(utils.SingleOrSlice(metadata.Publisher, metadata.Publishers)),
|
||||
PublishersSearch: utils.NormalizeTagsSearch(utils.SingleOrSlice(metadata.Publisher, metadata.Publishers)),
|
||||
Language: pgtype.Text{String: metadata.Language, Valid: metadata.Language != ""},
|
||||
DatePublished: pgtype.Date{Time: metadata.PublishDate, Valid: !metadata.PublishDate.IsZero()},
|
||||
Genre: pgtype.Text{String: metadata.Genre, Valid: metadata.Genre != ""},
|
||||
Contributors: metadata.Contributors,
|
||||
@@ -1121,6 +1142,55 @@ func (s *MediaScanner) processMediaFile(ctx context.Context, path string) (bool,
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// applyTitleDerivedSeries fills a missing series and/or ordinal from the
|
||||
// item's own subtitle or title text when they follow the unambiguous
|
||||
// Audible/Amazon "<series>, Book N" convention. Lowest-precedence source:
|
||||
// anything any other extractor set stays untouched. No display mutation —
|
||||
// title/subtitle render exactly as written; only the series fields fill.
|
||||
func applyTitleDerivedSeries(metadata *MediaMetadata) {
|
||||
if metadata == nil {
|
||||
return
|
||||
}
|
||||
needName := metadata.Series == ""
|
||||
needNum := metadata.SeriesNumber == 0
|
||||
if !needName && !needNum {
|
||||
return
|
||||
}
|
||||
|
||||
// Prefer the subtitle, then the post-colon segment of the title.
|
||||
tryTexts := []string{metadata.Subtitle}
|
||||
if _, tail, split := splitTitleOnSubtitle(metadata.Title); split {
|
||||
tryTexts = append(tryTexts, tail)
|
||||
}
|
||||
// A number parsed alongside a DIFFERENT series name belongs to that
|
||||
// name, not to the series we already have — accept it only when the
|
||||
// derived name matches the existing series (case-insensitive).
|
||||
for _, text := range tryTexts {
|
||||
name, num, hasNum, ok := deriveSeriesFromText(text)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
if name == metadata.Title {
|
||||
continue // degenerate: "the series" is the whole title
|
||||
}
|
||||
if needName {
|
||||
metadata.Series = name
|
||||
metadata.SeriesNumber = num
|
||||
needName = false
|
||||
needNum = false
|
||||
if !needName && !needNum {
|
||||
return
|
||||
}
|
||||
continue
|
||||
}
|
||||
if needNum && hasNum && strings.EqualFold(name, metadata.Series) {
|
||||
metadata.SeriesNumber = num
|
||||
needNum = false
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// extractCalibreSidecar checks for and parses a Calibre metadata.opf sidecar file
|
||||
func (s *MediaScanner) extractCalibreSidecar(path string) *MediaMetadata {
|
||||
// Get directory of media file
|
||||
@@ -1190,17 +1260,18 @@ func extractAudiobookshelfSidecar(path string) *MediaMetadata {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Series is tolerant: ABS/Calibre sidecars write an ARRAY of
|
||||
// {series, sequence} objects; Libation writes a plain STRING (or
|
||||
// null). An array-typed field made Libation's whole sidecar fail to
|
||||
// parse and silently fall to folder-structure metadata (found by
|
||||
// the audiobook ingest).
|
||||
// Series is tolerant across ALL on-disk shapes: ABS abmetadata writes
|
||||
// an array of STRINGS ("The Witcher #3"), some exports write an array
|
||||
// of {series, sequence} OBJECTS, Libation writes a plain STRING. The
|
||||
// string-array case used to fall through silently, dropping series for
|
||||
// every ABS-matched book (the A2c finding).
|
||||
var sidecar struct {
|
||||
Title string `json:"title"`
|
||||
Subtitle string `json:"subtitle"`
|
||||
Authors []string `json:"authors"`
|
||||
Narrators []string `json:"narrators"`
|
||||
Abridged *bool `json:"abridged"`
|
||||
Explicit *bool `json:"explicit"`
|
||||
Series json.RawMessage `json:"series"`
|
||||
Genres []string `json:"genres"`
|
||||
Tags []string `json:"tags"`
|
||||
@@ -1217,44 +1288,88 @@ func extractAudiobookshelfSidecar(path string) *MediaMetadata {
|
||||
return nil
|
||||
}
|
||||
|
||||
var seriesName, seriesSequence string
|
||||
var seriesArr []struct {
|
||||
Series string `json:"series"`
|
||||
Sequence string `json:"sequence"`
|
||||
}
|
||||
if len(sidecar.Series) > 0 {
|
||||
trimmed := strings.TrimSpace(string(sidecar.Series))
|
||||
if strings.HasPrefix(trimmed, "[") || strings.HasPrefix(trimmed, "{") {
|
||||
if err := json.Unmarshal(sidecar.Series, &seriesArr); err == nil && len(seriesArr) > 0 {
|
||||
seriesName = seriesArr[0].Series
|
||||
seriesSequence = seriesArr[0].Sequence
|
||||
}
|
||||
} else if trimmed != "null" && trimmed != "" {
|
||||
// Plain string form (Libation)
|
||||
seriesName = strings.Trim(trimmed, "\"")
|
||||
}
|
||||
}
|
||||
candidates := parseSeriesCandidates(sidecar.Series)
|
||||
|
||||
metadata := &MediaMetadata{
|
||||
Title: strings.TrimSpace(sidecar.Title),
|
||||
}
|
||||
// Subtitle joins the title Calibre-style ("Main: Subtitle") - there is no
|
||||
// separate subtitle column, and Calibre-sidecar books arrive pre-joined.
|
||||
if subtitle := strings.TrimSpace(sidecar.Subtitle); subtitle != "" && metadata.Title != "" {
|
||||
metadata.Title = metadata.Title + ": " + subtitle
|
||||
// Subtitle is a first-class column now (A2b) — never joined into the
|
||||
// title, never trimmed away.
|
||||
if subtitle := strings.TrimSpace(sidecar.Subtitle); subtitle != "" {
|
||||
metadata.Subtitle = subtitle
|
||||
}
|
||||
if len(sidecar.Authors) > 0 {
|
||||
// Full array is the write truth; keep the legacy single fill for
|
||||
// paths that still read metadata.Author.
|
||||
metadata.Authors = sidecar.Authors
|
||||
metadata.Author = strings.TrimSpace(sidecar.Authors[0])
|
||||
}
|
||||
if seriesName != "" {
|
||||
metadata.Series = strings.TrimSpace(seriesName)
|
||||
if index, err := strconv.ParseFloat(strings.TrimSpace(seriesSequence), 64); err == nil {
|
||||
metadata.SeriesNumber = index
|
||||
if sidecar.Publisher != nil && strings.TrimSpace(*sidecar.Publisher) != "" {
|
||||
metadata.Publisher = strings.TrimSpace(*sidecar.Publisher)
|
||||
metadata.Publishers = []string{metadata.Publisher}
|
||||
}
|
||||
if sidecar.Explicit != nil {
|
||||
metadata.Explicit = *sidecar.Explicit
|
||||
}
|
||||
|
||||
// Three-way series assignment (A2c §2.2):
|
||||
// primary bibliographic series -> series / series_number columns
|
||||
// additional bibliographic ones -> alternate_info JSONB (comic shape)
|
||||
// ordering constructs -> PlaylistSeries (curated collections;
|
||||
// never the series column)
|
||||
var extras, playlists []SeriesCandidate
|
||||
var primary *SeriesCandidate
|
||||
for i, c := range candidates {
|
||||
if isPlaylistSeriesName(c.Name) {
|
||||
playlists = append(playlists, c)
|
||||
continue
|
||||
}
|
||||
if primary == nil {
|
||||
primary = &candidates[i]
|
||||
continue
|
||||
}
|
||||
extras = append(extras, c)
|
||||
}
|
||||
if primary != nil {
|
||||
metadata.Series = primary.Name
|
||||
metadata.SeriesNumber = primary.Number
|
||||
if primary.HasNum && primary.Sequence != "" {
|
||||
seq := strings.TrimSpace(primary.Sequence)
|
||||
if _, err := strconv.ParseFloat(seq, 64); err != nil {
|
||||
// Lossy extraction ("1a", "Book 2"): keep the raw text so
|
||||
// nothing the source said is lost.
|
||||
metadata.AlternateInfo = preserveSeriesSequenceRaw(metadata.AlternateInfo, seq)
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(extras) > 0 && metadata.AlternateInfo == "" {
|
||||
e := extras[0]
|
||||
alt := map[string]interface{}{"alternate_series": e.Name}
|
||||
if e.HasNum {
|
||||
alt["alternate_number"] = e.Number
|
||||
}
|
||||
if b, err := json.Marshal(alt); err == nil {
|
||||
metadata.AlternateInfo = string(b)
|
||||
}
|
||||
}
|
||||
if len(playlists) > 0 {
|
||||
metadata.PlaylistSeries = playlists
|
||||
fmt.Printf("Series: %d playlist-style series entries on %q route to curated collections, not the series column\n", len(playlists), metadata.Title)
|
||||
}
|
||||
|
||||
// Smart genre (A2c §2.5): most-specific canonical genre wins the genre
|
||||
// column; every candidate (split parts included) stays in tags.
|
||||
if genre, _ := pickSmartGenre(sidecar.Genres); genre != "" {
|
||||
metadata.Genre = genre
|
||||
}
|
||||
if tags := append(append([]string{}, sidecar.Genres...), sidecar.Tags...); len(tags) > 0 {
|
||||
metadata.Tags = utils.NormalizeTags(tags)
|
||||
}
|
||||
if sidecar.Language != nil {
|
||||
// Display names ("English") normalize to ISO 639-1 ("en") — the
|
||||
// media_items.language convention.
|
||||
metadata.Language = normalizeLanguageCode(*sidecar.Language)
|
||||
}
|
||||
if sidecar.PublishedDate != nil {
|
||||
if date, err := time.Parse("2006-01-02", strings.TrimSpace(*sidecar.PublishedDate)); err == nil {
|
||||
metadata.PublishDate = date
|
||||
@@ -1271,7 +1386,10 @@ func extractAudiobookshelfSidecar(path string) *MediaMetadata {
|
||||
metadata.Publisher = strings.TrimSpace(*sidecar.Publisher)
|
||||
}
|
||||
if sidecar.Description != nil {
|
||||
metadata.Description = strings.TrimSpace(*sidecar.Description)
|
||||
// ABS descriptions are Audible HTML (already sanitized upstream, but
|
||||
// untrusted). bluemonday UGC policy keeps the basic formatting tags
|
||||
// the UI renders and strips everything else.
|
||||
metadata.Description = strings.TrimSpace(sanitizeSidecarDescription(*sidecar.Description))
|
||||
}
|
||||
if sidecar.ISBN != nil {
|
||||
metadata.ISBN = utils.NormalizeISBNSafe(strings.TrimSpace(*sidecar.ISBN))
|
||||
@@ -1287,13 +1405,32 @@ func extractAudiobookshelfSidecar(path string) *MediaMetadata {
|
||||
if sidecar.Abridged != nil {
|
||||
metadata.Abridged = *sidecar.Abridged
|
||||
}
|
||||
if sidecar.Language != nil {
|
||||
metadata.Language = strings.TrimSpace(*sidecar.Language)
|
||||
}
|
||||
|
||||
return metadata
|
||||
}
|
||||
|
||||
// sanitizeSidecarDescription normalizes untrusted sidecar HTML to a safe
|
||||
// subset. Plain text passes through unchanged.
|
||||
func sanitizeSidecarDescription(raw string) string {
|
||||
return bluemonday.UGCPolicy().Sanitize(raw)
|
||||
}
|
||||
|
||||
// preserveSeriesSequenceRaw records the un-parseable original sequence text
|
||||
// ("1a", "Book 2") inside the alternate_info JSONB so the float column can
|
||||
// stay an honest ordering key while no source information is lost.
|
||||
func preserveSeriesSequenceRaw(existingJSON, raw string) string {
|
||||
entry := map[string]interface{}{}
|
||||
if existingJSON != "" {
|
||||
_ = json.Unmarshal([]byte(existingJSON), &entry)
|
||||
}
|
||||
entry["series_sequence_raw"] = raw
|
||||
b, err := json.Marshal(entry)
|
||||
if err != nil {
|
||||
return existingJSON
|
||||
}
|
||||
return string(b)
|
||||
}
|
||||
|
||||
// mergeMetadata intelligently merges metadata from multiple sources
|
||||
// Priority: metadata.opf (Calibre) → embedded metadata → folder structure → filename
|
||||
// For comics: metadata.opf → ComicInfo.xml → folder structure → filename
|
||||
|
||||
Reference in new issue
Block a user