feat(scanner): fix ABS sidecar series parsing; three-way series mapping; smart genre; language codes; title-derived series
Phase 2a of the metadata overhaul (plan §2.1-2.3, §2.5-2.9). The headline bug: ABS writes sidecar series as an array of STRINGS (["The Witcher #3", ...]); the parser only accepted {series, sequence} OBJECT arrays, so every ABS-matched book silently lost its series (found live: 9 sidecars, zero series ingested; only folder-structure Malazan survived). parseSeriesCandidates now normalizes string arrays, object arrays and Libation plain strings. Three-way series assignment (plan §2.2): - primary bibliographic series -> series/series_number columns - additional real series (Drizzt: Icewind Dale #1 = Legend of Drizzt #4) -> alternate_info JSONB, comic-established shape - ordering constructs ('The Witcher (Reading Order)') -> PlaylistSeries candidates; NEVER the series column. Detection is explicit-marker-only ((reading order|chronological|publication order|in order|timeline) as parenthetical/suffix) — precision over recall; a series merely containing 'timeline' mid-name stays a series. Sequence policy: leading-number extraction ('Book 2' -> 2, '2, Dramatized Adaptation' -> 2, '1a' -> 1); lossy extractions preserve the raw text in alternate_info.series_sequence_raw. Smart genre (deterministic lexicon): compound Audible blobs split ('Epic, Action & Adventure, Military, Sword & Sorcery' -> 4 candidates), alias-normalized, format noise demoted ('Audiobook' never wins), most-specific canonical genre takes the genre column, all candidates stay in tags. Language: display names normalize to ISO 639-1 on ingest; ROOT-CAUSED the update-path drop — CreateMediaItem call site never set Language (explicit NULL over the column default), the update path did; create now carries it. Title-derived series fallback (§2.7): 'Baptism of Fire: The Witcher, Book 3' 2-level folders split conservatively (only when the post-colon tail matches ', Book N'), subtitle parsed for series+number, blanks-only precedence, matching-name number fill, zero display mutation. Works with no sidecar and no network. Ingest fixes: subtitle/explicit land in their columns (audio tag subtitle stops joining into title); full authors[] array (was [0]- only); sidecar descriptions sanitized via bluemonday UGC policy (vendored). metadata_overrides: verified scan-update path already applies ApplyMetadataOverrides; subtitle/explicit/authors covered by the Phase 1 override adaptation. Tests: new series_parsing_test.go (string-array sidecar e2e, Drizzt alternate_info, lossy raw preservation, playlist classification, title-derived fallback incl. no-mutation + blanks-only, genre lexicon, language map); audio-tag subtitle pin updated to column behavior; calibre/ABS 2.5-series_index pins widened per float change (changelog'd in phase 1). Deferred to 2b: auto-created playlist collections (§2.4) — settings toggle, collection upsert, ownership.
This commit is contained in:
John O'Keefe committed
1 parent
646477afad
commit
7e4c97f639
7 files changed
+862
-43
No files matched your search
@@ -0,0 +1,235 @@
|
||||
package services
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Series-metadata parsing helpers for the scanner (A2c overhaul).
|
||||
//
|
||||
// Sources disagree wildly on how a series and its ordinal are spelled:
|
||||
// ABS sidecars write ["Name #3"] string arrays or [{series, sequence}]
|
||||
// objects, Libation writes a bare string, Audible prefixes ordinals with
|
||||
// "Book ", Calibre uses decimal indices. These helpers normalize all of
|
||||
// them into candidates the scanner can rank and assign.
|
||||
|
||||
// SeriesCandidate is one parsed series membership from any source.
|
||||
type SeriesCandidate struct {
|
||||
Name string
|
||||
Sequence string // raw sequence text ("3", "1.5", "1a", "Book 2")
|
||||
Number float64 // leading number of Sequence (0 = none parsed)
|
||||
HasNum bool
|
||||
FromSidecar bool
|
||||
}
|
||||
|
||||
// Playlist markers that identify ordering constructs rather than
|
||||
// bibliographic series. Deliberately explicit: this classification routes
|
||||
// entries away from the series column into curated collections, so a false
|
||||
// positive creates user-visible objects. Prefer misses over false hits.
|
||||
var playlistMarkers = []string{
|
||||
"reading order",
|
||||
"chronological order",
|
||||
"publication order",
|
||||
"in order",
|
||||
"timeline",
|
||||
}
|
||||
|
||||
var (
|
||||
seriesSeqSuffixRe = regexp.MustCompile(`\s+#([^#\s]+)$`)
|
||||
// Leading ordinal inside a sequence string: "3", "1.5", "Book 2",
|
||||
// "2, Dramatized Adaptation" → 2. Mirrors ABS cleanSeriesSequence.
|
||||
seriesLeadingNumRe = regexp.MustCompile(`\d+(?:\.\d+)?`)
|
||||
// "<Series Name>, Book N" (also ; – : separators), Audible/Amazon style.
|
||||
deriveSeriesRe = regexp.MustCompile(`(?i)^\s*(.+?)\s*[,;:–-]\s*book\s+(\d+(?:\.\d+)?)\s*$`)
|
||||
// Conservative "Title: <name>, Book N" split — only when the post-colon
|
||||
// text is itself a series-and-number tail.
|
||||
titleSubtitleSplitRe = regexp.MustCompile(`(?i)^(.{1,200}?)\s*:\s*(.+,\s*book\s+\d+(?:\.\d+)?)$`)
|
||||
whitespaceRe = regexp.MustCompile(`\s+`)
|
||||
)
|
||||
|
||||
// splitSeriesString splits "Name #1a" into name + sequence, ABS
|
||||
// parseSeriesString-compatible: the sequence is anything after the last
|
||||
// " #" up to end-of-string, containing no whitespace or further '#'.
|
||||
func splitSeriesString(s string) (name, sequence string) {
|
||||
s = strings.TrimSpace(s)
|
||||
loc := seriesSeqSuffixRe.FindStringSubmatchIndex(s)
|
||||
if loc == nil {
|
||||
return s, ""
|
||||
}
|
||||
return strings.TrimSpace(s[:loc[0]]), s[loc[2]:loc[3]]
|
||||
}
|
||||
|
||||
// seriesSequenceNumber extracts the ordinal from a raw sequence string.
|
||||
// Returns the number, whether one was found, and whether information was
|
||||
// lost in the extraction (caller should preserve the raw text, e.g. "1a").
|
||||
func seriesSequenceNumber(sequence string) (num float64, found, lossy bool) {
|
||||
trimmed := strings.TrimSpace(sequence)
|
||||
if trimmed == "" {
|
||||
return 0, false, false
|
||||
}
|
||||
token := seriesLeadingNumRe.FindString(trimmed)
|
||||
if token == "" {
|
||||
return 0, false, false
|
||||
}
|
||||
num, err := strconv.ParseFloat(token, 64)
|
||||
if err != nil {
|
||||
return 0, false, false
|
||||
}
|
||||
// "1.5" tokenizes exactly (not lossy); "1a"/"Book 2" keep a raw tail.
|
||||
return num, true, trimmed != token
|
||||
}
|
||||
|
||||
// isPlaylistSeriesName reports whether the series name is an ordering
|
||||
// construct ("The Witcher (Reading Order)") rather than a bibliography.
|
||||
// Match discipline: the marker must be a parenthetical qualifier or a
|
||||
// trailing phrase — a series merely CONTAINING the word "timeline" mid-name
|
||||
// ("The Timeline of...") is a real series, not a playlist.
|
||||
func isPlaylistSeriesName(name string) bool {
|
||||
lower := strings.ToLower(strings.TrimSpace(name))
|
||||
for _, m := range playlistMarkers {
|
||||
if strings.HasSuffix(lower, m) {
|
||||
return true
|
||||
}
|
||||
if open := strings.Index(lower, "("); open >= 0 {
|
||||
if strings.Contains(lower[open:], m) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// parseSeriesCandidates normalizes the sidecar `series` field, which is
|
||||
// written as an array of STRINGS ("Name #3" — ABS abmetadata), an array of
|
||||
// OBJECTS ({series, sequence} — some exports), or a plain JSON string
|
||||
// (Libation), into candidates.
|
||||
func parseSeriesCandidates(raw json.RawMessage) []SeriesCandidate {
|
||||
trimmed := strings.TrimSpace(string(raw))
|
||||
if trimmed == "" || trimmed == "null" {
|
||||
return nil
|
||||
}
|
||||
var out []SeriesCandidate
|
||||
|
||||
add := func(name, sequence string, fromSidecar bool) {
|
||||
name = strings.TrimSpace(name)
|
||||
if name == "" {
|
||||
return
|
||||
}
|
||||
num, found, _ := seriesSequenceNumber(sequence)
|
||||
out = append(out, SeriesCandidate{
|
||||
Name: name, Sequence: sequence, Number: num, HasNum: found,
|
||||
FromSidecar: fromSidecar,
|
||||
})
|
||||
}
|
||||
|
||||
if strings.HasPrefix(trimmed, "[") {
|
||||
// Array of objects first ({"series": ..., "sequence": ...}).
|
||||
var objs []struct {
|
||||
Series string `json:"series"`
|
||||
Name string `json:"name"`
|
||||
Sequence string `json:"sequence"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &objs); err == nil && len(objs) > 0 {
|
||||
for _, o := range objs {
|
||||
name := o.Series
|
||||
if name == "" {
|
||||
name = o.Name
|
||||
}
|
||||
add(name, o.Sequence, true)
|
||||
}
|
||||
return out
|
||||
}
|
||||
// Array of strings: ABS abmetadata form "Name #3".
|
||||
var strs []string
|
||||
if err := json.Unmarshal(raw, &strs); err == nil {
|
||||
for _, s := range strs {
|
||||
name, seq := splitSeriesString(s)
|
||||
add(name, seq, true)
|
||||
}
|
||||
return out
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Plain JSON string (Libation): "Name" or "Name #3".
|
||||
var single string
|
||||
if err := json.Unmarshal(raw, &single); err == nil {
|
||||
name, seq := splitSeriesString(single)
|
||||
add(name, seq, true)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// deriveSeriesFromText extracts "<series>, Book N" from a title/subtitle
|
||||
// string (Audible/Amazon convention). Returns ok=false when the text is not
|
||||
// obviously a series tail — this fallback must not guess.
|
||||
func deriveSeriesFromText(text string) (name string, num float64, hasNum bool, ok bool) {
|
||||
if strings.TrimSpace(text) == "" {
|
||||
return "", 0, false, false
|
||||
}
|
||||
m := deriveSeriesRe.FindStringSubmatch(text)
|
||||
if m == nil {
|
||||
return "", 0, false, false
|
||||
}
|
||||
name = whitespaceRe.ReplaceAllString(strings.TrimSpace(m[1]), " ")
|
||||
if name == "" || len(name) > 200 || strings.HasPrefix(strings.ToLower(name), "book ") {
|
||||
return "", 0, false, false
|
||||
}
|
||||
num, err := strconv.ParseFloat(m[2], 64)
|
||||
if err != nil {
|
||||
return "", 0, false, false
|
||||
}
|
||||
return name, num, true, true
|
||||
}
|
||||
|
||||
// splitTitleOnSubtitle splits "Title: <name>, Book N" into title +
|
||||
// subtitle, but ONLY when the post-colon text is itself a series-and-number
|
||||
// tail — arbitrary colons in titles are never touched.
|
||||
func splitTitleOnSubtitle(folderTitle string) (title, subtitle string, split bool) {
|
||||
m := titleSubtitleSplitRe.FindStringSubmatch(strings.TrimSpace(folderTitle))
|
||||
if m == nil {
|
||||
return folderTitle, "", false
|
||||
}
|
||||
return strings.TrimSpace(m[1]), strings.TrimSpace(m[2]), true
|
||||
}
|
||||
|
||||
// normalizeLanguageCode maps sidecar/display language names to ISO 639-1
|
||||
// codes (the media_items.language convention). Values already in code form
|
||||
// pass through; unknown display names are kept verbatim when they fit the
|
||||
// VARCHAR(10) column, else dropped (empty) — never truncate mid-word.
|
||||
func normalizeLanguageCode(v string) string {
|
||||
v = strings.TrimSpace(v)
|
||||
if v == "" {
|
||||
return ""
|
||||
}
|
||||
if code, ok := languageNameToCode[strings.ToLower(v)]; ok {
|
||||
return code
|
||||
}
|
||||
lower := strings.ToLower(v)
|
||||
if len(lower) == 2 {
|
||||
return lower // already a code (or close enough to be harmless)
|
||||
}
|
||||
if len(v) <= 10 {
|
||||
return v
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
var languageNameToCode = map[string]string{
|
||||
"english": "en", "spanish": "es", "español": "es", "french": "fr",
|
||||
"français": "fr", "german": "de", "deutsch": "de", "italian": "it",
|
||||
"italiano": "it", "portuguese": "pt", "dutch": "nl", "russian": "ru",
|
||||
"ukrainian": "uk", "polish": "pl", "czech": "cs", "slovak": "sk",
|
||||
"swedish": "sv", "norwegian": "no", "danish": "da", "finnish": "fi",
|
||||
"icelandic": "is", "hungarian": "hu", "romanian": "ro", "bulgarian": "bg",
|
||||
"greek": "el", "turkish": "tr", "arabic": "ar", "hebrew": "he",
|
||||
"persian": "fa", "hindi": "hi", "bengali": "bn", "urdu": "ur",
|
||||
"thai": "th", "vietnamese": "vi", "indonesian": "id", "malay": "ms",
|
||||
"japanese": "ja", "korean": "ko", "chinese": "zh", "mandarin": "zh",
|
||||
"cantonese": "zh", "swahili": "sw", "afrikaans": "af", "catalan": "ca",
|
||||
"croatian": "hr", "serbian": "sr", "slovenian": "sl", "lithuanian": "lt",
|
||||
"latvian": "lv", "estonian": "et", "georgian": "ka", "armenian": "hy",
|
||||
"welsh": "cy", "irish": "ga", "basque": "eu", "galician": "gl",
|
||||
}
|
||||
Reference in new issue
Block a user