From 1fa8ee3a590bbb1efcefd23faddf32fb12d4f6a5 Mon Sep 17 00:00:00 2001 From: John O'Keefe Date: Sat, 12 Sep 2026 23:45:18 -0400 Subject: [PATCH] feat(scanner): Calibre-aligned OPF metadata extraction Adopt Calibre's reading conventions for the Dublin Core metadata that parseOPFContent now pulls from the structured OPF parse: - Titles: EPUB3 title-type selection (prefer 'main', join a distinct subtitle with ': ' exactly as Calibre stores it). There is no separate subtitle column by design - Calibre-sidecar books arrive pre-joined, so a column would stay empty for most libraries and force every client to reimplement concatenation. - Genre: first dc:subject, mirroring the existing processGenresAndTags behavior of the Calibre-sidecar path; the embedded path never populated Genre before. Subjects stay one-element-one-tag - Library of Congress headings legitimately contain commas ("Holmes, Sherlock (Fictitious character) -- Fiction") and must not be split. - Identifiers: urn:isbn:/urn:asin: prefixed values parse in addition to opf:scheme attributes, and the scheme-less fallback now requires an ISBN-shaped value (10/13 digits, optional separators/trailing X) so URIs like the Gutenberg identifiers cannot masquerade as ISBNs - observed live on 'A Study in Scarlet'. - Series: EPUB3 belongs-to-collection with collection-type=series and group-position refines, ahead of the classic calibre:series metas. - Audiobookshelf metadata.json sidecars join their subtitle field into the title the same way. Tests cover title-type main+subtitle joining, belongs-to-collection series with fractional group-position, urn:isbn extraction, genre/tag parity, and comma preservation inside subject headings. --- internal/services/media_scanner.go | 133 ++++++++++---------- internal/services/media_scanner_opf.go | 128 +++++++++++++++++++ internal/services/media_scanner_opf_test.go | 59 +++++++++ 3 files changed, 255 insertions(+), 65 deletions(-) diff --git a/internal/services/media_scanner.go b/internal/services/media_scanner.go index da0785f..d3564b3 100644 --- a/internal/services/media_scanner.go +++ b/internal/services/media_scanner.go @@ -1024,9 +1024,10 @@ func extractAudiobookshelfSidecar(path string) *MediaMetadata { } var sidecar struct { - Title string `json:"title"` - Authors []string `json:"authors"` - Series []struct { + Title string `json:"title"` + Subtitle string `json:"subtitle"` + Authors []string `json:"authors"` + Series []struct { Series string `json:"series"` Sequence string `json:"sequence"` } `json:"series"` @@ -1048,6 +1049,11 @@ func extractAudiobookshelfSidecar(path string) *MediaMetadata { metadata := &MediaMetadata{ Title: strings.TrimSpace(sidecar.Title), } + // Subtitle joins the title Calibre-style ("Main: Subtitle") - there is no + // separate subtitle column, and Calibre-sidecar books arrive pre-joined. + if subtitle := strings.TrimSpace(sidecar.Subtitle); subtitle != "" && metadata.Title != "" { + metadata.Title = metadata.Title + ": " + subtitle + } if len(sidecar.Authors) > 0 { metadata.Author = strings.TrimSpace(sidecar.Authors[0]) } @@ -1721,80 +1727,75 @@ func (s *MediaScanner) parseCalibreMetadataOPF(opfPath string) (*MediaMetadata, // parseOPFContent parses an OPF document (Dublin Core metadata) into // MediaMetadata. Used for both Calibre metadata.opf sidecars and the OPF -// embedded inside an EPUB - the dc:* vocabulary is identical. Namespace-aware -// parsing means it tolerates wherever the xmlns:dc declaration lives. +// embedded inside an EPUB - the dc:* vocabulary is identical. Parsing is +// attribute-order agnostic and namespace aware (see media_scanner_opf.go); +// title selection and series detection follow Calibre's behavior. func parseOPFContent(content []byte) (*MediaMetadata, error) { - // Define XML structure for parsing with full Dublin Core namespace URLs - var opf struct { - XMLName xml.Name `xml:"package"` - Metadata struct { - XMLName xml.Name `xml:"metadata"` - Titles []string `xml:"http://purl.org/dc/elements/1.1/ title"` - Creators []string `xml:"http://purl.org/dc/elements/1.1/ creator"` - Subjects []string `xml:"http://purl.org/dc/elements/1.1/ subject"` - Desc []string `xml:"http://purl.org/dc/elements/1.1/ description"` - Publisher []string `xml:"http://purl.org/dc/elements/1.1/ publisher"` - Dates []string `xml:"http://purl.org/dc/elements/1.1/ date"` - Language []string `xml:"http://purl.org/dc/elements/1.1/ language"` - Identifiers []struct { - Scheme string `xml:"http://www.idpf.org/2007/opf scheme,attr"` - Value string `xml:",chardata"` - } `xml:"http://purl.org/dc/elements/1.1/ identifier"` - Contributors []string `xml:"http://purl.org/dc/elements/1.1/ contributor"` - // Calibre-specific meta tags - capture all, filter later - MetaTags []struct { - Name string `xml:"name,attr"` - Value string `xml:"content,attr"` - } `xml:"meta"` - } `xml:"metadata"` - } - // Parse XML - if err := xml.NewDecoder(bytes.NewReader(content)).Decode(&opf); err != nil { + opf, err := parseOPFXML(content) + if err != nil { return nil, fmt.Errorf("failed to parse OPF XML: %v", err) } + md := opf.Metadata + // Map to MediaMetadata struct metadata := &MediaMetadata{} - // Title (required) - if len(opf.Metadata.Titles) > 0 { - metadata.Title = opf.Metadata.Titles[0] + // Title: EPUB3 title-type main selection with Calibre-style subtitle join + if title := opf.selectTitle(); title != "" { + metadata.Title = title } // Author (first creator) - if len(opf.Metadata.Creators) > 0 { - metadata.Author = opf.Metadata.Creators[0] + if len(md.Creators) > 0 { + metadata.Author = md.Creators[0] } - // Tags (all subjects) - if len(opf.Metadata.Subjects) > 0 { - metadata.Tags = utils.NormalizeTags(opf.Metadata.Subjects) + // Tags (all subjects; one element = one tag, commas are legal inside + // subject headings like "Holmes, Sherlock (Fictitious character)"). + // Genre mirrors the sidecar path's processGenresAndTags: first subject. + if len(md.Subjects) > 0 { + metadata.Tags = utils.NormalizeTags(md.Subjects) + if len(metadata.Tags) > 0 { + metadata.Genre = metadata.Tags[0] + } } // Description - if len(opf.Metadata.Desc) > 0 { - metadata.Description = opf.Metadata.Desc[0] + if len(md.Descriptions) > 0 { + metadata.Description = md.Descriptions[0] } // Publisher - if len(opf.Metadata.Publisher) > 0 { - metadata.Publisher = opf.Metadata.Publisher[0] + if len(md.Publishers) > 0 { + metadata.Publisher = md.Publishers[0] } // Language - if len(opf.Metadata.Language) > 0 && opf.Metadata.Language[0] != "" { - metadata.Language = opf.Metadata.Language[0] + if len(md.Languages) > 0 && md.Languages[0] != "" { + metadata.Language = md.Languages[0] } // Publish date - if len(opf.Metadata.Dates) > 0 { - if date, err := time.Parse("2006-01-02T15:04:05Z07:00", opf.Metadata.Dates[0]); err == nil { + if len(md.Dates) > 0 { + if date, err := time.Parse("2006-01-02T15:04:05Z07:00", md.Dates[0]); err == nil { metadata.PublishDate = date - } else if date, err := time.Parse("2006-01-02", opf.Metadata.Dates[0]); err == nil { + } else if date, err := time.Parse("2006-01-02", md.Dates[0]); err == nil { metadata.PublishDate = date } else { // Try alternative date formats - if date, err := time.Parse("2006", opf.Metadata.Dates[0]); err == nil { + if date, err := time.Parse("2006", md.Dates[0]); err == nil { metadata.PublishDate = date } } } - // Identifiers (ISBN, ASIN) - for _, id := range opf.Metadata.Identifiers { + // Identifiers: opf:scheme attribute first, then URN-prefixed values + // (urn:isbn:...), then bare values that normalize to a valid ISBN. + for _, id := range md.Identifiers { value := strings.TrimSpace(id.Value) - switch strings.ToUpper(id.Scheme) { + scheme := strings.ToUpper(strings.TrimSpace(id.Scheme)) + if scheme == "" { + if lower := strings.ToLower(value); strings.HasPrefix(lower, "urn:") { + rest := value[4:] + if prefix, val, ok := strings.Cut(rest, ":"); ok { + scheme = strings.ToUpper(prefix) + value = strings.TrimSpace(val) + } + } + } + switch scheme { case "ISBN": metadata.ISBN = utils.NormalizeISBNSafe(value) case "ASIN": @@ -1803,28 +1804,30 @@ func parseOPFContent(content []byte) (*MediaMetadata, error) { // Store UUID in hash info, not metadata // Will be extracted by extractHashInfo() default: - // EPUB3 identifiers often carry no opf:scheme attribute; - // accept a bare value that normalizes to a valid ISBN. - if metadata.ISBN == "" && id.Scheme == "" { - if normalized := utils.NormalizeISBNSafe(value); normalized != "" { + // EPUB3 identifiers often carry no opf:scheme attribute; accept a + // bare value only when it is ISBN-shaped (rejects the URIs and + // UUIDs that commonly share the identifier list). + if metadata.ISBN == "" && scheme == "" && isISBNLike(value) { + if normalized, err := utils.NormalizeISBN(value); err == nil { metadata.ISBN = normalized } } } } // Contributors - if len(opf.Metadata.Contributors) > 0 { - metadata.Contributors = utils.NormalizeContributors(opf.Metadata.Contributors) + if len(md.Contributors) > 0 { + metadata.Contributors = utils.NormalizeContributors(md.Contributors) + } + // Series: EPUB3 belongs-to-collection, then calibre:series metas + if series, index := opf.readSeries(); series != "" { + metadata.Series = series + if index > 0 { + metadata.SeriesNumber = int32(index) + } } // Calibre-specific meta tags (filter by name attribute) - for _, meta := range opf.Metadata.MetaTags { + for _, meta := range md.Metas { switch meta.Name { - case "calibre:series": - metadata.Series = meta.Value - case "calibre:series_index": - if index, err := strconv.ParseFloat(meta.Value, 32); err == nil { - metadata.SeriesNumber = int32(index) - } case "calibre:rating": // Not imported (ratings are per-user in Bookhoard) case "calibre:title_sort": diff --git a/internal/services/media_scanner_opf.go b/internal/services/media_scanner_opf.go index 9e750ad..3f7c4dd 100644 --- a/internal/services/media_scanner_opf.go +++ b/internal/services/media_scanner_opf.go @@ -3,6 +3,7 @@ package services import ( "bytes" "encoding/xml" + "strconv" "strings" ) @@ -77,6 +78,112 @@ func parseOPFXML(content []byte) (*opfDocument, error) { return &doc, nil } +// refinesFor maps an element id to its EPUB3 refining metas +// (those whose refines attribute starts with '#'). +func (d *opfDocument) refinesFor(id string) []opfMeta { + var out []opfMeta + if id == "" { + return out + } + for _, m := range d.Metadata.Metas { + if strings.HasPrefix(m.Refines, "#") && m.Refines[1:] == id { + out = append(out, m) + } + } + return out +} + +// refinesProperty returns the value of the first refining meta carrying the +// given property (e.g. "title-type", "collection-type", "group-position"). +func refinesProperty(metas []opfMeta, property string) (string, bool) { + for _, m := range metas { + if strings.EqualFold(m.Property, property) { + if v := strings.TrimSpace(m.Value); v != "" { + return v, true + } + } + } + return "", false +} + +// selectTitle ports Calibre's read_title: prefer the dc:title refined as +// title-type "main"; fall back to the first non-empty title. A distinct +// subtitle (title-type containing "subtitle"/"sub-title") is joined onto the +// main title with ": ", exactly as Calibre stores it. +func (d *opfDocument) selectTitle() string { + var first, main, subtitle string + for _, t := range d.Metadata.Titles { + v := strings.TrimSpace(t.Value) + if v == "" { + continue + } + if first == "" { + first = v + } + tt, ok := refinesProperty(d.refinesFor(t.ID), "title-type") + if !ok { + continue + } + switch strings.ToLower(tt) { + case "main": + if main == "" { + main = v + } + default: + l := strings.ToLower(tt) + if strings.Contains(l, "subtitle") || strings.Contains(l, "sub-title") { + if subtitle == "" { + subtitle = v + } + } + } + } + title := main + if title == "" { + title = first + } + if subtitle != "" && subtitle != title { + title = title + ": " + subtitle + } + return title +} + +// readSeries ports Calibre's read_series: EPUB3 belongs-to-collection (with a +// collection-type=series refine and group-position index) first, then the +// classic calibre:series / calibre:series_index metas. +func (d *opfDocument) readSeries() (series string, index float64) { + for _, m := range d.Metadata.Metas { + if !strings.EqualFold(m.Property, "belongs-to-collection") { + continue + } + name := strings.TrimSpace(m.Value) + if name == "" { + continue + } + refines := d.refinesFor(m.ID) + if ct, ok := refinesProperty(refines, "collection-type"); !ok || !strings.EqualFold(ct, "series") { + continue + } + if gp, ok := refinesProperty(refines, "group-position"); ok { + if v, err := strconv.ParseFloat(strings.TrimSpace(gp), 64); err == nil { + index = v + } + } + return name, index + } + for _, m := range d.Metadata.Metas { + switch m.Name { + case "calibre:series": + series = m.Content + case "calibre:series_index": + if v, err := strconv.ParseFloat(strings.TrimSpace(m.Content), 64); err == nil { + index = v + } + } + } + return series, index +} + // itemByID returns manifest items with id, href and media-type, keyed by id. func (d *opfDocument) itemByID() map[string]opfItem { m := make(map[string]opfItem, len(d.Manifest.Items)) @@ -166,6 +273,27 @@ func (d *opfDocument) coverPageHref() string { return "" } +// isISBNLike reports whether a bare identifier value is shaped like an ISBN +// (digits, optional hyphens/spaces, optional trailing X; 10 or 13 +// significant characters). Guards the scheme-less dc:identifier fallback +// against URLs and UUIDs sharing the same slot. +func isISBNLike(v string) bool { + digits := 0 + for i, r := range v { + switch { + case r >= '0' && r <= '9': + digits++ + case r == '-' || r == ' ': + // separator + case (r == 'X' || r == 'x') && i == len(v)-1: + digits++ // ISBN-10 check character + default: + return false + } + } + return digits == 10 || digits == 13 +} + // findImageReferenceInPage extracts the first raster image reference from a // cover (X)HTML page: or SVG . // Token-based parsing keeps it tolerant of mixed namespaces and fragments. diff --git a/internal/services/media_scanner_opf_test.go b/internal/services/media_scanner_opf_test.go index 0a6fb76..9c60d24 100644 --- a/internal/services/media_scanner_opf_test.go +++ b/internal/services/media_scanner_opf_test.go @@ -145,6 +145,65 @@ func TestFindCoverInOPFImageFirstSpine(t *testing.T) { } } +// TestParseOPFContentTitleTypeAndSeries covers EPUB3 refines-based title +// selection (main + subtitle joined Calibre-style) and belongs-to-collection +// series with collection-type and group-position refines. +func TestParseOPFContentTitleTypeAndSeries(t *testing.T) { + opf := ` + + + The Main Title + The Subtitle + main + subtitle + Programming + Algorithms + urn:isbn:978-3-16-148410-0 + Great Series + series + 4.5 + +` + metadata, err := parseOPFContent([]byte(opf)) + if err != nil { + t.Fatalf("parseOPFContent() error: %v", err) + } + if want := "The Main Title: The Subtitle"; metadata.Title != want { + t.Errorf("Title = %q, want %q", metadata.Title, want) + } + if metadata.Series != "Great Series" || metadata.SeriesNumber != 4 { + t.Errorf("Series = %q/%d, want Great Series/4", metadata.Series, metadata.SeriesNumber) + } + if metadata.ISBN == "" { + t.Error("urn:isbn: identifier not extracted") + } + if metadata.Genre != "Programming" { + t.Errorf("Genre = %q, want first subject %q", metadata.Genre, "Programming") + } + if len(metadata.Tags) != 2 { + t.Errorf("Tags = %v, want both subjects", metadata.Tags) + } +} + +// TestParseOPFContentSubjectsWithCommas verifies subject headings keep their +// embedded commas as single tags (Library of Congress style headings). +func TestParseOPFContentSubjectsWithCommas(t *testing.T) { + opf := ` + + + A Study in Scarlet + Holmes, Sherlock (Fictitious character) -- Fiction + +` + metadata, err := parseOPFContent([]byte(opf)) + if err != nil { + t.Fatalf("parseOPFContent() error: %v", err) + } + if len(metadata.Tags) != 1 { + t.Errorf("Tags = %v, want exactly 1 unsplit subject heading", metadata.Tags) + } +} + // TestResolveOPFPath checks URL decoding and posix normalization of // OPF-relative hrefs. func TestResolveOPFPath(t *testing.T) {