Files
bookhoard/internal/services/media_scanner_opf.go
T
John O'Keefe 1fa8ee3a59
Release / build-and-push (push) Successful in 3m5s
feat(scanner): Calibre-aligned OPF metadata extraction
Adopt Calibre's reading conventions for the Dublin Core metadata that
parseOPFContent now pulls from the structured OPF parse:

- Titles: EPUB3 title-type selection (prefer 'main', join a distinct
  subtitle with ': ' exactly as Calibre stores it). There is no separate
  subtitle column by design - Calibre-sidecar books arrive pre-joined,
  so a column would stay empty for most libraries and force every client
  to reimplement concatenation.
- Genre: first dc:subject, mirroring the existing processGenresAndTags
  behavior of the Calibre-sidecar path; the embedded path never
  populated Genre before. Subjects stay one-element-one-tag - Library
  of Congress headings legitimately contain commas ("Holmes, Sherlock
  (Fictitious character) -- Fiction") and must not be split.
- Identifiers: urn:isbn:/urn:asin: prefixed values parse in addition to
  opf:scheme attributes, and the scheme-less fallback now requires an
  ISBN-shaped value (10/13 digits, optional separators/trailing X) so
  URIs like the Gutenberg identifiers cannot masquerade as ISBNs -
  observed live on 'A Study in Scarlet'.
- Series: EPUB3 belongs-to-collection with collection-type=series and
  group-position refines, ahead of the classic calibre:series metas.
- Audiobookshelf metadata.json sidecars join their subtitle field into
  the title the same way.

Tests cover title-type main+subtitle joining, belongs-to-collection
series with fractional group-position, urn:isbn extraction, genre/tag
parity, and comma preservation inside subject headings.
2026-09-12 23:45:18 -04:00

328 lines
9.4 KiB
Go

package services
import (
"bytes"
"encoding/xml"
"strconv"
"strings"
)
// Calibre-modeled OPF parsing. The scanner previously scraped OPF content
// with attribute-order-sensitive regexes; real books serialize attributes in
// any order (e.g. Pragmatic/Pattinson EPUBs put id before properties and
// content before name), which silently defeated cover detection. Everything
// here is parsed with encoding/xml so attribute order and namespace prefix
// choices are irrelevant.
type opfDCValue struct {
ID string `xml:"id,attr"`
Value string `xml:",chardata"`
}
type opfIdentifier struct {
Scheme string `xml:"http://www.idpf.org/2007/opf scheme,attr"`
Value string `xml:",chardata"`
}
type opfMeta struct {
ID string `xml:"id,attr"`
Name string `xml:"name,attr"`
Content string `xml:"content,attr"`
Property string `xml:"property,attr"`
Refines string `xml:"refines,attr"`
Value string `xml:",chardata"`
}
type opfItem struct {
ID string `xml:"id,attr"`
Href string `xml:"href,attr"`
MediaType string `xml:"media-type,attr"`
Properties string `xml:"properties,attr"`
}
// opfDocument is a structured view of an OPF package document.
type opfDocument struct {
Metadata struct {
Titles []opfDCValue `xml:"http://purl.org/dc/elements/1.1/ title"`
Creators []string `xml:"http://purl.org/dc/elements/1.1/ creator"`
Subjects []string `xml:"http://purl.org/dc/elements/1.1/ subject"`
Descriptions []string `xml:"http://purl.org/dc/elements/1.1/ description"`
Publishers []string `xml:"http://purl.org/dc/elements/1.1/ publisher"`
Dates []string `xml:"http://purl.org/dc/elements/1.1/ date"`
Languages []string `xml:"http://purl.org/dc/elements/1.1/ language"`
Identifiers []opfIdentifier `xml:"http://purl.org/dc/elements/1.1/ identifier"`
Contributors []string `xml:"http://purl.org/dc/elements/1.1/ contributor"`
Metas []opfMeta `xml:"meta"`
} `xml:"metadata"`
Manifest struct {
Items []opfItem `xml:"item"`
} `xml:"manifest"`
Spine struct {
Itemrefs []struct {
IDRef string `xml:"idref,attr"`
} `xml:"itemref"`
} `xml:"spine"`
Guide struct {
References []struct {
Type string `xml:"type,attr"`
Href string `xml:"href,attr"`
} `xml:"reference"`
} `xml:"guide"`
}
func parseOPFXML(content []byte) (*opfDocument, error) {
var doc opfDocument
if err := xml.Unmarshal(content, &doc); err != nil {
return nil, err
}
return &doc, nil
}
// refinesFor maps an element id to its EPUB3 refining metas
// (those whose refines attribute starts with '#').
func (d *opfDocument) refinesFor(id string) []opfMeta {
var out []opfMeta
if id == "" {
return out
}
for _, m := range d.Metadata.Metas {
if strings.HasPrefix(m.Refines, "#") && m.Refines[1:] == id {
out = append(out, m)
}
}
return out
}
// refinesProperty returns the value of the first refining meta carrying the
// given property (e.g. "title-type", "collection-type", "group-position").
func refinesProperty(metas []opfMeta, property string) (string, bool) {
for _, m := range metas {
if strings.EqualFold(m.Property, property) {
if v := strings.TrimSpace(m.Value); v != "" {
return v, true
}
}
}
return "", false
}
// selectTitle ports Calibre's read_title: prefer the dc:title refined as
// title-type "main"; fall back to the first non-empty title. A distinct
// subtitle (title-type containing "subtitle"/"sub-title") is joined onto the
// main title with ": ", exactly as Calibre stores it.
func (d *opfDocument) selectTitle() string {
var first, main, subtitle string
for _, t := range d.Metadata.Titles {
v := strings.TrimSpace(t.Value)
if v == "" {
continue
}
if first == "" {
first = v
}
tt, ok := refinesProperty(d.refinesFor(t.ID), "title-type")
if !ok {
continue
}
switch strings.ToLower(tt) {
case "main":
if main == "" {
main = v
}
default:
l := strings.ToLower(tt)
if strings.Contains(l, "subtitle") || strings.Contains(l, "sub-title") {
if subtitle == "" {
subtitle = v
}
}
}
}
title := main
if title == "" {
title = first
}
if subtitle != "" && subtitle != title {
title = title + ": " + subtitle
}
return title
}
// readSeries ports Calibre's read_series: EPUB3 belongs-to-collection (with a
// collection-type=series refine and group-position index) first, then the
// classic calibre:series / calibre:series_index metas.
func (d *opfDocument) readSeries() (series string, index float64) {
for _, m := range d.Metadata.Metas {
if !strings.EqualFold(m.Property, "belongs-to-collection") {
continue
}
name := strings.TrimSpace(m.Value)
if name == "" {
continue
}
refines := d.refinesFor(m.ID)
if ct, ok := refinesProperty(refines, "collection-type"); !ok || !strings.EqualFold(ct, "series") {
continue
}
if gp, ok := refinesProperty(refines, "group-position"); ok {
if v, err := strconv.ParseFloat(strings.TrimSpace(gp), 64); err == nil {
index = v
}
}
return name, index
}
for _, m := range d.Metadata.Metas {
switch m.Name {
case "calibre:series":
series = m.Content
case "calibre:series_index":
if v, err := strconv.ParseFloat(strings.TrimSpace(m.Content), 64); err == nil {
index = v
}
}
}
return series, index
}
// itemByID returns manifest items with id, href and media-type, keyed by id.
func (d *opfDocument) itemByID() map[string]opfItem {
m := make(map[string]opfItem, len(d.Manifest.Items))
for _, it := range d.Manifest.Items {
if it.ID != "" && it.Href != "" && it.MediaType != "" {
m[it.ID] = it
}
}
return m
}
// firstSpineItem returns the manifest item for the first spine idref.
func (d *opfDocument) firstSpineItem() (opfItem, bool) {
if len(d.Spine.Itemrefs) == 0 {
return opfItem{}, false
}
item, ok := d.itemByID()[d.Spine.Itemrefs[0].IDRef]
return item, ok
}
// isRasterMedia reports whether a manifest media-type is an image but not an
// (X)HTML document - Calibre's guard against cover *pages* masquerading as
// cover images.
func isRasterMedia(mediaType string) bool {
mt := strings.ToLower(strings.TrimSpace(mediaType))
if mt == "" {
return false
}
if strings.Contains(mt, "xml") || strings.Contains(mt, "html") {
return false
}
return strings.HasPrefix(mt, "image/")
}
// findRasterCoverInOPF ports Calibre's read_raster_cover resolution order:
// 1. manifest item with properties containing "cover-image"
// 2. <meta name="cover" content="ID"> resolved through the manifest
// 3. the first spine item being a raster image itself (store manga)
//
// Returns the OPF-relative href of the cover image, or "".
func (d *opfDocument) findRasterCoverInOPF() string {
// 1. properties="cover-image" (space-separated property list)
for _, it := range d.Manifest.Items {
for _, prop := range strings.Fields(it.Properties) {
if strings.EqualFold(prop, "cover-image") && isRasterMedia(it.MediaType) {
return it.Href
}
}
}
// 2. meta name="cover" content=<manifest image id>
byID := d.itemByID()
for _, m := range d.Metadata.Metas {
if !strings.EqualFold(m.Name, "cover") {
continue
}
if it, ok := byID[strings.TrimSpace(m.Content)]; ok && isRasterMedia(it.MediaType) {
return it.Href
}
}
// 3. first spine item is itself an image (jpeg/webp/png per Calibre)
if it, ok := d.firstSpineItem(); ok {
mt := strings.ToLower(it.MediaType)
if mt == "image/jpeg" || mt == "image/webp" || mt == "image/png" {
return it.Href
}
}
return ""
}
// coverPageHref returns the OPF-relative href of the cover *page* document to
// mine for an embedded image: the guide's type="cover" reference when
// present, otherwise the first spine item (Calibre renders the latter).
func (d *opfDocument) coverPageHref() string {
for _, ref := range d.Guide.References {
if strings.EqualFold(ref.Type, "cover") && ref.Href != "" {
return ref.Href
}
}
if it, ok := d.firstSpineItem(); ok {
if it.Href != "" && !isRasterMedia(it.MediaType) {
return it.Href
}
}
return ""
}
// isISBNLike reports whether a bare identifier value is shaped like an ISBN
// (digits, optional hyphens/spaces, optional trailing X; 10 or 13
// significant characters). Guards the scheme-less dc:identifier fallback
// against URLs and UUIDs sharing the same slot.
func isISBNLike(v string) bool {
digits := 0
for i, r := range v {
switch {
case r >= '0' && r <= '9':
digits++
case r == '-' || r == ' ':
// separator
case (r == 'X' || r == 'x') && i == len(v)-1:
digits++ // ISBN-10 check character
default:
return false
}
}
return digits == 10 || digits == 13
}
// findImageReferenceInPage extracts the first raster image reference from a
// cover (X)HTML page: <img src="..."> or SVG <image xlink:href="...">.
// Token-based parsing keeps it tolerant of mixed namespaces and fragments.
// Returns the reference relative to the page document, or "".
func findImageReferenceInPage(pageContent []byte) string {
decoder := xml.NewDecoder(bytes.NewReader(pageContent))
for {
tok, err := decoder.Token()
if err != nil {
return ""
}
start, ok := tok.(xml.StartElement)
if !ok {
continue
}
switch strings.ToLower(start.Name.Local) {
case "img":
for _, a := range start.Attr {
if strings.EqualFold(a.Name.Local, "src") && strings.TrimSpace(a.Value) != "" {
return strings.TrimSpace(a.Value)
}
}
case "image":
for _, a := range start.Attr {
if strings.EqualFold(a.Name.Local, "href") && strings.TrimSpace(a.Value) != "" {
return strings.TrimSpace(a.Value)
}
}
}
}
}