Bookhoard/bookhoard
0
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases 18 Wiki Activity

feat(scanner): extract EPUB metadata from the embedded OPF directly

Browse Source 
go-epub's ReadBook parses every spine chapter and fails the entire call
if any single chapter (or the TOC) is malformed, discarding already-parsed
OPF metadata. For books like Pragmatic's 'A Common-Sense Guide' the OPF
holds good title/author/publisher/ISBN metadata that rescans then wrote
as blanks - success toast, no (visible) change.

Refactor to parse the EPUB's own OPF document with the same Dublin Core
machinery used for Calibre sidecars: parseCalibreMetadataOPF is now a thin
file wrapper around a reusable parseOPFContent([]byte), and the OPF lookup
previously inline in extractEPUBCover is shared via findOPFPathInZip. Since
metadata never touches chapter bodies, chapter damage cannot blank it.

Also picked up along the way: dc:language mapping and scheme-less
dc:identifier values that normalize to a valid ISBN (EPUB3 style).

Tests cover a Pragmatic-style EPUB (dc namespace on <metadata>, no
identifier scheme, deliberately malformed chapter) that must still yield
full metadata, plus series/date/subject OPF parsing.
This commit is contained in:
John O'Keefe
2026-09-12 17:49:30 -04:00
parent 7a9a66fcc5
commit 61681aac23
2 changed files with 248 additions and 117 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/services/media_scanner.go
+94 -117
View File
@@ -1347,89 +1347,38 @@ func (s *MediaScanner) extractMetadata(path string) (*MediaMetadata, error) {
}
}
// extractEPUBMetadata extracts metadata from an EPUB by parsing its embedded
// OPF document directly (container.xml → OPF → Dublin Core elements).
//
// It deliberately does NOT use a full-book parser: the previous go-epub
// implementation parsed every spine chapter and failed the whole call when any
// single chapter (or the TOC) was malformed, discarding perfectly good OPF
// metadata and leaving rescans writing blanks. The OPF holds all the metadata
// we need; chapter damage can no longer affect it.
func (s *MediaScanner) extractEPUBMetadata(path string) (*MediaMetadata, error) {
book, err := epub.ReadBook(path)
r, err := zip.OpenReader(path)
if err != nil {
return nil, fmt.Errorf("failed to open EPUB: %v", err)
}
metadata := &MediaMetadata{}
// Title
if title, err := book.Title(); err == nil && title != "" {
metadata.Title = title
}
// Author
if authors, err := book.MetadataByKey("creator"); err == nil && len(authors) > 0 {
metadata.Author = authors[0]
}
// Description
if descriptions, err := book.MetadataByKey("description"); err == nil && len(descriptions) > 0 {
metadata.Description = descriptions[0]
}
// Publisher
if publishers, err := book.MetadataByKey("publisher"); err == nil && len(publishers) > 0 {
metadata.Publisher = publishers[0]
}
// Series and series number (Calibre specific metadata)
if series, err := book.MetadataByKey("calibre:series"); err == nil && len(series) > 0 {
metadata.Series = series[0]
}
if seriesIndex, err := book.MetadataByKey("calibre:series_index"); err == nil && len(seriesIndex) > 0 {
if index, err := strconv.ParseFloat(seriesIndex[0], 32); err == nil {
metadata.SeriesNumber = int32(index)
defer func() {
if err := r.Close(); err != nil {
fmt.Printf("Warning: failed to close EPUB zip reader for %s: %v\n", path, err)
}
}()
opfPath := findOPFPathInZip(r.File)
if opfPath == "" {
return nil, fmt.Errorf("no OPF document found in EPUB %s", path)
}
opfContent, err := readFileFromZip(r.File, opfPath)
if err != nil {
return nil, fmt.Errorf("failed to read OPF %s from EPUB: %v", opfPath, err)
}
// Publish date
if dates, err := book.MetadataByKey("date"); err == nil && len(dates) > 0 {
if date, err := time.Parse("2006-01-02", dates[0]); err == nil {
metadata.PublishDate = date
} else {
// Try alternative date formats
if date, err := time.Parse("2006", dates[0]); err == nil {
metadata.PublishDate = date
}
}
metadata, err := parseOPFContent(opfContent)
if err != nil {
return nil, fmt.Errorf("failed to parse OPF in EPUB %s: %v", path, err)
}
// Contributors
if contributors, err := book.MetadataByKey("contributor"); err == nil && len(contributors) > 0 {
// Normalize contributors for display
metadata.Contributors = utils.NormalizeContributors(contributors)
}
// ISBN
if isbns, err := book.MetadataByKey("identifier"); err == nil && len(isbns) > 0 {
for _, isbn := range isbns {
if strings.Contains(strings.ToLower(isbn), "isbn") {
// Extract ISBN number from identifier like "isbn:978-3-16-148410-0"
isbnParts := strings.SplitN(isbn, ":", 2)
if len(isbnParts) == 2 {
metadata.ISBN = isbnParts[1]
break
}
}
if strings.Contains(strings.ToLower(isbn), "asin") {
// Extract ASIN from identifier like "asin:B08XXXXX"
asinParts := strings.SplitN(isbn, ":", 2)
if len(asinParts) == 2 {
metadata.ASIN = asinParts[1]
}
}
}
}
// Tags
if tags, err := book.MetadataByKey("subject"); err == nil && len(tags) > 0 {
// Normalize tags for display
metadata.Tags = utils.NormalizeTags(tags)
}
return metadata, nil
}
@@ -1604,7 +1553,7 @@ func (s *MediaScanner) LogProcessingIssue(
return err
}
// parseCalibreMetadataOPF parses a Calibre metadata.opf file and extracts metadata
// parseCalibreMetadataOPF parses a Calibre metadata.opf sidecar file.
func (s *MediaScanner) parseCalibreMetadataOPF(opfPath string) (*MediaMetadata, error) {
// Open file
file, err := os.Open(opfPath)
@@ -1616,6 +1565,18 @@ func (s *MediaScanner) parseCalibreMetadataOPF(opfPath string) (*MediaMetadata,
fmt.Printf("Warning: failed to close metadata.opf: %v\n", err)
}
}()
content, err := io.ReadAll(file)
if err != nil {
return nil, fmt.Errorf("failed to read metadata.opf: %v", err)
}
return parseOPFContent(content)
}
// parseOPFContent parses an OPF document (Dublin Core metadata) into
// MediaMetadata. Used for both Calibre metadata.opf sidecars and the OPF
// embedded inside an EPUB - the dc:* vocabulary is identical. Namespace-aware
// parsing means it tolerates wherever the xmlns:dc declaration lives.
func parseOPFContent(content []byte) (*MediaMetadata, error) {
// Define XML structure for parsing with full Dublin Core namespace URLs
var opf struct {
XMLName xml.Name `xml:"package"`
@@ -1641,8 +1602,8 @@ func (s *MediaScanner) parseCalibreMetadataOPF(opfPath string) (*MediaMetadata,
} `xml:"metadata"`
}
// Parse XML
if err := xml.NewDecoder(file).Decode(&opf); err != nil {
return nil, fmt.Errorf("failed to parse metadata.opf XML: %v", err)
if err := xml.NewDecoder(bytes.NewReader(content)).Decode(&opf); err != nil {
return nil, fmt.Errorf("failed to parse OPF XML: %v", err)
}
// Map to MediaMetadata struct
metadata := &MediaMetadata{}
@@ -1666,6 +1627,10 @@ func (s *MediaScanner) parseCalibreMetadataOPF(opfPath string) (*MediaMetadata,
if len(opf.Metadata.Publisher) > 0 {
metadata.Publisher = opf.Metadata.Publisher[0]
}
// Language
if len(opf.Metadata.Language) > 0 && opf.Metadata.Language[0] != "" {
metadata.Language = opf.Metadata.Language[0]
}
// Publish date
if len(opf.Metadata.Dates) > 0 {
if date, err := time.Parse("2006-01-02T15:04:05Z07:00", opf.Metadata.Dates[0]); err == nil {
@@ -1681,14 +1646,23 @@ func (s *MediaScanner) parseCalibreMetadataOPF(opfPath string) (*MediaMetadata,
}
// Identifiers (ISBN, ASIN)
for _, id := range opf.Metadata.Identifiers {
value := strings.TrimSpace(id.Value)
switch strings.ToUpper(id.Scheme) {
case "ISBN":
metadata.ISBN = utils.NormalizeISBNSafe(id.Value)
metadata.ISBN = utils.NormalizeISBNSafe(value)
case "ASIN":
metadata.ASIN = id.Value
metadata.ASIN = value
case "UUID", "CALIBRE":
// Store UUID in hash info, not metadata
// Will be extracted by extractHashInfo()
default:
// EPUB3 identifiers often carry no opf:scheme attribute;
// accept a bare value that normalizes to a valid ISBN.
if metadata.ISBN == "" && id.Scheme == "" {
if normalized := utils.NormalizeISBNSafe(value); normalized != "" {
metadata.ISBN = normalized
}
}
}
}
// Contributors
@@ -1715,6 +1689,45 @@ func (s *MediaScanner) parseCalibreMetadataOPF(opfPath string) (*MediaMetadata,
return metadata, nil
}
// findOPFPathInZip locates the OPF document inside an EPUB by reading
// META-INF/container.xml (string-scraped; we only need the rootfile
// full-path attribute). Returns "" when absent.
func findOPFPathInZip(files []*zip.File) string {
for _, f := range files {
if f.Name != "META-INF/container.xml" {
continue
}
rc, err := f.Open()
if err != nil {
return ""
}
content, readErr := io.ReadAll(rc)
if closeErr := rc.Close(); closeErr != nil {
fmt.Printf("Warning: failed to close META-INF/container.xml reader: %v\n", closeErr)
}
if readErr != nil {
return ""
}
opfStart := bytes.Index(content, []byte("<rootfile "))
if opfStart == -1 {
return ""
}
opfStartAttr := bytes.Index(content[opfStart:], []byte("full-path="))
if opfStartAttr == -1 {
return ""
}
opfStartAttr += len("full-path=")
quote := content[opfStart+opfStartAttr]
opfStartQuote := opfStart + opfStartAttr + 1
opfEndQuote := bytes.Index(content[opfStartQuote:], []byte{quote})
if opfEndQuote == -1 {
return ""
}
return string(content[opfStartQuote : opfStartQuote+opfEndQuote])
}
return ""
}
// extractEPUBCover extracts the cover image from an EPUB file.
// It looks for:
// 1. An item with properties="cover-image" in the manifest
@@ -1736,43 +1749,7 @@ func (s *MediaScanner) extractEPUBCover(epubPath string) (string, error) {
// Try to find cover image from OPF metadata
coverImageName := ""
// Attempt to read the OPF file to find cover reference
// First, find container.xml to locate the OPF
var opfPath string
for _, f := range r.File {
if f.Name == "META-INF/container.xml" {
rc, err := f.Open()
if err != nil {
continue
}
content, readErr := io.ReadAll(rc)
if closeErr := rc.Close(); closeErr != nil {
fmt.Printf("Warning: failed to close META-INF/container.xml reader in %s: %v\n", epubPath, closeErr)
}
if readErr != nil {
continue
}
// Parse container.xml to find OPF path
// Simple string search since we just need the path
opfStart := bytes.Index(content, []byte("<rootfile "))
if opfStart == -1 {
continue
}
opfStartAttr := bytes.Index(content[opfStart:], []byte("full-path="))
if opfStartAttr == -1 {
continue
}
opfStartAttr += len("full-path=")
quote := content[opfStart+opfStartAttr]
opfStartQuote := opfStart + opfStartAttr + 1
opfEndQuote := bytes.Index(content[opfStartQuote:], []byte{quote})
if opfEndQuote == -1 {
continue
}
opfPath = string(content[opfStartQuote : opfStartQuote+opfEndQuote])
break
}
}
opfPath := findOPFPathInZip(r.File)
if opfPath == "" {
// No OPF found, try common cover image paths
internal/services/media_scanner_metadata_test.go
+154
View File
@@ -0,0 +1,154 @@
package services
import (
"archive/zip"
"bytes"
"image"
"image/jpeg"
"os"
"path/filepath"
"testing"
"time"
)
// createTestJPEGBytes returns the bytes of a minimal valid JPEG.
func createTestJPEGBytes() string {
var buf bytes.Buffer
img := image.NewRGBA(image.Rect(0, 0, 1, 1))
if err := jpeg.Encode(&buf, img, nil); err != nil {
return ""
}
return buf.String()
}
// createPragmaticStyleEPUB builds an EPUB modeled on Pragmatic Bookshelf
// output: the dc namespace declared on the <metadata> element (not on
// <package>), a scheme-less ISBN identifier, an OPF-declared cover, and a
// deliberately malformed chapter body. The malformed chapter is the
// regression trigger: the previous go-epub-based extractor failed the whole
// book when any chapter was unparseable and wrote blank metadata.
func createPragmaticStyleEPUB(epubPath string) error {
file, err := os.Create(epubPath)
if err != nil {
return err
}
defer file.Close()
zipWriter := zip.NewWriter(file)
defer zipWriter.Close()
mimetypeW, err := zipWriter.CreateHeader(&zip.FileHeader{
Name: "mimetype",
Method: zip.Store,
})
if err != nil {
return err
}
mimetypeW.Write([]byte("application/epub+zip"))
files := map[string]string{
"META-INF/container.xml": `<?xml version="1.0"?><container version="1.0" xmlns="urn:oasis:names:tc:opendocument:xmlns:container"><rootfiles><rootfile full-path="OEBPS/content.opf" media-type="application/oebps-package+xml"/></rootfiles></container>`,
// dc namespace declared on <metadata>; identifiers carry no scheme attr
"OEBPS/content.opf": `<?xml version="1.0" encoding="UTF-8"?>
<package xmlns="http://www.idpf.org/2007/opf" version="3.0" unique-identifier="PubID">
<metadata xmlns:dc="http://purl.org/dc/elements/1.1/">
<dc:language>en</dc:language>
<dc:title>A Common-Sense Guide</dc:title>
<dc:creator>Jay Wengrow</dc:creator>
<dc:publisher>The Pragmatic Bookshelf, LLC</dc:publisher>
<dc:description>Content that makes you a better programmer.</dc:description>
<dc:subject>Programming</dc:subject>
<dc:identifier id="PubID">978-1-68050-722-8</dc:identifier>
<meta name="cover" content="cover-image"/>
</metadata>
<manifest>
<item id="cover-image" href="images/cover.jpg" media-type="image/jpeg"/>
<item id="ch1" href="ch1.xhtml" media-type="application/xhtml+xml"/>
</manifest>
<spine><itemref idref="ch1"/></spine>
</package>`,
// Malformed on purpose: unclosed tags
"OEBPS/ch1.xhtml": `<html><body><p>unclosed paragraph`,
"OEBPS/images/cover.jpg": createTestJPEGBytes(),
}
for name, content := range files {
w, err := zipWriter.Create(name)
if err != nil {
return err
}
if _, err := w.Write([]byte(content)); err != nil {
return err
}
}
return zipWriter.Close()
}
// TestExtractEPUBMetadataBrokenChapter guards the regression where one
// unparseable chapter made the extractor return nothing at all: metadata must
// come from the OPF regardless of chapter-body damage.
func TestExtractEPUBMetadataBrokenChapter(t *testing.T) {
tmpDir := t.TempDir()
epubPath := filepath.Join(tmpDir, "book.epub")
if err := createPragmaticStyleEPUB(epubPath); err != nil {
t.Fatalf("failed to create test EPUB: %v", err)
}
s := NewMediaScanner(nil)
metadata, err := s.extractEPUBMetadata(epubPath)
if err != nil {
t.Fatalf("extractEPUBMetadata() error: %v", err)
}
if metadata.Title != "A Common-Sense Guide" {
t.Errorf("Title = %q, want %q", metadata.Title, "A Common-Sense Guide")
}
if metadata.Author != "Jay Wengrow" {
t.Errorf("Author = %q, want %q", metadata.Author, "Jay Wengrow")
}
if metadata.Publisher != "The Pragmatic Bookshelf, LLC" {
t.Errorf("Publisher = %q, want %q", metadata.Publisher, "The Pragmatic Bookshelf, LLC")
}
if metadata.Description == "" {
t.Error("Description missing")
}
if metadata.Language != "en" {
t.Errorf("Language = %q, want %q", metadata.Language, "en")
}
// Scheme-less identifier that normalizes to a valid ISBN must be picked up
if metadata.ISBN == "" {
t.Error("ISBN missing (scheme-less dc:identifier fallback failed)")
}
}
func TestParseOPFContentCalibreSeries(t *testing.T) {
opf := `<?xml version="1.0"?>
<package xmlns="http://www.idpf.org/2007/opf" version="2.0" unique-identifier="id">
<metadata xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:opf="http://www.idpf.org/2007/opf">
<dc:title>Test Book</dc:title>
<dc:creator>Some Author</dc:creator>
<dc:date>2020-03-15</dc:date>
<dc:subject>Fiction</dc:subject>
<dc:subject>Classic</dc:subject>
<dc:identifier opf:scheme="ISBN">978-3-16-148410-0</dc:identifier>
<meta name="calibre:series" content="Great Series"/>
<meta name="calibre:series_index" content="2.5"/>
</metadata>
</package>`
metadata, err := parseOPFContent([]byte(opf))
if err != nil {
t.Fatalf("parseOPFContent() error: %v", err)
}
if metadata.Series != "Great Series" || metadata.SeriesNumber != 2 {
t.Errorf("Series = %q/%d, want Great Series/2", metadata.Series, metadata.SeriesNumber)
}
if metadata.ISBN == "" {
t.Error("schemed ISBN not extracted")
}
wantDate := time.Date(2020, 3, 15, 0, 0, 0, 0, time.UTC)
if !metadata.PublishDate.Equal(wantDate) {
t.Errorf("PublishDate = %v, want %v", metadata.PublishDate, wantDate)
}
if len(metadata.Tags) != 2 {
t.Errorf("Tags = %v, want 2 subjects", metadata.Tags)
}
}