From 14947608bee0f9d144bd52d577e19edf7c653b14 Mon Sep 17 00:00:00 2001 From: John O'Keefe Date: Mon, 12 Oct 2026 20:46:40 -0400 Subject: [PATCH] chore(scripts): add synthetic torture EPUB generator scripts/build_torture_epub.py deterministically builds a small EPUB exercising every CRE-xpointer to CFI conversion hazard observed in the wild, one chapter per hazard class: multi-text-node paragraphs (verse spans, inline markup), whitespace-only inline spans and comments, drop caps and footnote anchors, br-separated poetry with speaker and line-number spans, nested blockquotes/lists/tables/definition lists, surrogate pairs and ZWJ emoji, Hebrew niqqud, Arabic RTL, combining diacritics, invisible characters, HTML entities, CJK ruby, a 400- paragraph chapter with heavy phrase repetition for search disambiguation, and edge constructs (span-first paragraphs, empty spans, adjacent same-tag spans, deep same-tag nesting). Default output lands in the Ebooks library for real-device testing via OPDS; the script takes an output path argument for use as a test fixture. --- scripts/build_torture_epub.py | 220 ++++++++++++++++++++++++++++++++++ 1 file changed, 220 insertions(+) create mode 100644 scripts/build_torture_epub.py diff --git a/scripts/build_torture_epub.py b/scripts/build_torture_epub.py new file mode 100644 index 00000000..534d5b1d --- /dev/null +++ b/scripts/build_torture_epub.py @@ -0,0 +1,220 @@ +#!/usr/bin/env python3 +"""Build a synthetic 'torture test' EPUB exercising every CRE-xpointer -> EPUB-CFI +conversion hazard the server converter can encounter. + +Each chapter targets one hazard class; constructs carry comments naming the +hazard. Output: uploads/Ebooks/Bookhoard Test/CRE CFI Torture Test/CRE CFI Torture Test.epub +(or argv[1]). Deterministic output for reproducible tests. +""" +import os +import sys +import zipfile + +MIMETYPE = "application/epub+zip" + +CONTAINER = """ + + + + + +""" + +def page(name, title, body): + return f""" + + +{title} + + +

{title}

+{body} + + +""" + +CHAPTERS = [] + +# 1. The core bug shape: multi-text-node paragraphs (inline markup splitting text) +CHAPTERS.append(("split-text", "Split Text Nodes", """ +

In the beginning 1 God created the heaven and the earth 2 and the earth was without form.

+

A single sentence with several inline marked spans inside it.

+

First text node here.Immediately-following span text.Text that continues after the span.And a second span.Tail text node.

+

Quoted "double" and 'single' marks, an ampersand & entity, and a unicode em—dash.

+
+

Nested group paragraph one 1 with more text after the marker.

+

Nested group paragraph two, plain text only.

+
+""")) + +# 2. Whitespace-only text nodes + comments between elements (sibling-index drift) +CHAPTERS.append(("whitespace-comments", "Whitespace and Comments", """ + +

Tail paragraph after the list.

+""")) + +# 3. Drop caps + footnote anchors (single-char inline splits) +CHAPTERS.append(("dropcaps-footnotes", "Dropcaps and Footnotes", """ +

Once upon a time a drop cap split the first word from itself.

+

A claim requiring support1 and a second claim2 in the same paragraph.

+

Multiple internal links interleaved through text as anchors.

+
+

1. The first footnote, body text follows the backlink.

+

2. The second footnote with emphasis inside.

+
+""")) + +# 4. Poetry / drama: br-separated lines, speaker spans, line numbers +CHAPTERS.append(("poetry", "Poetry and Drama", """ +

HAMLET. To be, or not to be, that is the question:
Whether 'tis nobler in the mind to suffer
The slings and arrows of outrageous fortune.

+

1The curfew tolls the knell of parting day,
2The lowing herd wind slowly o'er the lea,
3The plowman homeward plods his weary way,
4And leaves the world to darkness and to me.

+""")) + +# 5. Deep nesting: quotes in quotes, lists in lists, tables with inline markup +CHAPTERS.append(("nested", "Nested Structures", """ +

Outer quotation text begins here 1 and continues.

+

Inner quotation, one level deeper with emphasis mid-sentence.

+

Outer quotation resumes after the inner block.

+
+ + + + +
Cell with inline markupSecond cell, plain
Cell three with a 7 markerCell four
+
Term one
Definition one with text.
Term two
Definition two with text.
+""")) + +# 6. Unicode hazards: surrogate pairs, combining marks, RTL, invisible chars, entities +CHAPTERS.append(("unicode", "Unicode Hazards", """ +

Emoji with surrogate pairs: 🎉🚀 and a ZWJ family 👨‍👩‍👧‍👦 plus math 𝛑𝕏 and flags 🇺🇸🇯🇵.

+

Hebrew with niqqud: בְּרֵאשִׁית בָּרָא אֱלֹהִים and a Midrash on דָּבָר מִשְׁפָּט אֱמֶת.

+

Arabic RTL with embedded LTR: يبدأ النص العربي هنا ثم 42 ثم ينتهي، مع تشكيل الكلمات المُعرَبة.

+

Decomposed e + combining acute: é vs precomposed é, and ĉ vs ĉ with combining circumflex.

+

Soft hyphen in­side a word, zero thin space, zero-width ​space, and trailing no-break space here.

+

Ampersand &, less <, greater >, apostrophe ', quote ", all rendered literally.

+""")) + +# 7. Ruby / CJK +CHAPTERS.append(("ruby", "Ruby Annotations", """ +

漢字かんじの読よみ方をテストする。

+

日本語のテキストには難むずかしい漢字かんじがたくさんあります。

+

The ruby base 漢 and its annotation かん are separate text nodes in raw XHTML.

+""")) + +# 8. Giant chapter with heavy phrase repetition (text-search disambiguation) +giant = ['

Opening paragraph of the giant chapter, unique opening sentence.

'] +for i in range(1, 401): + giant.append( + f'

Verse {i} And it came to pass in those days that a repeated phrase appeared again among the people. ' + f'Variation marker alpha-{i}: the sentence continues with unique tail number {i * 7} before ending.

' + ) +CHAPTERS.append(("giant", "Giant Repeated Chapter", "\n".join(giant))) + +# 9. Edge constructs: empties, same-tag adjacency, span-first paragraphs, deep same-tag nesting +deep = "

Deep nesting bottom text.

" +for lvl in range(10, 0, -1): + deep = f'
{deep}
' +CHAPTERS.append(("edge", "Edge Constructs", f""" +

Leading span before any direct text. Direct text follows the span.Tail span.

+

Adjacentsame-tagspans then plain text.

+

Text with an empty span mid-sentence and an empty emphasis after it.

+

+
{deep}
+

Paragraph with odd attribute values like "quotes" and 'apostrophes'.

+""")) + +CSS = """body { margin: 1em; } +.verse, .lnum, .fnref { font-size: 0.7em; vertical-align: super; color: #666; } +.speaker { font-variant: small-caps; } +.dropcap { font-size: 2.5em; font-weight: bold; float: left; line-height: 1; } +.line { display: block; margin-left: 2em; } +blockquote { margin: 1em 2em; border-left: 2px solid #ccc; padding-left: 1em; } +ruby rt { font-size: 0.5em; } +""" + +def build(out_path): + files = {} + files["mimetype"] = MIMETYPE + files["META-INF/container.xml"] = CONTAINER + files["css/style.css"] = CSS + + manifests, spines = [], [] + for idx, (name, title, body) in enumerate(CHAPTERS, start=1): + files[f"OPS/chapters/{name}.xhtml"] = page(name, title, body) + manifests.append(f' ') + spines.append(f' ') + + nav_items = "\n".join( + f'
  • {title}
    • ' + for name, title, _ in CHAPTERS + ) + files["OPS/nav.xhtml"] = f""" + + +Navigation + + + + +""" + nav_manifest = ' ' + files["OPS/content.opf"] = f""" + + + urn:uuid:1f2e3d4c-5b6a-7948-b7c6-d5e4f3a2b1c0 + CRE CFI Torture Test + Bookhoard Test + en + 2026-10-12T00:00:00Z + + +{nav_manifest} +{chr(10).join(manifests)} + + +{chr(10).join(spines)} + + +""" + + os.makedirs(os.path.dirname(out_path), exist_ok=True) + with zipfile.ZipFile(out_path, "w") as z: + z.writestr(zipfile.ZipInfo("mimetype"), MIMETYPE, compress_type=zipfile.ZIP_STORED) + for arcname in sorted(files): + if arcname == "mimetype": + continue + z.writestr(arcname, files[arcname], compress_type=zipfile.ZIP_DEFLATED) + print(f"wrote {out_path} ({os.path.getsize(out_path):,} bytes, {len(CHAPTERS)} chapters)") + +if __name__ == "__main__": + out = sys.argv[1] if len(sys.argv) > 1 else os.path.join( + os.path.dirname(__file__), "..", + "uploads/Ebooks/Bookhoard Test/CRE CFI Torture Test/CRE CFI Torture Test.epub") + build(os.path.normpath(out))