chore(scripts): add synthetic torture EPUB generator
scripts/build_torture_epub.py deterministically builds a small EPUB exercising every CRE-xpointer to CFI conversion hazard observed in the wild, one chapter per hazard class: multi-text-node paragraphs (verse spans, inline markup), whitespace-only inline spans and comments, drop caps and footnote anchors, br-separated poetry with speaker and line-number spans, nested blockquotes/lists/tables/definition lists, surrogate pairs and ZWJ emoji, Hebrew niqqud, Arabic RTL, combining diacritics, invisible characters, HTML entities, CJK ruby, a 400- paragraph chapter with heavy phrase repetition for search disambiguation, and edge constructs (span-first paragraphs, empty spans, adjacent same-tag spans, deep same-tag nesting). Default output lands in the Ebooks library for real-device testing via OPDS; the script takes an output path argument for use as a test fixture.
This commit is contained in:
John O'Keefe committed
1 parent
03a6040360
commit
14947608be
1 file changed
+220
@@ -0,0 +1,220 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Build a synthetic 'torture test' EPUB exercising every CRE-xpointer -> EPUB-CFI
|
||||
conversion hazard the server converter can encounter.
|
||||
|
||||
Each chapter targets one hazard class; constructs carry comments naming the
|
||||
hazard. Output: uploads/Ebooks/Bookhoard Test/CRE CFI Torture Test/CRE CFI Torture Test.epub
|
||||
(or argv[1]). Deterministic output for reproducible tests.
|
||||
"""
|
||||
import os
|
||||
import sys
|
||||
import zipfile
|
||||
|
||||
MIMETYPE = "application/epub+zip"
|
||||
|
||||
CONTAINER = """<?xml version="1.0" encoding="UTF-8"?>
|
||||
<container version="1.0" xmlns="urn:oasis:names:tc:opendocument:xmlns:container">
|
||||
<rootfiles>
|
||||
<rootfile full-path="OPS/content.opf" media-type="application/oebps-package+xml"/>
|
||||
</rootfiles>
|
||||
</container>
|
||||
"""
|
||||
|
||||
def page(name, title, body):
|
||||
return f"""<?xml version="1.0" encoding="UTF-8"?>
|
||||
<!DOCTYPE html>
|
||||
<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
|
||||
<head><title>{title}</title><link rel="stylesheet" type="text/css" href="../css/style.css"/></head>
|
||||
<body>
|
||||
<!-- chapter: {name} -->
|
||||
<h1 id="{name}-h">{title}</h1>
|
||||
{body}
|
||||
</body>
|
||||
</html>
|
||||
"""
|
||||
|
||||
CHAPTERS = []
|
||||
|
||||
# 1. The core bug shape: multi-text-node paragraphs (inline markup splitting text)
|
||||
CHAPTERS.append(("split-text", "Split Text Nodes", """
|
||||
<p id="st-p1">In the beginning <span class="verse">1</span> God created the heaven and the earth <span class="verse">2</span> and the earth was without form.</p>
|
||||
<p id="st-p2">A <em>single</em> sentence with <strong>several</strong> inline <span class="mark">marked</span> spans inside it.</p>
|
||||
<p id="st-p3">First text node here.<span class="cont">Immediately-following span text.</span>Text that continues after the span.<span class="cont2">And a second span.</span>Tail text node.</p>
|
||||
<p id="st-p4">Quoted "double" and 'single' marks, an ampersand & entity, and a unicode em—dash.</p>
|
||||
<div class="group" id="st-group">
|
||||
<p id="st-p5">Nested group paragraph one <span class="verse">1</span> with more text after the marker.</p>
|
||||
<p id="st-p6">Nested group paragraph two, plain text only.</p>
|
||||
</div>
|
||||
"""))
|
||||
|
||||
# 2. Whitespace-only text nodes + comments between elements (sibling-index drift)
|
||||
CHAPTERS.append(("whitespace-comments", "Whitespace and Comments", """
|
||||
<ul id="wc-list">
|
||||
<!-- comment directly inside list -->
|
||||
<li id="wc-li1">
|
||||
<p id="wc-p1">Indented markup with whitespace text nodes on every side.</p>
|
||||
<!-- comment between siblings -->
|
||||
<p id="wc-p2">Second indented paragraph with irregular internal spacing.</p>
|
||||
</li>
|
||||
<!-- another comment -->
|
||||
<li id="wc-li2">
|
||||
<p id="wc-p3">Paragraph after multiple comment and whitespace siblings.</p>
|
||||
</li>
|
||||
</ul>
|
||||
<p id="wc-p4">Tail paragraph after the list.</p>
|
||||
"""))
|
||||
|
||||
# 3. Drop caps + footnote anchors (single-char inline splits)
|
||||
CHAPTERS.append(("dropcaps-footnotes", "Dropcaps and Footnotes", """
|
||||
<p id="df-p1"><span class="dropcap">O</span>nce upon a time a drop cap split the first word from itself.</p>
|
||||
<p id="df-p2">A claim requiring support<span class="fnref"><a id="fnref-1" href="#fn-1">1</a></span> and a second claim<span class="fnref"><a id="fnref-2" href="#fn-2">2</a></span> in the same paragraph.</p>
|
||||
<p id="df-p3">Multiple <a href="#df-p1">internal links</a> interleaved <a href="#df-p2">through text</a> as anchors.</p>
|
||||
<div class="footnotes" id="df-notes">
|
||||
<p id="df-fn1"><a id="fn-1" href="#fnref-1">1.</a> The first footnote, body text follows the backlink.</p>
|
||||
<p id="df-fn2"><a id="fn-2" href="#fnref-2">2.</a> The second footnote with <em>emphasis</em> inside.</p>
|
||||
</div>
|
||||
"""))
|
||||
|
||||
# 4. Poetry / drama: br-separated lines, speaker spans, line numbers
|
||||
CHAPTERS.append(("poetry", "Poetry and Drama", """
|
||||
<p id="po-dramap"><span class="speaker">HAMLET.</span> To be, or not to be, that is the question:<br/>Whether 'tis nobler in the mind to suffer<br/>The slings and arrows of outrageous fortune.</p>
|
||||
<p id="po-verse1"><span class="line"><span class="lnum">1</span>The curfew tolls the knell of parting day,</span><br/><span class="line"><span class="lnum">2</span>The lowing herd wind slowly o'er the lea,</span><br/><span class="line"><span class="lnum">3</span>The plowman homeward plods his weary way,</span><br/><span class="line"><span class="lnum">4</span>And leaves the world to darkness and to me.</span></p>
|
||||
"""))
|
||||
|
||||
# 5. Deep nesting: quotes in quotes, lists in lists, tables with inline markup
|
||||
CHAPTERS.append(("nested", "Nested Structures", """
|
||||
<blockquote id="ne-q1"><p id="ne-q1p1">Outer quotation text begins here <span class="verse">1</span> and continues.</p>
|
||||
<blockquote id="ne-q2"><p id="ne-q2p1">Inner quotation, one level deeper <em>with emphasis</em> mid-sentence.</p></blockquote>
|
||||
<p id="ne-q1p2">Outer quotation resumes after the inner block.</p>
|
||||
</blockquote>
|
||||
<ul id="ne-list1">
|
||||
<li id="ne-li1">Level one item
|
||||
<ul id="ne-list2">
|
||||
<li id="ne-li2">Level two item
|
||||
<ul id="ne-list3">
|
||||
<li id="ne-li3">Level three item with <strong>bold</strong> text.</li>
|
||||
<li id="ne-li4">Level three sibling item.</li>
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
</li>
|
||||
<li id="ne-li5">Level one sibling item.</li>
|
||||
</ul>
|
||||
<table id="ne-table">
|
||||
<tr id="ne-tr1"><td id="ne-td1">Cell with <em>inline</em> markup</td><td id="ne-td2">Second cell, plain</td></tr>
|
||||
<tr id="ne-tr2"><td id="ne-td3">Cell three with a <span class="verse">7</span> marker</td><td id="ne-td4">Cell four</td></tr>
|
||||
</table>
|
||||
<dl id="ne-dl"><dt id="ne-dt1">Term one</dt><dd id="ne-dd1">Definition one with text.</dd><dt id="ne-dt2">Term two</dt><dd id="ne-dd2">Definition two with text.</dd></dl>
|
||||
"""))
|
||||
|
||||
# 6. Unicode hazards: surrogate pairs, combining marks, RTL, invisible chars, entities
|
||||
CHAPTERS.append(("unicode", "Unicode Hazards", """
|
||||
<p id="un-emoji">Emoji with surrogate pairs: 🎉🚀 and a ZWJ family 👨👩👧👦 plus math 𝛑𝕏 and flags 🇺🇸🇯🇵.</p>
|
||||
<p id="un-hebrew">Hebrew with niqqud: בְּרֵאשִׁית בָּרָא אֱלֹהִים and a Midrash on דָּבָר מִשְׁפָּט אֱמֶת.</p>
|
||||
<p id="un-arabic">Arabic RTL with embedded LTR: يبدأ النص العربي هنا ثم 42 ثم ينتهي، مع تشكيل الكلمات المُعرَبة.</p>
|
||||
<p id="un-combining">Decomposed e + combining acute: é vs precomposed é, and ĉ vs ĉ with combining circumflex.</p>
|
||||
<p id="un-invisible">Soft hyphen inside a word, zero thin space, zero-width space, and trailing no-break space here.</p>
|
||||
<p id="un-entities">Ampersand &amp;, less &lt;, greater &gt;, apostrophe &#39;, quote &quot;, all rendered literally.</p>
|
||||
"""))
|
||||
|
||||
# 7. Ruby / CJK
|
||||
CHAPTERS.append(("ruby", "Ruby Annotations", """
|
||||
<p id="ru-p1"><ruby>漢字<rt>かんじ</rt></ruby>の<ruby>読<rt>よ</rt></ruby>み<span>方</span>をテストする。</p>
|
||||
<p id="ru-p2">日本語のテキストには<ruby>難<rt>むずか</rt></ruby>しい<ruby>漢字<rt>かんじ</rt></ruby>がたくさんあります。</p>
|
||||
<p id="ru-p3">The ruby base 漢 and its annotation かん are separate text nodes in raw XHTML.</p>
|
||||
"""))
|
||||
|
||||
# 8. Giant chapter with heavy phrase repetition (text-search disambiguation)
|
||||
giant = ['<p id="gi-p0">Opening paragraph of the giant chapter, unique opening sentence.</p>']
|
||||
for i in range(1, 401):
|
||||
giant.append(
|
||||
f'<p id="gi-p{i}">Verse <span class="verse">{i}</span> And it came to pass in those days that a repeated phrase appeared again among the people. '
|
||||
f'Variation marker alpha-{i}: the sentence continues with unique tail number {i * 7} before ending.</p>'
|
||||
)
|
||||
CHAPTERS.append(("giant", "Giant Repeated Chapter", "\n".join(giant)))
|
||||
|
||||
# 9. Edge constructs: empties, same-tag adjacency, span-first paragraphs, deep same-tag nesting
|
||||
deep = "<p id=\"ed-deep\">Deep nesting bottom text.</p>"
|
||||
for lvl in range(10, 0, -1):
|
||||
deep = f'<div id="ed-lvl{lvl}">{deep}</div>'
|
||||
CHAPTERS.append(("edge", "Edge Constructs", f"""
|
||||
<p id="ed-spanfirst"><span class="lead">Leading span before any direct text.</span> Direct text follows the span.<span class="tail">Tail span.</span></p>
|
||||
<p id="ed-adjacent"><span class="a">Adjacent</span><span class="b">same-tag</span><span class="c">spans</span> then plain text.</p>
|
||||
<p id="ed-empty">Text with an empty span <span></span> mid-sentence and an empty emphasis <em></em> after it.</p>
|
||||
<p id="ed-emptyp"></p>
|
||||
<div id="ed-onlychild">{deep}</div>
|
||||
<p id="ed-attrs" data-why="not">Paragraph with odd attribute values like "quotes" and 'apostrophes'.</p>
|
||||
"""))
|
||||
|
||||
CSS = """body { margin: 1em; }
|
||||
.verse, .lnum, .fnref { font-size: 0.7em; vertical-align: super; color: #666; }
|
||||
.speaker { font-variant: small-caps; }
|
||||
.dropcap { font-size: 2.5em; font-weight: bold; float: left; line-height: 1; }
|
||||
.line { display: block; margin-left: 2em; }
|
||||
blockquote { margin: 1em 2em; border-left: 2px solid #ccc; padding-left: 1em; }
|
||||
ruby rt { font-size: 0.5em; }
|
||||
"""
|
||||
|
||||
def build(out_path):
|
||||
files = {}
|
||||
files["mimetype"] = MIMETYPE
|
||||
files["META-INF/container.xml"] = CONTAINER
|
||||
files["css/style.css"] = CSS
|
||||
|
||||
manifests, spines = [], []
|
||||
for idx, (name, title, body) in enumerate(CHAPTERS, start=1):
|
||||
files[f"OPS/chapters/{name}.xhtml"] = page(name, title, body)
|
||||
manifests.append(f' <item id="ch{idx}" href="chapters/{name}.xhtml" media-type="application/xhtml+xml"/>')
|
||||
spines.append(f' <itemref idref="ch{idx}"/>')
|
||||
|
||||
nav_items = "\n".join(
|
||||
f' <li><a href="chapters/{name}.xhtml">{title}</a></li>'
|
||||
for name, title, _ in CHAPTERS
|
||||
)
|
||||
files["OPS/nav.xhtml"] = f"""<?xml version="1.0" encoding="UTF-8"?>
|
||||
<!DOCTYPE html>
|
||||
<html xmlns="http://www.w3.org/1999/xhtml" xmlns:epub="http://www.idpf.org/2007/ops">
|
||||
<head><title>Navigation</title></head>
|
||||
<body>
|
||||
<nav epub:type="toc" id="toc"><h1>Contents</h1>
|
||||
<ol>
|
||||
{nav_items}
|
||||
</ol>
|
||||
</nav>
|
||||
</body>
|
||||
</html>
|
||||
"""
|
||||
nav_manifest = ' <item id="nav" href="nav.xhtml" media-type="application/xhtml+xml" properties="nav"/>'
|
||||
files["OPS/content.opf"] = f"""<?xml version="1.0" encoding="UTF-8"?>
|
||||
<package xmlns="http://www.idpf.org/2007/opf" version="3.0" unique-identifier="uid">
|
||||
<metadata xmlns:dc="http://purl.org/dc/elements/1.1/">
|
||||
<dc:identifier id="uid">urn:uuid:1f2e3d4c-5b6a-7948-b7c6-d5e4f3a2b1c0</dc:identifier>
|
||||
<dc:title>CRE CFI Torture Test</dc:title>
|
||||
<dc:creator>Bookhoard Test</dc:creator>
|
||||
<dc:language>en</dc:language>
|
||||
<meta property="dcterms:modified">2026-10-12T00:00:00Z</meta>
|
||||
</metadata>
|
||||
<manifest>
|
||||
{nav_manifest}
|
||||
{chr(10).join(manifests)}
|
||||
</manifest>
|
||||
<spine>
|
||||
{chr(10).join(spines)}
|
||||
</spine>
|
||||
</package>
|
||||
"""
|
||||
|
||||
os.makedirs(os.path.dirname(out_path), exist_ok=True)
|
||||
with zipfile.ZipFile(out_path, "w") as z:
|
||||
z.writestr(zipfile.ZipInfo("mimetype"), MIMETYPE, compress_type=zipfile.ZIP_STORED)
|
||||
for arcname in sorted(files):
|
||||
if arcname == "mimetype":
|
||||
continue
|
||||
z.writestr(arcname, files[arcname], compress_type=zipfile.ZIP_DEFLATED)
|
||||
print(f"wrote {out_path} ({os.path.getsize(out_path):,} bytes, {len(CHAPTERS)} chapters)")
|
||||
|
||||
if __name__ == "__main__":
|
||||
out = sys.argv[1] if len(sys.argv) > 1 else os.path.join(
|
||||
os.path.dirname(__file__), "..",
|
||||
"uploads/Ebooks/Bookhoard Test/CRE CFI Torture Test/CRE CFI Torture Test.epub")
|
||||
build(os.path.normpath(out))
|
||||
Reference in new issue
Block a user