bookhoard/scripts/build_torture_epub.py

#!/usr/bin/env python3
"""Build a synthetic 'torture test' EPUB exercising every CRE-xpointer -> EPUB-CFI
conversion hazard the server converter can encounter.

Each chapter targets one hazard class; constructs carry comments naming the
hazard. Output: uploads/Ebooks/Bookhoard Test/CRE CFI Torture Test/CRE CFI Torture Test.epub
(or argv[1]). Deterministic output for reproducible tests.
"""
import os
import sys
import zipfile

MIMETYPE = "application/epub+zip"

CONTAINER = """<?xml version="1.0" encoding="UTF-8"?>
<container version="1.0" xmlns="urn:oasis:names:tc:opendocument:xmlns:container">
  <rootfiles>
    <rootfile full-path="OPS/content.opf" media-type="application/oebps-package+xml"/>
  </rootfiles>
</container>
"""

def page(name, title, body):
    return f"""<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE html>
<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
<head><title>{title}</title><link rel="stylesheet" type="text/css" href="../css/style.css"/></head>
<body>
<!-- chapter: {name} -->
<h1 id="{name}-h">{title}</h1>
{body}
</body>
</html>
"""

CHAPTERS = []

# 1. The core bug shape: multi-text-node paragraphs (inline markup splitting text)
CHAPTERS.append(("split-text", "Split Text Nodes", """
<p id="st-p1">In the beginning <span class="verse">1</span> God created the heaven and the earth <span class="verse">2</span> and the earth was without form.</p>
<p id="st-p2">A <em>single</em> sentence with <strong>several</strong> inline <span class="mark">marked</span> spans inside it.</p>
<p id="st-p3">First text node here.<span class="cont">Immediately-following span text.</span>Text that continues after the span.<span class="cont2">And a second span.</span>Tail text node.</p>
<p id="st-p4">Quoted "double" and 'single' marks, an ampersand &amp; entity, and a unicode em—dash.</p>
<div class="group" id="st-group">
  <p id="st-p5">Nested group paragraph one <span class="verse">1</span> with more text after the marker.</p>
  <p id="st-p6">Nested group paragraph two, plain text only.</p>
</div>
"""))

# 2. Whitespace-only text nodes + comments between elements (sibling-index drift)
CHAPTERS.append(("whitespace-comments", "Whitespace and Comments", """
<ul id="wc-list">
  <!-- comment directly inside list -->
  <li id="wc-li1">
    <p id="wc-p1">Indented markup with whitespace text nodes on every side.</p>
    <!-- comment between siblings -->
    <p id="wc-p2">Second indented   paragraph with   irregular internal    spacing.</p>
  </li>
  <!-- another comment -->
  <li id="wc-li2">
    <p id="wc-p3">Paragraph after multiple comment and whitespace siblings.</p>
  </li>
</ul>
<p id="wc-p4">Tail paragraph after the list.</p>
"""))

# 3. Drop caps + footnote anchors (single-char inline splits)
CHAPTERS.append(("dropcaps-footnotes", "Dropcaps and Footnotes", """
<p id="df-p1"><span class="dropcap">O</span>nce upon a time a drop cap split the first word from itself.</p>
<p id="df-p2">A claim requiring support<span class="fnref"><a id="fnref-1" href="#fn-1">1</a></span> and a second claim<span class="fnref"><a id="fnref-2" href="#fn-2">2</a></span> in the same paragraph.</p>
<p id="df-p3">Multiple <a href="#df-p1">internal links</a> interleaved <a href="#df-p2">through text</a> as anchors.</p>
<div class="footnotes" id="df-notes">
  <p id="df-fn1"><a id="fn-1" href="#fnref-1">1.</a> The first footnote, body text follows the backlink.</p>
  <p id="df-fn2"><a id="fn-2" href="#fnref-2">2.</a> The second footnote with <em>emphasis</em> inside.</p>
</div>
"""))

# 4. Poetry / drama: br-separated lines, speaker spans, line numbers
CHAPTERS.append(("poetry", "Poetry and Drama", """
<p id="po-dramap"><span class="speaker">HAMLET.</span> To be, or not to be, that is the question:<br/>Whether 'tis nobler in the mind to suffer<br/>The slings and arrows of outrageous fortune.</p>
<p id="po-verse1"><span class="line"><span class="lnum">1</span>The curfew tolls the knell of parting day,</span><br/><span class="line"><span class="lnum">2</span>The lowing herd wind slowly o'er the lea,</span><br/><span class="line"><span class="lnum">3</span>The plowman homeward plods his weary way,</span><br/><span class="line"><span class="lnum">4</span>And leaves the world to darkness and to me.</span></p>
"""))

# 5. Deep nesting: quotes in quotes, lists in lists, tables with inline markup
CHAPTERS.append(("nested", "Nested Structures", """
<blockquote id="ne-q1"><p id="ne-q1p1">Outer quotation text begins here <span class="verse">1</span> and continues.</p>
  <blockquote id="ne-q2"><p id="ne-q2p1">Inner quotation, one level deeper <em>with emphasis</em> mid-sentence.</p></blockquote>
  <p id="ne-q1p2">Outer quotation resumes after the inner block.</p>
</blockquote>
<ul id="ne-list1">
  <li id="ne-li1">Level one item
    <ul id="ne-list2">
      <li id="ne-li2">Level two item
        <ul id="ne-list3">
          <li id="ne-li3">Level three item with <strong>bold</strong> text.</li>
          <li id="ne-li4">Level three sibling item.</li>
        </ul>
      </li>
    </ul>
  </li>
  <li id="ne-li5">Level one sibling item.</li>
</ul>
<table id="ne-table">
  <tr id="ne-tr1"><td id="ne-td1">Cell with <em>inline</em> markup</td><td id="ne-td2">Second cell, plain</td></tr>
  <tr id="ne-tr2"><td id="ne-td3">Cell three with a <span class="verse">7</span> marker</td><td id="ne-td4">Cell four</td></tr>
</table>
<dl id="ne-dl"><dt id="ne-dt1">Term one</dt><dd id="ne-dd1">Definition one with text.</dd><dt id="ne-dt2">Term two</dt><dd id="ne-dd2">Definition two with text.</dd></dl>
"""))

# 6. Unicode hazards: surrogate pairs, combining marks, RTL, invisible chars, entities
CHAPTERS.append(("unicode", "Unicode Hazards", """
<p id="un-emoji">Emoji with surrogate pairs: 🎉🚀 and a ZWJ family 👨‍👩‍👧‍👦 plus math 𝛑𝕏 and flags 🇺🇸🇯🇵.</p>
<p id="un-hebrew">Hebrew with niqqud: בְּרֵאשִׁית בָּרָא אֱלֹהִים and a Midrash on דָּבָר מִשְׁפָּט אֱמֶת.</p>
<p id="un-arabic">Arabic RTL with embedded LTR: يبدأ النص العربي هنا ثم 42 ثم ينتهي، مع تشكيل الكلمات المُعرَبة.</p>
<p id="un-combining">Decomposed e + combining acute: é vs precomposed é, and ĉ vs ĉ with combining circumflex.</p>
<p id="un-invisible">Soft hyphen in­side a word, zero&#8201;thin space, zero-width ​space, and trailing no-break space&#160;here.</p>
<p id="un-entities">Ampersand &amp;amp;, less &amp;lt;, greater &amp;gt;, apostrophe &amp;#39;, quote &amp;quot;, all rendered literally.</p>
"""))

# 7. Ruby / CJK
CHAPTERS.append(("ruby", "Ruby Annotations", """
<p id="ru-p1"><ruby>漢字<rt>かんじ</rt></ruby>の<ruby>読<rt>よ</rt></ruby>み<span>方</span>をテストする。</p>
<p id="ru-p2">日本語のテキストには<ruby>難<rt>むずか</rt></ruby>しい<ruby>漢字<rt>かんじ</rt></ruby>がたくさんあります。</p>
<p id="ru-p3">The ruby base 漢 and its annotation かん are separate text nodes in raw XHTML.</p>
"""))

# 8. Giant chapter with heavy phrase repetition (text-search disambiguation)
giant = ['<p id="gi-p0">Opening paragraph of the giant chapter, unique opening sentence.</p>']
for i in range(1, 401):
    giant.append(
        f'<p id="gi-p{i}">Verse <span class="verse">{i}</span> And it came to pass in those days that a repeated phrase appeared again among the people. '
        f'Variation marker alpha-{i}: the sentence continues with unique tail number {i * 7} before ending.</p>'
    )
CHAPTERS.append(("giant", "Giant Repeated Chapter", "\n".join(giant)))

# 9. Edge constructs: empties, same-tag adjacency, span-first paragraphs, deep same-tag nesting
deep = "<p id=\"ed-deep\">Deep nesting bottom text.</p>"
for lvl in range(10, 0, -1):
    deep = f'<div id="ed-lvl{lvl}">{deep}</div>'
CHAPTERS.append(("edge", "Edge Constructs", f"""
<p id="ed-spanfirst"><span class="lead">Leading span before any direct text.</span> Direct text follows the span.<span class="tail">Tail span.</span></p>
<p id="ed-adjacent"><span class="a">Adjacent</span><span class="b">same-tag</span><span class="c">spans</span> then plain text.</p>
<p id="ed-empty">Text with an empty span <span></span> mid-sentence and an empty emphasis <em></em> after it.</p>
<p id="ed-emptyp"></p>
<div id="ed-onlychild">{deep}</div>
<p id="ed-attrs" data-why="not">Paragraph with odd attribute values like &quot;quotes&quot; and &apos;apostrophes&apos;.</p>
"""))

CSS = """body { margin: 1em; }
.verse, .lnum, .fnref { font-size: 0.7em; vertical-align: super; color: #666; }
.speaker { font-variant: small-caps; }
.dropcap { font-size: 2.5em; font-weight: bold; float: left; line-height: 1; }
.line { display: block; margin-left: 2em; }
blockquote { margin: 1em 2em; border-left: 2px solid #ccc; padding-left: 1em; }
ruby rt { font-size: 0.5em; }
"""

def build(out_path):
    files = {}
    files["mimetype"] = MIMETYPE
    files["META-INF/container.xml"] = CONTAINER
    files["css/style.css"] = CSS

    manifests, spines = [], []
    for idx, (name, title, body) in enumerate(CHAPTERS, start=1):
        files[f"OPS/chapters/{name}.xhtml"] = page(name, title, body)
        manifests.append(f'    <item id="ch{idx}" href="chapters/{name}.xhtml" media-type="application/xhtml+xml"/>')
        spines.append(f'    <itemref idref="ch{idx}"/>')

    nav_items = "\n".join(
        f'      <li><a href="chapters/{name}.xhtml">{title}</a></li>'
        for name, title, _ in CHAPTERS
    )
    files["OPS/nav.xhtml"] = f"""<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE html>
<html xmlns="http://www.w3.org/1999/xhtml" xmlns:epub="http://www.idpf.org/2007/ops">
<head><title>Navigation</title></head>
<body>
<nav epub:type="toc" id="toc"><h1>Contents</h1>
  <ol>
{nav_items}
  </ol>
</nav>
</body>
</html>
"""
    nav_manifest = '    <item id="nav" href="nav.xhtml" media-type="application/xhtml+xml" properties="nav"/>'
    files["OPS/content.opf"] = f"""<?xml version="1.0" encoding="UTF-8"?>
<package xmlns="http://www.idpf.org/2007/opf" version="3.0" unique-identifier="uid">
  <metadata xmlns:dc="http://purl.org/dc/elements/1.1/">
    <dc:identifier id="uid">urn:uuid:1f2e3d4c-5b6a-7948-b7c6-d5e4f3a2b1c0</dc:identifier>
    <dc:title>CRE CFI Torture Test</dc:title>
    <dc:creator>Bookhoard Test</dc:creator>
    <dc:language>en</dc:language>
    <meta property="dcterms:modified">2026-10-12T00:00:00Z</meta>
  </metadata>
  <manifest>
{nav_manifest}
{chr(10).join(manifests)}
  </manifest>
  <spine>
{chr(10).join(spines)}
  </spine>
</package>
"""

    os.makedirs(os.path.dirname(out_path), exist_ok=True)
    with zipfile.ZipFile(out_path, "w") as z:
        z.writestr(zipfile.ZipInfo("mimetype"), MIMETYPE, compress_type=zipfile.ZIP_STORED)
        for arcname in sorted(files):
            if arcname == "mimetype":
                continue
            z.writestr(arcname, files[arcname], compress_type=zipfile.ZIP_DEFLATED)
    print(f"wrote {out_path} ({os.path.getsize(out_path):,} bytes, {len(CHAPTERS)} chapters)")

if __name__ == "__main__":
    out = sys.argv[1] if len(sys.argv) > 1 else os.path.join(
        os.path.dirname(__file__), "..",
        "uploads/Ebooks/Bookhoard Test/CRE CFI Torture Test/CRE CFI Torture Test.epub")
    build(os.path.normpath(out))