chore: ignore local export directory

The export/ directory holds database extracts (annotations, reading
positions, book metadata) pulled from production for debugging. The
contents are personal data and must never be committed.

.gitignore

@@ -79,3 +79,6 @@ web/static/vendor/
 VALID_TOKEN
 fresh_token.txt
 session-*.md
+
+# Local prod-export scratch: personal annotation/reading data, never commit
+/export/

internal/handlers/koreader.go

@@ -93,7 +93,22 @@ func (h *KOReaderHandler) convertHighlightPositions(ec annotationEpub, pos0, pos
 	if pos0 == "" || !ec.convertible() {
 		return "", ""
 	}
-	startLoc := wsync.ConvertToCanonical(wsync.LocatorSourceKOReader, pos0, percentage, contextText, ec.mediaItem.FormatGroup, ec.epubPath, "")
+	// Primary: anchor both endpoints by the selection text itself. One
+	// document match yields a same-spine ordered pair — the invariant the
+	// web reader's range-CFI builder needs. Independent per-endpoint
+	// conversion mis-anchors any selection past a block's first text node
+	// (crengine text-node indices don't survive to the raw XHTML).
+	startLoc := wsync.ConvertHighlightRangeToCanonical(ec.epubPath, ec.mediaItem.FormatGroup, pos0, pos1, contextText, percentage)
+	if startLoc.Precision == "exact" || startLoc.Precision == "structural" {
+		startCFI := webUsableCFI(startLoc)
+		endCFI := ""
+		if startCFI != "" && startLoc.EndCFI != "" {
+			endCFI = webUsableCFI(wsync.CanonicalLocator{CFI: startLoc.EndCFI, Precision: startLoc.Precision})
+		}
+		return startCFI, endCFI
+	}
+	// Fallback: legacy independent per-endpoint conversion.
+	startLoc = wsync.ConvertToCanonical(wsync.LocatorSourceKOReader, pos0, percentage, contextText, ec.mediaItem.FormatGroup, ec.epubPath, "")
 	startCFI := webUsableCFI(startLoc)
 	endCFI := ""
 	// A text-search start matched the selection text itself: its extent

internal/sync/cfi_converter.go

@@ -626,7 +626,12 @@ func flattenDocumentForSearch(body *html.Node) ([]rune, []docRunePos) {
 		lastBlock = block
 		off := 0
 		for _, r := range n.Data {
-			if unicode.IsSpace(r) {
+			// Zero-width format characters are line-break opportunities,
+			// not word characters: devices render them as breaks and
+			// capture plain spaces, so they must not fuse words in the
+			// search stream. ZWJ (U+200D) is exempt — it glues emoji
+			// sequences.
+			if unicode.IsSpace(r) || r == '\u200b' || r == '\u200c' || r == '\ufeff' {
 				pendingSpace = true
 			} else {
 				if pendingSpace && len(runes) > 0 {
@@ -644,9 +649,12 @@ func flattenDocumentForSearch(body *html.Node) ([]rune, []docRunePos) {
 	var walk func(n *html.Node)
 	walk = func(n *html.Node) {
 		if n.Type == html.TextNode {
-			if strings.TrimSpace(n.Data) != "" {
-				appendNode(n)
-			}
+			// Whitespace-only nodes still participate: a space carried in
+			// its own inline span (<span class="sp"> </span>) is a real
+			// word separator, and skipping it fuses adjacent words in the
+			// search stream. appendNode turns such nodes into pending
+			// space without emitting anything.
+			appendNode(n)
 			return
 		}
 		for c := n.FirstChild; c != nil; c = c.NextSibling {

internal/sync/cfi_highlight_range.go

@@ -0,0 +1,167 @@
+package sync
+
+// Text-anchored range conversion for device highlights.
+//
+// Why this exists: CRE xpointers address text as text()[N].offset, but
+// crengine's text-node enumeration does not survive to the raw XHTML on
+// disk (whitespace runs, entity merges, live-DOM splits), and the trailing
+// offset alone lands at a global block offset that is wrong for any anchor
+// past the first text node of a block. Converting pos0 and pos1
+// independently made it worse: mismatched start/end base paths and
+// backwards ranges — highlights that store but never paint (or paint in
+// the wrong place) in the web reader.
+//
+// The device's selection text, however, is a quote of the document. So for
+// highlights we anchor BOTH endpoints by locating that quote in the
+// flattened document text (whitespace-normalized, crossing block
+// boundaries), and use the device locators only as disambiguation hints
+// when the quote occurs more than once. Endpoints derived from one match
+// are same-spine and ordered by construction — the two properties the
+// reader's range-CFI builder needs.
+
+import (
+	"log"
+	"regexp"
+	"strings"
+	"unicode/utf8"
+
+	"golang.org/x/net/html"
+)
+
+// ConvertHighlightRange resolves a highlight's pos0/pos1 pair into a start
+// CFI plus a true range end by locating the selection text in the document.
+// It returns nil when the text cannot be located; callers fall back to the
+// legacy per-endpoint conversion.
+func (c *CFIConverter) ConvertHighlightRange(pos0, pos1, contextText string, storedPercentage float64) *ConversionResult {
+	if !IsCREXPointer(pos0) {
+		return nil
+	}
+	xp, err := ParseCREXPointer(pos0)
+	if err != nil {
+		return nil
+	}
+	doc, href, err := c.getContentDoc(xp.FragmentIndex)
+	if err != nil {
+		return nil
+	}
+	body := findBody(doc)
+	if body == nil {
+		return nil
+	}
+	spineIndex := xp.FragmentIndex - 1
+
+	// KOReader wraps partial-paragraph captures in curly braces; they are
+	// wire markers, not document text. Zero-width format characters in the
+	// capture are break opportunities the device rendered — normalize to
+	// spaces so both sides of the match agree.
+	ctx := normalizeWhitespace(strings.Trim(contextText, "{}"))
+	ctx = strings.NewReplacer("\u200b", " ", "\u200c", " ", "\ufeff", " ").Replace(ctx)
+	ctx = normalizeWhitespace(ctx)
+	if ctx == "" {
+		return nil
+	}
+
+	runes, poss := flattenDocumentForSearch(body)
+	text := string(runes)
+
+	words := strings.Fields(ctx)
+	quoted := make([]string, len(words))
+	for i, w := range words {
+		quoted[i] = regexp.QuoteMeta(w)
+	}
+	re, err := regexp.Compile(strings.Join(quoted, `\s+`))
+	if err != nil {
+		return nil
+	}
+	matches := re.FindAllStringIndex(text, -1)
+	if len(matches) == 0 {
+		log.Printf("Bookhoard: highlight range: selection %q not found in %s", truncateForLog(ctx, 60), href)
+		return nil
+	}
+	loc := pickHighlightMatch(matches, text, poss, xp, pos1, body)
+	if len(matches) > 1 {
+		log.Printf("Bookhoard: highlight range: %d matches for %q in %s, resolved by device hint", len(matches), truncateForLog(ctx, 40), href)
+	}
+
+	startRune := utf8.RuneCountInString(text[:loc[0]])
+	endRune := utf8.RuneCountInString(text[:loc[1]]) // exclusive
+
+	startPos := poss[startRune]
+	startCFI, err := buildCFI(spineIndex, startPos.node, startPos.off)
+	if err != nil || startCFI == "" {
+		log.Printf("Bookhoard: highlight range: buildCFI(start) failed: %v", err)
+		return nil
+	}
+	endCFI := ""
+	if endRune > startRune && endRune <= len(poss) {
+		endPos := poss[endRune-1]
+		endOff := endPos.off + 1
+		if n := utf8.RuneCountInString(endPos.node.Data); endOff > n {
+			endOff = n
+		}
+		if ec, err := buildCFI(spineIndex, endPos.node, endOff); err == nil && ec != "" {
+			endCFI = ec
+		}
+	}
+
+	log.Printf("Bookhoard: highlight range %q → %s (precision: exact)", truncateForLog(ctx, 60), startCFI)
+	return &ConversionResult{
+		EPUBCFI:    startCFI,
+		EndEPUBCFI: endCFI,
+		Href:       href,
+		Percentage: storedPercentage,
+		Precision:  "exact",
+	}
+}
+
+// pickHighlightMatch chooses among multiple occurrences of the selection.
+// pos1's element path (the end anchor's landing block) is the strongest
+// hint: the true occurrence is the one whose extent stays inside that
+// block. Without a usable hint, document order wins (first occurrence).
+func pickHighlightMatch(matches [][]int, text string, poss []docRunePos, xp *CREXPointer, pos1 string, body *html.Node) []int {
+	if len(matches) <= 1 {
+		return matches[0]
+	}
+	if xp1, err := ParseCREXPointer(pos1); err == nil && len(xp1.ElementPath) > 0 {
+		if elem := resolveCREElementPath(body, xp1.ElementPath); elem != nil {
+			lo, hi := elementFlatRange(poss, elem)
+			if lo >= 0 {
+				for _, m := range matches {
+					s := utf8.RuneCountInString(text[:m[0]])
+					e := utf8.RuneCountInString(text[:m[1]]) - 1
+					if s >= lo && e < hi {
+						return m
+					}
+				}
+			}
+		}
+	}
+	return matches[0]
+}
+
+// elementFlatRange returns the [lo, hi) rune range of the flattened stream
+// covered by elem's subtree, or (-1, 0) when elem contributes no text.
+func elementFlatRange(poss []docRunePos, elem *html.Node) (int, int) {
+	lo, hi := -1, -1
+	for i, p := range poss {
+		if isDescendantNode(p.node, elem) {
+			if lo < 0 {
+				lo = i
+			}
+			hi = i + 1
+		}
+	}
+	if lo < 0 {
+		return -1, 0
+	}
+	return lo, hi
+}
+
+func isDescendantNode(n, ancestor *html.Node) bool {
+	for p := n.Parent; p != nil; p = p.Parent {
+		if p == ancestor {
+			return true
+		}
+	}
+	return false
+}

internal/sync/cfi_repro_clean_test.go

@@ -0,0 +1,379 @@
+package sync
+
+// Manual repro harness for the CRE→CFI highlight conversion defects found in
+// "Clean" (Rosser K Powitzky): 28 device highlights whose stored web CFIs are
+// structurally inconsistent (mismatched start/end base paths, backwards
+// ranges). This test mirrors handlers.convertHighlightPositions exactly —
+// pos0 converted WITH the selection text as context, pos1 WITHOUT any — then
+// reports the result alongside the CFIs stored at ingest time and a
+// ground-truth oracle computed by locating the selection text in the document
+// itself. Skips silently when the book/export are not on this machine, like
+// cfi_verify_manual_test.go.
+
+import (
+	"fmt"
+	"encoding/json"
+	"os"
+	"strings"
+	"testing"
+
+	"golang.org/x/net/html"
+)
+
+const (
+	reproExport = "/home/nymusicman/Code/bookhoard/export/media_highlights.json"
+	reproEpub   = "/home/nymusicman/Code/bookhoard/uploads/Ebooks/Rosser K Powitzky/Clean_ How the Jewish Roots of Atone (77)/Clean_ How the Jewish Roots of - Rosser K Powitzky.epub"
+)
+
+type reproRow struct {
+	ID            string  `json:"id"`
+	StartPosition string  `json:"start_position"`
+	EndPosition   string  `json:"end_position"`
+	StoredStart   string  `json:"epubcfi_start"`
+	StoredEnd     string  `json:"epubcfi_end"`
+	SelectionText string  `json:"selection_text"`
+	PctStart      float64 `json:"percentage_start"`
+}
+
+// webUsable mirrors handlers.webUsableCFI (unexported there; cannot import
+// the handlers package from here without a cycle).
+func webUsable(precision, cfi string) string {
+	if precision != "structural" && precision != "exact" {
+		return ""
+	}
+	if !strings.HasPrefix(cfi, "epubcfi(") || !strings.HasSuffix(cfi, ")") {
+		return ""
+	}
+	return cfi
+}
+
+// reproExtendCFIByLength mirrors handlers.extendCFIByLength: advance the
+// trailing character offset of a point CFI by the UTF-16 length of text.
+func reproExtendCFIByLength(cfi, text string) string {
+	if cfi == "" || text == "" {
+		return ""
+	}
+	if !strings.HasSuffix(cfi, ")") {
+		return ""
+	}
+	inner := cfi[:len(cfi)-1]
+	idx := strings.LastIndex(inner, ":")
+	if idx < 0 {
+		return cfi
+	}
+	var off int
+	if _, err := fmtSscanInt(inner[idx+1:], &off); err != nil {
+		return cfi
+	}
+	return inner[:idx+1] + itoa(off+utf16Len(text)) + ")"
+}
+
+// cfiBase splits "epubcfi(...)" into (base, offset): base is everything
+// before the final ":N" character-offset terminator.
+func cfiBase(cfi string) (string, int, bool) {
+	if !strings.HasSuffix(cfi, ")") {
+		return "", 0, false
+	}
+	inner := cfi[:len(cfi)-1]
+	idx := strings.LastIndex(inner, ":")
+	if idx < 0 {
+		return inner, 0, true
+	}
+	off := 0
+	if _, err := fmtSscanInt(inner[idx+1:], &off); err != nil {
+		return inner, 0, true
+	}
+	return inner[:idx], off, true
+}
+
+func fmtSscanInt(s string, out *int) (int, error) {
+	n := 0
+	ok := false
+	for _, r := range s {
+		if r < '0' || r > '9' {
+			break
+		}
+		n = n*10 + int(r-'0')
+		ok = true
+	}
+	if !ok {
+		return 0, errReproNoInt
+	}
+	*out = n
+	return len(s), nil
+}
+
+var errReproNoInt = errReproNoIntErr{}
+
+type errReproNoIntErr struct{}
+
+func (errReproNoIntErr) Error() string { return "no integer" }
+
+func itoa(n int) string {
+	if n == 0 {
+		return "0"
+	}
+	neg := n < 0
+	if neg {
+		n = -n
+	}
+	var b [20]byte
+	i := len(b)
+	for n > 0 {
+		i--
+		b[i] = byte('0' + n%10)
+		n /= 10
+	}
+	if neg {
+		i--
+		b[i] = '-'
+	}
+	return string(b[i:])
+}
+
+// oracleCFIs locates the selection text in the document named by the start
+// xpointer's fragment and returns ground-truth (startCFI, endCFI, spineIndex).
+// Whitespace runs are collapsed on both sides before matching, so device
+// text with reflowed newlines still lands.
+func oracleCFIs(t *testing.T, conv *CFIConverter, row reproRow) (string, string, string) {
+	xp, err := ParseCREXPointer(row.StartPosition)
+	if err != nil {
+		return "", "", "unparseable start xpointer"
+	}
+	doc, href, err := conv.getContentDoc(xp.FragmentIndex)
+	if err != nil {
+		return "", "", "content doc missing: " + err.Error()
+	}
+	body := findBody(doc)
+	if body == nil {
+		return "", "", "no body"
+	}
+
+	// Flatten every text node in document order.
+	var segs []textSegment
+	var walk func(*html.Node)
+	walk = func(n *html.Node) {
+		if n.Type == html.TextNode {
+			segs = append(segs, textSegment{node: n, runes: []rune(n.Data)})
+		}
+		for c := n.FirstChild; c != nil; c = c.NextSibling {
+			walk(c)
+		}
+	}
+	walk(body)
+
+	var flat []rune
+	for _, s := range segs {
+		flat = append(flat, s.runes...)
+	}
+
+	needle := []rune(normalizeWhitespace(row.SelectionText))
+	if len(needle) == 0 {
+		return "", "", "empty selection text"
+	}
+	// Collapse whitespace in `flat` while mapping each kept rune back to a
+	// flat index.
+	var norm []rune
+	var normToFlat []int
+	lastWS := false
+	for i, r := range flat {
+		isWS := r == ' ' || r == '\t' || r == '\n' || r == '\r' || r == ' ' || r == '​'
+		if isWS {
+			if lastWS {
+				continue
+			}
+			norm = append(norm, ' ')
+			normToFlat = append(normToFlat, i)
+			lastWS = true
+			continue
+		}
+		norm = append(norm, r)
+		normToFlat = append(normToFlat, i)
+		lastWS = false
+	}
+
+	// Trim wrapped-quote artifacts KOReader adds to partial-paragraph text.
+	trimmed := needle
+	for len(trimmed) > 0 && (trimmed[0] == '{' || trimmed[0] == '}') {
+		trimmed = trimmed[1:]
+	}
+	for len(trimmed) > 0 && (trimmed[len(trimmed)-1] == '{' || trimmed[len(trimmed)-1] == '}') {
+		trimmed = trimmed[:len(trimmed)-1]
+	}
+
+	pos := indexRunes(norm, trimmed)
+	if pos < 0 {
+		return "", "", "selection text not found in doc " + href
+	}
+
+	flatStart := normToFlat[pos]
+	flatEnd := normToFlat[pos+len(trimmed)-1] + 1 // end is exclusive
+
+	nodeAt := func(flatIdx int) (*html.Node, int) {
+		acc := 0
+		for _, s := range segs {
+			n := len(s.runes)
+			if flatIdx < acc+n {
+				return s.node, flatIdx - acc
+			}
+			acc += n
+		}
+		return nil, 0
+	}
+
+	startNode, startOff := nodeAt(flatStart)
+	endNode, endOff := nodeAt(flatEnd)
+	if startNode == nil || endNode == nil {
+		return "", "", "flat index out of range"
+	}
+
+	spineIndex := xp.FragmentIndex - 1
+	startCFI, err := buildCFI(spineIndex, startNode, startOff)
+	if err != nil {
+		return "", "", "buildCFI start: " + err.Error()
+	}
+	endCFI, err := buildCFI(spineIndex, endNode, endOff)
+	if err != nil {
+		return "", "", "buildCFI end: " + err.Error()
+	}
+	return startCFI, endCFI, ""
+}
+
+func indexRunes(hay, needle []rune) int {
+	if len(needle) == 0 || len(needle) > len(hay) {
+		return -1
+	}
+	for i := 0; i+len(needle) <= len(hay); i++ {
+		match := true
+		for j := range needle {
+			if hay[i+j] != needle[j] {
+				match = false
+				break
+			}
+		}
+		if match {
+			return i
+		}
+	}
+	return -1
+}
+
+func TestManualCleanRepro(t *testing.T) {
+	raw, err := os.ReadFile(reproExport)
+	if err != nil {
+		t.Skipf("export json not present: %v", err)
+	}
+	if _, err := os.Stat(reproEpub); err != nil {
+		t.Skipf("book epub not present: %v", err)
+	}
+	var rows []reproRow
+	if err := json.Unmarshal(raw, &rows); err != nil {
+		t.Fatalf("unmarshal export: %v", err)
+	}
+
+	conv := cachedConverter(reproEpub)
+
+	stats := map[string]int{}
+	for _, row := range rows {
+		// Mirror handlers.convertHighlightPositions (text-anchored primary
+		// path, legacy per-endpoint fallback).
+		startLoc := ConvertHighlightRangeToCanonical(reproEpub, "reflowable", row.StartPosition, row.EndPosition, row.SelectionText, row.PctStart)
+		startCFI := webUsable(startLoc.Precision, startLoc.CFI)
+		endCFI := ""
+		if startCFI != "" && startLoc.EndCFI != "" {
+			endCFI = webUsable(startLoc.Precision, startLoc.EndCFI)
+		}
+		if startCFI == "" {
+			startLoc = ConvertToCanonical(LocatorSourceKOReader, row.StartPosition, row.PctStart, row.SelectionText, "reflowable", reproEpub, "")
+			startCFI = webUsable(startLoc.Precision, startLoc.CFI)
+			endCFI = ""
+			if startCFI != "" && startLoc.EndCFI != "" {
+				endCFI = startLoc.EndCFI
+			}
+			if endCFI == "" && row.EndPosition != "" {
+				endLoc := ConvertToCanonical(LocatorSourceKOReader, row.EndPosition, row.PctStart, "", "reflowable", reproEpub, "")
+				endCFI = webUsable(endLoc.Precision, endLoc.CFI)
+			}
+			if endCFI == "" && startCFI != "" && row.SelectionText != "" {
+				endCFI = reproExtendCFIByLength(startCFI, row.SelectionText)
+			}
+		}
+
+		oracleStart, oracleEnd, oracleNote := oracleCFIs(t, conv, row)
+		stats["oracleMiss:"+statusOr(oracleNote)]++
+
+		cls := func(a, b string) string {
+			ba, oa, okA := cfiBase(a)
+			bb, ob, okB := cfiBase(b)
+			switch {
+			case !okA || !okB || a == "" || b == "":
+				return "missing"
+			case ba != bb:
+				return "diff-base"
+			case ob <= oa:
+				return "backwards"
+			default:
+				return "healthy"
+			}
+		}
+		cur := cls(startCFI, endCFI)
+		sto := cls(row.StoredStart, row.StoredEnd)
+		ora := cls(oracleStart, oracleEnd)
+		stats["current:"+cur]++
+		stats["stored:"+sto]++
+		stats["oracle:"+ora]++
+
+		if os.Getenv("REPRO_VERBOSE") != "" || cur != "healthy" || sto != "healthy" {
+			t.Logf("row %s", row.ID[:8])
+			t.Logf("   raw:  %s  ->  %s", row.StartPosition, row.EndPosition)
+			t.Logf("   ctx:  %q", truncateRunes(row.SelectionText, 60))
+			t.Logf("   stored:   %-38s %-38s [%s]", row.StoredStart, row.StoredEnd, sto)
+			t.Logf("   current:  %-38s %-38s [%s]", startCFI, endCFI, cur)
+			t.Logf("   oracle:   %-38s %-38s [%s] %s", oracleStart, oracleEnd, ora, oracleNote)
+		}
+	}
+
+	t.Logf("summary: %v", stats)
+
+	// REPRO_EMIT_SQL=<path>: emit UPDATE statements carrying the NEW
+	// converter output into the database rows this export came from
+	// (one-off healing of rows written by the broken converter).
+	if out := os.Getenv("REPRO_EMIT_SQL"); out != "" {
+		var b strings.Builder
+		b.WriteString("BEGIN;\n")
+		for _, row := range rows {
+			startLoc := ConvertHighlightRangeToCanonical(reproEpub, "reflowable", row.StartPosition, row.EndPosition, row.SelectionText, row.PctStart)
+			startCFI := webUsable(startLoc.Precision, startLoc.CFI)
+			endCFI := ""
+			if startCFI != "" && startLoc.EndCFI != "" {
+				endCFI = webUsable(startLoc.Precision, startLoc.EndCFI)
+			}
+			if startCFI == "" {
+				continue // leave untouched; legacy fallback rows keep old CFIs
+			}
+			b.WriteString(fmt.Sprintf(
+				"UPDATE media_highlights SET epubcfi_start=%s, epubcfi_end=%s WHERE id='%s';\n",
+				pgQuote(startCFI), pgQuote(endCFI), row.ID,
+			))
+		}
+		b.WriteString("COMMIT;\n")
+		if err := os.WriteFile(out, []byte(b.String()), 0o600); err != nil {
+			t.Fatalf("write sql: %v", err)
+		}
+		t.Logf("wrote %s", out)
+	}
+}
+
+func pgQuote(s string) string {
+	if s == "" {
+		return "''"
+	}
+	return "'" + strings.ReplaceAll(s, "'", "''") + "'"
+}
+
+func statusOr(s string) string {
+	if s == "" {
+		return "found"
+	}
+	return s
+}

internal/sync/locators.go

@@ -125,6 +125,38 @@ func ConvertToCanonical(
 	}
 }
 
+// ConvertHighlightRangeToCanonical is the highlight-specific twin of
+// ConvertToCanonical: it anchors a device highlight's pos0/pos1 pair by the
+// selection text so both endpoints derive from one document match — same
+// spine, ordered — instead of two independent structural landings. Returns
+// a "fallback" locator with an empty CFI when the text cannot be located;
+// callers fall back to ConvertToCanonical per endpoint.
+func ConvertHighlightRangeToCanonical(
+	epubPath string,
+	formatGroup string,
+	pos0 string,
+	pos1 string,
+	contextText string,
+	percentage float64,
+) CanonicalLocator {
+	if !isConvertible(formatGroup) || epubPath == "" {
+		return CanonicalLocator{CFI: pos0, Precision: "passthrough", Percentage: percentage}
+	}
+	if !IsCREXPointer(pos0) {
+		return CanonicalLocator{CFI: pos0, Precision: "already-standard", Percentage: percentage}
+	}
+	result := cachedConverter(epubPath).ConvertHighlightRange(pos0, pos1, contextText, percentage)
+	if result == nil || result.EPUBCFI == "" {
+		return CanonicalLocator{CFI: "", Precision: "fallback", Percentage: percentage}
+	}
+	return CanonicalLocator{
+		CFI:        result.EPUBCFI,
+		EndCFI:     result.EndEPUBCFI,
+		Precision:  result.Precision,
+		Percentage: result.Percentage,
+	}
+}
+
 func ConvertFromCanonical(
 	source LocatorSource,
 	canonicalCFI string,

scripts/build_torture_epub.py

@@ -0,0 +1,220 @@
+#!/usr/bin/env python3
+"""Build a synthetic 'torture test' EPUB exercising every CRE-xpointer -> EPUB-CFI
+conversion hazard the server converter can encounter.
+
+Each chapter targets one hazard class; constructs carry comments naming the
+hazard. Output: uploads/Ebooks/Bookhoard Test/CRE CFI Torture Test/CRE CFI Torture Test.epub
+(or argv[1]). Deterministic output for reproducible tests.
+"""
+import os
+import sys
+import zipfile
+
+MIMETYPE = "application/epub+zip"
+
+CONTAINER = """<?xml version="1.0" encoding="UTF-8"?>
+<container version="1.0" xmlns="urn:oasis:names:tc:opendocument:xmlns:container">
+  <rootfiles>
+    <rootfile full-path="OPS/content.opf" media-type="application/oebps-package+xml"/>
+  </rootfiles>
+</container>
+"""
+
+def page(name, title, body):
+    return f"""<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE html>
+<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
+<head><title>{title}</title><link rel="stylesheet" type="text/css" href="../css/style.css"/></head>
+<body>
+<!-- chapter: {name} -->
+<h1 id="{name}-h">{title}</h1>
+{body}
+</body>
+</html>
+"""
+
+CHAPTERS = []
+
+# 1. The core bug shape: multi-text-node paragraphs (inline markup splitting text)
+CHAPTERS.append(("split-text", "Split Text Nodes", """
+<p id="st-p1">In the beginning <span class="verse">1</span> God created the heaven and the earth <span class="verse">2</span> and the earth was without form.</p>
+<p id="st-p2">A <em>single</em> sentence with <strong>several</strong> inline <span class="mark">marked</span> spans inside it.</p>
+<p id="st-p3">First text node here.<span class="cont">Immediately-following span text.</span>Text that continues after the span.<span class="cont2">And a second span.</span>Tail text node.</p>
+<p id="st-p4">Quoted "double" and 'single' marks, an ampersand &amp; entity, and a unicode em—dash.</p>
+<div class="group" id="st-group">
+  <p id="st-p5">Nested group paragraph one <span class="verse">1</span> with more text after the marker.</p>
+  <p id="st-p6">Nested group paragraph two, plain text only.</p>
+</div>
+"""))
+
+# 2. Whitespace-only text nodes + comments between elements (sibling-index drift)
+CHAPTERS.append(("whitespace-comments", "Whitespace and Comments", """
+<ul id="wc-list">
+  <!-- comment directly inside list -->
+  <li id="wc-li1">
+    <p id="wc-p1">Indented markup with whitespace text nodes on every side.</p>
+    <!-- comment between siblings -->
+    <p id="wc-p2">Second indented   paragraph with   irregular internal    spacing.</p>
+  </li>
+  <!-- another comment -->
+  <li id="wc-li2">
+    <p id="wc-p3">Paragraph after multiple comment and whitespace siblings.</p>
+  </li>
+</ul>
+<p id="wc-p4">Tail paragraph after the list.</p>
+"""))
+
+# 3. Drop caps + footnote anchors (single-char inline splits)
+CHAPTERS.append(("dropcaps-footnotes", "Dropcaps and Footnotes", """
+<p id="df-p1"><span class="dropcap">O</span>nce upon a time a drop cap split the first word from itself.</p>
+<p id="df-p2">A claim requiring support<span class="fnref"><a id="fnref-1" href="#fn-1">1</a></span> and a second claim<span class="fnref"><a id="fnref-2" href="#fn-2">2</a></span> in the same paragraph.</p>
+<p id="df-p3">Multiple <a href="#df-p1">internal links</a> interleaved <a href="#df-p2">through text</a> as anchors.</p>
+<div class="footnotes" id="df-notes">
+  <p id="df-fn1"><a id="fn-1" href="#fnref-1">1.</a> The first footnote, body text follows the backlink.</p>
+  <p id="df-fn2"><a id="fn-2" href="#fnref-2">2.</a> The second footnote with <em>emphasis</em> inside.</p>
+</div>
+"""))
+
+# 4. Poetry / drama: br-separated lines, speaker spans, line numbers
+CHAPTERS.append(("poetry", "Poetry and Drama", """
+<p id="po-dramap"><span class="speaker">HAMLET.</span> To be, or not to be, that is the question:<br/>Whether 'tis nobler in the mind to suffer<br/>The slings and arrows of outrageous fortune.</p>
+<p id="po-verse1"><span class="line"><span class="lnum">1</span>The curfew tolls the knell of parting day,</span><br/><span class="line"><span class="lnum">2</span>The lowing herd wind slowly o'er the lea,</span><br/><span class="line"><span class="lnum">3</span>The plowman homeward plods his weary way,</span><br/><span class="line"><span class="lnum">4</span>And leaves the world to darkness and to me.</span></p>
+"""))
+
+# 5. Deep nesting: quotes in quotes, lists in lists, tables with inline markup
+CHAPTERS.append(("nested", "Nested Structures", """
+<blockquote id="ne-q1"><p id="ne-q1p1">Outer quotation text begins here <span class="verse">1</span> and continues.</p>
+  <blockquote id="ne-q2"><p id="ne-q2p1">Inner quotation, one level deeper <em>with emphasis</em> mid-sentence.</p></blockquote>
+  <p id="ne-q1p2">Outer quotation resumes after the inner block.</p>
+</blockquote>
+<ul id="ne-list1">
+  <li id="ne-li1">Level one item
+    <ul id="ne-list2">
+      <li id="ne-li2">Level two item
+        <ul id="ne-list3">
+          <li id="ne-li3">Level three item with <strong>bold</strong> text.</li>
+          <li id="ne-li4">Level three sibling item.</li>
+        </ul>
+      </li>
+    </ul>
+  </li>
+  <li id="ne-li5">Level one sibling item.</li>
+</ul>
+<table id="ne-table">
+  <tr id="ne-tr1"><td id="ne-td1">Cell with <em>inline</em> markup</td><td id="ne-td2">Second cell, plain</td></tr>
+  <tr id="ne-tr2"><td id="ne-td3">Cell three with a <span class="verse">7</span> marker</td><td id="ne-td4">Cell four</td></tr>
+</table>
+<dl id="ne-dl"><dt id="ne-dt1">Term one</dt><dd id="ne-dd1">Definition one with text.</dd><dt id="ne-dt2">Term two</dt><dd id="ne-dd2">Definition two with text.</dd></dl>
+"""))
+
+# 6. Unicode hazards: surrogate pairs, combining marks, RTL, invisible chars, entities
+CHAPTERS.append(("unicode", "Unicode Hazards", """
+<p id="un-emoji">Emoji with surrogate pairs: 🎉🚀 and a ZWJ family 👨‍👩‍👧‍👦 plus math 𝛑𝕏 and flags 🇺🇸🇯🇵.</p>
+<p id="un-hebrew">Hebrew with niqqud: בְּרֵאשִׁית בָּרָא אֱלֹהִים and a Midrash on דָּבָר מִשְׁפָּט אֱמֶת.</p>
+<p id="un-arabic">Arabic RTL with embedded LTR: يبدأ النص العربي هنا ثم 42 ثم ينتهي، مع تشكيل الكلمات المُعرَبة.</p>
+<p id="un-combining">Decomposed e + combining acute: é vs precomposed é, and ĉ vs ĉ with combining circumflex.</p>
+<p id="un-invisible">Soft hyphen in­side a word, zero&#8201;thin space, zero-width ​space, and trailing no-break space&#160;here.</p>
+<p id="un-entities">Ampersand &amp;amp;, less &amp;lt;, greater &amp;gt;, apostrophe &amp;#39;, quote &amp;quot;, all rendered literally.</p>
+"""))
+
+# 7. Ruby / CJK
+CHAPTERS.append(("ruby", "Ruby Annotations", """
+<p id="ru-p1"><ruby>漢字<rt>かんじ</rt></ruby>の<ruby>読<rt>よ</rt></ruby>み<span>方</span>をテストする。</p>
+<p id="ru-p2">日本語のテキストには<ruby>難<rt>むずか</rt></ruby>しい<ruby>漢字<rt>かんじ</rt></ruby>がたくさんあります。</p>
+<p id="ru-p3">The ruby base 漢 and its annotation かん are separate text nodes in raw XHTML.</p>
+"""))
+
+# 8. Giant chapter with heavy phrase repetition (text-search disambiguation)
+giant = ['<p id="gi-p0">Opening paragraph of the giant chapter, unique opening sentence.</p>']
+for i in range(1, 401):
+    giant.append(
+        f'<p id="gi-p{i}">Verse <span class="verse">{i}</span> And it came to pass in those days that a repeated phrase appeared again among the people. '
+        f'Variation marker alpha-{i}: the sentence continues with unique tail number {i * 7} before ending.</p>'
+    )
+CHAPTERS.append(("giant", "Giant Repeated Chapter", "\n".join(giant)))
+
+# 9. Edge constructs: empties, same-tag adjacency, span-first paragraphs, deep same-tag nesting
+deep = "<p id=\"ed-deep\">Deep nesting bottom text.</p>"
+for lvl in range(10, 0, -1):
+    deep = f'<div id="ed-lvl{lvl}">{deep}</div>'
+CHAPTERS.append(("edge", "Edge Constructs", f"""
+<p id="ed-spanfirst"><span class="lead">Leading span before any direct text.</span> Direct text follows the span.<span class="tail">Tail span.</span></p>
+<p id="ed-adjacent"><span class="a">Adjacent</span><span class="b">same-tag</span><span class="c">spans</span> then plain text.</p>
+<p id="ed-empty">Text with an empty span <span></span> mid-sentence and an empty emphasis <em></em> after it.</p>
+<p id="ed-emptyp"></p>
+<div id="ed-onlychild">{deep}</div>
+<p id="ed-attrs" data-why="not">Paragraph with odd attribute values like &quot;quotes&quot; and &apos;apostrophes&apos;.</p>
+"""))
+
+CSS = """body { margin: 1em; }
+.verse, .lnum, .fnref { font-size: 0.7em; vertical-align: super; color: #666; }
+.speaker { font-variant: small-caps; }
+.dropcap { font-size: 2.5em; font-weight: bold; float: left; line-height: 1; }
+.line { display: block; margin-left: 2em; }
+blockquote { margin: 1em 2em; border-left: 2px solid #ccc; padding-left: 1em; }
+ruby rt { font-size: 0.5em; }
+"""
+
+def build(out_path):
+    files = {}
+    files["mimetype"] = MIMETYPE
+    files["META-INF/container.xml"] = CONTAINER
+    files["css/style.css"] = CSS
+
+    manifests, spines = [], []
+    for idx, (name, title, body) in enumerate(CHAPTERS, start=1):
+        files[f"OPS/chapters/{name}.xhtml"] = page(name, title, body)
+        manifests.append(f'    <item id="ch{idx}" href="chapters/{name}.xhtml" media-type="application/xhtml+xml"/>')
+        spines.append(f'    <itemref idref="ch{idx}"/>')
+
+    nav_items = "\n".join(
+        f'      <li><a href="chapters/{name}.xhtml">{title}</a></li>'
+        for name, title, _ in CHAPTERS
+    )
+    files["OPS/nav.xhtml"] = f"""<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE html>
+<html xmlns="http://www.w3.org/1999/xhtml" xmlns:epub="http://www.idpf.org/2007/ops">
+<head><title>Navigation</title></head>
+<body>
+<nav epub:type="toc" id="toc"><h1>Contents</h1>
+  <ol>
+{nav_items}
+  </ol>
+</nav>
+</body>
+</html>
+"""
+    nav_manifest = '    <item id="nav" href="nav.xhtml" media-type="application/xhtml+xml" properties="nav"/>'
+    files["OPS/content.opf"] = f"""<?xml version="1.0" encoding="UTF-8"?>
+<package xmlns="http://www.idpf.org/2007/opf" version="3.0" unique-identifier="uid">
+  <metadata xmlns:dc="http://purl.org/dc/elements/1.1/">
+    <dc:identifier id="uid">urn:uuid:1f2e3d4c-5b6a-7948-b7c6-d5e4f3a2b1c0</dc:identifier>
+    <dc:title>CRE CFI Torture Test</dc:title>
+    <dc:creator>Bookhoard Test</dc:creator>
+    <dc:language>en</dc:language>
+    <meta property="dcterms:modified">2026-10-12T00:00:00Z</meta>
+  </metadata>
+  <manifest>
+{nav_manifest}
+{chr(10).join(manifests)}
+  </manifest>
+  <spine>
+{chr(10).join(spines)}
+  </spine>
+</package>
+"""
+
+    os.makedirs(os.path.dirname(out_path), exist_ok=True)
+    with zipfile.ZipFile(out_path, "w") as z:
+        z.writestr(zipfile.ZipInfo("mimetype"), MIMETYPE, compress_type=zipfile.ZIP_STORED)
+        for arcname in sorted(files):
+            if arcname == "mimetype":
+                continue
+            z.writestr(arcname, files[arcname], compress_type=zipfile.ZIP_DEFLATED)
+    print(f"wrote {out_path} ({os.path.getsize(out_path):,} bytes, {len(CHAPTERS)} chapters)")
+
+if __name__ == "__main__":
+    out = sys.argv[1] if len(sys.argv) > 1 else os.path.join(
+        os.path.dirname(__file__), "..",
+        "uploads/Ebooks/Bookhoard Test/CRE CFI Torture Test/CRE CFI Torture Test.epub")
+    build(os.path.normpath(out))

web/src/reader/reader.ts

@@ -2382,41 +2382,86 @@ document.addEventListener("alpine:init", () => {
     // converter used to fall back to a document-start CFI when the end
     // xpointer didn't resolve exactly. In both cases the end is derived
     // from the start offset advanced by the text's UTF-16 length (EPUB
-    // CFI offsets are UTF-16 code units); multi-node selections just fail
-    // resolution harmlessly and fall back to the point CFI.
+    // CFI offsets are UTF-16 code units).
+    //
+    // The endpoints may legitimately anchor at different nodes (a
+    // selection crossing inline spans lands in two text nodes), so the
+    // range parent is the endpoints' longest shared step prefix — down to
+    // the bare spine step if needed — and each side emits its divergent
+    // remainder, per the EPUB CFI range grammar
+    // epubcfi(parent,start,end).
     toRangeCfi(start: string, end: string, text: string): string {
       if (!start) return end || start;
       if (start.includes(",")) return start; // already a range CFI
-      const re =
-        /^(epubcfi\(\/\d+\/\d+!\/\d+\/\d+(?:\[[^\]]*\])?)(\/(?:[^:)]+)?(?::(\d+))?)\)$/;
-      const ms = re.exec(start);
-      if (!ms) return start;
-      const base = ms[1];
-      const startLocal = ms[2];
-      const startOff = ms[3] ? parseInt(ms[3], 10) : -1;
+      const spineRe =
+        /^(epubcfi\([^!]*!)((?:\/\d+(?:\[[^\]]*\])?)+)(?::(\d+))?\)$/;
+      const parseCfi = (cfi: string) => {
+        const m = spineRe.exec(cfi);
+        if (!m) return null;
+        const steps = (m[2].match(/\/\d+(?:\[[^\]]*\])?/g) ?? []).map(
+          (s) => s,
+        );
+        return { spine: m[1], steps, off: m[3] ? parseInt(m[3], 10) : -1 };
+      };
+      const s = parseCfi(start);
+      if (!s || s.steps.length === 0) return start;
       const utf16len = [...(text ?? "")].reduce(
         (n, c) => n + (c.codePointAt(0)! > 0xffff ? 2 : 1),
         0,
       );
-      let endLocal = "";
-      if (end && !end.includes(",")) {
-        const me = re.exec(end);
-        if (me && me[1] === base) {
-          const endOff = me[3] ? parseInt(me[3], 10) : -1;
-          // Degenerate: end resolves to the document start (the old
-          // converter fallback) or sits before the start offset.
-          const degenerate =
-            endOff === 0 ||
-            (startOff >= 0 && endOff >= 0 && endOff < startOff);
-          if (!degenerate) endLocal = me[2];
+      // Range with the parent one step short of the terminal, so both
+      // subpaths keep at least their terminal step; offsets ride on the
+      // subpath terminators.
+      const rem = (steps: string[], off: number) =>
+        off >= 0 ? `${steps.join("")}:${off}` : steps.join("");
+      const advanceEnd = () => {
+        if (s.off < 0 || utf16len <= 0) return null;
+        const last = s.steps[s.steps.length - 1];
+        const parent = s.spine + s.steps.slice(0, -1).join("");
+        return `${parent},${rem([last], s.off)},${rem([last], s.off + utf16len)})`;
+      };
+      const e = end && !end.includes(",") ? parseCfi(end) : null;
+      if (!e || e.spine !== s.spine || e.steps.length === 0) {
+        const adv = advanceEnd();
+        return adv ?? start; // point CFI
+      }
+      let common = 0;
+      const maxCommon = Math.min(s.steps.length, e.steps.length);
+      while (common < maxCommon && s.steps[common] === e.steps[common]) {
+        common++;
+      }
+      if (common === maxCommon) common = maxCommon - 1;
+      // Same terminal node only when the single remaining steps are
+      // EQUAL — one step per side with different step values is a
+      // legitimate two-node range. Degenerate ends (document-start
+      // fallback of the old converter, or an end before the start) are
+      // repaired via the UTF-16 advance.
+      if (
+        common === s.steps.length - 1 &&
+        common === e.steps.length - 1 &&
+        s.steps[common] === e.steps[common]
+      ) {
+        const degenerate =
+          e.off === 0 || (s.off >= 0 && e.off >= 0 && e.off < s.off);
+        if (degenerate) {
+          const adv = advanceEnd();
+          if (adv) return adv;
+        }
+      } else {
+        // Different terminal nodes: require end after start in document
+        // order; otherwise repair.
+        const ai = parseInt(s.steps[common].slice(1), 10);
+        const bi = parseInt(e.steps[common].slice(1), 10);
+        if (bi < ai) {
+          const adv = advanceEnd();
+          return adv ?? start;
         }
       }
-      if (!endLocal) {
-        if (startOff < 0 || utf16len <= 0) return start; // point CFI
-        const cut = startLocal.lastIndexOf(":");
-        endLocal = `${startLocal.slice(0, cut)}:${startOff + utf16len}`;
-      }
-      return `${base},${startLocal},${endLocal})`;
+      const parent = s.spine + s.steps.slice(0, common).join("");
+      return `${parent},${rem(s.steps.slice(common), s.off)},${rem(
+        e.steps.slice(common),
+        e.off,
+      )})`;
     },
     async refreshAnnotations() {
       const token = getToken();