test(sync): add manual repro harness for CRE-to-CFI highlight conversion
TestManualCleanRepro replays a real device highlight export through the same conversion path the sync handler uses, diffs the output against the CFIs stored at ingest time, and computes a ground-truth oracle by locating each selection in the flattened document itself. Classifies every row as healthy / diff-base / backwards for both the converter and the stored data. Skips automatically when fixture files are not present on the machine (manual test, like cfi_verify_manual_test.go). REPRO_VERBOSE=1 prints per-row detail; REPRO_EMIT_SQL=<path> emits idempotent UPDATE statements that heal database rows written by the old converter from the preserved raw device positions - used to backfill dev and validated against production data.
This commit is contained in:
John O'Keefe committed
1 parent
93a468b435
commit
03a6040360
1 file changed
+379
@@ -0,0 +1,379 @@
|
||||
package sync
|
||||
|
||||
// Manual repro harness for the CRE→CFI highlight conversion defects found in
|
||||
// "Clean" (Rosser K Powitzky): 28 device highlights whose stored web CFIs are
|
||||
// structurally inconsistent (mismatched start/end base paths, backwards
|
||||
// ranges). This test mirrors handlers.convertHighlightPositions exactly —
|
||||
// pos0 converted WITH the selection text as context, pos1 WITHOUT any — then
|
||||
// reports the result alongside the CFIs stored at ingest time and a
|
||||
// ground-truth oracle computed by locating the selection text in the document
|
||||
// itself. Skips silently when the book/export are not on this machine, like
|
||||
// cfi_verify_manual_test.go.
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"golang.org/x/net/html"
|
||||
)
|
||||
|
||||
const (
|
||||
reproExport = "/home/nymusicman/Code/bookhoard/export/media_highlights.json"
|
||||
reproEpub = "/home/nymusicman/Code/bookhoard/uploads/Ebooks/Rosser K Powitzky/Clean_ How the Jewish Roots of Atone (77)/Clean_ How the Jewish Roots of - Rosser K Powitzky.epub"
|
||||
)
|
||||
|
||||
type reproRow struct {
|
||||
ID string `json:"id"`
|
||||
StartPosition string `json:"start_position"`
|
||||
EndPosition string `json:"end_position"`
|
||||
StoredStart string `json:"epubcfi_start"`
|
||||
StoredEnd string `json:"epubcfi_end"`
|
||||
SelectionText string `json:"selection_text"`
|
||||
PctStart float64 `json:"percentage_start"`
|
||||
}
|
||||
|
||||
// webUsable mirrors handlers.webUsableCFI (unexported there; cannot import
|
||||
// the handlers package from here without a cycle).
|
||||
func webUsable(precision, cfi string) string {
|
||||
if precision != "structural" && precision != "exact" {
|
||||
return ""
|
||||
}
|
||||
if !strings.HasPrefix(cfi, "epubcfi(") || !strings.HasSuffix(cfi, ")") {
|
||||
return ""
|
||||
}
|
||||
return cfi
|
||||
}
|
||||
|
||||
// reproExtendCFIByLength mirrors handlers.extendCFIByLength: advance the
|
||||
// trailing character offset of a point CFI by the UTF-16 length of text.
|
||||
func reproExtendCFIByLength(cfi, text string) string {
|
||||
if cfi == "" || text == "" {
|
||||
return ""
|
||||
}
|
||||
if !strings.HasSuffix(cfi, ")") {
|
||||
return ""
|
||||
}
|
||||
inner := cfi[:len(cfi)-1]
|
||||
idx := strings.LastIndex(inner, ":")
|
||||
if idx < 0 {
|
||||
return cfi
|
||||
}
|
||||
var off int
|
||||
if _, err := fmtSscanInt(inner[idx+1:], &off); err != nil {
|
||||
return cfi
|
||||
}
|
||||
return inner[:idx+1] + itoa(off+utf16Len(text)) + ")"
|
||||
}
|
||||
|
||||
// cfiBase splits "epubcfi(...)" into (base, offset): base is everything
|
||||
// before the final ":N" character-offset terminator.
|
||||
func cfiBase(cfi string) (string, int, bool) {
|
||||
if !strings.HasSuffix(cfi, ")") {
|
||||
return "", 0, false
|
||||
}
|
||||
inner := cfi[:len(cfi)-1]
|
||||
idx := strings.LastIndex(inner, ":")
|
||||
if idx < 0 {
|
||||
return inner, 0, true
|
||||
}
|
||||
off := 0
|
||||
if _, err := fmtSscanInt(inner[idx+1:], &off); err != nil {
|
||||
return inner, 0, true
|
||||
}
|
||||
return inner[:idx], off, true
|
||||
}
|
||||
|
||||
func fmtSscanInt(s string, out *int) (int, error) {
|
||||
n := 0
|
||||
ok := false
|
||||
for _, r := range s {
|
||||
if r < '0' || r > '9' {
|
||||
break
|
||||
}
|
||||
n = n*10 + int(r-'0')
|
||||
ok = true
|
||||
}
|
||||
if !ok {
|
||||
return 0, errReproNoInt
|
||||
}
|
||||
*out = n
|
||||
return len(s), nil
|
||||
}
|
||||
|
||||
var errReproNoInt = errReproNoIntErr{}
|
||||
|
||||
type errReproNoIntErr struct{}
|
||||
|
||||
func (errReproNoIntErr) Error() string { return "no integer" }
|
||||
|
||||
func itoa(n int) string {
|
||||
if n == 0 {
|
||||
return "0"
|
||||
}
|
||||
neg := n < 0
|
||||
if neg {
|
||||
n = -n
|
||||
}
|
||||
var b [20]byte
|
||||
i := len(b)
|
||||
for n > 0 {
|
||||
i--
|
||||
b[i] = byte('0' + n%10)
|
||||
n /= 10
|
||||
}
|
||||
if neg {
|
||||
i--
|
||||
b[i] = '-'
|
||||
}
|
||||
return string(b[i:])
|
||||
}
|
||||
|
||||
// oracleCFIs locates the selection text in the document named by the start
|
||||
// xpointer's fragment and returns ground-truth (startCFI, endCFI, spineIndex).
|
||||
// Whitespace runs are collapsed on both sides before matching, so device
|
||||
// text with reflowed newlines still lands.
|
||||
func oracleCFIs(t *testing.T, conv *CFIConverter, row reproRow) (string, string, string) {
|
||||
xp, err := ParseCREXPointer(row.StartPosition)
|
||||
if err != nil {
|
||||
return "", "", "unparseable start xpointer"
|
||||
}
|
||||
doc, href, err := conv.getContentDoc(xp.FragmentIndex)
|
||||
if err != nil {
|
||||
return "", "", "content doc missing: " + err.Error()
|
||||
}
|
||||
body := findBody(doc)
|
||||
if body == nil {
|
||||
return "", "", "no body"
|
||||
}
|
||||
|
||||
// Flatten every text node in document order.
|
||||
var segs []textSegment
|
||||
var walk func(*html.Node)
|
||||
walk = func(n *html.Node) {
|
||||
if n.Type == html.TextNode {
|
||||
segs = append(segs, textSegment{node: n, runes: []rune(n.Data)})
|
||||
}
|
||||
for c := n.FirstChild; c != nil; c = c.NextSibling {
|
||||
walk(c)
|
||||
}
|
||||
}
|
||||
walk(body)
|
||||
|
||||
var flat []rune
|
||||
for _, s := range segs {
|
||||
flat = append(flat, s.runes...)
|
||||
}
|
||||
|
||||
needle := []rune(normalizeWhitespace(row.SelectionText))
|
||||
if len(needle) == 0 {
|
||||
return "", "", "empty selection text"
|
||||
}
|
||||
// Collapse whitespace in `flat` while mapping each kept rune back to a
|
||||
// flat index.
|
||||
var norm []rune
|
||||
var normToFlat []int
|
||||
lastWS := false
|
||||
for i, r := range flat {
|
||||
isWS := r == ' ' || r == '\t' || r == '\n' || r == '\r' || r == ' ' || r == ''
|
||||
if isWS {
|
||||
if lastWS {
|
||||
continue
|
||||
}
|
||||
norm = append(norm, ' ')
|
||||
normToFlat = append(normToFlat, i)
|
||||
lastWS = true
|
||||
continue
|
||||
}
|
||||
norm = append(norm, r)
|
||||
normToFlat = append(normToFlat, i)
|
||||
lastWS = false
|
||||
}
|
||||
|
||||
// Trim wrapped-quote artifacts KOReader adds to partial-paragraph text.
|
||||
trimmed := needle
|
||||
for len(trimmed) > 0 && (trimmed[0] == '{' || trimmed[0] == '}') {
|
||||
trimmed = trimmed[1:]
|
||||
}
|
||||
for len(trimmed) > 0 && (trimmed[len(trimmed)-1] == '{' || trimmed[len(trimmed)-1] == '}') {
|
||||
trimmed = trimmed[:len(trimmed)-1]
|
||||
}
|
||||
|
||||
pos := indexRunes(norm, trimmed)
|
||||
if pos < 0 {
|
||||
return "", "", "selection text not found in doc " + href
|
||||
}
|
||||
|
||||
flatStart := normToFlat[pos]
|
||||
flatEnd := normToFlat[pos+len(trimmed)-1] + 1 // end is exclusive
|
||||
|
||||
nodeAt := func(flatIdx int) (*html.Node, int) {
|
||||
acc := 0
|
||||
for _, s := range segs {
|
||||
n := len(s.runes)
|
||||
if flatIdx < acc+n {
|
||||
return s.node, flatIdx - acc
|
||||
}
|
||||
acc += n
|
||||
}
|
||||
return nil, 0
|
||||
}
|
||||
|
||||
startNode, startOff := nodeAt(flatStart)
|
||||
endNode, endOff := nodeAt(flatEnd)
|
||||
if startNode == nil || endNode == nil {
|
||||
return "", "", "flat index out of range"
|
||||
}
|
||||
|
||||
spineIndex := xp.FragmentIndex - 1
|
||||
startCFI, err := buildCFI(spineIndex, startNode, startOff)
|
||||
if err != nil {
|
||||
return "", "", "buildCFI start: " + err.Error()
|
||||
}
|
||||
endCFI, err := buildCFI(spineIndex, endNode, endOff)
|
||||
if err != nil {
|
||||
return "", "", "buildCFI end: " + err.Error()
|
||||
}
|
||||
return startCFI, endCFI, ""
|
||||
}
|
||||
|
||||
func indexRunes(hay, needle []rune) int {
|
||||
if len(needle) == 0 || len(needle) > len(hay) {
|
||||
return -1
|
||||
}
|
||||
for i := 0; i+len(needle) <= len(hay); i++ {
|
||||
match := true
|
||||
for j := range needle {
|
||||
if hay[i+j] != needle[j] {
|
||||
match = false
|
||||
break
|
||||
}
|
||||
}
|
||||
if match {
|
||||
return i
|
||||
}
|
||||
}
|
||||
return -1
|
||||
}
|
||||
|
||||
func TestManualCleanRepro(t *testing.T) {
|
||||
raw, err := os.ReadFile(reproExport)
|
||||
if err != nil {
|
||||
t.Skipf("export json not present: %v", err)
|
||||
}
|
||||
if _, err := os.Stat(reproEpub); err != nil {
|
||||
t.Skipf("book epub not present: %v", err)
|
||||
}
|
||||
var rows []reproRow
|
||||
if err := json.Unmarshal(raw, &rows); err != nil {
|
||||
t.Fatalf("unmarshal export: %v", err)
|
||||
}
|
||||
|
||||
conv := cachedConverter(reproEpub)
|
||||
|
||||
stats := map[string]int{}
|
||||
for _, row := range rows {
|
||||
// Mirror handlers.convertHighlightPositions (text-anchored primary
|
||||
// path, legacy per-endpoint fallback).
|
||||
startLoc := ConvertHighlightRangeToCanonical(reproEpub, "reflowable", row.StartPosition, row.EndPosition, row.SelectionText, row.PctStart)
|
||||
startCFI := webUsable(startLoc.Precision, startLoc.CFI)
|
||||
endCFI := ""
|
||||
if startCFI != "" && startLoc.EndCFI != "" {
|
||||
endCFI = webUsable(startLoc.Precision, startLoc.EndCFI)
|
||||
}
|
||||
if startCFI == "" {
|
||||
startLoc = ConvertToCanonical(LocatorSourceKOReader, row.StartPosition, row.PctStart, row.SelectionText, "reflowable", reproEpub, "")
|
||||
startCFI = webUsable(startLoc.Precision, startLoc.CFI)
|
||||
endCFI = ""
|
||||
if startCFI != "" && startLoc.EndCFI != "" {
|
||||
endCFI = startLoc.EndCFI
|
||||
}
|
||||
if endCFI == "" && row.EndPosition != "" {
|
||||
endLoc := ConvertToCanonical(LocatorSourceKOReader, row.EndPosition, row.PctStart, "", "reflowable", reproEpub, "")
|
||||
endCFI = webUsable(endLoc.Precision, endLoc.CFI)
|
||||
}
|
||||
if endCFI == "" && startCFI != "" && row.SelectionText != "" {
|
||||
endCFI = reproExtendCFIByLength(startCFI, row.SelectionText)
|
||||
}
|
||||
}
|
||||
|
||||
oracleStart, oracleEnd, oracleNote := oracleCFIs(t, conv, row)
|
||||
stats["oracleMiss:"+statusOr(oracleNote)]++
|
||||
|
||||
cls := func(a, b string) string {
|
||||
ba, oa, okA := cfiBase(a)
|
||||
bb, ob, okB := cfiBase(b)
|
||||
switch {
|
||||
case !okA || !okB || a == "" || b == "":
|
||||
return "missing"
|
||||
case ba != bb:
|
||||
return "diff-base"
|
||||
case ob <= oa:
|
||||
return "backwards"
|
||||
default:
|
||||
return "healthy"
|
||||
}
|
||||
}
|
||||
cur := cls(startCFI, endCFI)
|
||||
sto := cls(row.StoredStart, row.StoredEnd)
|
||||
ora := cls(oracleStart, oracleEnd)
|
||||
stats["current:"+cur]++
|
||||
stats["stored:"+sto]++
|
||||
stats["oracle:"+ora]++
|
||||
|
||||
if os.Getenv("REPRO_VERBOSE") != "" || cur != "healthy" || sto != "healthy" {
|
||||
t.Logf("row %s", row.ID[:8])
|
||||
t.Logf(" raw: %s -> %s", row.StartPosition, row.EndPosition)
|
||||
t.Logf(" ctx: %q", truncateRunes(row.SelectionText, 60))
|
||||
t.Logf(" stored: %-38s %-38s [%s]", row.StoredStart, row.StoredEnd, sto)
|
||||
t.Logf(" current: %-38s %-38s [%s]", startCFI, endCFI, cur)
|
||||
t.Logf(" oracle: %-38s %-38s [%s] %s", oracleStart, oracleEnd, ora, oracleNote)
|
||||
}
|
||||
}
|
||||
|
||||
t.Logf("summary: %v", stats)
|
||||
|
||||
// REPRO_EMIT_SQL=<path>: emit UPDATE statements carrying the NEW
|
||||
// converter output into the database rows this export came from
|
||||
// (one-off healing of rows written by the broken converter).
|
||||
if out := os.Getenv("REPRO_EMIT_SQL"); out != "" {
|
||||
var b strings.Builder
|
||||
b.WriteString("BEGIN;\n")
|
||||
for _, row := range rows {
|
||||
startLoc := ConvertHighlightRangeToCanonical(reproEpub, "reflowable", row.StartPosition, row.EndPosition, row.SelectionText, row.PctStart)
|
||||
startCFI := webUsable(startLoc.Precision, startLoc.CFI)
|
||||
endCFI := ""
|
||||
if startCFI != "" && startLoc.EndCFI != "" {
|
||||
endCFI = webUsable(startLoc.Precision, startLoc.EndCFI)
|
||||
}
|
||||
if startCFI == "" {
|
||||
continue // leave untouched; legacy fallback rows keep old CFIs
|
||||
}
|
||||
b.WriteString(fmt.Sprintf(
|
||||
"UPDATE media_highlights SET epubcfi_start=%s, epubcfi_end=%s WHERE id='%s';\n",
|
||||
pgQuote(startCFI), pgQuote(endCFI), row.ID,
|
||||
))
|
||||
}
|
||||
b.WriteString("COMMIT;\n")
|
||||
if err := os.WriteFile(out, []byte(b.String()), 0o600); err != nil {
|
||||
t.Fatalf("write sql: %v", err)
|
||||
}
|
||||
t.Logf("wrote %s", out)
|
||||
}
|
||||
}
|
||||
|
||||
func pgQuote(s string) string {
|
||||
if s == "" {
|
||||
return "''"
|
||||
}
|
||||
return "'" + strings.ReplaceAll(s, "'", "''") + "'"
|
||||
}
|
||||
|
||||
func statusOr(s string) string {
|
||||
if s == "" {
|
||||
return "found"
|
||||
}
|
||||
return s
|
||||
}
|
||||
Reference in new issue
Block a user