fix(sync): normalize character offsets to UTF-16 at the wire; refresh book offset on every verified save
Offset currency policy, now explicit: EPUB CFI terminals, CRE text() offsets and the served char_offset handle are UTF-16 code units (the EPUB CFI spec, and what foliate/readium/KOReader/Kobo clients actually observe), while internal arithmetic — the book-wide character_offset column and percentage fractions — stays rune-based, consistent with TotalCharacters. For all-BMP books the currencies are identical, so no stored value changes; astral-plane text (emoji, rare CJK) no longer drifts. Boundaries converted: resolveCFIToNode interprets incoming CFI terminal offsets as UTF-16; textNodeAtUTF16Offset (née textNodeAtRuneOffset) interprets CRE text() offsets as UTF-16; buildCFI and buildCREXPointer emit UTF-16 terminals; blockCharOffset (the served char_offset) is UTF-16. Also fixes two character_offset column defects: heals wrote a BLOCK- relative offset into the book-wide column, and verified-but-unhealed saves (e.g. KOReader pushes) never refreshed it, leaving it stale behind the anchor. VerifyProgressAnchor now returns the verified book- wide rune offset and SaveProgress refreshes the column on every verified save. Tests: astral currency round trip (offset after an emoji must shift by one unit between currencies, in both heal and exact-verify directions) and book-offset ordering. The cmd/server/tests integration harness failures under docker (library folder 400 during setup) reproduce on the pre-change tree and are unrelated.
This commit is contained in:
John O'Keefe
+60
-26
@@ -26,6 +26,7 @@ package sync
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
"unicode/utf8"
|
||||
|
||||
"golang.org/x/net/html"
|
||||
)
|
||||
@@ -121,18 +122,45 @@ func cssSelectorFor(block *html.Node) string {
|
||||
return "body>" + strings.Join(segs, ">")
|
||||
}
|
||||
|
||||
// blockCharOffset computes the rune offset of (node, runeOff) within the
|
||||
// concatenated text of its block — the client-side scroll target.
|
||||
// blockCharOffset computes the offset of (node, runeOff) within the
|
||||
// concatenated text of its block, in UTF-16 code units — the client-side
|
||||
// scroll-target currency (JavaScript .length semantics).
|
||||
func blockCharOffset(block, node *html.Node, runeOff int) int {
|
||||
segments := collectInlineText(block)
|
||||
s := 0
|
||||
var sb strings.Builder
|
||||
for _, seg := range segments {
|
||||
if seg.node == node {
|
||||
return s + runeOff
|
||||
runes := seg.runes
|
||||
if runeOff < len(runes) {
|
||||
runes = runes[:runeOff]
|
||||
}
|
||||
sb.WriteString(string(runes))
|
||||
return runeToUTF16Index(sb.String(), utf8.RuneCountInString(sb.String()))
|
||||
}
|
||||
s += len(seg.runes)
|
||||
sb.WriteString(string(seg.runes))
|
||||
}
|
||||
return s + runeOff
|
||||
return utf16Len(sb.String())
|
||||
}
|
||||
|
||||
// bookCharOffset computes the book-wide rune offset of (node, runeOff) —
|
||||
// the reading_progress.character_offset column's currency, consistent with
|
||||
// TotalCharacters and the percentage derivations.
|
||||
func bookCharOffset(conv *CFIConverter, spineIndex int, node *html.Node, runeOff int) int {
|
||||
spine, err := conv.loadSpine()
|
||||
if err != nil {
|
||||
return 0
|
||||
}
|
||||
before := 0
|
||||
for i := 0; i < spineIndex && i < len(spine.items); i++ {
|
||||
doc, _, derr := conv.getContentDoc(i + 1)
|
||||
if derr != nil {
|
||||
continue
|
||||
}
|
||||
if b := findBody(doc); b != nil {
|
||||
before += countTextChars(b)
|
||||
}
|
||||
}
|
||||
return before + countTextCharsBefore(node) + runeOff
|
||||
}
|
||||
|
||||
// ProgressAnchor is the full server-computed apply handle for a stored
|
||||
@@ -149,15 +177,19 @@ type ProgressAnchor struct {
|
||||
|
||||
// VerifyProgressAnchor resolves a client-submitted standard CFI against
|
||||
// the EPUB, cross-checks the submitted context text, and heals the anchor
|
||||
// by text search on any mismatch.
|
||||
func VerifyProgressAnchor(epubPath, epubcfi, contextText string, percentage float64) (finalCFI string, cssSelector string, anchorHref string, charOffset *int, healedPct *float64, healed bool, err error) {
|
||||
// by text search on any mismatch. charOffset is the anchor's block-
|
||||
// relative UTF-16 offset (the served char_offset handle); bookOffset is
|
||||
// the anchor's book-wide rune offset (the character_offset column's
|
||||
// currency) — callers refresh the column from it on every verified save
|
||||
// so it never goes stale behind the anchor.
|
||||
func VerifyProgressAnchor(epubPath, epubcfi, contextText string, percentage float64) (finalCFI string, cssSelector string, anchorHref string, charOffset *int, bookOffset *int, healedPct *float64, healed bool, err error) {
|
||||
finalCFI = epubcfi
|
||||
anchorHref = ""
|
||||
|
||||
spineIndex, localSteps, err := parseEPUBCFI(epubcfi)
|
||||
if err != nil {
|
||||
cfi, sel, href, off, pct, healedFlag, herr := healFromContext(epubPath, contextText, percentage)
|
||||
return cfi, sel, href, off, pct, healedFlag, herr
|
||||
cfi, sel, href, off, book, pct, healedFlag, herr := healFromContext(epubPath, contextText, percentage)
|
||||
return cfi, sel, href, off, book, pct, healedFlag, herr
|
||||
}
|
||||
conv := cachedConverter(epubPath)
|
||||
doc, docHref, err := conv.getContentDoc(spineIndex + 1)
|
||||
@@ -174,40 +206,41 @@ func VerifyProgressAnchor(epubPath, epubcfi, contextText string, percentage floa
|
||||
serverCtx := blockContextText(block, node, runeOff)
|
||||
if contextMatches(serverCtx, contextText) {
|
||||
off := blockCharOffset(block, node, runeOff)
|
||||
return finalCFI, cssSelectorFor(block), anchorHref, &off, nil, false, nil
|
||||
book := bookCharOffset(conv, spineIndex, node, runeOff)
|
||||
return finalCFI, cssSelectorFor(block), anchorHref, &off, &book, nil, false, nil
|
||||
}
|
||||
|
||||
// Mismatch: heal by text search.
|
||||
hCFI, hPct, hSel, hHref, herr := healAnchorByText(epubPath, contextText, percentage, spineIndex)
|
||||
hCFI, hPct, hSel, hHref, hBook, herr := healAnchorByText(epubPath, contextText, percentage, spineIndex)
|
||||
if herr != nil {
|
||||
return finalCFI, "", hHref, nil, nil, false, fmt.Errorf("context mismatch (server %q vs client %q) and heal failed: %w",
|
||||
return finalCFI, "", hHref, nil, nil, nil, false, fmt.Errorf("context mismatch (server %q vs client %q) and heal failed: %w",
|
||||
truncateRunes(serverCtx, 40), truncateRunes(contextText, 40), herr)
|
||||
}
|
||||
hOff := blockCharOffsetFor(epubPath, hCFI)
|
||||
return hCFI, hSel, hHref, &hOff, &hPct, true, nil
|
||||
return hCFI, hSel, hHref, &hOff, &hBook, &hPct, true, nil
|
||||
}
|
||||
|
||||
func healFromContext(epubPath, contextText string, percentage float64) (string, string, string, *int, *float64, bool, error) {
|
||||
cfi, healedPct, sel, href, err := healAnchorByText(epubPath, contextText, percentage, -1)
|
||||
func healFromContext(epubPath, contextText string, percentage float64) (string, string, string, *int, *int, *float64, bool, error) {
|
||||
cfi, healedPct, sel, href, book, err := healAnchorByText(epubPath, contextText, percentage, -1)
|
||||
if err != nil {
|
||||
return "", "", "", nil, nil, false, err
|
||||
return "", "", "", nil, nil, nil, false, err
|
||||
}
|
||||
off := blockCharOffsetFor(epubPath, cfi)
|
||||
return cfi, sel, href, &off, &healedPct, true, nil
|
||||
return cfi, sel, href, &off, &book, &healedPct, true, nil
|
||||
}
|
||||
|
||||
func healAnchorByText(epubPath, contextText string, percentage float64, spineIndex int) (string, float64, string, string, error) {
|
||||
func healAnchorByText(epubPath, contextText string, percentage float64, spineIndex int) (string, float64, string, string, int, error) {
|
||||
if epubPath == "" {
|
||||
return "", 0, "", "", fmt.Errorf("no epub available for text anchoring")
|
||||
return "", 0, "", "", 0, fmt.Errorf("no epub available for text anchoring")
|
||||
}
|
||||
conv := cachedConverter(epubPath)
|
||||
spine, err := conv.loadSpine()
|
||||
if err != nil {
|
||||
return "", 0, "", "", err
|
||||
return "", 0, "", "", 0, err
|
||||
}
|
||||
needle := truncateRunes(normalizeWhitespace(contextText), 40)
|
||||
if len([]rune(needle)) < 12 {
|
||||
return "", 0, "", "", fmt.Errorf("context too short to anchor")
|
||||
return "", 0, "", "", 0, fmt.Errorf("context too short to anchor")
|
||||
}
|
||||
|
||||
type match struct {
|
||||
@@ -234,7 +267,7 @@ func healAnchorByText(epubPath, contextText string, percentage float64, spineInd
|
||||
}
|
||||
}
|
||||
if len(matches) == 0 || totalChars <= 0 {
|
||||
return "", 0, "", "", fmt.Errorf("context not found in book")
|
||||
return "", 0, "", "", 0, fmt.Errorf("context not found in book")
|
||||
}
|
||||
|
||||
best := matches[0]
|
||||
@@ -251,16 +284,17 @@ func healAnchorByText(epubPath, contextText string, percentage float64, spineInd
|
||||
}
|
||||
}
|
||||
|
||||
healedPct := (float64(charsBefore[best.spine]) + float64(countTextCharsBefore(best.node)+best.off)) / float64(totalChars)
|
||||
bookOff := charsBefore[best.spine] + countTextCharsBefore(best.node) + best.off
|
||||
healedPct := float64(bookOff) / float64(totalChars)
|
||||
cfi, err := buildCFI(best.spine, best.node, best.off)
|
||||
if err != nil {
|
||||
return "", 0, "", "", err
|
||||
return "", 0, "", "", 0, err
|
||||
}
|
||||
sel := ""
|
||||
if block := anchorBlock(best.node); block != nil {
|
||||
sel = cssSelectorFor(block)
|
||||
}
|
||||
return cfi, healedPct, sel, spine.items[best.spine].href, nil
|
||||
return cfi, healedPct, sel, spine.items[best.spine].href, bookOff, nil
|
||||
}
|
||||
|
||||
func blockCharOffsetFor(epubPath, cfi string) int {
|
||||
|
||||
Reference in New Issue
Block a user