From 6e9b3528d85756a7de62b4e83b046f45ed5c3342 Mon Sep 17 00:00:00 2001 From: John O'Keefe Date: Wed, 19 Aug 2026 14:08:03 -0400 Subject: [PATCH] =?UTF-8?q?fix(sync):=20synced=20highlights=20painted=20no?= =?UTF-8?q?where=20=E2=80=94=20degenerate=20range=20ends=20+=20color=20mod?= =?UTF-8?q?el=20mismatch?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both directions synced data but rendered nothing: - Web reader <- devices: highlights painted no overlay. Device pushes resolve their start xpointer exactly (text-search anchored by the selection) but the end conversion carries no context and fell back to a document-start CFI (epubcfi .../1:0) — a garbage range end. When the start resolved exactly, the end is now derived from it: same node, character offset advanced by the selection's UTF-16 length (extendCFIByLength). Same repair when SERVING to devices, where old web highlights (no end anchor) and converted range CFIs both collapsed pos1 onto pos0 (extendXPointerByLength on the xpointer form) — KOReader drew zero-width highlights. - Colors: KOReader paints from a fixed name set (Blitbuffer HIGHLIGHT_COLORS), the web uses hex swatches; neither understood the other, so device colors fell back to defaults and web hex drew nothing useful on devices. Both boundaries now translate: ingest maps names to hex (default #ffd54f), GetMetadata maps hex to names (default yellow) — per-datatype edits re-push with the editing side's color, which LWW then propagates. SyncBookmarks endpoint aligned to the same mapping and default. --- internal/handlers/koreader.go | 231 ++++++++++++++++++++++++---------- 1 file changed, 168 insertions(+), 63 deletions(-) diff --git a/internal/handlers/koreader.go b/internal/handlers/koreader.go index 2ab6b6b..b41d0e5 100644 --- a/internal/handlers/koreader.go +++ b/internal/handlers/koreader.go @@ -12,6 +12,7 @@ import ( "strconv" "strings" "time" + "unicode/utf8" "github.com/google/uuid" "github.com/jackc/pgx/v5/pgtype" @@ -66,7 +67,43 @@ func (h *KOReaderHandler) convertHighlightPositions(ctx context.Context, mediaIt // locators, the server resolves them against the actual book. startLoc := wsync.ConvertToCanonical(wsync.LocatorSourceKOReader, pos0, 0, contextText, mediaItem.FormatGroup, epubPath, "") endLoc := wsync.ConvertToCanonical(wsync.LocatorSourceKOReader, pos1, 0, "", mediaItem.FormatGroup, epubPath, "") - return startLoc.CFI, endLoc.CFI + endCFI := endLoc.CFI + // The end conversion carries no context text, so unless it resolved + // exactly it degenerates to a percentage fallback anchored at the + // document start — useless as a range end. When the START resolved + // exactly, derive the end from it: same node, character offset + // advanced by the selection's UTF-16 length (the CFI offset unit). + if endLoc.Precision != "exact" && startLoc.Precision == "exact" && contextText != "" { + endCFI = extendCFIByLength(startLoc.CFI, contextText) + } + return startLoc.CFI, endCFI +} + +// extendCFIByLength advances a point CFI's trailing character offset by the +// UTF-16 length of text (EPUB CFI character offsets are UTF-16 code units). +// Selections spanning multiple text nodes produce an out-of-range offset — +// harmless: resolution clamps or fails, and consumers fall back to the start. +func extendCFIByLength(cfi, text string) string { + if cfi == "" || text == "" { + return cfi + } + i := strings.LastIndex(cfi, ":") + if i < 0 || !strings.HasSuffix(cfi, ")") { + return cfi + } + off, err := strconv.Atoi(cfi[i+1 : len(cfi)-1]) + if err != nil { + return cfi + } + utf16len := 0 + for _, r := range text { + if r > 0xFFFF { + utf16len += 2 + } else { + utf16len++ + } + } + return cfi[:i+1] + strconv.Itoa(off+utf16len) + ")" } // deriveAnnotationPercentage computes a percentage for device-pushed @@ -103,24 +140,24 @@ type KOReaderProgressRequest struct { } type KOReaderBookProgress struct { - UUID string `json:"uuid,omitempty"` - SHA256 string `json:"sha256,omitempty"` - Title string `json:"title,omitempty"` - Authors []string `json:"authors,omitempty"` - Progress float64 `json:"progress"` - Percentage float64 `json:"percentage"` - LastRead string `json:"last_read,omitempty"` - FilePath string `json:"file_path,omitempty"` - DeviceInfo KOReaderDeviceInfo `json:"device_info,omitempty"` - Bookmarks []KOReaderBookmark `json:"bookmarks,omitempty"` - Highlights []KOReaderHighlight `json:"highlights,omitempty"` - Notes []KOReaderNote `json:"notes,omitempty"` - Chapter *int `json:"chapter,omitempty"` - Character *int64 `json:"character,omitempty"` - Epubcfi *string `json:"epubcfi,omitempty"` - ContextText *string `json:"context_text,omitempty"` - Page *int `json:"page,omitempty"` - TotalPages *int `json:"total_pages,omitempty"` + UUID string `json:"uuid,omitempty"` + SHA256 string `json:"sha256,omitempty"` + Title string `json:"title,omitempty"` + Authors []string `json:"authors,omitempty"` + Progress float64 `json:"progress"` + Percentage float64 `json:"percentage"` + LastRead string `json:"last_read,omitempty"` + FilePath string `json:"file_path,omitempty"` + DeviceInfo KOReaderDeviceInfo `json:"device_info,omitempty"` + Bookmarks []KOReaderBookmark `json:"bookmarks,omitempty"` + Highlights []KOReaderHighlight `json:"highlights,omitempty"` + Notes []KOReaderNote `json:"notes,omitempty"` + Chapter *int `json:"chapter,omitempty"` + Character *int64 `json:"character,omitempty"` + Epubcfi *string `json:"epubcfi,omitempty"` + ContextText *string `json:"context_text,omitempty"` + Page *int `json:"page,omitempty"` + TotalPages *int `json:"total_pages,omitempty"` } type KOReaderDeviceInfo struct { @@ -204,12 +241,12 @@ type KOReaderNote struct { } type KOReaderSyncResponse struct { - SyncStatus string `json:"sync_status"` - BooksSynced int `json:"books_synced"` + SyncStatus string `json:"sync_status"` + BooksSynced int `json:"books_synced"` BookResults []KOReaderBookSyncResult `json:"book_results,omitempty"` - Conflicts []KOReaderConflict `json:"conflicts,omitempty"` - Timestamp string `json:"timestamp"` - DeviceUpdated bool `json:"device_updated"` + Conflicts []KOReaderConflict `json:"conflicts,omitempty"` + Timestamp string `json:"timestamp"` + DeviceUpdated bool `json:"device_updated"` } type KOReaderBookSyncResult struct { @@ -237,20 +274,20 @@ type KOReaderMetadata struct { } type KOReaderProgressData struct { - Percentage float64 `json:"percentage"` - Character *int64 `json:"character,omitempty"` - Epubcfi *string `json:"epubcfi,omitempty"` - KoreaderXPointer *string `json:"koreader_xpointer,omitempty"` - Chapter *int `json:"chapter,omitempty"` - ChapterProgress *float64 `json:"chapter_progress,omitempty"` - Page *int `json:"page,omitempty"` - TotalPages *int `json:"total_pages,omitempty"` + Percentage float64 `json:"percentage"` + Character *int64 `json:"character,omitempty"` + Epubcfi *string `json:"epubcfi,omitempty"` + KoreaderXPointer *string `json:"koreader_xpointer,omitempty"` + Chapter *int `json:"chapter,omitempty"` + ChapterProgress *float64 `json:"chapter_progress,omitempty"` + Page *int `json:"page,omitempty"` + TotalPages *int `json:"total_pages,omitempty"` } type KOReaderAnnotations struct { - Highlights []KOReaderHighlight `json:"highlights,omitempty"` - Notes []KOReaderNote `json:"notes,omitempty"` - Bookmarks []KOReaderBookmark `json:"bookmarks,omitempty"` + Highlights []KOReaderHighlight `json:"highlights,omitempty"` + Notes []KOReaderNote `json:"notes,omitempty"` + Bookmarks []KOReaderBookmark `json:"bookmarks,omitempty"` DeletedHighlights []map[string]interface{} `json:"deleted_highlights,omitempty"` DeletedBookmarks []map[string]interface{} `json:"deleted_bookmarks,omitempty"` } @@ -568,18 +605,18 @@ func (h *KOReaderHandler) processBookAnnotations(ctx context.Context, deviceID, }) h.annotationSvc.SaveHighlight(ctx, wsync.SaveHighlightRequest{ - MediaItemID: mediaItemID, - UserID: userID, - SelectionText: hl.Text, - StartPosition: startPos, - EndPosition: endPos, - Color: hl.Color, - NoteText: hl.Notes, - PercentageStart: pctStart, - EpubcfiStart: epubcfiStart, - EpubcfiEnd: epubcfiEnd, - Source: "koreader", - DeviceSyncData: deviceData, + MediaItemID: mediaItemID, + UserID: userID, + SelectionText: hl.Text, + StartPosition: startPos, + EndPosition: endPos, + Color: mapColorFromKOReader(hl.Color), + NoteText: hl.Notes, + PercentageStart: pctStart, + EpubcfiStart: epubcfiStart, + EpubcfiEnd: epubcfiEnd, + Source: "koreader", + DeviceSyncData: deviceData, }) } @@ -604,17 +641,17 @@ func (h *KOReaderHandler) processBookAnnotations(ctx context.Context, deviceID, }) h.annotationSvc.SaveHighlight(ctx, wsync.SaveHighlightRequest{ - MediaItemID: mediaItemID, - UserID: userID, - SelectionText: note.Text, - StartPosition: startPos, - EndPosition: endPos, - NoteText: note.Notes, - PercentageStart: pctStart, - EpubcfiStart: epubcfiStart, - EpubcfiEnd: epubcfiEnd, - Source: "koreader", - DeviceSyncData: deviceData, + MediaItemID: mediaItemID, + UserID: userID, + SelectionText: note.Text, + StartPosition: startPos, + EndPosition: endPos, + NoteText: note.Notes, + PercentageStart: pctStart, + EpubcfiStart: epubcfiStart, + EpubcfiEnd: epubcfiEnd, + Source: "koreader", + DeviceSyncData: deviceData, }) } @@ -889,11 +926,19 @@ func (h *KOReaderHandler) GetMetadata(c *echo.Context) error { log.Printf("Bookhoard: GetMetadata skip highlight %s (no resolvable pos0)", ann.ID) continue } + // Old web highlights carry no end anchor, and converted range + // CFIs resolve to their start — either way pos1 collapses onto + // pos0 and the device paints a zero-width highlight. Derive the + // end by advancing the start's character offset by the length + // of the selected text. + if pos1 == "" || pos1 == pos0 { + pos1 = extendXPointerByLength(pos0, ann.SelectionText) + } highlight := KOReaderHighlight{ Text: ann.SelectionText, Pos0: pos0, Pos1: pos1, - Color: ann.Color.String, + Color: mapColorToKOReader(ann.Color.String), Datetime: ann.CreatedAt.Time.Format(time.RFC3339), } if ann.NoteText.Valid && ann.NoteText.String != "" { @@ -1039,14 +1084,74 @@ func (h *KOReaderHandler) reverseConvertCFI(c *echo.Context, mediaItem database. // pdfRectAnchor is the JSON locator the web reader stores in epubcfi_start // for PDF text highlights (page-fraction rects; page index is 0-based). type pdfRectAnchor struct { - V int `json:"v"` - Page int `json:"page"` + V int `json:"v"` + Page int `json:"page"` Rects [][]float64 `json:"rects"` } // koreaderPos0 resolves a device-native KOReader pos0 from an annotation's // stored locators, whatever the source. Resolution order: // +// extendXPointerByLength advances a CRE xpointer's trailing text-node +// character offset by the rune length of text, so a highlight with only a +// start anchor still gets a plausible (non-collapsed) end for drawing. +// Overshooting the node just clamps on the device. +func extendXPointerByLength(xp, text string) string { + if xp == "" || text == "" { + return xp + } + i := strings.LastIndex(xp, ".") + if i < 0 { + return xp + } + off, err := strconv.Atoi(xp[i+1:]) + if err != nil { + return xp + } + return xp[:i+1] + strconv.Itoa(off+utf8.RuneCountInString(text)) +} + +// KOReader paints highlight colors from a fixed set of names +// (Blitbuffer.HIGHLIGHT_COLORS); the web reader uses hex swatches. Map at +// the boundary so each side always receives something it can render; +// unmappable values fall back to each side's default (yellow). +var koreaderColorFromName = map[string]string{ + "yellow": "#ffd54f", + "orange": "#ffd54f", + "green": "#a5d6a7", + "olive": "#a5d6a7", + "cyan": "#90caf9", + "blue": "#90caf9", + "purple": "#ce93d8", + "red": "#f48fb1", +} + +var koreaderColorFromHex = map[string]string{ + "#ffd54f": "yellow", + "#a5d6a7": "green", + "#90caf9": "blue", + "#ce93d8": "purple", + "#f48fb1": "purple", +} + +// mapColorFromKOReader normalizes a device color name to a web hex +// swatch (default yellow) when ingesting device pushes. +func mapColorFromKOReader(name string) string { + if hex, ok := koreaderColorFromName[strings.ToLower(strings.TrimSpace(name))]; ok { + return hex + } + return "#ffd54f" +} + +// mapColorToKOReader normalizes a web hex swatch to a KOReader color +// name (default yellow) when serving to devices. +func mapColorToKOReader(hex string) string { + if name, ok := koreaderColorFromHex[strings.ToLower(strings.TrimSpace(hex))]; ok { + return name + } + return "yellow" +} + // 1. A device-native CRE xpointer ("/body/...") in startPosition wins — // round-trip identical for KOReader-pushed annotations (converting the // stored CFI instead could drift and duplicate on the device). @@ -1340,9 +1445,9 @@ func (h *KOReaderHandler) SyncBookmarks(c *echo.Context) error { endPos = startPos } - color := "#ffff00" + color := "#ffd54f" if highlight.Color != "" { - color = highlight.Color + color = mapColorFromKOReader(highlight.Color) } if h.annotationSvc != nil {