Bookhoard/bookhoard
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 12 Wiki Activity

fix(sync): web annotations never reached KOReader — bind 400s + unresolvable locators

Browse Source 
Two blockers, diagnosed by simulating the plugin against the live
server with real library books:

1. Every KOReader progress push carrying annotations failed the JSON
   bind with 400 ('cannot unmarshal string into ... chapter/page of
   type int') — the plugin sends chapter:'', page:'30', and for CRE
   documents page:'/body/...' — so annotation sync AND progress sync
   failed together. KOReader annotation chapter/page now use FlexInt,
   which accepts numbers, numeric strings, empty strings, and
   non-numeric strings (decoding to 0). The server is deliberately
   liberal here so thin clients can send raw bookmark data.

2. GetMetadata served locators KOReader cannot place, so pulled items
   were junk: web bookmarks leaked 'cfi:epubcfi(...)' positions, web
   PDF highlights had empty pos0 (skipped by the plugin, invisible),
   and web deletions carried no pos0 so tombstones never matched.
   New koreaderPos0 resolver handles every source: device-native
   xpointers pass through untouched (round-trip identical, verified),
   web PDF JSON anchors map to their page number, EPUB CFIs convert
   to CRE xpointers (selection text passed as text-search context for
   exact anchoring), 'page:N' positions strip to the bare number.
   Unresolvable annotations are skipped with a log line instead of
   poisoning devices; tombstones get pos0 injected from the new
   locator columns.

Also: thin clients omit per-annotation percentages (paging docs still
send arithmetic page/total); the server derives them — section
midpoint from the spine char distribution for CRE documents, page/
page-count for fixed formats.
This commit is contained in:
john-okeefe
2026-08-18 19:13:38 -04:00
parent 0670d904a0
commit f6e257e497
1 changed files with 174 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/handlers/koreader.go
+174 -29
View File
@@ -9,6 +9,8 @@ import (
"fmt"
"log"
"net/http"
"strconv"
"strings"
"time"
"github.com/google/uuid"
@@ -47,7 +49,7 @@ func (h *KOReaderHandler) SetAnnotationService(svc *wsync.AnnotationService) {
h.annotationSvc = svc
}
func (h *KOReaderHandler) convertHighlightPositions(ctx context.Context, mediaItemID pgtype.UUID, pos0, pos1 string) (string, string) {
func (h *KOReaderHandler) convertHighlightPositions(ctx context.Context, mediaItemID pgtype.UUID, pos0, pos1, contextText string) (string, string) {
if pos0 == "" || h.libraryService == nil {
return "", ""
}
@@ -59,11 +61,37 @@ func (h *KOReaderHandler) convertHighlightPositions(ctx context.Context, mediaIt
if err != nil || epubPath == "" {
return "", ""
}
startLoc := wsync.ConvertToCanonical(wsync.LocatorSourceKOReader, pos0, 0, "", mediaItem.FormatGroup, epubPath, "")
// The annotation's own text is the ideal anchor for the converter's
// text-search path: clients (thin, underpowered) send only raw
// locators, the server resolves them against the actual book.
startLoc := wsync.ConvertToCanonical(wsync.LocatorSourceKOReader, pos0, 0, contextText, mediaItem.FormatGroup, epubPath, "")
endLoc := wsync.ConvertToCanonical(wsync.LocatorSourceKOReader, pos1, 0, "", mediaItem.FormatGroup, epubPath, "")
return startLoc.CFI, endLoc.CFI
}
// deriveAnnotationPercentage computes a percentage for device-pushed
// annotations when the client didn't send one (thin clients skip their own
// per-annotation page lookups; arithmetic is only free on paging documents).
func (h *KOReaderHandler) deriveAnnotationPercentage(ctx context.Context, mediaItemID pgtype.UUID, pos0 string, page int) float64 {
mediaItem, err := h.db.GetMediaItem(ctx, mediaItemID)
if err != nil {
return 0
}
formatGroup := wsync.FormatGroup(mediaItem.FormatGroup)
if formatGroup == wsync.FormatGroupFixedLayout || formatGroup == wsync.FormatGroupComicArchive {
if page > 0 && mediaItem.PageCount.Valid && mediaItem.PageCount.Int32 > 0 {
return float64(page) / float64(mediaItem.PageCount.Int32)
}
return 0
}
if wsync.IsCREXPointer(pos0) && h.libraryService != nil {
if epubPath, err := h.libraryService.ResolveMediaPath(ctx, mediaItem.LibraryID, mediaItem.FilePath); err == nil && epubPath != "" {
return wsync.NewCFIConverter(epubPath).SectionPercentage(pos0)
}
}
return 0
}
func (h *KOReaderHandler) SetLibraryService(svc LibraryPathResolver) {
h.libraryService = svc
}
@@ -100,13 +128,48 @@ type KOReaderDeviceInfo struct {
DeviceModel string `json:"device_model,omitempty"`
}
// FlexInt tolerates the loose types KOReader clients send for optional
// numeric fields: JSON numbers, numeric strings ("30"), empty strings
// (""), or non-numeric strings ("/body/..." xpointers in `page` for CRE
// documents) — the latter decode to 0. Without this, a single annotation
// carrying chapter:"" or page:"/body/..." failed the whole request bind
// with a 400.
type FlexInt int
func (f *FlexInt) UnmarshalJSON(b []byte) error {
s := strings.TrimSpace(string(b))
if s == "null" || s == `""` {
*f = 0
return nil
}
if n, err := strconv.Atoi(s); err == nil {
*f = FlexInt(n)
return nil
}
if strings.HasPrefix(s, `"`) && strings.HasSuffix(s, `"`) {
inner := s[1 : len(s)-1]
if n, err := strconv.Atoi(inner); err == nil {
*f = FlexInt(n)
return nil
}
*f = 0
return nil
}
if fl, err := strconv.ParseFloat(s, 64); err == nil {
*f = FlexInt(int(fl))
return nil
}
*f = 0
return nil
}
type KOReaderBookmark struct {
Chapter int `json:"chapter,omitempty"`
Chapter FlexInt `json:"chapter,omitempty"`
Datetime string `json:"datetime,omitempty"`
Notes string `json:"notes,omitempty"`
Pos0 string `json:"pos0,omitempty"`
Pos1 string `json:"pos1,omitempty"`
Page int `json:"page,omitempty"`
Page FlexInt `json:"page,omitempty"`
Text string `json:"text,omitempty"`
Type string `json:"type,omitempty"`
Percentage *float64 `json:"percentage,omitempty"`
@@ -114,12 +177,12 @@ type KOReaderBookmark struct {
}
type KOReaderHighlight struct {
Chapter int `json:"chapter,omitempty"`
Chapter FlexInt `json:"chapter,omitempty"`
Datetime string `json:"datetime,omitempty"`
Notes string `json:"notes,omitempty"`
Pos0 string `json:"pos0,omitempty"`
Pos1 string `json:"pos1,omitempty"`
Page int `json:"page,omitempty"`
Page FlexInt `json:"page,omitempty"`
Text string `json:"text,omitempty"`
Type string `json:"type,omitempty"`
Color string `json:"color,omitempty"`
@@ -128,12 +191,12 @@ type KOReaderHighlight struct {
}
type KOReaderNote struct {
Chapter int `json:"chapter,omitempty"`
Chapter FlexInt `json:"chapter,omitempty"`
Datetime string `json:"datetime,omitempty"`
Notes string `json:"notes,omitempty"`
Pos0 string `json:"pos0,omitempty"`
Pos1 string `json:"pos1,omitempty"`
Page int `json:"page,omitempty"`
Page FlexInt `json:"page,omitempty"`
Text string `json:"text,omitempty"`
Type string `json:"type,omitempty"`
Percentage *float64 `json:"percentage,omitempty"`
@@ -486,12 +549,16 @@ func (h *KOReaderHandler) processBookAnnotations(ctx context.Context, deviceID,
for _, hl := range book.Highlights {
startPos := hl.Pos0
endPos := hl.Pos1
epubcfiStart, epubcfiEnd := h.convertHighlightPositions(ctx, mediaItemID, startPos, endPos)
// The highlight's own text anchors the conversion exactly.
epubcfiStart, epubcfiEnd := h.convertHighlightPositions(ctx, mediaItemID, startPos, endPos, hl.Text)
pctStart := 0.0
if hl.Percentage != nil {
pctStart = *hl.Percentage
}
if pctStart == 0 {
pctStart = h.deriveAnnotationPercentage(ctx, mediaItemID, startPos, int(hl.Page))
}
deviceData, _ := json.Marshal(map[string]interface{}{
"datetime": hl.Datetime,
@@ -519,12 +586,15 @@ func (h *KOReaderHandler) processBookAnnotations(ctx context.Context, deviceID,
for _, note := range book.Notes {
startPos := note.Pos0
endPos := note.Pos1
epubcfiStart, epubcfiEnd := h.convertHighlightPositions(ctx, mediaItemID, startPos, endPos)
epubcfiStart, epubcfiEnd := h.convertHighlightPositions(ctx, mediaItemID, startPos, endPos, note.Text)
pctStart := 0.0
if note.Percentage != nil {
pctStart = *note.Percentage
}
if pctStart == 0 {
pctStart = h.deriveAnnotationPercentage(ctx, mediaItemID, startPos, int(note.Page))
}
deviceData, _ := json.Marshal(map[string]interface{}{
"datetime": note.Datetime,
@@ -809,17 +879,15 @@ func (h *KOReaderHandler) GetMetadata(c *echo.Context) error {
for _, ann := range annotations {
if ann.AnnotationType == "highlight" {
pos0 := ann.StartPosition.String
pos1 := ann.EndPosition.String
if ann.EpubcfiStart.Valid && ann.EpubcfiStart.String != "" {
if converted := h.reverseConvertCFI(c, mediaItem, ann.EpubcfiStart.String); converted != "" {
pos0 = converted
}
}
if ann.EpubcfiEnd.Valid && ann.EpubcfiEnd.String != "" {
if converted := h.reverseConvertCFI(c, mediaItem, ann.EpubcfiEnd.String); converted != "" {
pos1 = converted
}
// Selection text doubles as the converter's text-search context.
pos0 := h.koreaderPos0(c, mediaItem, ann.StartPosition.String, ann.EpubcfiStart.String, ann.SelectionText)
pos1 := h.koreaderPos0(c, mediaItem, ann.EndPosition.String, ann.EpubcfiEnd.String, ann.SelectionText)
if pos0 == "" {
// Nothing the device could place — serving a locator it can't
// resolve would create junk bookmarks that re-push as
// duplicates, so skip instead.
log.Printf("Bookhoard: GetMetadata skip highlight %s (no resolvable pos0)", ann.ID)
continue
}
highlight := KOReaderHighlight{
Text: ann.SelectionText,
@@ -833,9 +901,14 @@ func (h *KOReaderHandler) GetMetadata(c *echo.Context) error {
}
annotationsResponse.Highlights = append(annotationsResponse.Highlights, highlight)
} else if ann.AnnotationType == "note" {
pos0 := h.koreaderPos0(c, mediaItem, ann.StartPosition.String, ann.EpubcfiStart.String, "")
if pos0 == "" {
log.Printf("Bookhoard: GetMetadata skip note %s (no resolvable pos0)", ann.ID)
continue
}
annotationsResponse.Notes = append(annotationsResponse.Notes, KOReaderNote{
Text: ann.SelectionText,
Pos0: ann.StartPosition.String,
Pos0: pos0,
Datetime: ann.CreatedAt.Time.Format(time.RFC3339),
})
}
@@ -846,9 +919,10 @@ func (h *KOReaderHandler) GetMetadata(c *echo.Context) error {
UserID: pgUserID,
})
for _, bm := range bookmarks {
pos0 := bm.Position.String
if pos0 == "" && bm.CfiPosition.Valid {
pos0 = bm.CfiPosition.String
pos0 := h.koreaderPos0(c, mediaItem, bm.Position.String, bm.CfiPosition.String, "")
if pos0 == "" {
log.Printf("Bookhoard: GetMetadata skip bookmark %s (no resolvable pos0)", bm.ID)
continue
}
koreaderBookmark := KOReaderBookmark{
Text: bm.Title,
@@ -860,7 +934,7 @@ func (h *KOReaderHandler) GetMetadata(c *echo.Context) error {
koreaderBookmark.Notes = bm.Notes.String
}
if bm.ChapterNumber.Valid {
koreaderBookmark.Chapter = int(bm.ChapterNumber.Int32)
koreaderBookmark.Chapter = FlexInt(bm.ChapterNumber.Int32)
}
annotationsResponse.Bookmarks = append(annotationsResponse.Bookmarks, koreaderBookmark)
}
@@ -880,6 +954,14 @@ func (h *KOReaderHandler) GetMetadata(c *echo.Context) error {
dd = map[string]interface{}{}
}
dd["dedup_key"] = ts.DedupKey.String
// KOReader deletes by matching pos0. Device-pushed annotations carry
// it in device_sync_data; web-created ones don't (their locator is
// converted at serve time), so resolve it from the stored columns.
if dd["pos0"] == nil || dd["pos0"] == "" {
if pos0 := h.koreaderPos0(c, mediaItem, ts.StartPosition.String, ts.EpubcfiStart.String, ""); pos0 != "" {
dd["pos0"] = pos0
}
}
if ts.AnnotationType == "highlight" {
annotationsResponse.DeletedHighlights = append(annotationsResponse.DeletedHighlights, dd)
} else if ts.AnnotationType == "bookmark" {
@@ -939,7 +1021,7 @@ func (h *KOReaderHandler) convertCFIToXPointer(c *echo.Context, mediaItem databa
}
}
func (h *KOReaderHandler) reverseConvertCFI(c *echo.Context, mediaItem database.MediaItems, epubcfi string) string {
func (h *KOReaderHandler) reverseConvertCFI(c *echo.Context, mediaItem database.MediaItems, epubcfi string, contextText string) string {
if h.libraryService == nil || epubcfi == "" {
return ""
}
@@ -947,13 +1029,76 @@ func (h *KOReaderHandler) reverseConvertCFI(c *echo.Context, mediaItem database.
if err != nil || epubPath == "" {
return ""
}
loc := wsync.ConvertFromCanonical(wsync.LocatorSourceKOReader, epubcfi, 0, "", mediaItem.FormatGroup, epubPath, "")
loc := wsync.ConvertFromCanonical(wsync.LocatorSourceKOReader, epubcfi, 0, contextText, mediaItem.FormatGroup, epubPath, "")
if loc.Position != "" && loc.Position != epubcfi {
return loc.Position
}
return ""
}
// pdfRectAnchor is the JSON locator the web reader stores in epubcfi_start
// for PDF text highlights (page-fraction rects; page index is 0-based).
type pdfRectAnchor struct {
V int `json:"v"`
Page int `json:"page"`
Rects [][]float64 `json:"rects"`
}
// koreaderPos0 resolves a device-native KOReader pos0 from an annotation's
// stored locators, whatever the source. Resolution order:
//
// 1. A device-native CRE xpointer ("/body/...") in startPosition wins —
// round-trip identical for KOReader-pushed annotations (converting the
// stored CFI instead could drift and duplicate on the device).
// 2. The web reader's PDF JSON anchor → bare page number (KOReader paging
// documents use the page number as pos0).
// 3. A stored EPUB CFI (epubcfi_start, or startPosition without the
// reader's "cfi:" prefix) → converted to a CRE xpointer, with
// contextText (the selection text) enabling the text-search fallback.
// 4. A "page:N" or bare-numeric position → the bare number.
//
// Returns "" when nothing usable exists; callers skip such annotations so
// devices never receive locators they cannot place.
func (h *KOReaderHandler) koreaderPos0(c *echo.Context, mediaItem database.MediaItems, startPosition, epubcfi, contextText string) string {
if wsync.IsCREXPointer(startPosition) {
return startPosition
}
if strings.HasPrefix(epubcfi, "{") {
var anchor pdfRectAnchor
if json.Unmarshal([]byte(epubcfi), &anchor) == nil && anchor.Page >= 0 {
return strconv.Itoa(anchor.Page)
}
}
cfi := epubcfi
if cfi == "" && strings.HasPrefix(startPosition, "cfi:") {
cfi = strings.TrimPrefix(startPosition, "cfi:")
}
if cfi != "" && wsync.IsStandardEPUBCFI(cfi) {
if converted := h.reverseConvertCFI(c, mediaItem, cfi, contextText); converted != "" {
return converted
}
// Conversion failed; fall through so numeric positions still work.
if wsync.IsCREXPointer(cfi) {
return cfi
}
}
if p := strings.TrimPrefix(startPosition, "page:"); p != "" && parsePageInt(p) >= 0 {
return p
}
return ""
}
func parsePageInt(s string) int64 {
var n int64
for _, r := range s {
if r < '0' || r > '9' {
return -1
}
n = n*10 + int64(r-'0')
}
return n
}
func (h *KOReaderHandler) GetLibrary(c *echo.Context) error {
device := c.Get("device").(database.Devices)
userID := device.UserID.Bytes
@@ -1201,7 +1346,7 @@ func (h *KOReaderHandler) SyncBookmarks(c *echo.Context) error {
}
if h.annotationSvc != nil {
epubcfiStart, epubcfiEnd := h.convertHighlightPositions(ctx, mediaItemID, highlight.Pos0, highlight.Pos1)
epubcfiStart, epubcfiEnd := h.convertHighlightPositions(ctx, mediaItemID, highlight.Pos0, highlight.Pos1, highlight.Text)
pctStart := 0.0
if highlight.Percentage != nil {