Merge branch 'test-fixture-cleanup': book-agnostic CFI converter tests

test(sync): replace book-specific CFI converter fixtures with a synthetic EPUB
Six converter tests pointed at absolute paths for 1984 and Crime and Punishment under uploads/ — books that don't exist on most checkouts (CI included), so the suite shipped with 5 permanently failing tests (and a sixth passing only by accident: the percentage-fallback path triggered by the missing file is the outcome it asserts). A writeTestEPUB helper now builds a minimal deterministic EPUB in t.TempDir() (zip → container.xml → OPF → 6-doc spine), so the tests exercise the real zip/OPF/spine/document pipeline with no external dependencies. The xpointer→CFI conversion, fragment-ID conversion, both round-trips (bare and context-text-anchored), and the text-search and percentage fallbacks all keep their original assertions, now against known document content. internal/sync is green for the first time on this machine.
2026-08-20 09:22:10 -04:00 · 2026-08-20 09:22:10 -04:00 · 2026-08-20 09:13:24 -04:00 · 2026-08-20 08:43:21 -04:00 · 2026-08-19 19:41:58 -04:00 · 2026-08-19 14:08:03 -04:00
8 changed files with 883 additions and 293 deletions
@@ -6852,7 +6852,11 @@ SELECT
    mh.dedup_key,
    'highlight' as annotation_type,
    mh.device_sync_data,
-    mh.deleted_at
+    mh.deleted_at,
+    mh.start_position,
+    mh.end_position,
+    mh.epubcfi_start,
+    mh.epubcfi_end
 FROM media_highlights mh
 WHERE mh.media_item_id = $1 AND mh.user_id = $2 AND mh.deleted = TRUE AND mh.deleted_at > $3
 UNION ALL
@@ -6861,7 +6865,11 @@ SELECT
    mn.dedup_key,
    'note' as annotation_type,
    mn.device_sync_data,
-    mn.deleted_at
+    mn.deleted_at,
+    mn.position as start_position,
+    NULL as end_position,
+    mn.epubcfi_location as epubcfi_start,
+    NULL as epubcfi_end
 FROM media_notes mn
 WHERE mn.media_item_id = $1 AND mn.user_id = $2 AND mn.deleted = TRUE AND mn.deleted_at > $3
 UNION ALL
@@ -6870,7 +6878,11 @@ SELECT
    mb.dedup_key,
    'bookmark' as annotation_type,
    mb.device_sync_data,
-    mb.deleted_at
+    mb.deleted_at,
+    mb.position as start_position,
+    NULL as end_position,
+    mb.cfi_position as epubcfi_start,
+    NULL as epubcfi_end
 FROM media_bookmarks mb
 WHERE mb.media_item_id = $1 AND mb.user_id = $2 AND mb.deleted = TRUE AND mb.deleted_at > $3
 ORDER BY deleted_at DESC
@@ -6888,6 +6900,10 @@ type GetTombstonedAnnotationsForBookRow struct {
 	AnnotationType string             `db:"annotation_type" json:"annotation_type"`
 	DeviceSyncData []byte             `db:"device_sync_data" json:"device_sync_data"`
 	DeletedAt      pgtype.Timestamptz `db:"deleted_at" json:"deleted_at"`
+	StartPosition  pgtype.Text        `db:"start_position" json:"start_position"`
+	EndPosition    pgtype.Text        `db:"end_position" json:"end_position"`
+	EpubcfiStart   pgtype.Text        `db:"epubcfi_start" json:"epubcfi_start"`
+	EpubcfiEnd     pgtype.Text        `db:"epubcfi_end" json:"epubcfi_end"`
 }

 func (q *Queries) GetTombstonedAnnotationsForBook(ctx context.Context, arg GetTombstonedAnnotationsForBookParams) ([]GetTombstonedAnnotationsForBookRow, error) {
@@ -6905,6 +6921,10 @@ func (q *Queries) GetTombstonedAnnotationsForBook(ctx context.Context, arg GetTo
 			&i.AnnotationType,
 			&i.DeviceSyncData,
 			&i.DeletedAt,
+			&i.StartPosition,
+			&i.EndPosition,
+			&i.EpubcfiStart,
+			&i.EpubcfiEnd,
 		); err != nil {
 			return nil, err
 		}
@@ -992,7 +992,11 @@ SELECT
    mh.dedup_key,
    'highlight' as annotation_type,
    mh.device_sync_data,
-    mh.deleted_at
+    mh.deleted_at,
+    mh.start_position,
+    mh.end_position,
+    mh.epubcfi_start,
+    mh.epubcfi_end
 FROM media_highlights mh
 WHERE mh.media_item_id = $1 AND mh.user_id = $2 AND mh.deleted = TRUE AND mh.deleted_at > $3
 UNION ALL
@@ -1001,7 +1005,11 @@ SELECT
    mn.dedup_key,
    'note' as annotation_type,
    mn.device_sync_data,
-    mn.deleted_at
+    mn.deleted_at,
+    mn.position as start_position,
+    NULL as end_position,
+    mn.epubcfi_location as epubcfi_start,
+    NULL as epubcfi_end
 FROM media_notes mn
 WHERE mn.media_item_id = $1 AND mn.user_id = $2 AND mn.deleted = TRUE AND mn.deleted_at > $3
 UNION ALL
@@ -1010,7 +1018,11 @@ SELECT
    mb.dedup_key,
    'bookmark' as annotation_type,
    mb.device_sync_data,
-    mb.deleted_at
+    mb.deleted_at,
+    mb.position as start_position,
+    NULL as end_position,
+    mb.cfi_position as epubcfi_start,
+    NULL as epubcfi_end
 FROM media_bookmarks mb
 WHERE mb.media_item_id = $1 AND mb.user_id = $2 AND mb.deleted = TRUE AND mb.deleted_at > $3
 ORDER BY deleted_at DESC;
@@ -9,7 +9,10 @@ import (
 	"fmt"
 	"log"
 	"net/http"
+	"strconv"
+	"strings"
 	"time"
+	"unicode/utf8"

 	"github.com/google/uuid"
 	"github.com/jackc/pgx/v5/pgtype"
@@ -47,7 +50,7 @@ func (h *KOReaderHandler) SetAnnotationService(svc *wsync.AnnotationService) {
 	h.annotationSvc = svc
 }

-func (h *KOReaderHandler) convertHighlightPositions(ctx context.Context, mediaItemID pgtype.UUID, pos0, pos1 string) (string, string) {
+func (h *KOReaderHandler) convertHighlightPositions(ctx context.Context, mediaItemID pgtype.UUID, pos0, pos1, contextText string) (string, string) {
 	if pos0 == "" || h.libraryService == nil {
 		return "", ""
 	}
@@ -59,9 +62,89 @@ func (h *KOReaderHandler) convertHighlightPositions(ctx context.Context, mediaIt
 	if err != nil || epubPath == "" {
 		return "", ""
 	}
-	startLoc := wsync.ConvertToCanonical(wsync.LocatorSourceKOReader, pos0, 0, "", mediaItem.FormatGroup, epubPath, "")
+	// The annotation's own text is the ideal anchor for the converter's
+	// text-search path: clients (thin, underpowered) send only raw
+	// locators, the server resolves them against the actual book.
+	startLoc := wsync.ConvertToCanonical(wsync.LocatorSourceKOReader, pos0, 0, contextText, mediaItem.FormatGroup, epubPath, "")
 	endLoc := wsync.ConvertToCanonical(wsync.LocatorSourceKOReader, pos1, 0, "", mediaItem.FormatGroup, epubPath, "")
-	return startLoc.CFI, endLoc.CFI
+	endCFI := endLoc.CFI
+	// The end conversion carries no context text, so unless it resolved
+	// exactly it degenerates to a percentage fallback anchored at the
+	// document start — useless as a range end. When the START resolved
+	// exactly, derive the end from it: same node, character offset
+	// advanced by the selection's UTF-16 length (the CFI offset unit).
+	if endLoc.Precision != "exact" && startLoc.Precision == "exact" && contextText != "" {
+		endCFI = extendCFIByLength(startLoc.CFI, contextText)
+	}
+	return startLoc.CFI, endCFI
+}
+
+// extendCFIByLength advances a point CFI's trailing character offset by the
+// UTF-16 length of text (EPUB CFI character offsets are UTF-16 code units).
+// Selections spanning multiple text nodes produce an out-of-range offset —
+// harmless: resolution clamps or fails, and consumers fall back to the start.
+func extendCFIByLength(cfi, text string) string {
+	if cfi == "" || text == "" {
+		return cfi
+	}
+	i := strings.LastIndex(cfi, ":")
+	if i < 0 || !strings.HasSuffix(cfi, ")") {
+		return cfi
+	}
+	off, err := strconv.Atoi(cfi[i+1 : len(cfi)-1])
+	if err != nil {
+		return cfi
+	}
+	utf16len := 0
+	for _, r := range text {
+		if r > 0xFFFF {
+			utf16len += 2
+		} else {
+			utf16len++
+		}
+	}
+	return cfi[:i+1] + strconv.Itoa(off+utf16len) + ")"
+}
+
+// existingHighlightColor returns the stored color of the highlight matching
+// the dedup key ("" when none) so device echoes that carry no color never
+// clobber the web color.
+func (h *KOReaderHandler) existingHighlightColor(ctx context.Context, mediaItemID, userID pgtype.UUID, dedupKey string) string {
+	if dedupKey == "" {
+		return ""
+	}
+	existing, err := h.db.GetMediaHighlightByDedupKey(ctx, database.GetMediaHighlightByDedupKeyParams{
+		UserID:      userID,
+		MediaItemID: mediaItemID,
+		DedupKey:    pgtype.Text{String: dedupKey, Valid: true},
+	})
+	if err != nil {
+		return ""
+	}
+	return existing.Color.String
+}
+
+// deriveAnnotationPercentage computes a percentage for device-pushed
+// annotations when the client didn't send one (thin clients skip their own
+// per-annotation page lookups; arithmetic is only free on paging documents).
+func (h *KOReaderHandler) deriveAnnotationPercentage(ctx context.Context, mediaItemID pgtype.UUID, pos0 string, page int) float64 {
+	mediaItem, err := h.db.GetMediaItem(ctx, mediaItemID)
+	if err != nil {
+		return 0
+	}
+	formatGroup := wsync.FormatGroup(mediaItem.FormatGroup)
+	if formatGroup == wsync.FormatGroupFixedLayout || formatGroup == wsync.FormatGroupComicArchive {
+		if page > 0 && mediaItem.PageCount.Valid && mediaItem.PageCount.Int32 > 0 {
+			return float64(page) / float64(mediaItem.PageCount.Int32)
+		}
+		return 0
+	}
+	if wsync.IsCREXPointer(pos0) && h.libraryService != nil {
+		if epubPath, err := h.libraryService.ResolveMediaPath(ctx, mediaItem.LibraryID, mediaItem.FilePath); err == nil && epubPath != "" {
+			return wsync.NewCFIConverter(epubPath).SectionPercentage(pos0)
+		}
+	}
+	return 0
 }

 func (h *KOReaderHandler) SetLibraryService(svc LibraryPathResolver) {
@@ -75,24 +158,24 @@ type KOReaderProgressRequest struct {
 }

 type KOReaderBookProgress struct {
-	UUID       string              `json:"uuid,omitempty"`
-	SHA256     string              `json:"sha256,omitempty"`
-	Title      string              `json:"title,omitempty"`
-	Authors    []string            `json:"authors,omitempty"`
-	Progress   float64             `json:"progress"`
-	Percentage float64             `json:"percentage"`
-	LastRead   string              `json:"last_read,omitempty"`
-	FilePath   string              `json:"file_path,omitempty"`
-	DeviceInfo KOReaderDeviceInfo  `json:"device_info,omitempty"`
-	Bookmarks  []KOReaderBookmark  `json:"bookmarks,omitempty"`
-	Highlights []KOReaderHighlight `json:"highlights,omitempty"`
-	Notes      []KOReaderNote      `json:"notes,omitempty"`
-	Chapter      *int                `json:"chapter,omitempty"`
-	Character    *int64              `json:"character,omitempty"`
-	Epubcfi      *string             `json:"epubcfi,omitempty"`
-	ContextText  *string             `json:"context_text,omitempty"`
-	Page         *int                `json:"page,omitempty"`
-	TotalPages   *int                `json:"total_pages,omitempty"`
+	UUID        string              `json:"uuid,omitempty"`
+	SHA256      string              `json:"sha256,omitempty"`
+	Title       string              `json:"title,omitempty"`
+	Authors     []string            `json:"authors,omitempty"`
+	Progress    float64             `json:"progress"`
+	Percentage  float64             `json:"percentage"`
+	LastRead    string              `json:"last_read,omitempty"`
+	FilePath    string              `json:"file_path,omitempty"`
+	DeviceInfo  KOReaderDeviceInfo  `json:"device_info,omitempty"`
+	Bookmarks   []KOReaderBookmark  `json:"bookmarks,omitempty"`
+	Highlights  []KOReaderHighlight `json:"highlights,omitempty"`
+	Notes       []KOReaderNote      `json:"notes,omitempty"`
+	Chapter     *int                `json:"chapter,omitempty"`
+	Character   *int64              `json:"character,omitempty"`
+	Epubcfi     *string             `json:"epubcfi,omitempty"`
+	ContextText *string             `json:"context_text,omitempty"`
+	Page        *int                `json:"page,omitempty"`
+	TotalPages  *int                `json:"total_pages,omitempty"`
 }

 type KOReaderDeviceInfo struct {
@@ -100,53 +183,91 @@ type KOReaderDeviceInfo struct {
 	DeviceModel     string `json:"device_model,omitempty"`
 }

+// FlexInt tolerates the loose types KOReader clients send for optional
+// numeric fields: JSON numbers, numeric strings ("30"), empty strings
+// (""), or non-numeric strings ("/body/..." xpointers in `page` for CRE
+// documents) — the latter decode to 0. Without this, a single annotation
+// carrying chapter:"" or page:"/body/..." failed the whole request bind
+// with a 400.
+type FlexInt int
+
+func (f *FlexInt) UnmarshalJSON(b []byte) error {
+	s := strings.TrimSpace(string(b))
+	if s == "null" || s == `""` {
+		*f = 0
+		return nil
+	}
+	if n, err := strconv.Atoi(s); err == nil {
+		*f = FlexInt(n)
+		return nil
+	}
+	if strings.HasPrefix(s, `"`) && strings.HasSuffix(s, `"`) {
+		inner := s[1 : len(s)-1]
+		if n, err := strconv.Atoi(inner); err == nil {
+			*f = FlexInt(n)
+			return nil
+		}
+		*f = 0
+		return nil
+	}
+	if fl, err := strconv.ParseFloat(s, 64); err == nil {
+		*f = FlexInt(int(fl))
+		return nil
+	}
+	*f = 0
+	return nil
+}
+
 type KOReaderBookmark struct {
-	Chapter    int      `json:"chapter,omitempty"`
+	Chapter    FlexInt  `json:"chapter,omitempty"`
 	Datetime   string   `json:"datetime,omitempty"`
 	Notes      string   `json:"notes,omitempty"`
 	Pos0       string   `json:"pos0,omitempty"`
 	Pos1       string   `json:"pos1,omitempty"`
-	Page       int      `json:"page,omitempty"`
+	Page       FlexInt  `json:"page,omitempty"`
 	Text       string   `json:"text,omitempty"`
 	Type       string   `json:"type,omitempty"`
 	Percentage *float64 `json:"percentage,omitempty"`
 	BookSHA256 string   `json:"book_sha256,omitempty"`
+	DedupKey   string   `json:"dedup_key,omitempty"`
 }

 type KOReaderHighlight struct {
-	Chapter    int      `json:"chapter,omitempty"`
+	Chapter    FlexInt  `json:"chapter,omitempty"`
 	Datetime   string   `json:"datetime,omitempty"`
 	Notes      string   `json:"notes,omitempty"`
 	Pos0       string   `json:"pos0,omitempty"`
 	Pos1       string   `json:"pos1,omitempty"`
-	Page       int      `json:"page,omitempty"`
+	Page       FlexInt  `json:"page,omitempty"`
 	Text       string   `json:"text,omitempty"`
 	Type       string   `json:"type,omitempty"`
 	Color      string   `json:"color,omitempty"`
 	Percentage *float64 `json:"percentage,omitempty"`
 	BookSHA256 string   `json:"book_sha256,omitempty"`
+	DedupKey   string   `json:"dedup_key,omitempty"`
 }

 type KOReaderNote struct {
-	Chapter    int      `json:"chapter,omitempty"`
+	Chapter    FlexInt  `json:"chapter,omitempty"`
 	Datetime   string   `json:"datetime,omitempty"`
 	Notes      string   `json:"notes,omitempty"`
 	Pos0       string   `json:"pos0,omitempty"`
 	Pos1       string   `json:"pos1,omitempty"`
-	Page       int      `json:"page,omitempty"`
+	Page       FlexInt  `json:"page,omitempty"`
 	Text       string   `json:"text,omitempty"`
 	Type       string   `json:"type,omitempty"`
 	Percentage *float64 `json:"percentage,omitempty"`
 	BookSHA256 string   `json:"book_sha256,omitempty"`
+	DedupKey   string   `json:"dedup_key,omitempty"`
 }

 type KOReaderSyncResponse struct {
-	SyncStatus    string                  `json:"sync_status"`
-	BooksSynced   int                     `json:"books_synced"`
+	SyncStatus    string                   `json:"sync_status"`
+	BooksSynced   int                      `json:"books_synced"`
 	BookResults   []KOReaderBookSyncResult `json:"book_results,omitempty"`
-	Conflicts     []KOReaderConflict      `json:"conflicts,omitempty"`
-	Timestamp     string                  `json:"timestamp"`
-	DeviceUpdated bool                    `json:"device_updated"`
+	Conflicts     []KOReaderConflict       `json:"conflicts,omitempty"`
+	Timestamp     string                   `json:"timestamp"`
+	DeviceUpdated bool                     `json:"device_updated"`
 }

 type KOReaderBookSyncResult struct {
@@ -174,20 +295,20 @@ type KOReaderMetadata struct {
 }

 type KOReaderProgressData struct {
-	Percentage         float64  `json:"percentage"`
-	Character          *int64   `json:"character,omitempty"`
-	Epubcfi            *string  `json:"epubcfi,omitempty"`
-	KoreaderXPointer   *string  `json:"koreader_xpointer,omitempty"`
-	Chapter            *int     `json:"chapter,omitempty"`
-	ChapterProgress    *float64 `json:"chapter_progress,omitempty"`
-	Page               *int     `json:"page,omitempty"`
-	TotalPages         *int     `json:"total_pages,omitempty"`
+	Percentage       float64  `json:"percentage"`
+	Character        *int64   `json:"character,omitempty"`
+	Epubcfi          *string  `json:"epubcfi,omitempty"`
+	KoreaderXPointer *string  `json:"koreader_xpointer,omitempty"`
+	Chapter          *int     `json:"chapter,omitempty"`
+	ChapterProgress  *float64 `json:"chapter_progress,omitempty"`
+	Page             *int     `json:"page,omitempty"`
+	TotalPages       *int     `json:"total_pages,omitempty"`
 }

 type KOReaderAnnotations struct {
-	Highlights        []KOReaderHighlight `json:"highlights,omitempty"`
-	Notes             []KOReaderNote      `json:"notes,omitempty"`
-	Bookmarks         []KOReaderBookmark  `json:"bookmarks,omitempty"`
+	Highlights        []KOReaderHighlight      `json:"highlights,omitempty"`
+	Notes             []KOReaderNote           `json:"notes,omitempty"`
+	Bookmarks         []KOReaderBookmark       `json:"bookmarks,omitempty"`
 	DeletedHighlights []map[string]interface{} `json:"deleted_highlights,omitempty"`
 	DeletedBookmarks  []map[string]interface{} `json:"deleted_bookmarks,omitempty"`
 }
@@ -486,12 +607,16 @@ func (h *KOReaderHandler) processBookAnnotations(ctx context.Context, deviceID,
 	for _, hl := range book.Highlights {
 		startPos := hl.Pos0
 		endPos := hl.Pos1
-		epubcfiStart, epubcfiEnd := h.convertHighlightPositions(ctx, mediaItemID, startPos, endPos)
+		// The highlight's own text anchors the conversion exactly.
+		epubcfiStart, epubcfiEnd := h.convertHighlightPositions(ctx, mediaItemID, startPos, endPos, hl.Text)

 		pctStart := 0.0
 		if hl.Percentage != nil {
 			pctStart = *hl.Percentage
 		}
+		if pctStart == 0 {
+			pctStart = h.deriveAnnotationPercentage(ctx, mediaItemID, startPos, int(hl.Page))
+		}

 		deviceData, _ := json.Marshal(map[string]interface{}{
 			"datetime": hl.Datetime,
@@ -500,31 +625,55 @@ func (h *KOReaderHandler) processBookAnnotations(ctx context.Context, deviceID,
 			"page":     hl.Page,
 		})

+		// Color semantics: devices render their own default and cannot
+		// round-trip web colors. An echo carries NO color — preserve the
+		// stored (web) color so round-trips never change it. A non-empty
+		// color means the user edited the highlight on the device: map the
+		// device color name and let it win.
+		color := ""
+		if hl.Color != "" {
+			color = mapColorFromKOReader(hl.Color)
+		}
+		dedupKey := hl.DedupKey
+		if dedupKey == "" {
+			dedupKey = wsync.ComputeDedupKey(hl.Text, epubcfiStart, startPos)
+		}
+		if color == "" {
+			color = h.existingHighlightColor(ctx, mediaItemID, userID, dedupKey)
+		}
+		if color == "" {
+			color = "#ffd54f"
+		}
+
 		h.annotationSvc.SaveHighlight(ctx, wsync.SaveHighlightRequest{
-			MediaItemID:      mediaItemID,
-			UserID:           userID,
-			SelectionText:    hl.Text,
-			StartPosition:    startPos,
-			EndPosition:      endPos,
-			Color:            hl.Color,
-			NoteText:         hl.Notes,
-			PercentageStart:  pctStart,
-			EpubcfiStart:     epubcfiStart,
-			EpubcfiEnd:       epubcfiEnd,
-			Source:           "koreader",
-			DeviceSyncData:   deviceData,
+			MediaItemID:     mediaItemID,
+			UserID:          userID,
+			SelectionText:   hl.Text,
+			StartPosition:   startPos,
+			EndPosition:     endPos,
+			Color:           color,
+			NoteText:        hl.Notes,
+			PercentageStart: pctStart,
+			EpubcfiStart:    epubcfiStart,
+			EpubcfiEnd:      epubcfiEnd,
+			Source:          "koreader",
+			DeviceSyncData:  deviceData,
+			DedupKey:        dedupKey,
 		})
 	}

 	for _, note := range book.Notes {
 		startPos := note.Pos0
 		endPos := note.Pos1
-		epubcfiStart, epubcfiEnd := h.convertHighlightPositions(ctx, mediaItemID, startPos, endPos)
+		epubcfiStart, epubcfiEnd := h.convertHighlightPositions(ctx, mediaItemID, startPos, endPos, note.Text)

 		pctStart := 0.0
 		if note.Percentage != nil {
 			pctStart = *note.Percentage
 		}
+		if pctStart == 0 {
+			pctStart = h.deriveAnnotationPercentage(ctx, mediaItemID, startPos, int(note.Page))
+		}

 		deviceData, _ := json.Marshal(map[string]interface{}{
 			"datetime": note.Datetime,
@@ -533,18 +682,25 @@ func (h *KOReaderHandler) processBookAnnotations(ctx context.Context, deviceID,
 			"page":     note.Page,
 		})

+		dedupKey := note.DedupKey
+		if dedupKey == "" {
+			dedupKey = wsync.ComputeDedupKey(note.Text, epubcfiStart, startPos)
+		}
+
 		h.annotationSvc.SaveHighlight(ctx, wsync.SaveHighlightRequest{
-			MediaItemID:      mediaItemID,
-			UserID:           userID,
-			SelectionText:    note.Text,
-			StartPosition:    startPos,
-			EndPosition:      endPos,
-			NoteText:         note.Notes,
-			PercentageStart:  pctStart,
-			EpubcfiStart:     epubcfiStart,
-			EpubcfiEnd:       epubcfiEnd,
-			Source:           "koreader",
-			DeviceSyncData:   deviceData,
+			MediaItemID:     mediaItemID,
+			UserID:          userID,
+			SelectionText:   note.Text,
+			StartPosition:   startPos,
+			EndPosition:     endPos,
+			Color:           h.existingHighlightColor(ctx, mediaItemID, userID, dedupKey),
+			NoteText:        note.Notes,
+			PercentageStart: pctStart,
+			EpubcfiStart:    epubcfiStart,
+			EpubcfiEnd:      epubcfiEnd,
+			Source:          "koreader",
+			DeviceSyncData:  deviceData,
+			DedupKey:        dedupKey,
 		})
 	}

@@ -562,6 +718,11 @@ func (h *KOReaderHandler) processBookAnnotations(ctx context.Context, deviceID,
 			"page":     bookmark.Page,
 		})

+		dedupKey := bookmark.DedupKey
+		if dedupKey == "" {
+			dedupKey = wsync.ComputeDedupKey(bookmark.Text, "", position)
+		}
+
 		h.annotationSvc.SaveBookmark(ctx, wsync.SaveBookmarkRequest{
 			MediaItemID:    mediaItemID,
 			UserID:         userID,
@@ -570,6 +731,7 @@ func (h *KOReaderHandler) processBookAnnotations(ctx context.Context, deviceID,
 			ChapterNumber:  int32(bookmark.Chapter),
 			Source:         "koreader",
 			DeviceSyncData: deviceData,
+			DedupKey:       dedupKey,
 		})
 	}
 }
@@ -809,34 +971,52 @@ func (h *KOReaderHandler) GetMetadata(c *echo.Context) error {

 	for _, ann := range annotations {
 		if ann.AnnotationType == "highlight" {
-			pos0 := ann.StartPosition.String
-			pos1 := ann.EndPosition.String
-			if ann.EpubcfiStart.Valid && ann.EpubcfiStart.String != "" {
-				if converted := h.reverseConvertCFI(c, mediaItem, ann.EpubcfiStart.String); converted != "" {
-					pos0 = converted
-				}
+			// Selection text doubles as the converter's text-search context.
+			pos0 := h.koreaderPos0(c, mediaItem, ann.StartPosition.String, ann.EpubcfiStart.String, ann.SelectionText)
+			pos1 := h.koreaderPos0(c, mediaItem, ann.EndPosition.String, ann.EpubcfiEnd.String, ann.SelectionText)
+			if pos0 == "" {
+				// Nothing the device could place — serving a locator it can't
+				// resolve would create junk bookmarks that re-push as
+				// duplicates, so skip instead.
+				log.Printf("Bookhoard: GetMetadata skip highlight %s (no resolvable pos0)", ann.ID)
+				continue
 			}
-			if ann.EpubcfiEnd.Valid && ann.EpubcfiEnd.String != "" {
-				if converted := h.reverseConvertCFI(c, mediaItem, ann.EpubcfiEnd.String); converted != "" {
-					pos1 = converted
-				}
+			// Old web highlights carry no end anchor, and converted range
+			// CFIs resolve to their start — either way pos1 collapses onto
+			// pos0 and the device paints a zero-width highlight. Derive the
+			// end by advancing the start's character offset by the length
+			// of the selected text.
+			if pos1 == "" || pos1 == pos0 {
+				pos1 = extendXPointerByLength(pos0, ann.SelectionText)
 			}
 			highlight := KOReaderHighlight{
-				Text:     ann.SelectionText,
-				Pos0:     pos0,
-				Pos1:     pos1,
-				Color:    ann.Color.String,
+				Text: ann.SelectionText,
+				Pos0: pos0,
+				Pos1: pos1,
+				// Web colors flow to the device, mapped to KOReader's named
+				// palette. Round-trip safety: the device suppresses the color
+				// when echoing un-edited applied entries (a pink→purple
+				// palette mismatch must not rewrite the stored hex), and an
+				// actual device edit pushes its color, which wins.
+				Color:    mapColorToKOReader(ann.Color.String),
 				Datetime: ann.CreatedAt.Time.Format(time.RFC3339),
+				DedupKey: ann.DedupKey.String,
 			}
 			if ann.NoteText.Valid && ann.NoteText.String != "" {
 				highlight.Notes = ann.NoteText.String
 			}
 			annotationsResponse.Highlights = append(annotationsResponse.Highlights, highlight)
 		} else if ann.AnnotationType == "note" {
+			pos0 := h.koreaderPos0(c, mediaItem, ann.StartPosition.String, ann.EpubcfiStart.String, "")
+			if pos0 == "" {
+				log.Printf("Bookhoard: GetMetadata skip note %s (no resolvable pos0)", ann.ID)
+				continue
+			}
 			annotationsResponse.Notes = append(annotationsResponse.Notes, KOReaderNote{
 				Text:     ann.SelectionText,
-				Pos0:     ann.StartPosition.String,
+				Pos0:     pos0,
 				Datetime: ann.CreatedAt.Time.Format(time.RFC3339),
+				DedupKey: ann.DedupKey.String,
 			})
 		}
 	}
@@ -846,21 +1026,23 @@ func (h *KOReaderHandler) GetMetadata(c *echo.Context) error {
 		UserID:      pgUserID,
 	})
 	for _, bm := range bookmarks {
-		pos0 := bm.Position.String
-		if pos0 == "" && bm.CfiPosition.Valid {
-			pos0 = bm.CfiPosition.String
+		pos0 := h.koreaderPos0(c, mediaItem, bm.Position.String, bm.CfiPosition.String, "")
+		if pos0 == "" {
+			log.Printf("Bookhoard: GetMetadata skip bookmark %s (no resolvable pos0)", bm.ID)
+			continue
 		}
 		koreaderBookmark := KOReaderBookmark{
 			Text:     bm.Title,
 			Pos0:     pos0,
 			Pos1:     pos0,
 			Datetime: bm.CreatedAt.Time.Format(time.RFC3339),
+			DedupKey: bm.DedupKey.String,
 		}
 		if bm.Notes.Valid && bm.Notes.String != "" {
 			koreaderBookmark.Notes = bm.Notes.String
 		}
 		if bm.ChapterNumber.Valid {
-			koreaderBookmark.Chapter = int(bm.ChapterNumber.Int32)
+			koreaderBookmark.Chapter = FlexInt(bm.ChapterNumber.Int32)
 		}
 		annotationsResponse.Bookmarks = append(annotationsResponse.Bookmarks, koreaderBookmark)
 	}
@@ -880,6 +1062,14 @@ func (h *KOReaderHandler) GetMetadata(c *echo.Context) error {
 			dd = map[string]interface{}{}
 		}
 		dd["dedup_key"] = ts.DedupKey.String
+		// KOReader deletes by matching pos0. Device-pushed annotations carry
+		// it in device_sync_data; web-created ones don't (their locator is
+		// converted at serve time), so resolve it from the stored columns.
+		if dd["pos0"] == nil || dd["pos0"] == "" {
+			if pos0 := h.koreaderPos0(c, mediaItem, ts.StartPosition.String, ts.EpubcfiStart.String, ""); pos0 != "" {
+				dd["pos0"] = pos0
+			}
+		}
 		if ts.AnnotationType == "highlight" {
 			annotationsResponse.DeletedHighlights = append(annotationsResponse.DeletedHighlights, dd)
 		} else if ts.AnnotationType == "bookmark" {
@@ -939,7 +1129,7 @@ func (h *KOReaderHandler) convertCFIToXPointer(c *echo.Context, mediaItem databa
 	}
 }

-func (h *KOReaderHandler) reverseConvertCFI(c *echo.Context, mediaItem database.MediaItems, epubcfi string) string {
+func (h *KOReaderHandler) reverseConvertCFI(c *echo.Context, mediaItem database.MediaItems, epubcfi string, contextText string) string {
 	if h.libraryService == nil || epubcfi == "" {
 		return ""
 	}
@@ -947,13 +1137,138 @@ func (h *KOReaderHandler) reverseConvertCFI(c *echo.Context, mediaItem database.
 	if err != nil || epubPath == "" {
 		return ""
 	}
-	loc := wsync.ConvertFromCanonical(wsync.LocatorSourceKOReader, epubcfi, 0, "", mediaItem.FormatGroup, epubPath, "")
+	loc := wsync.ConvertFromCanonical(wsync.LocatorSourceKOReader, epubcfi, 0, contextText, mediaItem.FormatGroup, epubPath, "")
 	if loc.Position != "" && loc.Position != epubcfi {
 		return loc.Position
 	}
 	return ""
 }

+// pdfRectAnchor is the JSON locator the web reader stores in epubcfi_start
+// for PDF text highlights (page-fraction rects; page index is 0-based).
+type pdfRectAnchor struct {
+	V     int         `json:"v"`
+	Page  int         `json:"page"`
+	Rects [][]float64 `json:"rects"`
+}
+
+// koreaderPos0 resolves a device-native KOReader pos0 from an annotation's
+// stored locators, whatever the source. Resolution order:
+//
+// extendXPointerByLength advances a CRE xpointer's trailing text-node
+// character offset by the rune length of text, so a highlight with only a
+// start anchor still gets a plausible (non-collapsed) end for drawing.
+// Overshooting the node just clamps on the device.
+func extendXPointerByLength(xp, text string) string {
+	if xp == "" || text == "" {
+		return xp
+	}
+	i := strings.LastIndex(xp, ".")
+	if i < 0 {
+		return xp
+	}
+	off, err := strconv.Atoi(xp[i+1:])
+	if err != nil {
+		return xp
+	}
+	return xp[:i+1] + strconv.Itoa(off+utf8.RuneCountInString(text))
+}
+
+// KOReader paints highlight colors from a fixed set of names
+// (Blitbuffer.HIGHLIGHT_COLORS); the web reader uses hex swatches. Map at
+// the boundary so each side always receives something it can render;
+// unmappable values fall back to each side's default (yellow).
+var koreaderColorFromName = map[string]string{
+	"yellow": "#ffd54f",
+	"orange": "#ffd54f",
+	"green":  "#a5d6a7",
+	"olive":  "#a5d6a7",
+	"cyan":   "#90caf9",
+	"blue":   "#90caf9",
+	"purple": "#ce93d8",
+	"red":    "#f48fb1",
+}
+
+// mapColorFromKOReader normalizes a device color name to a web hex
+// swatch (default yellow) when ingesting device pushes.
+func mapColorFromKOReader(name string) string {
+	if hex, ok := koreaderColorFromName[strings.ToLower(strings.TrimSpace(name))]; ok {
+		return hex
+	}
+	return "#ffd54f"
+}
+
+var koreaderColorFromHex = map[string]string{
+	"#ffd54f": "yellow",
+	"#a5d6a7": "green",
+	"#90caf9": "blue",
+	"#ce93d8": "purple",
+	"#f48fb1": "purple",
+}
+
+// mapColorToKOReader normalizes a web hex swatch to the nearest KOReader
+// color name (default yellow) when serving to devices. Pink maps to purple
+// (the palette's closest); round-trip drift is prevented on the device by
+// suppressing echo colors for un-edited applied entries.
+func mapColorToKOReader(hex string) string {
+	if name, ok := koreaderColorFromHex[strings.ToLower(strings.TrimSpace(hex))]; ok {
+		return name
+	}
+	return "yellow"
+}
+
+//  1. A device-native CRE xpointer ("/body/...") in startPosition wins —
+//     round-trip identical for KOReader-pushed annotations (converting the
+//     stored CFI instead could drift and duplicate on the device).
+//  2. The web reader's PDF JSON anchor → bare page number (KOReader paging
+//     documents use the page number as pos0).
+//  3. A stored EPUB CFI (epubcfi_start, or startPosition without the
+//     reader's "cfi:" prefix) → converted to a CRE xpointer, with
+//     contextText (the selection text) enabling the text-search fallback.
+//  4. A "page:N" or bare-numeric position → the bare number.
+//
+// Returns "" when nothing usable exists; callers skip such annotations so
+// devices never receive locators they cannot place.
+func (h *KOReaderHandler) koreaderPos0(c *echo.Context, mediaItem database.MediaItems, startPosition, epubcfi, contextText string) string {
+	if wsync.IsCREXPointer(startPosition) {
+		return startPosition
+	}
+	if strings.HasPrefix(epubcfi, "{") {
+		var anchor pdfRectAnchor
+		if json.Unmarshal([]byte(epubcfi), &anchor) == nil && anchor.Page >= 0 {
+			return strconv.Itoa(anchor.Page)
+		}
+	}
+	cfi := epubcfi
+	if cfi == "" && strings.HasPrefix(startPosition, "cfi:") {
+		cfi = strings.TrimPrefix(startPosition, "cfi:")
+	}
+	if cfi != "" && wsync.IsStandardEPUBCFI(cfi) {
+		if converted := h.reverseConvertCFI(c, mediaItem, cfi, contextText); converted != "" {
+			return converted
+		}
+		// Conversion failed; fall through so numeric positions still work.
+		if wsync.IsCREXPointer(cfi) {
+			return cfi
+		}
+	}
+	if p := strings.TrimPrefix(startPosition, "page:"); p != "" && parsePageInt(p) >= 0 {
+		return p
+	}
+	return ""
+}
+
+func parsePageInt(s string) int64 {
+	var n int64
+	for _, r := range s {
+		if r < '0' || r > '9' {
+			return -1
+		}
+		n = n*10 + int64(r-'0')
+	}
+	return n
+}
+
 func (h *KOReaderHandler) GetLibrary(c *echo.Context) error {
 	device := c.Get("device").(database.Devices)
 	userID := device.UserID.Bytes
@@ -1195,13 +1510,13 @@ func (h *KOReaderHandler) SyncBookmarks(c *echo.Context) error {
 			endPos = startPos
 		}

-		color := "#ffff00"
+		color := "#ffd54f"
 		if highlight.Color != "" {
-			color = highlight.Color
+			color = mapColorFromKOReader(highlight.Color)
 		}

 		if h.annotationSvc != nil {
-			epubcfiStart, epubcfiEnd := h.convertHighlightPositions(ctx, mediaItemID, highlight.Pos0, highlight.Pos1)
+			epubcfiStart, epubcfiEnd := h.convertHighlightPositions(ctx, mediaItemID, highlight.Pos0, highlight.Pos1, highlight.Text)

 			pctStart := 0.0
 			if highlight.Percentage != nil {
@@ -23,8 +23,8 @@ import (
 const TombstoneTTL = 30 * 24 * time.Hour

 type AnnotationService struct {
-	db      *database.Queries
-	connMgr *ConnectionManager
+	db       *database.Queries
+	connMgr  *ConnectionManager
 	settings *database.SettingsRegistry
 }

@@ -75,6 +75,11 @@ type SaveHighlightRequest struct {
 	Source           string
 	ModifiedAt       time.Time
 	DeviceSyncData   json.RawMessage
+	// DedupKey overrides the computed key when the client echoes back an
+	// annotation it received from us (device echoes carry device-native
+	// locators, so the computed key would never match the original row and
+	// every pull→push cycle would mint a duplicate).
+	DedupKey string
 }

 type SaveHighlightResult struct {
@@ -84,7 +89,10 @@ type SaveHighlightResult struct {
 }

 func (s *AnnotationService) SaveHighlight(ctx context.Context, req SaveHighlightRequest) (*SaveHighlightResult, error) {
-	dedupKey := ComputeDedupKey(req.SelectionText, req.EpubcfiStart, req.StartPosition)
+	dedupKey := req.DedupKey
+	if dedupKey == "" {
+		dedupKey = ComputeDedupKey(req.SelectionText, req.EpubcfiStart, req.StartPosition)
+	}

 	existing, err := s.db.GetMediaHighlightByDedupKey(ctx, database.GetMediaHighlightByDedupKeyParams{
 		UserID:      req.UserID,
@@ -124,22 +132,22 @@ func (s *AnnotationService) createHighlight(
 	deviceData := mergeDeviceSyncData(nil, req.Source, req.DeviceSyncData)

 	highlight, err := s.db.CreateMediaHighlightFull(ctx, database.CreateMediaHighlightFullParams{
-		MediaItemID:      req.MediaItemID,
-		UserID:           req.UserID,
-		SelectionText:    req.SelectionText,
-		StartPosition:    pgText(req.StartPosition),
-		EndPosition:      pgText(req.EndPosition),
-		Color:            pgText(req.Color),
-		NoteText:         pgText(req.NoteText),
-		PercentageStart:  pgFloat8(req.PercentageStart),
-		PercentageEnd:    pgFloat8(req.PercentageEnd),
-		EpubcfiStart:     pgText(req.EpubcfiStart),
-		EpubcfiEnd:       pgText(req.EpubcfiEnd),
-		ChapterReference: pgInt4(req.ChapterReference),
-		DedupKey:         pgtype.Text{String: dedupKey, Valid: true},
-		LastModifiedAt:   pgtype.Timestamptz{Time: modifiedAt, Valid: true},
+		MediaItemID:        req.MediaItemID,
+		UserID:             req.UserID,
+		SelectionText:      req.SelectionText,
+		StartPosition:      pgText(req.StartPosition),
+		EndPosition:        pgText(req.EndPosition),
+		Color:              pgText(req.Color),
+		NoteText:           pgText(req.NoteText),
+		PercentageStart:    pgFloat8(req.PercentageStart),
+		PercentageEnd:      pgFloat8(req.PercentageEnd),
+		EpubcfiStart:       pgText(req.EpubcfiStart),
+		EpubcfiEnd:         pgText(req.EpubcfiEnd),
+		ChapterReference:   pgInt4(req.ChapterReference),
+		DedupKey:           pgtype.Text{String: dedupKey, Valid: true},
+		LastModifiedAt:     pgtype.Timestamptz{Time: modifiedAt, Valid: true},
 		LastModifiedSource: pgtype.Text{String: req.Source, Valid: req.Source != ""},
-		DeviceSyncData:   deviceData,
+		DeviceSyncData:     deviceData,
 	})
 	if err != nil {
 		return nil, fmt.Errorf("create highlight: %w", err)
@@ -179,20 +187,20 @@ func (s *AnnotationService) applyLWW(
 	deviceData := mergeDeviceSyncData(existing.DeviceSyncData, req.Source, req.DeviceSyncData)

 	highlight, err := s.db.UpdateMediaHighlightForSync(ctx, database.UpdateMediaHighlightForSyncParams{
-		ID:               existing.ID,
-		SelectionText:    req.SelectionText,
-		StartPosition:    pgText(req.StartPosition),
-		EndPosition:      pgText(req.EndPosition),
-		Color:            pgText(req.Color),
-		NoteText:         pgText(req.NoteText),
-		PercentageStart:  pgFloat8(req.PercentageStart),
-		PercentageEnd:    pgFloat8(req.PercentageEnd),
-		EpubcfiStart:     pgText(req.EpubcfiStart),
-		EpubcfiEnd:       pgText(req.EpubcfiEnd),
-		ChapterReference: pgInt4(req.ChapterReference),
-		LastModifiedAt:   pgtype.Timestamptz{Time: modifiedAt, Valid: true},
+		ID:                 existing.ID,
+		SelectionText:      req.SelectionText,
+		StartPosition:      pgText(req.StartPosition),
+		EndPosition:        pgText(req.EndPosition),
+		Color:              pgText(req.Color),
+		NoteText:           pgText(req.NoteText),
+		PercentageStart:    pgFloat8(req.PercentageStart),
+		PercentageEnd:      pgFloat8(req.PercentageEnd),
+		EpubcfiStart:       pgText(req.EpubcfiStart),
+		EpubcfiEnd:         pgText(req.EpubcfiEnd),
+		ChapterReference:   pgInt4(req.ChapterReference),
+		LastModifiedAt:     pgtype.Timestamptz{Time: modifiedAt, Valid: true},
 		LastModifiedSource: pgtype.Text{String: req.Source, Valid: req.Source != ""},
-		DeviceSyncData:   deviceData,
+		DeviceSyncData:     deviceData,
 	})
 	if err != nil {
 		return nil, fmt.Errorf("update highlight: %w", err)
@@ -335,6 +343,7 @@ type SaveNoteRequest struct {
 	Source             string
 	ModifiedAt         time.Time
 	DeviceSyncData     []byte
+	DedupKey           string // overrides the computed key for device echoes
 }

 type SaveNoteResult struct {
@@ -348,7 +357,10 @@ func (s *AnnotationService) SaveNote(ctx context.Context, req SaveNoteRequest) (
 		return nil, errors.New("invalid user_id or media_item_id")
 	}

-	dedupKey := ComputeDedupKey(req.Content, req.EpubcfiLocation, req.Position)
+	dedupKey := req.DedupKey
+	if dedupKey == "" {
+		dedupKey = ComputeDedupKey(req.Content, req.EpubcfiLocation, req.Position)
+	}

 	existing, err := s.db.GetMediaNoteByDedupKey(ctx, database.GetMediaNoteByDedupKeyParams{
 		UserID:      req.UserID,
@@ -485,6 +497,9 @@ type SaveBookmarkRequest struct {
 	Source           string
 	ModifiedAt       time.Time
 	DeviceSyncData   json.RawMessage
+	// DedupKey overrides the computed key for device echoes (see
+	// SaveHighlightRequest).
+	DedupKey string
 }

 type SaveBookmarkResult struct {
@@ -494,7 +509,10 @@ type SaveBookmarkResult struct {
 }

 func (s *AnnotationService) SaveBookmark(ctx context.Context, req SaveBookmarkRequest) (*SaveBookmarkResult, error) {
-	dedupKey := ComputeDedupKey(req.Title, req.EpubcfiLocation, req.Position)
+	dedupKey := req.DedupKey
+	if dedupKey == "" {
+		dedupKey = ComputeDedupKey(req.Title, req.EpubcfiLocation, req.Position)
+	}

 	existing, err := s.db.GetMediaBookmarkByDedupKey(ctx, database.GetMediaBookmarkByDedupKeyParams{
 		UserID:      req.UserID,
@@ -530,21 +548,21 @@ func (s *AnnotationService) createBookmark(ctx context.Context, req SaveBookmark
 	deviceData := mergeDeviceSyncData(nil, req.Source, req.DeviceSyncData)

 	bm, err := s.db.CreateMediaBookmarkFull(ctx, database.CreateMediaBookmarkFullParams{
-		MediaItemID:      req.MediaItemID,
-		UserID:           req.UserID,
-		PageNumber:       pgInt4(req.PageNumber),
-		ChapterNumber:    pgInt4(req.ChapterNumber),
-		CfiPosition:      pgText(req.CFIPosition),
-		Title:            req.Title,
-		Position:         pgText(req.Position),
-		Notes:            pgText(req.Notes),
+		MediaItemID:        req.MediaItemID,
+		UserID:             req.UserID,
+		PageNumber:         pgInt4(req.PageNumber),
+		ChapterNumber:      pgInt4(req.ChapterNumber),
+		CfiPosition:        pgText(req.CFIPosition),
+		Title:              req.Title,
+		Position:           pgText(req.Position),
+		Notes:              pgText(req.Notes),
 		PercentageLocation: pgFloat8(req.PercentageLoc),
-		EpubcfiLocation:  pgText(req.EpubcfiLocation),
-		ChapterReference: pgInt4(req.ChapterReference),
-		DedupKey:         pgtype.Text{String: dedupKey, Valid: true},
-		LastModifiedAt:   pgtype.Timestamptz{Time: modifiedAt, Valid: true},
+		EpubcfiLocation:    pgText(req.EpubcfiLocation),
+		ChapterReference:   pgInt4(req.ChapterReference),
+		DedupKey:           pgtype.Text{String: dedupKey, Valid: true},
+		LastModifiedAt:     pgtype.Timestamptz{Time: modifiedAt, Valid: true},
 		LastModifiedSource: pgtype.Text{String: req.Source, Valid: req.Source != ""},
-		DeviceSyncData:   deviceData,
+		DeviceSyncData:     deviceData,
 	})
 	if err != nil {
 		return nil, fmt.Errorf("create bookmark: %w", err)
@@ -573,19 +591,19 @@ func (s *AnnotationService) applyBookmarkLWW(ctx context.Context, req SaveBookma
 	deviceData := mergeDeviceSyncData(existing.DeviceSyncData, req.Source, req.DeviceSyncData)

 	bm, err := s.db.UpdateMediaBookmarkForSync(ctx, database.UpdateMediaBookmarkForSyncParams{
-		ID:               existing.ID,
-		PageNumber:       pgInt4(req.PageNumber),
-		ChapterNumber:    pgInt4(req.ChapterNumber),
-		CfiPosition:      pgText(req.CFIPosition),
-		Title:            req.Title,
-		Position:         pgText(req.Position),
-		Notes:            pgText(req.Notes),
+		ID:                 existing.ID,
+		PageNumber:         pgInt4(req.PageNumber),
+		ChapterNumber:      pgInt4(req.ChapterNumber),
+		CfiPosition:        pgText(req.CFIPosition),
+		Title:              req.Title,
+		Position:           pgText(req.Position),
+		Notes:              pgText(req.Notes),
 		PercentageLocation: pgFloat8(req.PercentageLoc),
-		EpubcfiLocation:  pgText(req.EpubcfiLocation),
-		ChapterReference: pgInt4(req.ChapterReference),
-		LastModifiedAt:   pgtype.Timestamptz{Time: modifiedAt, Valid: true},
+		EpubcfiLocation:    pgText(req.EpubcfiLocation),
+		ChapterReference:   pgInt4(req.ChapterReference),
+		LastModifiedAt:     pgtype.Timestamptz{Time: modifiedAt, Valid: true},
 		LastModifiedSource: pgtype.Text{String: req.Source, Valid: req.Source != ""},
-		DeviceSyncData:   deviceData,
+		DeviceSyncData:     deviceData,
 	})
 	if err != nil {
 		return nil, fmt.Errorf("update bookmark: %w", err)
@@ -11,6 +11,7 @@ import (
 	"regexp"
 	"strconv"
 	"strings"
+	"sync"
 	"unicode/utf8"

 	"golang.org/x/net/html"
@@ -19,6 +20,9 @@ import (
 type CFIConverter struct {
 	epubPath string
 	cache    *spineCache
+	// mu guards the lazily-built spine/doc caches: converter instances are
+	// shared across concurrent requests via the package cache in locators.go.
+	mu sync.Mutex
 }

 type spineItem struct {
@@ -37,6 +41,8 @@ func NewCFIConverter(epubPath string) *CFIConverter {
 }

 func (c *CFIConverter) loadSpine() (*spineCache, error) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
 	if c.cache != nil {
 		return c.cache, nil
 	}
@@ -94,6 +100,8 @@ func (c *CFIConverter) getContentDoc(fragmentIndex int) (*html.Node, string, err
 	item := spine.items[spineIndex]
 	href := item.href

+	c.mu.Lock()
+	defer c.mu.Unlock()
 	if cached, ok := spine.docCache[href]; ok {
 		return cached, href, nil
 	}
@@ -243,6 +251,46 @@ type ConversionResult struct {
 	Precision  string
 }

+// SectionPercentage derives an approximate book-wide percentage for a CRE
+// xpointer from the char distribution across the spine: the midpoint of the
+// document it points into. Precision is per-section, which is what
+// percentage_start is used for (ordering/filtering) — and it lets thin
+// clients skip their own per-annotation page lookups entirely.
+func (c *CFIConverter) SectionPercentage(xpointer string) float64 {
+	xp, err := ParseCREXPointer(xpointer)
+	if err != nil {
+		return 0
+	}
+	spine, err := c.loadSpine()
+	if err != nil {
+		return 0
+	}
+	total := 0
+	charCounts := make([]int, len(spine.items))
+	for i := range spine.items {
+		doc, _, docErr := c.getContentDoc(i + 1)
+		if docErr != nil {
+			continue
+		}
+		if b := findBody(doc); b != nil {
+			charCounts[i] = countTextChars(b)
+			total += charCounts[i]
+		}
+	}
+	if total <= 0 {
+		return 0
+	}
+	idx := xp.FragmentIndex - 1
+	if idx < 0 || idx >= len(spine.items) {
+		return 0
+	}
+	before := 0
+	for i := 0; i < idx; i++ {
+		before += charCounts[i]
+	}
+	return (float64(before) + float64(charCounts[idx])/2) / float64(total)
+}
+
 func (c *CFIConverter) ConvertCREToStandard(xpointer string, storedPercentage float64, contextText string) (*ConversionResult, error) {
 	if IsCREFragmentID(xpointer) {
 		return c.convertFragmentID(xpointer, storedPercentage)
@@ -884,8 +932,8 @@ func readZipFile(zr *zip.Reader, name string) ([]byte, error) {
 }

 type opfContainer struct {
-	XMLName   xml.Name   `xml:"container"`
-	RootFiles []opfRoot  `xml:"rootfiles>rootfile"`
+	XMLName   xml.Name  `xml:"container"`
+	RootFiles []opfRoot `xml:"rootfiles>rootfile"`
 }

 type opfRoot struct {
@@ -906,8 +954,8 @@ func extractOPFPath(data []byte) (string, error) {
 }

 type xmlPackage struct {
-	XMLName xml.Name `xml:"package"`
-	Spine   xmlSpine `xml:"spine"`
+	XMLName  xml.Name    `xml:"package"`
+	Spine    xmlSpine    `xml:"spine"`
 	Manifest xmlManifest `xml:"manifest"`
 }

@@ -1036,9 +1084,9 @@ func preprocessXHTML(input string) string {
 }

 type cfiStep struct {
-	Index    int
-	ID       string
-	Offset   int
+	Index     int
+	ID        string
+	Offset    int
 	HasOffset bool
 }

@@ -1,6 +1,8 @@
 package sync

 import (
+	"archive/zip"
+	"os"
 	"strings"
 	"testing"

@@ -9,7 +11,7 @@ import (

 func TestParseCREXPointer(t *testing.T) {
 	tests := []struct {
-		input   string
+		input    string
 		wantFrag int
 		wantPath int
 		wantChar int
@@ -72,9 +74,9 @@ func TestIsCREFragmentID(t *testing.T) {

 func TestParseCREFragmentID(t *testing.T) {
 	tests := []struct {
-		input       string
-		wantSpine   int
-		wantAnchor  string
+		input      string
+		wantSpine  int
+		wantAnchor string
 	}{
 		{"#_doc_fragment_5_ link2HCH0002", 5, "link2HCH0002"},
 		{"#_doc_fragment_0_", 0, ""},
@@ -116,57 +118,224 @@ func TestIsStandardEPUBCFI(t *testing.T) {
 	}
 }

-func TestConvert1984(t *testing.T) {
-	epubPath := "/home/nymusicman/Code/bookhoard/uploads/Ebooks/George Orwell/1984 (126)/1984 - George Orwell.epub"
-	c := NewCFIConverter(epubPath)
+// writeTestEPUB builds a minimal, deterministic EPUB in a temp dir so the
+// conversion tests exercise the real zip→OPF→spine→document pipeline
+// without depending on books in a particular machine's uploads/ tree.
+//
+// Spine: doc1..doc6. doc2 carries the Dashwood sentence used for exact and
+// text-search anchoring; doc6 has an id anchor for fragment-ID conversion.
+func writeTestEPUB(t *testing.T) string {
+	t.Helper()

-	xp := "/body/DocFragment[2]/body/div/p[5]/text().500"
-	result, err := c.ConvertCREToStandard(xp, 0.01, "")
+	type spineDoc struct {
+		name string
+		body string
+	}
+	docs := []spineDoc{
+		{"doc1.xhtml", "<body><div><p>Chapter one opening page.</p></div></body>"},
+		{"doc2.xhtml", "<body><div><p>The family of Dashwood had long been settled in Sussex.</p><p>Their estate was large, and their residence was at Norland Park.</p></div></body>"},
+		{"doc3.xhtml", "<body><div><p>Chapter three contents.</p></div></body>"},
+		{"doc4.xhtml", "<body><div><p>Chapter four contents.</p></div></body>"},
+		{"doc5.xhtml", "<body><div><p>Chapter five contents.</p></div></body>"},
+		{"doc6.xhtml", "<body><div><p id=\"link2HCH0002\">He was neither fit to be a husband nor a father.</p></div></body>"},
+	}
+
+	containerXML := `<?xml version="1.0"?>
+<container version="1.0" xmlns="urn:oasis:names:tc:opendocument:xmlns:container">
+  <rootfiles>
+    <rootfile full-path="OEBPS/content.opf" media-type="application/oebps-package+xml"/>
+  </rootfiles>
+</container>`
+
+	manifest := ""
+	spineRefs := ""
+	for _, d := range docs {
+		id := d.name[:len(d.name)-len(".xhtml")]
+		manifest += "    <item id=\"" + id + "\" href=\"" + d.name + "\" media-type=\"application/xhtml+xml\"/>\n"
+		spineRefs += "    <itemref idref=\"" + id + "\"/>\n"
+	}
+	opf := `<?xml version="1.0" encoding="UTF-8"?>
+<package xmlns="http://www.idpf.org/2007/opf" version="3.0" unique-identifier="uid">
+  <metadata xmlns:dc="http://purl.org/dc/elements/1.1/">
+    <dc:identifier id="uid">test-bookhoard-fixture</dc:identifier>
+    <dc:title>Fixture</dc:title>
+  </metadata>
+  <manifest>
+` + manifest + `  </manifest>
+  <spine>
+` + spineRefs + `  </spine>
+</package>`
+
+	path := t.TempDir() + "/fixture.epub"
+	f, err := os.Create(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer f.Close()
+	zw := zip.NewWriter(f)
+	write := func(name, content string) {
+		w, err := zw.Create(name)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if _, err := w.Write([]byte(content)); err != nil {
+			t.Fatal(err)
+		}
+	}
+	write("META-INF/container.xml", containerXML)
+	write("OEBPS/content.opf", opf)
+	for _, d := range docs {
+		write("OEBPS/"+d.name, "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<html xmlns=\"http://www.w3.org/1999/xhtml\">"+d.body+"</html>\n")
+	}
+	if err := zw.Close(); err != nil {
+		t.Fatal(err)
+	}
+	return path
+}
+
+const fixtureSentence = "The family of Dashwood had long been settled in Sussex."
+
+func TestConvertXPointerToCFI(t *testing.T) {
+	c := NewCFIConverter(writeTestEPUB(t))
+
+	xp := "/body/DocFragment[2]/body/div[1]/p[1]/text().10"
+	result, err := c.ConvertCREToStandard(xp, 0.05, "")
 	if err != nil {
 		t.Fatalf("ConvertCREToStandard error: %v", err)
 	}
 	t.Logf("Input: %s", xp)
 	t.Logf("EPUBCFI: %s", result.EPUBCFI)
-	t.Logf("Href: %s", result.Href)
 	t.Logf("Precision: %s", result.Precision)
-	t.Logf("Percentage: %.4f", result.Percentage)

 	if result.Precision == "percentage" {
 		t.Error("expected better than percentage precision")
 	}
+	if result.EPUBCFI == "" {
+		t.Error("expected non-empty epubcfi")
+	}
 }

-func TestConvertCrimeAndPunishmentFragmentID(t *testing.T) {
-	epubPath := "/home/nymusicman/Code/bookhoard/uploads/Ebooks/Fyodor Dostoyevsky/Crime and Punishment (103)/Crime and Punishment - Fyodor Dostoyevsky.epub"
-	c := NewCFIConverter(epubPath)
+func TestConvertFragmentID(t *testing.T) {
+	c := NewCFIConverter(writeTestEPUB(t))

-	xp := "#_doc_fragment_5_ link2HCH0002"
-	result, err := c.ConvertCREToStandard(xp, 0.0303, "")
+	frag := "#_doc_fragment_5_ link2HCH0002"
+	result, err := c.ConvertCREToStandard(frag, 0.9, "")
 	if err != nil {
 		t.Fatalf("ConvertCREToStandard error: %v", err)
 	}
-	t.Logf("Input: %s", xp)
-	t.Logf("EPUBCFI: %s", result.EPUBCFI)
+	t.Logf("Input: %s", frag)
 	t.Logf("Href: %s", result.Href)
 	t.Logf("Precision: %s", result.Precision)
-	t.Logf("Percentage: %.4f", result.Percentage)

-	if result.Precision == "percentage" {
-		t.Error("expected better than percentage precision")
+	if result.Precision != "element" {
+		t.Errorf("expected element precision, got %s", result.Precision)
 	}
 	if result.Href == "" {
 		t.Error("expected non-empty href")
 	}
-	if result.Precision != "element" {
-		t.Errorf("expected element precision, got %s", result.Precision)
+	if !strings.Contains(result.Href, "doc6.xhtml#link2HCH0002") {
+		t.Errorf("expected doc6.xhtml#link2HCH0002 href, got %s", result.Href)
 	}
 }

+func TestRoundTripXPointer(t *testing.T) {
+	c := NewCFIConverter(writeTestEPUB(t))
+
+	originalXP := "/body/DocFragment[2]/body/div[1]/p[1]/text().10"
+	forward, err := c.ConvertCREToStandard(originalXP, 0.05, "")
+	if err != nil {
+		t.Fatalf("forward conversion error: %v", err)
+	}
+	if forward.EPUBCFI == "" {
+		t.Fatal("forward conversion produced empty epubcfi")
+	}
+	t.Logf("Forward: %s → %s", originalXP, forward.EPUBCFI)
+
+	reverse, err := c.ConvertStandardToCRE(forward.EPUBCFI, forward.Percentage, "")
+	if err != nil {
+		t.Fatalf("reverse conversion error: %v", err)
+	}
+	if reverse.XPointer == "" {
+		t.Fatal("reverse conversion produced empty XPointer")
+	}
+	t.Logf("Reverse: %s → %s", forward.EPUBCFI, reverse.XPointer)
+
+	if reverse.Precision != "exact" {
+		t.Errorf("expected exact precision, got %s", reverse.Precision)
+	}
+}
+
+func TestRoundTripWithContextText(t *testing.T) {
+	c := NewCFIConverter(writeTestEPUB(t))
+
+	originalXP := "/body/DocFragment[2]/body/div[1]/p[2]/text().3"
+	forward, err := c.ConvertCREToStandard(originalXP, 0.06, fixtureSentence)
+	if err != nil {
+		t.Fatalf("forward conversion error: %v", err)
+	}
+	if forward.EPUBCFI == "" {
+		t.Fatal("forward conversion produced empty epubcfi")
+	}
+	t.Logf("Forward: %s → %s", originalXP, forward.EPUBCFI)
+
+	reverse, err := c.ConvertStandardToCRE(forward.EPUBCFI, forward.Percentage, fixtureSentence)
+	if err != nil {
+		t.Fatalf("reverse conversion error: %v", err)
+	}
+	if reverse.XPointer == "" {
+		t.Fatal("reverse conversion produced empty XPointer")
+	}
+	t.Logf("Reverse: %s → %s", forward.EPUBCFI, reverse.XPointer)
+
+	if reverse.Precision != "exact" {
+		t.Errorf("expected exact precision, got %s", reverse.Precision)
+	}
+}
+
+func TestReverseTextSearchFallback(t *testing.T) {
+	c := NewCFIConverter(writeTestEPUB(t))
+
+	// Unresolvable steps in a CFI that still parses to spine doc2
+	// (spine index 1): the text search must anchor on the sentence.
+	reverse, err := c.ConvertStandardToCRE("epubcfi(/6/4!/4/99999/1:0)", 0.05, fixtureSentence)
+	if err != nil {
+		t.Fatalf("reverse conversion error: %v", err)
+	}
+	t.Logf("Text search fallback XPointer: %s", reverse.XPointer)
+	t.Logf("Precision: %s", reverse.Precision)
+
+	if reverse.Precision != "exact" {
+		t.Errorf("expected exact precision from text search, got %s", reverse.Precision)
+	}
+	if reverse.XPointer == "" {
+		t.Error("expected non-empty XPointer from text search")
+	}
+	if !strings.Contains(reverse.XPointer, "DocFragment[2]") {
+		t.Errorf("expected fallback into DocFragment[2], got %s", reverse.XPointer)
+	}
+}
+
+func TestReversePercentageFallback(t *testing.T) {
+	c := NewCFIConverter(writeTestEPUB(t))
+
+	reverse, err := c.ConvertStandardToCRE("epubcfi(/6/4!/4/99999/1:0)", 0.5, "")
+	if err != nil {
+		t.Fatalf("reverse conversion error: %v", err)
+	}
+	t.Logf("Percentage fallback precision: %s", reverse.Precision)
+
+	if reverse.Precision != "percentage" {
+		t.Errorf("expected percentage precision, got %s with XPointer %s", reverse.Precision, reverse.XPointer)
+	}
+	if reverse.XPointer != "" {
+		t.Error("expected empty XPointer for percentage fallback")
+	}
+}
 func TestParseEPUBCFI(t *testing.T) {
 	tests := []struct {
-		input      string
-		wantSpine  int
-		wantSteps  int
+		input     string
+		wantSpine int
+		wantSteps int
 	}{
 		{"epubcfi(/6/12!/4/2/90/1:7)", 5, 4},
 		{"epubcfi(/6/4!/4/2/1:0)", 1, 3},
@@ -213,103 +382,6 @@ func TestParseEPUBCFIInvalid(t *testing.T) {
 	}
 }

-func TestRoundTrip1984(t *testing.T) {
-	epubPath := "/home/nymusicman/Code/bookhoard/uploads/Ebooks/George Orwell/1984 (126)/1984 - George Orwell.epub"
-	c := NewCFIConverter(epubPath)
-
-	originalXP := "/body/DocFragment[2]/body/div/p[5]/text().500"
-	forward, err := c.ConvertCREToStandard(originalXP, 0.01, "")
-	if err != nil {
-		t.Fatalf("forward conversion error: %v", err)
-	}
-	if forward.EPUBCFI == "" {
-		t.Fatal("forward conversion produced empty epubcfi")
-	}
-	t.Logf("Forward: %s → %s", originalXP, forward.EPUBCFI)
-
-	reverse, err := c.ConvertStandardToCRE(forward.EPUBCFI, forward.Percentage, "")
-	if err != nil {
-		t.Fatalf("reverse conversion error: %v", err)
-	}
-	if reverse.XPointer == "" {
-		t.Fatal("reverse conversion produced empty XPointer")
-	}
-	t.Logf("Reverse: %s → %s", forward.EPUBCFI, reverse.XPointer)
-	t.Logf("Reverse precision: %s", reverse.Precision)
-
-	if reverse.Precision != "exact" {
-		t.Errorf("expected exact precision, got %s", reverse.Precision)
-	}
-}
-
-func TestRoundTripCP(t *testing.T) {
-	epubPath := "/home/nymusicman/Code/bookhoard/uploads/Ebooks/Fyodor Dostoyevsky/Crime and Punishment (103)/Crime and Punishment - Fyodor Dostoyevsky.epub"
-	c := NewCFIConverter(epubPath)
-
-	originalXP := "/body/DocFragment[6]/body/div/p[47]/text().2399"
-	contextText := "Raskolnikov was not used to crowds, and, as we said before, he avoided society of every sort, more especially of l"
-	forward, err := c.ConvertCREToStandard(originalXP, 0.0579, contextText)
-	if err != nil {
-		t.Fatalf("forward conversion error: %v", err)
-	}
-	if forward.EPUBCFI == "" {
-		t.Fatal("forward conversion produced empty epubcfi")
-	}
-	t.Logf("Forward: %s → %s", originalXP, forward.EPUBCFI)
-
-	reverse, err := c.ConvertStandardToCRE(forward.EPUBCFI, forward.Percentage, contextText)
-	if err != nil {
-		t.Fatalf("reverse conversion error: %v", err)
-	}
-	if reverse.XPointer == "" {
-		t.Fatal("reverse conversion produced empty XPointer")
-	}
-	t.Logf("Reverse: %s → %s", forward.EPUBCFI, reverse.XPointer)
-	t.Logf("Reverse precision: %s", reverse.Precision)
-
-	if reverse.Precision != "exact" {
-		t.Errorf("expected exact precision, got %s", reverse.Precision)
-	}
-}
-
-func TestReverseTextSearchFallback(t *testing.T) {
-	epubPath := "/home/nymusicman/Code/bookhoard/uploads/Ebooks/Fyodor Dostoyevsky/Crime and Punishment (103)/Crime and Punishment - Fyodor Dostoyevsky.epub"
-	c := NewCFIConverter(epubPath)
-
-	contextText := "Raskolnikov was not used to crowds, and, as we said before, he avoided society of every sort, more especially of l"
-	reverse, err := c.ConvertStandardToCRE("epubcfi(/6/12!/4/99999/1:0)", 0.0579, contextText)
-	if err != nil {
-		t.Fatalf("reverse conversion error: %v", err)
-	}
-	t.Logf("Text search fallback XPointer: %s", reverse.XPointer)
-	t.Logf("Precision: %s", reverse.Precision)
-
-	if reverse.Precision != "exact" {
-		t.Errorf("expected exact precision from text search, got %s", reverse.Precision)
-	}
-	if reverse.XPointer == "" {
-		t.Error("expected non-empty XPointer from text search")
-	}
-}
-
-func TestReversePercentageFallback(t *testing.T) {
-	epubPath := "/home/nymusicman/Code/bookhoard/uploads/Ebooks/George Orwell/1984 (126)/1984 - George Orwell.epub"
-	c := NewCFIConverter(epubPath)
-
-	reverse, err := c.ConvertStandardToCRE("epubcfi(/6/12!/4/99999/1:0)", 0.5, "")
-	if err != nil {
-		t.Fatalf("reverse conversion error: %v", err)
-	}
-	t.Logf("Percentage fallback precision: %s", reverse.Precision)
-
-	if reverse.Precision != "percentage" {
-		t.Errorf("expected percentage precision, got %s with XPointer %s", reverse.Precision, reverse.XPointer)
-	}
-	if reverse.XPointer != "" {
-		t.Error("expected empty XPointer for percentage fallback")
-	}
-}
-
 func TestFindTextInNode_SingleTextNode(t *testing.T) {
 	doc := parseTestHTML(`<html><body><p>Hello world this is a test</p></body></html>`)
 	body := findBody(doc)
@@ -1,6 +1,9 @@
 package sync

-import "log"
+import (
+	"log"
+	"sync"
+)

 type LocatorSource string

@@ -26,6 +29,35 @@ func isConvertible(formatGroup string) bool {
 	return formatGroup == string(FormatGroupReflowable)
 }

+// Converters parse and cache the whole EPUB (spine + content docs), so
+// creating one per annotation re-reads the book for every entry. A small
+// bounded cache lets one request — or several — share a single parse.
+// Servers are the right place for this work: clients stay thin.
+var (
+	converterMu    sync.Mutex
+	converterCache = map[string]*CFIConverter{}
+	converterOrder []string // insertion order for eviction
+)
+
+const maxCachedConverters = 8
+
+func cachedConverter(epubPath string) *CFIConverter {
+	converterMu.Lock()
+	defer converterMu.Unlock()
+	if c, ok := converterCache[epubPath]; ok {
+		return c
+	}
+	c := NewCFIConverter(epubPath)
+	converterCache[epubPath] = c
+	converterOrder = append(converterOrder, epubPath)
+	for len(converterOrder) > maxCachedConverters {
+		oldest := converterOrder[0]
+		converterOrder = converterOrder[1:]
+		delete(converterCache, oldest)
+	}
+	return c
+}
+
 func ConvertToCanonical(
 	source LocatorSource,
 	devicePos string,
@@ -48,7 +80,7 @@ func ConvertToCanonical(
 		if !IsCREXPointer(devicePos) {
 			return CanonicalLocator{CFI: devicePos, Precision: "already-standard", Percentage: percentage}
 		}
-		converter := NewCFIConverter(epubPath)
+		converter := cachedConverter(epubPath)
 		result, err := converter.ConvertCREToStandard(devicePos, percentage, contextText)
 		if err != nil || result == nil {
 			log.Printf("Bookhoard: locator CRE→CFI conversion failed: %v", err)
@@ -101,7 +133,7 @@ func ConvertFromCanonical(

 	switch source {
 	case LocatorSourceKOReader:
-		converter := NewCFIConverter(epubPath)
+		converter := cachedConverter(epubPath)
 		result, err := converter.ConvertStandardToCRE(canonicalCFI, percentage, contextText)
 		if err != nil || result == nil {
 			log.Printf("Bookhoard: locator CFI→CRE conversion failed: %v", err)
@@ -412,6 +412,8 @@ document.addEventListener("alpine:init", () => {
      note: string;
      color: string;
      cfi: string;
+      cfiEnd: string;
+      renderCfi: string;
      percentage: number;
      pdfPage: number;
      pdfRects: number[][];
@@ -443,6 +445,7 @@ document.addEventListener("alpine:init", () => {
      y: 0,
      text: "",
      cfi: "",
+      cfiEnd: "",
      id: "",
      color: "#ffd54f",
      note: "",
@@ -701,8 +704,15 @@ document.addEventListener("alpine:init", () => {
            const text = sel.toString().replace(/\s+/g, " ").trim();
            if (!text) return;
            let cfi: string;
+            let cfiEnd: string;
            try {
              cfi = this.view.getCFI(index, range);
+              // Collapse to the end point for a distinct end anchor —
+              // KOReader sync renders the highlight box from pos0/pos1, and
+              // pos1 == pos0 would be a degenerate (zero-length) range.
+              const endRange = range.cloneRange();
+              endRange.collapse(false);
+              cfiEnd = this.view.getCFI(index, endRange);
            } catch {
              return;
            }
@@ -715,6 +725,7 @@ document.addEventListener("alpine:init", () => {
              y: (iframeRect?.top ?? 0) + rect.top,
              text,
              cfi,
+              cfiEnd,
            });
          };
          doc.addEventListener(
@@ -827,6 +838,7 @@ document.addEventListener("alpine:init", () => {
          y: (iframeRect?.top ?? 0) + rect.top,
          text: h.text,
          cfi: h.cfi,
+          cfiEnd: h.cfiEnd,
          id: h.id,
          color: h.color,
          note: h.note,
@@ -1070,6 +1082,7 @@ document.addEventListener("alpine:init", () => {
      y: number;
      text: string;
      cfi: string;
+      cfiEnd?: string;
      id?: string;
      color?: string;
      note?: string;
@@ -1080,6 +1093,7 @@ document.addEventListener("alpine:init", () => {
      p.mode = opts.mode;
      p.text = opts.text;
      p.cfi = opts.cfi;
+      p.cfiEnd = opts.cfiEnd ?? "";
      p.id = opts.id ?? "";
      p.color = opts.color || "#ffd54f";
      p.note = opts.note ?? "";
@@ -1109,7 +1123,7 @@ document.addEventListener("alpine:init", () => {
        } else {
          this.view
            ?.addAnnotation({
-              value: hl.cfi,
+              value: hl.renderCfi || hl.cfi,
              color: hl.color,
              note: hl.note,
              id: hl.id,
@@ -1141,17 +1155,68 @@ document.addEventListener("alpine:init", () => {
          /* not ours; leave as-is */
        }
      }
+      const cfiEnd = r.epubcfi_end ?? "";
      return {
        id: r.id,
        text: r.selection_text ?? "",
        note: r.note_text ?? "",
        color: r.color ?? "#ffff00",
        cfi,
+        cfiEnd,
+        // Rendering/navigating anchor: device-synced highlights store
+        // POINT CFIs (epubcfi(/6/N!/4/2[id]/8/1:1)), which resolve to a
+        // collapsed range and paint nothing. Foliate's overlayer needs a
+        // RANGE CFI — same shape getCFI() produces natively
+        // (epubcfi(/6/N!/4/2[id],/8/1:1,/8/1:67)) — synthesized here from
+        // the stored start and end points when both share a base path.
+        renderCfi: this.toRangeCfi(cfi, cfiEnd, r.selection_text ?? ""),
        percentage: r.percentage_start ?? 0,
        pdfPage,
        pdfRects,
      };
    },
+    // Build a foliate-renderable RANGE CFI from stored (possibly point)
+    // CFIs. Repairs two stale shapes using the selection text: a missing
+    // end (old web highlights), and a degenerate end — the device-push
+    // converter used to fall back to a document-start CFI when the end
+    // xpointer didn't resolve exactly. In both cases the end is derived
+    // from the start offset advanced by the text's UTF-16 length (EPUB
+    // CFI offsets are UTF-16 code units); multi-node selections just fail
+    // resolution harmlessly and fall back to the point CFI.
+    toRangeCfi(start: string, end: string, text: string): string {
+      if (!start) return end || start;
+      if (start.includes(",")) return start; // already a range CFI
+      const re =
+        /^(epubcfi\(\/\d+\/\d+!\/\d+\/\d+(?:\[[^\]]*\])?)(\/(?:[^:)]+)?(?::(\d+))?)\)$/;
+      const ms = re.exec(start);
+      if (!ms) return start;
+      const base = ms[1];
+      const startLocal = ms[2];
+      const startOff = ms[3] ? parseInt(ms[3], 10) : -1;
+      const utf16len = [...(text ?? "")].reduce(
+        (n, c) => n + (c.codePointAt(0)! > 0xffff ? 2 : 1),
+        0,
+      );
+      let endLocal = "";
+      if (end && !end.includes(",")) {
+        const me = re.exec(end);
+        if (me && me[1] === base) {
+          const endOff = me[3] ? parseInt(me[3], 10) : -1;
+          // Degenerate: end resolves to the document start (the old
+          // converter fallback) or sits before the start offset.
+          const degenerate =
+            endOff === 0 ||
+            (startOff >= 0 && endOff >= 0 && endOff < startOff);
+          if (!degenerate) endLocal = me[2];
+        }
+      }
+      if (!endLocal) {
+        if (startOff < 0 || utf16len <= 0) return start; // point CFI
+        const cut = startLocal.lastIndexOf(":");
+        endLocal = `${startLocal.slice(0, cut)}:${startOff + utf16len}`;
+      }
+      return `${base},${startLocal},${endLocal})`;
+    },
    async refreshAnnotations() {
      const token = getToken();
      if (!token || !this.mediaItemId) return;
@@ -1205,6 +1270,7 @@ document.addEventListener("alpine:init", () => {
              start_position: "",
              end_position: "",
              epubcfi_start: p.pdfPage >= 0 ? pdfAnchor : p.cfi,
+              epubcfi_end: p.pdfPage >= 0 ? pdfAnchor : p.cfiEnd,
              color,
              note_text: "",
              percentage_start: this.lastRelocateDetail?.fraction ?? 0,
@@ -1230,7 +1296,10 @@ document.addEventListener("alpine:init", () => {
          }
        } else {
          this.view?.addAnnotation({
-            value: p.cfi,
+            value:
+              p.pdfPage >= 0
+                ? ""
+                : this.toRangeCfi(p.cfi, p.cfiEnd, p.text) || p.cfi,
            color,
            note: "",
            id: row.id,
@@ -1263,6 +1332,7 @@ document.addEventListener("alpine:init", () => {
              start_position: "",
              end_position: "",
              epubcfi_start: anchor,
+              epubcfi_end: p.pdfPage >= 0 ? anchor : p.cfiEnd,
              color: p.color,
              note_text: p.note,
            }),
@@ -1282,7 +1352,7 @@ document.addEventListener("alpine:init", () => {
          });
        } else {
          this.view?.addAnnotation({
-            value: p.cfi,
+            value: this.toRangeCfi(p.cfi, p.cfiEnd, p.text) || p.cfi,
            color: p.color,
            note: p.note,
            id: p.id,
@@ -1324,6 +1394,7 @@ document.addEventListener("alpine:init", () => {
    },
    goToHighlight(hl: {
      cfi: string;
+      renderCfi: string;
      pdfPage: number;
    }) {
      if (hl.pdfPage >= 0) {
@@ -1331,9 +1402,11 @@ document.addEventListener("alpine:init", () => {
        this.pushBackStack();
        this.view?.goTo?.(hl.pdfPage);
        this.closeDrawers();
-      } else if (hl.cfi) {
+      } else if (hl.renderCfi || hl.cfi) {
        this.pushBackStack();
-        this.view?.showAnnotation({ value: hl.cfi })?.catch?.(() => {});
+        this.view
+          ?.showAnnotation({ value: hl.renderCfi || hl.cfi })
+          ?.catch?.(() => {});
        this.closeDrawers();
      }
    },
Author	SHA1	Message	Date
john-okeefe	995ccb50bb	Merge branch 'test-fixture-cleanup': book-agnostic CFI converter tests Release / build-and-push (push) Successful in 2m36s Details	2026-08-20 09:22:10 -04:00
john-okeefe	4ab947f7db	test(sync): replace book-specific CFI converter fixtures with a synthetic EPUB Six converter tests pointed at absolute paths for 1984 and Crime and Punishment under uploads/ — books that don't exist on most checkouts (CI included), so the suite shipped with 5 permanently failing tests (and a sixth passing only by accident: the percentage-fallback path triggered by the missing file is the outcome it asserts). A writeTestEPUB helper now builds a minimal deterministic EPUB in t.TempDir() (zip → container.xml → OPF → 6-doc spine), so the tests exercise the real zip/OPF/spine/document pipeline with no external dependencies. The xpointer→CFI conversion, fragment-ID conversion, both round-trips (bare and context-text-anchored), and the text-search and percentage fallbacks all keep their original assertions, now against known document content. internal/sync is green for the first time on this machine.	2026-08-20 09:22:10 -04:00
john-okeefe	f07c93e582	Merge branch 'sync-annotations-fix': bidirectional annotation sync for KOReader Server-side (8 commits): web annotations finally reach KOReader and vice versa. Fixed the 400 bind failures on every annotation-carrying push (loose client types), resolved device-native pos0 locators for every source (device xpointers pass through round-trip identical, web CFIs convert to CRE xpointers with text-search anchoring, PDF anchors map to pages), derived degenerate range ends from selection length, echo-deduplication via served dedup keys (pull→push cycles converge instead of minting duplicates), web↔device color mapping at both boundaries with echo suppression (web colors flow to devices, round-trips never drift them, device edits win), drawer-based annotation classification, and tombstone propagation that can't cross-delete. Perf: parsed-EPUB converter cache (bounded, locked). Plugin-side (bookhoard.koplugin @ 4ea3966): dual-model annotation store (KOReader 2024.07+ v2 ui.annotation + legacy v1), thin-client collection (no per-annotation CRE lookups), dedup-key identity matching, device-default coloring for applied highlights with datetime_updated-based echo suppression, and native-shaped AnnotationsModified dispatches (fixes a ReaderThumbnail crash and paints immediately instead of after restart).	2026-08-20 09:13:24 -04:00
john-okeefe	178fb2eb37	feat(sync): serve web highlight colors to KOReader (mapped to its palette) Reverses the earlier "no colors to the device" decision now that the echo machinery makes it safe: GetMetadata maps the stored web hex to KOReader's fixed color names (#ce93d8→purple, #90caf9→blue, #a5d6a7→green, #ffd54f→yellow; pink maps to purple as the closest — round-trip drift is prevented on the device by echo suppression, and a device edit still wins). mapColorToKOReader restored for serving; ingest (name→hex, preserve-on-echo) unchanged.	2026-08-20 08:43:21 -04:00
john-okeefe	dafcadd211	fix(sync): echo dedup + color semantics + classification for KOReader round-trips Echo duplication: devices push their full annotation list on every sync, and an echo of a web-created annotation computed a different dedup key than the original (device locators differ from web locators) — every pull→push cycle minted a duplicate row, and cleaning those up on the web tombstoned them back to the device, deleting the just-applied copies. That was the "web highlights never appear on KOReader" experience. GetMetadata now serves each annotation's dedup_key; the device stores it on the applied entry and echoes it in pushes; SaveHighlight/SaveBookmark/SaveNote accept a DedupKey override so echoes converge onto the original row (verified: pull → echo push creates no rows, LWW skips identical content). Color semantics (per user preference): devices render their own default and cannot round-trip web colors, so GetMetadata no longer serves colors at all — every highlight syncs regardless of its web color and the device draws its default. An echo carries no color; ingest then PRESERVES the stored web color (existingHighlightColor lookup by dedup key) so round-trips never change it. A non-empty device color means the user edited the highlight there: it maps name→hex (green→#a5d6a7, default yellow) and wins. Verified: echo kept #ffd54f; a simulated device edit with "green" updated the web row to #a5d6a7. Classification: KOReader auto-fills text="in Chapter X" on page bookmarks (ReaderAnnotation:updateItemByXPointer), so the plugin's text-presence classification turned every echoed bookmark into a junk highlight on the web. v2 classification now keys off the drawer field (present = highlight/note, absent = bookmark with its label in note).	2026-08-19 19:41:58 -04:00
john-okeefe	50ec2bebf2	fix(reader): render device-synced highlights — synthesize range CFIs Device-synced highlights stored POINT CFIs (epubcfi(.../8/1:1)); the overlayer resolves those to a collapsed range and paints nothing, so KOReader-made highlights were listed in the drawer but invisible on the page. mapHighlightRow now builds a renderCfi: a proper RANGE CFI (epubcfi(base,/start,/end)) synthesized from the stored start/end points. It also repairs stale rows: missing ends (old web highlights) and degenerate document-start ends (the old converter fallback) are derived from the start offset plus the selection text's UTF-16 length. All overlay drawing, navigation (showAnnotation), and the post-create/post-edit re-adds use renderCfi. Verified in-browser against live device-synced rows: the paginator's overlayer paints the highlight rects after the fix.	2026-08-19 14:08:03 -04:00
john-okeefe	6e9b3528d8	fix(sync): synced highlights painted nowhere — degenerate range ends + color model mismatch Both directions synced data but rendered nothing: - Web reader <- devices: highlights painted no overlay. Device pushes resolve their start xpointer exactly (text-search anchored by the selection) but the end conversion carries no context and fell back to a document-start CFI (epubcfi .../1:0) — a garbage range end. When the start resolved exactly, the end is now derived from it: same node, character offset advanced by the selection's UTF-16 length (extendCFIByLength). Same repair when SERVING to devices, where old web highlights (no end anchor) and converted range CFIs both collapsed pos1 onto pos0 (extendXPointerByLength on the xpointer form) — KOReader drew zero-width highlights. - Colors: KOReader paints from a fixed name set (Blitbuffer HIGHLIGHT_COLORS), the web uses hex swatches; neither understood the other, so device colors fell back to defaults and web hex drew nothing useful on devices. Both boundaries now translate: ingest maps names to hex (default #ffd54f), GetMetadata maps hex to names (default yellow) — per-datatype edits re-push with the editing side's color, which LWW then propagates. SyncBookmarks endpoint aligned to the same mapping and default.	2026-08-19 14:08:03 -04:00
john-okeefe	97e546b2a4	feat(reader): send end-anchor CFI for EPUB highlights Web highlights stored only epubcfi_start, so devices received degenerate pos0 == pos1 (zero-length) highlight ranges. The reader now collapses the selection range to its end point for a second CFI and stores it as epubcfi_end (PDF rect anchors reuse the JSON anchor for both ends).	2026-08-18 19:13:51 -04:00
john-okeefe	1585aa1073	perf(sync): share parsed EPUBs across conversions, make converters concurrency-safe ConvertToCanonical/ConvertFromCanonical built a fresh CFIConverter per call, and each annotation converts twice (pos0+pos1) — a book with 200 highlights re-opened and re-parsed the EPUB 400+ times per sync, and again per metadata pull. A bounded 8-entry cache keyed by path now shares converters (the parsing work belongs on the server; clients stay thin). CFIConverter gained a mutex around its lazily built spine/doc caches since instances are now shared between concurrent requests. Adds CFIConverter.SectionPercentage: book-wide percentage for a CRE xpointer from the spine char distribution (midpoint of its document) — the server-side counterpart to dropping per-annotation getPageFromXPointer lookups from the plugin.	2026-08-18 19:13:51 -04:00
john-okeefe	f6e257e497	fix(sync): web annotations never reached KOReader — bind 400s + unresolvable locators Two blockers, diagnosed by simulating the plugin against the live server with real library books: 1. Every KOReader progress push carrying annotations failed the JSON bind with 400 ('cannot unmarshal string into ... chapter/page of type int') — the plugin sends chapter:'', page:'30', and for CRE documents page:'/body/...' — so annotation sync AND progress sync failed together. KOReader annotation chapter/page now use FlexInt, which accepts numbers, numeric strings, empty strings, and non-numeric strings (decoding to 0). The server is deliberately liberal here so thin clients can send raw bookmark data. 2. GetMetadata served locators KOReader cannot place, so pulled items were junk: web bookmarks leaked 'cfi:epubcfi(...)' positions, web PDF highlights had empty pos0 (skipped by the plugin, invisible), and web deletions carried no pos0 so tombstones never matched. New koreaderPos0 resolver handles every source: device-native xpointers pass through untouched (round-trip identical, verified), web PDF JSON anchors map to their page number, EPUB CFIs convert to CRE xpointers (selection text passed as text-search context for exact anchoring), 'page:N' positions strip to the bare number. Unresolvable annotations are skipped with a log line instead of poisoning devices; tombstones get pos0 injected from the new locator columns. Also: thin clients omit per-annotation percentages (paging docs still send arithmetic page/total); the server derives them — section midpoint from the spine char distribution for CRE documents, page/ page-count for fixed formats.	2026-08-18 19:13:38 -04:00
john-okeefe	0670d904a0	feat(db): locator columns for tombstoned annotations GetTombstonedAnnotationsForBook now also returns each tombstone's start_position/end_position and epubcfi_start/end (note: position/ epubcfi_location, bookmark: position/cfi_position), so serving code can resolve a device-native locator for deletions of web-created annotations, whose device_sync_data carries no pos0.	2026-08-18 19:13:23 -04:00