fix(sync): echo dedup + color semantics + classification for KOReader round-trips

Echo duplication: devices push their full annotation list on every
sync, and an echo of a web-created annotation computed a different
dedup key than the original (device locators differ from web locators)
— every pull→push cycle minted a duplicate row, and cleaning those up
on the web tombstoned them back to the device, deleting the
just-applied copies. That was the "web highlights never appear on
KOReader" experience. GetMetadata now serves each annotation's
dedup_key; the device stores it on the applied entry and echoes it in
pushes; SaveHighlight/SaveBookmark/SaveNote accept a DedupKey
override so echoes converge onto the original row (verified: pull →
echo push creates no rows, LWW skips identical content).

Color semantics (per user preference): devices render their own
default and cannot round-trip web colors, so GetMetadata no longer
serves colors at all — every highlight syncs regardless of its web
color and the device draws its default. An echo carries no color;
ingest then PRESERVES the stored web color (existingHighlightColor
lookup by dedup key) so round-trips never change it. A non-empty
device color means the user edited the highlight there: it maps
name→hex (green→#a5d6a7, default yellow) and wins. Verified: echo
kept #ffd54f; a simulated device edit with "green" updated the web
row to #a5d6a7.

Classification: KOReader auto-fills text="in Chapter X" on page
bookmarks (ReaderAnnotation:updateItemByXPointer), so the plugin's
text-presence classification turned every echoed bookmark into a junk
highlight on the web. v2 classification now keys off the drawer field
(present = highlight/note, absent = bookmark with its label in note).
This commit is contained in:
2026-08-19 19:41:58 -04:00
parent 50ec2bebf2
commit dafcadd211
2 changed files with 141 additions and 79 deletions
+63 -19
View File
@@ -106,6 +106,24 @@ func extendCFIByLength(cfi, text string) string {
return cfi[:i+1] + strconv.Itoa(off+utf16len) + ")" return cfi[:i+1] + strconv.Itoa(off+utf16len) + ")"
} }
// existingHighlightColor returns the stored color of the highlight matching
// the dedup key ("" when none) so device echoes that carry no color never
// clobber the web color.
func (h *KOReaderHandler) existingHighlightColor(ctx context.Context, mediaItemID, userID pgtype.UUID, dedupKey string) string {
if dedupKey == "" {
return ""
}
existing, err := h.db.GetMediaHighlightByDedupKey(ctx, database.GetMediaHighlightByDedupKeyParams{
UserID: userID,
MediaItemID: mediaItemID,
DedupKey: pgtype.Text{String: dedupKey, Valid: true},
})
if err != nil {
return ""
}
return existing.Color.String
}
// deriveAnnotationPercentage computes a percentage for device-pushed // deriveAnnotationPercentage computes a percentage for device-pushed
// annotations when the client didn't send one (thin clients skip their own // annotations when the client didn't send one (thin clients skip their own
// per-annotation page lookups; arithmetic is only free on paging documents). // per-annotation page lookups; arithmetic is only free on paging documents).
@@ -211,6 +229,7 @@ type KOReaderBookmark struct {
Type string `json:"type,omitempty"` Type string `json:"type,omitempty"`
Percentage *float64 `json:"percentage,omitempty"` Percentage *float64 `json:"percentage,omitempty"`
BookSHA256 string `json:"book_sha256,omitempty"` BookSHA256 string `json:"book_sha256,omitempty"`
DedupKey string `json:"dedup_key,omitempty"`
} }
type KOReaderHighlight struct { type KOReaderHighlight struct {
@@ -225,6 +244,7 @@ type KOReaderHighlight struct {
Color string `json:"color,omitempty"` Color string `json:"color,omitempty"`
Percentage *float64 `json:"percentage,omitempty"` Percentage *float64 `json:"percentage,omitempty"`
BookSHA256 string `json:"book_sha256,omitempty"` BookSHA256 string `json:"book_sha256,omitempty"`
DedupKey string `json:"dedup_key,omitempty"`
} }
type KOReaderNote struct { type KOReaderNote struct {
@@ -238,6 +258,7 @@ type KOReaderNote struct {
Type string `json:"type,omitempty"` Type string `json:"type,omitempty"`
Percentage *float64 `json:"percentage,omitempty"` Percentage *float64 `json:"percentage,omitempty"`
BookSHA256 string `json:"book_sha256,omitempty"` BookSHA256 string `json:"book_sha256,omitempty"`
DedupKey string `json:"dedup_key,omitempty"`
} }
type KOReaderSyncResponse struct { type KOReaderSyncResponse struct {
@@ -604,19 +625,40 @@ func (h *KOReaderHandler) processBookAnnotations(ctx context.Context, deviceID,
"page": hl.Page, "page": hl.Page,
}) })
// Color semantics: devices render their own default and cannot
// round-trip web colors. An echo carries NO color — preserve the
// stored (web) color so round-trips never change it. A non-empty
// color means the user edited the highlight on the device: map the
// device color name and let it win.
color := ""
if hl.Color != "" {
color = mapColorFromKOReader(hl.Color)
}
dedupKey := hl.DedupKey
if dedupKey == "" {
dedupKey = wsync.ComputeDedupKey(hl.Text, epubcfiStart, startPos)
}
if color == "" {
color = h.existingHighlightColor(ctx, mediaItemID, userID, dedupKey)
}
if color == "" {
color = "#ffd54f"
}
h.annotationSvc.SaveHighlight(ctx, wsync.SaveHighlightRequest{ h.annotationSvc.SaveHighlight(ctx, wsync.SaveHighlightRequest{
MediaItemID: mediaItemID, MediaItemID: mediaItemID,
UserID: userID, UserID: userID,
SelectionText: hl.Text, SelectionText: hl.Text,
StartPosition: startPos, StartPosition: startPos,
EndPosition: endPos, EndPosition: endPos,
Color: mapColorFromKOReader(hl.Color), Color: color,
NoteText: hl.Notes, NoteText: hl.Notes,
PercentageStart: pctStart, PercentageStart: pctStart,
EpubcfiStart: epubcfiStart, EpubcfiStart: epubcfiStart,
EpubcfiEnd: epubcfiEnd, EpubcfiEnd: epubcfiEnd,
Source: "koreader", Source: "koreader",
DeviceSyncData: deviceData, DeviceSyncData: deviceData,
DedupKey: dedupKey,
}) })
} }
@@ -640,18 +682,25 @@ func (h *KOReaderHandler) processBookAnnotations(ctx context.Context, deviceID,
"page": note.Page, "page": note.Page,
}) })
dedupKey := note.DedupKey
if dedupKey == "" {
dedupKey = wsync.ComputeDedupKey(note.Text, epubcfiStart, startPos)
}
h.annotationSvc.SaveHighlight(ctx, wsync.SaveHighlightRequest{ h.annotationSvc.SaveHighlight(ctx, wsync.SaveHighlightRequest{
MediaItemID: mediaItemID, MediaItemID: mediaItemID,
UserID: userID, UserID: userID,
SelectionText: note.Text, SelectionText: note.Text,
StartPosition: startPos, StartPosition: startPos,
EndPosition: endPos, EndPosition: endPos,
Color: h.existingHighlightColor(ctx, mediaItemID, userID, dedupKey),
NoteText: note.Notes, NoteText: note.Notes,
PercentageStart: pctStart, PercentageStart: pctStart,
EpubcfiStart: epubcfiStart, EpubcfiStart: epubcfiStart,
EpubcfiEnd: epubcfiEnd, EpubcfiEnd: epubcfiEnd,
Source: "koreader", Source: "koreader",
DeviceSyncData: deviceData, DeviceSyncData: deviceData,
DedupKey: dedupKey,
}) })
} }
@@ -669,6 +718,11 @@ func (h *KOReaderHandler) processBookAnnotations(ctx context.Context, deviceID,
"page": bookmark.Page, "page": bookmark.Page,
}) })
dedupKey := bookmark.DedupKey
if dedupKey == "" {
dedupKey = wsync.ComputeDedupKey(bookmark.Text, "", position)
}
h.annotationSvc.SaveBookmark(ctx, wsync.SaveBookmarkRequest{ h.annotationSvc.SaveBookmark(ctx, wsync.SaveBookmarkRequest{
MediaItemID: mediaItemID, MediaItemID: mediaItemID,
UserID: userID, UserID: userID,
@@ -677,6 +731,7 @@ func (h *KOReaderHandler) processBookAnnotations(ctx context.Context, deviceID,
ChapterNumber: int32(bookmark.Chapter), ChapterNumber: int32(bookmark.Chapter),
Source: "koreader", Source: "koreader",
DeviceSyncData: deviceData, DeviceSyncData: deviceData,
DedupKey: dedupKey,
}) })
} }
} }
@@ -938,8 +993,12 @@ func (h *KOReaderHandler) GetMetadata(c *echo.Context) error {
Text: ann.SelectionText, Text: ann.SelectionText,
Pos0: pos0, Pos0: pos0,
Pos1: pos1, Pos1: pos1,
Color: mapColorToKOReader(ann.Color.String), // No color served: devices render their own default and
// cannot round-trip web colors — the web color only changes
// when the highlight is edited on the device (push carries
// the device color, ingested with the name→hex map).
Datetime: ann.CreatedAt.Time.Format(time.RFC3339), Datetime: ann.CreatedAt.Time.Format(time.RFC3339),
DedupKey: ann.DedupKey.String,
} }
if ann.NoteText.Valid && ann.NoteText.String != "" { if ann.NoteText.Valid && ann.NoteText.String != "" {
highlight.Notes = ann.NoteText.String highlight.Notes = ann.NoteText.String
@@ -955,6 +1014,7 @@ func (h *KOReaderHandler) GetMetadata(c *echo.Context) error {
Text: ann.SelectionText, Text: ann.SelectionText,
Pos0: pos0, Pos0: pos0,
Datetime: ann.CreatedAt.Time.Format(time.RFC3339), Datetime: ann.CreatedAt.Time.Format(time.RFC3339),
DedupKey: ann.DedupKey.String,
}) })
} }
} }
@@ -974,6 +1034,7 @@ func (h *KOReaderHandler) GetMetadata(c *echo.Context) error {
Pos0: pos0, Pos0: pos0,
Pos1: pos0, Pos1: pos0,
Datetime: bm.CreatedAt.Time.Format(time.RFC3339), Datetime: bm.CreatedAt.Time.Format(time.RFC3339),
DedupKey: bm.DedupKey.String,
} }
if bm.Notes.Valid && bm.Notes.String != "" { if bm.Notes.Valid && bm.Notes.String != "" {
koreaderBookmark.Notes = bm.Notes.String koreaderBookmark.Notes = bm.Notes.String
@@ -1126,14 +1187,6 @@ var koreaderColorFromName = map[string]string{
"red": "#f48fb1", "red": "#f48fb1",
} }
var koreaderColorFromHex = map[string]string{
"#ffd54f": "yellow",
"#a5d6a7": "green",
"#90caf9": "blue",
"#ce93d8": "purple",
"#f48fb1": "purple",
}
// mapColorFromKOReader normalizes a device color name to a web hex // mapColorFromKOReader normalizes a device color name to a web hex
// swatch (default yellow) when ingesting device pushes. // swatch (default yellow) when ingesting device pushes.
func mapColorFromKOReader(name string) string { func mapColorFromKOReader(name string) string {
@@ -1143,15 +1196,6 @@ func mapColorFromKOReader(name string) string {
return "#ffd54f" return "#ffd54f"
} }
// mapColorToKOReader normalizes a web hex swatch to a KOReader color
// name (default yellow) when serving to devices.
func mapColorToKOReader(hex string) string {
if name, ok := koreaderColorFromHex[strings.ToLower(strings.TrimSpace(hex))]; ok {
return name
}
return "yellow"
}
// 1. A device-native CRE xpointer ("/body/...") in startPosition wins — // 1. A device-native CRE xpointer ("/body/...") in startPosition wins —
// round-trip identical for KOReader-pushed annotations (converting the // round-trip identical for KOReader-pushed annotations (converting the
// stored CFI instead could drift and duplicate on the device). // stored CFI instead could drift and duplicate on the device).
+21 -3
View File
@@ -75,6 +75,11 @@ type SaveHighlightRequest struct {
Source string Source string
ModifiedAt time.Time ModifiedAt time.Time
DeviceSyncData json.RawMessage DeviceSyncData json.RawMessage
// DedupKey overrides the computed key when the client echoes back an
// annotation it received from us (device echoes carry device-native
// locators, so the computed key would never match the original row and
// every pull→push cycle would mint a duplicate).
DedupKey string
} }
type SaveHighlightResult struct { type SaveHighlightResult struct {
@@ -84,7 +89,10 @@ type SaveHighlightResult struct {
} }
func (s *AnnotationService) SaveHighlight(ctx context.Context, req SaveHighlightRequest) (*SaveHighlightResult, error) { func (s *AnnotationService) SaveHighlight(ctx context.Context, req SaveHighlightRequest) (*SaveHighlightResult, error) {
dedupKey := ComputeDedupKey(req.SelectionText, req.EpubcfiStart, req.StartPosition) dedupKey := req.DedupKey
if dedupKey == "" {
dedupKey = ComputeDedupKey(req.SelectionText, req.EpubcfiStart, req.StartPosition)
}
existing, err := s.db.GetMediaHighlightByDedupKey(ctx, database.GetMediaHighlightByDedupKeyParams{ existing, err := s.db.GetMediaHighlightByDedupKey(ctx, database.GetMediaHighlightByDedupKeyParams{
UserID: req.UserID, UserID: req.UserID,
@@ -335,6 +343,7 @@ type SaveNoteRequest struct {
Source string Source string
ModifiedAt time.Time ModifiedAt time.Time
DeviceSyncData []byte DeviceSyncData []byte
DedupKey string // overrides the computed key for device echoes
} }
type SaveNoteResult struct { type SaveNoteResult struct {
@@ -348,7 +357,10 @@ func (s *AnnotationService) SaveNote(ctx context.Context, req SaveNoteRequest) (
return nil, errors.New("invalid user_id or media_item_id") return nil, errors.New("invalid user_id or media_item_id")
} }
dedupKey := ComputeDedupKey(req.Content, req.EpubcfiLocation, req.Position) dedupKey := req.DedupKey
if dedupKey == "" {
dedupKey = ComputeDedupKey(req.Content, req.EpubcfiLocation, req.Position)
}
existing, err := s.db.GetMediaNoteByDedupKey(ctx, database.GetMediaNoteByDedupKeyParams{ existing, err := s.db.GetMediaNoteByDedupKey(ctx, database.GetMediaNoteByDedupKeyParams{
UserID: req.UserID, UserID: req.UserID,
@@ -485,6 +497,9 @@ type SaveBookmarkRequest struct {
Source string Source string
ModifiedAt time.Time ModifiedAt time.Time
DeviceSyncData json.RawMessage DeviceSyncData json.RawMessage
// DedupKey overrides the computed key for device echoes (see
// SaveHighlightRequest).
DedupKey string
} }
type SaveBookmarkResult struct { type SaveBookmarkResult struct {
@@ -494,7 +509,10 @@ type SaveBookmarkResult struct {
} }
func (s *AnnotationService) SaveBookmark(ctx context.Context, req SaveBookmarkRequest) (*SaveBookmarkResult, error) { func (s *AnnotationService) SaveBookmark(ctx context.Context, req SaveBookmarkRequest) (*SaveBookmarkResult, error) {
dedupKey := ComputeDedupKey(req.Title, req.EpubcfiLocation, req.Position) dedupKey := req.DedupKey
if dedupKey == "" {
dedupKey = ComputeDedupKey(req.Title, req.EpubcfiLocation, req.Position)
}
existing, err := s.db.GetMediaBookmarkByDedupKey(ctx, database.GetMediaBookmarkByDedupKeyParams{ existing, err := s.db.GetMediaBookmarkByDedupKey(ctx, database.GetMediaBookmarkByDedupKeyParams{
UserID: req.UserID, UserID: req.UserID,