fix(sync): echo dedup + color semantics + classification for KOReader round-trips
Echo duplication: devices push their full annotation list on every sync, and an echo of a web-created annotation computed a different dedup key than the original (device locators differ from web locators) — every pull→push cycle minted a duplicate row, and cleaning those up on the web tombstoned them back to the device, deleting the just-applied copies. That was the "web highlights never appear on KOReader" experience. GetMetadata now serves each annotation's dedup_key; the device stores it on the applied entry and echoes it in pushes; SaveHighlight/SaveBookmark/SaveNote accept a DedupKey override so echoes converge onto the original row (verified: pull → echo push creates no rows, LWW skips identical content). Color semantics (per user preference): devices render their own default and cannot round-trip web colors, so GetMetadata no longer serves colors at all — every highlight syncs regardless of its web color and the device draws its default. An echo carries no color; ingest then PRESERVES the stored web color (existingHighlightColor lookup by dedup key) so round-trips never change it. A non-empty device color means the user edited the highlight there: it maps name→hex (green→#a5d6a7, default yellow) and wins. Verified: echo kept #ffd54f; a simulated device edit with "green" updated the web row to #a5d6a7. Classification: KOReader auto-fills text="in Chapter X" on page bookmarks (ReaderAnnotation:updateItemByXPointer), so the plugin's text-presence classification turned every echoed bookmark into a junk highlight on the web. v2 classification now keys off the drawer field (present = highlight/note, absent = bookmark with its label in note).
This commit is contained in:
@@ -106,6 +106,24 @@ func extendCFIByLength(cfi, text string) string {
|
|||||||
return cfi[:i+1] + strconv.Itoa(off+utf16len) + ")"
|
return cfi[:i+1] + strconv.Itoa(off+utf16len) + ")"
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// existingHighlightColor returns the stored color of the highlight matching
|
||||||
|
// the dedup key ("" when none) so device echoes that carry no color never
|
||||||
|
// clobber the web color.
|
||||||
|
func (h *KOReaderHandler) existingHighlightColor(ctx context.Context, mediaItemID, userID pgtype.UUID, dedupKey string) string {
|
||||||
|
if dedupKey == "" {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
existing, err := h.db.GetMediaHighlightByDedupKey(ctx, database.GetMediaHighlightByDedupKeyParams{
|
||||||
|
UserID: userID,
|
||||||
|
MediaItemID: mediaItemID,
|
||||||
|
DedupKey: pgtype.Text{String: dedupKey, Valid: true},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
return existing.Color.String
|
||||||
|
}
|
||||||
|
|
||||||
// deriveAnnotationPercentage computes a percentage for device-pushed
|
// deriveAnnotationPercentage computes a percentage for device-pushed
|
||||||
// annotations when the client didn't send one (thin clients skip their own
|
// annotations when the client didn't send one (thin clients skip their own
|
||||||
// per-annotation page lookups; arithmetic is only free on paging documents).
|
// per-annotation page lookups; arithmetic is only free on paging documents).
|
||||||
@@ -211,6 +229,7 @@ type KOReaderBookmark struct {
|
|||||||
Type string `json:"type,omitempty"`
|
Type string `json:"type,omitempty"`
|
||||||
Percentage *float64 `json:"percentage,omitempty"`
|
Percentage *float64 `json:"percentage,omitempty"`
|
||||||
BookSHA256 string `json:"book_sha256,omitempty"`
|
BookSHA256 string `json:"book_sha256,omitempty"`
|
||||||
|
DedupKey string `json:"dedup_key,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type KOReaderHighlight struct {
|
type KOReaderHighlight struct {
|
||||||
@@ -225,6 +244,7 @@ type KOReaderHighlight struct {
|
|||||||
Color string `json:"color,omitempty"`
|
Color string `json:"color,omitempty"`
|
||||||
Percentage *float64 `json:"percentage,omitempty"`
|
Percentage *float64 `json:"percentage,omitempty"`
|
||||||
BookSHA256 string `json:"book_sha256,omitempty"`
|
BookSHA256 string `json:"book_sha256,omitempty"`
|
||||||
|
DedupKey string `json:"dedup_key,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type KOReaderNote struct {
|
type KOReaderNote struct {
|
||||||
@@ -238,6 +258,7 @@ type KOReaderNote struct {
|
|||||||
Type string `json:"type,omitempty"`
|
Type string `json:"type,omitempty"`
|
||||||
Percentage *float64 `json:"percentage,omitempty"`
|
Percentage *float64 `json:"percentage,omitempty"`
|
||||||
BookSHA256 string `json:"book_sha256,omitempty"`
|
BookSHA256 string `json:"book_sha256,omitempty"`
|
||||||
|
DedupKey string `json:"dedup_key,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type KOReaderSyncResponse struct {
|
type KOReaderSyncResponse struct {
|
||||||
@@ -604,19 +625,40 @@ func (h *KOReaderHandler) processBookAnnotations(ctx context.Context, deviceID,
|
|||||||
"page": hl.Page,
|
"page": hl.Page,
|
||||||
})
|
})
|
||||||
|
|
||||||
|
// Color semantics: devices render their own default and cannot
|
||||||
|
// round-trip web colors. An echo carries NO color — preserve the
|
||||||
|
// stored (web) color so round-trips never change it. A non-empty
|
||||||
|
// color means the user edited the highlight on the device: map the
|
||||||
|
// device color name and let it win.
|
||||||
|
color := ""
|
||||||
|
if hl.Color != "" {
|
||||||
|
color = mapColorFromKOReader(hl.Color)
|
||||||
|
}
|
||||||
|
dedupKey := hl.DedupKey
|
||||||
|
if dedupKey == "" {
|
||||||
|
dedupKey = wsync.ComputeDedupKey(hl.Text, epubcfiStart, startPos)
|
||||||
|
}
|
||||||
|
if color == "" {
|
||||||
|
color = h.existingHighlightColor(ctx, mediaItemID, userID, dedupKey)
|
||||||
|
}
|
||||||
|
if color == "" {
|
||||||
|
color = "#ffd54f"
|
||||||
|
}
|
||||||
|
|
||||||
h.annotationSvc.SaveHighlight(ctx, wsync.SaveHighlightRequest{
|
h.annotationSvc.SaveHighlight(ctx, wsync.SaveHighlightRequest{
|
||||||
MediaItemID: mediaItemID,
|
MediaItemID: mediaItemID,
|
||||||
UserID: userID,
|
UserID: userID,
|
||||||
SelectionText: hl.Text,
|
SelectionText: hl.Text,
|
||||||
StartPosition: startPos,
|
StartPosition: startPos,
|
||||||
EndPosition: endPos,
|
EndPosition: endPos,
|
||||||
Color: mapColorFromKOReader(hl.Color),
|
Color: color,
|
||||||
NoteText: hl.Notes,
|
NoteText: hl.Notes,
|
||||||
PercentageStart: pctStart,
|
PercentageStart: pctStart,
|
||||||
EpubcfiStart: epubcfiStart,
|
EpubcfiStart: epubcfiStart,
|
||||||
EpubcfiEnd: epubcfiEnd,
|
EpubcfiEnd: epubcfiEnd,
|
||||||
Source: "koreader",
|
Source: "koreader",
|
||||||
DeviceSyncData: deviceData,
|
DeviceSyncData: deviceData,
|
||||||
|
DedupKey: dedupKey,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -640,18 +682,25 @@ func (h *KOReaderHandler) processBookAnnotations(ctx context.Context, deviceID,
|
|||||||
"page": note.Page,
|
"page": note.Page,
|
||||||
})
|
})
|
||||||
|
|
||||||
|
dedupKey := note.DedupKey
|
||||||
|
if dedupKey == "" {
|
||||||
|
dedupKey = wsync.ComputeDedupKey(note.Text, epubcfiStart, startPos)
|
||||||
|
}
|
||||||
|
|
||||||
h.annotationSvc.SaveHighlight(ctx, wsync.SaveHighlightRequest{
|
h.annotationSvc.SaveHighlight(ctx, wsync.SaveHighlightRequest{
|
||||||
MediaItemID: mediaItemID,
|
MediaItemID: mediaItemID,
|
||||||
UserID: userID,
|
UserID: userID,
|
||||||
SelectionText: note.Text,
|
SelectionText: note.Text,
|
||||||
StartPosition: startPos,
|
StartPosition: startPos,
|
||||||
EndPosition: endPos,
|
EndPosition: endPos,
|
||||||
|
Color: h.existingHighlightColor(ctx, mediaItemID, userID, dedupKey),
|
||||||
NoteText: note.Notes,
|
NoteText: note.Notes,
|
||||||
PercentageStart: pctStart,
|
PercentageStart: pctStart,
|
||||||
EpubcfiStart: epubcfiStart,
|
EpubcfiStart: epubcfiStart,
|
||||||
EpubcfiEnd: epubcfiEnd,
|
EpubcfiEnd: epubcfiEnd,
|
||||||
Source: "koreader",
|
Source: "koreader",
|
||||||
DeviceSyncData: deviceData,
|
DeviceSyncData: deviceData,
|
||||||
|
DedupKey: dedupKey,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -669,6 +718,11 @@ func (h *KOReaderHandler) processBookAnnotations(ctx context.Context, deviceID,
|
|||||||
"page": bookmark.Page,
|
"page": bookmark.Page,
|
||||||
})
|
})
|
||||||
|
|
||||||
|
dedupKey := bookmark.DedupKey
|
||||||
|
if dedupKey == "" {
|
||||||
|
dedupKey = wsync.ComputeDedupKey(bookmark.Text, "", position)
|
||||||
|
}
|
||||||
|
|
||||||
h.annotationSvc.SaveBookmark(ctx, wsync.SaveBookmarkRequest{
|
h.annotationSvc.SaveBookmark(ctx, wsync.SaveBookmarkRequest{
|
||||||
MediaItemID: mediaItemID,
|
MediaItemID: mediaItemID,
|
||||||
UserID: userID,
|
UserID: userID,
|
||||||
@@ -677,6 +731,7 @@ func (h *KOReaderHandler) processBookAnnotations(ctx context.Context, deviceID,
|
|||||||
ChapterNumber: int32(bookmark.Chapter),
|
ChapterNumber: int32(bookmark.Chapter),
|
||||||
Source: "koreader",
|
Source: "koreader",
|
||||||
DeviceSyncData: deviceData,
|
DeviceSyncData: deviceData,
|
||||||
|
DedupKey: dedupKey,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -938,8 +993,12 @@ func (h *KOReaderHandler) GetMetadata(c *echo.Context) error {
|
|||||||
Text: ann.SelectionText,
|
Text: ann.SelectionText,
|
||||||
Pos0: pos0,
|
Pos0: pos0,
|
||||||
Pos1: pos1,
|
Pos1: pos1,
|
||||||
Color: mapColorToKOReader(ann.Color.String),
|
// No color served: devices render their own default and
|
||||||
|
// cannot round-trip web colors — the web color only changes
|
||||||
|
// when the highlight is edited on the device (push carries
|
||||||
|
// the device color, ingested with the name→hex map).
|
||||||
Datetime: ann.CreatedAt.Time.Format(time.RFC3339),
|
Datetime: ann.CreatedAt.Time.Format(time.RFC3339),
|
||||||
|
DedupKey: ann.DedupKey.String,
|
||||||
}
|
}
|
||||||
if ann.NoteText.Valid && ann.NoteText.String != "" {
|
if ann.NoteText.Valid && ann.NoteText.String != "" {
|
||||||
highlight.Notes = ann.NoteText.String
|
highlight.Notes = ann.NoteText.String
|
||||||
@@ -955,6 +1014,7 @@ func (h *KOReaderHandler) GetMetadata(c *echo.Context) error {
|
|||||||
Text: ann.SelectionText,
|
Text: ann.SelectionText,
|
||||||
Pos0: pos0,
|
Pos0: pos0,
|
||||||
Datetime: ann.CreatedAt.Time.Format(time.RFC3339),
|
Datetime: ann.CreatedAt.Time.Format(time.RFC3339),
|
||||||
|
DedupKey: ann.DedupKey.String,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -974,6 +1034,7 @@ func (h *KOReaderHandler) GetMetadata(c *echo.Context) error {
|
|||||||
Pos0: pos0,
|
Pos0: pos0,
|
||||||
Pos1: pos0,
|
Pos1: pos0,
|
||||||
Datetime: bm.CreatedAt.Time.Format(time.RFC3339),
|
Datetime: bm.CreatedAt.Time.Format(time.RFC3339),
|
||||||
|
DedupKey: bm.DedupKey.String,
|
||||||
}
|
}
|
||||||
if bm.Notes.Valid && bm.Notes.String != "" {
|
if bm.Notes.Valid && bm.Notes.String != "" {
|
||||||
koreaderBookmark.Notes = bm.Notes.String
|
koreaderBookmark.Notes = bm.Notes.String
|
||||||
@@ -1126,14 +1187,6 @@ var koreaderColorFromName = map[string]string{
|
|||||||
"red": "#f48fb1",
|
"red": "#f48fb1",
|
||||||
}
|
}
|
||||||
|
|
||||||
var koreaderColorFromHex = map[string]string{
|
|
||||||
"#ffd54f": "yellow",
|
|
||||||
"#a5d6a7": "green",
|
|
||||||
"#90caf9": "blue",
|
|
||||||
"#ce93d8": "purple",
|
|
||||||
"#f48fb1": "purple",
|
|
||||||
}
|
|
||||||
|
|
||||||
// mapColorFromKOReader normalizes a device color name to a web hex
|
// mapColorFromKOReader normalizes a device color name to a web hex
|
||||||
// swatch (default yellow) when ingesting device pushes.
|
// swatch (default yellow) when ingesting device pushes.
|
||||||
func mapColorFromKOReader(name string) string {
|
func mapColorFromKOReader(name string) string {
|
||||||
@@ -1143,15 +1196,6 @@ func mapColorFromKOReader(name string) string {
|
|||||||
return "#ffd54f"
|
return "#ffd54f"
|
||||||
}
|
}
|
||||||
|
|
||||||
// mapColorToKOReader normalizes a web hex swatch to a KOReader color
|
|
||||||
// name (default yellow) when serving to devices.
|
|
||||||
func mapColorToKOReader(hex string) string {
|
|
||||||
if name, ok := koreaderColorFromHex[strings.ToLower(strings.TrimSpace(hex))]; ok {
|
|
||||||
return name
|
|
||||||
}
|
|
||||||
return "yellow"
|
|
||||||
}
|
|
||||||
|
|
||||||
// 1. A device-native CRE xpointer ("/body/...") in startPosition wins —
|
// 1. A device-native CRE xpointer ("/body/...") in startPosition wins —
|
||||||
// round-trip identical for KOReader-pushed annotations (converting the
|
// round-trip identical for KOReader-pushed annotations (converting the
|
||||||
// stored CFI instead could drift and duplicate on the device).
|
// stored CFI instead could drift and duplicate on the device).
|
||||||
|
|||||||
@@ -75,6 +75,11 @@ type SaveHighlightRequest struct {
|
|||||||
Source string
|
Source string
|
||||||
ModifiedAt time.Time
|
ModifiedAt time.Time
|
||||||
DeviceSyncData json.RawMessage
|
DeviceSyncData json.RawMessage
|
||||||
|
// DedupKey overrides the computed key when the client echoes back an
|
||||||
|
// annotation it received from us (device echoes carry device-native
|
||||||
|
// locators, so the computed key would never match the original row and
|
||||||
|
// every pull→push cycle would mint a duplicate).
|
||||||
|
DedupKey string
|
||||||
}
|
}
|
||||||
|
|
||||||
type SaveHighlightResult struct {
|
type SaveHighlightResult struct {
|
||||||
@@ -84,7 +89,10 @@ type SaveHighlightResult struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (s *AnnotationService) SaveHighlight(ctx context.Context, req SaveHighlightRequest) (*SaveHighlightResult, error) {
|
func (s *AnnotationService) SaveHighlight(ctx context.Context, req SaveHighlightRequest) (*SaveHighlightResult, error) {
|
||||||
dedupKey := ComputeDedupKey(req.SelectionText, req.EpubcfiStart, req.StartPosition)
|
dedupKey := req.DedupKey
|
||||||
|
if dedupKey == "" {
|
||||||
|
dedupKey = ComputeDedupKey(req.SelectionText, req.EpubcfiStart, req.StartPosition)
|
||||||
|
}
|
||||||
|
|
||||||
existing, err := s.db.GetMediaHighlightByDedupKey(ctx, database.GetMediaHighlightByDedupKeyParams{
|
existing, err := s.db.GetMediaHighlightByDedupKey(ctx, database.GetMediaHighlightByDedupKeyParams{
|
||||||
UserID: req.UserID,
|
UserID: req.UserID,
|
||||||
@@ -335,6 +343,7 @@ type SaveNoteRequest struct {
|
|||||||
Source string
|
Source string
|
||||||
ModifiedAt time.Time
|
ModifiedAt time.Time
|
||||||
DeviceSyncData []byte
|
DeviceSyncData []byte
|
||||||
|
DedupKey string // overrides the computed key for device echoes
|
||||||
}
|
}
|
||||||
|
|
||||||
type SaveNoteResult struct {
|
type SaveNoteResult struct {
|
||||||
@@ -348,7 +357,10 @@ func (s *AnnotationService) SaveNote(ctx context.Context, req SaveNoteRequest) (
|
|||||||
return nil, errors.New("invalid user_id or media_item_id")
|
return nil, errors.New("invalid user_id or media_item_id")
|
||||||
}
|
}
|
||||||
|
|
||||||
dedupKey := ComputeDedupKey(req.Content, req.EpubcfiLocation, req.Position)
|
dedupKey := req.DedupKey
|
||||||
|
if dedupKey == "" {
|
||||||
|
dedupKey = ComputeDedupKey(req.Content, req.EpubcfiLocation, req.Position)
|
||||||
|
}
|
||||||
|
|
||||||
existing, err := s.db.GetMediaNoteByDedupKey(ctx, database.GetMediaNoteByDedupKeyParams{
|
existing, err := s.db.GetMediaNoteByDedupKey(ctx, database.GetMediaNoteByDedupKeyParams{
|
||||||
UserID: req.UserID,
|
UserID: req.UserID,
|
||||||
@@ -485,6 +497,9 @@ type SaveBookmarkRequest struct {
|
|||||||
Source string
|
Source string
|
||||||
ModifiedAt time.Time
|
ModifiedAt time.Time
|
||||||
DeviceSyncData json.RawMessage
|
DeviceSyncData json.RawMessage
|
||||||
|
// DedupKey overrides the computed key for device echoes (see
|
||||||
|
// SaveHighlightRequest).
|
||||||
|
DedupKey string
|
||||||
}
|
}
|
||||||
|
|
||||||
type SaveBookmarkResult struct {
|
type SaveBookmarkResult struct {
|
||||||
@@ -494,7 +509,10 @@ type SaveBookmarkResult struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (s *AnnotationService) SaveBookmark(ctx context.Context, req SaveBookmarkRequest) (*SaveBookmarkResult, error) {
|
func (s *AnnotationService) SaveBookmark(ctx context.Context, req SaveBookmarkRequest) (*SaveBookmarkResult, error) {
|
||||||
dedupKey := ComputeDedupKey(req.Title, req.EpubcfiLocation, req.Position)
|
dedupKey := req.DedupKey
|
||||||
|
if dedupKey == "" {
|
||||||
|
dedupKey = ComputeDedupKey(req.Title, req.EpubcfiLocation, req.Position)
|
||||||
|
}
|
||||||
|
|
||||||
existing, err := s.db.GetMediaBookmarkByDedupKey(ctx, database.GetMediaBookmarkByDedupKeyParams{
|
existing, err := s.db.GetMediaBookmarkByDedupKey(ctx, database.GetMediaBookmarkByDedupKeyParams{
|
||||||
UserID: req.UserID,
|
UserID: req.UserID,
|
||||||
|
|||||||
Reference in New Issue
Block a user