Bookhoard/bookhoard
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 12 Wiki Activity

feat(sync): wire annotation sync into all device and web handlers

Browse Source 
Complete the annotation sync pipeline across all ingest and serve paths.
Previously, annotations sent inline with KOReader progress pushes were
silently discarded, and no annotations were ever served back to devices.

INGEST (device → server):

KOReader (koreader.go):
  - Add processBookAnnotations helper that processes inline highlights,
    notes, and bookmarks from every progress push (immediate + checkpoint)
  - Highlights get CRE→CFI position conversion before SaveHighlight
  - KOReader 'notes' (text + notes) stored as highlights with NoteText
    to ensure correct round-trip classification
  - Bookmarks routed through SaveBookmark with device sync data
  - Called from both updateProgressForBook and handleCheckpointSync

Kobo (kobo.go):
  - Markup handler: annotations and bookmarks route through
    AnnotationService (SaveHighlight/SaveBookmark)
  - Bookmark handler: same routing with device sync data
  - SyncFromServer handler: same routing
  - All handlers fall back to direct DB calls when annotationSvc == nil

Web reader (media.go):
  - CreateMediaHighlight → SaveHighlight (Source="web", ModifiedAt=now)
  - CreateMediaNote → SaveNote (Source="web")
  - DeleteMediaHighlight → TombstoneHighlightByID
  - DeleteMediaNote → TombstoneNoteByID (was hard delete, now tombstone)
  - All fall back to old behavior when annotationSvc == nil

SERVE (server → device):

KOReader GetMetadata (koreader.go):
  - Query and serve bookmarks from media_bookmarks table (was missing)
  - Serve deleted_highlights and deleted_bookmarks arrays containing
    device_sync_data + dedup_key for client-side deletion
  - Highlights/notes already served with reverse CFI conversion

Kobo Markup handler (kobo.go):
  - Track processed books during sync
  - Query tombstones per book, extract bookmark_id from device_sync_data
  - Return DeletedAnnotations array in KoboSyncStatus response

Conflict resolution (conflicts.go):
  - Enable annotation conflict types in ResolveConflict handler
  - Add applyAnnotationResolution dispatching to:
    applyHighlightResolution / applyBookmarkResolution / applyNoteResolution
  - Each looks up by dedup_key and applies winner's fields
  - Allow manual override of auto_resolved conflicts
    (changed check from != "unresolved" to == "user_resolved")

Infrastructure:
  - AnnotationService field + SetAnnotationService in router Config
  - Inject AnnotationService into KOReader, Kobo, Media handlers
  - Start tombstone purger goroutine in main.go (24h interval)
  - Test helpers: construct AnnotationService in test setup
This commit is contained in:
john-okeefe
2026-07-29 14:49:19 -04:00
parent 635a9439cb
commit 75b33fdae6
8 changed files with 772 additions and 110 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/handlers/koreader.go
+316 -36
View File
@@ -4,6 +4,7 @@ import (
"bookhoard/internal/database"
wsync "bookhoard/internal/sync"
"context"
"encoding/json"
"fmt"
"log"
"net/http"
@@ -19,6 +20,7 @@ type KOReaderHandler struct {
connManager *wsync.ConnectionManager
queue *wsync.SyncQueueProcessor
progressSvc *wsync.ProgressService
annotationSvc *wsync.AnnotationService
libraryService LibraryPathResolver
}
@@ -34,6 +36,27 @@ func (h *KOReaderHandler) SetProgressService(svc *wsync.ProgressService) {
h.progressSvc = svc
}
func (h *KOReaderHandler) SetAnnotationService(svc *wsync.AnnotationService) {
h.annotationSvc = svc
}
func (h *KOReaderHandler) convertHighlightPositions(ctx context.Context, mediaItemID pgtype.UUID, pos0, pos1 string) (string, string) {
if pos0 == "" || h.libraryService == nil {
return "", ""
}
mediaItem, err := h.db.GetMediaItem(ctx, mediaItemID)
if err != nil {
return "", ""
}
epubPath, err := h.libraryService.ResolveMediaPath(ctx, mediaItem.LibraryID, mediaItem.FilePath)
if err != nil || epubPath == "" {
return "", ""
}
startLoc := wsync.ConvertToCanonical(wsync.LocatorSourceKOReader, pos0, 0, "", mediaItem.FormatGroup, epubPath, "")
endLoc := wsync.ConvertToCanonical(wsync.LocatorSourceKOReader, pos1, 0, "", mediaItem.FormatGroup, epubPath, "")
return startLoc.CFI, endLoc.CFI
}
func (h *KOReaderHandler) SetLibraryService(svc LibraryPathResolver) {
h.libraryService = svc
}
@@ -154,9 +177,11 @@ type KOReaderProgressData struct {
}
type KOReaderAnnotations struct {
Highlights []KOReaderHighlight `json:"highlights,omitempty"`
Notes []KOReaderNote `json:"notes,omitempty"`
Bookmarks []KOReaderBookmark `json:"bookmarks,omitempty"`
Highlights []KOReaderHighlight `json:"highlights,omitempty"`
Notes []KOReaderNote `json:"notes,omitempty"`
Bookmarks []KOReaderBookmark `json:"bookmarks,omitempty"`
DeletedHighlights []map[string]interface{} `json:"deleted_highlights,omitempty"`
DeletedBookmarks []map[string]interface{} `json:"deleted_bookmarks,omitempty"`
}
type KOReaderLibraryResponse struct {
@@ -396,6 +421,7 @@ func (h *KOReaderHandler) handleCheckpointSync(c *echo.Context, device database.
if synced {
booksEnqueued++
}
h.processBookAnnotations(c.Request().Context(), device.ID, userID, mediaItemID, book)
bookResults = append(bookResults, KOReaderBookSyncResult{
SHA256: book.SHA256,
BookUUID: uuid.UUID(mediaItemID.Bytes).String(),
@@ -441,6 +467,102 @@ func (h *KOReaderHandler) enqueueProgressForBook(c *echo.Context, deviceID pgtyp
return h.queue.EnqueueProgress(update)
}
func (h *KOReaderHandler) processBookAnnotations(ctx context.Context, deviceID, userID, mediaItemID pgtype.UUID, book KOReaderBookProgress) {
if h.annotationSvc == nil {
return
}
for _, hl := range book.Highlights {
startPos := hl.Pos0
endPos := hl.Pos1
epubcfiStart, epubcfiEnd := h.convertHighlightPositions(ctx, mediaItemID, startPos, endPos)
pctStart := 0.0
if hl.Percentage != nil {
pctStart = *hl.Percentage
}
deviceData, _ := json.Marshal(map[string]interface{}{
"datetime": hl.Datetime,
"pos0": hl.Pos0,
"pos1": hl.Pos1,
"page": hl.Page,
})
h.annotationSvc.SaveHighlight(ctx, wsync.SaveHighlightRequest{
MediaItemID: mediaItemID,
UserID: userID,
SelectionText: hl.Text,
StartPosition: startPos,
EndPosition: endPos,
Color: hl.Color,
NoteText: hl.Notes,
PercentageStart: pctStart,
EpubcfiStart: epubcfiStart,
EpubcfiEnd: epubcfiEnd,
Source: "koreader",
DeviceSyncData: deviceData,
})
}
for _, note := range book.Notes {
startPos := note.Pos0
endPos := note.Pos1
epubcfiStart, epubcfiEnd := h.convertHighlightPositions(ctx, mediaItemID, startPos, endPos)
pctStart := 0.0
if note.Percentage != nil {
pctStart = *note.Percentage
}
deviceData, _ := json.Marshal(map[string]interface{}{
"datetime": note.Datetime,
"pos0": note.Pos0,
"pos1": note.Pos1,
"page": note.Page,
})
h.annotationSvc.SaveHighlight(ctx, wsync.SaveHighlightRequest{
MediaItemID: mediaItemID,
UserID: userID,
SelectionText: note.Text,
StartPosition: startPos,
EndPosition: endPos,
NoteText: note.Notes,
PercentageStart: pctStart,
EpubcfiStart: epubcfiStart,
EpubcfiEnd: epubcfiEnd,
Source: "koreader",
DeviceSyncData: deviceData,
})
}
for _, bookmark := range book.Bookmarks {
position := ""
if bookmark.Pos0 != "" {
position = bookmark.Pos0
} else if bookmark.Page > 0 {
position = fmt.Sprintf("page:%d", bookmark.Page)
}
deviceData, _ := json.Marshal(map[string]interface{}{
"datetime": bookmark.Datetime,
"pos0": bookmark.Pos0,
"page": bookmark.Page,
})
h.annotationSvc.SaveBookmark(ctx, wsync.SaveBookmarkRequest{
MediaItemID: mediaItemID,
UserID: userID,
Title: bookmark.Text,
Position: position,
ChapterNumber: int32(bookmark.Chapter),
Source: "koreader",
DeviceSyncData: deviceData,
})
}
}
func (h *KOReaderHandler) updateProgressForBook(c *echo.Context, deviceID pgtype.UUID, userID pgtype.UUID, mediaItemID pgtype.UUID, book KOReaderBookProgress) error {
ctx := c.Request().Context()
@@ -522,7 +644,11 @@ func (h *KOReaderHandler) updateProgressForBook(c *echo.Context, deviceID pgtype
}
_, err := h.progressSvc.SaveProgress(ctx, saveReq)
return err
if err != nil {
return err
}
h.processBookAnnotations(ctx, deviceID, userID, mediaItemID, book)
return nil
}
_, err := h.db.UpdateUniversalProgress(ctx, database.UpdateUniversalProgressParams{
@@ -558,6 +684,8 @@ func (h *KOReaderHandler) updateProgressForBook(c *echo.Context, deviceID pgtype
},
)
h.processBookAnnotations(ctx, deviceID, userID, mediaItemID, book)
return nil
}
@@ -657,7 +785,7 @@ func (h *KOReaderHandler) GetMetadata(c *echo.Context) error {
progressData.TotalPages = &progress
}
annotations, err := h.db.GetAnnotationsForBook(c.Request().Context(), database.GetAnnotationsForBookParams{
annotations, err := h.db.GetActiveAnnotationsForBook(c.Request().Context(), database.GetActiveAnnotationsForBookParams{
MediaItemID: pgBookUUID,
UserID: pgUserID,
})
@@ -670,13 +798,29 @@ func (h *KOReaderHandler) GetMetadata(c *echo.Context) error {
for _, ann := range annotations {
if ann.AnnotationType == "highlight" {
annotationsResponse.Highlights = append(annotationsResponse.Highlights, KOReaderHighlight{
pos0 := ann.StartPosition.String
pos1 := ann.EndPosition.String
if ann.EpubcfiStart.Valid && ann.EpubcfiStart.String != "" {
if converted := h.reverseConvertCFI(c, mediaItem, ann.EpubcfiStart.String); converted != "" {
pos0 = converted
}
}
if ann.EpubcfiEnd.Valid && ann.EpubcfiEnd.String != "" {
if converted := h.reverseConvertCFI(c, mediaItem, ann.EpubcfiEnd.String); converted != "" {
pos1 = converted
}
}
highlight := KOReaderHighlight{
Text: ann.SelectionText,
Pos0: ann.StartPosition.String,
Pos1: ann.EndPosition.String,
Pos0: pos0,
Pos1: pos1,
Color: ann.Color.String,
Datetime: ann.CreatedAt.Time.Format(time.RFC3339),
})
}
if ann.NoteText.Valid && ann.NoteText.String != "" {
highlight.Notes = ann.NoteText.String
}
annotationsResponse.Highlights = append(annotationsResponse.Highlights, highlight)
} else if ann.AnnotationType == "note" {
annotationsResponse.Notes = append(annotationsResponse.Notes, KOReaderNote{
Text: ann.SelectionText,
@@ -686,6 +830,52 @@ func (h *KOReaderHandler) GetMetadata(c *echo.Context) error {
}
}
bookmarks, _ := h.db.GetMediaBookmarks(c.Request().Context(), database.GetMediaBookmarksParams{
MediaItemID: pgBookUUID,
UserID: pgUserID,
})
for _, bm := range bookmarks {
pos0 := bm.Position.String
if pos0 == "" && bm.CfiPosition.Valid {
pos0 = bm.CfiPosition.String
}
koreaderBookmark := KOReaderBookmark{
Text: bm.Title,
Pos0: pos0,
Pos1: pos0,
Datetime: bm.CreatedAt.Time.Format(time.RFC3339),
}
if bm.Notes.Valid && bm.Notes.String != "" {
koreaderBookmark.Notes = bm.Notes.String
}
if bm.ChapterNumber.Valid {
koreaderBookmark.Chapter = int(bm.ChapterNumber.Int32)
}
annotationsResponse.Bookmarks = append(annotationsResponse.Bookmarks, koreaderBookmark)
}
cutoff := pgtype.Timestamptz{Time: time.Now().Add(-wsync.TombstoneTTL), Valid: true}
tombstones, _ := h.db.GetTombstonedAnnotationsForBook(c.Request().Context(), database.GetTombstonedAnnotationsForBookParams{
MediaItemID: pgBookUUID,
UserID: pgUserID,
DeletedAt: cutoff,
})
for _, ts := range tombstones {
var dd map[string]interface{}
if len(ts.DeviceSyncData) > 0 {
json.Unmarshal(ts.DeviceSyncData, &dd)
}
if dd == nil {
dd = map[string]interface{}{}
}
dd["dedup_key"] = ts.DedupKey.String
if ts.AnnotationType == "highlight" {
annotationsResponse.DeletedHighlights = append(annotationsResponse.DeletedHighlights, dd)
} else if ts.AnnotationType == "bookmark" {
annotationsResponse.DeletedBookmarks = append(annotationsResponse.DeletedBookmarks, dd)
}
}
lastSync := "never"
if progress.LastSyncTimestamp.Valid {
lastSync = progress.LastSyncTimestamp.Time.Format(time.RFC3339)
@@ -737,6 +927,21 @@ func (h *KOReaderHandler) convertCFIToXPointer(c *echo.Context, mediaItem databa
}
}
func (h *KOReaderHandler) reverseConvertCFI(c *echo.Context, mediaItem database.MediaItems, epubcfi string) string {
if h.libraryService == nil || epubcfi == "" {
return ""
}
epubPath, err := h.libraryService.ResolveMediaPath(c.Request().Context(), mediaItem.LibraryID, mediaItem.FilePath)
if err != nil || epubPath == "" {
return ""
}
loc := wsync.ConvertFromCanonical(wsync.LocatorSourceKOReader, epubcfi, 0, "", mediaItem.FormatGroup, epubPath, "")
if loc.Position != "" && loc.Position != epubcfi {
return loc.Position
}
return ""
}
func (h *KOReaderHandler) GetLibrary(c *echo.Context) error {
device := c.Get("device").(database.Devices)
userID := device.UserID.Bytes
@@ -776,7 +981,7 @@ func (h *KOReaderHandler) GetLibrary(c *echo.Context) error {
}
}
annotations, _ := h.db.GetAnnotationsForBook(c.Request().Context(), database.GetAnnotationsForBookParams{
annotations, _ := h.db.GetActiveAnnotationsForBook(c.Request().Context(), database.GetActiveAnnotationsForBookParams{
MediaItemID: pgItemUUID,
UserID: pgUserID,
})
@@ -876,15 +1081,36 @@ func (h *KOReaderHandler) SyncBookmarks(c *echo.Context) error {
position = fmt.Sprintf("page:%d", bookmark.Page)
}
_, err := h.db.CreateMediaNote(ctx, database.CreateMediaNoteParams{
MediaItemID: mediaItemID,
UserID: pgUserID,
Content: bookmark.Text,
Position: pgtype.Text{String: position, Valid: position != ""},
})
if h.annotationSvc != nil {
deviceData, _ := json.Marshal(map[string]interface{}{
"datetime": bookmark.Datetime,
"pos0": bookmark.Pos0,
"page": bookmark.Page,
})
if err == nil {
bookmarksSynced++
result, err := h.annotationSvc.SaveBookmark(ctx, wsync.SaveBookmarkRequest{
MediaItemID: mediaItemID,
UserID: pgUserID,
Title: bookmark.Text,
Position: position,
ChapterNumber: int32(bookmark.Chapter),
Source: "koreader",
DeviceSyncData: deviceData,
})
if err == nil && result.Outcome != wsync.SaveOutcomeDeleted {
bookmarksSynced++
}
} else {
_, err := h.db.CreateMediaNote(ctx, database.CreateMediaNoteParams{
MediaItemID: mediaItemID,
UserID: pgUserID,
Content: bookmark.Text,
Position: pgtype.Text{String: position, Valid: position != ""},
})
if err == nil {
bookmarksSynced++
}
}
}
@@ -906,15 +1132,35 @@ func (h *KOReaderHandler) SyncBookmarks(c *echo.Context) error {
position = fmt.Sprintf("page:%d", note.Page)
}
_, err := h.db.CreateMediaNote(ctx, database.CreateMediaNoteParams{
MediaItemID: mediaItemID,
UserID: pgUserID,
Content: note.Notes,
Position: pgtype.Text{String: position, Valid: position != ""},
})
if h.annotationSvc != nil {
deviceData, _ := json.Marshal(map[string]interface{}{
"datetime": note.Datetime,
"pos0": note.Pos0,
"page": note.Page,
})
if err == nil {
notesSynced++
result, err := h.annotationSvc.SaveNote(ctx, wsync.SaveNoteRequest{
MediaItemID: mediaItemID,
UserID: pgUserID,
Content: note.Notes,
Position: position,
Source: "koreader",
DeviceSyncData: deviceData,
})
if err == nil && result.Outcome != wsync.SaveOutcomeDeleted {
notesSynced++
}
} else {
_, err := h.db.CreateMediaNote(ctx, database.CreateMediaNoteParams{
MediaItemID: mediaItemID,
UserID: pgUserID,
Content: note.Notes,
Position: pgtype.Text{String: position, Valid: position != ""},
})
if err == nil {
notesSynced++
}
}
}
@@ -941,17 +1187,51 @@ func (h *KOReaderHandler) SyncBookmarks(c *echo.Context) error {
color = highlight.Color
}
_, err := h.db.CreateMediaHighlight(ctx, database.CreateMediaHighlightParams{
MediaItemID: mediaItemID,
UserID: pgUserID,
SelectionText: highlight.Text,
StartPosition: pgtype.Text{String: startPos, Valid: startPos != ""},
EndPosition: pgtype.Text{String: endPos, Valid: endPos != ""},
Color: pgtype.Text{String: color, Valid: true},
})
if h.annotationSvc != nil {
epubcfiStart, epubcfiEnd := h.convertHighlightPositions(ctx, mediaItemID, highlight.Pos0, highlight.Pos1)
if err == nil {
highlightsSynced++
pctStart := 0.0
if highlight.Percentage != nil {
pctStart = *highlight.Percentage
}
deviceData, _ := json.Marshal(map[string]interface{}{
"datetime": highlight.Datetime,
"pos0": highlight.Pos0,
"pos1": highlight.Pos1,
"page": highlight.Page,
})
result, err := h.annotationSvc.SaveHighlight(ctx, wsync.SaveHighlightRequest{
MediaItemID: mediaItemID,
UserID: pgUserID,
SelectionText: highlight.Text,
StartPosition: startPos,
EndPosition: endPos,
Color: color,
NoteText: highlight.Notes,
PercentageStart: pctStart,
EpubcfiStart: epubcfiStart,
EpubcfiEnd: epubcfiEnd,
Source: "koreader",
DeviceSyncData: deviceData,
})
if err == nil && result.Outcome != wsync.SaveOutcomeDeleted {
highlightsSynced++
}
} else {
_, err := h.db.CreateMediaHighlight(ctx, database.CreateMediaHighlightParams{
MediaItemID: mediaItemID,
UserID: pgUserID,
SelectionText: highlight.Text,
StartPosition: pgtype.Text{String: startPos, Valid: startPos != ""},
EndPosition: pgtype.Text{String: endPos, Valid: endPos != ""},
Color: pgtype.Text{String: color, Valid: true},
})
if err == nil {
highlightsSynced++
}
}
}