Bookhoard/bookhoard
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 12 Wiki Activity

feat(sync): wire annotation sync into all device and web handlers

Browse Source 
Complete the annotation sync pipeline across all ingest and serve paths.
Previously, annotations sent inline with KOReader progress pushes were
silently discarded, and no annotations were ever served back to devices.

INGEST (device → server):

KOReader (koreader.go):
  - Add processBookAnnotations helper that processes inline highlights,
    notes, and bookmarks from every progress push (immediate + checkpoint)
  - Highlights get CRE→CFI position conversion before SaveHighlight
  - KOReader 'notes' (text + notes) stored as highlights with NoteText
    to ensure correct round-trip classification
  - Bookmarks routed through SaveBookmark with device sync data
  - Called from both updateProgressForBook and handleCheckpointSync

Kobo (kobo.go):
  - Markup handler: annotations and bookmarks route through
    AnnotationService (SaveHighlight/SaveBookmark)
  - Bookmark handler: same routing with device sync data
  - SyncFromServer handler: same routing
  - All handlers fall back to direct DB calls when annotationSvc == nil

Web reader (media.go):
  - CreateMediaHighlight → SaveHighlight (Source="web", ModifiedAt=now)
  - CreateMediaNote → SaveNote (Source="web")
  - DeleteMediaHighlight → TombstoneHighlightByID
  - DeleteMediaNote → TombstoneNoteByID (was hard delete, now tombstone)
  - All fall back to old behavior when annotationSvc == nil

SERVE (server → device):

KOReader GetMetadata (koreader.go):
  - Query and serve bookmarks from media_bookmarks table (was missing)
  - Serve deleted_highlights and deleted_bookmarks arrays containing
    device_sync_data + dedup_key for client-side deletion
  - Highlights/notes already served with reverse CFI conversion

Kobo Markup handler (kobo.go):
  - Track processed books during sync
  - Query tombstones per book, extract bookmark_id from device_sync_data
  - Return DeletedAnnotations array in KoboSyncStatus response

Conflict resolution (conflicts.go):
  - Enable annotation conflict types in ResolveConflict handler
  - Add applyAnnotationResolution dispatching to:
    applyHighlightResolution / applyBookmarkResolution / applyNoteResolution
  - Each looks up by dedup_key and applies winner's fields
  - Allow manual override of auto_resolved conflicts
    (changed check from != "unresolved" to == "user_resolved")

Infrastructure:
  - AnnotationService field + SetAnnotationService in router Config
  - Inject AnnotationService into KOReader, Kobo, Media handlers
  - Start tombstone purger goroutine in main.go (24h interval)
  - Test helpers: construct AnnotationService in test setup
This commit is contained in:
john-okeefe
2026-07-29 14:49:19 -04:00
parent 635a9439cb
commit 75b33fdae6
8 changed files with 772 additions and 110 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/handlers/kobo.go
+230 -61
View File
@@ -3,6 +3,7 @@ package handlers
import (
"bookhoard/internal/database"
wsync "bookhoard/internal/sync"
"encoding/json"
"fmt"
"log"
"net/http"
@@ -19,6 +20,7 @@ type KoboHandler struct {
db *database.Queries
connManager *wsync.ConnectionManager
progressSvc *wsync.ProgressService
annotationSvc *wsync.AnnotationService
libraryService LibraryPathResolver
}
@@ -30,6 +32,10 @@ func (h *KoboHandler) SetProgressService(svc *wsync.ProgressService) {
h.progressSvc = svc
}
func (h *KoboHandler) SetAnnotationService(svc *wsync.AnnotationService) {
h.annotationSvc = svc
}
func (h *KoboHandler) SetLibraryService(svc LibraryPathResolver) {
h.libraryService = svc
}
@@ -245,9 +251,16 @@ type KoboInitResponse struct {
}
type KoboSyncStatus struct {
Status string `json:"Status"`
MarkupsSynced int `json:"MarkupsSynced"`
BookmarksSynced int `json:"BookmarksSynced"`
Status string `json:"Status"`
MarkupsSynced int `json:"MarkupsSynced"`
BookmarksSynced int `json:"BookmarksSynced"`
DeletedAnnotations []KoboDeletedAnnotation `json:"DeletedAnnotations,omitempty"`
}
type KoboDeletedAnnotation struct {
ContentId string `json:"ContentId"`
BookmarkId string `json:"BookmarkId"`
Type string `json:"Type"`
}
type KoboServerSyncData struct {
@@ -311,7 +324,7 @@ func (h *KoboHandler) Initialization(c *echo.Context) error {
}
bookmarkCount := 0
annotations, _ := h.db.GetAnnotationsForBook(c.Request().Context(), database.GetAnnotationsForBookParams{
annotations, _ := h.db.GetActiveAnnotationsForBook(c.Request().Context(), database.GetActiveAnnotationsForBookParams{
MediaItemID: pgtype.UUID{Bytes: item.ID.Bytes, Valid: true},
UserID: pgUserID,
})
@@ -403,6 +416,7 @@ func (h *KoboHandler) Markup(c *echo.Context) error {
markupsSynced := 0
bookmarksSynced := 0
unlinkedBooks := 0
processedBooks := make(map[pgtype.UUID]string)
for _, readingSync := range req.ReadingSync {
bookhoardUUID, err, _ := h.mapContentIdToBookhoardUUID(c, readingSync.ContentId, deviceUUID)
@@ -412,6 +426,7 @@ func (h *KoboHandler) Markup(c *echo.Context) error {
}
pgMediaUUID := pgtype.UUID{Bytes: bookhoardUUID, Valid: true}
processedBooks[pgMediaUUID] = readingSync.ContentId
percentage := readingSync.PercentRead / 100.0
// Kobo only sends a percentage. For fixed-layout & comic formats the page
@@ -465,29 +480,72 @@ func (h *KoboHandler) Markup(c *echo.Context) error {
}
pgMediaUUID := pgtype.UUID{Bytes: bookhoardUUID, Valid: true}
processedBooks[pgMediaUUID] = bookmarkSync.ContentId
switch bookmarkSync.BookmarkType {
case "annotation":
if bookmarkSync.BookmarkText != "" {
h.db.CreateMediaHighlight(c.Request().Context(), database.CreateMediaHighlightParams{
MediaItemID: pgMediaUUID,
UserID: pgUserID,
SelectionText: bookmarkSync.BookmarkText,
StartPosition: pgtype.Text{String: bookmarkSync.BookmarkId, Valid: true},
EndPosition: pgtype.Text{String: bookmarkSync.BookmarkId, Valid: true},
Color: pgtype.Text{String: "#ffff00", Valid: true},
})
bookmarksSynced++
if h.annotationSvc != nil {
deviceData, _ := json.Marshal(map[string]interface{}{
"bookmark_id": bookmarkSync.BookmarkId,
"date_created": bookmarkSync.DateCreated,
})
result, err := h.annotationSvc.SaveHighlight(c.Request().Context(), wsync.SaveHighlightRequest{
MediaItemID: pgMediaUUID,
UserID: pgUserID,
SelectionText: bookmarkSync.BookmarkText,
StartPosition: bookmarkSync.BookmarkId,
EndPosition: bookmarkSync.BookmarkId,
Color: "#ffff00",
NoteText: bookmarkSync.BookmarkTitle,
Source: "kobo",
DeviceSyncData: deviceData,
})
if err == nil && result.Outcome != wsync.SaveOutcomeDeleted {
bookmarksSynced++
}
} else {
h.db.CreateMediaHighlight(c.Request().Context(), database.CreateMediaHighlightParams{
MediaItemID: pgMediaUUID,
UserID: pgUserID,
SelectionText: bookmarkSync.BookmarkText,
StartPosition: pgtype.Text{String: bookmarkSync.BookmarkId, Valid: true},
EndPosition: pgtype.Text{String: bookmarkSync.BookmarkId, Valid: true},
Color: pgtype.Text{String: "#ffff00", Valid: true},
})
bookmarksSynced++
}
}
case "bookmark":
if bookmarkSync.BookmarkText != "" {
h.db.CreateMediaNote(c.Request().Context(), database.CreateMediaNoteParams{
MediaItemID: pgMediaUUID,
UserID: pgUserID,
Content: bookmarkSync.BookmarkText,
Position: pgtype.Text{String: bookmarkSync.BookmarkId, Valid: true},
})
bookmarksSynced++
if h.annotationSvc != nil {
deviceData, _ := json.Marshal(map[string]interface{}{
"bookmark_id": bookmarkSync.BookmarkId,
"date_created": bookmarkSync.DateCreated,
})
result, err := h.annotationSvc.SaveBookmark(c.Request().Context(), wsync.SaveBookmarkRequest{
MediaItemID: pgMediaUUID,
UserID: pgUserID,
Title: bookmarkSync.BookmarkText,
Position: bookmarkSync.BookmarkId,
ChapterNumber: int32(bookmarkSync.Chapter),
Source: "kobo",
DeviceSyncData: deviceData,
})
if err == nil && result.Outcome != wsync.SaveOutcomeDeleted {
bookmarksSynced++
}
} else {
h.db.CreateMediaNote(c.Request().Context(), database.CreateMediaNoteParams{
MediaItemID: pgMediaUUID,
UserID: pgUserID,
Content: bookmarkSync.BookmarkText,
Position: pgtype.Text{String: bookmarkSync.BookmarkId, Valid: true},
})
bookmarksSynced++
}
}
case "last-read-place":
if bookmarkSync.BookmarkId != "" {
@@ -564,6 +622,32 @@ func (h *KoboHandler) Markup(c *echo.Context) error {
BookmarksSynced: bookmarksSynced,
}
if h.annotationSvc != nil && len(processedBooks) > 0 {
cutoff := pgtype.Timestamptz{Time: time.Now().Add(-wsync.TombstoneTTL), Valid: true}
for mediaItemID, contentId := range processedBooks {
tombstones, _ := h.db.GetTombstonedAnnotationsForBook(c.Request().Context(), database.GetTombstonedAnnotationsForBookParams{
MediaItemID: mediaItemID,
UserID: pgUserID,
DeletedAt: cutoff,
})
for _, ts := range tombstones {
var dd map[string]interface{}
if len(ts.DeviceSyncData) > 0 {
json.Unmarshal(ts.DeviceSyncData, &dd)
}
bookmarkID, _ := dd["bookmark_id"].(string)
if bookmarkID == "" {
continue
}
response.DeletedAnnotations = append(response.DeletedAnnotations, KoboDeletedAnnotation{
ContentId: contentId,
BookmarkId: bookmarkID,
Type: ts.AnnotationType,
})
}
}
}
// Include unlinked books count if any
if unlinkedBooks > 0 {
// For now, just log it. In production, this should trigger an alert
@@ -606,25 +690,67 @@ func (h *KoboHandler) Bookmark(c *echo.Context) error {
switch bookmarkSync.BookmarkType {
case "annotation":
if bookmarkSync.BookmarkText != "" {
h.db.CreateMediaHighlight(c.Request().Context(), database.CreateMediaHighlightParams{
MediaItemID: pgMediaUUID,
UserID: pgUserID,
SelectionText: bookmarkSync.BookmarkText,
StartPosition: pgtype.Text{String: bookmarkSync.BookmarkId, Valid: true},
EndPosition: pgtype.Text{String: bookmarkSync.BookmarkId, Valid: true},
Color: pgtype.Text{String: "#ffff00", Valid: true},
})
bookmarksSynced++
if h.annotationSvc != nil {
deviceData, _ := json.Marshal(map[string]interface{}{
"bookmark_id": bookmarkSync.BookmarkId,
"date_created": bookmarkSync.DateCreated,
})
result, err := h.annotationSvc.SaveHighlight(c.Request().Context(), wsync.SaveHighlightRequest{
MediaItemID: pgMediaUUID,
UserID: pgUserID,
SelectionText: bookmarkSync.BookmarkText,
StartPosition: bookmarkSync.BookmarkId,
EndPosition: bookmarkSync.BookmarkId,
Color: "#ffff00",
NoteText: bookmarkSync.BookmarkTitle,
Source: "kobo",
DeviceSyncData: deviceData,
})
if err == nil && result.Outcome != wsync.SaveOutcomeDeleted {
bookmarksSynced++
}
} else {
h.db.CreateMediaHighlight(c.Request().Context(), database.CreateMediaHighlightParams{
MediaItemID: pgMediaUUID,
UserID: pgUserID,
SelectionText: bookmarkSync.BookmarkText,
StartPosition: pgtype.Text{String: bookmarkSync.BookmarkId, Valid: true},
EndPosition: pgtype.Text{String: bookmarkSync.BookmarkId, Valid: true},
Color: pgtype.Text{String: "#ffff00", Valid: true},
})
bookmarksSynced++
}
}
case "bookmark":
if bookmarkSync.BookmarkText != "" {
h.db.CreateMediaNote(c.Request().Context(), database.CreateMediaNoteParams{
MediaItemID: pgMediaUUID,
UserID: pgUserID,
Content: bookmarkSync.BookmarkText,
Position: pgtype.Text{String: bookmarkSync.BookmarkId, Valid: true},
})
bookmarksSynced++
if h.annotationSvc != nil {
deviceData, _ := json.Marshal(map[string]interface{}{
"bookmark_id": bookmarkSync.BookmarkId,
"date_created": bookmarkSync.DateCreated,
})
result, err := h.annotationSvc.SaveBookmark(c.Request().Context(), wsync.SaveBookmarkRequest{
MediaItemID: pgMediaUUID,
UserID: pgUserID,
Title: bookmarkSync.BookmarkText,
Position: bookmarkSync.BookmarkId,
ChapterNumber: int32(bookmarkSync.Chapter),
Source: "kobo",
DeviceSyncData: deviceData,
})
if err == nil && result.Outcome != wsync.SaveOutcomeDeleted {
bookmarksSynced++
}
} else {
h.db.CreateMediaNote(c.Request().Context(), database.CreateMediaNoteParams{
MediaItemID: pgMediaUUID,
UserID: pgUserID,
Content: bookmarkSync.BookmarkText,
Position: pgtype.Text{String: bookmarkSync.BookmarkId, Valid: true},
})
bookmarksSynced++
}
}
}
}
@@ -756,36 +882,79 @@ func (h *KoboHandler) SyncFromServer(c *echo.Context) error {
for _, bookmark := range syncData.Bookmarks {
if bookmark.BookmarkType == "bookmark" {
h.db.CreateMediaNote(c.Request().Context(), database.CreateMediaNoteParams{
MediaItemID: pgMediaUUID,
UserID: pgUserID,
Content: bookmark.BookmarkText,
Position: pgtype.Text{String: bookmark.BookmarkId, Valid: true},
})
bookmarksSent++
if h.annotationSvc != nil {
result, err := h.annotationSvc.SaveBookmark(c.Request().Context(), wsync.SaveBookmarkRequest{
MediaItemID: pgMediaUUID,
UserID: pgUserID,
Title: bookmark.BookmarkText,
Position: bookmark.BookmarkId,
Source: "kobo",
})
if err == nil && result.Outcome != wsync.SaveOutcomeDeleted {
bookmarksSent++
}
} else {
h.db.CreateMediaNote(c.Request().Context(), database.CreateMediaNoteParams{
MediaItemID: pgMediaUUID,
UserID: pgUserID,
Content: bookmark.BookmarkText,
Position: pgtype.Text{String: bookmark.BookmarkId, Valid: true},
})
bookmarksSent++
}
} else if bookmark.BookmarkType == "annotation" {
h.db.CreateMediaHighlight(c.Request().Context(), database.CreateMediaHighlightParams{
MediaItemID: pgMediaUUID,
UserID: pgUserID,
SelectionText: bookmark.BookmarkText,
StartPosition: pgtype.Text{String: bookmark.BookmarkId, Valid: true},
EndPosition: pgtype.Text{String: bookmark.BookmarkId, Valid: true},
Color: pgtype.Text{String: "#ffff00", Valid: true},
})
highlightsSent++
if h.annotationSvc != nil {
result, err := h.annotationSvc.SaveHighlight(c.Request().Context(), wsync.SaveHighlightRequest{
MediaItemID: pgMediaUUID,
UserID: pgUserID,
SelectionText: bookmark.BookmarkText,
StartPosition: bookmark.BookmarkId,
EndPosition: bookmark.BookmarkId,
Color: "#ffff00",
Source: "kobo",
})
if err == nil && result.Outcome != wsync.SaveOutcomeDeleted {
highlightsSent++
}
} else {
h.db.CreateMediaHighlight(c.Request().Context(), database.CreateMediaHighlightParams{
MediaItemID: pgMediaUUID,
UserID: pgUserID,
SelectionText: bookmark.BookmarkText,
StartPosition: pgtype.Text{String: bookmark.BookmarkId, Valid: true},
EndPosition: pgtype.Text{String: bookmark.BookmarkId, Valid: true},
Color: pgtype.Text{String: "#ffff00", Valid: true},
})
highlightsSent++
}
}
}
for _, highlight := range syncData.Highlights {
h.db.CreateMediaHighlight(c.Request().Context(), database.CreateMediaHighlightParams{
MediaItemID: pgMediaUUID,
UserID: pgUserID,
SelectionText: highlight.BookmarkText,
StartPosition: pgtype.Text{String: highlight.BookmarkId, Valid: true},
EndPosition: pgtype.Text{String: highlight.BookmarkId, Valid: true},
Color: pgtype.Text{String: "#ffff00", Valid: true},
})
highlightsSent++
if h.annotationSvc != nil {
result, err := h.annotationSvc.SaveHighlight(c.Request().Context(), wsync.SaveHighlightRequest{
MediaItemID: pgMediaUUID,
UserID: pgUserID,
SelectionText: highlight.BookmarkText,
StartPosition: highlight.BookmarkId,
EndPosition: highlight.BookmarkId,
Color: "#ffff00",
Source: "kobo",
})
if err == nil && result.Outcome != wsync.SaveOutcomeDeleted {
highlightsSent++
}
} else {
h.db.CreateMediaHighlight(c.Request().Context(), database.CreateMediaHighlightParams{
MediaItemID: pgMediaUUID,
UserID: pgUserID,
SelectionText: highlight.BookmarkText,
StartPosition: pgtype.Text{String: highlight.BookmarkId, Valid: true},
EndPosition: pgtype.Text{String: highlight.BookmarkId, Valid: true},
Color: pgtype.Text{String: "#ffff00", Valid: true},
})
highlightsSent++
}
}
}