feat(sync): add AnnotationService with dedup, LWW, and tombstone management
AnnotationService is the central service for cross-device annotation sync.
It provides SaveHighlight, SaveNote, and SaveBookmark methods that handle
the full sync lifecycle:
Identity (3-layer):
1. Server UUID (primary key)
2. Per-device native ID stored in device_sync_data JSONB
3. Content dedup_key: sha1(normalize(selection_text) + bucket_position)
- CFI character offsets are stripped for bucketing so the same
highlight at slightly different offsets still deduplicates
- Raw positions are preserved in the DB for precise restoration
Resolution policy (LWW):
- When the incoming annotation has an explicit ModifiedAt timestamp,
last_modified_at wins
- When the device sends zero ModifiedAt (creation time only), field-diff
mode compares content fields (text/color/note/percentage) — if all
match, the save is skipped; if any differ, the save is applied with
server-receive-time as the new last_modified_at
Conflict detection:
- When incoming and existing annotations have different sources (e.g.
koreader vs kobo) and content differs, an auto_resolved sync_conflict
is recorded with both sides' data for audit trail
- Broadcasts a WebSocket conflict notification for real-time UI updates
Tombstone management:
- Delete-wins: tombstoned annotations block recreation from stale pushes
- 30-day TTL before physical purge
- PurgeExpiredTombstones method + StartTombstonePurger goroutine (24h ticker)
Add locators.go with unified bidirectional CFI conversion:
ConvertToCanonical / ConvertFromCanonical
- CRE XPointer <-> standard EPUB CFI (for KOReader)
- KEPUB CFI passthrough (for Kobo)
- Skips non-reflowable formats (PDF, CBZ, fixed-layout EPUBs)
Add 25 unit tests covering:
- Dedup key determinism, text normalization, position sensitivity
- Offset insensitivity (CFI char-offset bucketing)
- Device sync data merge (preserves existing, overwrites same source)
- Cross-source detection
- LWW comparison (newer wins, older skipped, fallback to updated_at)
- Field-diff mode (identical content skipped, changes applied)
- Tombstone TTL constant
- CRE XPointer parsing and classification
- Standard EPUB CFI classification
This commit is contained in:
@@ -0,0 +1,343 @@
|
||||
package sync
|
||||
|
||||
import (
|
||||
"bookhoard/internal/database"
|
||||
"encoding/json"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgtype"
|
||||
)
|
||||
|
||||
func TestComputeDedupKey_Deterministic(t *testing.T) {
|
||||
k1 := ComputeDedupKey("Hello World", "epubcfi(/6/4!/4/10/3:100)", "")
|
||||
k2 := ComputeDedupKey("Hello World", "epubcfi(/6/4!/4/10/3:100)", "")
|
||||
if k1 != k2 {
|
||||
t.Errorf("same input should produce same key: %q vs %q", k1, k2)
|
||||
}
|
||||
}
|
||||
|
||||
func TestComputeDedupKey_Normalization(t *testing.T) {
|
||||
cases := [][]string{
|
||||
{"Hello World", " Hello World "},
|
||||
{"HELLO WORLD", "hello world"},
|
||||
{"Hello World", "Hello World"},
|
||||
{"Hello\t\nWorld", "Hello World"},
|
||||
}
|
||||
cfi := "epubcfi(/6/4!/4/10/3:100)"
|
||||
for _, c := range cases {
|
||||
k1 := ComputeDedupKey(c[0], cfi, "")
|
||||
k2 := ComputeDedupKey(c[1], cfi, "")
|
||||
if k1 != k2 {
|
||||
t.Errorf("normalized texts should match: %q vs %q → %q vs %q", c[0], c[1], k1, k2)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestComputeDedupKey_PositionSensitivity(t *testing.T) {
|
||||
text := "same text"
|
||||
k1 := ComputeDedupKey(text, "epubcfi(/6/4!/4/10/3:100)", "")
|
||||
k2 := ComputeDedupKey(text, "epubcfi(/6/4!/4/20/3:100)", "")
|
||||
if k1 == k2 {
|
||||
t.Error("different element paths should produce different keys")
|
||||
}
|
||||
}
|
||||
|
||||
func TestComputeDedupKey_OffsetInsensitive(t *testing.T) {
|
||||
text := "same text"
|
||||
base := "epubcfi(/6/4!/4/10/3:100)"
|
||||
offsetShift := "epubcfi(/6/4!/4/10/3:200)"
|
||||
k1 := ComputeDedupKey(text, base, "")
|
||||
k2 := ComputeDedupKey(text, offsetShift, "")
|
||||
if k1 != k2 {
|
||||
t.Error("same element path with different char offsets should produce same key (bucket)")
|
||||
}
|
||||
}
|
||||
|
||||
func TestComputeDedupKey_FallbackToRawPosition(t *testing.T) {
|
||||
text := "same text"
|
||||
k1 := ComputeDedupKey(text, "", "page:42")
|
||||
k2 := ComputeDedupKey(text, "", "page:42")
|
||||
if k1 != k2 {
|
||||
t.Error("same raw position should produce same key")
|
||||
}
|
||||
k3 := ComputeDedupKey(text, "", "page:99")
|
||||
if k1 == k3 {
|
||||
t.Error("different raw positions should produce different keys")
|
||||
}
|
||||
}
|
||||
|
||||
func TestComputeDedupKey_DifferentTextSamePosition(t *testing.T) {
|
||||
cfi := "epubcfi(/6/4!/4/10/3:100)"
|
||||
k1 := ComputeDedupKey("first highlight", cfi, "")
|
||||
k2 := ComputeDedupKey("second highlight", cfi, "")
|
||||
if k1 == k2 {
|
||||
t.Error("different selection text should produce different keys")
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizeText(t *testing.T) {
|
||||
cases := []struct{ in, want string }{
|
||||
{"Hello World", "hello world"},
|
||||
{" Hello World ", "hello world"},
|
||||
{"Hello\t\nWorld", "hello world"},
|
||||
{"", ""},
|
||||
{" ", ""},
|
||||
}
|
||||
for _, c := range cases {
|
||||
got := normalizeText(c.in)
|
||||
if got != c.want {
|
||||
t.Errorf("normalizeText(%q) = %q, want %q", c.in, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestBucketPosition(t *testing.T) {
|
||||
cases := []struct{ in, want string }{
|
||||
{"epubcfi(/6/4!/4/10/3:100)", "epubcfi(/6/4!/4/10/3"},
|
||||
{"epubcfi(/6/4!/4/10/3:0)", "epubcfi(/6/4!/4/10/3"},
|
||||
{"page:42", "page:42"},
|
||||
{"short", "short"},
|
||||
{"", ""},
|
||||
}
|
||||
for _, c := range cases {
|
||||
got := bucketPosition(c.in)
|
||||
if got != c.want {
|
||||
t.Errorf("bucketPosition(%q) = %q, want %q", c.in, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestBucketPosition_LongString(t *testing.T) {
|
||||
long := "this_is_a_very_long_position_string_that_exceeds_fifty_characters_total"
|
||||
got := bucketPosition(long)
|
||||
if len(got) > 50 {
|
||||
t.Errorf("bucketPosition should truncate to <=50 chars, got %d", len(got))
|
||||
}
|
||||
if got != long[:50] {
|
||||
t.Errorf("bucketPosition truncated wrong: got %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMergeDeviceSyncData_NewEntry(t *testing.T) {
|
||||
result := mergeDeviceSyncData(nil, "koreader", json.RawMessage(`{"datetime":"2024-01-01"}`))
|
||||
var m map[string]interface{}
|
||||
if err := json.Unmarshal(result, &m); err != nil {
|
||||
t.Fatalf("unmarshal failed: %v", err)
|
||||
}
|
||||
entry, ok := m["koreader"]
|
||||
if !ok {
|
||||
t.Fatal("expected koreader entry")
|
||||
}
|
||||
entryMap := entry.(map[string]interface{})
|
||||
if entryMap["datetime"] != "2024-01-01" {
|
||||
t.Errorf("unexpected datetime: %v", entryMap["datetime"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestMergeDeviceSyncData_PreservesExisting(t *testing.T) {
|
||||
existing := []byte(`{"koreader":{"datetime":"2024-01-01"}}`)
|
||||
result := mergeDeviceSyncData(existing, "kobo", json.RawMessage(`{"bookmark_id":"abc"}`))
|
||||
var m map[string]interface{}
|
||||
if err := json.Unmarshal(result, &m); err != nil {
|
||||
t.Fatalf("unmarshal failed: %v", err)
|
||||
}
|
||||
if _, ok := m["koreader"]; !ok {
|
||||
t.Error("koreader entry should be preserved")
|
||||
}
|
||||
if _, ok := m["kobo"]; !ok {
|
||||
t.Error("kobo entry should be added")
|
||||
}
|
||||
}
|
||||
|
||||
func TestMergeDeviceSyncData_OverwritesSameSource(t *testing.T) {
|
||||
existing := []byte(`{"koreader":{"datetime":"old"}}`)
|
||||
result := mergeDeviceSyncData(existing, "koreader", json.RawMessage(`{"datetime":"new"}`))
|
||||
var m map[string]interface{}
|
||||
json.Unmarshal(result, &m)
|
||||
entry := m["koreader"].(map[string]interface{})
|
||||
if entry["datetime"] != "new" {
|
||||
t.Errorf("expected overwritten datetime 'new', got %v", entry["datetime"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsCrossSource(t *testing.T) {
|
||||
if isCrossSource("koreader", pgtype.Text{String: "kobo", Valid: true}) != true {
|
||||
t.Error("different sources should be cross-source")
|
||||
}
|
||||
if isCrossSource("koreader", pgtype.Text{String: "koreader", Valid: true}) != false {
|
||||
t.Error("same sources should not be cross-source")
|
||||
}
|
||||
if isCrossSource("", pgtype.Text{String: "koreader", Valid: true}) != false {
|
||||
t.Error("empty incoming source should not be cross-source")
|
||||
}
|
||||
if isCrossSource("koreader", pgtype.Text{Valid: false}) != false {
|
||||
t.Error("invalid existing source should not be cross-source")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompareIncoming_FieldDiff_Identical(t *testing.T) {
|
||||
svc := &AnnotationService{}
|
||||
req := SaveHighlightRequest{
|
||||
SelectionText: "hello",
|
||||
Color: "#ffff00",
|
||||
NoteText: "a note",
|
||||
PercentageStart: 10.5,
|
||||
PercentageEnd: 11.0,
|
||||
}
|
||||
existing := pgHighlights("hello", "#ffff00", "a note", 10.5, 11.0)
|
||||
newer, changed := svc.compareIncoming(req, existing)
|
||||
if newer {
|
||||
t.Error("identical content should not be newer")
|
||||
}
|
||||
if changed {
|
||||
t.Error("identical content should not be changed")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompareIncoming_FieldDiff_DifferentText(t *testing.T) {
|
||||
svc := &AnnotationService{}
|
||||
req := SaveHighlightRequest{
|
||||
SelectionText: "edited text",
|
||||
}
|
||||
existing := pgHighlights("original text", "#ffff00", "", 0, 0)
|
||||
newer, changed := svc.compareIncoming(req, existing)
|
||||
if !newer {
|
||||
t.Error("different content should be newer")
|
||||
}
|
||||
if !changed {
|
||||
t.Error("different content should be changed")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompareIncoming_FieldDiff_DifferentColor(t *testing.T) {
|
||||
svc := &AnnotationService{}
|
||||
req := SaveHighlightRequest{
|
||||
SelectionText: "same",
|
||||
Color: "#ff0000",
|
||||
}
|
||||
existing := pgHighlights("same", "#ffff00", "", 0, 0)
|
||||
_, changed := svc.compareIncoming(req, existing)
|
||||
if !changed {
|
||||
t.Error("different color should be detected as changed")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompareIncoming_LWW_NewerWins(t *testing.T) {
|
||||
svc := &AnnotationService{}
|
||||
now := time.Now()
|
||||
req := SaveHighlightRequest{
|
||||
SelectionText: "same",
|
||||
ModifiedAt: now.Add(1 * time.Hour),
|
||||
}
|
||||
existing := pgHighlights("same", "", "", 0, 0)
|
||||
existing.LastModifiedAt = pgtype.Timestamptz{Time: now, Valid: true}
|
||||
newer, changed := svc.compareIncoming(req, existing)
|
||||
if !newer {
|
||||
t.Error("future timestamp should be newer")
|
||||
}
|
||||
if !changed {
|
||||
t.Error("LWW mode should always report changed=true")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompareIncoming_LWW_OlderSkipped(t *testing.T) {
|
||||
svc := &AnnotationService{}
|
||||
now := time.Now()
|
||||
req := SaveHighlightRequest{
|
||||
SelectionText: "same",
|
||||
ModifiedAt: now.Add(-1 * time.Hour),
|
||||
}
|
||||
existing := pgHighlights("same", "", "", 0, 0)
|
||||
existing.LastModifiedAt = pgtype.Timestamptz{Time: now, Valid: true}
|
||||
newer, _ := svc.compareIncoming(req, existing)
|
||||
if newer {
|
||||
t.Error("past timestamp should not be newer")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompareIncoming_LWW_FallsBackToUpdatedAt(t *testing.T) {
|
||||
svc := &AnnotationService{}
|
||||
now := time.Now()
|
||||
req := SaveHighlightRequest{
|
||||
SelectionText: "same",
|
||||
ModifiedAt: now.Add(1 * time.Hour),
|
||||
}
|
||||
existing := pgHighlights("same", "", "", 0, 0)
|
||||
existing.LastModifiedAt = pgtype.Timestamptz{Valid: false}
|
||||
existing.UpdatedAt = pgtype.Timestamptz{Time: now, Valid: true}
|
||||
newer, _ := svc.compareIncoming(req, existing)
|
||||
if !newer {
|
||||
t.Error("should fall back to updated_at when last_modified_at is invalid")
|
||||
}
|
||||
}
|
||||
|
||||
func TestPgText(t *testing.T) {
|
||||
if pgText("").Valid {
|
||||
t.Error("empty string should produce invalid pgtype.Text")
|
||||
}
|
||||
v := pgText("hello")
|
||||
if !v.Valid || v.String != "hello" {
|
||||
t.Errorf("expected valid 'hello', got %+v", v)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPgFloat8(t *testing.T) {
|
||||
if pgFloat8(0).Valid {
|
||||
t.Error("zero should produce invalid pgtype.Float8")
|
||||
}
|
||||
v := pgFloat8(1.5)
|
||||
if !v.Valid || v.Float64 != 1.5 {
|
||||
t.Errorf("expected valid 1.5, got %+v", v)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPgInt4(t *testing.T) {
|
||||
if pgInt4(0).Valid {
|
||||
t.Error("zero should produce invalid pgtype.Int4")
|
||||
}
|
||||
v := pgInt4(3)
|
||||
if !v.Valid || v.Int32 != 3 {
|
||||
t.Errorf("expected valid 3, got %+v", v)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFloatEq(t *testing.T) {
|
||||
if !floatEq(0, pgtype.Float8{Valid: false}) {
|
||||
t.Error("0 vs invalid should be equal")
|
||||
}
|
||||
if !floatEq(10.5, pgtype.Float8{Float64: 10.5, Valid: true}) {
|
||||
t.Error("10.5 vs 10.5 should be equal")
|
||||
}
|
||||
if floatEq(10.6, pgtype.Float8{Float64: 10.5, Valid: true}) {
|
||||
t.Error("10.6 vs 10.5 should not be equal")
|
||||
}
|
||||
}
|
||||
|
||||
func TestTextEq(t *testing.T) {
|
||||
if !textEq("", pgtype.Text{Valid: false}) {
|
||||
t.Error("empty vs invalid should be equal")
|
||||
}
|
||||
if !textEq("hi", pgtype.Text{String: "hi", Valid: true}) {
|
||||
t.Error("same strings should be equal")
|
||||
}
|
||||
if textEq("hi", pgtype.Text{String: "bye", Valid: true}) {
|
||||
t.Error("different strings should not be equal")
|
||||
}
|
||||
}
|
||||
|
||||
func TestTombstoneTTL(t *testing.T) {
|
||||
if TombstoneTTL != 30*24*time.Hour {
|
||||
t.Errorf("expected 30 days, got %v", TombstoneTTL)
|
||||
}
|
||||
}
|
||||
|
||||
func pgHighlights(text, color, note string, pctStart, pctEnd float64) database.MediaHighlights {
|
||||
return database.MediaHighlights{
|
||||
SelectionText: text,
|
||||
Color: pgtype.Text{String: color, Valid: color != ""},
|
||||
NoteText: pgtype.Text{String: note, Valid: note != ""},
|
||||
PercentageStart: pgtype.Float8{Float64: pctStart, Valid: pctStart != 0},
|
||||
PercentageEnd: pgtype.Float8{Float64: pctEnd, Valid: pctEnd != 0},
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user