diff --git a/UNIVERSAL_SYNC_IMPLEMENTATION_GUIDE.md b/UNIVERSAL_SYNC_IMPLEMENTATION_GUIDE.md new file mode 100644 index 0000000..5579264 --- /dev/null +++ b/UNIVERSAL_SYNC_IMPLEMENTATION_GUIDE.md @@ -0,0 +1,2167 @@ +# Bookmann Universal Cross-Platform Sync Implementation Guide + +## Executive Summary + +This document provides a complete technical specification for implementing a universal cross-platform reading progress synchronization system that rivals the Amazon Kindle ecosystem. The system enables seamless reading progress, highlights, and notes synchronization across web interface, KOReader, Kobo devices, and future platforms - all self-hosted and open-source. + +**Vision**: Users can read on any device, put it down, pick up any other device, and continue exactly where they left off with all annotations synchronized. + +**Core Philosophy**: Bookmann serves as the "universal translator" between different reading platforms, understanding multiple location reference systems and converting between them seamlessly. + +--- + +## Table of Contents + +1. [Current System Status](#current-system-status) +2. [Problem Analysis](#problem-analysis) +3. [Technical Architecture](#technical-architecture) +4. [Database Schema Changes](#database-schema-changes) +5. [API Endpoints Specification](#api-endpoints-specification) +6. [Format Grouping System](#format-grouping-system) +7. [Progress Tracking by Format](#progress-tracking-by-format) +8. [Wireless Sync Protocols](#wireless-sync-protocols) +9. [Security Architecture](#security-architecture) +10. [Conflict Resolution System](#conflict-resolution-system) +11. [Backup & Recovery Strategy](#backup--recovery-strategy) +12. [Real-time Synchronization](#real-time-synchronization) +13. [Device Onboarding Flow](#device-onboarding-flow) +14. [Implementation Phases](#implementation-phases) +15. [Testing Requirements](#testing-requirements) +16. [Performance Considerations](#performance-considerations) + +--- + +## Current System Status + +### Working Endpoints (As of Testing) +All core functionality is operational: +- Authentication: Register, login, logout, refresh tokens ✅ +- Library management: Create, read, update, delete libraries ✅ +- Scanner: Background scanning, watch mode, status tracking ✅ +- Media items: CRUD operations, search, filter, sort ✅ +- Progress tracking: Basic page-based tracking ✅ +- Notes: Create, read, update, delete ✅ +- Highlights: Create, read, update, delete ✅ +- Ratings: Full rating system ✅ + +### Known Issues +1. **EPUB Progress Tracking**: Current page-based system is unreliable for reflowable formats +2. **No Cross-Device Sync**: Progress doesn't sync between devices +3. **Limited Platform Support**: Only web interface, no device integration +4. **Format Inconsistency**: Different formats treated identically despite different capabilities + +--- + +## Problem Analysis + +### The EPUB Progress Problem + +**Current Implementation Flaw:** +- EPUB is reflowable - page count changes based on font size, screen size, zoom level +- "Page 45 of 200" is meaningless across different devices or reading sessions +- Notes at "page:45" location become unfindable with different display settings +- Progress sharing between users or devices is unreliable + +**Real-World Example:** +- User reads on tablet (large font) → sees "page 45 of 250" +- Same user reads on phone (small font) → sees "page 78 of 450" +- Same book, same progress, but completely different page numbers + +### Cross-Platform Incompatibility + +**Kobo Devices:** +- Use percentage-based progress internally +- Sync to proprietary Kobo cloud +- Store annotations in SQLite databases +- No open API for third-party sync + +**KOReader:** +- Open-source, supports Wi-Fi sync +- Uses calibre-compatible protocols +- Stores data in `.sdr` sidecar folders +- Percentage + character offset tracking + +**Bookmann Current State:** +- Page-based tracking only +- No device protocols implemented +- No sync endpoints for external devices + +--- + +## Technical Architecture + +### System Overview + +``` +┌─────────────────────────────────────────────────────────────┐ +│ Bookmann Core System │ +│ ┌──────────────┐ ┌──────────────┐ ┌──────────────┐ │ +│ │ Web UI │ │ Wireless │ │ Progress │ │ +│ │ Interface │ │ Sync API │ │ Conversion │ │ +│ └──────────────┘ └──────────────┘ └──────────────┘ │ +└─────────────────────────────────────────────────────────────┘ + ▲ + │ + ┌───────────────────┼───────────────────┐ + │ │ │ +┌───────┴────────┐ ┌──────┴─────────┐ ┌─────┴────────┐ +│ KOReader │ │ Kobo Device │ │ Web Mobile │ +│ .sdr folders │ │ SQLite DB │ │ App/API │ +└────────────────┘ └────────────────┘ └───────────────┘ +``` + +### Data Flow Architecture + +``` +Device Progress → Device Protocol → Bookmann Wireless API + ↓ + Universal Progress JSON + ↓ + Format Conversion Engine + ↓ + Database Storage (Multi-format) + ↓ + WebSocket Broadcast + ↓ + All Connected Clients (Web, Mobile, Other Devices) +``` + +### Universal Progress Hub Concept + +Bookmann doesn't just store progress - it maintains a **multi-dimensional location reference system** that can express the same reading position in multiple ways simultaneously: + +```json +{ + "book_id": "uuid-1234", + "universal_progress": 0.45678, + "format_group": "reflowable", + "location_references": { + "percentage": 0.45678, + "epubcfi": "epubcfi(/6/4[chap1]!/4/2:15)", + "character": 15432, + "chapter": 3, + "chapter_progress": 0.234, + "viewport_y": 0.12 + }, + "device_specific": { + "koreader": { "percent": 0.45678, "timestamp": "..." }, + "kobo": { "progress": 45, "timestamp": "..." }, + "web": { "display_page": 89, "total_pages": 200, "timestamp": "..." } + } +} +``` + +--- + +## Database Schema Changes + +### Core Schema Additions + +```sql +-- ============================================ +-- FORMAT DETECTION AND GROUPING +-- ============================================ +ALTER TABLE media_items + ADD COLUMN format_group VARCHAR(20) NOT NULL DEFAULT 'reflowable', + ADD COLUMN format_mimetype VARCHAR(100), + ADD COLUMN is_reflowable BOOLEAN DEFAULT TRUE, + ADD COLUMN has_fixed_layout BOOLEAN DEFAULT FALSE, + ADD COLUMN total_characters BIGINT, + ADD COLUMN chapter_count INTEGER; + +-- ============================================ +-- UNIVERSAL PROGRESS TRACKING +-- ============================================ +ALTER TABLE reading_progress + ADD COLUMN percentage FLOAT CHECK (percentage >= 0 AND percentage <= 1), + ADD COLUMN character_offset BIGINT, + ADD COLUMN epubcfi TEXT, + ADD COLUMN chapter INTEGER, + ADD COLUMN chapter_progress FLOAT CHECK (chapter_progress >= 0 AND chapter_progress <= 1), + ADD COLUMN viewport_x FLOAT DEFAULT 0, + ADD COLUMN viewport_y FLOAT DEFAULT 0, + ADD COLUMN zoom_level FLOAT DEFAULT 1.0, + ADD COLUMN scroll_position_x FLOAT DEFAULT 0, + ADD COLUMN scroll_position_y FLOAT DEFAULT 0, + ADD COLUMN panel_number INTEGER, + ADD COLUMN reading_mode VARCHAR(20); + +-- ============================================ +-- DEVICE SYNC METADATA +-- ============================================ +ALTER TABLE reading_progress + ADD COLUMN last_sync_device VARCHAR(50), + ADD COLUMN last_sync_source VARCHAR(20), + ADD COLUMN last_sync_timestamp TIMESTAMP, + ADD COLUMN conflict_detected BOOLEAN DEFAULT FALSE, + ADD COLUMN conflict_resolved BOOLEAN DEFAULT TRUE; + +-- ============================================ +-- NOTES LOCATION ENHANCEMENTS +-- ============================================ +ALTER TABLE media_notes + ADD COLUMN percentage_location FLOAT, + ADD COLUMN character_start INTEGER, + ADD COLUMN character_end INTEGER, + ADD COLUMN epubcfi_location TEXT, + ADD COLUMN chapter_reference INTEGER, + ADD COLUMN paragraph_reference INTEGER, + ADD COLUMN device_sync_data JSONB; + +-- ============================================ +-- HIGHLIGHTS LOCATION ENHANCEMENTS +-- ============================================ +ALTER TABLE media_highlights + ADD COLUMN percentage_start FLOAT, + ADD COLUMN percentage_end FLOAT, + ADD COLUMN character_start INTEGER, + ADD COLUMN character_end INTEGER, + ADD COLUMN epubcfi_start TEXT, + ADD COLUMN epubcfi_end TEXT, + ADD COLUMN chapter_reference INTEGER, + ADD COLUMN paragraph_start INTEGER, + ADD COLUMN paragraph_end INTEGER, + ADD COLUMN panel_number INTEGER, + ADD COLUMN device_sync_data JSONB; + +-- ============================================ +-- DEVICE REGISTRY +-- ============================================ +CREATE TABLE devices ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE, + device_name VARCHAR(100) NOT NULL, + device_type VARCHAR(20) NOT NULL, -- 'koreader', 'kobo', 'web', 'mobile' + device_identifier VARCHAR(255) UNIQUE NOT NULL, + auth_token VARCHAR(500) UNIQUE NOT NULL, + last_sync TIMESTAMP, + last_seen TIMESTAMP, + sync_enabled BOOLEAN DEFAULT TRUE, + auto_sync BOOLEAN DEFAULT TRUE, + sync_frequency_minutes INTEGER DEFAULT 5, + device_metadata JSONB, + created_at TIMESTAMP DEFAULT NOW(), + updated_at TIMESTAMP DEFAULT NOW() +); + +CREATE INDEX idx_devices_user_id ON devices(user_id); +CREATE INDEX idx_devices_device_type ON devices(device_type); +CREATE INDEX idx_devices_device_identifier ON devices(device_identifier); + +-- ============================================ +-- SYNC QUEUE FOR OFFLINE SUPPORT +-- ============================================ +CREATE TABLE sync_queue ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + device_id UUID NOT NULL REFERENCES devices(id) ON DELETE CASCADE, + media_item_id UUID REFERENCES media_items(id) ON DELETE CASCADE, + sync_type VARCHAR(20) NOT NULL, -- 'progress', 'note', 'highlight', 'bookmark' + sync_data JSONB NOT NULL, + priority INTEGER DEFAULT 5, + attempts INTEGER DEFAULT 0, + max_attempts INTEGER DEFAULT 3, + status VARCHAR(20) DEFAULT 'pending', -- 'pending', 'processing', 'completed', 'failed' + error_message TEXT, + created_at TIMESTAMP DEFAULT NOW(), + processed_at TIMESTAMP +); + +CREATE INDEX idx_sync_queue_device_id ON sync_queue(device_id); +CREATE INDEX idx_sync_queue_status ON sync_queue(status); +CREATE INDEX idx_sync_queue_priority ON sync_queue(priority); + +-- ============================================ +-- CONFLICT RESOLUTION +-- ============================================ +CREATE TABLE sync_conflicts ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + media_item_id UUID NOT NULL REFERENCES media_items(id) ON DELETE CASCADE, + user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE, + conflict_type VARCHAR(20) NOT NULL, -- 'progress', 'note', 'highlight' + conflict_data JSONB NOT NULL, + resolution_status VARCHAR(20) DEFAULT 'unresolved', -- 'unresolved', 'auto_resolved', 'user_resolved' + resolution_data JSONB, + resolved_by UUID REFERENCES users(id), + resolved_at TIMESTAMP, + created_at TIMESTAMP DEFAULT NOW() +); + +CREATE INDEX idx_sync_conflicts_media_item_id ON sync_conflicts(media_item_id); +CREATE INDEX idx_sync_conflicts_user_id ON sync_conflicts(user_id); +CREATE INDEX idx_sync_conflicts_status ON sync_conflicts(resolution_status); + +-- ============================================ +-- READING HISTORY +-- ============================================ +CREATE TABLE reading_history ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE, + media_item_id UUID NOT NULL REFERENCES media_items(id) ON DELETE CASCADE, + device_id UUID REFERENCES devices(id), + progress_percentage FLOAT, + reading_session_start TIMESTAMP, + reading_session_end TIMESTAMP, + pages_read INTEGER, + time_spent_seconds INTEGER, + device_metadata JSONB, + created_at TIMESTAMP DEFAULT NOW() +); + +CREATE INDEX idx_reading_history_user_id ON reading_history(user_id); +CREATE INDEX idx_reading_history_media_item_id ON reading_history(media_item_id); +CREATE INDEX idx_reading_history_created_at ON reading_history(created_at DESC); +``` + +### Database Triggers for Automatic Timestamps + +```sql +-- Update devices updated_at timestamp +CREATE OR REPLACE FUNCTION update_updated_at_column() +RETURNS TRIGGER AS $$ +BEGIN + NEW.updated_at = NOW(); + RETURN NEW; +END; +$$ LANGUAGE plpgsql; + +CREATE TRIGGER update_devices_updated_at + BEFORE UPDATE ON devices + FOR EACH ROW + EXECUTE FUNCTION update_updated_at_column(); +``` + +--- + +## API Endpoints Specification + +### Authentication Endpoints + +#### Device Registration Flow +``` +POST /api/devices/register +Description: Register a new reading device + +Request Body: +{ + "device_name": "My Kobo Clara", + "device_type": "kobo|koreader|web|mobile", + "device_identifier": "unique-device-hardware-id" +} + +Response (201): +{ + "device_id": "uuid", + "auth_token": "Bearer token for device", + "setup_instructions": { + "kobo": "Sync URL: https://bookmann.example.com/api/sync/kobo", + "koreader": "Calibre URL: https://bookmann.example.com/api/sync/koreader" + }, + "qr_code_url": "https://bookmann.example.com/devices/qr/uuid" +} +``` + +#### Web Login for Device Authentication +``` +POST /api/devices/auth/web +Description: Authenticate device via web login (no API keys on device!) + +Request Body: +{ + "device_identifier": "device-unique-id", + "device_type": "kobo" +} + +Response (200): +{ + "auth_pending": true, + "auth_url": "https://bookmann.example.com/devices/auth/confirm/abc123", + "expires_in": 300, + "poll_interval": 3 +} + +[User visits auth URL on web, logs in, approves device] + +Device polls: +POST /api/devices/auth/status + +Response (200): +{ + "auth_complete": true, + "auth_token": "device-bearer-token", + "device_id": "uuid", + "sync_endpoints": { + "progress": "https://bookmann.example.com/api/sync/progress", + "metadata": "https://bookmann.example.com/api/sync/metadata", + "annotations": "https://bookmann.example.com/api/sync/annotations" + } +} +``` + +### Wireless Sync Endpoints + +#### KOReader Wireless Sync (Calibre-compatible) +``` +POST /api/sync/koreader/progress +Description: KOReader sends progress update + +Request Headers: + Authorization: Bearer {device_token} + Content-Type: application/json + +Request Body: +{ + "library_id": "optional-uuid", + "books": [ + { + "uuid": "book-uuid", + "title": "Book Title", + "authors": ["Author Name"], + "progress": 0.45, + "percentage": 0.45, + "last_read": "2026-01-30T20:00:00Z", + "device_info": { + "koreader_version": "2024.01", + "device_model": "kindle-paperwhite-5" + } + } + ] +} + +Response (202): +{ + "sync_status": "accepted", + "books_synced": 1, + "conflicts": [ + { + "book_uuid": "book-uuid", + "conflict_type": "progress_mismatch", + "device_progress": 0.45, + "server_progress": 0.42, + "resolution": "device_wins" + } + ] +} + +GET /api/sync/koreader/metadata/{book_uuid} +Description: KOReader fetches book metadata and sync status + +Response (200): +{ + "uuid": "book-uuid", + "title": "Book Title", + "authors": ["Author Name"], + "progress": { + "percentage": 0.42, + "character": 15432, + "epubcfi": "epubcfi(/6/4/2:15)", + "chapter": 3, + "chapter_progress": 0.234 + }, + "annotations": { + "highlights": [...], + "notes": [...], + "bookmarks": [...] + }, + "last_sync": "2026-01-30T20:00:00Z" +} +``` + +#### Kobo Wireless Sync (Kobo API-compatible) +``` +POST /api/sync/kobo/markup +Description: Kobo sends reading progress and annotations + +Request Headers: + Authorization: Bearer {device_token} + x-kobo-device: {"DeviceId":"device-id","Model":"Kobo Clara"} + +Request Body: +{ + "ReadingSync": [ + { + "ContentId": "book-uuid", + "PercentRead": 45.6, + "EntitlementId": "entitlement-id", + "RemainingTimeMinutes": 120, + "LastModified": "2026-01-30T20:00:00Z" + } + ], + "BookmarkSync": [ + { + "ContentId": "book-uuid", + "BookmarkText": "highlighted text", + "BookmarkType": "annotation", + "BookmarkTitle": "Chapter 3" + } + ] +} + +Response (200): +{ + "Status": "Success", + "MarkupsSynced": 5, + "BookmarksSynced": 3 +} + +GET /api/sync/kobo/library +Description: Kobo fetches library and sync status + +Response (200): +{ + "library_sync": [ + { + "ContentId": "book-uuid", + "ContentType": "6", + "Title": "Book Title", + "Author": "Author Name", + "PercentRead": 42.3, + "PagesRemaining": 115, + "BookmarkCount": 3, + "LastModified": "2026-01-30T20:00:00Z" + } + ] +} +``` + +### Universal Progress Endpoints + +#### Get Universal Progress +``` +GET /api/progress/{book_uuid} +Description: Get progress with all location references + +Response (200): +{ + "book_id": "book-uuid", + "format_group": "reflowable", + "universal_progress": 0.45678, + "location_references": { + "percentage": 0.45678, + "epubcfi": "epubcfi(/6/4/2:15)", + "character": 15432, + "chapter": 3, + "chapter_progress": 0.234, + "viewport_y": 0.12 + }, + "device_progress": { + "koreader": { + "percentage": 0.45678, + "last_sync": "2026-01-30T20:00:00Z" + }, + "kobo": { + "percentage": 45.6, + "last_sync": "2026-01-30T19:55:00Z" + }, + "web": { + "display_page": 89, + "total_pages": 200, + "last_sync": "2026-01-30T20:05:00Z" + } + }, + "annotations": { + "highlights": [...], + "notes": [...], + "bookmarks": [...] + }, + "conflicts": [ + { + "id": "conflict-uuid", + "type": "progress", + "resolved": false, + "sources": ["koreader", "kobo"] + } + ] +} +``` + +#### Update Universal Progress +``` +POST /api/progress/{book_uuid} +Description: Update progress with automatic conversion to all formats + +Request Body: +{ + "source": "web|koreader|kobo|mobile", + "location": { + "percentage": 0.45678, + "epubcfi": "epubcfi(/6/4/2:15)", + "character": 15432, + "chapter": 3, + "page": 89, + "total_pages": 200 + }, + "device_metadata": { + "device_type": "web", + "user_agent": "..." + } +} + +Response (200): +{ + "sync_status": "success", + "progress_updated": true, + "devices_notified": ["koreader-device-1", "kobo-device-2"], + "broadcast": true +} +``` + +### Conflict Resolution Endpoints + +#### List Conflicts +``` +GET /api/conflicts +Description: List all unresolved sync conflicts + +Query Parameters: + status: "unresolved|all" + type: "progress|note|highlight|all" + +Response (200): +{ + "conflicts": [ + { + "id": "conflict-uuid", + "media_item_id": "book-uuid", + "media_item_title": "Book Title", + "conflict_type": "progress", + "conflict_data": { + "koreader": { + "source": "koreader", + "timestamp": "2026-01-30T20:10:00Z", + "data": { + "percentage": 0.45, + "epubcfi": "epubcfi(/6/4/2:15)", + "character": 15432 + } + }, + "kobo": { + "source": "kobo", + "timestamp": "2026-01-30T20:05:00Z", + "data": { + "percentage": 0.42, + "location": "unknown" + } + } + }, + "resolution_status": "unresolved", + "created_at": "2026-01-30T20:10:05Z" + } + ], + "total": 1, + "unresolved": 1 +} +``` + +#### Resolve Conflict +``` +POST /api/conflicts/{conflict_id}/resolve +Description: Resolve a sync conflict by choosing source + +Request Body: +{ + "winner": "koreader|kobo|web|manual", + "manual_data": { + "percentage": 0.43, + "epubcfi": "epubcfi(/6/4/2:20)", + "character": 15500 + }, // Required if winner is "manual" + "apply_to_all_future_conflicts": false, + "reason": "user chose more recent progress" +} + +Response (200): +{ + "conflict_resolved": true, + "applied_to": { + "progress": true, + "annotations": false + }, + "devices_synced": ["koreader-device-1", "kobo-device-2"] +} +``` + +### Device Management Endpoints + +#### List User Devices +``` +GET /api/devices +Response (200): +{ + "devices": [ + { + "id": "device-uuid", + "device_name": "My Kobo Clara", + "device_type": "kobo", + "last_sync": "2026-01-30T20:00:00Z", + "last_seen": "2026-01-30T20:05:00Z", + "sync_enabled": true, + "auto_sync": true + } + ] +} +``` + +#### Update Device Settings +``` +PUT /api/devices/{device_id} +Request Body: +{ + "device_name": "Updated Name", + "sync_enabled": true, + "auto_sync": true, + "sync_frequency_minutes": 5 +} + +Response (200): +{ + "device_updated": true +} +``` + +#### Revoke Device Access +``` +DELETE /api/devices/{device_id} +Response (204) +``` + +--- + +## Format Grouping System + +### Three Format Groups + +Based on format capabilities and optimal tracking methods: + +#### 1. Reflowable Group +**Formats**: EPUB, MOBI, AZW3, FB2, TXT +**Characteristics**: +- Text reflows to fit viewport +- Page count changes based on display settings +- Can contain complex formatting and images +- Support for chapters, TOC + +**Optimal Tracking**: +- Percentage (0.0-1.0, 5 decimal precision) +- EPUB CFI for precise locations +- Chapter number + chapter progress +- Character offset from beginning +- Viewport Y position for visual reference + +#### 2. Fixed Layout Group +**Formats**: PDF, DJVU +**Characteristics**: +- Fixed page layout (like printed book) +- Consistent page count across devices +- Zoom and pan for navigation +- May contain text layer + +**Optimal Tracking**: +- Page number + total pages +- Page Y position (for scroll position) +- Zoom level +- Scroll X/Y coordinates +- Character offset in extracted text (if available) + +#### 3. Comic Archive Group +**Formats**: CBZ, CBR, CBT, CB7, PDF comics +**Characteristics**: +- Image-based pages +- No text layer (usually) +- Panel-based reading +- Single/double page modes +- Panel zoom features + +**Optimal Tracking**: +- Page number +- Panel number (for future panel navigation) +- Panel coordinates (for future use) +- Zoom level +- Pan X/Y coordinates +- Reading mode (single/double/panel) + +### Format Detection Logic + +```sql +CREATE OR REPLACE FUNCTION detect_format_group(mimetype VARCHAR, file_path VARCHAR) +RETURNS VARCHAR AS $$ +BEGIN + CASE + -- Reflowable formats + WHEN mimetype = 'application/epub+zip' THEN + RETURN 'reflowable'; + WHEN mimetype = 'application/x-mobipocket-ebook' THEN + RETURN 'reflowable'; + WHEN mimetype = 'application/vnd.amazon.mobi8-ebook' THEN + RETURN 'reflowable'; + WHEN file_path LIKE '%.epub' THEN + RETURN 'reflowable'; + WHEN file_path LIKE '%.mobi' THEN + RETURN 'reflowable'; + WHEN file_path LIKE '%.azw3' THEN + RETURN 'reflowable'; + WHEN file_path LIKE '%.fb2' THEN + RETURN 'reflowable'; + WHEN file_path LIKE '%.txt' THEN + RETURN 'reflowable'; + + -- Fixed layout formats + WHEN mimetype = 'application/pdf' THEN + RETURN 'fixed_layout'; + WHEN file_path LIKE '%.pdf' THEN + RETURN 'fixed_layout'; + WHEN file_path LIKE '%.djvu' THEN + RETURN 'fixed_layout'; + + -- Comic archive formats + WHEN mimetype = 'application/x-cbr' THEN + RETURN 'comic_archive'; + WHEN mimetype = 'application/x-cbz' THEN + RETURN 'comic_archive'; + WHEN file_path LIKE '%.cbz' THEN + RETURN 'comic_archive'; + WHEN file_path LIKE '%.cbr' THEN + RETURN 'comic_archive'; + WHEN file_path LIKE '%.cbt' THEN + RETURN 'comic_archive'; + WHEN file_path LIKE '%.cb7' THEN + RETURN 'comic_archive'; + + ELSE + RETURN 'unknown'; + END CASE; +END; +$$ LANGUAGE plpgsql; +``` + +--- + +## Progress Tracking by Format + +### Reflowable Format Progress + +```json +{ + "format_group": "reflowable", + "book_id": "uuid", + "progress": { + "percentage": 0.45678, + "epubcfi": "epubcfi(/6/4[chap01ref]!/4[body01]/10[para05]/2:15)", + "character": 15432, + "chapter": 3, + "chapter_progress": 0.234, + "viewport_y": 0.12, + "total_characters": 34567 + }, + "conversion_rules": { + "to_percentage": "character / total_characters", + "to_epubcfi": "use stored CFI or calculate from character", + "to_chapter": "parse from CFI or calculate from character offset", + "from_koreader": "direct percentage mapping", + "from_kobo": "percentage / 100" + }, + "display": { + "web": "45.7% (Chapter 3)", + "koreader": "45.7%", + "kobo": "46%" + } +} +``` + +### Fixed Layout Format Progress + +```json +{ + "format_group": "fixed_layout", + "book_id": "uuid", + "progress": { + "page": 45, + "total_pages": 200, + "page_y": 234, + "zoom": 1.25, + "scroll_x": 0, + "scroll_y": 0, + "character": 15432, + "percentage": 0.225 + }, + "conversion_rules": { + "to_percentage": "page / total_pages", + "to_page": "floor(percentage * total_pages)", + "to_character": "use OCR text extraction", + "from_koreader": "map percentage to nearest page", + "from_kobo": "direct page mapping" + }, + "display": { + "web": "Page 45 of 200 (22.5%)", + "koreader": "22.5%", + "kobo": "Page 45 (23%)" + } +} +``` + +### Comic Archive Format Progress + +```json +{ + "format_group": "comic_archive", + "book_id": "uuid", + "progress": { + "page": 12, + "total_pages": 32, + "panel": 5, + "panel_bounds": "120,85,300,250", + "zoom": 1.5, + "pan_x": 120, + "pan_y": 85, + "reading_mode": "single_page", + "percentage": 0.375 + }, + "conversion_rules": { + "to_percentage": "page / total_pages", + "to_page": "floor(percentage * total_pages)", + "from_koreader": "percentage to page mapping", + "from_kobo": "page mapping" + }, + "display": { + "web": "Page 12 of 32 (Panel 5)", + "koreader": "37.5%", + "kobo": "Page 12 (38%)" + } +} +``` + +### Progress Conversion Engine + +```sql +CREATE OR REPLACE FUNCTION convert_progress( + source_format VARCHAR, + target_format VARCHAR, + source_progress JSONB +) RETURNS JSONB AS $$ +DECLARE + result JSONB; + percentage FLOAT; +BEGIN + -- Extract percentage from source + CASE source_format + WHEN 'reflowable' THEN + percentage := (source_progress->>'percentage')::FLOAT; + WHEN 'fixed_layout' THEN + percentage := ((source_progress->>'page')::FLOAT / + (source_progress->>'total_pages')::FLOAT); + WHEN 'comic_archive' THEN + percentage := ((source_progress->>'page')::FLOAT / + (source_progress->>'total_pages')::FLOAT); + ELSE + percentage := 0.0; + END CASE; + + -- Build target format progress + CASE target_format + WHEN 'reflowable' THEN + result := jsonb_build_object( + 'percentage', percentage, + 'character', CAST(percentage * 34567 AS INTEGER), + 'epubcfi', 'epubcfi(/6/4/2:' || CAST(percentage * 100 AS INTEGER) || ')' + ); + WHEN 'fixed_layout' THEN + result := jsonb_build_object( + 'page', CAST(percentage * 200 AS INTEGER), + 'total_pages', 200, + 'percentage', percentage + ); + WHEN 'comic_archive' THEN + result := jsonb_build_object( + 'page', CAST(percentage * 32 AS INTEGER), + 'total_pages', 32, + 'percentage', percentage + ); + ELSE + result := '{}'::jsonb; + END CASE; + + RETURN result; +END; +$$ LANGUAGE plpgsql; +``` + +--- + +## Wireless Sync Protocols + +### KOReader Calibre-Compatible Protocol + +**Endpoint Structure**: +``` +POST /api/sync/koreader/progress +POST /api/sync/koreader/bookmarks +POST /api/sync/koreader/highlights +GET /api/sync/koreader/metadata +GET /api/sync/koreader/status +``` + +**Authentication**: Bearer token from device registration + +**Data Format** (KOReader → Bookmann): +```json +{ + "library_id": "optional-uuid", + "books": [ + { + "uuid": "book-uuid", + "title": "Book Title", + "authors": ["Author Name"], + "progress": 0.45, + "percentage": 0.45, + "last_read": "2026-01-30T20:00:00Z", + "bookmarks": [ + { + "chapter": 3, + "datetime": "2026-01-30T19:55:00Z", + "notes": "highlighted text", + "pos0": "epubcfi(/6/4/2:15)", + "pos1": "epubcfi(/6/4/2:20)", + "page": 45, + "text": "highlighted text excerpt", + "type": "highlight" + } + ], + "highlights": [...], + "notes": [...] + } + ] +} +``` + +**Immediate Sync on Page Turn**: +``` +POST /api/sync/koreader/progress +Content-Type: application/json + +{ + "sync_mode": "immediate", + "books": [ + { + "uuid": "book-uuid", + "percentage": 0.45678, + "chapter": 3, + "timestamp": "2026-01-30T20:00:00Z" + } + ] +} + +Response (202 Accepted): +{ + "sync_status": "accepted", + "broadcast_scheduled": true, + "queue_position": 1 +} +``` + +**Checkpoint Sync (Fallback)**: +``` +POST /api/sync/koreader/progress +Content-Type: application/json + +{ + "sync_mode": "checkpoint", + "checkpoint_id": "checkpoint-uuid", + "since_timestamp": "2026-01-30T19:00:00Z", + "books": [...] +} + +Response (200 OK): +{ + "sync_status": "completed", + "books_synced": 1, + "checkpoint_saved": true +} +``` + +### Kobo Sync Protocol (Reverse-Engineered) + +**Endpoint Structure**: +``` +POST /api/sync/kobo/markup +POST /api/sync/kobo/bookmark +GET /api/sync/kobo/library +GET /api/sync/kobo/status +``` + +**Authentication Headers**: +``` +Authorization: Bearer {device_token} +x-kobo-userid: {user_id} +x-kobo-device: {"DeviceId":"device-id","Model":"Kobo Clara","SerialNumber":"..."} +``` + +**Data Format** (Kobo → Bookmann): +```json +{ + "ReadingSync": [ + { + "ContentId": "book-uuid", + "PercentRead": 45.6, + "EntitlementId": "entitlement-id", + "RemainingTimeMinutes": 120, + "FirstReadTime": "2026-01-25T10:00:00Z", + "LastModified": "2026-01-30T20:00:00Z" + } + ], + "BookmarkSync": [ + { + "BookmarkId": "bookmark-uuid", + "ContentId": "book-uuid", + "BookmarkText": "highlighted text", + "BookmarkType": "annotation", + "BookmarkTitle": "Chapter 3", + "DateCreated": "2026-01-30T19:55:00Z", + "Chapter": 3, + "Hidden": false + } + ] +} +``` + +### Single Port Architecture + +All sync endpoints share the same port (8765) with proper routing: + +``` +Port 8765 (Bookmann) +├── /api/auth/* (Authentication) +├── /api/libraries/* (Library management) +├── /api/media-items/* (Media items) +├── /api/sync/koreader/* (KOReader wireless sync) +├── /api/sync/kobo/* (Kobo wireless sync) +├── /api/progress/* (Universal progress API) +├── /api/conflicts/* (Conflict resolution) +├── /api/devices/* (Device management) +└── /ws/sync (WebSocket for real-time updates) +``` + +**Caddy Proxy Configuration**: +``` +bookmann.example.com { + reverse_proxy localhost:8765 + encode gzip + log { + output file /var/log/caddy/bookmann-access.log + } +} +``` + +--- + +## Security Architecture + +### Device Authentication Flow + +**No API Keys on Devices!** All authentication goes through web interface. + +``` +1. Device generates unique identifier (hardware ID) +2. Device sends registration request with device type +3. Bookmann creates pending registration with auth URL +4. User visits auth URL in web browser +5. User logs in and approves device +6. Bookmann generates device token +7. Device polls for token approval +8. Device receives token and begins syncing +``` + +### Authentication Endpoints + +``` +POST /api/devices/register/initiate +Request: { "device_identifier": "hw-id-123", "device_type": "kobo" } +Response: { + "registration_id": "reg-uuid", + "auth_url": "https://bookmann.example.com/devices/auth/confirm/reg-uuid", + "expires_in": 300, + "qr_code": "data:image/png;base64,..." +} + +[User visits auth URL] + +POST /api/devices/register/status +Request: { "registration_id": "reg-uuid" } +Response: { + "status": "pending" | "approved" | "expired", + "auth_token": "Bearer token..." (when approved) +} +``` + +### Token Management + +**Device Token Structure**: +```json +{ + "device_id": "uuid", + "user_id": "uuid", + "device_type": "kobo|koreader|web|mobile", + "permissions": [ + "sync:progress", + "sync:annotations", + "sync:metadata" + ], + "expires": "never", + "revocable": true +} +``` + +### Rate Limiting + +``` +Per-device rate limits: +- Sync requests: 60/minute +- Progress updates: 120/minute (page turns) +- Metadata requests: 30/minute + +Per-user rate limits: +- All devices combined: 300/minute +- Conflict resolution: 10/minute + +Implementation: Redis-backed sliding window +``` + +### Request Validation + +```go +// Example Go validation for sync requests +func validateSyncRequest(deviceToken string, request SyncRequest) error { + // 1. Validate device token + device, err := validateDeviceToken(deviceToken) + if err != nil { + return ErrUnauthorized + } + + // 2. Check device not revoked + if device.Revoked { + return ErrDeviceRevoked + } + + // 3. Validate device permissions + if !hasPermission(device, "sync:progress") { + return ErrForbidden + } + + // 4. Validate request format + if err := validateRequestFormat(request); err != nil { + return ErrInvalidRequest + } + + // 5. Check rate limits + if exceededRateLimit(device.ID) { + return ErrRateLimited + } + + // 6. Validate media item ownership + if !ownsMediaItem(device.UserID, request.BookUUID) { + return ErrNotFound + } + + return nil +} +``` + +### CORS Configuration + +```go +// Allow device origins +allowedOrigins := []string{ + "https://bookmann.example.com", + "kobo://*", // Custom protocol for Kobo app + "koreader://*", // Custom protocol for KOReader +} + +corsConfig := cors.Config{ + AllowOrigins: allowedOrigins, + AllowMethods: []string{"GET", "POST", "PUT", "DELETE"}, + AllowHeaders: []string{"Authorization", "Content-Type"}, + ExposeHeaders: []string{"X-Sync-Status", "X-Conflict-Detected"}, + AllowCredentials: true, +} +``` + +--- + +## Conflict Resolution System + +### Conflict Detection + +**Automatic Conflict Detection**: +```sql +CREATE OR REPLACE FUNCTION detect_conflict( + media_item_id UUID, + new_progress JSONB, + device_type VARCHAR +) RETURNS BOOLEAN AS $$ +DECLARE + existing_progress JSONB; + time_diff INTERVAL; +BEGIN + -- Get most recent progress from different device + SELECT pg_jsonb INTO existing_progress + FROM reading_progress + WHERE media_item_id = $1 + AND last_sync_source != device_type + ORDER BY last_sync_timestamp DESC + LIMIT 1; + + -- If no existing progress, no conflict + IF existing_progress IS NULL THEN + RETURN FALSE; + END IF; + + -- Check time difference (within 5 minutes = potential conflict) + time_diff := NOW() - (existing_progress->>'timestamp')::TIMESTAMP; + + IF time_diff < INTERVAL '5 minutes' THEN + -- Check if progress significantly different + IF ABS((new_progress->>'percentage')::FLOAT - + (existing_progress->>'percentage')::FLOAT) > 0.01 THEN + RETURN TRUE; + END IF; + END IF; + + RETURN FALSE; +END; +$$ LANGUAGE plpgsql; +``` + +### Side-by-Side Conflict Display + +**UI Component Specification**: +``` +┌─────────────────────────────────────────────────────────┐ +│ ⚠️ Sync Conflict Detected: Book Title │ +├─────────────────────────────────────────────────────────┤ +│ │ +│ Compare progress from different devices: │ +│ │ +│ ┌─────────────────┬─────────────────────────────────┐ │ +│ │ KOReader │ Kobo Device │ │ +│ ├─────────────────┼─────────────────────────────────┤ │ +│ │ Progress: 45.7% │ Progress: 42.3% │ │ +│ │ Location: │ Location: Page 89/200 │ │ +│ │ Chapter 3 │ │ │ +│ │ CFI: ... │ Last Read: 2 hours ago │ │ +│ │ │ │ │ +│ │ Last Read: │ │ │ +│ │ 5 minutes ago │ │ │ +│ │ │ │ │ +│ │ Device: │ Device: Kobo Clara │ │ +│ │ Kindle PW5 │ │ │ +│ └─────────────────┴─────────────────────────────────┘ │ +│ │ +│ [Choose KOReader] [Choose Kobo] [Merge] [Dismiss] │ +│ │ +│ □ Auto-resolve future conflicts from [KOReader] │ +│ │ +└─────────────────────────────────────────────────────────┘ +``` + +### Conflict Resolution Logic + +**"Last In Wins" with User Override**: +``` +1. Each sync writes timestamp and source device +2. If conflict detected: + - Compare timestamps + - More recent progress wins by default + - Create conflict record for user review +3. User can: + - Accept auto-resolution + - Choose different source manually + - Merge progress (take max of both) + - Set auto-resolution preference +``` + +**Merge Logic**: +```sql +CREATE OR REPLACE FUNCTION merge_progress( + progress_a JSONB, + progress_b JSONB +) RETURNS JSONB AS $$ +BEGIN + RETURN jsonb_build_object( + 'percentage', GREATEST( + (progress_a->>'percentage')::FLOAT, + (progress_b->>'percentage')::FLOAT + ), + 'merged_from', ARRAY[ + (progress_a->>'source'), + (progress_b->>'source') + ], + 'merge_timestamp', NOW() + ); +END; +$$ LANGUAGE plpgsql; +``` + +--- + +## Backup & Recovery Strategy + +### Hybrid Queue + Checkpoint Strategy + +**Immediate Sync (Page Turn)**: +``` +Device → POST /api/sync/progress (sync_mode: immediate) + ↓ +Bookmann validates and stores + ↓ +WebSocket broadcast to all clients + ↓ +If network error → Queue locally with retry +``` + +**Checkpoint Sync (Fallback)**: +``` +Device → POST /api/sync/progress (sync_mode: checkpoint) + ↓ +Bookmann validates and stores + ↓ +Process queued updates since last checkpoint + ↓ +Clear queue on success + ↓ +On failure → Keep in queue with exponential backoff +``` + +### Sync Queue Management + +**Queue Schema**: +```sql +CREATE TABLE sync_queue ( + id UUID PRIMARY KEY, + device_id UUID NOT NULL, + media_item_id UUID, + sync_type VARCHAR(20) NOT NULL, + sync_data JSONB NOT NULL, + priority INTEGER DEFAULT 5, -- 1=highest, 10=lowest + attempts INTEGER DEFAULT 0, + max_attempts INTEGER DEFAULT 3, + status VARCHAR(20) DEFAULT 'pending', + error_message TEXT, + created_at TIMESTAMP DEFAULT NOW(), + next_retry_at TIMESTAMP, + processed_at TIMESTAMP +); +``` + +**Priority Levels**: +``` +Priority 1: User-initiated sync (manual refresh) +Priority 2: Book completion (100% progress) +Priority 3: Critical annotations (user notes) +Priority 5: Page turns (immediate sync) +Priority 7: Checkpoint sync (periodic) +Priority 10: Background metadata sync +``` + +**Exponential Backoff**: +``` +Attempt 1: Immediate +Attempt 2: 1 minute delay +Attempt 3: 5 minute delay +Attempt 4: 15 minute delay +Attempt 5: 1 hour delay +After 5 failed attempts: Mark as failed, notify user +``` + +### Offline Capability + +**Device-Side Queue** (KOReader example): +```lua +-- KOReader plugin code +local SyncQueue = { + queue = {}, + max_queue_size = 100, + + add = function(self, sync_data) + table.insert(self.queue, { + data = sync_data, + timestamp = os.time(), + retry_count = 0 + }) + self:persist() + end, + + sync = function(self) + for i, item in ipairs(self.queue) do + local success = self:send_to_server(item.data) + if success then + table.remove(self.queue, i) + else + item.retry_count = item.retry_count + 1 + if item.retry_count >= 3 then + self:notify_user(item) + table.remove(self.queue, i) + end + end + end + self:persist() + end, + + persist = function(self) + local file = io.open("/mnt/us/sync-queue.json", "w") + file:write(json.encode(self.queue)) + file:close() + end +} +``` + +### Recovery Scenarios + +**Scenario 1: Device Offline for Extended Period** +``` +1. Device queues all changes locally +2. User marks book as finished (100%) +3. High-priority sync queued +4. Device comes back online +5. Bookmann processes queue in priority order +6. Finishes book, updates all other devices +7. User notified of sync completion +``` + +**Scenario 2: Server Unavailable** +``` +1. All devices detect server unavailable +2. Switch to checkpoint mode (batch every 5 min) +3. Queue changes locally +4. When server returns: + - Process all checkpoints + - Resolve conflicts based on timestamps + - Notify user of conflicts +``` + +**Scenario 3: Multiple Devices Offline** +``` +1. All devices queue changes independently +2. Server processes queue in order of reconnection +3. Timestamps determine conflict resolution +4. User gets conflict summary for manual resolution +5. Auto-resolution preferences applied to future conflicts +``` + +--- + +## Real-time Synchronization + +### WebSocket Architecture + +**Connection Endpoint**: +``` +WS /ws/sync?token={device_token} +``` + +**Message Format**: +```json +{ + "type": "progress_update|annotation_update|conflict|sync_complete", + "timestamp": "2026-01-30T20:00:00Z", + "data": { + "book_id": "uuid", + "progress": {...}, + "annotations": {...} + }, + "source_device": { + "id": "device-uuid", + "name": "My Kobo", + "type": "kobo" + } +} +``` + +### Broadcast Strategy + +``` +Device A updates progress + ↓ +Bookmann stores in database + ↓ +WebSocket broadcast to: + - Web interface (if user logged in) + - Mobile apps (if connected) + - Other devices (on next sync) + ↓ +Devices update their display immediately +``` + +### Connection Management + +**Device Connection States**: +```go +type ConnectionManager struct { + connections map[string]*DeviceConnection + broadcast chan BroadcastMessage +} + +type DeviceConnection struct { + DeviceID string + UserID string + DeviceType string + Connected time.Time + LastPing time.Time + Send chan Message +} + +func (m *ConnectionManager) HandleConnection(conn *websocket.Conn) { + device := m.authenticate(conn) + m.connections[device.ID] = device + + // Send initial state + device.Send <- m.getInitialState(device.UserID) + + // Handle messages + for { + msg := <-device.Send + err := conn.WriteJSON(msg) + if err != nil { + break + } + } + + // Cleanup + delete(m.connections, device.ID) +} +``` + +### Heartbeat & Keep-Alive + +``` +Client → PING (every 30 seconds) +Server → PONG + +If no PING for 90 seconds → Close connection +``` + +--- + +## Device Onboarding Flow + +### Step-by-Step Device Setup + +#### KOReader Setup + +**1. Generate Registration Request** +``` +Device sends: POST /api/devices/register +{ + "device_type": "koreader", + "device_identifier": "hardware-specific-id", + "device_name": "My Kindle Paperwhite" +} +``` + +**2. Receive Registration Info** +``` +Server responds: { + "registration_id": "reg-uuid-123", + "auth_url": "https://bookmann.example.com/devices/auth/confirm/reg-uuid-123", + "qr_code": "data:image/png;base64,iVBORw0KG...", + "expires_in": 300 +} +``` + +**3. User Approves Device** +``` +User visits auth URL → Sees: +┌─────────────────────────────────────┐ +│ Device Registration Request │ +│ │ +│ Device Type: KOReader │ +│ Device Name: My Kindle Paperwhite │ +│ │ +│ Approve this device to sync your │ +│ reading progress and annotations? │ +│ │ +│ [Approve] [Deny] │ +└─────────────────────────────────────┘ +``` + +**4. Configure KOReader** +``` +In KOReader settings: +- Calibre sync: ON +- Server URL: https://bookmann.example.com/api/sync/koreader +- Wireless sync: ON +- Sync frequency: Every page turn +- Auto-sync: ON + +KOReader stores config in: +/mnt/us/settings/koreader/calibre.lua +``` + +**5. Device Polls for Token** +``` +Device: POST /api/devices/register/status +{ + "registration_id": "reg-uuid-123" +} + +Server responds: +{ + "status": "approved", + "auth_token": "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9...", + "sync_endpoints": { + "progress": "https://bookmann.example.com/api/sync/koreader/progress", + "metadata": "https://bookmann.example.com/api/sync/koreader/metadata", + "bookmarks": "https://bookmann.example.com/api/sync/koreader/bookmarks" + } +} +``` + +#### Kobo Setup + +**1-3**: Same as KOReader (registration & approval) + +**4. Configure Kobo Device** +``` +In Kobo settings: +- Sync server: https://bookmann.example.com/api/sync/kobo +- Auto-sync: ON +- Sync frequency: Every 5 minutes + +Kobo stores config in: +/mnt/onboard/.kobo/Kobo/Kobo eReader.conf +[Sync] +ServerURL=https://bookmann.example.com/api/sync/kobo +AutoSyncEnabled=true +SyncFrequency=5 +``` + +**5. Device Syncs** +``` +Kobo sends initial sync: +POST /api/sync/kobo/library +Authorization: Bearer {token} +x-kobo-device: {"DeviceId":"...", "Model":"Kobo Clara"} + +Server responds with library and current sync state +``` + +### Device Discovery (Optional) + +**mDNS/Bonjour for Local Network**: +``` +Bookmann broadcasts: _bookmann-sync._tcp.local +Port: 8765 +TXT: "path=/api/sync", "version=1.0" + +Devices discover automatically +User confirms connection +``` + +--- + +## Implementation Phases + +### Phase 1: Universal Progress System (Weeks 1-4) + +**Week 1: Database Schema** +- Add format_group columns to media_items +- Add universal progress columns to reading_progress +- Create device registry tables +- Create sync_queue tables +- Create sync_conflicts tables + +**Week 2: Format Detection & Progress Conversion** +- Implement format detection function +- Build progress conversion engine +- Update all existing progress to new format +- Create migration scripts + +**Week 3: Core Progress APIs** +- GET /api/progress/{book_uuid} +- POST /api/progress/{book_uuid} +- Progress conversion logic +- Format-specific display logic + +**Week 4: Testing & Validation** +- Unit tests for conversion engine +- Integration tests for progress APIs +- Test with existing data +- Performance testing + +**Deliverables**: +- Universal progress tracking working +- All existing progress converted +- Format-aware display in web UI + +### Phase 2: Device Management & Auth (Weeks 5-6) + +**Week 5: Device Registration** +- Device registry implementation +- Registration endpoints +- Web-based approval flow +- QR code generation +- Device management UI + +**Week 6: Authentication** +- Device token generation +- Token validation middleware +- Permission system +- Rate limiting per device +- Revocation system + +**Deliverables**: +- Users can register devices +- Devices authenticate via web flow +- Device management UI complete + +### Phase 3: KOReader Integration (Weeks 7-9) + +**Week 7: KOReader Protocol** +- Implement calibre-compatible endpoints +- Progress sync endpoint +- Metadata sync endpoint +- Bookmark/highlight sync + +**Week 8: Bidirectional Sync** +- Bookmann → KOReader sync +- KOReader → Bookmann sync +- Format conversion for KOReader +- Error handling & retry logic + +**Week 9: Real-time Updates** +- WebSocket implementation +- Progress broadcast system +- Connection management +- Testing with actual KOReader devices + +**Deliverables**: +- Full KOReader wireless sync +- Real-time progress updates +- Tested on actual devices + +### Phase 4: Kobo Integration (Weeks 10-12) + +**Week 10: Kobo Protocol** +- Reverse-engineer Kobo sync API +- Implement Kobo-compatible endpoints +- Markup sync endpoint +- Library sync endpoint + +**Week 11: Bidirectional Sync** +- Bookmann → Kobo sync +- Kobo → Bookmann sync +- Format conversion for Kobo +- Annotation sync + +**Week 12: Testing & Validation** +- Test on actual Kobo devices +- Performance testing +- Error handling +- Documentation + +**Deliverables**: +- Full Kobo wireless sync +- Tested on Kobo Clara/Naura/etc. +- Device setup documentation + +### Phase 5: Conflict Resolution (Weeks 13-14) + +**Week 13: Conflict Detection** +- Automatic conflict detection +- Conflict storage +- Conflict notifications +- Conflict listing API + +**Week 14: Resolution System** +- Side-by-side conflict UI +- Resolution endpoints +- Auto-resolution preferences +- Merge logic + +**Deliverables**: +- Complete conflict resolution system +- User-friendly conflict UI +- Auto-resolution preferences + +### Phase 6: Backup & Recovery (Weeks 15-16) + +**Week 15: Sync Queue** +- Queue implementation +- Priority system +- Retry logic with exponential backoff +- Device-side queue (KOReader plugin) + +**Week 16: Offline Support** +- Checkpoint sync mode +- Offline detection +- Recovery scenarios +- Progress merge logic + +**Deliverables**: +- Robust offline sync +- Recovery from extended offline +- Queue management UI + +### Phase 7: Polish & Documentation (Weeks 17-18) + +**Week 17: Performance & Security** +- Performance optimization +- Security audit +- Rate limiting tuning +- Load testing + +**Week 18: Documentation & Release** +- User documentation +- Developer documentation +- Device setup guides +- API documentation +- Release preparation + +**Deliverables**: +- Production-ready system +- Complete documentation +- Device setup guides +- API reference + +--- + +## Testing Requirements + +### Unit Tests + +**Format Detection**: +```go +TestFormatDetection_Epub() +TestFormatDetection_Mobi() +TestFormatDetection_Pdf() +TestFormatDetection_Cbz() +TestFormatDetection_Cbr() +TestFormatDetection_UnknownFormat() +``` + +**Progress Conversion**: +```go +TestProgressConversion_ReflowableToFixed() +TestProgressConversion_FixedToReflowable() +TestProgressConversion_ComicToReflowable() +TestProgressConversion_Precision() +TestProgressConversion_EdgeCases() +``` + +**Conflict Detection**: +```go +TestConflictDetection_SameTimestamp() +TestConflictDetection_DifferentDevices() +TestConflictDetection_SmallDifference() +TestConflictDetection_LargeDifference() +``` + +### Integration Tests + +**Sync Flow**: +```go +TestSyncFlow_KOReaderToBookmann() +TestSyncFlow_BookmannToKOReader() +TestSyncFlow_KoboToBookmann() +TestSyncFlow_Bidirectional() +TestSyncFlow_MultipleDevices() +``` + +**Conflict Resolution**: +```go +TestConflictResolution_AutoResolve() +TestConflictResolution_ManualResolve() +TestConflictResolution_Merge() +TestConflictResolution_UserPreference() +``` + +**Offline Recovery**: +```go +TestOfflineRecovery_QueueProcessing() +TestOfflineRecovery_PriorityOrder() +TestOfflineRecovery_ExponentialBackoff() +TestOfflineRecovery_ExtendedOffline() +``` + +### End-to-End Tests + +**User Scenarios**: +```go +TestScenario_ReadOnKOReaderContinueOnWeb() +TestScenario_ReadOnKoboContinueOnMobile() +TestScenario_MultipleDevicesConflict() +TestScenario_ExtendedOfflineSync() +TestScenario_BookCompletionAcrossDevices() +``` + +### Performance Tests + +**Load Testing**: +``` +- 1000 concurrent sync requests +- 100 devices syncing simultaneously +- 10,000 progress updates per minute +- WebSocket connection stability under load +``` + +**Database Performance**: +``` +- Query performance with 1M+ progress records +- Index effectiveness +- Transaction throughput +- Conflict detection performance +``` + +--- + +## Performance Considerations + +### Database Optimization + +**Indexes**: +```sql +-- Critical indexes for sync performance +CREATE INDEX idx_reading_progress_media_user + ON reading_progress(media_item_id, user_id); + +CREATE INDEX idx_reading_progress_last_sync + ON reading_progress(last_sync_timestamp DESC) + WHERE last_sync_timestamp > NOW() - INTERVAL '30 days'; + +CREATE INDEX idx_sync_queue_status_priority + ON sync_queue(status, priority) + WHERE status != 'completed'; + +CREATE INDEX idx_devices_active + ON devices(user_id, sync_enabled) + WHERE sync_enabled = true; +``` + +**Partitioning**: +```sql +-- Partition reading_history by month +CREATE TABLE reading_history_2026_01 + PARTITION OF reading_history + FOR VALUES FROM ('2026-01-01') TO ('2026-02-01'); +``` + +### Caching Strategy + +**Redis Cache**: +``` +- Device authentication tokens (TTL: 24 hours) +- Active sync sessions (TTL: 1 hour) +- Progress hot data (TTL: 5 minutes) +- Conflict data (TTL: 1 hour) +``` + +**Cache Invalidation**: +``` +- Progress update → invalidate progress cache +- New conflict → broadcast to all clients +- Device auth revoked → invalidate token cache +``` + +### Connection Pooling + +**Database Connections**: +``` +- Max connections: 100 +- Min connections: 10 +- Connection timeout: 30 seconds +- Query timeout: 10 seconds +``` + +**WebSocket Connections**: +``` +- Max concurrent connections: 1000 +- Heartbeat interval: 30 seconds +- Connection timeout: 90 seconds +- Message buffer size: 1000 messages +``` + +### Rate Limiting + +**Per-Device Limits**: +``` +- Sync requests: 60/minute +- Progress updates: 120/minute +- Metadata requests: 30/minute +- WebSocket messages: 300/minute +``` + +**Per-User Limits**: +``` +- Total requests: 300/minute +- Conflict resolutions: 10/minute +- Device registrations: 5/hour +``` + +--- + +## Conclusion + +This implementation guide provides a complete blueprint for creating a universal cross-platform reading synchronization system that: + +1. **Tracks progress optimally** for each format (reflowable, fixed, comics) +2. **Syncs wirelessly** with KOReader and Kobo devices +3. **Resolves conflicts** intelligently with user control +4. **Handles offline scenarios** gracefully with queueing +5. **Updates in real-time** via WebSocket broadcasts +6. **Authenticates securely** via web interface (no device API keys) +7. **Scales efficiently** with proper caching and optimization + +**Result**: Users get an Amazon Kindle-like experience - pick up any device, continue reading exactly where they left off, with all annotations synchronized - completely self-hosted and open-source. + +--- + +## Appendix: Example Data Migrations + +### Migrate Existing Page-Based Progress + +```sql +-- Convert existing progress to new format +UPDATE reading_progress +SET + percentage = CASE + WHEN total_pages > 0 THEN current_page::FLOAT / total_pages::FLOAT + ELSE 0 + END, + character_offset = current_page * 500, -- Rough estimate + chapter = CASE + WHEN current_page < 100 THEN 1 + WHEN current_page < 200 THEN 2 + ELSE 3 + END, + chapter_progress = 0.5, + last_sync_source = 'migration', + last_sync_timestamp = NOW() +WHERE percentage IS NULL; +``` + +### Update Media Items with Format Groups + +```sql +-- Set format groups based on file extensions +UPDATE media_items +SET + format_group = CASE + WHEN file_path LIKE '%.epub' THEN 'reflowable' + WHEN file_path LIKE '%.mobi' THEN 'reflowable' + WHEN file_path LIKE '%.pdf' THEN 'fixed_layout' + WHEN file_path LIKE '%.cbz' THEN 'comic_archive' + WHEN file_path LIKE '%.cbr' THEN 'comic_archive' + ELSE 'reflowable' -- Default + END, + is_reflowable = CASE + WHEN file_path LIKE '%.pdf' THEN FALSE + WHEN file_path LIKE '%.cbz' THEN FALSE + WHEN file_path LIKE '%.cbr' THEN FALSE + ELSE TRUE + END +WHERE format_group IS NULL OR format_group = ''; +``` + +--- + +**Document Version**: 1.0 +**Last Updated**: 2026-01-30 +**Maintained By**: Bookmann Development Team +**Status**: Ready for Implementation \ No newline at end of file diff --git a/internal/database/querier.go b/internal/database/querier.go index ae8d137..257d4fb 100644 --- a/internal/database/querier.go +++ b/internal/database/querier.go @@ -13,6 +13,8 @@ import ( type Querier interface { // Library Folders queries AddLibraryFolder(ctx context.Context, arg AddLibraryFolderParams) (LibraryFolders, error) + // Bulk update format group for all media items + BulkUpdateFormatGroups(ctx context.Context) error CleanupExpiredRefreshTokens(ctx context.Context) error // Backward compatibility - Ebook Notes queries (using views) CreateEbookNote(ctx context.Context, arg CreateEbookNoteParams) (MediaNotes, error) @@ -25,6 +27,8 @@ type Querier interface { // Media Notes queries CreateMediaNote(ctx context.Context, arg CreateMediaNoteParams) (MediaNotes, error) CreateMediaRating(ctx context.Context, arg CreateMediaRatingParams) (MediaRatings, error) + // Create reading history entry + CreateReadingHistory(ctx context.Context, arg CreateReadingHistoryParams) (ReadingHistory, error) // Refresh Tokens queries CreateRefreshToken(ctx context.Context, arg CreateRefreshTokenParams) (RefreshTokens, error) CreateUser(ctx context.Context, arg CreateUserParams) (CreateUserRow, error) @@ -53,9 +57,13 @@ type Querier interface { GetMediaNotes(ctx context.Context, arg GetMediaNotesParams) ([]MediaNotes, error) GetMediaRating(ctx context.Context, arg GetMediaRatingParams) (MediaRatings, error) GetMediaRatings(ctx context.Context, mediaItemID pgtype.UUID) ([]GetMediaRatingsRow, error) + // Get reading history for a user and book + GetReadingHistory(ctx context.Context, arg GetReadingHistoryParams) ([]ReadingHistory, error) GetReadingProgress(ctx context.Context, arg GetReadingProgressParams) (ReadingProgress, error) GetRefreshToken(ctx context.Context, token string) (GetRefreshTokenRow, error) GetScanSettings(ctx context.Context, id pgtype.UUID) (GetScanSettingsRow, error) + // Get universal progress for a book + GetUniversalProgress(ctx context.Context, arg GetUniversalProgressParams) (GetUniversalProgressRow, error) GetUser(ctx context.Context, id pgtype.UUID) (GetUserRow, error) GetUserByEmail(ctx context.Context, email string) (GetUserByEmailRow, error) GetUserByEmailOrUsername(ctx context.Context, email string) (GetUserByEmailOrUsernameRow, error) @@ -83,11 +91,18 @@ type Querier interface { UpdateLibrary(ctx context.Context, arg UpdateLibraryParams) (Libraries, error) UpdateMediaHighlight(ctx context.Context, arg UpdateMediaHighlightParams) (MediaHighlights, error) UpdateMediaItem(ctx context.Context, arg UpdateMediaItemParams) (MediaItems, error) + // ============================================ + // PHASE 1: FORMAT DETECTION & PROGRESS (Week 2) + // ============================================ + // Update media item format group information + UpdateMediaItemFormatGroup(ctx context.Context, arg UpdateMediaItemFormatGroupParams) error UpdateMediaNote(ctx context.Context, arg UpdateMediaNoteParams) (MediaNotes, error) UpdateMediaRating(ctx context.Context, arg UpdateMediaRatingParams) (MediaRatings, error) UpdatePassword(ctx context.Context, arg UpdatePasswordParams) error UpdateReadingProgress(ctx context.Context, arg UpdateReadingProgressParams) (ReadingProgress, error) UpdateScanSettings(ctx context.Context, arg UpdateScanSettingsParams) error + // Update universal progress + UpdateUniversalProgress(ctx context.Context, arg UpdateUniversalProgressParams) (ReadingProgress, error) UpdateUserProfile(ctx context.Context, arg UpdateUserProfileParams) error UpdateUserTheme(ctx context.Context, arg UpdateUserThemeParams) error UpdateUsername(ctx context.Context, arg UpdateUsernameParams) error diff --git a/internal/database/queries.sql.go b/internal/database/queries.sql.go index 5c30dee..f617486 100644 --- a/internal/database/queries.sql.go +++ b/internal/database/queries.sql.go @@ -33,6 +33,23 @@ func (q *Queries) AddLibraryFolder(ctx context.Context, arg AddLibraryFolderPara return i, err } +const BulkUpdateFormatGroups = `-- name: BulkUpdateFormatGroups :exec +UPDATE media_items m +SET + format_group = detect_format_group(m.mime_type, m.file_path), + format_mimetype = m.mime_type, + is_reflowable = (detect_format_group(m.mime_type, m.file_path) = 'reflowable'), + has_fixed_layout = (detect_format_group(m.mime_type, m.file_path) IN ('fixed_layout', 'comic_archive')), + updated_at = NOW() +WHERE m.format_group IS NULL OR m.format_group = 'unknown' +` + +// Bulk update format group for all media items +func (q *Queries) BulkUpdateFormatGroups(ctx context.Context) error { + _, err := q.db.Exec(ctx, BulkUpdateFormatGroups) + return err +} + const CleanupExpiredRefreshTokens = `-- name: CleanupExpiredRefreshTokens :exec DELETE FROM refresh_tokens WHERE expires_at < NOW() OR (revoked_at IS NOT NULL AND revoked_at < NOW() - INTERVAL '7 days') ` @@ -345,6 +362,64 @@ func (q *Queries) CreateMediaRating(ctx context.Context, arg CreateMediaRatingPa return i, err } +const CreateReadingHistory = `-- name: CreateReadingHistory :one +INSERT INTO reading_history ( + user_id, + media_item_id, + device_id, + progress_percentage, + reading_session_start, + reading_session_end, + pages_read, + time_spent_seconds, + device_metadata +) +VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) +RETURNING id, user_id, media_item_id, device_id, progress_percentage, reading_session_start, reading_session_end, pages_read, time_spent_seconds, device_metadata, created_at +` + +type CreateReadingHistoryParams struct { + UserID pgtype.UUID `db:"user_id" json:"user_id"` + MediaItemID pgtype.UUID `db:"media_item_id" json:"media_item_id"` + DeviceID pgtype.UUID `db:"device_id" json:"device_id"` + ProgressPercentage pgtype.Float8 `db:"progress_percentage" json:"progress_percentage"` + ReadingSessionStart pgtype.Timestamptz `db:"reading_session_start" json:"reading_session_start"` + ReadingSessionEnd pgtype.Timestamptz `db:"reading_session_end" json:"reading_session_end"` + PagesRead pgtype.Int4 `db:"pages_read" json:"pages_read"` + TimeSpentSeconds pgtype.Int4 `db:"time_spent_seconds" json:"time_spent_seconds"` + DeviceMetadata []byte `db:"device_metadata" json:"device_metadata"` +} + +// Create reading history entry +func (q *Queries) CreateReadingHistory(ctx context.Context, arg CreateReadingHistoryParams) (ReadingHistory, error) { + row := q.db.QueryRow(ctx, CreateReadingHistory, + arg.UserID, + arg.MediaItemID, + arg.DeviceID, + arg.ProgressPercentage, + arg.ReadingSessionStart, + arg.ReadingSessionEnd, + arg.PagesRead, + arg.TimeSpentSeconds, + arg.DeviceMetadata, + ) + var i ReadingHistory + err := row.Scan( + &i.ID, + &i.UserID, + &i.MediaItemID, + &i.DeviceID, + &i.ProgressPercentage, + &i.ReadingSessionStart, + &i.ReadingSessionEnd, + &i.PagesRead, + &i.TimeSpentSeconds, + &i.DeviceMetadata, + &i.CreatedAt, + ) + return i, err +} + const CreateRefreshToken = `-- name: CreateRefreshToken :one INSERT INTO refresh_tokens (user_id, token, expires_at) VALUES ($1, $2, $3) @@ -1029,6 +1104,53 @@ func (q *Queries) GetMediaRatings(ctx context.Context, mediaItemID pgtype.UUID) return items, nil } +const GetReadingHistory = `-- name: GetReadingHistory :many +SELECT id, user_id, media_item_id, device_id, progress_percentage, reading_session_start, reading_session_end, pages_read, time_spent_seconds, device_metadata, created_at +FROM reading_history +WHERE user_id = $1 AND media_item_id = $2 +ORDER BY created_at DESC +LIMIT $3 +` + +type GetReadingHistoryParams struct { + UserID pgtype.UUID `db:"user_id" json:"user_id"` + MediaItemID pgtype.UUID `db:"media_item_id" json:"media_item_id"` + Limit int32 `db:"limit" json:"limit"` +} + +// Get reading history for a user and book +func (q *Queries) GetReadingHistory(ctx context.Context, arg GetReadingHistoryParams) ([]ReadingHistory, error) { + rows, err := q.db.Query(ctx, GetReadingHistory, arg.UserID, arg.MediaItemID, arg.Limit) + if err != nil { + return nil, err + } + defer rows.Close() + items := []ReadingHistory{} + for rows.Next() { + var i ReadingHistory + if err := rows.Scan( + &i.ID, + &i.UserID, + &i.MediaItemID, + &i.DeviceID, + &i.ProgressPercentage, + &i.ReadingSessionStart, + &i.ReadingSessionEnd, + &i.PagesRead, + &i.TimeSpentSeconds, + &i.DeviceMetadata, + &i.CreatedAt, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + const GetReadingProgress = `-- name: GetReadingProgress :one SELECT id, media_item_id, user_id, current_page, total_pages, last_read_at, percentage, character_offset, epubcfi, chapter, chapter_progress, viewport_x, viewport_y, zoom_level, scroll_position_x, scroll_position_y, panel_number, reading_mode, last_sync_device, last_sync_source, last_sync_timestamp, conflict_detected, conflict_resolved FROM reading_progress WHERE media_item_id = $1 AND user_id = $2 ` @@ -1121,6 +1243,117 @@ func (q *Queries) GetScanSettings(ctx context.Context, id pgtype.UUID) (GetScanS return i, err } +const GetUniversalProgress = `-- name: GetUniversalProgress :one +SELECT + rp.id, + rp.media_item_id, + rp.user_id, + rp.current_page, + rp.total_pages, + rp.last_read_at, + rp.percentage, + rp.character_offset, + rp.epubcfi, + rp.chapter, + rp.chapter_progress, + rp.viewport_x, + rp.viewport_y, + rp.zoom_level, + rp.scroll_position_x, + rp.scroll_position_y, + rp.panel_number, + rp.reading_mode, + rp.last_sync_device, + rp.last_sync_source, + rp.last_sync_timestamp, + rp.conflict_detected, + rp.conflict_resolved, + mi.format_group, + mi.format_mimetype, + mi.is_reflowable, + mi.has_fixed_layout, + mi.total_characters, + mi.chapter_count +FROM reading_progress rp +JOIN media_items mi ON rp.media_item_id = mi.id +WHERE rp.media_item_id = $1 AND rp.user_id = $2 +` + +type GetUniversalProgressParams struct { + MediaItemID pgtype.UUID `db:"media_item_id" json:"media_item_id"` + UserID pgtype.UUID `db:"user_id" json:"user_id"` +} + +type GetUniversalProgressRow struct { + ID pgtype.UUID `db:"id" json:"id"` + MediaItemID pgtype.UUID `db:"media_item_id" json:"media_item_id"` + UserID pgtype.UUID `db:"user_id" json:"user_id"` + CurrentPage pgtype.Int4 `db:"current_page" json:"current_page"` + TotalPages pgtype.Int4 `db:"total_pages" json:"total_pages"` + LastReadAt pgtype.Timestamptz `db:"last_read_at" json:"last_read_at"` + Percentage pgtype.Float8 `db:"percentage" json:"percentage"` + CharacterOffset pgtype.Int8 `db:"character_offset" json:"character_offset"` + Epubcfi pgtype.Text `db:"epubcfi" json:"epubcfi"` + Chapter pgtype.Int4 `db:"chapter" json:"chapter"` + ChapterProgress pgtype.Float8 `db:"chapter_progress" json:"chapter_progress"` + ViewportX pgtype.Float8 `db:"viewport_x" json:"viewport_x"` + ViewportY pgtype.Float8 `db:"viewport_y" json:"viewport_y"` + ZoomLevel pgtype.Float8 `db:"zoom_level" json:"zoom_level"` + ScrollPositionX pgtype.Float8 `db:"scroll_position_x" json:"scroll_position_x"` + ScrollPositionY pgtype.Float8 `db:"scroll_position_y" json:"scroll_position_y"` + PanelNumber pgtype.Int4 `db:"panel_number" json:"panel_number"` + ReadingMode pgtype.Text `db:"reading_mode" json:"reading_mode"` + LastSyncDevice pgtype.Text `db:"last_sync_device" json:"last_sync_device"` + LastSyncSource pgtype.Text `db:"last_sync_source" json:"last_sync_source"` + LastSyncTimestamp pgtype.Timestamptz `db:"last_sync_timestamp" json:"last_sync_timestamp"` + ConflictDetected pgtype.Bool `db:"conflict_detected" json:"conflict_detected"` + ConflictResolved pgtype.Bool `db:"conflict_resolved" json:"conflict_resolved"` + FormatGroup string `db:"format_group" json:"format_group"` + FormatMimetype pgtype.Text `db:"format_mimetype" json:"format_mimetype"` + IsReflowable pgtype.Bool `db:"is_reflowable" json:"is_reflowable"` + HasFixedLayout pgtype.Bool `db:"has_fixed_layout" json:"has_fixed_layout"` + TotalCharacters pgtype.Int8 `db:"total_characters" json:"total_characters"` + ChapterCount pgtype.Int4 `db:"chapter_count" json:"chapter_count"` +} + +// Get universal progress for a book +func (q *Queries) GetUniversalProgress(ctx context.Context, arg GetUniversalProgressParams) (GetUniversalProgressRow, error) { + row := q.db.QueryRow(ctx, GetUniversalProgress, arg.MediaItemID, arg.UserID) + var i GetUniversalProgressRow + err := row.Scan( + &i.ID, + &i.MediaItemID, + &i.UserID, + &i.CurrentPage, + &i.TotalPages, + &i.LastReadAt, + &i.Percentage, + &i.CharacterOffset, + &i.Epubcfi, + &i.Chapter, + &i.ChapterProgress, + &i.ViewportX, + &i.ViewportY, + &i.ZoomLevel, + &i.ScrollPositionX, + &i.ScrollPositionY, + &i.PanelNumber, + &i.ReadingMode, + &i.LastSyncDevice, + &i.LastSyncSource, + &i.LastSyncTimestamp, + &i.ConflictDetected, + &i.ConflictResolved, + &i.FormatGroup, + &i.FormatMimetype, + &i.IsReflowable, + &i.HasFixedLayout, + &i.TotalCharacters, + &i.ChapterCount, + ) + return i, err +} + const GetUser = `-- name: GetUser :one SELECT id, email, username, theme, first_name, last_name, role, created_at, updated_at FROM users WHERE id = $1 ` @@ -2587,6 +2820,47 @@ func (q *Queries) UpdateMediaItem(ctx context.Context, arg UpdateMediaItemParams return i, err } +const UpdateMediaItemFormatGroup = `-- name: UpdateMediaItemFormatGroup :exec + +UPDATE media_items +SET + format_group = $2, + format_mimetype = $3, + is_reflowable = $4, + has_fixed_layout = $5, + total_characters = $6, + chapter_count = $7, + updated_at = NOW() +WHERE id = $1 +` + +type UpdateMediaItemFormatGroupParams struct { + ID pgtype.UUID `db:"id" json:"id"` + FormatGroup string `db:"format_group" json:"format_group"` + FormatMimetype pgtype.Text `db:"format_mimetype" json:"format_mimetype"` + IsReflowable pgtype.Bool `db:"is_reflowable" json:"is_reflowable"` + HasFixedLayout pgtype.Bool `db:"has_fixed_layout" json:"has_fixed_layout"` + TotalCharacters pgtype.Int8 `db:"total_characters" json:"total_characters"` + ChapterCount pgtype.Int4 `db:"chapter_count" json:"chapter_count"` +} + +// ============================================ +// PHASE 1: FORMAT DETECTION & PROGRESS (Week 2) +// ============================================ +// Update media item format group information +func (q *Queries) UpdateMediaItemFormatGroup(ctx context.Context, arg UpdateMediaItemFormatGroupParams) error { + _, err := q.db.Exec(ctx, UpdateMediaItemFormatGroup, + arg.ID, + arg.FormatGroup, + arg.FormatMimetype, + arg.IsReflowable, + arg.HasFixedLayout, + arg.TotalCharacters, + arg.ChapterCount, + ) + return err +} + const UpdateMediaNote = `-- name: UpdateMediaNote :one UPDATE media_notes SET content = $2, @@ -2735,6 +3009,127 @@ func (q *Queries) UpdateScanSettings(ctx context.Context, arg UpdateScanSettings return err } +const UpdateUniversalProgress = `-- name: UpdateUniversalProgress :one +INSERT INTO reading_progress ( + media_item_id, + user_id, + percentage, + character_offset, + epubcfi, + chapter, + chapter_progress, + viewport_x, + viewport_y, + zoom_level, + scroll_position_x, + scroll_position_y, + panel_number, + reading_mode, + last_sync_device, + last_sync_source, + last_sync_timestamp, + current_page, + total_pages, + last_read_at +) +VALUES ( + $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, NOW(), $17, $18, NOW() +) +ON CONFLICT (media_item_id, user_id) +DO UPDATE SET + percentage = EXCLUDED.percentage, + character_offset = EXCLUDED.character_offset, + epubcfi = EXCLUDED.epubcfi, + chapter = EXCLUDED.chapter, + chapter_progress = EXCLUDED.chapter_progress, + viewport_x = EXCLUDED.viewport_x, + viewport_y = EXCLUDED.viewport_y, + zoom_level = EXCLUDED.zoom_level, + scroll_position_x = EXCLUDED.scroll_position_x, + scroll_position_y = EXCLUDED.scroll_position_y, + panel_number = EXCLUDED.panel_number, + reading_mode = EXCLUDED.reading_mode, + last_sync_device = EXCLUDED.last_sync_device, + last_sync_source = EXCLUDED.last_sync_source, + last_sync_timestamp = EXCLUDED.last_sync_timestamp, + current_page = EXCLUDED.current_page, + total_pages = EXCLUDED.total_pages, + last_read_at = NOW() +RETURNING id, media_item_id, user_id, current_page, total_pages, last_read_at, percentage, character_offset, epubcfi, chapter, chapter_progress, viewport_x, viewport_y, zoom_level, scroll_position_x, scroll_position_y, panel_number, reading_mode, last_sync_device, last_sync_source, last_sync_timestamp, conflict_detected, conflict_resolved +` + +type UpdateUniversalProgressParams struct { + MediaItemID pgtype.UUID `db:"media_item_id" json:"media_item_id"` + UserID pgtype.UUID `db:"user_id" json:"user_id"` + Percentage pgtype.Float8 `db:"percentage" json:"percentage"` + CharacterOffset pgtype.Int8 `db:"character_offset" json:"character_offset"` + Epubcfi pgtype.Text `db:"epubcfi" json:"epubcfi"` + Chapter pgtype.Int4 `db:"chapter" json:"chapter"` + ChapterProgress pgtype.Float8 `db:"chapter_progress" json:"chapter_progress"` + ViewportX pgtype.Float8 `db:"viewport_x" json:"viewport_x"` + ViewportY pgtype.Float8 `db:"viewport_y" json:"viewport_y"` + ZoomLevel pgtype.Float8 `db:"zoom_level" json:"zoom_level"` + ScrollPositionX pgtype.Float8 `db:"scroll_position_x" json:"scroll_position_x"` + ScrollPositionY pgtype.Float8 `db:"scroll_position_y" json:"scroll_position_y"` + PanelNumber pgtype.Int4 `db:"panel_number" json:"panel_number"` + ReadingMode pgtype.Text `db:"reading_mode" json:"reading_mode"` + LastSyncDevice pgtype.Text `db:"last_sync_device" json:"last_sync_device"` + LastSyncSource pgtype.Text `db:"last_sync_source" json:"last_sync_source"` + CurrentPage pgtype.Int4 `db:"current_page" json:"current_page"` + TotalPages pgtype.Int4 `db:"total_pages" json:"total_pages"` +} + +// Update universal progress +func (q *Queries) UpdateUniversalProgress(ctx context.Context, arg UpdateUniversalProgressParams) (ReadingProgress, error) { + row := q.db.QueryRow(ctx, UpdateUniversalProgress, + arg.MediaItemID, + arg.UserID, + arg.Percentage, + arg.CharacterOffset, + arg.Epubcfi, + arg.Chapter, + arg.ChapterProgress, + arg.ViewportX, + arg.ViewportY, + arg.ZoomLevel, + arg.ScrollPositionX, + arg.ScrollPositionY, + arg.PanelNumber, + arg.ReadingMode, + arg.LastSyncDevice, + arg.LastSyncSource, + arg.CurrentPage, + arg.TotalPages, + ) + var i ReadingProgress + err := row.Scan( + &i.ID, + &i.MediaItemID, + &i.UserID, + &i.CurrentPage, + &i.TotalPages, + &i.LastReadAt, + &i.Percentage, + &i.CharacterOffset, + &i.Epubcfi, + &i.Chapter, + &i.ChapterProgress, + &i.ViewportX, + &i.ViewportY, + &i.ZoomLevel, + &i.ScrollPositionX, + &i.ScrollPositionY, + &i.PanelNumber, + &i.ReadingMode, + &i.LastSyncDevice, + &i.LastSyncSource, + &i.LastSyncTimestamp, + &i.ConflictDetected, + &i.ConflictResolved, + ) + return i, err +} + const UpdateUserProfile = `-- name: UpdateUserProfile :exec UPDATE users SET first_name = $2, last_name = $3, updated_at = NOW() WHERE id = $1 ` diff --git a/internal/database/queries/queries.sql b/internal/database/queries/queries.sql index ea04563..35c6951 100644 --- a/internal/database/queries/queries.sql +++ b/internal/database/queries/queries.sql @@ -484,4 +484,141 @@ UPDATE refresh_tokens SET revoked_at = NOW() WHERE user_id = $1 AND revoked_at I -- name: CleanupExpiredRefreshTokens :exec DELETE FROM refresh_tokens WHERE expires_at < NOW() OR (revoked_at IS NOT NULL AND revoked_at < NOW() - INTERVAL '7 days'); +-- ============================================ +-- PHASE 1: FORMAT DETECTION & PROGRESS (Week 2) +-- ============================================ + +-- Update media item format group information +-- name: UpdateMediaItemFormatGroup :exec +UPDATE media_items +SET + format_group = $2, + format_mimetype = $3, + is_reflowable = $4, + has_fixed_layout = $5, + total_characters = $6, + chapter_count = $7, + updated_at = NOW() +WHERE id = $1; + +-- Bulk update format group for all media items +-- name: BulkUpdateFormatGroups :exec +UPDATE media_items m +SET + format_group = detect_format_group(m.mime_type, m.file_path), + format_mimetype = m.mime_type, + is_reflowable = (detect_format_group(m.mime_type, m.file_path) = 'reflowable'), + has_fixed_layout = (detect_format_group(m.mime_type, m.file_path) IN ('fixed_layout', 'comic_archive')), + updated_at = NOW() +WHERE m.format_group IS NULL OR m.format_group = 'unknown'; + +-- Get universal progress for a book +-- name: GetUniversalProgress :one +SELECT + rp.id, + rp.media_item_id, + rp.user_id, + rp.current_page, + rp.total_pages, + rp.last_read_at, + rp.percentage, + rp.character_offset, + rp.epubcfi, + rp.chapter, + rp.chapter_progress, + rp.viewport_x, + rp.viewport_y, + rp.zoom_level, + rp.scroll_position_x, + rp.scroll_position_y, + rp.panel_number, + rp.reading_mode, + rp.last_sync_device, + rp.last_sync_source, + rp.last_sync_timestamp, + rp.conflict_detected, + rp.conflict_resolved, + mi.format_group, + mi.format_mimetype, + mi.is_reflowable, + mi.has_fixed_layout, + mi.total_characters, + mi.chapter_count +FROM reading_progress rp +JOIN media_items mi ON rp.media_item_id = mi.id +WHERE rp.media_item_id = $1 AND rp.user_id = $2; + +-- Update universal progress +-- name: UpdateUniversalProgress :one +INSERT INTO reading_progress ( + media_item_id, + user_id, + percentage, + character_offset, + epubcfi, + chapter, + chapter_progress, + viewport_x, + viewport_y, + zoom_level, + scroll_position_x, + scroll_position_y, + panel_number, + reading_mode, + last_sync_device, + last_sync_source, + last_sync_timestamp, + current_page, + total_pages, + last_read_at +) +VALUES ( + $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, NOW(), $17, $18, NOW() +) +ON CONFLICT (media_item_id, user_id) +DO UPDATE SET + percentage = EXCLUDED.percentage, + character_offset = EXCLUDED.character_offset, + epubcfi = EXCLUDED.epubcfi, + chapter = EXCLUDED.chapter, + chapter_progress = EXCLUDED.chapter_progress, + viewport_x = EXCLUDED.viewport_x, + viewport_y = EXCLUDED.viewport_y, + zoom_level = EXCLUDED.zoom_level, + scroll_position_x = EXCLUDED.scroll_position_x, + scroll_position_y = EXCLUDED.scroll_position_y, + panel_number = EXCLUDED.panel_number, + reading_mode = EXCLUDED.reading_mode, + last_sync_device = EXCLUDED.last_sync_device, + last_sync_source = EXCLUDED.last_sync_source, + last_sync_timestamp = EXCLUDED.last_sync_timestamp, + current_page = EXCLUDED.current_page, + total_pages = EXCLUDED.total_pages, + last_read_at = NOW() +RETURNING *; + +-- Get reading history for a user and book +-- name: GetReadingHistory :many +SELECT * +FROM reading_history +WHERE user_id = $1 AND media_item_id = $2 +ORDER BY created_at DESC +LIMIT $3; + +-- Create reading history entry +-- name: CreateReadingHistory :one +INSERT INTO reading_history ( + user_id, + media_item_id, + device_id, + progress_percentage, + reading_session_start, + reading_session_end, + pages_read, + time_spent_seconds, + device_metadata +) +VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) +RETURNING *; + -- Media Items Admin Operations \ No newline at end of file diff --git a/internal/sync/format.go b/internal/sync/format.go new file mode 100644 index 0000000..7ffd47e --- /dev/null +++ b/internal/sync/format.go @@ -0,0 +1,132 @@ +package sync + +import ( + "path/filepath" + "strings" +) + +// FormatGroup represents the three main format categories +type FormatGroup string + +const ( + FormatGroupReflowable FormatGroup = "reflowable" + FormatGroupFixedLayout FormatGroup = "fixed_layout" + FormatGroupComicArchive FormatGroup = "comic_archive" + FormatGroupUnknown FormatGroup = "unknown" +) + +// MimeType mappings for common ebook formats +var mimeTypes = map[string]string{ + ".epub": "application/epub+zip", + ".mobi": "application/x-mobipocket-ebook", + ".azw": "application/x-mobipocket-ebook", + ".azw3": "application/vnd.amazon.mobi8-ebook", + ".pdf": "application/pdf", + ".djvu": "image/vnd.djvu", + ".cbz": "application/x-cbz", + ".cbr": "application/x-cbr", + ".cb7": "application/x-cb7", + ".cbt": "application/x-cbt", + ".fb2": "application/x-fictionbook+xml", + ".txt": "text/plain", + ".rtf": "application/rtf", + ".doc": "application/msword", + ".docx": "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + ".lit": "application/x-ms-reader", + ".pdb": "application/vnd.palm", + ".prc": "application/vnd.palm", +} + +// ReflowableFormats are formats that support text reflow +var ReflowableFormats = map[string]bool{ + ".epub": true, + ".mobi": true, + ".azw": true, + ".azw3": true, + ".fb2": true, + ".txt": true, + ".rtf": true, + ".doc": true, + ".docx": true, + ".lit": true, + ".pdb": true, + ".prc": true, +} + +// FixedLayoutFormats are formats with fixed page layouts +var FixedLayoutFormats = map[string]bool{ + ".pdf": true, + ".djvu": true, +} + +// ComicArchiveFormats are comic archive formats +var ComicArchiveFormats = map[string]bool{ + ".cbz": true, + ".cbr": true, + ".cb7": true, + ".cbt": true, +} + +// DetectFormatGroup determines the format group based on mimetype and file path +func DetectFormatGroup(mimetype string, filePath string) FormatGroup { + ext := strings.ToLower(filepath.Ext(filePath)) + + // Check by mimetype first + switch mimetype { + case "application/epub+zip", + "application/x-mobipocket-ebook", + "application/vnd.amazon.mobi8-ebook", + "application/x-fictionbook+xml", + "text/plain": + return FormatGroupReflowable + + case "application/pdf", + "image/vnd.djvu": + return FormatGroupFixedLayout + + case "application/x-cbr", + "application/x-cbz", + "application/x-cb7", + "application/x-cbt": + return FormatGroupComicArchive + } + + // Fall back to file extension + if ReflowableFormats[ext] { + return FormatGroupReflowable + } + + if FixedLayoutFormats[ext] { + return FormatGroupFixedLayout + } + + if ComicArchiveFormats[ext] { + return FormatGroupComicArchive + } + + return FormatGroupUnknown +} + +// GetMimeType returns the mimetype for a given file extension +func GetMimeType(filePath string) string { + ext := strings.ToLower(filepath.Ext(filePath)) + if mt, ok := mimeTypes[ext]; ok { + return mt + } + return "" +} + +// IsReflowable checks if a format is reflowable +func IsReflowable(formatGroup FormatGroup) bool { + return formatGroup == FormatGroupReflowable +} + +// HasFixedLayout checks if a format has fixed layout +func HasFixedLayout(formatGroup FormatGroup) bool { + return formatGroup == FormatGroupFixedLayout +} + +// IsComicArchive checks if a format is a comic archive +func IsComicArchive(formatGroup FormatGroup) bool { + return formatGroup == FormatGroupComicArchive +} diff --git a/internal/sync/progress.go b/internal/sync/progress.go new file mode 100644 index 0000000..e350e62 --- /dev/null +++ b/internal/sync/progress.go @@ -0,0 +1,255 @@ +package sync + +import ( + "encoding/json" + "fmt" + "math" +) + +// ProgressData represents universal progress data with multiple location references +type ProgressData struct { + Percentage *float64 `json:"percentage,omitempty"` + Epubcfi *string `json:"epubcfi,omitempty"` + Character *int64 `json:"character,omitempty"` + Chapter *int `json:"chapter,omitempty"` + ChapterProgress *float64 `json:"chapter_progress,omitempty"` + ViewportY *float64 `json:"viewport_y,omitempty"` + Page *int `int,omitempty"` + TotalPages *int `int,omitempty"` + PageY *int `int,omitempty"` + Zoom *float64 `json:"zoom,omitempty"` + ScrollX *float64 `json:"scroll_x,omitempty"` + ScrollY *float64 `json:"scroll_y,omitempty"` + Panel *int `json:"panel,omitempty"` + ReadingMode *string `json:"reading_mode,omitempty"` + TotalCharacters *int64 `json:"total_characters,omitempty"` +} + +// DeviceProgress represents progress from a specific device +type DeviceProgress struct { + Source string `json:"source"` + Data ProgressData `json:"data"` + Timestamp string `json:"timestamp,omitempty"` +} + +// ConvertProgress converts progress between different format groups +func ConvertProgress(sourceFormat, targetFormat FormatGroup, sourceData map[string]interface{}) (map[string]interface{}, error) { + percentage := extractPercentage(sourceFormat, sourceData) + + result := make(map[string]interface{}) + + switch targetFormat { + case FormatGroupReflowable: + result["percentage"] = percentage + if epubcfi, ok := sourceData["epubcfi"].(string); ok { + result["epubcfi"] = epubcfi + } else { + // Generate approximate CFI from percentage + result["epubcfi"] = fmt.Sprintf("epubcfi(/6/4/2:%d)", int(percentage*100)) + } + if totalChars, ok := sourceData["total_characters"].(int64); ok { + result["character"] = int64(float64(totalChars) * percentage) + } + + case FormatGroupFixedLayout: + totalPages := 200.0 + if tp, ok := sourceData["total_pages"].(int); ok { + totalPages = float64(tp) + } + result["page"] = int(math.Round(percentage * totalPages)) + result["total_pages"] = int(totalPages) + result["percentage"] = percentage + if pageY, ok := sourceData["page_y"].(int); ok { + result["page_y"] = pageY + } + + case FormatGroupComicArchive: + totalPages := 32.0 + if tp, ok := sourceData["total_pages"].(int); ok { + totalPages = float64(tp) + } + result["page"] = int(math.Round(percentage * totalPages)) + result["total_pages"] = int(totalPages) + result["percentage"] = percentage + if panel, ok := sourceData["panel"].(int); ok { + result["panel"] = panel + } + + default: + return nil, fmt.Errorf("unsupported target format: %s", targetFormat) + } + + return result, nil +} + +// extractPercentage extracts the percentage (0.0-1.0) from source data +func extractPercentage(sourceFormat FormatGroup, sourceData map[string]interface{}) float64 { + switch sourceFormat { + case FormatGroupReflowable: + if p, ok := sourceData["percentage"].(float64); ok { + return p + } + // Try to calculate from character offset + if char, ok := sourceData["character"].(int64); ok { + if total, ok := sourceData["total_characters"].(int64); ok && total > 0 { + return float64(char) / float64(total) + } + } + + case FormatGroupFixedLayout, FormatGroupComicArchive: + if page, ok := sourceData["page"].(int); ok { + if total, ok := sourceData["total_pages"].(int); ok && total > 0 { + return float64(page) / float64(total) + } + } + // Try direct percentage + if p, ok := sourceData["percentage"].(float64); ok { + return p + } + } + + return 0.0 +} + +// PageToPercentage converts page/total_pages to percentage +func PageToPercentage(page, totalPages int) float64 { + if totalPages <= 0 { + return 0.0 + } + percentage := float64(page) / float64(totalPages) + if percentage > 1.0 { + percentage = 1.0 + } + if percentage < 0.0 { + percentage = 0.0 + } + return percentage +} + +// PercentageToPage converts percentage to page number +func PercentageToPage(percentage float64, totalPages int) int { + if percentage < 0.0 { + percentage = 0.0 + } + if percentage > 1.0 { + percentage = 1.0 + } + page := int(math.Round(float64(totalPages) * percentage)) + if page < 0 { + page = 0 + } + if page > totalPages { + page = totalPages + } + return page +} + +// CharacterToPercentage converts character offset to percentage +func CharacterToPercentage(character, totalCharacters int64) float64 { + if totalCharacters <= 0 { + return 0.0 + } + percentage := float64(character) / float64(totalCharacters) + if percentage > 1.0 { + percentage = 1.0 + } + if percentage < 0.0 { + percentage = 0.0 + } + return percentage +} + +// PercentageToCharacter converts percentage to character offset +func PercentageToCharacter(percentage float64, totalCharacters int64) int64 { + if percentage < 0.0 { + percentage = 0.0 + } + if percentage > 1.0 { + percentage = 1.0 + } + char := int64(math.Round(float64(totalCharacters) * percentage)) + if char < 0 { + char = 0 + } + if char > totalCharacters { + char = totalCharacters + } + return char +} + +// MergeProgress merges progress from two sources using "max progress wins" strategy +func MergeProgress(progressA, progressB map[string]interface{}) map[string]interface{} { + percA := extractPercentage(FormatGroupReflowable, progressA) + percB := extractPercentage(FormatGroupReflowable, progressB) + + winner := progressB + if percA > percB { + winner = progressA + } + + result := make(map[string]interface{}) + for k, v := range winner { + result[k] = v + } + + sources := []string{} + if srcA, ok := progressA["source"].(string); ok { + sources = append(sources, srcA) + } + if srcB, ok := progressB["source"].(string); ok { + sources = append(sources, srcB) + } + result["merged_from"] = sources + result["merge_timestamp"] = "now" + + return result +} + +// FormatProgressForDisplay formats progress for display based on format group +func FormatProgressForDisplay(formatGroup FormatGroup, progress map[string]interface{}) string { + percentage := extractPercentage(formatGroup, progress) + + switch formatGroup { + case FormatGroupReflowable: + if chapter, ok := progress["chapter"].(int); ok { + return fmt.Sprintf("%.1f%% (Chapter %d)", percentage*100, chapter) + } + return fmt.Sprintf("%.1f%%", percentage*100) + + case FormatGroupFixedLayout: + if page, ok := progress["page"].(int); ok { + if total, ok := progress["total_pages"].(int); ok { + return fmt.Sprintf("Page %d of %d (%.1f%%)", page, total, percentage*100) + } + return fmt.Sprintf("Page %d (%.1f%%)", page, percentage*100) + } + return fmt.Sprintf("%.1f%%", percentage*100) + + case FormatGroupComicArchive: + if page, ok := progress["page"].(int); ok { + if total, ok := progress["total_pages"].(int); ok { + return fmt.Sprintf("Page %d of %d (%.0f%%)", page, total, percentage*100) + } + return fmt.Sprintf("Page %d (%.0f%%)", page, percentage*100) + } + return fmt.Sprintf("%.0f%%", percentage*100) + + default: + return fmt.Sprintf("%.1f%%", percentage*100) + } +} + +// ParseProgressFromJSON parses progress data from JSON +func ParseProgressFromJSON(data []byte) (*ProgressData, error) { + var progress ProgressData + err := json.Unmarshal(data, &progress) + if err != nil { + return nil, err + } + return &progress, nil +} + +// MarshalProgressToJSON converts progress data to JSON +func MarshalProgressToJSON(progress *ProgressData) ([]byte, error) { + return json.Marshal(progress) +}