From d28002e4cbd117218c00ba70bf7b1ac85f81644d Mon Sep 17 00:00:00 2001 From: John O'Keefe Date: Sat, 31 Jan 2026 22:32:09 -0500 Subject: [PATCH] feat(sync): Add Kobo/Koreader sync and conflict handling - Add Kobo markup/sync endpoints for bookshelves - Add Koreader progress sync with SHA-256 support - Add sync conflict detection and resolution - Update ebook scanner for better file matching --- internal/handlers/conflicts.go | 30 ++- internal/handlers/kobo.go | 268 ++++++++++++++++++++++---- internal/handlers/koreader.go | 290 ++++++++++++++++++++--------- internal/services/ebook_scanner.go | 274 ++++++++++++++++++++++++++- 4 files changed, 728 insertions(+), 134 deletions(-) diff --git a/internal/handlers/conflicts.go b/internal/handlers/conflicts.go index e9df7fa..579b9f7 100644 --- a/internal/handlers/conflicts.go +++ b/internal/handlers/conflicts.go @@ -64,7 +64,7 @@ type ConflictResolveResponse struct { DevicesSynced []string `json:"devices_synced"` } -func (h *ConflictHandler) ListConflicts(c echo.Context) error { +func (h *ConflictHandler) GetConflictsData(c echo.Context) ([]ConflictDetailResponse, int, int, error) { user := c.Get("user").(database.Users) status := c.QueryParam("status") @@ -80,14 +80,11 @@ func (h *ConflictHandler) ListConflicts(c echo.Context) error { }) if err != nil && err != pgx.ErrNoRows { - return echo.NewHTTPError(http.StatusInternalServerError, "failed to list conflicts") + return nil, 0, 0, err } - response := ConflictListResponse{ - Conflicts: make([]ConflictDetailResponse, 0), - Total: len(conflicts), - Unresolved: 0, - } + response := make([]ConflictDetailResponse, 0, len(conflicts)) + unresolvedCount := 0 for _, conflict := range conflicts { var conflictData map[string]ConflictSourceData @@ -116,13 +113,26 @@ func (h *ConflictHandler) ListConflicts(c echo.Context) error { } } - response.Conflicts = append(response.Conflicts, detail) + response = append(response, detail) if conflict.ResolutionStatus.String == "unresolved" { - response.Unresolved++ + unresolvedCount++ } } - return c.JSON(http.StatusOK, response) + return response, len(response), unresolvedCount, nil +} + +func (h *ConflictHandler) ListConflicts(c echo.Context) error { + conflicts, total, unresolved, err := h.GetConflictsData(c) + if err != nil { + return echo.NewHTTPError(http.StatusInternalServerError, "failed to list conflicts") + } + + return c.JSON(http.StatusOK, ConflictListResponse{ + Conflicts: conflicts, + Total: total, + Unresolved: unresolved, + }) } func (h *ConflictHandler) GetConflict(c echo.Context) error { diff --git a/internal/handlers/kobo.go b/internal/handlers/kobo.go index 1284cd6..65e94bb 100644 --- a/internal/handlers/kobo.go +++ b/internal/handlers/kobo.go @@ -3,9 +3,12 @@ package handlers import ( "bookmann/internal/database" wsync "bookmann/internal/sync" + "crypto/sha256" + "encoding/hex" "encoding/json" "fmt" "net/http" + "regexp" "time" "github.com/google/uuid" @@ -22,6 +25,161 @@ func NewKoboHandler(db *database.Queries, connManager *wsync.ConnectionManager) return &KoboHandler{db: db, connManager: connManager} } +// mapContentIdToBookmannUUID maps Kobo ContentId to Bookmann UUID with multiple fallback strategies +// Phase 6: Enhanced Kobo Sync - ContentId Mapping Logic +func (h *KoboHandler) mapContentIdToBookmannUUID(ctx echo.Context, contentId string, deviceID uuid.UUID) (uuid.UUID, error, string) { + // Step 1: Try direct ContentId lookup in device_catalogs table + catalog, err := h.db.GetDeviceCatalogByKoboContentId(ctx.Request().Context(), contentId) + if err == nil && catalog.ID.Valid { + // Found! Use canonical Bookmann UUID + return uuid.UUID(catalog.BookmannUuid.Bytes), nil, "catalog_match" + } + + // Step 2: ContentId not found - check if it looks like a SHA-256 hash + if len(contentId) == 64 && looksLikeSHA256(contentId) { + // Try to find media item by SHA-256 + mediaItem, err := h.db.GetMediaItemBySHA256(ctx.Request().Context(), pgtype.Text{String: contentId, Valid: true}) + if err == nil { + // Found by SHA-256! Create device catalog entry for future lookups + _, _ = h.db.CreateDeviceCatalog(ctx.Request().Context(), database.CreateDeviceCatalogParams{ + DeviceID: pgtype.UUID{Bytes: deviceID, Valid: true}, + MediaItemID: pgtype.UUID{Bytes: mediaItem.ID.Bytes, Valid: true}, + BookmannUuid: pgtype.UUID{Bytes: mediaItem.ID.Bytes, Valid: true}, + KoboContentID: contentId, + ContentIDType: pgtype.Text{String: "sha256", Valid: true}, + Available: pgtype.Bool{Bool: true, Valid: true}, + DeliveryDate: pgtype.Timestamptz{Time: time.Now(), Valid: true}, + DeliveryMethod: pgtype.Text{String: "sync", Valid: true}, + }) + return uuid.UUID(mediaItem.ID.Bytes), nil, "sha256_match" + } + } + + // Step 3: Try to parse as UUID directly + if parsedUUID, err := uuid.Parse(contentId); err == nil { + // Check if this UUID exists in media_items + mediaItem, err := h.db.GetMediaItem(ctx.Request().Context(), pgtype.UUID{Bytes: parsedUUID, Valid: true}) + if err == nil { + // Create device catalog entry + _, _ = h.db.CreateDeviceCatalog(ctx.Request().Context(), database.CreateDeviceCatalogParams{ + DeviceID: pgtype.UUID{Bytes: deviceID, Valid: true}, + MediaItemID: pgtype.UUID{Bytes: mediaItem.ID.Bytes, Valid: true}, + BookmannUuid: pgtype.UUID{Bytes: mediaItem.ID.Bytes, Valid: true}, + KoboContentID: contentId, + ContentIDType: pgtype.Text{String: "bookmann_uuid", Valid: true}, + Available: pgtype.Bool{Bool: true, Valid: true}, + DeliveryDate: pgtype.Timestamptz{Time: time.Now(), Valid: true}, + DeliveryMethod: pgtype.Text{String: "sync", Valid: true}, + }) + return parsedUUID, nil, "uuid_match" + } + } + + // Step 4: Not found - return error for unlinked book + return uuid.Nil, fmt.Errorf("unlinked book: ContentId %s not found", contentId), "unlinked" +} + +// mapBookmannUUIDToKoboContentId maps Bookmann UUID to Kobo ContentId +// Creates new entry in device_catalogs if not exists +func (h *KoboHandler) mapBookmannUUIDToKoboContentId(c echo.Context, bookmannUUID uuid.UUID, deviceID uuid.UUID) (string, error) { + // Check if catalog entry already exists + catalog, err := h.db.GetDeviceCatalogByBookmannUUID(c.Request().Context(), database.GetDeviceCatalogByBookmannUUIDParams{ + DeviceID: pgtype.UUID{Bytes: deviceID, Valid: true}, + BookmannUuid: pgtype.UUID{Bytes: bookmannUUID, Valid: true}, + }) + if err == nil && catalog.ID.Valid { + return catalog.KoboContentID, nil + } + + // Get media item to check for existing Kobo content ID + mediaItem, err := h.db.GetMediaItem(c.Request().Context(), pgtype.UUID{Bytes: bookmannUUID, Valid: true}) + if err != nil { + return "", err + } + + // Generate Kobo ContentId based on priority: + // 1. Use existing kobo_content_id from media_items + // 2. Use existing entitlement_id from media_items + // 3. Generate new "kobo_" prefixed UUID + var koboContentId string + contentIdType := "bookmann_generated" + + if mediaItem.KoboContentID.Valid && mediaItem.KoboContentID.String != "" { + koboContentId = mediaItem.KoboContentID.String + contentIdType = "kobo_metadata" + } else if mediaItem.EntitlementID.Valid && mediaItem.EntitlementID.String != "" { + koboContentId = mediaItem.EntitlementID.String + contentIdType = "entitlement_id" + } else { + koboContentId = "kobo_" + uuid.New().String() + contentIdType = "kobo_generated" + } + + // Create device catalog entry + _, err = h.db.CreateDeviceCatalog(c.Request().Context(), database.CreateDeviceCatalogParams{ + DeviceID: pgtype.UUID{Bytes: deviceID, Valid: true}, + MediaItemID: pgtype.UUID{Bytes: bookmannUUID, Valid: true}, + BookmannUuid: pgtype.UUID{Bytes: bookmannUUID, Valid: true}, + KoboContentID: koboContentId, + ContentIDType: pgtype.Text{String: contentIdType, Valid: true}, + Available: pgtype.Bool{Bool: true, Valid: true}, + DeliveryDate: pgtype.Timestamptz{Time: time.Now(), Valid: true}, + DeliveryMethod: pgtype.Text{String: "opds", Valid: true}, + }) + + if err != nil { + return "", err + } + + return koboContentId, nil +} + +// getCollectionMetadataForBook retrieves collection names for a book +func (h *KoboHandler) getCollectionMetadataForBook(c echo.Context, bookmannUUID uuid.UUID, deviceID uuid.UUID) ([]string, error) { + device := c.Get("device").(database.Devices) + pgDeviceID := pgtype.UUID{Bytes: device.ID.Bytes, Valid: true} + + // Get collections for this book + collections, err := h.db.GetCollectionsForBook(c.Request().Context(), pgtype.UUID{Bytes: bookmannUUID, Valid: true}) + if err != nil { + return []string{}, nil + } + + var collectionNames []string + + // For each collection, check if there's a device-specific shelf mapping + for _, collection := range collections { + mapping, err := h.db.GetDeviceShelfMapping(c.Request().Context(), database.GetDeviceShelfMappingParams{ + DeviceID: pgDeviceID, + CollectionID: pgtype.UUID{Bytes: collection.ID.Bytes, Valid: true}, + }) + if err == nil && mapping.ID.Valid && mapping.DeviceShelfName.Valid { + // Use device-specific shelf name + collectionNames = append(collectionNames, mapping.DeviceShelfName.String) + } else if collection.Name != "" { + // Fall back to collection name + collectionNames = append(collectionNames, collection.Name) + } + } + + return collectionNames, nil +} + +// looksLikeSHA256 checks if a string looks like a SHA-256 hash +func looksLikeSHA256(s string) bool { + if len(s) != 64 { + return false + } + matched, _ := regexp.MatchString("^[0-9a-fA-F]{64}$", s) + return matched +} + +// calculateFileSHA256 calculates SHA-256 hash of file path +func calculateFileSHA256(filePath string) string { + hash := sha256.Sum256([]byte(filePath)) + return hex.EncodeToString(hash[:]) +} + type KoboDeviceInfo struct { DeviceID string `json:"DeviceId"` Model string `json:"Model"` @@ -55,18 +213,20 @@ type KoboMarkupRequest struct { } type KoboLibraryBook struct { - ContentId string `json:"ContentId"` - ContentType string `json:"ContentType"` - Title string `json:"Title"` - Author string `json:"Author"` - PercentRead float64 `json:"PercentRead"` - PagesRemaining *int `json:"PagesRemaining,omitempty"` - BookmarkCount int `json:"BookmarkCount"` - LastModified string `json:"LastModified"` - EntitlementId string `json:"EntitlementId,omitempty"` - Revision int `json:"Revision"` - MimeType string `json:"MimeType"` - FileSize int64 `json:"FileSize"` + ContentId string `json:"ContentId"` + ContentType string `json:"ContentType"` + Title string `json:"Title"` + Author string `json:"Author"` + PercentRead float64 `json:"PercentRead"` + PagesRemaining *int `json:"PagesRemaining,omitempty"` + BookmarkCount int `json:"BookmarkCount"` + LastModified string `json:"LastModified"` + EntitlementId string `json:"EntitlementId,omitempty"` + Revision int `json:"Revision"` + MimeType string `json:"MimeType"` + FileSize int64 `json:"FileSize"` + Categories []string `json:"Categories,omitempty"` + BookmannUUID string `json:"BookmannUUID,omitempty"` } type KoboLibraryResponse struct { @@ -110,8 +270,10 @@ type KoboAnalyticsTest struct { func (h *KoboHandler) Initialization(c echo.Context) error { device := c.Get("device").(database.Devices) userID := device.UserID.Bytes + deviceID := device.ID.Bytes pgUserID := pgtype.UUID{Bytes: userID, Valid: true} + deviceUUID := uuid.UUID(deviceID) mediaItems, err := h.db.GetUserMediaItemsForSync(c.Request().Context(), pgUserID) if err != nil { @@ -122,6 +284,8 @@ func (h *KoboHandler) Initialization(c echo.Context) error { librarySync := []KoboLibraryBook{} for _, item := range mediaItems { + bookmannUUID := uuid.UUID(item.ID.Bytes) + progress, _ := h.db.GetUniversalProgress(c.Request().Context(), database.GetUniversalProgressParams{ MediaItemID: pgtype.UUID{Bytes: item.ID.Bytes, Valid: true}, UserID: pgUserID, @@ -154,10 +318,15 @@ func (h *KoboHandler) Initialization(c echo.Context) error { author = item.Author.String } - entitlementId := item.EntitlementID.String - - if entitlementId == "" || !item.EntitlementID.Valid { - entitlementId = "kobo_" + uuid.UUID(item.ID.Bytes).String() + // Phase 6: Use ContentId mapping instead of direct UUID + koboContentId, err := h.mapBookmannUUIDToKoboContentId(c, bookmannUUID, deviceUUID) + if err != nil { + // Fallback to entitlement_id or generate new one + if item.EntitlementID.Valid && item.EntitlementID.String != "" { + koboContentId = item.EntitlementID.String + } else { + koboContentId = "kobo_" + bookmannUUID.String() + } } mimeType := item.MimeType.String @@ -175,8 +344,11 @@ func (h *KoboHandler) Initialization(c echo.Context) error { revision = int(item.RevisionNumber.Int32) } + // Phase 6: Get collection metadata for this book + collections, _ := h.getCollectionMetadataForBook(c, bookmannUUID, deviceUUID) + librarySync = append(librarySync, KoboLibraryBook{ - ContentId: uuid.UUID(item.ID.Bytes).String(), + ContentId: koboContentId, ContentType: "6", Title: item.Title, Author: author, @@ -184,10 +356,12 @@ func (h *KoboHandler) Initialization(c echo.Context) error { PagesRemaining: pagesRemaining, BookmarkCount: bookmarkCount, LastModified: lastModified, - EntitlementId: entitlementId, + EntitlementId: koboContentId, Revision: revision, MimeType: mimeType, FileSize: fileSize, + Categories: collections, + BookmannUUID: bookmannUUID.String(), }) } @@ -205,8 +379,10 @@ func (h *KoboHandler) LibrarySync(c echo.Context) error { func (h *KoboHandler) Markup(c echo.Context) error { device := c.Get("device").(database.Devices) userID := device.UserID.Bytes + deviceID := device.ID.Bytes pgUserID := pgtype.UUID{Bytes: userID, Valid: true} + deviceUUID := uuid.UUID(deviceID) var req KoboMarkupRequest if err := c.Bind(&req); err != nil { @@ -217,14 +393,19 @@ func (h *KoboHandler) Markup(c echo.Context) error { markupsSynced := 0 bookmarksSynced := 0 + unlinkedBooks := 0 for _, readingSync := range req.ReadingSync { - mediaUUID, err := uuid.Parse(readingSync.ContentId) + // Phase 6: Use ContentId mapping with fallback logic + bookmannUUID, err, _ := h.mapContentIdToBookmannUUID(c, readingSync.ContentId, deviceUUID) if err != nil { + // Unlinked book detected + unlinkedBooks++ + // TODO: Create unlinked book entry for manual resolution continue } - pgMediaUUID := pgtype.UUID{Bytes: mediaUUID, Valid: true} + pgMediaUUID := pgtype.UUID{Bytes: bookmannUUID, Valid: true} percentage := readingSync.PercentRead / 100.0 @@ -240,7 +421,7 @@ func (h *KoboHandler) Markup(c echo.Context) error { markupsSynced++ h.connManager.BroadcastProgressUpdate( - mediaUUID, + bookmannUUID, percentage, wsync.SourceDevice{ ID: uuid.UUID(userID).String(), @@ -252,12 +433,14 @@ func (h *KoboHandler) Markup(c echo.Context) error { } for _, bookmarkSync := range req.BookmarkSync { - mediaUUID, err := uuid.Parse(bookmarkSync.ContentId) + // Phase 6: Use ContentId mapping with fallback logic + bookmannUUID, err, _ := h.mapContentIdToBookmannUUID(c, bookmarkSync.ContentId, deviceUUID) if err != nil { + // Unlinked book - skip continue } - pgMediaUUID := pgtype.UUID{Bytes: mediaUUID, Valid: true} + pgMediaUUID := pgtype.UUID{Bytes: bookmannUUID, Valid: true} switch bookmarkSync.BookmarkType { case "annotation": @@ -292,18 +475,29 @@ func (h *KoboHandler) Markup(c echo.Context) error { }) } - return c.JSON(http.StatusOK, KoboSyncStatus{ + response := KoboSyncStatus{ Status: "Success", MarkupsSynced: markupsSynced, BookmarksSynced: bookmarksSynced, - }) + } + + // Include unlinked books count if any + if unlinkedBooks > 0 { + // For now, just log it. In production, this should trigger an alert + // and the unlinked books should be stored for manual resolution + response.Status = "Partial" + } + + return c.JSON(http.StatusOK, response) } func (h *KoboHandler) Bookmark(c echo.Context) error { device := c.Get("device").(database.Devices) userID := device.UserID.Bytes + deviceID := device.ID.Bytes pgUserID := pgtype.UUID{Bytes: userID, Valid: true} + deviceUUID := uuid.UUID(deviceID) var req struct { BookmarkSync []KoboBookmarkSync `json:"BookmarkSync"` @@ -317,12 +511,14 @@ func (h *KoboHandler) Bookmark(c echo.Context) error { bookmarksSynced := 0 for _, bookmarkSync := range req.BookmarkSync { - mediaUUID, err := uuid.Parse(bookmarkSync.ContentId) + // Phase 6: Use ContentId mapping with fallback logic + bookmannUUID, err, _ := h.mapContentIdToBookmannUUID(c, bookmarkSync.ContentId, deviceUUID) if err != nil { + // Unlinked book - skip continue } - pgMediaUUID := pgtype.UUID{Bytes: mediaUUID, Valid: true} + pgMediaUUID := pgtype.UUID{Bytes: bookmannUUID, Valid: true} switch bookmarkSync.BookmarkType { case "annotation": @@ -367,8 +563,10 @@ func (h *KoboHandler) Bookmark(c echo.Context) error { func (h *KoboHandler) AnalyticsGettests(c echo.Context) error { device := c.Get("device").(database.Devices) userID := device.UserID.Bytes + deviceID := device.ID.Bytes pgUserID := pgtype.UUID{Bytes: userID, Valid: true} + deviceUUID := uuid.UUID(deviceID) var req []KoboAnalyticsTest if err := c.Bind(&req); err != nil { @@ -378,12 +576,14 @@ func (h *KoboHandler) AnalyticsGettests(c echo.Context) error { } for _, test := range req { - mediaUUID, err := uuid.Parse(test.ContentId) + // Phase 6: Use ContentId mapping with fallback logic + bookmannUUID, err, _ := h.mapContentIdToBookmannUUID(c, test.ContentId, deviceUUID) if err != nil { + // Unlinked book - skip continue } - pgMediaUUID := pgtype.UUID{Bytes: mediaUUID, Valid: true} + pgMediaUUID := pgtype.UUID{Bytes: bookmannUUID, Valid: true} percentage := test.PercentRead / 100.0 _, err = h.db.UpdateUniversalProgress(c.Request().Context(), database.UpdateUniversalProgressParams{ @@ -396,7 +596,7 @@ func (h *KoboHandler) AnalyticsGettests(c echo.Context) error { if err == nil { h.connManager.BroadcastProgressUpdate( - mediaUUID, + bookmannUUID, percentage, wsync.SourceDevice{ ID: uuid.UUID(userID).String(), @@ -436,8 +636,10 @@ func parseKoboDeviceHeader(c echo.Context) (KoboDeviceInfo, error) { func (h *KoboHandler) SyncFromServer(c echo.Context) error { device := c.Get("device").(database.Devices) userID := device.UserID.Bytes + deviceID := device.ID.Bytes pgUserID := pgtype.UUID{Bytes: userID, Valid: true} + deviceUUID := uuid.UUID(deviceID) var req []KoboServerSyncData if err := c.Bind(&req); err != nil { @@ -451,12 +653,14 @@ func (h *KoboHandler) SyncFromServer(c echo.Context) error { highlightsSent := 0 for _, syncData := range req { - mediaUUID, err := uuid.Parse(syncData.ContentId) + // Phase 6: Use ContentId mapping with fallback logic + bookmannUUID, err, _ := h.mapContentIdToBookmannUUID(c, syncData.ContentId, deviceUUID) if err != nil { + // Unlinked book - skip continue } - pgMediaUUID := pgtype.UUID{Bytes: mediaUUID, Valid: true} + pgMediaUUID := pgtype.UUID{Bytes: bookmannUUID, Valid: true} percentage := syncData.PercentRead / 100.0 diff --git a/internal/handlers/koreader.go b/internal/handlers/koreader.go index feb1410..341024d 100644 --- a/internal/handlers/koreader.go +++ b/internal/handlers/koreader.go @@ -32,6 +32,7 @@ type KOReaderProgressRequest struct { type KOReaderBookProgress struct { UUID string `json:"uuid,omitempty"` + SHA256 string `json:"sha256,omitempty"` Title string `json:"title,omitempty"` Authors []string `json:"authors,omitempty"` Progress float64 `json:"progress"` @@ -64,6 +65,7 @@ type KOReaderBookmark struct { Text string `json:"text,omitempty"` Type string `json:"type,omitempty"` Percentage *float64 `json:"percentage,omitempty"` + BookSHA256 string `json:"book_sha256,omitempty"` } type KOReaderHighlight struct { @@ -77,6 +79,7 @@ type KOReaderHighlight struct { Type string `json:"type,omitempty"` Color string `json:"color,omitempty"` Percentage *float64 `json:"percentage,omitempty"` + BookSHA256 string `json:"book_sha256,omitempty"` } type KOReaderNote struct { @@ -89,6 +92,7 @@ type KOReaderNote struct { Text string `json:"text,omitempty"` Type string `json:"type,omitempty"` Percentage *float64 `json:"percentage,omitempty"` + BookSHA256 string `json:"book_sha256,omitempty"` } type KOReaderSyncResponse struct { @@ -182,46 +186,14 @@ func (h *KOReaderHandler) SyncProgress(c echo.Context) error { conflicts := []KOReaderConflict{} for _, book := range req.Books { - var mediaUUID uuid.UUID - var err error + mediaItemID, _ := h.resolveBookToMediaItem(c, device.ID, pgUserID, book) + if !mediaItemID.Valid { + continue + } - if book.UUID != "" { - mediaUUID, err = uuid.Parse(book.UUID) - if err != nil { - continue - } - - mediaItem, err := h.db.GetMediaItem(c.Request().Context(), pgtype.UUID{Bytes: mediaUUID, Valid: true}) - if err == nil { - err = h.updateProgressForBook(c, pgUserID, mediaItem.ID, book) - if err == nil { - booksSynced++ - } - } - } else if book.FilePath != "" { - mediaItem, err := h.db.GetMediaItemByFilePath(c.Request().Context(), book.FilePath) - if err == nil { - err = h.updateProgressForBook(c, pgUserID, mediaItem.ID, book) - if err == nil { - booksSynced++ - } - } - } else if book.Title != "" { - mediaItems, err := h.db.ListMediaItems(c.Request().Context(), database.ListMediaItemsParams{ - Limit: 100, - Offset: 0, - }) - if err == nil { - for _, mi := range mediaItems { - if mi.Title == book.Title && (book.Authors == nil || mi.Author.String == book.Authors[0]) { - err = h.updateProgressForBook(c, pgUserID, mi.ID, book) - if err == nil { - booksSynced++ - } - break - } - } - } + err := h.updateProgressForBook(c, pgUserID, mediaItemID, book) + if err == nil { + booksSynced++ } } @@ -251,50 +223,136 @@ func (h *KOReaderHandler) SyncProgress(c echo.Context) error { }) } +func (h *KOReaderHandler) resolveBookToMediaItem(c echo.Context, deviceID pgtype.UUID, userID pgtype.UUID, book KOReaderBookProgress) (pgtype.UUID, float64) { + ctx := c.Request().Context() + + // Priority 1: UUID provided (highest confidence - 1.0) + if book.UUID != "" { + mediaUUID, err := uuid.Parse(book.UUID) + if err == nil { + mediaItem, err := h.db.GetMediaItem(ctx, pgtype.UUID{Bytes: mediaUUID, Valid: true}) + if err == nil { + // Create device file alias if FilePath is also provided + if book.FilePath != "" { + h.createDeviceFileAlias(c, deviceID, mediaItem.ID, book) + } + return mediaItem.ID, 1.0 + } + } + } + + // Priority 2: SHA-256 provided (medium confidence - 0.9) + if book.SHA256 != "" && len(book.SHA256) == 64 { + mediaItem, err := h.db.GetMediaItemBySHA256(ctx, pgtype.Text{String: book.SHA256, Valid: true}) + if err == nil { + // Create device file alias if FilePath is provided + if book.FilePath != "" { + h.createDeviceFileAlias(c, deviceID, mediaItem.ID, book) + } + return mediaItem.ID, 0.9 + } + } + + // Priority 3: FilePath provided (check existing alias or create new - confidence 0.7) + if book.FilePath != "" { + // Check if alias already exists + alias, err := h.db.GetDeviceFileAlias(ctx, database.GetDeviceFileAliasParams{ + DeviceID: deviceID, + FilePath: book.FilePath, + }) + if err == nil { + return alias.MediaItemID, alias.ConfidenceScore.Float64 + } + + // Try to find by file path + mediaItem, err := h.db.GetMediaItemByFilePath(ctx, book.FilePath) + if err == nil { + // Create new alias + confidence := 0.7 + h.createDeviceFileAlias(c, deviceID, mediaItem.ID, book) + return mediaItem.ID, confidence + } + } + + // Priority 4: Title + Author match (lowest confidence - 0.5) + if book.Title != "" { + mediaItems, err := h.db.ListMediaItems(ctx, database.ListMediaItemsParams{ + Limit: 100, + Offset: 0, + }) + if err == nil { + for _, mi := range mediaItems { + if mi.Title == book.Title { + // Check author match if provided + if len(book.Authors) > 0 && mi.Author.Valid { + if mi.Author.String == book.Authors[0] { + // Create device file alias if FilePath is provided + if book.FilePath != "" { + h.createDeviceFileAlias(c, deviceID, mi.ID, book) + } + return mi.ID, 0.5 + } + } else { + // Title only match + if book.FilePath != "" { + h.createDeviceFileAlias(c, deviceID, mi.ID, book) + } + return mi.ID, 0.4 + } + } + } + } + } + + return pgtype.UUID{}, 0.0 +} + +func (h *KOReaderHandler) createDeviceFileAlias(c echo.Context, deviceID pgtype.UUID, mediaItemID pgtype.UUID, book KOReaderBookProgress) { + ctx := c.Request().Context() + + if book.FilePath == "" { + return + } + + sha256 := book.SHA256 + if sha256 == "" { + // Try to get SHA256 from media item + mediaItem, err := h.db.GetMediaItem(ctx, mediaItemID) + if err == nil && mediaItem.FileSha256.Valid { + sha256 = mediaItem.FileSha256.String + } + } + + confidence := 0.7 + // Check if alias already exists + _, err := h.db.GetDeviceFileAlias(ctx, database.GetDeviceFileAliasParams{ + DeviceID: deviceID, + FilePath: book.FilePath, + }) + if err != nil { + // Create new alias + _, _ = h.db.CreateDeviceFileAlias(ctx, database.CreateDeviceFileAliasParams{ + MediaItemID: mediaItemID, + DeviceID: deviceID, + FilePath: book.FilePath, + FileSha256: pgtype.Text{String: sha256, Valid: sha256 != ""}, + ConfidenceScore: pgtype.Float8{Float64: confidence, Valid: true}, + }) + } +} + func (h *KOReaderHandler) handleCheckpointSync(c echo.Context, device database.Devices, userID pgtype.UUID, req KOReaderProgressRequest) error { booksEnqueued := 0 for _, book := range req.Books { - var mediaUUID uuid.UUID - var err error + mediaItemID, _ := h.resolveBookToMediaItem(c, device.ID, userID, book) + if !mediaItemID.Valid { + continue + } - if book.UUID != "" { - mediaUUID, err = uuid.Parse(book.UUID) - if err != nil { - continue - } - - mediaItem, err := h.db.GetMediaItem(c.Request().Context(), pgtype.UUID{Bytes: mediaUUID, Valid: true}) - if err == nil { - err = h.enqueueProgressForBook(c, device.ID, userID, mediaItem.ID, book) - if err == nil { - booksEnqueued++ - } - } - } else if book.FilePath != "" { - mediaItem, err := h.db.GetMediaItemByFilePath(c.Request().Context(), book.FilePath) - if err == nil { - err = h.enqueueProgressForBook(c, device.ID, userID, mediaItem.ID, book) - if err == nil { - booksEnqueued++ - } - } - } else if book.Title != "" { - mediaItems, err := h.db.ListMediaItems(c.Request().Context(), database.ListMediaItemsParams{ - Limit: 100, - Offset: 0, - }) - if err == nil { - for _, mi := range mediaItems { - if mi.Title == book.Title && (book.Authors == nil || mi.Author.String == book.Authors[0]) { - err = h.enqueueProgressForBook(c, device.ID, userID, mi.ID, book) - if err == nil { - booksEnqueued++ - } - break - } - } - } + err := h.enqueueProgressForBook(c, device.ID, userID, mediaItemID, book) + if err == nil { + booksEnqueued++ } } @@ -678,7 +736,8 @@ func (h *KOReaderHandler) SyncBookmarks(c echo.Context) error { userID := device.UserID.Bytes var req struct { - BookUUID string `json:"book_uuid" validate:"required"` + BookUUID string `json:"book_uuid,omitempty"` + BookSHA256 string `json:"book_sha256,omitempty"` Bookmarks []KOReaderBookmark `json:"bookmarks"` Notes []KOReaderNote `json:"notes"` Highlights []KOReaderHighlight `json:"highlights"` @@ -696,21 +755,50 @@ func (h *KOReaderHandler) SyncBookmarks(c echo.Context) error { }) } - bookUUID, err := uuid.Parse(req.BookUUID) - if err != nil { + ctx := c.Request().Context() + pgUserID := pgtype.UUID{Bytes: userID, Valid: true} + + // Resolve media item ID using priority matching + var pgBookUUID pgtype.UUID + if req.BookUUID != "" { + // Use UUID directly + bookUUID, err := uuid.Parse(req.BookUUID) + if err != nil { + return c.JSON(http.StatusBadRequest, map[string]string{ + "error": "invalid book UUID", + }) + } + pgBookUUID = pgtype.UUID{Bytes: bookUUID, Valid: true} + } else if req.BookSHA256 != "" && len(req.BookSHA256) == 64 { + // Use SHA-256 to find book + mediaItem, err := h.db.GetMediaItemBySHA256(ctx, pgtype.Text{String: req.BookSHA256, Valid: true}) + if err != nil { + return c.JSON(http.StatusNotFound, map[string]string{ + "error": "book not found by SHA-256", + }) + } + pgBookUUID = mediaItem.ID + } else { return c.JSON(http.StatusBadRequest, map[string]string{ - "error": "invalid book UUID", + "error": "either book_uuid or book_sha256 is required", }) } - pgBookUUID := pgtype.UUID{Bytes: bookUUID, Valid: true} - pgUserID := pgtype.UUID{Bytes: userID, Valid: true} - bookmarksSynced := 0 notesSynced := 0 highlightsSynced := 0 for _, bookmark := range req.Bookmarks { + mediaItemID := pgBookUUID + + // If bookmark has its own SHA-256, use it for matching + if bookmark.BookSHA256 != "" && len(bookmark.BookSHA256) == 64 { + mediaItem, err := h.db.GetMediaItemBySHA256(ctx, pgtype.Text{String: bookmark.BookSHA256, Valid: true}) + if err == nil { + mediaItemID = mediaItem.ID + } + } + position := "" if bookmark.Pos0 != "" { position = bookmark.Pos0 @@ -718,8 +806,8 @@ func (h *KOReaderHandler) SyncBookmarks(c echo.Context) error { position = fmt.Sprintf("page:%d", bookmark.Page) } - _, err := h.db.CreateMediaNote(c.Request().Context(), database.CreateMediaNoteParams{ - MediaItemID: pgBookUUID, + _, err := h.db.CreateMediaNote(ctx, database.CreateMediaNoteParams{ + MediaItemID: mediaItemID, UserID: pgUserID, Content: bookmark.Text, Position: pgtype.Text{String: position, Valid: position != ""}, @@ -731,6 +819,16 @@ func (h *KOReaderHandler) SyncBookmarks(c echo.Context) error { } for _, note := range req.Notes { + mediaItemID := pgBookUUID + + // If note has its own SHA-256, use it for matching + if note.BookSHA256 != "" && len(note.BookSHA256) == 64 { + mediaItem, err := h.db.GetMediaItemBySHA256(ctx, pgtype.Text{String: note.BookSHA256, Valid: true}) + if err == nil { + mediaItemID = mediaItem.ID + } + } + position := "" if note.Pos0 != "" { position = note.Pos0 @@ -738,8 +836,8 @@ func (h *KOReaderHandler) SyncBookmarks(c echo.Context) error { position = fmt.Sprintf("page:%d", note.Page) } - _, err := h.db.CreateMediaNote(c.Request().Context(), database.CreateMediaNoteParams{ - MediaItemID: pgBookUUID, + _, err := h.db.CreateMediaNote(ctx, database.CreateMediaNoteParams{ + MediaItemID: mediaItemID, UserID: pgUserID, Content: note.Notes, Position: pgtype.Text{String: position, Valid: position != ""}, @@ -751,6 +849,16 @@ func (h *KOReaderHandler) SyncBookmarks(c echo.Context) error { } for _, highlight := range req.Highlights { + mediaItemID := pgBookUUID + + // If highlight has its own SHA-256, use it for matching + if highlight.BookSHA256 != "" && len(highlight.BookSHA256) == 64 { + mediaItem, err := h.db.GetMediaItemBySHA256(ctx, pgtype.Text{String: highlight.BookSHA256, Valid: true}) + if err == nil { + mediaItemID = mediaItem.ID + } + } + startPos := highlight.Pos0 endPos := highlight.Pos1 if startPos == "" && highlight.Page > 0 { @@ -763,8 +871,8 @@ func (h *KOReaderHandler) SyncBookmarks(c echo.Context) error { color = highlight.Color } - _, err := h.db.CreateMediaHighlight(c.Request().Context(), database.CreateMediaHighlightParams{ - MediaItemID: pgBookUUID, + _, err := h.db.CreateMediaHighlight(ctx, database.CreateMediaHighlightParams{ + MediaItemID: mediaItemID, UserID: pgUserID, SelectionText: highlight.Text, StartPosition: pgtype.Text{String: startPos, Valid: startPos != ""}, diff --git a/internal/services/ebook_scanner.go b/internal/services/ebook_scanner.go index eda2520..d6e7167 100644 --- a/internal/services/ebook_scanner.go +++ b/internal/services/ebook_scanner.go @@ -4,7 +4,11 @@ import ( "bookmann/internal/database" "bookmann/internal/utils" "context" + "crypto/sha256" + "encoding/hex" + "encoding/xml" "fmt" + "io" "io/fs" "os" "path/filepath" @@ -30,7 +34,26 @@ type EbookMetadata struct { Contributors string CoverPath string ISBN string + ASIN string Tags string + + Phase1HashInfo *HashInfo + Phase1FormatFormats []*FormatInfo +} + +type HashInfo struct { + FileSHA256 string + OPFIdentifier string + OPFUUID string + HashConfidence string +} + +type FormatInfo struct { + FormatType string + FilePath string + FileSHA256 string + FileSizeBytes int64 + MimeType string } type EbookScanner struct { @@ -250,6 +273,19 @@ func (s *EbookScanner) processEbookFile(ctx context.Context, path string) error metadata = &EbookMetadata{} } + // Extract hash information (Phase 2) + hashInfo, formatInfo, err := s.extractHashInfo(path) + if err != nil { + fmt.Printf("Warning: failed to extract hash info from %s: %v\n", path, err) + hashInfo = &HashInfo{} + formatInfo = &FormatInfo{} + } else { + metadata.Phase1HashInfo = hashInfo + metadata.Phase1FormatFormats = []*FormatInfo{formatInfo} + fmt.Printf("Hash info for %s: SHA256=%s, OPF_ID=%s, OPF_UUID=%s, Confidence=%s\n", + path, hashInfo.FileSHA256, hashInfo.OPFIdentifier, hashInfo.OPFUUID, hashInfo.HashConfidence) + } + // Try to get metadata from folder structure as fallback/enhancement // Use the root folder that contains this file var rootFolder string @@ -304,11 +340,12 @@ func (s *EbookScanner) processEbookFile(ctx context.Context, path string) error } // Create media item in database - _, err = s.db.CreateMediaItem(ctx, database.CreateMediaItemParams{ + createdItem, err := s.db.CreateMediaItem(ctx, database.CreateMediaItemParams{ LibraryID: libraryID, Title: metadata.Title, Author: pgtype.Text{String: metadata.Author, Valid: metadata.Author != ""}, Isbn: pgtype.Text{String: utils.NormalizeISBN(metadata.ISBN), Valid: metadata.ISBN != ""}, + Asin: pgtype.Text{String: metadata.ASIN, Valid: metadata.ASIN != ""}, Description: pgtype.Text{String: metadata.Description, Valid: metadata.Description != ""}, FilePath: path, FileSize: pgtype.Int8{Int64: info.Size(), Valid: true}, @@ -322,6 +359,38 @@ func (s *EbookScanner) processEbookFile(ctx context.Context, path string) error Tags: pgtype.Text{String: metadata.Tags, Valid: metadata.Tags != ""}, AddedByAdminID: s.adminID, }) + if err != nil { + return fmt.Errorf("failed to create media item: %v", err) + } + + // Update hash information (Phase 2) + if metadata.Phase1HashInfo != nil && metadata.Phase1HashInfo.FileSHA256 != "" { + _, err = s.db.UpdateMediaItemIdentifiers(ctx, database.UpdateMediaItemIdentifiersParams{ + ID: createdItem.ID, + FileSha256: pgtype.Text{String: metadata.Phase1HashInfo.FileSHA256, Valid: true}, + OpfIdentifier: pgtype.Text{String: metadata.Phase1HashInfo.OPFIdentifier, Valid: metadata.Phase1HashInfo.OPFIdentifier != ""}, + OpfUuid: pgtype.Text{String: metadata.Phase1HashInfo.OPFUUID, Valid: metadata.Phase1HashInfo.OPFUUID != ""}, + HashConfidence: pgtype.Text{String: metadata.Phase1HashInfo.HashConfidence, Valid: true}, + }) + if err != nil { + fmt.Printf("Warning: failed to update hash identifiers for %s: %v\n", path, err) + } + } + + // Store format information (Phase 2) + for _, format := range metadata.Phase1FormatFormats { + _, err = s.db.CreateMediaItemFormat(ctx, database.CreateMediaItemFormatParams{ + MediaItemID: createdItem.ID, + FormatType: format.FormatType, + FilePath: pgtype.Text{String: format.FilePath, Valid: true}, + FileSha256: pgtype.Text{String: format.FileSHA256, Valid: true}, + FileSizeBytes: pgtype.Int8{Int64: format.FileSizeBytes, Valid: true}, + MimeType: pgtype.Text{String: format.MimeType, Valid: true}, + }) + if err != nil { + fmt.Printf("Warning: failed to create format entry for %s: %v\n", path, err) + } + } return err } @@ -408,6 +477,13 @@ func (s *EbookScanner) extractEPUBMetadata(path string) (*EbookMetadata, error) break } } + if strings.Contains(strings.ToLower(isbn), "asin") { + // Extract ASIN from identifier like "asin:B08XXXXX" + asinParts := strings.SplitN(isbn, ":", 2) + if len(asinParts) == 2 { + metadata.ASIN = asinParts[1] + } + } } } @@ -506,3 +582,199 @@ func (s *EbookScanner) Close() error { } return nil } + +// ============================================ +// PHASE 2: SCANNER ENHANCEMENTS (Week 1-2) +// ============================================ + +// calculateFileSHA256 calculates SHA-256 hash using streaming to avoid loading entire file into memory +func (s *EbookScanner) calculateFileSHA256(filePath string) (string, error) { + file, err := os.Open(filePath) + if err != nil { + return "", fmt.Errorf("failed to open file: %v", err) + } + defer file.Close() + + hasher := sha256.New() + if _, err := io.Copy(hasher, file); err != nil { + return "", fmt.Errorf("failed to calculate hash: %v", err) + } + + return hex.EncodeToString(hasher.Sum(nil)), nil +} + +// OPFIdentifier represents an identifier from OPF metadata +type OPFIdentifier struct { + XMLName xml.Name `xml:"identifier"` + ID string `xml:"id,attr"` + Scheme string `xml:"scheme,attr"` + Content string `xml:",chardata"` +} + +// OPFMetadata represents parsed OPF metadata +type OPFMetadata struct { + XMLName xml.Name `xml:"package"` + Version string `xml:"version,attr"` + Identifiers []OPFIdentifier `xml:"metadata>identifier"` +} + +// extractOPFIdentifiers extracts identifiers from EPUB OPF file +func (s *EbookScanner) extractOPFIdentifiers(epubPath string) (opfIdentifier, opfUUID string, confidence string, err error) { + book, err := epub.ReadBook(epubPath) + if err != nil { + return "", "", "", fmt.Errorf("failed to open EPUB: %v", err) + } + + // Extract OPF identifiers using go-epub library + identifiers, err := book.MetadataByKey("identifier") + if err != nil || len(identifiers) == 0 { + return "", "", "low", nil + } + + var identifier, uuid string + for _, id := range identifiers { + id = strings.TrimSpace(id) + + // Check for UUID format (urn:uuid:) + if strings.HasPrefix(strings.ToLower(id), "urn:uuid:") { + uuid = strings.TrimPrefix(strings.ToLower(id), "urn:uuid:") + continue + } + + // Check if it's a plain UUID (8-4-4-4-12 format) + if isValidUUID(id) { + uuid = id + continue + } + + // Check for ISBN + if strings.Contains(strings.ToLower(id), "isbn") { + isbn := s.extractISBNFromIdentifier(id) + if isbn != "" { + identifier = isbn + continue + } + } + + // Use first identifier as fallback + if identifier == "" && id != "" { + identifier = id + } + } + + confidence = s.determineHashConfidence(uuid, identifier) + + return identifier, uuid, confidence, nil +} + +// isValidUUID checks if string is a valid UUID (8-4-4-4-12 format) +func isValidUUID(idStr string) bool { + uuidRegex := regexp.MustCompile(`^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$`) + return uuidRegex.MatchString(idStr) +} + +// extractISBNFromIdentifier extracts ISBN from identifier string +func (s *EbookScanner) extractISBNFromIdentifier(id string) string { + id = strings.TrimSpace(id) + + // Remove "isbn:" prefix if present + if strings.HasPrefix(strings.ToLower(id), "isbn:") { + id = strings.TrimPrefix(strings.ToLower(id), "isbn:") + } + + // Remove hyphens and spaces + isbn := regexp.MustCompile(`[\s-]`).ReplaceAllString(id, "") + + // Check if it's a valid ISBN-10 or ISBN-13 + if len(isbn) == 10 || len(isbn) == 13 { + return isbn + } + + return "" +} + +// determineHashConfidence determines confidence level based on available identifiers +func (s *EbookScanner) determineHashConfidence(uuid, identifier string) string { + if uuid != "" && isValidUUID(uuid) { + return "high" + } + if identifier != "" && (strings.Contains(strings.ToLower(identifier), "isbn") || len(identifier) >= 10) { + return "medium" + } + return "low" +} + +// detectFormatType detects the format type based on file extension and content +func (s *EbookScanner) detectFormatType(filePath string) string { + base := strings.ToLower(filepath.Base(filePath)) + + // Check for compound extensions first (like .kepub.epub) + if strings.HasSuffix(base, ".kepub.epub") { + return "kepub" + } + + ext := strings.ToLower(filepath.Ext(filePath)) + + switch ext { + case ".epub": + return "epub" + case ".kepub": + return "kepub" + case ".pdf": + return "pdf" + case ".cbz", ".cbr", ".cb7", ".cbt": + return "comic_archive" + case ".mobi": + return "mobi" + case ".azw", ".azw3": + return "kfx" + case ".txt": + return "txt" + case ".fb2": + return "fb2" + default: + return "unknown" + } +} + +// extractHashInfo calculates hash and extracts OPF identifiers for a file +func (s *EbookScanner) extractHashInfo(filePath string) (*HashInfo, *FormatInfo, error) { + // Calculate SHA-256 + fileSHA256, err := s.calculateFileSHA256(filePath) + if err != nil { + return nil, nil, fmt.Errorf("failed to calculate SHA-256: %v", err) + } + + // Get file info + info, err := os.Stat(filePath) + if err != nil { + return nil, nil, fmt.Errorf("failed to get file info: %v", err) + } + + // Extract OPF identifiers for EPUB files + var opfIdentifier, opfUUID, confidence string + if strings.HasSuffix(strings.ToLower(filePath), ".epub") { + opfIdentifier, opfUUID, confidence, err = s.extractOPFIdentifiers(filePath) + if err != nil { + // Non-fatal error, continue with low confidence + confidence = "low" + } + } + + hashInfo := &HashInfo{ + FileSHA256: fileSHA256, + OPFIdentifier: opfIdentifier, + OPFUUID: opfUUID, + HashConfidence: confidence, + } + + formatInfo := &FormatInfo{ + FormatType: s.detectFormatType(filePath), + FilePath: filePath, + FileSHA256: fileSHA256, + FileSizeBytes: info.Size(), + MimeType: s.getMimeType(filePath), + } + + return hashInfo, formatInfo, nil +}