Bookhoard/bookhoard
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 12 Wiki Activity

feat(sync): Add Kobo/Koreader sync and conflict handling

Browse Source 
- Add Kobo markup/sync endpoints for bookshelves
- Add Koreader progress sync with SHA-256 support
- Add sync conflict detection and resolution
- Update ebook scanner for better file matching
This commit is contained in:
john-okeefe
2026-01-31 22:32:09 -05:00
parent 63008001c6
commit d28002e4cb
4 changed files with 728 additions and 134 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/handlers/conflicts.go
+20 -10
View File
@@ -64,7 +64,7 @@ type ConflictResolveResponse struct {
DevicesSynced []string `json:"devices_synced"`
}
func (h *ConflictHandler) ListConflicts(c echo.Context) error {
func (h *ConflictHandler) GetConflictsData(c echo.Context) ([]ConflictDetailResponse, int, int, error) {
user := c.Get("user").(database.Users)
status := c.QueryParam("status")
@@ -80,14 +80,11 @@ func (h *ConflictHandler) ListConflicts(c echo.Context) error {
})
if err != nil && err != pgx.ErrNoRows {
return echo.NewHTTPError(http.StatusInternalServerError, "failed to list conflicts")
return nil, 0, 0, err
}
response := ConflictListResponse{
Conflicts: make([]ConflictDetailResponse, 0),
Total: len(conflicts),
Unresolved: 0,
}
response := make([]ConflictDetailResponse, 0, len(conflicts))
unresolvedCount := 0
for _, conflict := range conflicts {
var conflictData map[string]ConflictSourceData
@@ -116,13 +113,26 @@ func (h *ConflictHandler) ListConflicts(c echo.Context) error {
}
}
response.Conflicts = append(response.Conflicts, detail)
response = append(response, detail)
if conflict.ResolutionStatus.String == "unresolved" {
response.Unresolved++
unresolvedCount++
}
}
return c.JSON(http.StatusOK, response)
return response, len(response), unresolvedCount, nil
}
func (h *ConflictHandler) ListConflicts(c echo.Context) error {
conflicts, total, unresolved, err := h.GetConflictsData(c)
if err != nil {
return echo.NewHTTPError(http.StatusInternalServerError, "failed to list conflicts")
}
return c.JSON(http.StatusOK, ConflictListResponse{
Conflicts: conflicts,
Total: total,
Unresolved: unresolved,
})
}
func (h *ConflictHandler) GetConflict(c echo.Context) error {
internal/handlers/kobo.go
+224 -20
View File
@@ -3,9 +3,12 @@ package handlers
import (
"bookmann/internal/database"
wsync "bookmann/internal/sync"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"fmt"
"net/http"
"regexp"
"time"
"github.com/google/uuid"
@@ -22,6 +25,161 @@ func NewKoboHandler(db *database.Queries, connManager *wsync.ConnectionManager)
return &KoboHandler{db: db, connManager: connManager}
}
// mapContentIdToBookmannUUID maps Kobo ContentId to Bookmann UUID with multiple fallback strategies
// Phase 6: Enhanced Kobo Sync - ContentId Mapping Logic
func (h *KoboHandler) mapContentIdToBookmannUUID(ctx echo.Context, contentId string, deviceID uuid.UUID) (uuid.UUID, error, string) {
// Step 1: Try direct ContentId lookup in device_catalogs table
catalog, err := h.db.GetDeviceCatalogByKoboContentId(ctx.Request().Context(), contentId)
if err == nil && catalog.ID.Valid {
// Found! Use canonical Bookmann UUID
return uuid.UUID(catalog.BookmannUuid.Bytes), nil, "catalog_match"
}
// Step 2: ContentId not found - check if it looks like a SHA-256 hash
if len(contentId) == 64 && looksLikeSHA256(contentId) {
// Try to find media item by SHA-256
mediaItem, err := h.db.GetMediaItemBySHA256(ctx.Request().Context(), pgtype.Text{String: contentId, Valid: true})
if err == nil {
// Found by SHA-256! Create device catalog entry for future lookups
_, _ = h.db.CreateDeviceCatalog(ctx.Request().Context(), database.CreateDeviceCatalogParams{
DeviceID: pgtype.UUID{Bytes: deviceID, Valid: true},
MediaItemID: pgtype.UUID{Bytes: mediaItem.ID.Bytes, Valid: true},
BookmannUuid: pgtype.UUID{Bytes: mediaItem.ID.Bytes, Valid: true},
KoboContentID: contentId,
ContentIDType: pgtype.Text{String: "sha256", Valid: true},
Available: pgtype.Bool{Bool: true, Valid: true},
DeliveryDate: pgtype.Timestamptz{Time: time.Now(), Valid: true},
DeliveryMethod: pgtype.Text{String: "sync", Valid: true},
})
return uuid.UUID(mediaItem.ID.Bytes), nil, "sha256_match"
}
}
// Step 3: Try to parse as UUID directly
if parsedUUID, err := uuid.Parse(contentId); err == nil {
// Check if this UUID exists in media_items
mediaItem, err := h.db.GetMediaItem(ctx.Request().Context(), pgtype.UUID{Bytes: parsedUUID, Valid: true})
if err == nil {
// Create device catalog entry
_, _ = h.db.CreateDeviceCatalog(ctx.Request().Context(), database.CreateDeviceCatalogParams{
DeviceID: pgtype.UUID{Bytes: deviceID, Valid: true},
MediaItemID: pgtype.UUID{Bytes: mediaItem.ID.Bytes, Valid: true},
BookmannUuid: pgtype.UUID{Bytes: mediaItem.ID.Bytes, Valid: true},
KoboContentID: contentId,
ContentIDType: pgtype.Text{String: "bookmann_uuid", Valid: true},
Available: pgtype.Bool{Bool: true, Valid: true},
DeliveryDate: pgtype.Timestamptz{Time: time.Now(), Valid: true},
DeliveryMethod: pgtype.Text{String: "sync", Valid: true},
})
return parsedUUID, nil, "uuid_match"
}
}
// Step 4: Not found - return error for unlinked book
return uuid.Nil, fmt.Errorf("unlinked book: ContentId %s not found", contentId), "unlinked"
}
// mapBookmannUUIDToKoboContentId maps Bookmann UUID to Kobo ContentId
// Creates new entry in device_catalogs if not exists
func (h *KoboHandler) mapBookmannUUIDToKoboContentId(c echo.Context, bookmannUUID uuid.UUID, deviceID uuid.UUID) (string, error) {
// Check if catalog entry already exists
catalog, err := h.db.GetDeviceCatalogByBookmannUUID(c.Request().Context(), database.GetDeviceCatalogByBookmannUUIDParams{
DeviceID: pgtype.UUID{Bytes: deviceID, Valid: true},
BookmannUuid: pgtype.UUID{Bytes: bookmannUUID, Valid: true},
})
if err == nil && catalog.ID.Valid {
return catalog.KoboContentID, nil
}
// Get media item to check for existing Kobo content ID
mediaItem, err := h.db.GetMediaItem(c.Request().Context(), pgtype.UUID{Bytes: bookmannUUID, Valid: true})
if err != nil {
return "", err
}
// Generate Kobo ContentId based on priority:
// 1. Use existing kobo_content_id from media_items
// 2. Use existing entitlement_id from media_items
// 3. Generate new "kobo_" prefixed UUID
var koboContentId string
contentIdType := "bookmann_generated"
if mediaItem.KoboContentID.Valid && mediaItem.KoboContentID.String != "" {
koboContentId = mediaItem.KoboContentID.String
contentIdType = "kobo_metadata"
} else if mediaItem.EntitlementID.Valid && mediaItem.EntitlementID.String != "" {
koboContentId = mediaItem.EntitlementID.String
contentIdType = "entitlement_id"
} else {
koboContentId = "kobo_" + uuid.New().String()
contentIdType = "kobo_generated"
}
// Create device catalog entry
_, err = h.db.CreateDeviceCatalog(c.Request().Context(), database.CreateDeviceCatalogParams{
DeviceID: pgtype.UUID{Bytes: deviceID, Valid: true},
MediaItemID: pgtype.UUID{Bytes: bookmannUUID, Valid: true},
BookmannUuid: pgtype.UUID{Bytes: bookmannUUID, Valid: true},
KoboContentID: koboContentId,
ContentIDType: pgtype.Text{String: contentIdType, Valid: true},
Available: pgtype.Bool{Bool: true, Valid: true},
DeliveryDate: pgtype.Timestamptz{Time: time.Now(), Valid: true},
DeliveryMethod: pgtype.Text{String: "opds", Valid: true},
})
if err != nil {
return "", err
}
return koboContentId, nil
}
// getCollectionMetadataForBook retrieves collection names for a book
func (h *KoboHandler) getCollectionMetadataForBook(c echo.Context, bookmannUUID uuid.UUID, deviceID uuid.UUID) ([]string, error) {
device := c.Get("device").(database.Devices)
pgDeviceID := pgtype.UUID{Bytes: device.ID.Bytes, Valid: true}
// Get collections for this book
collections, err := h.db.GetCollectionsForBook(c.Request().Context(), pgtype.UUID{Bytes: bookmannUUID, Valid: true})
if err != nil {
return []string{}, nil
}
var collectionNames []string
// For each collection, check if there's a device-specific shelf mapping
for _, collection := range collections {
mapping, err := h.db.GetDeviceShelfMapping(c.Request().Context(), database.GetDeviceShelfMappingParams{
DeviceID: pgDeviceID,
CollectionID: pgtype.UUID{Bytes: collection.ID.Bytes, Valid: true},
})
if err == nil && mapping.ID.Valid && mapping.DeviceShelfName.Valid {
// Use device-specific shelf name
collectionNames = append(collectionNames, mapping.DeviceShelfName.String)
} else if collection.Name != "" {
// Fall back to collection name
collectionNames = append(collectionNames, collection.Name)
}
}
return collectionNames, nil
}
// looksLikeSHA256 checks if a string looks like a SHA-256 hash
func looksLikeSHA256(s string) bool {
if len(s) != 64 {
return false
}
matched, _ := regexp.MatchString("^[0-9a-fA-F]{64}$", s)
return matched
}
// calculateFileSHA256 calculates SHA-256 hash of file path
func calculateFileSHA256(filePath string) string {
hash := sha256.Sum256([]byte(filePath))
return hex.EncodeToString(hash[:])
}
type KoboDeviceInfo struct {
DeviceID string `json:"DeviceId"`
Model string `json:"Model"`
@@ -67,6 +225,8 @@ type KoboLibraryBook struct {
Revision int `json:"Revision"`
MimeType string `json:"MimeType"`
FileSize int64 `json:"FileSize"`
Categories []string `json:"Categories,omitempty"`
BookmannUUID string `json:"BookmannUUID,omitempty"`
}
type KoboLibraryResponse struct {
@@ -110,8 +270,10 @@ type KoboAnalyticsTest struct {
func (h *KoboHandler) Initialization(c echo.Context) error {
device := c.Get("device").(database.Devices)
userID := device.UserID.Bytes
deviceID := device.ID.Bytes
pgUserID := pgtype.UUID{Bytes: userID, Valid: true}
deviceUUID := uuid.UUID(deviceID)
mediaItems, err := h.db.GetUserMediaItemsForSync(c.Request().Context(), pgUserID)
if err != nil {
@@ -122,6 +284,8 @@ func (h *KoboHandler) Initialization(c echo.Context) error {
librarySync := []KoboLibraryBook{}
for _, item := range mediaItems {
bookmannUUID := uuid.UUID(item.ID.Bytes)
progress, _ := h.db.GetUniversalProgress(c.Request().Context(), database.GetUniversalProgressParams{
MediaItemID: pgtype.UUID{Bytes: item.ID.Bytes, Valid: true},
UserID: pgUserID,
@@ -154,10 +318,15 @@ func (h *KoboHandler) Initialization(c echo.Context) error {
author = item.Author.String
}
entitlementId := item.EntitlementID.String
if entitlementId == "" || !item.EntitlementID.Valid {
entitlementId = "kobo_" + uuid.UUID(item.ID.Bytes).String()
// Phase 6: Use ContentId mapping instead of direct UUID
koboContentId, err := h.mapBookmannUUIDToKoboContentId(c, bookmannUUID, deviceUUID)
if err != nil {
// Fallback to entitlement_id or generate new one
if item.EntitlementID.Valid && item.EntitlementID.String != "" {
koboContentId = item.EntitlementID.String
} else {
koboContentId = "kobo_" + bookmannUUID.String()
}
}
mimeType := item.MimeType.String
@@ -175,8 +344,11 @@ func (h *KoboHandler) Initialization(c echo.Context) error {
revision = int(item.RevisionNumber.Int32)
}
// Phase 6: Get collection metadata for this book
collections, _ := h.getCollectionMetadataForBook(c, bookmannUUID, deviceUUID)
librarySync = append(librarySync, KoboLibraryBook{
ContentId: uuid.UUID(item.ID.Bytes).String(),
ContentId: koboContentId,
ContentType: "6",
Title: item.Title,
Author: author,
@@ -184,10 +356,12 @@ func (h *KoboHandler) Initialization(c echo.Context) error {
PagesRemaining: pagesRemaining,
BookmarkCount: bookmarkCount,
LastModified: lastModified,
EntitlementId: entitlementId,
EntitlementId: koboContentId,
Revision: revision,
MimeType: mimeType,
FileSize: fileSize,
Categories: collections,
BookmannUUID: bookmannUUID.String(),
})
}
@@ -205,8 +379,10 @@ func (h *KoboHandler) LibrarySync(c echo.Context) error {
func (h *KoboHandler) Markup(c echo.Context) error {
device := c.Get("device").(database.Devices)
userID := device.UserID.Bytes
deviceID := device.ID.Bytes
pgUserID := pgtype.UUID{Bytes: userID, Valid: true}
deviceUUID := uuid.UUID(deviceID)
var req KoboMarkupRequest
if err := c.Bind(&req); err != nil {
@@ -217,14 +393,19 @@ func (h *KoboHandler) Markup(c echo.Context) error {
markupsSynced := 0
bookmarksSynced := 0
unlinkedBooks := 0
for _, readingSync := range req.ReadingSync {
mediaUUID, err := uuid.Parse(readingSync.ContentId)
// Phase 6: Use ContentId mapping with fallback logic
bookmannUUID, err, _ := h.mapContentIdToBookmannUUID(c, readingSync.ContentId, deviceUUID)
if err != nil {
// Unlinked book detected
unlinkedBooks++
// TODO: Create unlinked book entry for manual resolution
continue
}
pgMediaUUID := pgtype.UUID{Bytes: mediaUUID, Valid: true}
pgMediaUUID := pgtype.UUID{Bytes: bookmannUUID, Valid: true}
percentage := readingSync.PercentRead / 100.0
@@ -240,7 +421,7 @@ func (h *KoboHandler) Markup(c echo.Context) error {
markupsSynced++
h.connManager.BroadcastProgressUpdate(
mediaUUID,
bookmannUUID,
percentage,
wsync.SourceDevice{
ID: uuid.UUID(userID).String(),
@@ -252,12 +433,14 @@ func (h *KoboHandler) Markup(c echo.Context) error {
}
for _, bookmarkSync := range req.BookmarkSync {
mediaUUID, err := uuid.Parse(bookmarkSync.ContentId)
// Phase 6: Use ContentId mapping with fallback logic
bookmannUUID, err, _ := h.mapContentIdToBookmannUUID(c, bookmarkSync.ContentId, deviceUUID)
if err != nil {
// Unlinked book - skip
continue
}
pgMediaUUID := pgtype.UUID{Bytes: mediaUUID, Valid: true}
pgMediaUUID := pgtype.UUID{Bytes: bookmannUUID, Valid: true}
switch bookmarkSync.BookmarkType {
case "annotation":
@@ -292,18 +475,29 @@ func (h *KoboHandler) Markup(c echo.Context) error {
})
}
return c.JSON(http.StatusOK, KoboSyncStatus{
response := KoboSyncStatus{
Status: "Success",
MarkupsSynced: markupsSynced,
BookmarksSynced: bookmarksSynced,
})
}
// Include unlinked books count if any
if unlinkedBooks > 0 {
// For now, just log it. In production, this should trigger an alert
// and the unlinked books should be stored for manual resolution
response.Status = "Partial"
}
return c.JSON(http.StatusOK, response)
}
func (h *KoboHandler) Bookmark(c echo.Context) error {
device := c.Get("device").(database.Devices)
userID := device.UserID.Bytes
deviceID := device.ID.Bytes
pgUserID := pgtype.UUID{Bytes: userID, Valid: true}
deviceUUID := uuid.UUID(deviceID)
var req struct {
BookmarkSync []KoboBookmarkSync `json:"BookmarkSync"`
@@ -317,12 +511,14 @@ func (h *KoboHandler) Bookmark(c echo.Context) error {
bookmarksSynced := 0
for _, bookmarkSync := range req.BookmarkSync {
mediaUUID, err := uuid.Parse(bookmarkSync.ContentId)
// Phase 6: Use ContentId mapping with fallback logic
bookmannUUID, err, _ := h.mapContentIdToBookmannUUID(c, bookmarkSync.ContentId, deviceUUID)
if err != nil {
// Unlinked book - skip
continue
}
pgMediaUUID := pgtype.UUID{Bytes: mediaUUID, Valid: true}
pgMediaUUID := pgtype.UUID{Bytes: bookmannUUID, Valid: true}
switch bookmarkSync.BookmarkType {
case "annotation":
@@ -367,8 +563,10 @@ func (h *KoboHandler) Bookmark(c echo.Context) error {
func (h *KoboHandler) AnalyticsGettests(c echo.Context) error {
device := c.Get("device").(database.Devices)
userID := device.UserID.Bytes
deviceID := device.ID.Bytes
pgUserID := pgtype.UUID{Bytes: userID, Valid: true}
deviceUUID := uuid.UUID(deviceID)
var req []KoboAnalyticsTest
if err := c.Bind(&req); err != nil {
@@ -378,12 +576,14 @@ func (h *KoboHandler) AnalyticsGettests(c echo.Context) error {
}
for _, test := range req {
mediaUUID, err := uuid.Parse(test.ContentId)
// Phase 6: Use ContentId mapping with fallback logic
bookmannUUID, err, _ := h.mapContentIdToBookmannUUID(c, test.ContentId, deviceUUID)
if err != nil {
// Unlinked book - skip
continue
}
pgMediaUUID := pgtype.UUID{Bytes: mediaUUID, Valid: true}
pgMediaUUID := pgtype.UUID{Bytes: bookmannUUID, Valid: true}
percentage := test.PercentRead / 100.0
_, err = h.db.UpdateUniversalProgress(c.Request().Context(), database.UpdateUniversalProgressParams{
@@ -396,7 +596,7 @@ func (h *KoboHandler) AnalyticsGettests(c echo.Context) error {
if err == nil {
h.connManager.BroadcastProgressUpdate(
mediaUUID,
bookmannUUID,
percentage,
wsync.SourceDevice{
ID: uuid.UUID(userID).String(),
@@ -436,8 +636,10 @@ func parseKoboDeviceHeader(c echo.Context) (KoboDeviceInfo, error) {
func (h *KoboHandler) SyncFromServer(c echo.Context) error {
device := c.Get("device").(database.Devices)
userID := device.UserID.Bytes
deviceID := device.ID.Bytes
pgUserID := pgtype.UUID{Bytes: userID, Valid: true}
deviceUUID := uuid.UUID(deviceID)
var req []KoboServerSyncData
if err := c.Bind(&req); err != nil {
@@ -451,12 +653,14 @@ func (h *KoboHandler) SyncFromServer(c echo.Context) error {
highlightsSent := 0
for _, syncData := range req {
mediaUUID, err := uuid.Parse(syncData.ContentId)
// Phase 6: Use ContentId mapping with fallback logic
bookmannUUID, err, _ := h.mapContentIdToBookmannUUID(c, syncData.ContentId, deviceUUID)
if err != nil {
// Unlinked book - skip
continue
}
pgMediaUUID := pgtype.UUID{Bytes: mediaUUID, Valid: true}
pgMediaUUID := pgtype.UUID{Bytes: bookmannUUID, Valid: true}
percentage := syncData.PercentRead / 100.0
internal/handlers/koreader.go
+179 -71
View File
@@ -32,6 +32,7 @@ type KOReaderProgressRequest struct {
type KOReaderBookProgress struct {
UUID string `json:"uuid,omitempty"`
SHA256 string `json:"sha256,omitempty"`
Title string `json:"title,omitempty"`
Authors []string `json:"authors,omitempty"`
Progress float64 `json:"progress"`
@@ -64,6 +65,7 @@ type KOReaderBookmark struct {
Text string `json:"text,omitempty"`
Type string `json:"type,omitempty"`
Percentage *float64 `json:"percentage,omitempty"`
BookSHA256 string `json:"book_sha256,omitempty"`
}
type KOReaderHighlight struct {
@@ -77,6 +79,7 @@ type KOReaderHighlight struct {
Type string `json:"type,omitempty"`
Color string `json:"color,omitempty"`
Percentage *float64 `json:"percentage,omitempty"`
BookSHA256 string `json:"book_sha256,omitempty"`
}
type KOReaderNote struct {
@@ -89,6 +92,7 @@ type KOReaderNote struct {
Text string `json:"text,omitempty"`
Type string `json:"type,omitempty"`
Percentage *float64 `json:"percentage,omitempty"`
BookSHA256 string `json:"book_sha256,omitempty"`
}
type KOReaderSyncResponse struct {
@@ -182,48 +186,16 @@ func (h *KOReaderHandler) SyncProgress(c echo.Context) error {
conflicts := []KOReaderConflict{}
for _, book := range req.Books {
var mediaUUID uuid.UUID
var err error
if book.UUID != "" {
mediaUUID, err = uuid.Parse(book.UUID)
if err != nil {
mediaItemID, _ := h.resolveBookToMediaItem(c, device.ID, pgUserID, book)
if !mediaItemID.Valid {
continue
}
mediaItem, err := h.db.GetMediaItem(c.Request().Context(), pgtype.UUID{Bytes: mediaUUID, Valid: true})
if err == nil {
err = h.updateProgressForBook(c, pgUserID, mediaItem.ID, book)
err := h.updateProgressForBook(c, pgUserID, mediaItemID, book)
if err == nil {
booksSynced++
}
}
} else if book.FilePath != "" {
mediaItem, err := h.db.GetMediaItemByFilePath(c.Request().Context(), book.FilePath)
if err == nil {
err = h.updateProgressForBook(c, pgUserID, mediaItem.ID, book)
if err == nil {
booksSynced++
}
}
} else if book.Title != "" {
mediaItems, err := h.db.ListMediaItems(c.Request().Context(), database.ListMediaItemsParams{
Limit: 100,
Offset: 0,
})
if err == nil {
for _, mi := range mediaItems {
if mi.Title == book.Title && (book.Authors == nil || mi.Author.String == book.Authors[0]) {
err = h.updateProgressForBook(c, pgUserID, mi.ID, book)
if err == nil {
booksSynced++
}
break
}
}
}
}
}
_, err := h.db.UpdateDeviceLastSync(c.Request().Context(), device.ID)
if err != nil {
@@ -251,51 +223,137 @@ func (h *KOReaderHandler) SyncProgress(c echo.Context) error {
})
}
func (h *KOReaderHandler) handleCheckpointSync(c echo.Context, device database.Devices, userID pgtype.UUID, req KOReaderProgressRequest) error {
booksEnqueued := 0
for _, book := range req.Books {
var mediaUUID uuid.UUID
var err error
func (h *KOReaderHandler) resolveBookToMediaItem(c echo.Context, deviceID pgtype.UUID, userID pgtype.UUID, book KOReaderBookProgress) (pgtype.UUID, float64) {
ctx := c.Request().Context()
// Priority 1: UUID provided (highest confidence - 1.0)
if book.UUID != "" {
mediaUUID, err = uuid.Parse(book.UUID)
if err != nil {
continue
mediaUUID, err := uuid.Parse(book.UUID)
if err == nil {
mediaItem, err := h.db.GetMediaItem(ctx, pgtype.UUID{Bytes: mediaUUID, Valid: true})
if err == nil {
// Create device file alias if FilePath is also provided
if book.FilePath != "" {
h.createDeviceFileAlias(c, deviceID, mediaItem.ID, book)
}
return mediaItem.ID, 1.0
}
}
}
mediaItem, err := h.db.GetMediaItem(c.Request().Context(), pgtype.UUID{Bytes: mediaUUID, Valid: true})
// Priority 2: SHA-256 provided (medium confidence - 0.9)
if book.SHA256 != "" && len(book.SHA256) == 64 {
mediaItem, err := h.db.GetMediaItemBySHA256(ctx, pgtype.Text{String: book.SHA256, Valid: true})
if err == nil {
err = h.enqueueProgressForBook(c, device.ID, userID, mediaItem.ID, book)
if err == nil {
booksEnqueued++
// Create device file alias if FilePath is provided
if book.FilePath != "" {
h.createDeviceFileAlias(c, deviceID, mediaItem.ID, book)
}
return mediaItem.ID, 0.9
}
}
} else if book.FilePath != "" {
mediaItem, err := h.db.GetMediaItemByFilePath(c.Request().Context(), book.FilePath)
// Priority 3: FilePath provided (check existing alias or create new - confidence 0.7)
if book.FilePath != "" {
// Check if alias already exists
alias, err := h.db.GetDeviceFileAlias(ctx, database.GetDeviceFileAliasParams{
DeviceID: deviceID,
FilePath: book.FilePath,
})
if err == nil {
err = h.enqueueProgressForBook(c, device.ID, userID, mediaItem.ID, book)
return alias.MediaItemID, alias.ConfidenceScore.Float64
}
// Try to find by file path
mediaItem, err := h.db.GetMediaItemByFilePath(ctx, book.FilePath)
if err == nil {
booksEnqueued++
// Create new alias
confidence := 0.7
h.createDeviceFileAlias(c, deviceID, mediaItem.ID, book)
return mediaItem.ID, confidence
}
}
} else if book.Title != "" {
mediaItems, err := h.db.ListMediaItems(c.Request().Context(), database.ListMediaItemsParams{
// Priority 4: Title + Author match (lowest confidence - 0.5)
if book.Title != "" {
mediaItems, err := h.db.ListMediaItems(ctx, database.ListMediaItemsParams{
Limit: 100,
Offset: 0,
})
if err == nil {
for _, mi := range mediaItems {
if mi.Title == book.Title && (book.Authors == nil || mi.Author.String == book.Authors[0]) {
err = h.enqueueProgressForBook(c, device.ID, userID, mi.ID, book)
if mi.Title == book.Title {
// Check author match if provided
if len(book.Authors) > 0 && mi.Author.Valid {
if mi.Author.String == book.Authors[0] {
// Create device file alias if FilePath is provided
if book.FilePath != "" {
h.createDeviceFileAlias(c, deviceID, mi.ID, book)
}
return mi.ID, 0.5
}
} else {
// Title only match
if book.FilePath != "" {
h.createDeviceFileAlias(c, deviceID, mi.ID, book)
}
return mi.ID, 0.4
}
}
}
}
}
return pgtype.UUID{}, 0.0
}
func (h *KOReaderHandler) createDeviceFileAlias(c echo.Context, deviceID pgtype.UUID, mediaItemID pgtype.UUID, book KOReaderBookProgress) {
ctx := c.Request().Context()
if book.FilePath == "" {
return
}
sha256 := book.SHA256
if sha256 == "" {
// Try to get SHA256 from media item
mediaItem, err := h.db.GetMediaItem(ctx, mediaItemID)
if err == nil && mediaItem.FileSha256.Valid {
sha256 = mediaItem.FileSha256.String
}
}
confidence := 0.7
// Check if alias already exists
_, err := h.db.GetDeviceFileAlias(ctx, database.GetDeviceFileAliasParams{
DeviceID: deviceID,
FilePath: book.FilePath,
})
if err != nil {
// Create new alias
_, _ = h.db.CreateDeviceFileAlias(ctx, database.CreateDeviceFileAliasParams{
MediaItemID: mediaItemID,
DeviceID: deviceID,
FilePath: book.FilePath,
FileSha256: pgtype.Text{String: sha256, Valid: sha256 != ""},
ConfidenceScore: pgtype.Float8{Float64: confidence, Valid: true},
})
}
}
func (h *KOReaderHandler) handleCheckpointSync(c echo.Context, device database.Devices, userID pgtype.UUID, req KOReaderProgressRequest) error {
booksEnqueued := 0
for _, book := range req.Books {
mediaItemID, _ := h.resolveBookToMediaItem(c, device.ID, userID, book)
if !mediaItemID.Valid {
continue
}
err := h.enqueueProgressForBook(c, device.ID, userID, mediaItemID, book)
if err == nil {
booksEnqueued++
}
break
}
}
}
}
}
_, err := h.db.UpdateDeviceLastSync(c.Request().Context(), device.ID)
@@ -678,7 +736,8 @@ func (h *KOReaderHandler) SyncBookmarks(c echo.Context) error {
userID := device.UserID.Bytes
var req struct {
BookUUID string `json:"book_uuid" validate:"required"`
BookUUID string `json:"book_uuid,omitempty"`
BookSHA256 string `json:"book_sha256,omitempty"`
Bookmarks []KOReaderBookmark `json:"bookmarks"`
Notes []KOReaderNote `json:"notes"`
Highlights []KOReaderHighlight `json:"highlights"`
@@ -696,21 +755,50 @@ func (h *KOReaderHandler) SyncBookmarks(c echo.Context) error {
})
}
ctx := c.Request().Context()
pgUserID := pgtype.UUID{Bytes: userID, Valid: true}
// Resolve media item ID using priority matching
var pgBookUUID pgtype.UUID
if req.BookUUID != "" {
// Use UUID directly
bookUUID, err := uuid.Parse(req.BookUUID)
if err != nil {
return c.JSON(http.StatusBadRequest, map[string]string{
"error": "invalid book UUID",
})
}
pgBookUUID := pgtype.UUID{Bytes: bookUUID, Valid: true}
pgUserID := pgtype.UUID{Bytes: userID, Valid: true}
pgBookUUID = pgtype.UUID{Bytes: bookUUID, Valid: true}
} else if req.BookSHA256 != "" && len(req.BookSHA256) == 64 {
// Use SHA-256 to find book
mediaItem, err := h.db.GetMediaItemBySHA256(ctx, pgtype.Text{String: req.BookSHA256, Valid: true})
if err != nil {
return c.JSON(http.StatusNotFound, map[string]string{
"error": "book not found by SHA-256",
})
}
pgBookUUID = mediaItem.ID
} else {
return c.JSON(http.StatusBadRequest, map[string]string{
"error": "either book_uuid or book_sha256 is required",
})
}
bookmarksSynced := 0
notesSynced := 0
highlightsSynced := 0
for _, bookmark := range req.Bookmarks {
mediaItemID := pgBookUUID
// If bookmark has its own SHA-256, use it for matching
if bookmark.BookSHA256 != "" && len(bookmark.BookSHA256) == 64 {
mediaItem, err := h.db.GetMediaItemBySHA256(ctx, pgtype.Text{String: bookmark.BookSHA256, Valid: true})
if err == nil {
mediaItemID = mediaItem.ID
}
}
position := ""
if bookmark.Pos0 != "" {
position = bookmark.Pos0
@@ -718,8 +806,8 @@ func (h *KOReaderHandler) SyncBookmarks(c echo.Context) error {
position = fmt.Sprintf("page:%d", bookmark.Page)
}
_, err := h.db.CreateMediaNote(c.Request().Context(), database.CreateMediaNoteParams{
MediaItemID: pgBookUUID,
_, err := h.db.CreateMediaNote(ctx, database.CreateMediaNoteParams{
MediaItemID: mediaItemID,
UserID: pgUserID,
Content: bookmark.Text,
Position: pgtype.Text{String: position, Valid: position != ""},
@@ -731,6 +819,16 @@ func (h *KOReaderHandler) SyncBookmarks(c echo.Context) error {
}
for _, note := range req.Notes {
mediaItemID := pgBookUUID
// If note has its own SHA-256, use it for matching
if note.BookSHA256 != "" && len(note.BookSHA256) == 64 {
mediaItem, err := h.db.GetMediaItemBySHA256(ctx, pgtype.Text{String: note.BookSHA256, Valid: true})
if err == nil {
mediaItemID = mediaItem.ID
}
}
position := ""
if note.Pos0 != "" {
position = note.Pos0
@@ -738,8 +836,8 @@ func (h *KOReaderHandler) SyncBookmarks(c echo.Context) error {
position = fmt.Sprintf("page:%d", note.Page)
}
_, err := h.db.CreateMediaNote(c.Request().Context(), database.CreateMediaNoteParams{
MediaItemID: pgBookUUID,
_, err := h.db.CreateMediaNote(ctx, database.CreateMediaNoteParams{
MediaItemID: mediaItemID,
UserID: pgUserID,
Content: note.Notes,
Position: pgtype.Text{String: position, Valid: position != ""},
@@ -751,6 +849,16 @@ func (h *KOReaderHandler) SyncBookmarks(c echo.Context) error {
}
for _, highlight := range req.Highlights {
mediaItemID := pgBookUUID
// If highlight has its own SHA-256, use it for matching
if highlight.BookSHA256 != "" && len(highlight.BookSHA256) == 64 {
mediaItem, err := h.db.GetMediaItemBySHA256(ctx, pgtype.Text{String: highlight.BookSHA256, Valid: true})
if err == nil {
mediaItemID = mediaItem.ID
}
}
startPos := highlight.Pos0
endPos := highlight.Pos1
if startPos == "" && highlight.Page > 0 {
@@ -763,8 +871,8 @@ func (h *KOReaderHandler) SyncBookmarks(c echo.Context) error {
color = highlight.Color
}
_, err := h.db.CreateMediaHighlight(c.Request().Context(), database.CreateMediaHighlightParams{
MediaItemID: pgBookUUID,
_, err := h.db.CreateMediaHighlight(ctx, database.CreateMediaHighlightParams{
MediaItemID: mediaItemID,
UserID: pgUserID,
SelectionText: highlight.Text,
StartPosition: pgtype.Text{String: startPos, Valid: startPos != ""},
internal/services/ebook_scanner.go
+273 -1
View File
@@ -4,7 +4,11 @@ import (
"bookmann/internal/database"
"bookmann/internal/utils"
"context"
"crypto/sha256"
"encoding/hex"
"encoding/xml"
"fmt"
"io"
"io/fs"
"os"
"path/filepath"
@@ -30,7 +34,26 @@ type EbookMetadata struct {
Contributors string
CoverPath string
ISBN string
ASIN string
Tags string
Phase1HashInfo *HashInfo
Phase1FormatFormats []*FormatInfo
}
type HashInfo struct {
FileSHA256 string
OPFIdentifier string
OPFUUID string
HashConfidence string
}
type FormatInfo struct {
FormatType string
FilePath string
FileSHA256 string
FileSizeBytes int64
MimeType string
}
type EbookScanner struct {
@@ -250,6 +273,19 @@ func (s *EbookScanner) processEbookFile(ctx context.Context, path string) error
metadata = &EbookMetadata{}
}
// Extract hash information (Phase 2)
hashInfo, formatInfo, err := s.extractHashInfo(path)
if err != nil {
fmt.Printf("Warning: failed to extract hash info from %s: %v\n", path, err)
hashInfo = &HashInfo{}
formatInfo = &FormatInfo{}
} else {
metadata.Phase1HashInfo = hashInfo
metadata.Phase1FormatFormats = []*FormatInfo{formatInfo}
fmt.Printf("Hash info for %s: SHA256=%s, OPF_ID=%s, OPF_UUID=%s, Confidence=%s\n",
path, hashInfo.FileSHA256, hashInfo.OPFIdentifier, hashInfo.OPFUUID, hashInfo.HashConfidence)
}
// Try to get metadata from folder structure as fallback/enhancement
// Use the root folder that contains this file
var rootFolder string
@@ -304,11 +340,12 @@ func (s *EbookScanner) processEbookFile(ctx context.Context, path string) error
}
// Create media item in database
_, err = s.db.CreateMediaItem(ctx, database.CreateMediaItemParams{
createdItem, err := s.db.CreateMediaItem(ctx, database.CreateMediaItemParams{
LibraryID: libraryID,
Title: metadata.Title,
Author: pgtype.Text{String: metadata.Author, Valid: metadata.Author != ""},
Isbn: pgtype.Text{String: utils.NormalizeISBN(metadata.ISBN), Valid: metadata.ISBN != ""},
Asin: pgtype.Text{String: metadata.ASIN, Valid: metadata.ASIN != ""},
Description: pgtype.Text{String: metadata.Description, Valid: metadata.Description != ""},
FilePath: path,
FileSize: pgtype.Int8{Int64: info.Size(), Valid: true},
@@ -322,6 +359,38 @@ func (s *EbookScanner) processEbookFile(ctx context.Context, path string) error
Tags: pgtype.Text{String: metadata.Tags, Valid: metadata.Tags != ""},
AddedByAdminID: s.adminID,
})
if err != nil {
return fmt.Errorf("failed to create media item: %v", err)
}
// Update hash information (Phase 2)
if metadata.Phase1HashInfo != nil && metadata.Phase1HashInfo.FileSHA256 != "" {
_, err = s.db.UpdateMediaItemIdentifiers(ctx, database.UpdateMediaItemIdentifiersParams{
ID: createdItem.ID,
FileSha256: pgtype.Text{String: metadata.Phase1HashInfo.FileSHA256, Valid: true},
OpfIdentifier: pgtype.Text{String: metadata.Phase1HashInfo.OPFIdentifier, Valid: metadata.Phase1HashInfo.OPFIdentifier != ""},
OpfUuid: pgtype.Text{String: metadata.Phase1HashInfo.OPFUUID, Valid: metadata.Phase1HashInfo.OPFUUID != ""},
HashConfidence: pgtype.Text{String: metadata.Phase1HashInfo.HashConfidence, Valid: true},
})
if err != nil {
fmt.Printf("Warning: failed to update hash identifiers for %s: %v\n", path, err)
}
}
// Store format information (Phase 2)
for _, format := range metadata.Phase1FormatFormats {
_, err = s.db.CreateMediaItemFormat(ctx, database.CreateMediaItemFormatParams{
MediaItemID: createdItem.ID,
FormatType: format.FormatType,
FilePath: pgtype.Text{String: format.FilePath, Valid: true},
FileSha256: pgtype.Text{String: format.FileSHA256, Valid: true},
FileSizeBytes: pgtype.Int8{Int64: format.FileSizeBytes, Valid: true},
MimeType: pgtype.Text{String: format.MimeType, Valid: true},
})
if err != nil {
fmt.Printf("Warning: failed to create format entry for %s: %v\n", path, err)
}
}
return err
}
@@ -408,6 +477,13 @@ func (s *EbookScanner) extractEPUBMetadata(path string) (*EbookMetadata, error)
break
}
}
if strings.Contains(strings.ToLower(isbn), "asin") {
// Extract ASIN from identifier like "asin:B08XXXXX"
asinParts := strings.SplitN(isbn, ":", 2)
if len(asinParts) == 2 {
metadata.ASIN = asinParts[1]
}
}
}
}
@@ -506,3 +582,199 @@ func (s *EbookScanner) Close() error {
}
return nil
}
// ============================================
// PHASE 2: SCANNER ENHANCEMENTS (Week 1-2)
// ============================================
// calculateFileSHA256 calculates SHA-256 hash using streaming to avoid loading entire file into memory
func (s *EbookScanner) calculateFileSHA256(filePath string) (string, error) {
file, err := os.Open(filePath)
if err != nil {
return "", fmt.Errorf("failed to open file: %v", err)
}
defer file.Close()
hasher := sha256.New()
if _, err := io.Copy(hasher, file); err != nil {
return "", fmt.Errorf("failed to calculate hash: %v", err)
}
return hex.EncodeToString(hasher.Sum(nil)), nil
}
// OPFIdentifier represents an identifier from OPF metadata
type OPFIdentifier struct {
XMLName xml.Name `xml:"identifier"`
ID string `xml:"id,attr"`
Scheme string `xml:"scheme,attr"`
Content string `xml:",chardata"`
}
// OPFMetadata represents parsed OPF metadata
type OPFMetadata struct {
XMLName xml.Name `xml:"package"`
Version string `xml:"version,attr"`
Identifiers []OPFIdentifier `xml:"metadata>identifier"`
}
// extractOPFIdentifiers extracts identifiers from EPUB OPF file
func (s *EbookScanner) extractOPFIdentifiers(epubPath string) (opfIdentifier, opfUUID string, confidence string, err error) {
book, err := epub.ReadBook(epubPath)
if err != nil {
return "", "", "", fmt.Errorf("failed to open EPUB: %v", err)
}
// Extract OPF identifiers using go-epub library
identifiers, err := book.MetadataByKey("identifier")
if err != nil || len(identifiers) == 0 {
return "", "", "low", nil
}
var identifier, uuid string
for _, id := range identifiers {
id = strings.TrimSpace(id)
// Check for UUID format (urn:uuid:)
if strings.HasPrefix(strings.ToLower(id), "urn:uuid:") {
uuid = strings.TrimPrefix(strings.ToLower(id), "urn:uuid:")
continue
}
// Check if it's a plain UUID (8-4-4-4-12 format)
if isValidUUID(id) {
uuid = id
continue
}
// Check for ISBN
if strings.Contains(strings.ToLower(id), "isbn") {
isbn := s.extractISBNFromIdentifier(id)
if isbn != "" {
identifier = isbn
continue
}
}
// Use first identifier as fallback
if identifier == "" && id != "" {
identifier = id
}
}
confidence = s.determineHashConfidence(uuid, identifier)
return identifier, uuid, confidence, nil
}
// isValidUUID checks if string is a valid UUID (8-4-4-4-12 format)
func isValidUUID(idStr string) bool {
uuidRegex := regexp.MustCompile(`^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$`)
return uuidRegex.MatchString(idStr)
}
// extractISBNFromIdentifier extracts ISBN from identifier string
func (s *EbookScanner) extractISBNFromIdentifier(id string) string {
id = strings.TrimSpace(id)
// Remove "isbn:" prefix if present
if strings.HasPrefix(strings.ToLower(id), "isbn:") {
id = strings.TrimPrefix(strings.ToLower(id), "isbn:")
}
// Remove hyphens and spaces
isbn := regexp.MustCompile(`[\s-]`).ReplaceAllString(id, "")
// Check if it's a valid ISBN-10 or ISBN-13
if len(isbn) == 10 || len(isbn) == 13 {
return isbn
}
return ""
}
// determineHashConfidence determines confidence level based on available identifiers
func (s *EbookScanner) determineHashConfidence(uuid, identifier string) string {
if uuid != "" && isValidUUID(uuid) {
return "high"
}
if identifier != "" && (strings.Contains(strings.ToLower(identifier), "isbn") || len(identifier) >= 10) {
return "medium"
}
return "low"
}
// detectFormatType detects the format type based on file extension and content
func (s *EbookScanner) detectFormatType(filePath string) string {
base := strings.ToLower(filepath.Base(filePath))
// Check for compound extensions first (like .kepub.epub)
if strings.HasSuffix(base, ".kepub.epub") {
return "kepub"
}
ext := strings.ToLower(filepath.Ext(filePath))
switch ext {
case ".epub":
return "epub"
case ".kepub":
return "kepub"
case ".pdf":
return "pdf"
case ".cbz", ".cbr", ".cb7", ".cbt":
return "comic_archive"
case ".mobi":
return "mobi"
case ".azw", ".azw3":
return "kfx"
case ".txt":
return "txt"
case ".fb2":
return "fb2"
default:
return "unknown"
}
}
// extractHashInfo calculates hash and extracts OPF identifiers for a file
func (s *EbookScanner) extractHashInfo(filePath string) (*HashInfo, *FormatInfo, error) {
// Calculate SHA-256
fileSHA256, err := s.calculateFileSHA256(filePath)
if err != nil {
return nil, nil, fmt.Errorf("failed to calculate SHA-256: %v", err)
}
// Get file info
info, err := os.Stat(filePath)
if err != nil {
return nil, nil, fmt.Errorf("failed to get file info: %v", err)
}
// Extract OPF identifiers for EPUB files
var opfIdentifier, opfUUID, confidence string
if strings.HasSuffix(strings.ToLower(filePath), ".epub") {
opfIdentifier, opfUUID, confidence, err = s.extractOPFIdentifiers(filePath)
if err != nil {
// Non-fatal error, continue with low confidence
confidence = "low"
}
}
hashInfo := &HashInfo{
FileSHA256: fileSHA256,
OPFIdentifier: opfIdentifier,
OPFUUID: opfUUID,
HashConfidence: confidence,
}
formatInfo := &FormatInfo{
FormatType: s.detectFormatType(filePath),
FilePath: filePath,
FileSHA256: fileSHA256,
FileSizeBytes: info.Size(),
MimeType: s.getMimeType(filePath),
}
return hashInfo, formatInfo, nil
}