feat(sync): Add Kobo/Koreader sync and conflict handling
- Add Kobo markup/sync endpoints for bookshelves - Add Koreader progress sync with SHA-256 support - Add sync conflict detection and resolution - Update ebook scanner for better file matching
This commit is contained in:
@@ -4,7 +4,11 @@ import (
|
||||
"bookmann/internal/database"
|
||||
"bookmann/internal/utils"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/xml"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
@@ -30,7 +34,26 @@ type EbookMetadata struct {
|
||||
Contributors string
|
||||
CoverPath string
|
||||
ISBN string
|
||||
ASIN string
|
||||
Tags string
|
||||
|
||||
Phase1HashInfo *HashInfo
|
||||
Phase1FormatFormats []*FormatInfo
|
||||
}
|
||||
|
||||
type HashInfo struct {
|
||||
FileSHA256 string
|
||||
OPFIdentifier string
|
||||
OPFUUID string
|
||||
HashConfidence string
|
||||
}
|
||||
|
||||
type FormatInfo struct {
|
||||
FormatType string
|
||||
FilePath string
|
||||
FileSHA256 string
|
||||
FileSizeBytes int64
|
||||
MimeType string
|
||||
}
|
||||
|
||||
type EbookScanner struct {
|
||||
@@ -250,6 +273,19 @@ func (s *EbookScanner) processEbookFile(ctx context.Context, path string) error
|
||||
metadata = &EbookMetadata{}
|
||||
}
|
||||
|
||||
// Extract hash information (Phase 2)
|
||||
hashInfo, formatInfo, err := s.extractHashInfo(path)
|
||||
if err != nil {
|
||||
fmt.Printf("Warning: failed to extract hash info from %s: %v\n", path, err)
|
||||
hashInfo = &HashInfo{}
|
||||
formatInfo = &FormatInfo{}
|
||||
} else {
|
||||
metadata.Phase1HashInfo = hashInfo
|
||||
metadata.Phase1FormatFormats = []*FormatInfo{formatInfo}
|
||||
fmt.Printf("Hash info for %s: SHA256=%s, OPF_ID=%s, OPF_UUID=%s, Confidence=%s\n",
|
||||
path, hashInfo.FileSHA256, hashInfo.OPFIdentifier, hashInfo.OPFUUID, hashInfo.HashConfidence)
|
||||
}
|
||||
|
||||
// Try to get metadata from folder structure as fallback/enhancement
|
||||
// Use the root folder that contains this file
|
||||
var rootFolder string
|
||||
@@ -304,11 +340,12 @@ func (s *EbookScanner) processEbookFile(ctx context.Context, path string) error
|
||||
}
|
||||
|
||||
// Create media item in database
|
||||
_, err = s.db.CreateMediaItem(ctx, database.CreateMediaItemParams{
|
||||
createdItem, err := s.db.CreateMediaItem(ctx, database.CreateMediaItemParams{
|
||||
LibraryID: libraryID,
|
||||
Title: metadata.Title,
|
||||
Author: pgtype.Text{String: metadata.Author, Valid: metadata.Author != ""},
|
||||
Isbn: pgtype.Text{String: utils.NormalizeISBN(metadata.ISBN), Valid: metadata.ISBN != ""},
|
||||
Asin: pgtype.Text{String: metadata.ASIN, Valid: metadata.ASIN != ""},
|
||||
Description: pgtype.Text{String: metadata.Description, Valid: metadata.Description != ""},
|
||||
FilePath: path,
|
||||
FileSize: pgtype.Int8{Int64: info.Size(), Valid: true},
|
||||
@@ -322,6 +359,38 @@ func (s *EbookScanner) processEbookFile(ctx context.Context, path string) error
|
||||
Tags: pgtype.Text{String: metadata.Tags, Valid: metadata.Tags != ""},
|
||||
AddedByAdminID: s.adminID,
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create media item: %v", err)
|
||||
}
|
||||
|
||||
// Update hash information (Phase 2)
|
||||
if metadata.Phase1HashInfo != nil && metadata.Phase1HashInfo.FileSHA256 != "" {
|
||||
_, err = s.db.UpdateMediaItemIdentifiers(ctx, database.UpdateMediaItemIdentifiersParams{
|
||||
ID: createdItem.ID,
|
||||
FileSha256: pgtype.Text{String: metadata.Phase1HashInfo.FileSHA256, Valid: true},
|
||||
OpfIdentifier: pgtype.Text{String: metadata.Phase1HashInfo.OPFIdentifier, Valid: metadata.Phase1HashInfo.OPFIdentifier != ""},
|
||||
OpfUuid: pgtype.Text{String: metadata.Phase1HashInfo.OPFUUID, Valid: metadata.Phase1HashInfo.OPFUUID != ""},
|
||||
HashConfidence: pgtype.Text{String: metadata.Phase1HashInfo.HashConfidence, Valid: true},
|
||||
})
|
||||
if err != nil {
|
||||
fmt.Printf("Warning: failed to update hash identifiers for %s: %v\n", path, err)
|
||||
}
|
||||
}
|
||||
|
||||
// Store format information (Phase 2)
|
||||
for _, format := range metadata.Phase1FormatFormats {
|
||||
_, err = s.db.CreateMediaItemFormat(ctx, database.CreateMediaItemFormatParams{
|
||||
MediaItemID: createdItem.ID,
|
||||
FormatType: format.FormatType,
|
||||
FilePath: pgtype.Text{String: format.FilePath, Valid: true},
|
||||
FileSha256: pgtype.Text{String: format.FileSHA256, Valid: true},
|
||||
FileSizeBytes: pgtype.Int8{Int64: format.FileSizeBytes, Valid: true},
|
||||
MimeType: pgtype.Text{String: format.MimeType, Valid: true},
|
||||
})
|
||||
if err != nil {
|
||||
fmt.Printf("Warning: failed to create format entry for %s: %v\n", path, err)
|
||||
}
|
||||
}
|
||||
|
||||
return err
|
||||
}
|
||||
@@ -408,6 +477,13 @@ func (s *EbookScanner) extractEPUBMetadata(path string) (*EbookMetadata, error)
|
||||
break
|
||||
}
|
||||
}
|
||||
if strings.Contains(strings.ToLower(isbn), "asin") {
|
||||
// Extract ASIN from identifier like "asin:B08XXXXX"
|
||||
asinParts := strings.SplitN(isbn, ":", 2)
|
||||
if len(asinParts) == 2 {
|
||||
metadata.ASIN = asinParts[1]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -506,3 +582,199 @@ func (s *EbookScanner) Close() error {
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ============================================
|
||||
// PHASE 2: SCANNER ENHANCEMENTS (Week 1-2)
|
||||
// ============================================
|
||||
|
||||
// calculateFileSHA256 calculates SHA-256 hash using streaming to avoid loading entire file into memory
|
||||
func (s *EbookScanner) calculateFileSHA256(filePath string) (string, error) {
|
||||
file, err := os.Open(filePath)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to open file: %v", err)
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
hasher := sha256.New()
|
||||
if _, err := io.Copy(hasher, file); err != nil {
|
||||
return "", fmt.Errorf("failed to calculate hash: %v", err)
|
||||
}
|
||||
|
||||
return hex.EncodeToString(hasher.Sum(nil)), nil
|
||||
}
|
||||
|
||||
// OPFIdentifier represents an identifier from OPF metadata
|
||||
type OPFIdentifier struct {
|
||||
XMLName xml.Name `xml:"identifier"`
|
||||
ID string `xml:"id,attr"`
|
||||
Scheme string `xml:"scheme,attr"`
|
||||
Content string `xml:",chardata"`
|
||||
}
|
||||
|
||||
// OPFMetadata represents parsed OPF metadata
|
||||
type OPFMetadata struct {
|
||||
XMLName xml.Name `xml:"package"`
|
||||
Version string `xml:"version,attr"`
|
||||
Identifiers []OPFIdentifier `xml:"metadata>identifier"`
|
||||
}
|
||||
|
||||
// extractOPFIdentifiers extracts identifiers from EPUB OPF file
|
||||
func (s *EbookScanner) extractOPFIdentifiers(epubPath string) (opfIdentifier, opfUUID string, confidence string, err error) {
|
||||
book, err := epub.ReadBook(epubPath)
|
||||
if err != nil {
|
||||
return "", "", "", fmt.Errorf("failed to open EPUB: %v", err)
|
||||
}
|
||||
|
||||
// Extract OPF identifiers using go-epub library
|
||||
identifiers, err := book.MetadataByKey("identifier")
|
||||
if err != nil || len(identifiers) == 0 {
|
||||
return "", "", "low", nil
|
||||
}
|
||||
|
||||
var identifier, uuid string
|
||||
for _, id := range identifiers {
|
||||
id = strings.TrimSpace(id)
|
||||
|
||||
// Check for UUID format (urn:uuid:)
|
||||
if strings.HasPrefix(strings.ToLower(id), "urn:uuid:") {
|
||||
uuid = strings.TrimPrefix(strings.ToLower(id), "urn:uuid:")
|
||||
continue
|
||||
}
|
||||
|
||||
// Check if it's a plain UUID (8-4-4-4-12 format)
|
||||
if isValidUUID(id) {
|
||||
uuid = id
|
||||
continue
|
||||
}
|
||||
|
||||
// Check for ISBN
|
||||
if strings.Contains(strings.ToLower(id), "isbn") {
|
||||
isbn := s.extractISBNFromIdentifier(id)
|
||||
if isbn != "" {
|
||||
identifier = isbn
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
// Use first identifier as fallback
|
||||
if identifier == "" && id != "" {
|
||||
identifier = id
|
||||
}
|
||||
}
|
||||
|
||||
confidence = s.determineHashConfidence(uuid, identifier)
|
||||
|
||||
return identifier, uuid, confidence, nil
|
||||
}
|
||||
|
||||
// isValidUUID checks if string is a valid UUID (8-4-4-4-12 format)
|
||||
func isValidUUID(idStr string) bool {
|
||||
uuidRegex := regexp.MustCompile(`^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$`)
|
||||
return uuidRegex.MatchString(idStr)
|
||||
}
|
||||
|
||||
// extractISBNFromIdentifier extracts ISBN from identifier string
|
||||
func (s *EbookScanner) extractISBNFromIdentifier(id string) string {
|
||||
id = strings.TrimSpace(id)
|
||||
|
||||
// Remove "isbn:" prefix if present
|
||||
if strings.HasPrefix(strings.ToLower(id), "isbn:") {
|
||||
id = strings.TrimPrefix(strings.ToLower(id), "isbn:")
|
||||
}
|
||||
|
||||
// Remove hyphens and spaces
|
||||
isbn := regexp.MustCompile(`[\s-]`).ReplaceAllString(id, "")
|
||||
|
||||
// Check if it's a valid ISBN-10 or ISBN-13
|
||||
if len(isbn) == 10 || len(isbn) == 13 {
|
||||
return isbn
|
||||
}
|
||||
|
||||
return ""
|
||||
}
|
||||
|
||||
// determineHashConfidence determines confidence level based on available identifiers
|
||||
func (s *EbookScanner) determineHashConfidence(uuid, identifier string) string {
|
||||
if uuid != "" && isValidUUID(uuid) {
|
||||
return "high"
|
||||
}
|
||||
if identifier != "" && (strings.Contains(strings.ToLower(identifier), "isbn") || len(identifier) >= 10) {
|
||||
return "medium"
|
||||
}
|
||||
return "low"
|
||||
}
|
||||
|
||||
// detectFormatType detects the format type based on file extension and content
|
||||
func (s *EbookScanner) detectFormatType(filePath string) string {
|
||||
base := strings.ToLower(filepath.Base(filePath))
|
||||
|
||||
// Check for compound extensions first (like .kepub.epub)
|
||||
if strings.HasSuffix(base, ".kepub.epub") {
|
||||
return "kepub"
|
||||
}
|
||||
|
||||
ext := strings.ToLower(filepath.Ext(filePath))
|
||||
|
||||
switch ext {
|
||||
case ".epub":
|
||||
return "epub"
|
||||
case ".kepub":
|
||||
return "kepub"
|
||||
case ".pdf":
|
||||
return "pdf"
|
||||
case ".cbz", ".cbr", ".cb7", ".cbt":
|
||||
return "comic_archive"
|
||||
case ".mobi":
|
||||
return "mobi"
|
||||
case ".azw", ".azw3":
|
||||
return "kfx"
|
||||
case ".txt":
|
||||
return "txt"
|
||||
case ".fb2":
|
||||
return "fb2"
|
||||
default:
|
||||
return "unknown"
|
||||
}
|
||||
}
|
||||
|
||||
// extractHashInfo calculates hash and extracts OPF identifiers for a file
|
||||
func (s *EbookScanner) extractHashInfo(filePath string) (*HashInfo, *FormatInfo, error) {
|
||||
// Calculate SHA-256
|
||||
fileSHA256, err := s.calculateFileSHA256(filePath)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("failed to calculate SHA-256: %v", err)
|
||||
}
|
||||
|
||||
// Get file info
|
||||
info, err := os.Stat(filePath)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("failed to get file info: %v", err)
|
||||
}
|
||||
|
||||
// Extract OPF identifiers for EPUB files
|
||||
var opfIdentifier, opfUUID, confidence string
|
||||
if strings.HasSuffix(strings.ToLower(filePath), ".epub") {
|
||||
opfIdentifier, opfUUID, confidence, err = s.extractOPFIdentifiers(filePath)
|
||||
if err != nil {
|
||||
// Non-fatal error, continue with low confidence
|
||||
confidence = "low"
|
||||
}
|
||||
}
|
||||
|
||||
hashInfo := &HashInfo{
|
||||
FileSHA256: fileSHA256,
|
||||
OPFIdentifier: opfIdentifier,
|
||||
OPFUUID: opfUUID,
|
||||
HashConfidence: confidence,
|
||||
}
|
||||
|
||||
formatInfo := &FormatInfo{
|
||||
FormatType: s.detectFormatType(filePath),
|
||||
FilePath: filePath,
|
||||
FileSHA256: fileSHA256,
|
||||
FileSizeBytes: info.Size(),
|
||||
MimeType: s.getMimeType(filePath),
|
||||
}
|
||||
|
||||
return hashInfo, formatInfo, nil
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user