From 9fc5d3baea4e8ee65206034aea6e5a5a86020456 Mon Sep 17 00:00:00 2001 From: John O'Keefe Date: Fri, 23 Jan 2026 09:02:53 -0500 Subject: [PATCH] feat: implement multiple folder ebook scanner and API handlers - Create EbookScanner service with multiple folder support - Update scanner to handle array of folder paths instead of single path - Add EPUB metadata extraction for rich ebook information - Implement folder monitoring with fsnotify for real-time updates - Add API endpoints for folder management: - POST /api/auth/ebook-folders (add folder) - GET /api/auth/ebook-folders (list folders) - DELETE /api/auth/ebook-folders/:folderPath (remove folder) - Update scanner endpoints to accept folder_paths array - Add go-epub and fsnotify dependencies for metadata extraction and file watching --- backend/go.mod | 6 +- backend/go.sum | 6 +- backend/internal/handlers/auth.go | 93 ++++++ backend/internal/handlers/ebook.go | 83 +++++- backend/internal/services/ebook_scanner.go | 312 +++++++++++++++++++++ 5 files changed, 493 insertions(+), 7 deletions(-) create mode 100644 backend/internal/services/ebook_scanner.go diff --git a/backend/go.mod b/backend/go.mod index 08229b2..80a7958 100644 --- a/backend/go.mod +++ b/backend/go.mod @@ -3,9 +3,10 @@ module bookmann go 1.25 require ( + github.com/ArcadiaLin/go-epub v0.1.1 + github.com/fsnotify/fsnotify v1.9.0 github.com/go-playground/validator/v10 v10.30.1 - - + github.com/golang-jwt/jwt v3.2.2+incompatible github.com/google/uuid v1.4.0 github.com/jackc/pgx/v5 v5.4.3 github.com/labstack/echo/v4 v4.11.3 @@ -16,7 +17,6 @@ require ( github.com/gabriel-vasile/mimetype v1.4.12 // indirect github.com/go-playground/locales v0.14.1 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect - github.com/golang-jwt/jwt v3.2.2+incompatible // indirect github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect github.com/jackc/puddle/v2 v2.2.1 // indirect diff --git a/backend/go.sum b/backend/go.sum index 58508c8..9d33cf5 100644 --- a/backend/go.sum +++ b/backend/go.sum @@ -1,6 +1,10 @@ +github.com/ArcadiaLin/go-epub v0.1.1 h1:13roe62tarrZ1Y1QTxE+Bzd/NKlChhRzegLVmU5Hgws= +github.com/ArcadiaLin/go-epub v0.1.1/go.mod h1:GY09AG6jnEbsYytkw6VeICLOt25F1GQDGXjYS7cxNhU= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k= +github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= github.com/gabriel-vasile/mimetype v1.4.12 h1:e9hWvmLYvtp846tLHam2o++qitpguFiYCKbn0w9jyqw= github.com/gabriel-vasile/mimetype v1.4.12/go.mod h1:d+9Oxyo1wTzWdyVUPMmXFvp4F9tea18J8ufA774AB3s= github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= @@ -13,8 +17,6 @@ github.com/go-playground/validator/v10 v10.30.1 h1:f3zDSN/zOma+w6+1Wswgd9fLkdwy0 github.com/go-playground/validator/v10 v10.30.1/go.mod h1:oSuBIQzuJxL//3MelwSLD5hc2Tu889bF0Idm9Dg26cM= github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY= github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= -github.com/golang-jwt/jwt/v5 v5.3.0 h1:pv4AsKCKKZuqlgs5sUmn4x8UlGa0kEVt/puTpKx9vvo= -github.com/golang-jwt/jwt/v5 v5.3.0/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= github.com/google/uuid v1.4.0 h1:MtMxsa51/r9yyhkyLsVeVt0B+BGQZzpQiTQ4eHZ8bc4= github.com/google/uuid v1.4.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= diff --git a/backend/internal/handlers/auth.go b/backend/internal/handlers/auth.go index 0d64ab4..cb28537 100644 --- a/backend/internal/handlers/auth.go +++ b/backend/internal/handlers/auth.go @@ -315,6 +315,99 @@ func (h *AuthHandler) ListUsers(c echo.Context) error { return c.JSON(http.StatusOK, userList) } +type AddEbookFolderRequest struct { + FolderPath string `json:"folder_path" validate:"required"` +} + +type EbookFolderResponse struct { + ID string `json:"id"` + UserID string `json:"user_id"` + FolderPath string `json:"folder_path"` + CreatedAt string `json:"created_at"` +} + +// AddEbookFolder handles POST /api/auth/ebook-folders +func (h *AuthHandler) AddEbookFolder(c echo.Context) error { + userID := c.Get("user_id").(string) + userUUID, err := uuid.Parse(userID) + if err != nil { + return c.JSON(http.StatusBadRequest, map[string]string{"error": "invalid user id"}) + } + + var req AddEbookFolderRequest + if err := c.Bind(&req); err != nil { + return c.JSON(http.StatusBadRequest, map[string]string{"error": "invalid request"}) + } + if err := c.Validate(&req); err != nil { + return c.JSON(http.StatusBadRequest, map[string]string{"error": err.Error()}) + } + + folder, err := h.db.AddUserEbookFolder(c.Request().Context(), database.AddUserEbookFolderParams{ + UserID: pgtype.UUID{Bytes: userUUID, Valid: true}, + FolderPath: req.FolderPath, + }) + if err != nil { + return c.JSON(http.StatusInternalServerError, map[string]string{"error": err.Error()}) + } + + return c.JSON(http.StatusCreated, EbookFolderResponse{ + ID: uuid.UUID(folder.ID.Bytes).String(), + UserID: uuid.UUID(folder.UserID.Bytes).String(), + FolderPath: folder.FolderPath, + CreatedAt: folder.CreatedAt.Time.Format("2006-01-02T15:04:05Z07:00"), + }) +} + +// GetEbookFolders handles GET /api/auth/ebook-folders +func (h *AuthHandler) GetEbookFolders(c echo.Context) error { + userID := c.Get("user_id").(string) + userUUID, err := uuid.Parse(userID) + if err != nil { + return c.JSON(http.StatusBadRequest, map[string]string{"error": "invalid user id"}) + } + + folders, err := h.db.GetUserEbookFolders(c.Request().Context(), pgtype.UUID{Bytes: userUUID, Valid: true}) + if err != nil { + return c.JSON(http.StatusInternalServerError, map[string]string{"error": err.Error()}) + } + + var response []EbookFolderResponse + for _, folder := range folders { + response = append(response, EbookFolderResponse{ + ID: uuid.UUID(folder.ID.Bytes).String(), + UserID: uuid.UUID(folder.UserID.Bytes).String(), + FolderPath: folder.FolderPath, + CreatedAt: folder.CreatedAt.Time.Format("2006-01-02T15:04:05Z07:00"), + }) + } + + return c.JSON(http.StatusOK, response) +} + +// DeleteEbookFolder handles DELETE /api/auth/ebook-folders/:folderPath +func (h *AuthHandler) DeleteEbookFolder(c echo.Context) error { + userID := c.Get("user_id").(string) + userUUID, err := uuid.Parse(userID) + if err != nil { + return c.JSON(http.StatusBadRequest, map[string]string{"error": "invalid user id"}) + } + + folderPath := c.Param("folderPath") + if folderPath == "" { + return c.JSON(http.StatusBadRequest, map[string]string{"error": "folder path is required"}) + } + + err = h.db.DeleteUserEbookFolder(c.Request().Context(), database.DeleteUserEbookFolderParams{ + UserID: pgtype.UUID{Bytes: userUUID, Valid: true}, + FolderPath: folderPath, + }) + if err != nil { + return c.JSON(http.StatusInternalServerError, map[string]string{"error": err.Error()}) + } + + return c.JSON(http.StatusOK, map[string]string{"message": "ebook folder removed"}) +} + func (h *AuthHandler) generateJWT(userID string) (string, error) { claims := jwtgo.MapClaims{ "user_id": userID, diff --git a/backend/internal/handlers/ebook.go b/backend/internal/handlers/ebook.go index 008053f..4298973 100644 --- a/backend/internal/handlers/ebook.go +++ b/backend/internal/handlers/ebook.go @@ -2,8 +2,11 @@ package handlers import ( "bookmann/internal/database" + "bookmann/internal/services" + "context" "net/http" "strconv" + "sync" "time" "github.com/google/uuid" @@ -12,12 +15,20 @@ import ( ) type Handler struct { - db *database.Queries + db *database.Queries + scanner *services.EbookScanner + ctx context.Context + cancel context.CancelFunc + mu sync.Mutex } func NewHandler(db *database.Queries) *Handler { + ctx, cancel := context.WithCancel(context.Background()) return &Handler{ - db: db, + db: db, + scanner: services.NewEbookScanner(db), + ctx: ctx, + cancel: cancel, } } @@ -49,6 +60,11 @@ func SetupRoutes(g *echo.Group, db *database.Queries) { g.PUT("/ebooks/:id/rating", h.CreateOrUpdateEbookRating) g.DELETE("/ebooks/:id/rating", h.DeleteEbookRating) g.GET("/ebooks/:id/ratings", h.GetEbookRatings) + + // Scanner routes + g.POST("/scanner/scan", h.ScanEbooks) + g.POST("/scanner/start", h.StartScanner) + g.POST("/scanner/stop", h.StopScanner) } // ListEbooks handles GET /api/ebooks @@ -407,3 +423,66 @@ func (h *Handler) GetEbookRatings(c echo.Context) error { return c.JSON(http.StatusOK, ratings) } + +// ScanEbooksRequest represents the request for scanning ebooks +type ScanEbooksRequest struct { + FolderPaths []string `json:"folder_paths" validate:"required,min=1"` +} + +// ScanEbooks handles POST /api/scanner/scan +func (h *Handler) ScanEbooks(c echo.Context) error { + var req ScanEbooksRequest + if err := c.Bind(&req); err != nil { + return c.JSON(http.StatusBadRequest, map[string]string{"error": "invalid request"}) + } + if err := c.Validate(&req); err != nil { + return c.JSON(http.StatusBadRequest, map[string]string{"error": err.Error()}) + } + + // Set the folder paths for scanning + if err := h.scanner.SetFolders(req.FolderPaths); err != nil { + return c.JSON(http.StatusBadRequest, map[string]string{"error": "invalid folder paths: " + err.Error()}) + } + + // Perform the scan + if err := h.scanner.ScanFolders(h.ctx); err != nil { + return c.JSON(http.StatusInternalServerError, map[string]string{"error": "scan failed: " + err.Error()}) + } + + return c.JSON(http.StatusOK, map[string]string{"message": "scan completed"}) +} + +// StartScanner handles POST /api/scanner/start +func (h *Handler) StartScanner(c echo.Context) error { + h.mu.Lock() + defer h.mu.Unlock() + + var req ScanEbooksRequest + if err := c.Bind(&req); err != nil { + return c.JSON(http.StatusBadRequest, map[string]string{"error": "invalid request"}) + } + if err := c.Validate(&req); err != nil { + return c.JSON(http.StatusBadRequest, map[string]string{"error": err.Error()}) + } + + // Set the folder paths + if err := h.scanner.SetFolders(req.FolderPaths); err != nil { + return c.JSON(http.StatusBadRequest, map[string]string{"error": "invalid folder paths: " + err.Error()}) + } + + // Start watching for changes + h.scanner.WatchChanges(h.ctx) + + return c.JSON(http.StatusOK, map[string]string{"message": "scanner started"}) +} + +// StopScanner handles POST /api/scanner/stop +func (h *Handler) StopScanner(c echo.Context) error { + h.mu.Lock() + defer h.mu.Unlock() + + h.cancel() + h.ctx, h.cancel = context.WithCancel(context.Background()) + + return c.JSON(http.StatusOK, map[string]string{"message": "scanner stopped"}) +} diff --git a/backend/internal/services/ebook_scanner.go b/backend/internal/services/ebook_scanner.go new file mode 100644 index 0000000..62b183e --- /dev/null +++ b/backend/internal/services/ebook_scanner.go @@ -0,0 +1,312 @@ +package services + +import ( + "bookmann/internal/database" + "context" + "fmt" + "io/fs" + "os" + "path/filepath" + "strings" + "time" + + epub "github.com/ArcadiaLin/go-epub" + "github.com/fsnotify/fsnotify" + "github.com/jackc/pgx/v5/pgtype" +) + +type EbookMetadata struct { + Title string + Author string + Description string + Series string + SeriesNumber int32 + Publisher string + PublishDate time.Time + Contributors string + CoverPath string +} + +type EbookScanner struct { + db *database.Queries + watcher *fsnotify.Watcher + folders []string +} + +func NewEbookScanner(db *database.Queries) *EbookScanner { + watcher, err := fsnotify.NewWatcher() + if err != nil { + panic(fmt.Sprintf("Failed to create file watcher: %v", err)) + } + + return &EbookScanner{ + db: db, + watcher: watcher, + } +} + +func (s *EbookScanner) SetFolders(folders []string) error { + s.folders = folders + + // Remove old watch if exists + if s.watcher != nil { + s.watcher.Close() + } + + // Create new watcher + watcher, err := fsnotify.NewWatcher() + if err != nil { + return fmt.Errorf("failed to create watcher: %v", err) + } + s.watcher = watcher + + // Add all folders to watch + for _, folder := range folders { + if err := s.watcher.Add(folder); err != nil { + fmt.Printf("Warning: failed to watch folder %s: %v\n", folder, err) + } + } + + return nil +} + +func (s *EbookScanner) ScanFolders(ctx context.Context) error { + if len(s.folders) == 0 { + return fmt.Errorf("no folders set") + } + + for _, folder := range s.folders { + err := filepath.WalkDir(folder, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + + if d.IsDir() { + // Also watch subdirectories + if err := s.watcher.Add(path); err != nil { + fmt.Printf("Warning: failed to watch subdirectory %s: %v\n", path, err) + } + return nil + } + + // Check if it's an ebook file + if s.isEbookFile(path) { + if err := s.processEbookFile(ctx, path); err != nil { + fmt.Printf("Error processing ebook %s: %v\n", path, err) + } + } + + return nil + }) + if err != nil { + return fmt.Errorf("failed to scan folder %s: %v", folder, err) + } + } + + return nil +} + +func (s *EbookScanner) isEbookFile(path string) bool { + ext := strings.ToLower(filepath.Ext(path)) + switch ext { + case ".epub", ".pdf", ".mobi", ".azw3", ".fb2", ".txt": + return true + default: + return false + } +} + +func (s *EbookScanner) processEbookFile(ctx context.Context, path string) error { + // Get file info + info, err := os.Stat(path) + if err != nil { + return fmt.Errorf("failed to get file info: %v", err) + } + + // Check if ebook already exists in database + existingEbook, err := s.getEbookByFilePath(ctx, path) + if err == nil { + // Ebook exists, check if file has changed (by size) + if existingEbook.FileSize.Int64 != info.Size() { + return s.updateEbook(ctx, existingEbook.ID, path, info) + } + return nil // Skip if already exists and size matches + } else if err.Error() != "sql: no rows in result set" { + // Some other error occurred + return fmt.Errorf("failed to check if ebook exists: %v", err) + } + // Ebook doesn't exist, continue with creation + + // Extract metadata + metadata, err := s.extractMetadata(path) + if err != nil { + fmt.Printf("Warning: failed to extract metadata from %s: %v\n", path, err) + // Continue with basic metadata + metadata = &EbookMetadata{ + Title: filepath.Base(path), + Author: "Unknown", + } + } + + // Create ebook in database + _, err = s.db.CreateEbook(ctx, database.CreateEbookParams{ + Title: metadata.Title, + Author: pgtype.Text{String: metadata.Author, Valid: metadata.Author != ""}, + Description: pgtype.Text{String: metadata.Description, Valid: metadata.Description != ""}, + FilePath: path, + FileSize: pgtype.Int8{Int64: info.Size(), Valid: true}, + MimeType: pgtype.Text{String: s.getMimeType(path), Valid: true}, + CoverImagePath: pgtype.Text{String: metadata.CoverPath, Valid: metadata.CoverPath != ""}, + Series: pgtype.Text{String: metadata.Series, Valid: metadata.Series != ""}, + SeriesNumber: pgtype.Int4{Int32: metadata.SeriesNumber, Valid: metadata.SeriesNumber > 0}, + Publisher: pgtype.Text{String: metadata.Publisher, Valid: metadata.Publisher != ""}, + DatePublished: pgtype.Date{Time: metadata.PublishDate, Valid: !metadata.PublishDate.IsZero()}, + Contributors: pgtype.Text{String: metadata.Contributors, Valid: metadata.Contributors != ""}, + }) + + return err +} + +func (s *EbookScanner) extractMetadata(path string) (*EbookMetadata, error) { + ext := strings.ToLower(filepath.Ext(path)) + + switch ext { + case ".epub": + return s.extractEPUBMetadata(path) + default: + // For other formats, return basic metadata + return &EbookMetadata{ + Title: filepath.Base(path), + }, nil + } +} + +func (s *EbookScanner) extractEPUBMetadata(path string) (*EbookMetadata, error) { + book, err := epub.ReadBook(path) + if err != nil { + return nil, fmt.Errorf("failed to open EPUB: %v", err) + } + + metadata := &EbookMetadata{} + + // Title + if title, err := book.Title(); err == nil && title != "" { + metadata.Title = title + } + + // Author + if authors, err := book.MetadataByKey("creator"); err == nil && len(authors) > 0 { + metadata.Author = authors[0] + } + + // Description + if descriptions, err := book.MetadataByKey("description"); err == nil && len(descriptions) > 0 { + metadata.Description = descriptions[0] + } + + // Publisher + if publishers, err := book.MetadataByKey("publisher"); err == nil && len(publishers) > 0 { + metadata.Publisher = publishers[0] + } + + // Publish date + if dates, err := book.MetadataByKey("date"); err == nil && len(dates) > 0 { + if date, err := time.Parse("2006-01-02", dates[0]); err == nil { + metadata.PublishDate = date + } + } + + // Contributors + if contributors, err := book.MetadataByKey("contributor"); err == nil && len(contributors) > 0 { + metadata.Contributors = strings.Join(contributors, ", ") + } + + return metadata, nil +} + +func (s *EbookScanner) updateEbook(ctx context.Context, ebookID pgtype.UUID, filePath string, info os.FileInfo) error { + metadata, err := s.extractMetadata(filePath) + if err != nil { + fmt.Printf("Warning: failed to extract metadata from %s: %v\n", filePath, err) + metadata = &EbookMetadata{ + Title: filepath.Base(filePath), + } + } + + _, err = s.db.UpdateEbook(ctx, database.UpdateEbookParams{ + ID: ebookID, + Title: metadata.Title, + Author: pgtype.Text{String: metadata.Author, Valid: metadata.Author != ""}, + Isbn: pgtype.Text{}, // Keep existing ISBN + Description: pgtype.Text{String: metadata.Description, Valid: metadata.Description != ""}, + CoverImagePath: pgtype.Text{String: metadata.CoverPath, Valid: metadata.CoverPath != ""}, + Series: pgtype.Text{String: metadata.Series, Valid: metadata.Series != ""}, + SeriesNumber: pgtype.Int4{Int32: metadata.SeriesNumber, Valid: metadata.SeriesNumber > 0}, + Tags: pgtype.Text{}, // Keep existing tags + Asin: pgtype.Text{}, // Keep existing ASIN + DatePublished: pgtype.Date{Time: metadata.PublishDate, Valid: !metadata.PublishDate.IsZero()}, + Publisher: pgtype.Text{String: metadata.Publisher, Valid: metadata.Publisher != ""}, + Contributors: pgtype.Text{String: metadata.Contributors, Valid: metadata.Contributors != ""}, + }) + + return err +} + +func (s *EbookScanner) getEbookByFilePath(ctx context.Context, filePath string) (database.Ebooks, error) { + return s.db.GetEbookByFilePath(ctx, filePath) +} + +func (s *EbookScanner) getMimeType(path string) string { + ext := strings.ToLower(filepath.Ext(path)) + switch ext { + case ".epub": + return "application/epub+zip" + case ".pdf": + return "application/pdf" + case ".mobi": + return "application/x-mobipocket-ebook" + case ".azw3": + return "application/vnd.amazon.ebook" + case ".fb2": + return "application/x-fictionbook+xml" + case ".txt": + return "text/plain" + default: + return "application/octet-stream" + } +} + +func (s *EbookScanner) WatchChanges(ctx context.Context) { + go func() { + for { + select { + case event, ok := <-s.watcher.Events: + if !ok { + return + } + if event.Has(fsnotify.Create) || event.Has(fsnotify.Write) { + if s.isEbookFile(event.Name) { + fmt.Printf("New/modified ebook detected: %s\n", event.Name) + if err := s.processEbookFile(ctx, event.Name); err != nil { + fmt.Printf("Error processing modified ebook %s: %v\n", event.Name, err) + } + } + } + case err, ok := <-s.watcher.Errors: + if !ok { + return + } + fmt.Printf("Watcher error: %v\n", err) + case <-ctx.Done(): + return + } + } + }() +} + +func (s *EbookScanner) Close() error { + if s.watcher != nil { + return s.watcher.Close() + } + return nil +}