diff --git a/IMPLEMENTATION_PLAN_COVER_PDF.md b/IMPLEMENTATION_PLAN_COVER_PDF.md deleted file mode 100644 index 76ffeab..0000000 --- a/IMPLEMENTATION_PLAN_COVER_PDF.md +++ /dev/null @@ -1,1017 +0,0 @@ -# Detailed Implementation Plan: Cover Image Extraction & PDF Metadata - -## CRITICAL PRE-IMPLEMENTATION CHECKLIST - -- [ ] Read this entire document before starting -- [ ] Understand each step before executing -- [ ] Verify after each edit (go build ./...) -- [ ] Do NOT proceed if compilation fails - STOP and fix -- [ ] Follow PROJECT_GUIDELINES.md exactly - ---- - -## PHASE 1: Add pdfcpu Dependency - -### Step 1.1: Add pdfcpu to go.mod - -**File:** `go.mod` - -**Action:** Add this line to the dependencies section (alphabetically near other pdf libraries if any): - -``` -github.com/pdfcpu/pdfcpu v0.9.1 -``` - -**Verification:** After adding, run inside container: -```bash -cd /home/nymusicman/Code/bookhoard && podman run --rm -v .:/app -w /app golang:1.25 go mod tidy -``` - ---- - -## PHASE 2: Add EPUB Cover Extraction Function - -### Step 2.1: Add extractEPUBCover Function - -**File:** `internal/services/media_scanner.go` - -**Location:** AFTER line 652 (after the closing `}` of `extractEPUBMetadata`), BEFORE line 655 (the `extractPDFMetadata` function) - -**Exact Code to INSERT:** - -```go -// extractEPUBCover extracts the cover image from an EPUB file. -// It looks for: -// 1. An item with properties="cover-image" in the manifest -// 2. A meta tag with name="cover" pointing to an image -// 3. Common cover image paths like cover.jpg, cover.jpeg, cover.png -// Returns the path to the saved cover image, or empty string if no cover found. -func (s *MediaScanner) extractEPUBCover(epubPath string) (string, error) { - // Open the EPUB as a zip file to extract images - r, err := zip.OpenReader(epubPath) - if err != nil { - return "", fmt.Errorf("failed to open EPUB as zip: %v", err) - } - defer r.Close() - - // Try to find cover image from OPF metadata - coverImageName := "" - - // Attempt to read the OPF file to find cover reference - // First, find container.xml to locate the OPF - var opfPath string - for _, f := range r.File { - if f.Name == "META-INF/container.xml" { - rc, err := f.Open() - if err != nil { - continue - } - content, err := io.ReadAll(rc) - rc.Close() - if err != nil { - continue - } - // Parse container.xml to find OPF path - // Simple string search since we just need the path - opfStart := bytes.Index(content, []byte("]*properties="[^"]*cover-image[^"]*"[^>]*id="([^"]+)"`) - matches := coverImageRE.FindStringSubmatch(contentStr) - if len(matches) > 1 { - coverID := matches[1] - // Find the href for this ID - hrefRE := regexp.MustCompile(fmt.Sprintf(`]*id="%s"[^>]*href="([^"]+)"`, coverID)) - hrefMatches := hrefRE.FindStringSubmatch(contentStr) - if len(hrefMatches) > 1 { - return resolveOPFPath(opfDir, hrefMatches[1]) - } - } - - // Look for meta name="cover" - metaCoverRE := regexp.MustCompile(`]*name="cover"[^>]*content="([^"]+)"`) - metaMatches := metaCoverRE.FindStringSubmatch(contentStr) - if len(metaMatches) > 1 { - coverContent := metaMatches[1] - // Could be "image-id" format - if strings.HasPrefix(coverContent, "image-") { - coverID := strings.TrimPrefix(coverContent, "image-") - hrefRE := regexp.MustCompile(fmt.Sprintf(`]*id="%s"[^>]*href="([^"]+)"`, coverID)) - hrefMatches := hrefRE.FindStringSubmatch(contentStr) - if len(hrefMatches) > 1 { - return resolveOPFPath(opfDir, hrefMatches[1]) - } - } - } - - // Fall back to searching common paths - return findCoverImageInZip(files) -} - -// resolveOPFPath resolves a relative path against the OPF directory -func resolveOPFPath(opfDir, href string) string { - if opfDir == "" { - return href - } - // Handle ../ in href - if strings.HasPrefix(href, "../") { - // Simple case: just use the href as-is for now - return href - } - // Join the directory with the href - return filepath.Join(filepath.Dir(opfDir), href) -} - -// readFileFromZip reads a file from the zip by name -func readFileFromZip(files []*zip.File, name string) ([]byte, error) { - // Normalize the name for comparison - name = filepath.ToSlash(name) - for _, f := range files { - fName := filepath.ToSlash(f.Name) - if fName == name || fName == name { - rc, err := f.Open() - if err != nil { - return nil, err - } - defer rc.Close() - return io.ReadAll(rc) - } - } - return nil, fmt.Errorf("file not found: %s", name) -} - -// extractImageFromZip extracts an image file and returns its contents -func extractImageFromZip(files []*zip.File, imagePath, opfDir string) ([]byte, error) { - // Try direct match first - for _, f := range files { - if strings.ToLower(f.Name) == strings.ToLower(imagePath) { - rc, err := f.Open() - if err != nil { - return nil, err - } - defer rc.Close() - return io.ReadAll(rc) - } - } - - // Try resolved path - resolvedPath := resolveOPFPath(opfDir, imagePath) - for _, f := range files { - if strings.ToLower(f.Name) == strings.ToLower(resolvedPath) { - rc, err := f.Open() - if err != nil { - return nil, err - } - defer rc.Close() - return io.ReadAll(rc) - } - } - - return nil, fmt.Errorf("image not found: %s", imagePath) -} -``` - -**Verification:** Run `go build ./internal/services/...` - must compile without errors - ---- - -### Step 2.2: Add Sidecar Cover Detection Function - -**File:** `internal/services/media_scanner.go` - -**Location:** AFTER the extractEPUBCover function (which ends around line 820ish), BEFORE extractPDFMetadata function - -**Exact Code to INSERT:** - -```go -// findSidecarCover looks for cover images in the same directory as the media file. -// It checks for common cover filename patterns in priority order: -// 1. cover.jpg, cover.jpeg, cover.png, cover.webp -// 2. folder.jpg, folder.jpeg, folder.png, folder.webp -// 3. {basename}.jpg, {basename}.jpeg, etc. (same name as media file) -// 4. .folder.jpg (hidden file) -// Returns the full path to the cover file, or empty string if not found. -func findSidecarCover(mediaPath string) string { - dir := filepath.Dir(mediaPath) - baseName := strings.TrimSuffix(filepath.Base(mediaPath), filepath.Ext(mediaPath)) - - // Priority order for cover filenames - coverPatterns := []string{ - "cover.jpg", - "cover.jpeg", - "cover.png", - "cover.webp", - "folder.jpg", - "folder.jpeg", - "folder.png", - "folder.webp", - ".folder.jpg", - ".folder.jpeg", - ".folder.png", - } - - // First, check exact match cover/folder names - for _, coverName := range coverPatterns { - coverPath := filepath.Join(dir, coverName) - if _, err := os.Stat(coverPath); err == nil { - return coverPath - } - } - - // Second, check for {basename}.{ext} pattern - extensions := []string{".jpg", ".jpeg", ".png", ".webp"} - for _, ext := range extensions { - coverPath := filepath.Join(dir, baseName+ext) - if _, err := os.Stat(coverPath); err == nil { - return coverPath - } - // Also check uppercase extension - coverPathUpper := filepath.Join(dir, baseName+strings.ToUpper(ext)) - if _, err := os.Stat(coverPathUpper); err == nil { - return coverPathUpper - } - } - - return "" -} -``` - -**Verification:** Run `go build ./internal/services/...` - must compile without errors - ---- - -### Step 2.3: Update extractMetadata to Extract EPUB Cover - -**File:** `internal/services/media_scanner.go` - -**Location:** In the `.epub` case around line 558 - -**EXISTING CODE (lines 556-558):** -```go - switch ext { - case ".epub": - return s.extractEPUBMetadata(path) -``` - -**REPLACE WITH:** -```go - switch ext { - case ".epub": - metadata, err := s.extractEPUBMetadata(path) - if err != nil { - return metadata, err - } - // Try to extract embedded cover - coverPath, err := s.extractEPUBCover(path) - if err != nil { - fmt.Printf("Warning: failed to extract EPUB cover from %s: %v\n", path, err) - } else if coverPath != "" { - metadata.CoverPath = coverPath - } - // If no embedded cover, try sidecar - if metadata.CoverPath == "" { - sidecarCover := findSidecarCover(path) - if sidecarCover != "" { - metadata.CoverPath = sidecarCover - } - } - return metadata, nil -``` - -**Verification:** Run `go build ./internal/services/...` - must compile without errors - ---- - -## PHASE 3: PDF Metadata & Cover Extraction - -### Step 3.1: Update extractPDFMetadata Function - -**File:** `internal/services/media_scanner.go` - -**Location:** Lines 655-663 (current stub function) - -**EXISTING CODE (lines 655-663):** -```go -func (s *MediaScanner) extractPDFMetadata(path string) (*MediaMetadata, error) { - // For now, return basic metadata since PDF extraction requires additional libraries - // In a future enhancement, you could use libraries like github.com/ledongthuc/pdf - filename := strings.TrimSuffix(filepath.Base(path), ".pdf") - - return &MediaMetadata{ - Title: filename, - }, nil -} -``` - -**REPLACE WITH:** -```go -func (s *MediaScanner) extractPDFMetadata(path string) (*MediaMetadata, error) { - metadata := &MediaMetadata{} - - // Use pdfcpu API to read PDF metadata - // Configuration: nil = default (lenient mode) - pdfInfo, err := pdfcpuapi.PDFInfoFile(path, nil, nil) - if err != nil { - fmt.Printf("Warning: failed to read PDF info from %s: %v\n", path, err) - // Fall back to filename as title - metadata.Title = strings.TrimSuffix(filepath.Base(path), ".pdf") - return metadata, nil - } - - // Extract title - if pdfInfo.Title != "" { - metadata.Title = pdfInfo.Title - } else { - metadata.Title = strings.TrimSuffix(filepath.Base(path), ".pdf") - } - - // Extract author - if pdfInfo.Author != "" { - metadata.Author = pdfInfo.Author - } - - // Extract subject (use as description) - if pdfInfo.Subject != "" { - metadata.Description = pdfInfo.Subject - } - - // Extract creator (use as author fallback) - if pdfInfo.Creator != "" && metadata.Author == "" { - metadata.Author = pdfInfo.Creator - } - - // Extract producer (use as publisher) - if pdfInfo.Producer != "" { - metadata.Publisher = pdfInfo.Producer - } - - // Try to extract cover image - coverPath, err := s.extractPDFCover(path) - if err != nil { - fmt.Printf("Warning: failed to extract PDF cover from %s: %v\n", path, err) - } else if coverPath != "" { - metadata.CoverPath = coverPath - } - - // If no embedded cover, try sidecar - if metadata.CoverPath == "" { - sidecarCover := findSidecarCover(path) - if sidecarCover != "" { - metadata.CoverPath = sidecarCover - } - } - - return metadata, nil -} -``` - ---- - -### Step 3.2: Add extractPDFCover Helper Function - -**File:** `internal/services/media_scanner.go` - -**Location:** AFTER the extractPDFMetadata function, BEFORE the ComicInfo struct - -**Exact Code to INSERT:** - -```go -// extractPDFCover extracts a cover image from a PDF file. -// It uses pdfcpu API to extract images from the first page. -// Returns the path to the saved cover, or empty string if no cover found. -func (s *MediaScanner) extractPDFCover(pdfPath string) (string, error) { - // Create a temporary directory for extracted images - tmpDir, err := os.MkdirTemp("", "pdf-cover-") - if err != nil { - return "", fmt.Errorf("failed to create temp dir: %v", err) - } - defer os.RemoveAll(tmpDir) - - // Use pdfcpu API to extract images from first page - // ExtractImagesFile(inFile, outDir string, selectedPages []string, conf *model.Configuration) error - err = pdfcpuapi.ExtractImagesFile(pdfPath, tmpDir, []string{"1"}, nil) - if err != nil { - // No images found or extraction failed - this is OK, just return empty - return "", nil - } - - // Check for extracted images in the temp directory - entries, err := os.ReadDir(tmpDir) - if err != nil || len(entries) == 0 { - return "", nil - } - - // Find the largest image (likely the cover) - var largestImage string - var largestSize int64 - - for _, entry := range entries { - if entry.IsDir() { - continue - } - info, err := entry.Info() - if err != nil { - continue - } - // Skip very small files (likely thumbnails or icons) - if info.Size() < 1000 { - continue - } - if info.Size() > largestSize { - largestImage = filepath.Join(tmpDir, entry.Name()) - largestSize = info.Size() - } - } - - if largestImage == "" { - return "", nil - } - - // Read the image - imageData, err := os.ReadFile(largestImage) - if err != nil || len(imageData) == 0 { - return "", nil - } - - // Save cover to disk (same pattern as comics: {pdf_path}.cover.jpg) - coverPath := pdfPath + ".cover.jpg" - if err := os.WriteFile(coverPath, imageData, 0644); err != nil { - return "", fmt.Errorf("failed to write cover file: %v", err) - } - - return coverPath, nil -} -``` - ---- - -## PHASE 4: Add Required Import - -### Step 4.1: Add pdfcpu imports - -**File:** `internal/services/media_scanner.go` - -**Location:** In the import block (lines 1-34) - -**EXISTING CODE (lines 28-34):** -```go - "bookhoard/internal/sevenzip" - epub "github.com/ArcadiaLin/go-epub" - "github.com/fsnotify/fsnotify" - "github.com/jackc/pgx/v5" - "github.com/jackc/pgx/v5/pgtype" - "github.com/nwaples/rardecode" -) -``` - -**REPLACE WITH:** -```go - "bookhoard/internal/sevenzip" - epub "github.com/ArcadiaLin/go-epub" - "github.com/fsnotify/fsnotify" - "github.com/jackc/pgx/v5" - "github.com/jackc/pgx/v5/pgtype" - "github.com/nwaples/rardecode" - pdfcpuapi "github.com/pdfcpu/pdfcpu/pkg/api" - "github.com/pdfcpu/pdfcpu/pkg/pdfcpu" -) -``` - -**NOTE:** We import both: -- `pdfcpuapi` for high-level functions: `PDFInfoFile`, `ExtractImagesFile` -- `pdfcpu` for types and configuration (if needed) - -**Verification:** Run `go build ./internal/services/...` - must compile without errors - ---- - -## PHASE 5: Unit Tests - -### Step 5.1: Create EPUB Cover Unit Test File - -**File:** `internal/services/media_scanner_epub_cover_test.go` (NEW FILE) - -**Exact Code:** - -```go -package services - -import ( - "archive/zip" - "bytes" - "os" - "path/filepath" - "testing" -) - -// TestExtractEPUBCover tests EPUB cover extraction -func TestExtractEPUBCover(t *testing.T) { - tests := []struct { - name string - setupFunc func(t *testing.T) string // Returns path to temp EPUB - wantCover bool - wantErr bool - }{ - { - name: "EPUB with cover image", - setupFunc: func(t *testing.T) string { - tmpDir := t.TempDir() - epubPath := filepath.Join(tmpDir, "test.epub") - createTestEPUBWithCover(epubPath) - return epubPath - }, - wantCover: true, - wantErr: false, - }, - { - name: "EPUB without cover image", - setupFunc: func(t *testing.T) string { - tmpDir := t.TempDir() - epubPath := filepath.Join(tmpDir, "test.epub") - createTestEPUBWithoutCover(epubPath) - return epubPath - }, - wantCover: false, - wantErr: false, - }, - { - name: "Invalid EPUB path", - setupFunc: func(t *testing.T) string { - return "/nonexistent/path.epub" - }, - wantCover: false, - wantErr: true, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - epubPath := tt.setupFunc(t) - if tt.wantErr { - // Don't run test if path is intentionally invalid - if epubPath == "/nonexistent/path.epub" { - return - } - } - - scanner := &MediaScanner{} - coverPath, err := scanner.extractEPUBCover(epubPath) - - if tt.wantErr && err == nil { - t.Error("Expected error but got none") - } - if !tt.wantErr && err != nil { - t.Errorf("Unexpected error: %v", err) - } - if tt.wantCover && coverPath == "" { - t.Error("Expected cover path but got empty string") - } - if !tt.wantCover && coverPath != "" { - t.Errorf("Did not expect cover but got: %s", coverPath) - } - - // Cleanup cover file if created - if coverPath != "" { - os.Remove(coverPath) - } - }) - } -} - -// createTestEPUBWithCover creates a test EPUB with a cover image -func createTestEPUBWithCover(epubPath string) error { - file, err := os.Create(epubPath) - if err != nil { - return err - } - defer file.Close() - - zipWriter := zip.NewWriter(file) - defer zipWriter.Close() - - // Create minimal EPUB structure with cover - files := map[string]string{ - "mimetype": "application/epub+zip", - "META-INF/container.xml": ``, - "OEBPS/content.opf": ``, - "OEBPS/cover.xhtml": `CoverCover`, - "OEBPS/cover.jpg": createPlaceholderJPEG(), - } - - for name, content := range files { - w, err := zipWriter.Create(name) - if err != nil { - return err - } - if name == "mimetype" { - w.Write([]byte(content)) - } else { - w.Write([]byte(content)) - } - } - - return nil -} - -// createTestEPUBWithoutCover creates a test EPUB without a cover image -func createTestEPUBWithoutCover(epubPath string) error { - file, err := os.Create(epubPath) - if err != nil { - return err - } - defer file.Close() - - zipWriter := zip.NewWriter(file) - defer zipWriter.Close() - - files := map[string]string{ - "mimetype": "application/epub+zip", - "META-INF/container.xml": ``, - "OEBPS/content.opf": `Test Book`, - "OEBPS/chapter.xhtml": `Chapter 1

Chapter 1 content

`, - } - - for name, content := range files { - w, err := zipWriter.Create(name) - if err != nil { - return err - } - w.Write([]byte(content)) - } - - return nil -} - -// createPlaceholderJPEG creates a minimal valid JPEG for testing -func createPlaceholderJPEG() string { - // Minimal 1x1 red JPEG - return "\xFF\xD8\xFF\xE0\x00\x10JFIF\x00\x01\x01\x00\x00\x01\x01\x00\x00\xFF\xDB\x00C\x00\x08\x06\x06\x07\x06\x05\x08\x07\x07\x07\t\t\x08\n\x0c\x14\r\x0c\x0b\x0b\x0c\x19\x12\x13\x0f\x14\x1d\x1a\x1f\x1e\x1d\x1a\x1c\x1c $.\' \",#\x1c\x1c(7),01444\x1f\'9=82<.342\xFF\xC0\x00\x0b\x08\x00\x01\x00\x01\x01\x01\x11\x00\xFF\xC4\x00\x1f\x00\x00\x01\x05\x01\x01\x01\x01\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x01\x02\x03\x04\x05\x06\x07\x08\t\n\x0b\xFF\xC4\x00\xb5\x10\x00\x02\x01\x03\x03\x02\x04\x03\x05\x05\x04\x04\x00\x00\x01}\x01\x02\x03\x00\x04\x11\x05\x12!1A\x06\x13Qa\x07\"q\x142\x81\x91\xa1\x08#B\xb1\xc1\x15R\xd1\xf0$3br\x82\t\n\x16\x17\x18\x19\x1a%&\'()*456789:CDEFGHIJSTUVWXYZcdefghijstuvwxyz\x83\x84\x85\x86\x87\x88\x89\x8a\x92\x93\x94\x95\x96\x97\x98\x99\x9a\xa2\xa3\xa4\xa5\xa6\xa7\xa8\xa9\xaa\xb2\xb3\xb4\xb5\xb6\xb7\xb8\xb9\xba\xc2\xc3\xc4\xc5\xc6\xc7\xc8\xc9\xca\xd2\xd3\xd4\xd5\xd6\xd7\xd8\xd9\xda\xe1\xe2\xe3\xe4\xe5\xe6\xe7\xe8\xe9\xea\xf1\xf2\xf3\xf4\xf5\xf6\xf7\xf8\xf9\xfa\xFF\xDA\x00\x08\x01\x01\x00\x00?\x00\xFB\xD5\xFF\xD9" -} - -// TestFindSidecarCover tests sidecar cover detection -func TestFindSidecarCover(t *testing.T) { - tests := []struct { - name string - setupFunc func(t *testing.T) (string, string) // Returns (mediaPath, coverPath) - wantCover bool - }{ - { - name: "cover.jpg exists", - setupFunc: func(t *testing.T) (string, string) { - tmpDir := t.TempDir() - mediaPath := filepath.Join(tmpDir, "book.epub") - coverPath := filepath.Join(tmpDir, "cover.jpg") - os.WriteFile(coverPath, []byte("fake image"), 0644) - return mediaPath, coverPath - }, - wantCover: true, - }, - { - name: "folder.jpg exists", - setupFunc: func(t *testing.T) (string, string) { - tmpDir := t.TempDir() - mediaPath := filepath.Join(tmpDir, "book.epub") - coverPath := filepath.Join(tmpDir, "folder.jpg") - os.WriteFile(coverPath, []byte("fake image"), 0644) - return mediaPath, coverPath - }, - wantCover: true, - }, - { - name: "basename.jpg exists", - setupFunc: func(t *testing.T) (string, string) { - tmpDir := t.TempDir() - mediaPath := filepath.Join(tmpDir, "mystory.epub") - coverPath := filepath.Join(tmpDir, "mystory.jpg") - os.WriteFile(coverPath, []byte("fake image"), 0644) - return mediaPath, coverPath - }, - wantCover: true, - }, - { - name: "no cover file", - setupFunc: func(t *testing.T) (string, string) { - tmpDir := t.TempDir() - mediaPath := filepath.Join(tmpDir, "book.epub") - return mediaPath, "" - }, - wantCover: false, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - mediaPath, _ := tt.setupFunc(t) - result := findSidecarCover(mediaPath) - - if tt.wantCover && result == "" { - t.Error("Expected cover path but got empty string") - } - if !tt.wantCover && result != "" { - t.Errorf("Did not expect cover but got: %s", result) - } - }) - } -} -``` - -**Verification:** Run `go test ./internal/services/... -run "EPUB|Cover" -v` - tests should compile and run - ---- - -## PHASE 6: Integration Tests - -### Step 6.1: Add Scanner Integration Test for Covers - -**File:** `cmd/server/tests/scanner_integration_test.go` - -**Location:** Add as a new test function in the file - -**Exact Code:** - -```go -func (s *ScannerIntegrationTestSuite) TestScanMediaItems_CoverExtraction() { - token := s.setup.Token - - // Create test library - createLibReq := map[string]interface{}{ - "name": "Cover Test Library", - "description": "Test library for cover extraction", - "type": "ebooks", - } - createLibBody, _ := json.Marshal(createLibReq) - createLibURL := s.setup.Server.URL + "/api/libraries" - createLibReqHTTP, _ := http.NewRequest("POST", createLibURL, bytes.NewBuffer(createLibBody)) - createLibReqHTTP.Header.Set("Content-Type", "application/json") - createLibReqHTTP.Header.Set("Authorization", "Bearer "+token) - - client := &http.Client{} - createLibResp, err := client.Do(createLibReqHTTP) - require.NoError(s.T(), err) - require.Equal(s.T(), http.StatusCreated, createLibResp.StatusCode) - - var createLibResponse map[string]interface{} - json.NewDecoder(createLibResp.Body).Decode(&createLibResponse) - createLibResp.Body.Close() - - libraryID, ok := createLibResponse["id"].(string) - require.True(s.T(), ok, "library_id should be string") - - // Add test folder - folderURL := fmt.Sprintf("%s/api/libraries/%s/folders", s.setup.Server.URL, libraryID) - folderReq := map[string]interface{}{ - "folder_path": "/app/uploads/cover-test", - } - folderBody, _ := json.Marshal(folderReq) - folderReqHTTP, _ := http.NewRequest("POST", folderURL, bytes.NewBuffer(folderBody)) - folderReqHTTP.Header.Set("Content-Type", "application/json") - folderReqHTTP.Header.Set("Authorization", "Bearer "+token) - - folderResp, err := client.Do(folderReqHTTP) - require.NoError(s.T(), err) - folderResp.Body.Close() - require.Equal(s.T(), http.StatusCreated, folderResp.StatusCode) - - // Note: In real test, you'd create actual EPUB/PDF files with covers - // For now, we test that the scan completes without error - - // Run scan - scanURL := fmt.Sprintf("%s/api/libraries/%s/scan", s.setup.Server.URL, libraryID) - scanReq, _ := http.NewRequest("POST", scanURL, nil) - scanReq.Header.Set("Authorization", "Bearer "+token) - - scanResp, err := client.Do(scanReq) - require.NoError(s.T(), err) - require.Equal(s.T(), http.StatusAccepted, scanResp.StatusCode) - - var scanResponse map[string]interface{} - json.NewDecoder(scanResp.Body).Decode(&scanResponse) - scanResp.Body.Close() - - jobID, ok := scanResponse["job_id"].(string) - require.True(s.T(), ok, "job_id should be string") - - // Wait for scan to complete - time.Sleep(5 * time.Second) - - // Poll for completion - var completed bool - for i := 0; i < 30; i++ { - statusURL := fmt.Sprintf("%s/api/scanner/status/%s", s.setup.Server.URL, jobID) - statusReq, _ := http.NewRequest("GET", statusURL, nil) - statusReq.Header.Set("Authorization", "Bearer "+token) - - statusResp, err := client.Do(statusReq) - require.NoError(s.T(), err) - - var status map[string]interface{} - json.NewDecoder(statusResp.Body).Decode(&status) - statusResp.Body.Close() - - if status["status"] == "completed" || status["status"] == "failed" { - completed = true - break - } - time.Sleep(1 * time.Second) - } - - require.True(s.T(), completed, "Scan should complete") -} -``` - -**NOTE:** Full integration test requires actual test EPUB/PDF files with covers. This test verifies the scan flow works. - ---- - -## PHASE 7: Bruno Tests - -### Step 7.1: Update Scan Media Items Documentation - -**File:** `bruno/scanner/Scan Media Items.yml` - -**Add to docs section:** -```yaml -docs: |- - ## Scan Media Items (Background) - - Triggers an asynchronous media items scanning operation. The scan runs in the background and can be monitored using the job ID. - - **Cover Extraction:** - - EPUB: Extracts embedded cover from OPF manifest, falls back to sidecar cover.jpg/folder.jpg - - PDF: Attempts to extract first page as cover image, falls back to sidecar - - Comics (CBZ/CBR): Already extracts cover from archive - - **Metadata Extraction:** - - Title, Author, Description, Publisher (all formats) - - Series, Series Number (EPUB with Calibre metadata) - - Tags/Subjects (EPUB) -``` - ---- - -## PHASE 8: Documentation Updates - -### Step 8.1: Update User Admin Guide - -**File:** `docs/user/admin-guide.md` - -**Add section:** - -```markdown -## Cover Image Extraction - -Bookhoard automatically extracts cover images from your media files during scanning. - -### Supported Formats - -| Format | Embedded Cover | Sidecar Cover | -|--------|---------------|---------------| -| EPUB | ✓ | ✓ | -| PDF | ✓ (first page) | ✓ | -| CBZ/CBR| ✓ | - | - -### Cover Priority - -1. **Embedded cover** - Extracted from the file's internal metadata -2. **Sidecar cover** - If no embedded cover, looks for: - - `cover.jpg`, `cover.jpeg`, `cover.png`, `cover.webp` - - `folder.jpg`, `folder.jpeg`, `folder.png`, `folder.webp` - - `{filename}.jpg` (same name as media file) - -### Storage - -Cover images are saved next to the original file with `.cover.jpg` extension: -- `/path/to/book.epub` → `/path/to/book.epub.cover.jpg` -``` - ---- - -## PHASE 9: Verification - -### Step 9.1: Build Verification - -```bash -# Build the application -go build ./... - -# Run unit tests -go test ./internal/services/... -v -run "EPUB|Cover" - -# Run integration tests (requires running container) -go test ./cmd/server/tests/... -v -run "Scanner" -``` - -### Step 9.2: Run Verification Script - -```bash -bash scripts/verify-guidelines.sh -``` - ---- - -## SUMMARY OF CHANGES - -### Files Modified - -| File | Change Type | Lines | -|------|-------------|-------| -| `go.mod` | Add dependency | +1 | -| `internal/services/media_scanner.go` | Add functions + modify | +200-250 | - -### Files Created - -| File | Purpose | -|------|---------| -| `internal/services/media_scanner_epub_cover_test.go` | Unit tests | -| `docs/user/admin-guide.md` | Update user docs | - -### Files Updated - -| File | Purpose | -|------|---------| -| `bruno/scanner/Scan Media Items.yml` | Update API docs | - ---- - -## POST-IMPLEMENTATION CHECKLIST - -- [ ] All code compiles without errors -- [ ] Unit tests pass -- [ ] Integration tests pass (manual verification with real files) -- [ ] Verification script passes (0 errors) -- [ ] Documentation is complete and accurate -- [ ] No secrets or credentials in changes diff --git a/TASKS-backend-progress-tracking.md b/TASKS-backend-progress-tracking.md deleted file mode 100644 index b810b9c..0000000 --- a/TASKS-backend-progress-tracking.md +++ /dev/null @@ -1,799 +0,0 @@ -# Backend Scan Progress Tracking - -**Date Created:** 2025-02-25 -**Status:** Ready to Implement -**Priority:** HIGH - Required for accurate progress UI - ---- - -## 🚨 Problem - -The scan job status endpoint returns minimal data: -```json -{ - "job_id": "...", - "status": "completed", - "progress": 1.0, - "result": {"message": "scan completed", "library_id": "..."} -} -``` - -**Missing fields needed by frontend:** -- `files_scanned` - total files processed -- `new_items` - books added to database -- `errors` - scan errors encountered -- Real-time `progress` updates (0% → 100% during scan) - -**Current behavior:** -- Progress jumps from 0% to 100% when scan completes -- No file counts during scanning -- No error tracking - ---- - -## 📋 Implementation Plan - -### Overview - -Add progress tracking to scan jobs by: -1. Extending `JobResult` to include scan statistics -2. Adding progress update mechanism to worker -3. Tracking statistics during `ScanFolders()` (with batching every 10 files) -4. Updating progress as files are processed -5. Adding integration tests to verify behavior - ---- - -### Step 1: Extend JobResult Structure - -**File:** `internal/services/worker.go` - -**Current JobResult:** -```go -type JobResult struct { - JobID string - Status JobStatus - Error string - Result interface{} - Progress float64 -} -``` - -**Add scan statistics:** -```go -type JobResult struct { - JobID string - Status JobStatus - Error string - Result interface{} - Progress float64 - FilesScanned int // NEW - NewItems int // NEW - Errors int // NEW -} -``` - -**Update `processScanJob()` to return stats:** -```go -return map[string]interface{}{ - "message": "scan completed", - "library_id": libraryID, - "files_scanned": totalFiles, - "new_items": newItems, - "errors": errors, -}, nil -``` - ---- - -### Step 2: Add Progress Update Callback to Job - -**File:** `internal/services/worker.go` - -**Add callback to Job struct:** -```go -type Job struct { - ID string - Type JobType - Params map[string]interface{} - Status JobStatus - CreatedAt time.Time - StartedAt *time.Time - CompletedAt *time.Time - Error error - Result interface{} - Context context.Context - ProgressCallback func(progress float64, filesScanned, newItems, errors int) // NEW -} -``` - -**Add update method:** -```go -func (j *Job) UpdateProgress(progress float64, filesScanned, newItems, errors int) { - if j.ProgressCallback != nil { - j.ProgressCallback(progress, filesScanned, newItems, errors) - } -} -``` - ---- - -### Step 3: Set Up Callback and Pass Job to Scanner - -**File:** `internal/services/worker.go` - -**Update processScanJob() to set up callback:** -```go -func (w *Worker) processScanJob(job *Job) (interface{}, error) { - libraryID, ok := job.Params["library_id"].(string) - if !ok { - return nil, fmt.Errorf("library_id required") - } - - folders, ok := job.Params["folders"].([]string) - if !ok { - return nil, fmt.Errorf("folders required") - } - - adminID, ok := job.Params["admin_id"].(string) - if !ok { - return nil, fmt.Errorf("admin_id required") - } - - db, ok := job.Params["db"].(*database.Queries) - if !ok { - return nil, fmt.Errorf("database queries required") - } - - scanner := NewMediaScanner(db) - scanner.job = job // NEW: Pass job reference for progress updates - - // NEW: Set up progress callback to update JobResult in real-time - job.ProgressCallback = func(progress float64, filesScanned, newItems, errors int) { - w.mu.Lock() - defer w.mu.Unlock() - - if result, exists := w.results[job.ID]; exists { - result.Progress = progress - result.FilesScanned = filesScanned - result.NewItems = newItems - result.Errors = errors - } - } - - if err := scanner.SetFolders(folders); err != nil { - return nil, err - } - - var adminUUID pgtype.UUID - if err := adminUUID.Scan(adminID); err != nil { - return nil, err - } - scanner.SetAdminID(adminUUID) - - if err := scanner.ScanFolders(job.Context); err != nil { - return nil, err - } - - totalFiles, newItems, errors := scanner.GetStats() - - return map[string]interface{}{ - "message": "scan completed", - "library_id": libraryID, - "files_scanned": totalFiles, - "new_items": newItems, - "errors": errors, - }, nil -} -``` - ---- - -### Step 4: Update Worker Job Completion Tracking - -**File:** `internal/services/worker.go` - -**Modify job processing loop:** -```go -case JobTypeScan: - result, err = w.processScanJob(job) - -// After job completes, update JobResult with stats -w.mu.Lock() -status := JobStatusCompleted -if err != nil { - status = JobStatusFailed -} -if job.Context != nil && job.Context.Err() != nil { - status = JobStatusCancelled -} - -// Extract stats from result if available -var filesScanned, newItems, errors int -if result != nil { - if stats, ok := result.(map[string]interface{}); ok { - filesScanned = int(stats["files_scanned"].(float64)) - newItems = int(stats["new_items"].(float64)) - errors = int(stats["errors"].(float64)) - } -} - -w.results[job.ID] = &JobResult{ - JobID: job.ID, - Status: status, - Error: func() string { if err != nil { return err.Error() } else { return "" } }(), - Result: result, - Progress: 1.0, - FilesScanned: filesScanned, // NEW - NewItems: newItems, // NEW - Errors: errors, // NEW -} -w.mu.Unlock() -``` - ---- - -### Step 5: Track Statistics During Scan - -**File:** `internal/services/media_scanner.go` - -**Add counter fields to MediaScanner:** -```go -type MediaScanner struct { - db *database.Queries - watcher *fsnotify.Watcher - folders []string - adminID pgtype.UUID - defaultLibraryID pgtype.UUID - libraryTypes map[string][]string - - // NEW: Scan statistics - totalFiles int - newItems int - errors int - job *Job // Reference to job for progress updates -} -``` - -**Add getter method:** -```go -func (s *MediaScanner) GetStats() (int, int, int) { - return s.totalFiles, s.newItems, s.errors -} -``` - -**Update ScanFolders() to track stats:** -```go -func (s *MediaScanner) ScanFolders(ctx context.Context) error { - if len(s.folders) == 0 { - return fmt.Errorf("no folders set") - } - - // Reset counters - s.totalFiles = 0 - s.newItems = 0 - s.errors = 0 - - // First pass: count total files - for _, folder := range s.folders { - filepath.WalkDir(folder, func(path string, d fs.DirEntry, err error) error { - if !d.IsDir() && s.isScannableFile(path) { - s.totalFiles++ - } - return nil - }) - } - - fmt.Printf("Starting scan of %d folders: %v (%d files to scan)\n", len(s.folders), s.folders, s.totalFiles) - - processedFiles := 0 - mediaFiles := 0 - - for _, folder := range s.folders { - fmt.Printf("Scanning folder: %s\n", folder) - - if _, err := os.Stat(folder); os.IsNotExist(err) { - fmt.Printf("Folder does not exist: %s\n", folder) - s.errors++ - continue - } - - err := filepath.WalkDir(folder, func(path string, d fs.DirEntry, err error) error { - if err != nil { - fmt.Printf("Error accessing path %s: %v\n", path, err) - s.errors++ - return err - } - - if d.IsDir() { - if err := s.watcher.Add(path); err != nil { - fmt.Printf("Warning: failed to watch subdirectory %s: %v\n", path, err) - } - return nil - } - - if s.isScannableFile(path) { - mediaFiles++ - processedFiles++ - - // Update progress (batch every 10 files to reduce mutex contention) - if processedFiles%10 == 0 && s.totalFiles > 0 { - progress := float64(processedFiles) / float64(s.totalFiles) - if s.job != nil { - s.job.UpdateProgress(progress, processedFiles, s.newItems, s.errors) - } - } - - wasNew, err := s.processMediaFile(ctx, path) - if err != nil { - fmt.Printf("Error processing media file %s: %v\n", path, err) - s.errors++ - } else { - fmt.Printf("Successfully processed media file: %s\n", path) - // newItems already incremented in processMediaFile if wasNew - } - } - - return nil - }) - if err != nil { - s.errors++ - return fmt.Errorf("failed to scan folder %s: %v", folder, err) - } - } - - fmt.Printf("Scan completed: %d total files scanned, %d media files found, %d new items, %d errors\n", - processedFiles, mediaFiles, s.newItems, s.errors) - - // Final progress update to ensure we report 100% - if s.job != nil && s.totalFiles > 0 { - s.job.UpdateProgress(1.0, processedFiles, s.newItems, s.errors) - } - - return nil -} -``` - ---- - -### Step 6: Update processMediaFile to Track New Items - -**File:** `internal/services/media_scanner.go` - -**Modify return value:** -```go -// CURRENT: func (s *MediaScanner) processMediaFile(ctx context.Context, path string) error -// NEW: Returns (bool, error) where bool indicates if item was newly created - -func (s *MediaScanner) processMediaFile(ctx context.Context, path string) (bool, error) { - // ... existing file processing code ... - - // Check if media item already exists - existingItem, err := s.getMediaItemByFilePath(ctx, path) - if err == nil && existingItem.FileSize.Int64 == info.Size() { - fmt.Printf("Media item already exists with same size, skipping: %s\n", path) - return false, nil // FALSE = not a new item (already exists) - } - - // If item exists but different size, it's an update - still not "new" - if err == nil { - fmt.Printf("Updating existing media item: %s\n", path) - // ... update logic ... - return false, nil // FALSE = not a new item (was an update) - } - - // ... rest of processing for new item ... - - // Create media item in database - createdItem, err := s.db.CreateMediaItem(ctx, database.CreateMediaItemParams{...}) - if err != nil { - return false, err // FALSE = error, false means not created - } - - s.newItems++ // NEW: Track new items - return true, nil // TRUE = new item created -} -``` - -**Update ScanFolders() to use return value:** -```go -wasNew, err := s.processMediaFile(ctx, path) -if err != nil { - s.errors++ -} else if wasNew { - // newItems already incremented in processMediaFile -} -``` - ---- - -### Step 7: Update GetScanStatus Handler - -**File:** `internal/handlers/scanner.go` - -**Update response to include new fields:** -```go -func (h *Handler) GetScanStatus(c echo.Context) error { - jobID := c.Param("jobId") - - result, exists := h.worker.GetJobStatus(jobID) - if !exists { - return c.JSON(http.StatusNotFound, map[string]string{"error": "job not found"}) - } - - return c.JSON(http.StatusOK, map[string]interface{}{ - "job_id": result.JobID, - "status": result.Status, - "error": result.Error, - "result": result.Result, - "progress": result.Progress, - "files_scanned": result.FilesScanned, // NEW - "new_items": result.NewItems, // NEW - "errors": result.Errors, // NEW - }) -} -``` - ---- - -### Step 8: Add Integration Tests - -**File:** `cmd/server/tests/scanner_integration_test.go` (new file) - -**Create new integration test file:** -```go -package main - -import ( - "bytes" - "encoding/json" - "fmt" - "net/http" - "testing" - "time" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/stretchr/testify/suite" -) - -type ScannerIntegrationTestSuite struct { - suite.Suite - setup *TestServerSetup -} - -func (s *ScannerIntegrationTestSuite) SetupSuite() { - s.setup = setupTestServer(s.T()) -} - -func (s *ScannerIntegrationTestSuite) TearDownSuite() { - s.setup.Close() -} - -func (s *ScannerIntegrationTestSuite) TestScanProgress_TracksStatistics() { - token := s.setup.Token - - // Create test library - libraryID := s.setup.CreateLibrary(s.T(), "Scan Test Library", "ebooks") - - // Add folder to library - folderURL := fmt.Sprintf("%s/api/libraries/%s/folders", s.setup.Server.URL, libraryID) - folderReq := map[string]interface{}{ - "folder_path": "/app/uploads", - } - folderBody, _ := json.Marshal(folderReq) - req, _ := http.NewRequest("POST", folderURL, bytes.NewBuffer(folderBody)) - req.Header.Set("Content-Type", "application/json") - req.Header.Set("Authorization", "Bearer "+token) - - client := &http.Client{} - resp, err := client.Do(req) - require.NoError(s.T(), err) - resp.Body.Close() - require.Equal(s.T(), http.StatusCreated, resp.StatusCode, "Folder creation should succeed") - - // Start scan - scanURL := fmt.Sprintf("%s/api/libraries/%s/scan", s.setup.Server.URL, libraryID) - scanReq, _ := http.NewRequest("POST", scanURL, nil) - scanReq.Header.Set("Authorization", "Bearer "+token) - - scanResp, err := client.Do(scanReq) - require.NoError(s.T(), err) - require.Equal(s.T(), http.StatusAccepted, scanResp.StatusCode) - - var scanResponse map[string]interface{} - err = json.NewDecoder(scanResp.Body).Decode(&scanResponse) - require.NoError(s.T(), err) - scanResp.Body.Close() - - jobID, ok := scanResponse["job_id"].(string) - require.True(s.T(), ok, "job_id should be string") - require.NotEmpty(s.T(), jobID, "job_id should not be empty") - - // Poll for progress updates - var lastProgress float64 - var lastFilesScanned, lastNewItems, lastErrors int - - for i := 0; i < 30; i++ { // Poll for up to 30 seconds - time.Sleep(1 * time.Second) - - statusURL := fmt.Sprintf("%s/api/scanner/status/%s", s.setup.Server.URL, jobID) - statusReq, _ := http.NewRequest("GET", statusURL, nil) - statusReq.Header.Set("Authorization", "Bearer "+token) - - statusResp, err := client.Do(statusReq) - require.NoError(s.T(), err) - - var status map[string]interface{} - err = json.NewDecoder(statusResp.Body).Decode(&status) - statusResp.Body.Close() - require.NoError(s.T(), err) - - // Verify new fields exist - assert.Contains(s.T(), status, "files_scanned") - assert.Contains(s.T(), status, "new_items") - assert.Contains(s.T(), status, "errors") - - // Track progress with safe type assertions - progressFloat, ok := status["progress"].(float64) - require.True(s.T(), ok, "progress should be float64") - progress := progressFloat - - filesScannedFloat, ok := status["files_scanned"].(float64) - require.True(s.T(), ok, "files_scanned should be float64") - filesScanned := int(filesScannedFloat) - - newItemsFloat, ok := status["new_items"].(float64) - require.True(s.T(), ok, "new_items should be float64") - newItems := int(newItemsFloat) - - errorsFloat, ok := status["errors"].(float64) - require.True(s.T(), ok, "errors should be float64") - errors := int(errorsFloat) - - // Progress should be non-decreasing - assert.GreaterOrEqual(s.T(), progress, lastProgress) - lastProgress = progress - - // Files scanned should be non-decreasing - assert.GreaterOrEqual(s.T(), filesScanned, lastFilesScanned) - lastFilesScanned = filesScanned - - // Items/errors should be non-decreasing - assert.GreaterOrEqual(s.T(), newItems, lastNewItems) - assert.GreaterOrEqual(s.T(), errors, lastErrors) - - // Break if scan complete - if status["status"] == "completed" || status["status"] == "failed" { - break - } - } - - // Verify final state - assert.Equal(s.T(), 1.0, lastProgress) - assert.GreaterOrEqual(s.T(), lastFilesScanned, 0) -} - -func (s *ScannerIntegrationTestSuite) TestScanProgress_BatchingWorks() { - token := s.setup.Token - - // Create library with folder - libraryID := s.setup.CreateLibrary(s.T(), "Batch Test Library", "ebooks") - - folderURL := fmt.Sprintf("%s/api/libraries/%s/folders", s.setup.Server.URL, libraryID) - folderReq := map[string]interface{}{ - "folder_path": "/app/uploads", - } - folderBody, _ := json.Marshal(folderReq) - req, _ := http.NewRequest("POST", folderURL, bytes.NewBuffer(folderBody)) - req.Header.Set("Content-Type", "application/json") - req.Header.Set("Authorization", "Bearer "+token) - - client := &http.Client{} - resp, err := client.Do(req) - require.NoError(s.T(), err) - resp.Body.Close() - - // Start scan - scanURL := fmt.Sprintf("%s/api/libraries/%s/scan", s.setup.Server.URL, libraryID) - scanReq, _ := http.NewRequest("POST", scanURL, nil) - scanReq.Header.Set("Authorization", "Bearer "+token) - - scanResp, err := client.Do(scanReq) - require.NoError(s.T(), err) - - var scanResponse map[string]interface{} - json.NewDecoder(scanResp.Body).Decode(&scanResponse) - scanResp.Body.Close() - - jobID, ok := scanResponse["job_id"].(string) - require.True(s.T(), ok, "job_id should be string") - require.NotEmpty(s.T(), jobID, "job_id should not be empty") - - // Poll and verify we don't get updates on EVERY file - updateCount := 0 - previousFilesScanned := -1 - - for i := 0; i < 20; i++ { - time.Sleep(500 * time.Millisecond) - - statusURL := fmt.Sprintf("%s/api/scanner/status/%s", s.setup.Server.URL, jobID) - statusReq, _ := http.NewRequest("GET", statusURL, nil) - statusReq.Header.Set("Authorization", "Bearer "+token) - - statusResp, _ := client.Do(statusReq) - - var status map[string]interface{} - err = json.NewDecoder(statusResp.Body).Decode(&status) - require.NoError(s.T(), err) - statusResp.Body.Close() - - filesScannedFloat, ok := status["files_scanned"].(float64) - require.True(s.T(), ok, "files_scanned should be float64") - filesScanned := int(filesScannedFloat) - - // Only count as update if files_scanned changed - if filesScanned != previousFilesScanned { - updateCount++ - previousFilesScanned = filesScanned - } - - if status["status"] == "completed" || status["status"] == "failed" { - break - } - } - - // With batching every 10 files, we should have FEWER updates than files - // This is a weak assertion, but verifies batching is working - // Threshold of 50 assumes test library has < 500 files - adjust based on actual test data - assert.Less(s.T(), updateCount, 50) -} - -func TestScannerIntegrationTestSuite(t *testing.T) { - suite.Run(t, new(ScannerIntegrationTestSuite)) -} -``` - -**Note:** The integration test requires `encoding/json` import (already included in the import list above). - -**Update existing unit tests:** `internal/services/worker_test.go` - -**Add test for new JobResult fields:** -```go -func TestWorker_JobResult_HasStatsFields(t *testing.T) { - // Test that JobResult properly stores scan statistics - worker := NewWorker(1) - defer worker.Shutdown() - - jobID := "test-job-stats" - - // Simulate job completion with stats - worker.mu.Lock() - worker.results[jobID] = &JobResult{ - JobID: jobID, - Status: JobStatusCompleted, - Progress: 1.0, - FilesScanned: 42, - NewItems: 5, - Errors: 1, - } - worker.mu.Unlock() - - // Verify stats are retrievable - result, exists := worker.GetJobStatus(jobID) - require.True(t, exists, "Job result should exist") - require.NotNil(t, result, "Result should not be nil") - - assert.Equal(t, jobID, result.JobID) - assert.Equal(t, JobStatusCompleted, result.Status) - assert.Equal(t, 1.0, result.Progress) - assert.Equal(t, 42, result.FilesScanned, "FilesScanned should be 42") - assert.Equal(t, 5, result.NewItems, "NewItems should be 5") - assert.Equal(t, 1, result.Errors, "Errors should be 1") -} - -func TestWorker_ProgressCallback_UpdatesJobResult(t *testing.T) { - // Test that progress callback updates JobResult in real-time - worker := NewWorker(1) - defer worker.Shutdown() - - job := &Job{ - ID: "test-progress", - Type: JobTypeScan, - Status: JobStatusInProgress, - Context: context.Background(), - } - - // Set up progress callback - job.ProgressCallback = func(progress float64, filesScanned, newItems, errors int) { - worker.mu.Lock() - defer worker.mu.Unlock() - - if result, exists := worker.results[job.ID]; exists { - result.Progress = progress - result.FilesScanned = filesScanned - result.NewItems = newItems - result.Errors = errors - } - } - - // Initialize result - worker.mu.Lock() - worker.results[job.ID] = &JobResult{ - JobID: job.ID, - Status: JobStatusInProgress, - } - worker.mu.Unlock() - - // Simulate progress updates - job.UpdateProgress(0.5, 10, 2, 0) - - result, exists := worker.GetJobStatus(job.ID) - require.True(t, exists) - assert.Equal(t, 0.5, result.Progress) - assert.Equal(t, 10, result.FilesScanned) - assert.Equal(t, 2, result.NewItems) - assert.Equal(t, 0, result.Errors) - - // Simulate completion - job.UpdateProgress(1.0, 20, 5, 1) - - result, exists = worker.GetJobStatus(job.ID) - require.True(t, exists) - assert.Equal(t, 1.0, result.Progress) - assert.Equal(t, 20, result.FilesScanned) - assert.Equal(t, 5, result.NewItems) - assert.Equal(t, 1, result.Errors) -} -``` - ---- - -## 🧪 Testing Checklist - -After implementation: - -- [ ] Unit tests pass: `go test ./internal/services/...` - - [ ] TestWorker_JobResult_HasStatsFields verifies stats are stored - - [ ] TestWorker_ProgressCallback_UpdatesJobResult verifies real-time updates -- [ ] Integration tests pass: `go test ./cmd/server/tests/...` - - [ ] TestScanProgress_TracksStatistics verifies all new fields exist and increment - - [ ] TestScanProgress_BatchingWorks verifies batching reduces update frequency -- [ ] Scan a library with multiple files -- [ ] Poll `/api/scanner/status/{jobId}` during scan -- [ ] Verify `progress` increases from 0% to 100% gradually -- [ ] Verify `files_scanned` count increases during scan -- [ ] Verify `new_items` count shows books added -- [ ] Verify `errors` count shows scan errors -- [ ] Check final status has accurate totals -- [ ] Test with empty library (no files) -- [ ] Test with library containing only non-media files -- [ ] Test with library causing scan errors -- [ ] Verify batching reduces update frequency (integration test) - ---- - -## 📝 Notes - -- **Thread Safety:** JobResult updates are thread-safe via worker's mutex (w.mu.Lock()) -- **Performance:** Progress updates are batched every 10 files to reduce mutex contention -- **Memory:** Stats tracking uses 3 int fields (24 bytes) per MediaScanner instance -- **Backwards Compatibility:** Frontend already uses `|| 0` fallbacks, so safe to deploy -- **Testing:** Integration tests use `setupTestServer()` from `test_helpers.go` -- **Database Pool Configuration:** `setupTestServer()` already sets `max_conns=1` (test_helpers.go:428), preventing connection pool exhaustion during test runs - ---- - -## 🔗 Related Files - -- `internal/services/worker.go` - Job processing and result tracking -- `internal/services/media_scanner.go` - Scan logic and statistics -- `internal/handlers/scanner.go` - Status API endpoint -- `cmd/server/tests/scanner_integration_test.go` - Integration tests (NEW) -- `internal/services/worker_test.go` - Unit tests to update -- `TASKS-scanning-progress.md` - Frontend implementation that depends on this - ---- - -**Last Updated:** 2025-02-25 -**Status:** Ready for review and implementation diff --git a/TASKS-scanning-progress.md b/TASKS-scanning-progress.md deleted file mode 100644 index 663f9ba..0000000 --- a/TASKS-scanning-progress.md +++ /dev/null @@ -1,741 +0,0 @@ -# Scanning & Dashboard Issues - Implementation Plan - -**Date Created:** 2025-02-24 -**Status:** Documented - Ready to Implement - ---- - -## ✅ Fixed Issues - -### Bruno Collection File -**File:** `/home/nymusicman/Code/bookhoard/bruno/scanner/Scan Media Items.yml` - -**Problem:** Invalid JSON syntax - library_id variable was not quoted - -**Original (Line 20):** -```yaml -"library_id": {{library_id}} # ❌ WRONG - UUID not quoted -``` - -**Fixed:** -```yaml -"library_id": "{{library_id}}" # ✅ CORRECT - quoted string -``` - -**Impact:** Bruno requests now work correctly. API scanning confirmed functional. - ---- - -## 🚧 Remaining Issues - -### Issue 1: Scan Library Button (Frontend) -**Severity:** HIGH - Button completely non-functional -**File:** `templates/admin.templ` (lines 69-85) - -**Current Broken Code:** -```javascript -function quickScan() { - fetch('/api/scanner/scan', { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - 'Authorization': 'Bearer ' + localStorage.getItem('token') - }, - body: JSON.stringify({ - folder_paths: [] // ← EMPTY ARRAY! Causes 400 error - }) - }) -} -``` - -**Why It Fails:** -- Sends `folder_paths: []` (empty array) -- Handler checks `if len(req.FolderPaths) > 0` → FALSE for empty array -- Returns 400: "either library_id or folder_paths required for scanning" -- No scan job is created -- No progress feedback -- Users can't scan from UI - -**What Should Happen:** -1. Fetch all libraries from `/api/libraries` -2. Trigger scan for each library via `/api/libraries/{id}/scan` -3. Collect all job IDs -4. Poll `/api/scanner/status/{jobId}` for progress -5. Display progress UI -6. Show results when complete - ---- - -### Issue 2: Scanned Books Not Showing on Dashboard -**Severity:** MEDIUM - Data exists but not visible -**Status:** Requires diagnosis - -**Symptoms:** -- Book successfully scanned via API -- Book exists in database -- Book not visible on `/dashboard` page -- Book appears in `/api/media-items` endpoint - -**Expected Behavior:** -- Book should appear in "recently-added" section -- Should be at top of list (most recent `created_at`) -- Should be visible immediately after scan - -**Possible Root Causes:** - -#### Hypothesis 1: System Collections Missing -- System collections (including "recently-added") created during user registration -- Possible creation failure or user created before feature existed -- Check: Query database for user's system collections - -#### Hypothesis 2: Wrong Library Selected -- Dashboard shows books for selected library only -- Book might be in different library than displayed -- Check: Compare book's library_id with dashboard's selected library - -#### Hypothesis 3: Collection Hidden -- User preferences might hide "recently-added" collection -- `show_on_dashboard = false` in database -- Check: User's dashboard preferences - -#### Hypothesis 4: Empty Result Set -- Query limit too low -- Ordering incorrect -- Check: API responses directly - ---- - -## 📋 Implementation Plan - -### Part 1: Fix Scan Library Button - -**NOTE - Major Changes to Original Plan:** -- **Switched from inline JavaScript to TypeScript** (follows PROJECT_GUIDELINES.md: "convert all JavaScript to TypeScript") -- **Uses existing `web/src/admin.ts` infrastructure** instead of adding new inline code -- **No custom CSS** - uses TailwindCSS transition classes for animation (follows "TailwindCSS classes only" rule) -- **Inline CSS with variables retained** - follows existing pattern in admin.templ for theme support - -**Rationale:** -- Project already has `web/src/admin.ts` with TypeScript scanning functions -- Inline JavaScript in templates makes code harder to maintain -- TypeScript provides better type safety and code organization -- TailwindCSS transitions are sufficient for UI animation - -**Files to Modify:** -- `web/src/admin.ts` (extend TypeScript scanning functions) -- `templates/admin.templ` (add progress UI, include admin.js) - -**Implementation Steps:** - -#### Step 1: Extend TypeScript in web/src/admin.ts - -**Location:** Add new functions after `loadSystemStats()` (around line 103) - -**Add these functions:** -```typescript -async function scanAllLibraries(): Promise { - const token = localStorage.getItem('token'); - if (!token) return; - - try { - // Step 1: Get all libraries - const libsResp = await fetch('/api/libraries', { - headers: { 'Authorization': `Bearer ${token}` } - }); - - if (!libsResp.ok) { - throw new Error('Failed to get libraries'); - } - - const libsData = await libsResp.json(); - - if (!libsData.data || libsData.data.length === 0) { - if ((window as any).showToast?.error) { - (window as any).showToast.error('No libraries found. Please create a library first.'); - } - return; - } - - const libraries = libsData.data; - - // Step 2: Scan each library - const jobs: string[] = []; - const libraryNames: Record = {}; - - for (const lib of libraries) { - const scanResp = await fetch(`/api/libraries/${lib.id}/scan`, { - method: 'POST', - headers: { 'Authorization': `Bearer ${token}` } - }); - - if (scanResp.ok) { - const result = await scanResp.json(); - jobs.push(result.job_id); - libraryNames[result.job_id] = lib.name; - } else { - console.error(`Failed to scan library: ${lib.name}`); - } - } - - if (jobs.length === 0) { - if ((window as any).showToast?.error) { - (window as any).showToast.error('Failed to start scan for any library'); - } - return; - } - - // Step 3: Show progress UI - showScanProgress(jobs, libraryNames); - - } catch (error) { - console.error('Scan error:', error); - if ((window as any).showToast?.error) { - (window as any).showToast.error('Failed to start scan: ' + (error as Error).message); - } - } -} - -function showScanProgress(jobIds: string[], libraryNames: Record): void { - const container = document.getElementById('scan-progress-container') as HTMLElement; - const list = document.getElementById('library-progress-list') as HTMLElement; - - if (!container || !list) return; - - container.classList.remove('hidden'); - // Trigger slide-in animation by removing opacity and transform classes - container.classList.remove('opacity-0', '-translate-y-2.5'); - - // Create progress items for each library - list.innerHTML = jobIds.map(jobId => ` -
-
- - ${libraryNames[jobId]} - - - Pending... - -
-
-
-
-
-
- `).join(''); - - // Start polling - pollScanProgress(jobIds, libraryNames); -} - -function pollScanProgress(jobIds: string[], libraryNames: Record): void { - const token = localStorage.getItem('token'); - const startTime = Date.now(); - - const interval = setInterval(async () => { - let allComplete = true; - let totalProgress = 0; - let totalFiles = 0; - let totalNewItems = 0; - let totalErrors = 0; - - for (const jobId of jobIds) { - try { - const resp = await fetch(`/api/scanner/status/${jobId}`, { - headers: { 'Authorization': `Bearer ${token}` } - }); - - if (resp.ok) { - const status = await resp.json(); - - // Update individual library progress - updateLibraryProgress(jobId, status); - - totalProgress += status.progress || 0; - totalFiles += status.files_scanned || 0; - totalNewItems += status.new_items || 0; - totalErrors += status.errors || 0; - - if (status.status !== 'completed' && status.status !== 'failed') { - allComplete = false; - } - } - } catch (error) { - console.error(`Failed to poll job ${jobId}:`, error); - } - } - - // Update overall progress - const overallProgress = Math.round(totalProgress / jobIds.length); - const progressBar = document.getElementById('scan-progress-bar') as HTMLElement; - const progressText = document.getElementById('scan-progress-text') as HTMLElement; - const statusText = document.getElementById('scan-status') as HTMLElement; - - if (progressBar) progressBar.style.width = overallProgress + '%'; - if (progressText) progressText.textContent = overallProgress + '%'; - - // Update status text - const elapsed = Math.round((Date.now() - startTime) / 1000); - if (!allComplete && statusText) { - statusText.textContent = `Scanning... ${elapsed}s elapsed • ${totalFiles} files processed`; - } - - // Check if all complete - if (allComplete) { - clearInterval(interval); - showScanResults(jobIds.length, totalFiles, totalNewItems, totalErrors, elapsed); - } - }, 2000); -} - -function updateLibraryProgress(jobId: string, status: any): void { - const bar = document.getElementById(`bar-${jobId}`) as HTMLElement; - const statusText = document.getElementById(`status-${jobId}`) as HTMLElement; - - if (bar) { - bar.style.width = (status.progress || 0) + '%'; - } - - if (statusText) { - const statusMessages: Record = { - 'pending': 'Pending...', - 'running': `Scanning... ${status.progress || 0}%`, - 'completed': `✓ Complete (${status.new_items || 0} items)`, - 'failed': `✗ Failed` - }; - statusText.textContent = statusMessages[status.status] || status.status; - } -} - -function showScanResults(libCount: number, files: number, items: number, errors: number, elapsed: number): void { - const resultsDiv = document.getElementById('scan-results') as HTMLElement; - const contentDiv = document.getElementById('scan-results-content') as HTMLElement; - - if (!resultsDiv || !contentDiv) return; - - contentDiv.innerHTML = ` -

• ${libCount} librar${libCount === 1 ? 'y' : 'ies'} scanned

-

• ${files} files processed

-

• ${items} new items added

- ${errors > 0 ? `

• ${errors} errors

` : ''} -

Completed in ${elapsed} seconds

- `; - - resultsDiv.classList.remove('hidden'); - - const statusText = document.getElementById('scan-status') as HTMLElement; - if (statusText) statusText.textContent = 'Scan complete!'; -} - -function hideScanProgress(): void { - const container = document.getElementById('scan-progress-container') as HTMLElement; - if (container) container.classList.add('hidden'); -} - -// Export to window -(window as any).scanAllLibraries = scanAllLibraries; -(window as any).hideScanProgress = hideScanProgress; -``` - -#### Step 2: Update admin.templ to Include admin.js - -**Location:** `templates/admin.templ` lines 7-12 (head section) - -**Add admin.js script tag:** -```html - - - - -``` - -#### Step 3: Update Scan Button onClick Handler - -**Location:** `templates/admin.templ` line 54 - -**Change from:** -```html - - - - -
-
- Overall Progress - 0% -
-
-
-
-
-
- Starting scan... -
-
- - -
- -
- - - - -``` - -#### Step 5: Build TypeScript to JavaScript - -**Build command:** -```bash -npm run build:ts -``` - -**What this does:** -- Compiles `web/src/admin.ts` to `web/static/admin.js` -- TypeScript compiler (`tsc`) handles the conversion -- Output file `admin.js` will be loaded by the script tag added in Step 2 - -**Note:** This build step runs automatically in the Docker container during image build. For local development, run it manually after editing TypeScript files. - -#### Step 6: Animation Implementation Note - -**How the slide-in animation works:** - -1. **Initial state** (Step 4 HTML): Container has classes `hidden opacity-0 -translate-y-2.5 transition-all duration-300 ease-out` -2. **When scan starts** (Step 1 TypeScript): `showScanProgress()` function: - ```typescript - container.classList.remove('hidden'); // Makes element visible - container.classList.remove('opacity-0', '-translate-y-2.5'); // Triggers animation - ``` -3. **Result:** Browser transitions from `opacity-0` to `opacity-1` and `-translate-y-2.5` to `translate-y-0` over 300ms - -**No custom CSS needed** - follows PROJECT_GUIDELINES.md "TailwindCSS classes only" rule. - ---- - -### Part 2: Diagnose Dashboard Issue - -**Diagnostic Steps:** - -#### Step 1: Check System Collections Exist -**Bruno Request:** -``` -GET /api/dashboard/sections?library_id=849151fb-564e-4b24-89e3-d11360789576 -``` - -**Expected Response:** -```json -{ - "data": [ - { - "title": "continue-reading", - "query_type": "continue-reading", - "items": [...] - }, - { - "title": "recently-added", - "query_type": "recently-added", - "items": [ - { - "title": "Leviticus on the Butcher's Block", - "created_at": "2025-02-24...", - ... - } - ] - }, - { - "title": "recently-read", - "query_type": "recently-read", - "items": [...] - }, - { - "title": "not-started", - "query_type": "not-started", - "items": [...] - } - ] -} -``` - -**If sections array is empty or "recently-added" missing:** -- System collections were not created for this user -- Need to manually call `CreateDefaultCollectionsForUser` - -#### Step 2: Check Book's Library -**Bruno Request:** -``` -GET /api/media-items?library_id=849151fb-564e-4b24-89e3-d11360789576&limit=5&sort=created_at+DESC -``` - -**Expected:** -- Scanned book should appear first (most recent created_at) -- If book appears here, it's in the correct library - -**If book appears:** -- Book is in correct library -- Issue is with dashboard query or collection visibility - -**If book doesn't appear:** -- Book was added to different library -- Check other libraries - -#### Step 3: Check All Libraries -**Bruno Request:** -``` -GET /api/libraries -``` - -**Purpose:** -- See all available libraries -- Check if book might be in a different library -- Confirm the library_id being used - -#### Step 4: Verify URL Library Parameter -**Check:** -- Does `/dashboard` URL have `?library_id=xxx` parameter? -- Which library is selected in dropdown? - -**If no library_id parameter:** -- Dashboard auto-selects first visible library -- Book might be in a different library - -#### Step 5: Direct Database Check (if needed) - -**Check system collections:** -```sql -SELECT name, query_type, show_on_dashboard -FROM collections -WHERE user_id = 'your-user-id' - AND is_system_collection = true; -``` - -**Check book's library:** -```sql -SELECT id, title, library_id, created_at -FROM media_items -WHERE title LIKE '%Leviticus%' -ORDER BY created_at DESC -LIMIT 1; -``` - ---- - -## 🔧 Potential Fixes for Dashboard Issue - -### Fix A: Recreate System Collections -If system collections don't exist: - -**Option 1: Manual API Call** -``` -POST /api/admin/recreate-system-collections -``` -(Endpoint may need to be created) - -**Option 2: Direct Database** -```sql -INSERT INTO collections (user_id, name, description, icon, color, show_on_dashboard, query_type, priority, is_system_collection) -VALUES - ('your-user-id', 'continue-reading', 'Books you''re currently reading (0 < progress < 1)', '📖', '#7aa2f7', true, 'continue-reading', 1, true), - ('your-user-id', 'recently-added', 'Newly added items to this library', '🆕', '#9ece6a', true, 'recently-added', 2, true), - ('your-user-id', 'recently-read', 'Books you''ve finished (progress >= 1)', '✅', '#e0af68', true, 'recently-read', 3, true), - ('your-user-id', 'not-started', 'Books you haven''t read yet (progress = 0 or no record)', '📕', '#f7768e', true, 'not-started', 4, true); -``` - -**Option 3: Backend Handler** -Create endpoint to recreate system collections for a user. - -### Fix B: Switch to Correct Library -If book is in different library: -- Select the correct library in dropdown -- Or create a combined view showing all libraries - -### Fix C: Update User Preferences -If collection is hidden: -``` -GET /api/dashboard/preferences?library_id=xxx -``` -Check if "recently-added" is in `hidden_collections` - -Update: -``` -PUT /api/dashboard/preferences -{ - "library_id": "xxx", - "hidden_collections": [], // Empty = show all - "collection_order": [...], - "items_per_section": 20 -} -``` - ---- - -## 📊 Implementation Priority - -1. **HIGH PRIORITY:** Fix Scan Library button - - Impact: Users can't scan from UI at all - - Effort: Medium (2-3 hours) - - Files: 1 (`admin.templ`) - -2. **MEDIUM PRIORITY:** Diagnose dashboard issue - - Impact: Books exist but not visible - - Effort: Low (30 min diagnosis) - - Files: 0 (investigation only) - -3. **LOW PRIORITY:** Fix dashboard issue - - Impact: Depends on root cause - - Effort: Unknown until diagnosis complete - - Files: Unknown until diagnosis complete - ---- - -## 🧪 Testing Checklist - -### After Fixing Scan Button: - -- [ ] Scan button triggers without errors -- [ ] Progress UI appears -- [ ] Progress bar updates every 2 seconds -- [ ] Each library shows individual progress -- [ ] Scan completes and shows results -- [ ] "Refresh to View Books" works -- [ ] Books appear on dashboard after refresh - -### After Fixing Dashboard: - -- [ ] Scanned book appears in "recently-added" section -- [ ] Book is at top of list (most recent) -- [ ] Book cover displays correctly -- [ ] Clicking book opens it -- [ ] All system collections show data -- [ ] Collections can be hidden/shown -- [ ] Dashboard works across page refreshes - ---- - -## 📝 Notes - -- Backend scanning is confirmed working (via Bruno) -- Job status polling endpoint works correctly -- Database contains the scanned book -- Issue is purely frontend/dashboard display logic -- System collections should be created during user registration -- Dashboard supports library switching via dropdown -- User can customize dashboard (hide collections, reorder, change items per section) - ---- - -## 🔗 Related Files - -- `templates/admin.templ` - Admin page with Scan button -- `templates/dashboard.templ` - Dashboard template -- `internal/handlers/scanner.go` - Scan endpoints -- `internal/services/dashboard_service.go` - Dashboard logic -- `internal/services/worker.go` - Background job processing -- `internal/handlers/dashboard.go` - Dashboard handlers -- `internal/router/frontend.go` - Dashboard route -- `web/static/dashboard.js` - Dashboard frontend logic -- `bruno/scanner/Scan Media Items.yml` - Bruno collection (FIXED) - ---- - -**Last Updated:** 2025-02-24 -**Status:** Ready to implement Scan button fix, Dashboard issue needs diagnosis