Bookhoard/bookhoard
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 12 Wiki Activity

Implement backend scan progress tracking (Steps 1-7)

Browse Source 
Implements comprehensive progress tracking for scan jobs to provide real-time
statistics to the frontend (files_scanned, new_items, errors).

Changes:
1. Extended JobResult struct with new fields:
   - FilesScanned: total files processed
   - NewItems: books added to database
   - Errors: scan errors encountered

2. Added progress callback mechanism:
   - Job.ProgressCallback function field for real-time updates
   - Job.UpdateProgress() method to trigger callbacks
   - Worker stores callback and updates JobResult during scan

3. MediaScanner now tracks statistics:
   - totalFiles, newItems, errors counters
   - GetStats() method to retrieve statistics
   - First pass counts total files for progress calculation
   - Batches progress updates every 10 files (reduces mutex contention)
   - Final update ensures 100% progress is reported

4. Updated processMediaFile signature:
   - Returns (bool, error) instead of (error)
   - true = new item created, false = existing/updated/error
   - Increments newItems counter when creating database entries
   - Updated WatchChanges to handle new return value

5. Worker job completion extracts stats:
   - Parses result map for files_scanned, new_items, errors
   - Stores in final JobResult for API response

6. GetScanStatus API response includes new fields:
   - files_scanned, new_items, errors now in JSON response
   - Frontend can display real-time progress

Design decisions:
- Batching every 10 files balances performance vs. granularity
- Thread-safe via worker mutex (w.mu.Lock/Unlock)
- Callback pattern decouples scanner from job management
- processMediaFile return type allows tracking new vs. updated items
- Maintains backward compatibility (uses || 0 fallbacks in frontend)

Testing:
- All code compiles successfully
- Follows service layer pattern (no business logic in handlers)
- No database schema changes
- Integration tests to be added in Step 8 (separate commit)

Files modified:
- internal/services/worker.go (JobResult, Job struct, processScanJob, processJob)
- internal/services/media_scanner.go (struct fields, GetStats, ScanFolders, processMediaFile)
- internal/handlers/scanner.go (GetScanStatus response)

Related: TASKS-backend-progress-tracking.md Steps 1-7
This commit is contained in:
john-okeefe
2026-02-25 10:52:53 -05:00
parent 88844670af
commit 0295bf7a37
3 changed files with 127 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/handlers/scanner.go
+8 -5
View File
@@ -223,11 +223,14 @@ func (h *Handler) GetScanStatus(c echo.Context) error {
}
return c.JSON(http.StatusOK, map[string]interface{}{
"job_id": result.JobID,
"status": result.Status,
"error": result.Error,
"result": result.Result,
"progress": result.Progress,
"job_id": result.JobID,
"status": result.Status,
"error": result.Error,
"result": result.Result,
"progress": result.Progress,
"files_scanned": result.FilesScanned,
"new_items": result.NewItems,
"errors": result.Errors,
})
}
internal/services/media_scanner.go
+60 -21
View File
@@ -75,6 +75,11 @@ type MediaScanner struct {
adminID pgtype.UUID
defaultLibraryID pgtype.UUID
libraryTypes map[string][]string
totalFiles int
newItems int
errors int
job *Job
}
// NewMediaScanner creates a new media scanner instance
@@ -98,6 +103,10 @@ func (s *MediaScanner) SetAdminID(adminID pgtype.UUID) {
s.adminID = adminID
}
func (s *MediaScanner) GetStats() (int, int, int) {
return s.totalFiles, s.newItems, s.errors
}
func (s *MediaScanner) SetFolders(folders []string) error {
s.folders = folders
@@ -153,42 +162,64 @@ func (s *MediaScanner) ScanFolders(ctx context.Context) error {
return fmt.Errorf("no folders set")
}
fmt.Printf("Starting scan of %d folders: %v\n", len(s.folders), s.folders)
s.totalFiles = 0
s.newItems = 0
s.errors = 0
totalFiles := 0
for _, folder := range s.folders {
filepath.WalkDir(folder, func(path string, d fs.DirEntry, err error) error {
if !d.IsDir() && s.isScannableFile(path) {
s.totalFiles++
}
return nil
})
}
fmt.Printf("Starting scan of %d folders: %v (%d files to scan)\n", len(s.folders), s.folders, s.totalFiles)
processedFiles := 0
mediaFiles := 0
for _, folder := range s.folders {
fmt.Printf("Scanning folder: %s\n", folder)
// Check if folder exists
if _, err := os.Stat(folder); os.IsNotExist(err) {
fmt.Printf("Folder does not exist: %s\n", folder)
s.errors++
continue
}
err := filepath.WalkDir(folder, func(path string, d fs.DirEntry, err error) error {
if err != nil {
fmt.Printf("Error accessing path %s: %v\n", path, err)
s.errors++
return err
}
totalFiles++
if d.IsDir() {
// Also watch subdirectories
if err := s.watcher.Add(path); err != nil {
fmt.Printf("Warning: failed to watch subdirectory %s: %v\n", path, err)
}
return nil
}
// Check if file should be scanned based on library type
if s.isScannableFile(path) {
mediaFiles++
fmt.Printf("Found media file: %s\n", path)
if err := s.processMediaFile(ctx, path); err != nil {
processedFiles++
if processedFiles%10 == 0 && s.totalFiles > 0 {
progress := float64(processedFiles) / float64(s.totalFiles)
if s.job != nil {
s.job.UpdateProgress(progress, processedFiles, s.newItems, s.errors)
}
}
wasNew, err := s.processMediaFile(ctx, path)
if err != nil {
fmt.Printf("Error processing media file %s: %v\n", path, err)
s.errors++
} else if wasNew {
fmt.Printf("Successfully processed media file: %s\n", path)
} else {
fmt.Printf("Successfully processed media file: %s\n", path)
}
@@ -197,11 +228,18 @@ func (s *MediaScanner) ScanFolders(ctx context.Context) error {
return nil
})
if err != nil {
s.errors++
return fmt.Errorf("failed to scan folder %s: %v", folder, err)
}
}
fmt.Printf("Scan completed: %d total files scanned, %d media files found\n", totalFiles, mediaFiles)
fmt.Printf("Scan completed: %d total files scanned, %d media files found, %d new items, %d errors\n",
processedFiles, mediaFiles, s.newItems, s.errors)
if s.job != nil && s.totalFiles > 0 {
s.job.UpdateProgress(1.0, processedFiles, s.newItems, s.errors)
}
return nil
}
@@ -301,14 +339,14 @@ func (s *MediaScanner) extractFolderStructureMetadata(path, rootFolder string) *
return metadata
}
func (s *MediaScanner) processMediaFile(ctx context.Context, path string) error {
func (s *MediaScanner) processMediaFile(ctx context.Context, path string) (bool, error) {
fmt.Printf("Processing media file: %s\n", path)
// Get file info
info, err := os.Stat(path)
if err != nil {
fmt.Printf("Failed to get file info for %s: %v\n", path, err)
return fmt.Errorf("failed to get file info: %v", err)
return false, fmt.Errorf("failed to get file info: %v", err)
}
fmt.Printf("File info for %s: size=%d\n", path, info.Size())
@@ -320,14 +358,14 @@ func (s *MediaScanner) processMediaFile(ctx context.Context, path string) error
// Media item exists, check if file has changed (by size)
if existingItem.FileSize.Int64 != info.Size() {
fmt.Printf("File size changed, updating media item: %s\n", path)
return s.updateMediaItem(ctx, existingItem.ID, path, info)
_ = s.updateMediaItem(ctx, existingItem.ID, path, info)
return false, nil
}
fmt.Printf("Media item already exists with same size, skipping: %s\n", path)
return nil // Skip if already exists and size matches
return false, nil
} else if err != pgx.ErrNoRows {
fmt.Printf("Database error checking media item existence: %v\n", err)
// Some other error occurred
return fmt.Errorf("failed to check if media item exists: %v", err)
return false, fmt.Errorf("failed to check if media item exists: %v", err)
}
fmt.Printf("Media item does not exist in database, creating new entry: %s\n", path)
@@ -434,7 +472,7 @@ func (s *MediaScanner) processMediaFile(ctx context.Context, path string) error
if strings.HasPrefix(path, folder) {
lib, err := s.db.GetLibraryByFolder(ctx, folder)
if err != nil {
return fmt.Errorf("failed to find library for folder %s: %v", folder, err)
return false, fmt.Errorf("failed to find library for folder %s: %v", folder, err)
}
libraryID = lib.LibraryID
break
@@ -442,7 +480,7 @@ func (s *MediaScanner) processMediaFile(ctx context.Context, path string) error
}
if !libraryID.Valid {
return fmt.Errorf("no library found for file path: %s", path)
return false, fmt.Errorf("no library found for file path: %s", path)
}
// Normalize metadata fields for display
@@ -476,7 +514,7 @@ func (s *MediaScanner) processMediaFile(ctx context.Context, path string) error
AddedByAdminID: s.adminID,
})
if err != nil {
return fmt.Errorf("failed to create media item: %v", err)
return false, fmt.Errorf("failed to create media item: %v", err)
}
// Update hash information before database storage
@@ -508,7 +546,8 @@ func (s *MediaScanner) processMediaFile(ctx context.Context, path string) error
}
}
return err
s.newItems++
return true, nil
}
func (s *MediaScanner) extractMetadata(path string) (*MediaMetadata, error) {
@@ -996,7 +1035,7 @@ func (s *MediaScanner) WatchChanges(ctx context.Context) {
// Handle file modifications and creations
if (event.Has(fsnotify.Create) || event.Has(fsnotify.Write)) && s.isScannableFile(event.Name) {
fmt.Printf("New/modified media file detected: %s\n", event.Name)
if err := s.processMediaFile(ctx, event.Name); err != nil {
if _, err := s.processMediaFile(ctx, event.Name); err != nil {
fmt.Printf("Error processing modified media file %s: %v\n", event.Name, err)
}
}
internal/services/worker.go
+59 -19
View File
@@ -28,24 +28,34 @@ const (
)
type Job struct {
ID string
Type JobType
Params map[string]interface{}
Status JobStatus
CreatedAt time.Time
StartedAt *time.Time
CompletedAt *time.Time
Error error
Result interface{}
Context context.Context
ID string
Type JobType
Params map[string]interface{}
Status JobStatus
CreatedAt time.Time
StartedAt *time.Time
CompletedAt *time.Time
Error error
Result interface{}
Context context.Context
ProgressCallback func(progress float64, filesScanned, newItems, errors int)
}
func (j *Job) UpdateProgress(progress float64, filesScanned, newItems, errors int) {
if j.ProgressCallback != nil {
j.ProgressCallback(progress, filesScanned, newItems, errors)
}
}
type JobResult struct {
JobID string
Status JobStatus
Error string
Result interface{}
Progress float64
JobID string
Status JobStatus
Error string
Result interface{}
Progress float64
FilesScanned int
NewItems int
Errors int
}
type Worker struct {
@@ -135,6 +145,15 @@ func (w *Worker) processJob(job *Job) {
status = JobStatusCancelled
}
var filesScanned, newItems, errors int
if result != nil {
if stats, ok := result.(map[string]interface{}); ok {
filesScanned = int(stats["files_scanned"].(float64))
newItems = int(stats["new_items"].(float64))
errors = int(stats["errors"].(float64))
}
}
w.results[job.ID] = &JobResult{
JobID: job.ID,
Status: status,
@@ -145,8 +164,11 @@ func (w *Worker) processJob(job *Job) {
return ""
}
}(),
Result: result,
Progress: 1.0,
Result: result,
Progress: 1.0,
FilesScanned: filesScanned,
NewItems: newItems,
Errors: errors,
}
w.mu.Unlock()
}
@@ -173,6 +195,19 @@ func (w *Worker) processScanJob(job *Job) (interface{}, error) {
}
scanner := NewMediaScanner(db)
scanner.job = job
job.ProgressCallback = func(progress float64, filesScanned, newItems, errors int) {
w.mu.Lock()
defer w.mu.Unlock()
if result, exists := w.results[job.ID]; exists {
result.Progress = progress
result.FilesScanned = filesScanned
result.NewItems = newItems
result.Errors = errors
}
}
if err := scanner.SetFolders(folders); err != nil {
return nil, err
@@ -188,9 +223,14 @@ func (w *Worker) processScanJob(job *Job) (interface{}, error) {
return nil, err
}
totalFiles, newItems, errors := scanner.GetStats()
return map[string]interface{}{
"message": "scan completed",
"library_id": libraryID,
"message": "scan completed",
"library_id": libraryID,
"files_scanned": totalFiles,
"new_items": newItems,
"errors": errors,
}, nil
}