Implement backend scan progress tracking (Steps 1-7)
Implements comprehensive progress tracking for scan jobs to provide real-time statistics to the frontend (files_scanned, new_items, errors). Changes: 1. Extended JobResult struct with new fields: - FilesScanned: total files processed - NewItems: books added to database - Errors: scan errors encountered 2. Added progress callback mechanism: - Job.ProgressCallback function field for real-time updates - Job.UpdateProgress() method to trigger callbacks - Worker stores callback and updates JobResult during scan 3. MediaScanner now tracks statistics: - totalFiles, newItems, errors counters - GetStats() method to retrieve statistics - First pass counts total files for progress calculation - Batches progress updates every 10 files (reduces mutex contention) - Final update ensures 100% progress is reported 4. Updated processMediaFile signature: - Returns (bool, error) instead of (error) - true = new item created, false = existing/updated/error - Increments newItems counter when creating database entries - Updated WatchChanges to handle new return value 5. Worker job completion extracts stats: - Parses result map for files_scanned, new_items, errors - Stores in final JobResult for API response 6. GetScanStatus API response includes new fields: - files_scanned, new_items, errors now in JSON response - Frontend can display real-time progress Design decisions: - Batching every 10 files balances performance vs. granularity - Thread-safe via worker mutex (w.mu.Lock/Unlock) - Callback pattern decouples scanner from job management - processMediaFile return type allows tracking new vs. updated items - Maintains backward compatibility (uses || 0 fallbacks in frontend) Testing: - All code compiles successfully - Follows service layer pattern (no business logic in handlers) - No database schema changes - Integration tests to be added in Step 8 (separate commit) Files modified: - internal/services/worker.go (JobResult, Job struct, processScanJob, processJob) - internal/services/media_scanner.go (struct fields, GetStats, ScanFolders, processMediaFile) - internal/handlers/scanner.go (GetScanStatus response) Related: TASKS-backend-progress-tracking.md Steps 1-7
This commit is contained in:
+59
-19
@@ -28,24 +28,34 @@ const (
|
||||
)
|
||||
|
||||
type Job struct {
|
||||
ID string
|
||||
Type JobType
|
||||
Params map[string]interface{}
|
||||
Status JobStatus
|
||||
CreatedAt time.Time
|
||||
StartedAt *time.Time
|
||||
CompletedAt *time.Time
|
||||
Error error
|
||||
Result interface{}
|
||||
Context context.Context
|
||||
ID string
|
||||
Type JobType
|
||||
Params map[string]interface{}
|
||||
Status JobStatus
|
||||
CreatedAt time.Time
|
||||
StartedAt *time.Time
|
||||
CompletedAt *time.Time
|
||||
Error error
|
||||
Result interface{}
|
||||
Context context.Context
|
||||
ProgressCallback func(progress float64, filesScanned, newItems, errors int)
|
||||
}
|
||||
|
||||
func (j *Job) UpdateProgress(progress float64, filesScanned, newItems, errors int) {
|
||||
if j.ProgressCallback != nil {
|
||||
j.ProgressCallback(progress, filesScanned, newItems, errors)
|
||||
}
|
||||
}
|
||||
|
||||
type JobResult struct {
|
||||
JobID string
|
||||
Status JobStatus
|
||||
Error string
|
||||
Result interface{}
|
||||
Progress float64
|
||||
JobID string
|
||||
Status JobStatus
|
||||
Error string
|
||||
Result interface{}
|
||||
Progress float64
|
||||
FilesScanned int
|
||||
NewItems int
|
||||
Errors int
|
||||
}
|
||||
|
||||
type Worker struct {
|
||||
@@ -135,6 +145,15 @@ func (w *Worker) processJob(job *Job) {
|
||||
status = JobStatusCancelled
|
||||
}
|
||||
|
||||
var filesScanned, newItems, errors int
|
||||
if result != nil {
|
||||
if stats, ok := result.(map[string]interface{}); ok {
|
||||
filesScanned = int(stats["files_scanned"].(float64))
|
||||
newItems = int(stats["new_items"].(float64))
|
||||
errors = int(stats["errors"].(float64))
|
||||
}
|
||||
}
|
||||
|
||||
w.results[job.ID] = &JobResult{
|
||||
JobID: job.ID,
|
||||
Status: status,
|
||||
@@ -145,8 +164,11 @@ func (w *Worker) processJob(job *Job) {
|
||||
return ""
|
||||
}
|
||||
}(),
|
||||
Result: result,
|
||||
Progress: 1.0,
|
||||
Result: result,
|
||||
Progress: 1.0,
|
||||
FilesScanned: filesScanned,
|
||||
NewItems: newItems,
|
||||
Errors: errors,
|
||||
}
|
||||
w.mu.Unlock()
|
||||
}
|
||||
@@ -173,6 +195,19 @@ func (w *Worker) processScanJob(job *Job) (interface{}, error) {
|
||||
}
|
||||
|
||||
scanner := NewMediaScanner(db)
|
||||
scanner.job = job
|
||||
|
||||
job.ProgressCallback = func(progress float64, filesScanned, newItems, errors int) {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
|
||||
if result, exists := w.results[job.ID]; exists {
|
||||
result.Progress = progress
|
||||
result.FilesScanned = filesScanned
|
||||
result.NewItems = newItems
|
||||
result.Errors = errors
|
||||
}
|
||||
}
|
||||
|
||||
if err := scanner.SetFolders(folders); err != nil {
|
||||
return nil, err
|
||||
@@ -188,9 +223,14 @@ func (w *Worker) processScanJob(job *Job) (interface{}, error) {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
totalFiles, newItems, errors := scanner.GetStats()
|
||||
|
||||
return map[string]interface{}{
|
||||
"message": "scan completed",
|
||||
"library_id": libraryID,
|
||||
"message": "scan completed",
|
||||
"library_id": libraryID,
|
||||
"files_scanned": totalFiles,
|
||||
"new_items": newItems,
|
||||
"errors": errors,
|
||||
}, nil
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user