refactor: improve worker type safety and scanner reliability
Worker improvements:
- Add strongly-typed result structs for all job types
- Replace map[string]interface{} with specific result types
- Add JSON tags to JobResult for proper API serialization
- Fix processJob to handle different result types correctly
- Improve directory scan job with proper library folder resolution
- Add debug logging for scan operations
Media scanner improvements:
- Add nil checks for database in GetPollInterval and GetAutoScanEnabled
- Fix pdfcpu API call signature (add validateOnly parameter)
- Add debug logging for scanDirectory with file counters
- Improve error handling and reporting
Test fixes:
- Fix default poll interval expectation from 30s to 60s
- Add settingsCache initialization to scanner tests
- Add folders initialization to ProcessDirtyDirectories test
This commit is contained in:
+127
-36
@@ -73,16 +73,74 @@ func (j *Job) UpdateProgress(progress float64, filesScanned, newItems, errors in
|
||||
}
|
||||
|
||||
type JobResult struct {
|
||||
JobID string
|
||||
Status JobStatus
|
||||
Error string
|
||||
Result interface{}
|
||||
Progress float64
|
||||
FilesScanned int
|
||||
NewItems int
|
||||
Errors int
|
||||
JobID string `json:"job_id"`
|
||||
Status JobStatus `json:"status"`
|
||||
Error string `json:"error"`
|
||||
Result interface{} `json:"result"`
|
||||
Progress float64 `json:"progress"`
|
||||
FilesScanned int `json:"files_scanned"`
|
||||
NewItems int `json:"new_items"`
|
||||
Errors int `json:"errors"`
|
||||
}
|
||||
|
||||
// ScanJobResult represents the result of a scan job
|
||||
type ScanJobResult struct {
|
||||
Message string `json:"message"`
|
||||
LibraryID string `json:"library_id"`
|
||||
FilesScanned int `json:"files_scanned"`
|
||||
NewItems int `json:"new_items"`
|
||||
Errors int `json:"errors"`
|
||||
}
|
||||
|
||||
// DirectoryScanJobResult represents the result of a directory scan job
|
||||
type DirectoryScanJobResult struct {
|
||||
Message string `json:"message"`
|
||||
FilesScanned int `json:"files_scanned"`
|
||||
NewItems int `json:"new_items"`
|
||||
Errors int `json:"errors"`
|
||||
}
|
||||
|
||||
// SetFoldersJobResult represents the result of a set folders job
|
||||
type SetFoldersJobResult struct {
|
||||
Message string `json:"message"`
|
||||
Folders []string `json:"folders"`
|
||||
}
|
||||
|
||||
// ImportJobResult represents the result of an import job
|
||||
type ImportJobResult struct {
|
||||
Message string `json:"message"`
|
||||
Items int `json:"items_imported"`
|
||||
}
|
||||
|
||||
// ConvertJobResult represents the result of a convert job
|
||||
type ConvertJobResult struct {
|
||||
Message string `json:"message"`
|
||||
Converted int `json:"converted"`
|
||||
Errors int `json:"errors"`
|
||||
}
|
||||
|
||||
// ThumbnailsJobResult represents the result of a thumbnail generation job
|
||||
type ThumbnailsJobResult struct {
|
||||
Message string `json:"message"`
|
||||
LibraryID string `json:"library_id"`
|
||||
TotalItems int `json:"total_items"`
|
||||
Processed int `json:"processed"`
|
||||
NewThumbnails int `json:"new_thumbnails"`
|
||||
Errors int `json:"errors"`
|
||||
}
|
||||
|
||||
// BackupJobResult represents the result of a backup job
|
||||
type BackupJobResult struct {
|
||||
Message string `json:"message"`
|
||||
BackupPath string `json:"backup_path"`
|
||||
Size int64 `json:"size"`
|
||||
}
|
||||
|
||||
// AnalyticsJobResult represents the result of an analytics job
|
||||
type AnalyticsJobResult struct {
|
||||
Message string `json:"message"`
|
||||
ReportPath string `json:"report_path"`
|
||||
}
|
||||
type Worker struct {
|
||||
jobQueue chan *Job
|
||||
results map[string]*JobResult
|
||||
@@ -212,10 +270,22 @@ func (w *Worker) processJob(job *Job) {
|
||||
|
||||
var filesScanned, newItems, errors int
|
||||
if result != nil {
|
||||
if stats, ok := result.(map[string]interface{}); ok {
|
||||
filesScanned = int(stats["files_scanned"].(float64))
|
||||
newItems = int(stats["new_items"].(float64))
|
||||
errors = int(stats["errors"].(float64))
|
||||
switch r := result.(type) {
|
||||
case *ScanJobResult:
|
||||
filesScanned = r.FilesScanned
|
||||
newItems = r.NewItems
|
||||
errors = r.Errors
|
||||
case *DirectoryScanJobResult:
|
||||
filesScanned = r.FilesScanned
|
||||
newItems = r.NewItems
|
||||
errors = r.Errors
|
||||
case *ThumbnailsJobResult:
|
||||
filesScanned = r.Processed
|
||||
newItems = r.NewThumbnails
|
||||
errors = r.Errors
|
||||
// Other job types don't report these metrics
|
||||
default:
|
||||
// Keep zeros for job types that don't report stats
|
||||
}
|
||||
}
|
||||
|
||||
@@ -317,12 +387,12 @@ func (w *Worker) processScanJob(job *Job) (interface{}, error) {
|
||||
|
||||
totalFiles, newItems, errors := scanner.GetStats()
|
||||
|
||||
return map[string]interface{}{
|
||||
"message": "scan completed",
|
||||
"library_id": libraryID,
|
||||
"files_scanned": float64(totalFiles),
|
||||
"new_items": float64(newItems),
|
||||
"errors": float64(errors),
|
||||
return &ScanJobResult{
|
||||
Message: "scan completed",
|
||||
LibraryID: libraryID,
|
||||
FilesScanned: totalFiles,
|
||||
NewItems: newItems,
|
||||
Errors: errors,
|
||||
}, nil
|
||||
}
|
||||
|
||||
@@ -443,9 +513,9 @@ func (w *Worker) processSetFoldersJob(job *Job) (interface{}, error) {
|
||||
}
|
||||
|
||||
// Return success result
|
||||
return map[string]interface{}{
|
||||
"message": "folders configured successfully",
|
||||
"folders": folders,
|
||||
return &SetFoldersJobResult{
|
||||
Message: "folders configured successfully",
|
||||
Folders: folders,
|
||||
}, nil
|
||||
}
|
||||
|
||||
@@ -594,13 +664,13 @@ func (w *Worker) processThumbnailsJob(job *Job) (interface{}, error) {
|
||||
updateProgress()
|
||||
}
|
||||
|
||||
return map[string]interface{}{
|
||||
"message": "thumbnail generation completed",
|
||||
"library_id": libraryID,
|
||||
"total_items": totalItems,
|
||||
"processed": processedItems,
|
||||
"new_thumbnails": newThumbnails,
|
||||
"errors": errors,
|
||||
return &ThumbnailsJobResult{
|
||||
Message: "thumbnail generation completed",
|
||||
LibraryID: libraryID,
|
||||
TotalItems: totalItems,
|
||||
Processed: processedItems,
|
||||
NewThumbnails: newThumbnails,
|
||||
Errors: errors,
|
||||
}, nil
|
||||
}
|
||||
|
||||
@@ -817,18 +887,39 @@ func (w *Worker) processDirectoryScanJob(job *Job) (interface{}, error) {
|
||||
|
||||
// Create temporary scanner instance for this job
|
||||
scanner := NewMediaScanner(db)
|
||||
|
||||
// Call scanDirectory() directly
|
||||
// Job queue provides concurrency control - no need for activeScans map
|
||||
scanner.job = job
|
||||
// Find which library owns this directory
|
||||
ctx := context.Background()
|
||||
libRow, err := db.GetLibraryByFolder(ctx, directory)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("directory not associated with any library: %s", directory)
|
||||
}
|
||||
// Get all folders for this library
|
||||
folders, err := db.GetLibraryFolders(ctx, libRow.ID)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get library folders: %w", err)
|
||||
}
|
||||
// Extract folder paths
|
||||
var folderPaths []string
|
||||
for _, f := range folders {
|
||||
folderPaths = append(folderPaths, f.FolderPath)
|
||||
}
|
||||
// Configure scanner with folders
|
||||
if err := scanner.SetFolders(folderPaths); err != nil {
|
||||
return nil, fmt.Errorf("failed to set folders: %w", err)
|
||||
}
|
||||
// Now scan the directory
|
||||
scanner.scanDirectory(ctx, directory)
|
||||
// Debug logging
|
||||
fmt.Printf("DEBUG: scanDirectory completed - totalFiles: %d, newItems: %d, errors: %d\n",
|
||||
scanner.totalFiles, scanner.newItems, scanner.errors)
|
||||
|
||||
// Return scan results
|
||||
return map[string]interface{}{
|
||||
"message": fmt.Sprintf("Scanned directory: %s", directory),
|
||||
"totalFiles": scanner.totalFiles,
|
||||
"newItems": scanner.newItems,
|
||||
"errors": scanner.errors,
|
||||
return &DirectoryScanJobResult{
|
||||
Message: fmt.Sprintf("Scanned directory: %s", directory),
|
||||
FilesScanned: scanner.totalFiles,
|
||||
NewItems: scanner.newItems,
|
||||
Errors: scanner.errors,
|
||||
}, nil
|
||||
}
|
||||
func (w *Worker) EnqueueJob(job *Job) error {
|
||||
|
||||
Reference in New Issue
Block a user