Compare commits

..
1 Commits
Author SHA1 Message Date
John O'Keefe 795225f097 fix(scanner): reconcile archive pass per library, not per folder
Release / build-and-push (push) Successful in 3m5s
The archive pass looped over the scan's folders and, inside each
iteration, compared the ENTIRE library's item list against that one
folder's walk. With two registered folders every item living under the
other folder was flagged "missing (1/2)" on every reconciling scan —
prod's audiobook library (the only multi-folder one) showed 743/743
flagged — and nothing reached the two-consecutive-miss threshold only
because each scan's walk clears missing_scan_count before the pass
re-flags it. With three or more folders the defect stops bouncing: a
later iteration sees an earlier iteration's count-1 marks on items its
own walk lacks and archives them mid-pass — one registered folder away
from archiving a third of a library per scan. Single-folder libraries
were unaffected, which is why comics/ebooks/manga showed zero flags.

Folder-book presence was suspected alongside it (audioBookRoots read
during the comparison), but the map is written for every >=2-track
folder-book root on every walk (processAudioBookFile, since 6216bd9b);
the flagged folder-books were victims of the same cross-folder
comparison. The roots attribution did carry the same unchecked
strings.HasPrefix boundary bug as the rest of the scanner (8th site,
variable named `root`, missed by the earlier sweep); it now uses
folderContains.

The pass now groups folders by owning library and, per library,
compares ListMediaItemsByLibraryIncludingArchived against the UNION of
all that library's folders' walks. The coverage (partial-scope) guard,
the walk-failure skip, and the mark/archive/purge item loop are
unchanged.

Regression test (archive_multifolder_test.go) builds a two-folder
audiobooks library — single-file book in one folder, folder-of-tracks
book in the other — and scans twice: pre-fix both items are flagged
missing on both scans (verified by running the test against the old
code), post-fix neither is ever flagged or archived.
2026-10-28 01:30:41 -04:00
2 changed files with 141 additions and 16 deletions

No files matched your search

@@ -0,0 +1,87 @@
package main
// Regression tests for the archive reconciliation pass: a multi-folder
// library must be reconciled against the UNION of its folders' walks, not
// once per folder. The old per-folder loop listed every library item
// against each single folder's walk, so with two registered folders every
// item living under the other folder was flagged "missing (1/2)" on every
// scan (prod: 743/743 audiobooks stuck at count 1 — unarchivable only
// because each scan's walk kept clearing them), and with three or more
// folders the crossed marks would have reached the archive threshold
// mid-pass and archived real content. Folder-book roots (directory
// file_path) must also count as present across folder boundaries — the
// file walk never emits a directory.
import (
"context"
"fmt"
"path/filepath"
"testing"
"time"
"bookhoard/internal/database"
"bookhoard/internal/services"
"github.com/jackc/pgx/v5/pgtype"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// scanTestFolders runs a synchronous scan over several folders — the
// multi-folder shape the archive pass must get right (scanTestFolder
// covers the single-folder case).
func scanTestFolders(t *testing.T, setup *TestServerSetup, adminID pgtype.UUID, folders []string) {
t.Helper()
scanner := services.NewMediaScanner(setup.DB)
defer func() { _ = scanner.Close() }()
scanner.SetAdminID(adminID)
require.NoError(t, scanner.SetFolders(folders, false))
require.NoError(t, scanner.ScanFolders(context.Background()))
}
func TestArchivePassMultiFolderLibraryKeepsEverythingPresent(t *testing.T) {
setup := setupTestServer(t)
t.Cleanup(func() { _ = setup.Close() })
libID, folderA := createTypedTestLibrary(t, setup, fmt.Sprintf("Test Archive MultiFolder %d", time.Now().UnixNano()), "audiobooks")
t.Cleanup(func() { deleteLibraries(t, setup, libID) }) // after setup.Close defer (LIFO): pool still open
// Second registered folder in the SAME library — the shape the
// per-folder reconciliation got wrong.
folderB := t.TempDir()
_, err := setup.DB.AddLibraryFolder(context.Background(), database.AddLibraryFolderParams{
LibraryID: libID,
FolderPath: folderB,
})
require.NoError(t, err)
admin, _ := moveTestAdminID(t, setup)
// Folder A holds a lone single-file book; folder B holds a
// folder-of-tracks book whose file_path is a DIRECTORY.
writeTestTrack(t, filepath.Join(folderA, "Author A", "Lone Book", "lonely.mp3"))
writeTestTrack(t, filepath.Join(folderB, "Author B", "Folder Book", "01 First.mp3"))
writeTestTrack(t, filepath.Join(folderB, "Author B", "Folder Book", "02 Second.mp3"))
// Two consecutive scans: the first proves no flags are raised, the
// second proves the flag/walk bounce (clear → re-flag) is gone too.
for scanNo := 1; scanNo <= 2; scanNo++ {
scanTestFolders(t, setup, admin, []string{folderA, folderB})
items := listLibraryAudioItems(t, setup, libID)
var singleFile, folderBook bool
for _, item := range items {
assert.False(t, item.ArchivedAt.Valid, "scan %d: item %q must not be archived", scanNo, item.FilePath)
assert.Equal(t, int32(0), item.MissingScanCount, "scan %d: item %q must not be flagged missing", scanNo, item.FilePath)
switch item.FilePath {
case "Author A/Lone Book/lonely.mp3":
singleFile = true
case "Author B/Folder Book":
folderBook = true
}
}
require.True(t, singleFile, "scan %d: single-file book must exist", scanNo)
require.True(t, folderBook, "scan %d: folder book must exist at its directory path", scanNo)
require.Len(t, items, 2, "scan %d: exactly the two books, no per-track strays", scanNo)
}
}
+54 -16
View File
@@ -831,18 +831,47 @@ func (s *MediaScanner) ScanFolders(ctx context.Context) error {
// retention window (ARCHIVE_RETENTION_DAYS; 0 = manual purge only). // retention window (ARCHIVE_RETENTION_DAYS; 0 = manual purge only).
// Every branch logs - the previous hard-delete cleanup failed silently // Every branch logs - the previous hard-delete cleanup failed silently
// and left orphaned rows undetected. // and left orphaned rows undetected.
// Group this scan's folders by owning library: the reconciliation is
// per LIBRARY, comparing every library item against the union of all
// its folders' walks. The old per-folder loop listed the whole
// library's items against each single folder's walk, so with two
// folders every item living under the other folder was flagged
// missing on every scan (prod: 743/743 audiobooks stuck at
// "missing (1/2)", unarchivable only because the walk kept clearing
// them) — and with three or more folders earlier iterations' marks
// crossed later ones' walks and archived real content mid-pass.
scannedSet := make(map[string]bool, len(s.folders))
for _, scanned := range s.folders {
scannedSet[scanned] = true
}
type archiveScope struct {
libraryID pgtype.UUID
name string
folders []string
}
scopes := make([]archiveScope, 0, len(s.folders))
scopeIndex := make(map[pgtype.UUID]int, len(s.folders))
for _, folder := range s.folders { for _, folder := range s.folders {
lib, err := s.db.GetLibraryByFolder(ctx, folder)
if err != nil {
fmt.Printf("[ARCHIVE] Warning: no library found for folder %s, skipping it in archive pass: %v\n", folder, err)
continue
}
idx, seen := scopeIndex[lib.LibraryID]
if !seen {
idx = len(scopes)
scopeIndex[lib.LibraryID] = idx
scopes = append(scopes, archiveScope{libraryID: lib.LibraryID, name: lib.Name})
}
scopes[idx].folders = append(scopes[idx].folders, folder)
}
for _, scope := range scopes {
if ctx.Err() != nil { if ctx.Err() != nil {
fmt.Printf("[ARCHIVE] Scan cancelled, stopping archive pass\n") fmt.Printf("[ARCHIVE] Scan cancelled, stopping archive pass\n")
return ctx.Err() return ctx.Err()
} }
libraryID := scope.libraryID
lib, err := s.db.GetLibraryByFolder(ctx, folder)
if err != nil {
fmt.Printf("[ARCHIVE] Warning: no library found for folder %s, skipping archive pass: %v\n", folder, err)
continue
}
libraryID := lib.LibraryID
// Partial-scope guard (per-folder scans, 2026-10): the // Partial-scope guard (per-folder scans, 2026-10): the
// reconciliation compares EVERY library item against this scan's // reconciliation compares EVERY library item against this scan's
@@ -855,10 +884,6 @@ func (s *MediaScanner) ScanFolders(ctx context.Context) error {
fmt.Printf("[ARCHIVE] Warning: failed to list folders for library, skipping archive pass: %v\n", regErr) fmt.Printf("[ARCHIVE] Warning: failed to list folders for library, skipping archive pass: %v\n", regErr)
continue continue
} }
scannedSet := make(map[string]bool, len(s.folders))
for _, scanned := range s.folders {
scannedSet[scanned] = true
}
covered := 0 covered := 0
for _, rf := range registeredFolders { for _, rf := range registeredFolders {
if scannedSet[rf.FolderPath] { if scannedSet[rf.FolderPath] {
@@ -867,7 +892,7 @@ func (s *MediaScanner) ScanFolders(ctx context.Context) error {
} }
if covered < len(registeredFolders) { if covered < len(registeredFolders) {
fmt.Printf("[ARCHIVE] Skipping reconciliation for %s: scan covered %d of %d registered folders\n", fmt.Printf("[ARCHIVE] Skipping reconciliation for %s: scan covered %d of %d registered folders\n",
lib.Name, covered, len(registeredFolders)) scope.name, covered, len(registeredFolders))
continue continue
} }
@@ -877,8 +902,11 @@ func (s *MediaScanner) ScanFolders(ctx context.Context) error {
continue continue
} }
// Build set of scanned file paths for this folder // Build the set of scanned paths across ALL of this library's
// folders.
scannedPaths := make(map[string]bool) scannedPaths := make(map[string]bool)
walkFailed := false
for _, folder := range scope.folders {
if err := filepath.WalkDir(folder, func(path string, d os.DirEntry, walkErr error) error { if err := filepath.WalkDir(folder, func(path string, d os.DirEntry, walkErr error) error {
if walkErr != nil { if walkErr != nil {
return walkErr return walkErr
@@ -889,14 +917,24 @@ func (s *MediaScanner) ScanFolders(ctx context.Context) error {
return nil return nil
}); err != nil { }); err != nil {
fmt.Printf("[ARCHIVE] Warning: failed to walk directory %s, skipping archive pass: %v\n", folder, err) fmt.Printf("[ARCHIVE] Warning: failed to walk directory %s, skipping archive pass: %v\n", folder, err)
continue // Skip to next folder to avoid false archivals walkFailed = true
break
}
}
if walkFailed {
continue // Skip this library to avoid false archivals
} }
// Folder-book roots count as present: the item's file_path is the // Folder-book roots count as present: the item's file_path is the
// DIRECTORY, which the file walk above never emits. // DIRECTORY, which the file walk above never emits. Attribution is
// boundary-safe — a sibling folder whose name merely extends this
// one's must not contribute its roots here.
for root := range s.audioBookRoots { for root := range s.audioBookRoots {
if strings.HasPrefix(root, folder) { for _, folder := range scope.folders {
if folderContains(folder, root) {
scannedPaths[s.getRelativePath(root)] = true scannedPaths[s.getRelativePath(root)] = true
break
}
} }
} }