diff --git a/cmd/server/tests/phase13_test.go b/cmd/server/tests/phase13_test.go new file mode 100644 index 00000000..0e7442ed --- /dev/null +++ b/cmd/server/tests/phase13_test.go @@ -0,0 +1,174 @@ +package main + +// Phase 13 (faceted plan): force rescan = original-scan semantics, and +// the manually_edited filter. +// +// Force rescan contract: re-review every field on every book from all +// sources (sidecars → folder structure → tags → fallbacks) on multi- +// file books too — the pre-Phase-13 skip is gone — with +// metadata_overrides as the only protection. A force pass also stamps +// the fresh manifest with the current parser version (success-only) +// and closes the book's open ledger issues. + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "os" + "path/filepath" + "testing" + "time" + + "bookhoard/internal/database" + "bookhoard/internal/services" + + "github.com/jackc/pgx/v5/pgtype" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// forceScanTestFolder runs a FORCE scan over one folder. +func forceScanTestFolder(t *testing.T, setup *TestServerSetup, adminID pgtype.UUID, folder string) { + t.Helper() + scanner := services.NewMediaScanner(setup.DB) + defer func() { _ = scanner.Close() }() + scanner.SetAdminID(adminID) + require.NoError(t, scanner.SetFolders([]string{folder}, false)) + scanner.SetForce(true) + require.NoError(t, scanner.ScanFolders(context.Background())) +} + +func TestForceRescanMultifileSidecarApply(t *testing.T) { + setup := setupTestServer(t) + t.Cleanup(func() { _ = setup.Close() }) + + ctx := context.Background() + admin, _ := moveTestAdminID(t, setup) + libID, folder := audioTestLibrary(t, setup, fmt.Sprintf("Test Force Rescan %d", time.Now().UnixNano())) + t.Cleanup(func() { deleteLibraries(t, setup, libID) }) + + writeTestTrack(t, filepath.Join(folder, "Force Author", "Force Title", "01 First.mp3")) + writeTestTrack(t, filepath.Join(folder, "Force Author", "Force Title", "02 Second.mp3")) + + scanTestFolder(t, setup, admin, folder) + rel := filepath.ToSlash("Force Author/Force Title") + book, found := findAudioItemByPath(t, setup, libID, rel) + require.True(t, found, "folder book must import") + require.Equal(t, "Force Title", book.Title) + + // Corrupt, then FORCE: the pre-Phase-13 skip is gone — the full + // chain re-derives the title from the folder basename. + corruptItemColumn(t, setup, book.ID, "title", "ZZZ") + forceScanTestFolder(t, setup, admin, folder) + book, found = findAudioItemByPath(t, setup, libID, rel) + require.True(t, found) + require.Equal(t, "Force Title", book.Title, "force rescan must re-apply folder-book metadata") + require.Equal(t, 1, storedManifestEv(t, setup, book.ID), "force rescan stamps the current parser version") + + // metadata_overrides is the ONLY protection: a recorded override + // survives the force pass untouched. + _, err := setup.DBPool.Exec(ctx, + `UPDATE media_items SET metadata_overrides = ARRAY['title'], title = 'Manual Title' WHERE id = $1`, + book.ID) + require.NoError(t, err) + forceScanTestFolder(t, setup, admin, folder) + book, _ = findAudioItemByPath(t, setup, libID, rel) + require.Equal(t, "Manual Title", book.Title, "metadata_overrides protect manual edits under force rescan") +} + +func TestForceRescanSingleFileSidecarReapply(t *testing.T) { + setup := setupTestServer(t) + t.Cleanup(func() { _ = setup.Close() }) + + admin, _ := moveTestAdminID(t, setup) + libID, folder := createTypedTestLibrary(t, setup, fmt.Sprintf("Test Force Single %d", time.Now().UnixNano()), "ebooks") + t.Cleanup(func() { deleteLibraries(t, setup, libID) }) + + bookDir := filepath.Join(folder, "Force Probe Author", "Force Probe Book") + require.NoError(t, os.MkdirAll(bookDir, 0o755)) + opf := ` + + + Force Probe Book + Force Probe Author + +` + require.NoError(t, os.WriteFile(filepath.Join(bookDir, "metadata.opf"), []byte(opf), 0o644)) + require.NoError(t, os.WriteFile(filepath.Join(bookDir, "Force Probe Book.epub"), []byte("dummy epub content"), 0o644)) + + scanTestFolder(t, setup, admin, folder) + rel := filepath.ToSlash("Force Probe Author/Force Probe Book/Force Probe Book.epub") + book, found := findAudioItemByPath(t, setup, libID, rel) + require.True(t, found) + + // Single-file force rescan already ran the full chain — pin it, and + // pin the NEW part: the manifest comes out stamped. + corruptItemColumn(t, setup, book.ID, "title", "ZZZ") + forceScanTestFolder(t, setup, admin, folder) + book, found = findAudioItemByPath(t, setup, libID, rel) + require.True(t, found) + require.Equal(t, "Force Probe Book", book.Title, "force rescan re-derives sidecar metadata (single-file)") + require.Equal(t, 1, storedManifestEv(t, setup, book.ID), "force rescan stamps the manifest") +} + +func TestManuallyEditedFilter(t *testing.T) { + setup := setupTestServer(t) + t.Cleanup(func() { _ = setup.Close() }) + + ctx := context.Background() + libID, _ := createTypedTestLibrary(t, setup, fmt.Sprintf("Test Manual Edits %d", time.Now().UnixNano()), "ebooks") + t.Cleanup(func() { deleteLibraries(t, setup, libID) }) + + createBook := func(path string) pgtype.UUID { + created, err := setup.DB.CreateMediaItem(ctx, database.CreateMediaItemParams{ + LibraryID: libID, + Title: path, + Authors: []string{}, + FilePath: path, + FileSize: pgtype.Int8{Int64: 64, Valid: true}, + MimeType: pgtype.Text{String: "application/epub+zip", Valid: true}, + }) + require.NoError(t, err) + return created.ID + } + editedID := createBook("edited-book.epub") + cleanID := createBook("clean-book.epub") + _, err := setup.DBPool.Exec(ctx, + `UPDATE media_items SET metadata_overrides = ARRAY['title'] WHERE id = $1`, + editedID) + require.NoError(t, err) + + qfIDs := func(op string) []string { + status, rows := qfSearch(t, setup, setup.Token, map[string][]string{ + "qf": {qfAll(qfRule("manually_edited", op))}, + "library_id": {libID.String()}, + }) + require.Equal(t, http.StatusOK, status) + ids := make([]string, 0, len(rows)) + for _, r := range rows { + ids = append(ids, r["id"].(string)) + } + return ids + } + + trueIDs := qfIDs("is_true") + require.Len(t, trueIDs, 1) + assert.Equal(t, editedID.String(), trueIDs[0], "is_true matches only manually-edited books") + + falseIDs := qfIDs("is_false") + require.Len(t, falseIDs, 1) + assert.Equal(t, cleanID.String(), falseIDs[0], "is_false matches only untouched books") + + // The deep-linkable JSON shape round-trips through the qf parser + // (arity-none rule: no values on the wire). + var parsed struct { + Mode string `json:"mode"` + Children []struct { + Field string `json:"field"` + } `json:"children"` + } + require.NoError(t, json.Unmarshal([]byte(qfAll(qfRule("manually_edited", "is_true"))), &parsed)) + require.Len(t, parsed.Children, 1) + assert.Equal(t, "manually_edited", parsed.Children[0].Field) +} diff --git a/docs/contract_pins.md b/docs/contract_pins.md index ffbbfbe7..b036a41c 100644 --- a/docs/contract_pins.md +++ b/docs/contract_pins.md @@ -395,3 +395,22 @@ cover_image_path, has_conflict, format_group, badge`. open issues. Unchanged-skip scans do NOT resolve (extraction never re-ran — the degraded metadata is still degraded). A repeat failure reopens a resolved row (upsert sets resolved=false). + +## 2026-10-21 — Phase 13: force rescan = original scan; the manual-edits filter +- **Force rescan contract change (owner-directed)**: force rescan now + behaves like an original import — the full metadata chain (sidecars → + folder structure → tags → fallbacks) re-lands on MULTI-FILE books + too (the pre-Phase-13 editorial-keep skip is retired; the owner's + metadata_overrides count in the single digits, and overrides are the + only protection, honored inside the apply). A force pass also STAMPS + each re-derived book's manifest with the current parser version + (success-only) and closes its open processing issues. Single-file + force rescan already behaved this way — now pinned by test. +- **NEW filter field** `manually_edited` ("Manually edited", boolean): + is_true = `cardinality(mi.metadata_overrides) > 0`, is_false = `= 0`. + Composed-engine (qf) only — no flat param, no value menu, no + suggestion surface; family lens inherited. +- Deployment ordering note (owner Q&A): set scan_extractor_version + BEFORE Rescan All — force rescan stamps books with the current + setting, so bump-first + force = one pass of work; force-first + + bump = the trickle re-derives everything again (wasted, not harmful). diff --git a/internal/services/filter_query.go b/internal/services/filter_query.go index 7a071880..98746a43 100644 --- a/internal/services/filter_query.go +++ b/internal/services/filter_query.go @@ -749,6 +749,7 @@ func (s *SearchService) ComposedSearch(ctx context.Context, params SearchParams) b.joinBlock() + ` WHERE ` + whereSQL + composedSortOrder(sortBind, qBind, exactBind) + ` ` + limitBind + ` OFFSET ` + offsetBind + fmt.Printf("[DEBUG COMPOSED] %s\n", sqlStr) rows, err := s.raw.Query(ctx, sqlStr, b.args...) if err != nil { diff --git a/internal/services/filter_registry.go b/internal/services/filter_registry.go index 5fc46de2..77c432b2 100644 --- a/internal/services/filter_registry.go +++ b/internal/services/filter_registry.go @@ -249,6 +249,18 @@ var filterFields = []FilterField{ {Key: "abridged", Label: "Abridged", Group: GroupBookFacts, Kind: KindBool, Ops: []string{OpIsTrue, OpIsFalse}, DefaultOp: OpIsTrue, Clause: clauseBoolFactory("mi.abridged"), Suggest: suggestNone}, + {Key: "manually_edited", Label: "Manually edited", Group: GroupBookFacts, Kind: KindBool, + Ops: []string{OpIsTrue, OpIsFalse}, DefaultOp: OpIsTrue, + // metadata_overrides is NOT NULL TEXT[] — '{}' means untouched. + Clause: func(r FilterRule, b *filterBuilder) string { + switch r.Op { + case OpIsTrue: + return "(cardinality(mi.metadata_overrides) > 0)" + case OpIsFalse: + return "(cardinality(mi.metadata_overrides) = 0)" + } + return "TRUE" + }, Suggest: suggestNone}, {Key: "cover", Label: "Cover", Group: GroupBookFacts, Kind: KindPresence, Ops: []string{OpHas, OpMissing}, DefaultOp: OpHas, Clause: clauseCover, Suggest: suggestNone}, diff --git a/internal/services/media_scanner.go b/internal/services/media_scanner.go index b01c32d5..1828beee 100644 --- a/internal/services/media_scanner.go +++ b/internal/services/media_scanner.go @@ -1490,6 +1490,15 @@ func (s *MediaScanner) processExistingRow(ctx context.Context, existingItem data // Use UPDATE instead of DELETE+INSERT to preserve created_at if err := s.updateMediaItem(ctx, existingItem, path); err != nil { fmt.Printf("Warning: failed to update existing media item: %v\n", err) + } else { + // Force rescan = original-scan semantics (faceted plan + // Phase 13): stamp the fresh manifest with the current + // parser version and close open ledger issues — success- + // only, same truthfulness rule as the bulk path. + dir := bookSidecarDir(path) + s.storeManifest(ctx, libraryID, &scanEntry{ID: existingItem.ID, FilePath: s.getRelativePath(path)}, + hashSidecarManifest(path, dir, statSidecarManifest(path, dir))) + s.resolveIssuesForItem(ctx, existingItem.ID) } // Recompute hash identifiers too - a force rescan is the admin's // backfill tool and must refresh stale or missing hashes. @@ -1541,6 +1550,16 @@ func (s *MediaScanner) processExistingViaScanState(ctx context.Context, entry *s } if err := s.updateMediaItem(ctx, existingItem, path); err != nil { fmt.Printf("Warning: failed to update existing media item: %v\n", err) + } else { + // Force rescan = original-scan semantics (faceted plan + // Phase 13): the full extraction just re-derived this row, + // so stamp the fresh manifest with the current parser + // version and close its open ledger issues. Success-only — + // a stamp over a failed update would claim parser work + // that never ran. + dir := bookSidecarDir(path) + s.storeManifest(ctx, libraryID, entry, hashSidecarManifest(path, dir, statSidecarManifest(path, dir))) + s.resolveIssuesForItem(ctx, entry.ID) } // Recompute hash identifiers too - a force rescan is the admin's // backfill tool and must refresh stale or missing hashes. diff --git a/internal/services/media_scanner_audio_multifile.go b/internal/services/media_scanner_audio_multifile.go index 9af41b07..0417a1a2 100644 --- a/internal/services/media_scanner_audio_multifile.go +++ b/internal/services/media_scanner_audio_multifile.go @@ -951,10 +951,11 @@ func (s *MediaScanner) processAudioFolderBook(ctx context.Context, rootDir strin // ordering): the generic update's format-group pass derives the // mime from the sentinel (none) and chapters from // metadata.ChapterCount (unset) — the focused payload write below - // must win. Force rescan keeps the editorial-keep contract (the - // per-book RescanMediaItem is the editorial tool), so it skips - // the apply even when the ladder flagged a drift it caused. - if manifestChanged && !s.forceRescan { + // must win. Force rescan applies TOO (faceted plan Phase 13: + // force rescan = original-scan semantics — the full chain + // re-lands on every field, with metadata_overrides as the only + // protection, honored inside the apply). + if manifestChanged || s.forceRescan { if applyErr := s.applyMultiFileSidecarEdit(ctx, existing, sentinel, metadata); applyErr != nil { fmt.Printf("Warning: %v\n", applyErr) } else { @@ -1191,10 +1192,16 @@ func (s *MediaScanner) processZipAudioBook(ctx context.Context, zipPath string, changed := existing.FileSize.Int64 != totalSize || !audioPayloadTracksEqual(existing.AudioMetadata, payload) if changed || s.forceRescan || manifestChanged { // Editorial apply BEFORE the payload write — see the folder - // processor for the format-group ordering rationale. - if manifestChanged && !s.forceRescan { + // processor for the format-group ordering rationale. Force + // rescan applies too (Phase 13: original-scan semantics, + // metadata_overrides honored inside the apply). + if manifestChanged || s.forceRescan { if applyErr := s.applyMultiFileSidecarEdit(ctx, existing, zipPath, metadata); applyErr != nil { fmt.Printf("Warning: %v\n", applyErr) + } else { + // Ledger self-heal (Phase 11): the book's metadata + // re-applied successfully — close its open issues. + s.resolveIssuesForItem(ctx, existing.ID) } s.storeMultiFileManifest(ctx, libraryID, zipPath, zipPath, filepath.Dir(zipPath)) } diff --git a/web/src/filter_builder.ts b/web/src/filter_builder.ts index ba169a2e..5593c4d0 100644 --- a/web/src/filter_builder.ts +++ b/web/src/filter_builder.ts @@ -135,6 +135,7 @@ export const FIELD_GROUPS: { group: string; fields: FieldMeta[] }[] = [ F("file_availability", "File availability", "Book facts", "fixed", ["is", "is_not"], "is", false, ["available", "missing"]), F("cover", "Cover", "Book facts", "none", ["has", "missing"], "has"), + F("manually_edited", "Manually edited", "Book facts", "none", ["is_true", "is_false"], "is_true"), F("added_by", "Added by", "Book facts", "id", ["is", "has", "missing"], "has", true), ], },