John O'Keefe
91b0c45dd4
Parser fixes (7e4c97f6, genre model v2) never reached rows ingested
under an older parser: the sidecar never changed, so every fingerprint
said 'unchanged'. The manifest JSONB now carries the reserved 'ev'
stamp — the parser version that produced the item's row.
Semantics (all pinned by tests):
- ABSENT ev decodes as 1 ('v1-or-unknown'); stamping is truthful-only:
create paths + post-extraction stores stamp the setting's current
value; deploy-day backfill and the extract-failure guard stay UNSET
(a row never claims a version its data can't prove — a mid-backfill
bump self-heals next poll); repoint reconciliation preserves the
stored ev (stat-only refresh re-extracted nothing).
- NEW setting scan_extractor_version (seeded + registry + getter,
int default 1, category scanner): a manifest whose ev != the setting
re-extracts through the SAME change path (full extraction →
updateMediaItemWithMetadata → fresh stamped manifest). Sidecar-less
books get an entries-less manifest as their ev home (create: stamped;
first-sighting: unset).
- Catch-up bounded: evReextractBudgetPerScan=100, atomic (watcher pass
can overlap a poll scan), reset per ScanFolders — amortized +
resumable across polls; no force rescan, no flood.
- Multifile books included: the ev gate rides multifileSidecarChanged's
shared ladder; ev-driven manifestChanged drives the editorial apply
+ stamped re-store.
Settings API: the unified GET/PUT /api/system/settings now lists the
key (registry-driven, auto-surfaces in the admin UI). Test harness:
the registry is now Loaded + wired into the SystemSettingsHandler
(main.go parity) — settings writes through the API were 503s before.
Verified: unit (ev normalization, budget CAS, GetExtractorVersion
cache paths, registry clamping); integration state machine —
create-stamp ev=1 → equality skips (corruption probe persists) →
wiped scan-state backfills UNSET without re-extract → bump re-extracts
ONCE (sidecar value restored, ev=2) → stable (second corruption
persists); multifile twin green; adjacent suites (Audiobook, Calibre,
CrossLibraryMove, SchemaIdempotency) green; live app restart re-scans
1198 files with 0 changed (deploy-day rule), setting listed at
/api/system/settings.
183 lines
8.8 KiB
Go
183 lines
8.8 KiB
Go
package main
|
|
|
|
// Extractor-version (ev) mechanism, faceted plan Phase 10: parser fixes
|
|
// (7e4c97f6, genre model v2) must reach rows ingested under an older
|
|
// parser without a force rescan. The manifest's ev stamp is the
|
|
// per-item provenance; scan_extractor_version is the current parser's
|
|
// claim; a mismatch re-extracts on a later scan, bounded per scan.
|
|
//
|
|
// State machine pinned here, driven through REAL scans over a temp
|
|
// library (the corruption probe makes re-extraction observable: the
|
|
// only writer that restores the sidecar values is a re-extraction):
|
|
//
|
|
// create → stamped ev=1 (current parser, truthful)
|
|
// equal → no re-extract (corruption persists)
|
|
// wiped → backfill stores ev UNSET, never stamps, never re-extracts
|
|
// bump → re-extract ONCE (corruption healed, ev=2)
|
|
// equal → stable (second corruption persists)
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"net/http"
|
|
"os"
|
|
"path/filepath"
|
|
"strconv"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/jackc/pgx/v5/pgtype"
|
|
"github.com/stretchr/testify/require"
|
|
)
|
|
|
|
// storedManifestEv reads the "ev" stamp from an item's stored manifest
|
|
// (0 = absent — the backfill's v1-or-unknown marker).
|
|
func storedManifestEv(t *testing.T, setup *TestServerSetup, itemID pgtype.UUID) int {
|
|
t.Helper()
|
|
row, err := setup.DB.GetMediaScanState(context.Background(), itemID)
|
|
require.NoError(t, err, "scan-state row must exist — every book needs an ev home")
|
|
var m map[string]any
|
|
require.NoError(t, json.Unmarshal(row.Manifest, &m))
|
|
ev, _ := m["ev"].(float64)
|
|
return int(ev)
|
|
}
|
|
|
|
// corruptItemColumn scribbles a scan-owned column on a TEST-CREATED item
|
|
// so a later re-extraction is observable (the sidecar rewrites it back).
|
|
// The row is test data cascade-deleted with the library — never real rows.
|
|
func corruptItemColumn(t *testing.T, setup *TestServerSetup, itemID pgtype.UUID, column, value string) {
|
|
t.Helper()
|
|
tag, err := setup.DBPool.Exec(context.Background(),
|
|
fmt.Sprintf("UPDATE media_items SET %s = $1 WHERE id = $2", column), value, itemID)
|
|
require.NoError(t, err)
|
|
require.Equal(t, int64(1), tag.RowsAffected(), "corruption must hit exactly the test item")
|
|
}
|
|
|
|
|
|
func setExtractorVersion(t *testing.T, setup *TestServerSetup, v int) {
|
|
t.Helper()
|
|
body, _ := json.Marshal(map[string]string{"key": "scan_extractor_version", "value": strconv.Itoa(v)})
|
|
req, _ := http.NewRequest("PUT", setup.Server.URL+"/api/system/settings", bytes.NewBuffer(body))
|
|
req.Header.Set("Content-Type", "application/json")
|
|
req.Header.Set("Authorization", "Bearer "+setup.Token)
|
|
resp, err := http.DefaultClient.Do(req)
|
|
require.NoError(t, err)
|
|
defer func() { _ = resp.Body.Close() }()
|
|
require.Equal(t, http.StatusOK, resp.StatusCode, "settings API is the only sanctioned write path")
|
|
}
|
|
|
|
func TestExtractorVersionReextract(t *testing.T) {
|
|
setup := setupTestServer(t)
|
|
t.Cleanup(func() { _ = setup.Close() })
|
|
|
|
ctx := context.Background()
|
|
admin, _ := moveTestAdminID(t, setup)
|
|
libID, folder := createTypedTestLibrary(t, setup, fmt.Sprintf("Test EV Library %d", time.Now().UnixNano()), "ebooks")
|
|
t.Cleanup(func() { deleteLibraries(t, setup, libID) })
|
|
|
|
// metadata.opf sidecar: the re-extraction's provenance source.
|
|
bookDir := filepath.Join(folder, "EV Probe Author", "EV Probe Book")
|
|
require.NoError(t, os.MkdirAll(bookDir, 0o755))
|
|
opf := `<?xml version='1.0' encoding='utf-8'?>
|
|
<package xmlns="http://www.idpf.org/2007/opf" version="2.0">
|
|
<metadata xmlns:dc="http://purl.org/dc/elements/1.1/">
|
|
<dc:title>EV Probe Book</dc:title>
|
|
<dc:creator>EV Probe Author</dc:creator>
|
|
</metadata>
|
|
</package>`
|
|
require.NoError(t, os.WriteFile(filepath.Join(bookDir, "metadata.opf"), []byte(opf), 0o644))
|
|
require.NoError(t, os.WriteFile(filepath.Join(bookDir, "EV Probe Book.epub"), []byte("dummy epub content"), 0o644))
|
|
|
|
scanTestFolder(t, setup, admin, folder)
|
|
item, found := findAudioItemByPath(t, setup, libID, filepath.ToSlash("EV Probe Author/EV Probe Book/EV Probe Book.epub"))
|
|
require.True(t, found, "sidecar book must import")
|
|
|
|
// Create stamps the current parser version (truthful).
|
|
require.Equal(t, 1, storedManifestEv(t, setup, item.ID), "create must stamp the default ev=1")
|
|
|
|
// Equality → skip: corruption persists (nothing re-extracts at ev=1).
|
|
// NOTE: `author` is trigger-owned (sync_author_publisher_display
|
|
// rewrites it from the authors array on every UPDATE) — corrupt a
|
|
// trigger-free column the re-extraction rewrites: title.
|
|
corruptItemColumn(t, setup, item.ID, "title", "ZZZ corrupted")
|
|
scanTestFolder(t, setup, admin, folder)
|
|
item, found = findAudioItemByPath(t, setup, libID, filepath.ToSlash("EV Probe Author/EV Probe Book/EV Probe Book.epub"))
|
|
require.True(t, found)
|
|
require.Equal(t, "ZZZ corrupted", item.Title, "equal ev must NOT re-extract (corruption persists)")
|
|
require.Equal(t, 1, storedManifestEv(t, setup, item.ID))
|
|
|
|
// Wipe the scan state (test-owned row) to simulate a pre-release
|
|
// book: backfill re-fingerprints WITHOUT re-extraction and WITHOUT
|
|
// stamping — the row never claims a version it can't prove.
|
|
_, err := setup.DBPool.Exec(ctx, "DELETE FROM media_scan_state WHERE media_item_id = $1", item.ID)
|
|
require.NoError(t, err)
|
|
scanTestFolder(t, setup, admin, folder)
|
|
require.Equal(t, 0, storedManifestEv(t, setup, item.ID), "backfill must store ev UNSET (v1-or-unknown)")
|
|
item, _ = findAudioItemByPath(t, setup, libID, filepath.ToSlash("EV Probe Author/EV Probe Book/EV Probe Book.epub"))
|
|
require.Equal(t, "ZZZ corrupted", item.Title, "backfill must NOT re-extract")
|
|
|
|
// Bump via the SETTINGS API; the absent ev rides v1 → mismatch →
|
|
// re-extract once: the corruption heals and the fresh manifest
|
|
// carries ev=2.
|
|
setExtractorVersion(t, setup, 2)
|
|
t.Cleanup(func() { setExtractorVersion(t, setup, 1) })
|
|
scanTestFolder(t, setup, admin, folder)
|
|
item, found = findAudioItemByPath(t, setup, libID, filepath.ToSlash("EV Probe Author/EV Probe Book/EV Probe Book.epub"))
|
|
require.True(t, found)
|
|
require.Equal(t, "EV Probe Book", item.Title, "ev mismatch must re-extract (sidecar value restored)")
|
|
require.Equal(t, 2, storedManifestEv(t, setup, item.ID), "post-extraction store stamps the new ev")
|
|
|
|
// Then stable: the second corruption persists at ev=2 == setting 2.
|
|
corruptItemColumn(t, setup, item.ID, "title", "ZZZ corrupted again")
|
|
scanTestFolder(t, setup, admin, folder)
|
|
item, _ = findAudioItemByPath(t, setup, libID, filepath.ToSlash("EV Probe Author/EV Probe Book/EV Probe Book.epub"))
|
|
require.Equal(t, "ZZZ corrupted again", item.Title, "equal ev must NOT re-extract again (stable)")
|
|
require.Equal(t, 2, storedManifestEv(t, setup, item.ID))
|
|
}
|
|
|
|
func TestExtractorVersionMultifile(t *testing.T) {
|
|
setup := setupTestServer(t)
|
|
t.Cleanup(func() { _ = setup.Close() })
|
|
|
|
admin, _ := moveTestAdminID(t, setup)
|
|
libID, folder := audioTestLibrary(t, setup, fmt.Sprintf("Test EV Audio %d", time.Now().UnixNano()))
|
|
t.Cleanup(func() { deleteLibraries(t, setup, libID) })
|
|
|
|
// A canonical folder book, sidecar-less: the Phase 6 seed gives it
|
|
// an entries-less manifest — now also its ev home.
|
|
writeTestTrack(t, filepath.Join(folder, "EV Audio Author", "EV Audio Title", "01 First.mp3"))
|
|
writeTestTrack(t, filepath.Join(folder, "EV Audio Author", "EV Audio Title", "02 Second.mp3"))
|
|
|
|
scanTestFolder(t, setup, admin, folder)
|
|
book, found := findAudioItemByPath(t, setup, libID, filepath.ToSlash("EV Audio Author/EV Audio Title"))
|
|
require.True(t, found, "folder book must import")
|
|
require.Equal(t, "EV Audio Title", book.Title)
|
|
require.Equal(t, 1, storedManifestEv(t, setup, book.ID), "multifile create must stamp the default ev=1")
|
|
|
|
// Equality → skip: the multifile ladder's unchanged gate stays shut.
|
|
corruptItemColumn(t, setup, book.ID, "title", "ZZZ corrupted")
|
|
scanTestFolder(t, setup, admin, folder)
|
|
book, _ = findAudioItemByPath(t, setup, libID, filepath.ToSlash("EV Audio Author/EV Audio Title"))
|
|
require.Equal(t, "ZZZ corrupted", book.Title, "equal ev must NOT re-apply (corruption persists)")
|
|
require.Equal(t, 1, storedManifestEv(t, setup, book.ID))
|
|
|
|
// Bump → the ev gate fires through multifileSidecarChanged → the
|
|
// editorial apply re-derives the title from the folder basename.
|
|
setExtractorVersion(t, setup, 2)
|
|
t.Cleanup(func() { setExtractorVersion(t, setup, 1) })
|
|
scanTestFolder(t, setup, admin, folder)
|
|
book, found = findAudioItemByPath(t, setup, libID, filepath.ToSlash("EV Audio Author/EV Audio Title"))
|
|
require.True(t, found)
|
|
require.Equal(t, "EV Audio Title", book.Title, "ev mismatch must re-extract the folder book")
|
|
require.Equal(t, 2, storedManifestEv(t, setup, book.ID), "change-path store stamps the new ev")
|
|
|
|
// Then stable.
|
|
corruptItemColumn(t, setup, book.ID, "title", "ZZZ corrupted again")
|
|
scanTestFolder(t, setup, admin, folder)
|
|
book, _ = findAudioItemByPath(t, setup, libID, filepath.ToSlash("EV Audio Author/EV Audio Title"))
|
|
require.Equal(t, "ZZZ corrupted again", book.Title, "equal ev must NOT re-apply again (stable)")
|
|
require.Equal(t, 2, storedManifestEv(t, setup, book.ID))
|
|
}
|