From 03cb4c78694222fa48ed7cba58c8321561b407a8 Mon Sep 17 00:00:00 2001 From: John O'Keefe Date: Fri, 14 Aug 2026 08:53:01 -0400 Subject: [PATCH] feat(admin): startup hash backfill and hash-conflict resolution API Complete the SHA-256 lifecycle for preexisting databases: items imported before hashing existed get hashed automatically, and any content duplicates discovered in the process land on the new admin Hash Conflicts page for an explicit keep/merge decision. HashBackfillService (runs once 30s after startup, independent of auto-scan): - hashes every media_items row where file_sha256 IS NULL, resolving each path through LibraryService; per-item failures are logged and skipped so one unreadable file cannot block the pass - no-op once everything is hashed (logged and skipped) - finishes with a conflict sweep flagging every content-duplicate group via FindHashConflictGroups + CreateHashConflict; the sweep runs after the per-item pass because a preexisting pair only becomes detectable once both sides have their hash API (admin-only): - GET /api/admin/hash-conflicts - pending groups with member items and usage counts - POST /api/admin/hash-conflicts/:id/resolve - action=keep_all, or action=keep with keep_uuid: validates the uuid belongs to the group, re-parents every other copy's child rows onto the kept item (reparent_media_item_children), deletes the losers, and records the resolution + resolving admin; accepts form or JSON bodies and returns the htmx resolved fragment Page route /admin/hash-conflicts (admin-only) renders the template with hydrated conflict data; HashConflictsHandler wired into the router Config and constructed in main. Verified end-to-end against the live database: duplicate detection, pending listing, keep_all resolution, merge path (re-parent + delete), and - critically - a resolved group is not re-flagged by a later sweep (upsert no-op). Database restored afterward. --- cmd/server/main.go | 2 + internal/handlers/hash_conflicts.go | 255 ++++++++++++++++++++++++++++ internal/router/frontend.go | 62 +++++++ internal/router/router.go | 16 ++ internal/services/hash_backfill.go | 119 +++++++++++++ 5 files changed, 454 insertions(+) create mode 100644 internal/handlers/hash_conflicts.go create mode 100644 internal/services/hash_backfill.go diff --git a/cmd/server/main.go b/cmd/server/main.go index 829173b..7e52504 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -104,6 +104,7 @@ func main() { deviceAuthMiddleware := middleware.NewDeviceAuthMiddleware(queries) deviceAuthMiddleware.SetSettings(registry) processingIssuesHandler := handlers.NewProcessingIssuesHandler(queries) + hashConflictsHandler := handlers.NewHashConflictsHandler(queries) // Create WebSocket connection manager connManager := sync.NewConnectionManager() @@ -202,6 +203,7 @@ func main() { MediaHandler: mediaHandler, MatchingHandler: matchingHandler, ProcessingIssuesHandler: processingIssuesHandler, + HashConflictsHandler: hashConflictsHandler, KOReaderHandler: koreaderHandler, WSHandler: wsHandler, ConflictHandler: conflictHandler, diff --git a/internal/handlers/hash_conflicts.go b/internal/handlers/hash_conflicts.go new file mode 100644 index 0000000..bc50c88 --- /dev/null +++ b/internal/handlers/hash_conflicts.go @@ -0,0 +1,255 @@ +package handlers + +import ( + "bookhoard/internal/database" + "fmt" + "net/http" + "time" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5/pgtype" + "github.com/labstack/echo/v5" +) + +// HashConflictsHandler serves the admin Hash Conflicts page API: listing +// content-duplicate groups (same library + SHA-256 at different paths) and +// resolving them by keeping every copy or merging all but one. +type HashConflictsHandler struct { + db *database.Queries +} + +func NewHashConflictsHandler(db *database.Queries) *HashConflictsHandler { + return &HashConflictsHandler{db: db} +} + +// HashConflictItem is one copy in a conflict group, hydrated with per-item +// user-data counts so the admin can make an informed keep/merge choice. +type HashConflictItem struct { + ID uuid.UUID `json:"id"` + Title string `json:"title"` + Author string `json:"author,omitempty"` + FilePath string `json:"file_path"` + FileSize int64 `json:"file_size,omitempty"` + CreatedAt string `json:"created_at"` + ProgressCount int64 `json:"progress_count"` + HighlightCount int64 `json:"highlight_count"` + BookmarkCount int64 `json:"bookmark_count"` + NoteCount int64 `json:"note_count"` + CollectionCount int64 `json:"collection_count"` +} + +// HashConflictResponse is one pending conflict group. +type HashConflictResponse struct { + ID string `json:"id"` + LibraryID string `json:"library_id"` + LibraryName string `json:"library_name"` + SHA256 string `json:"sha256"` + CreatedAt string `json:"created_at"` + Items []HashConflictItem `json:"items"` +} + +// ListHashConflicts returns all pending hash conflict groups with their member +// items and usage counts. +// GET /api/admin/hash-conflicts +func (h *HashConflictsHandler) ListHashConflicts(c *echo.Context) error { + ctx := c.Request().Context() + + pending, err := h.db.ListPendingHashConflicts(ctx) + if err != nil { + return c.JSON(http.StatusInternalServerError, map[string]string{ + "error": "failed to list hash conflicts", + }) + } + + conflicts := make([]HashConflictResponse, 0, len(pending)) + for _, p := range pending { + resp := HashConflictResponse{ + ID: uuid.UUID(p.ID.Bytes).String(), + LibraryID: uuid.UUID(p.LibraryID.Bytes).String(), + LibraryName: p.LibraryName, + SHA256: p.FileSha256, + CreatedAt: p.CreatedAt.Time.Format(time.RFC3339), + Items: []HashConflictItem{}, + } + items, err := h.db.ListMediaItemsBySHA256AndLibrary(ctx, database.ListMediaItemsBySHA256AndLibraryParams{ + FileSha256: pgtype.Text{String: p.FileSha256, Valid: true}, + LibraryID: p.LibraryID, + }) + if err != nil { + continue + } + + for _, mi := range items { + counts, err := h.db.GetMediaItemUsageCounts(ctx, mi.ID) + if err != nil { + counts = database.GetMediaItemUsageCountsRow{} + } + resp.Items = append(resp.Items, HashConflictItem{ + ID: uuid.UUID(mi.ID.Bytes), + Title: mi.Title, + Author: mi.Author.String, + FilePath: mi.FilePath, + FileSize: mi.FileSize.Int64, + CreatedAt: mi.CreatedAt.Time.Format(time.RFC3339), + ProgressCount: counts.ProgressCount, + HighlightCount: counts.HighlightsCount, + BookmarkCount: counts.BookmarksCount, + NoteCount: counts.NotesCount, + CollectionCount: counts.CollectionsCount, + }) + } + + conflicts = append(conflicts, resp) + } + + return c.JSON(http.StatusOK, map[string]interface{}{ + "conflicts": conflicts, + "total": len(conflicts), + }) +} + +// ResolveHashConflict resolves one conflict group. +// +// Form/JSON fields: +// - action=keep_all both copies are intentional; dismiss +// - action=keep&keep_uuid= merge every other copy's child rows into the +// kept item (progress, highlights, bookmarks, +// notes, collections, ...) and delete the losers +// +// POST /api/admin/hash-conflicts/:id/resolve +func (h *HashConflictsHandler) ResolveHashConflict(c *echo.Context) error { + ctx := c.Request().Context() + + conflictID, err := uuid.Parse(c.Param("id")) + if err != nil { + return c.JSON(http.StatusBadRequest, map[string]string{"error": "invalid conflict ID"}) + } + pgConflictID := pgtype.UUID{Bytes: conflictID, Valid: true} + + conflict, err := h.db.GetHashConflict(ctx, pgConflictID) + if err != nil { + return c.JSON(http.StatusNotFound, map[string]string{"error": "conflict not found"}) + } + if conflict.Status != "pending" { + return c.JSON(http.StatusConflict, map[string]string{"error": "conflict already resolved"}) + } + + action := c.FormValue("action") + keepUUIDStr := c.FormValue("keep_uuid") + if action == "" { + // Also accept a JSON body (htmx sends form-encoded, API clients may send JSON) + var body struct { + Action string `json:"action"` + KeepUUID string `json:"keep_uuid"` + } + if err := c.Bind(&body); err == nil && body.Action != "" { + action = body.Action + if keepUUIDStr == "" { + keepUUIDStr = body.KeepUUID + } + } + } + + var pgUserID pgtype.UUID + if userID, ok := c.Get("user_id").(string); ok && userID != "" { + if u, err := uuid.Parse(userID); err == nil { + pgUserID = pgtype.UUID{Bytes: u, Valid: true} + } + } + + switch action { + case "keep_all": + if err := h.db.ResolveHashConflict(ctx, database.ResolveHashConflictParams{ + ID: pgConflictID, + Resolution: pgtype.Text{String: "keep_all", Valid: true}, + ResolvedBy: pgUserID, + }); err != nil { + return c.JSON(http.StatusInternalServerError, map[string]string{"error": "failed to resolve conflict"}) + } + return renderResolved(c, "All copies kept.") + + case "keep": + keepUUID, err := uuid.Parse(keepUUIDStr) + if err != nil { + return c.JSON(http.StatusBadRequest, map[string]string{"error": "keep_uuid is required for action=keep"}) + } + pgKeepUUID := pgtype.UUID{Bytes: keepUUID, Valid: true} + + // Validate the kept item belongs to this conflict group. + items, err := h.db.ListMediaItemsBySHA256AndLibrary(ctx, database.ListMediaItemsBySHA256AndLibraryParams{ + FileSha256: pgtype.Text{String: conflict.FileSha256, Valid: true}, + LibraryID: conflict.LibraryID, + }) + if err != nil { + return c.JSON(http.StatusInternalServerError, map[string]string{"error": "failed to load conflict group"}) + } + + keepValid := false + for _, mi := range items { + if mi.ID.Bytes == pgKeepUUID.Bytes { + keepValid = true + break + } + } + if !keepValid { + return c.JSON(http.StatusBadRequest, map[string]string{"error": "keep_uuid is not part of this conflict"}) + } + + merged := 0 + for _, mi := range items { + if mi.ID.Bytes == pgKeepUUID.Bytes { + continue + } + if err := h.db.ReparentMediaItemChildren(ctx, database.ReparentMediaItemChildrenParams{ + Column1: pgKeepUUID, + Column2: mi.ID, + }); err != nil { + return c.JSON(http.StatusInternalServerError, map[string]string{ + "error": fmt.Sprintf("failed to merge %q: %v", mi.FilePath, err), + }) + } + if err := h.db.DeleteMediaItem(ctx, mi.ID); err != nil { + return c.JSON(http.StatusInternalServerError, map[string]string{ + "error": fmt.Sprintf("failed to delete %q: %v", mi.FilePath, err), + }) + } + merged++ + } + + if err := h.db.ResolveHashConflict(ctx, database.ResolveHashConflictParams{ + ID: pgConflictID, + Resolution: pgtype.Text{String: "kept:" + keepUUID.String(), Valid: true}, + ResolvedBy: pgUserID, + }); err != nil { + return c.JSON(http.StatusInternalServerError, map[string]string{"error": "failed to resolve conflict"}) + } + + return renderResolved(c, fmt.Sprintf("Merged %d duplicate cop%s - all reading data preserved.", + merged, map[bool]string{true: "y", false: "ies"}[merged == 1])) + + default: + return c.JSON(http.StatusBadRequest, map[string]string{"error": "action must be 'keep_all' or 'keep'"}) + } +} + +// renderResolved returns the htmx fragment swapped in place of a conflict card. +// Built inline (rather than via the templates package) because templates +// imports handlers and a back-import would be a cycle. +func renderResolved(c *echo.Context, message string) error { + html := fmt.Sprintf(` +
+ + + +
+

Conflict resolved

+

%s

+
+
`, message) + return c.HTML(http.StatusOK, html) +} diff --git a/internal/router/frontend.go b/internal/router/frontend.go index d8741d6..556ef69 100644 --- a/internal/router/frontend.go +++ b/internal/router/frontend.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "encoding/json" + "fmt" "log" "net/http" "strconv" @@ -944,6 +945,67 @@ func registerFrontendRoutes(cfg *Config) { return c.HTML(http.StatusOK, buf.String()) })) + // Admin hash conflicts page: content-duplicate groups flagged during hash + // backfill or rescan, resolved by keeping all copies or merging into one. + frontendProtected.GET("/admin/hash-conflicts", handlers.AdminMiddleware(func(c *echo.Context) error { + user, err := getTemplateUserWithTheme(c, cfg) + if err != nil { + return renderErrorPage(c, "Error loading user", "user_load_error") + } + + pending, err := cfg.Queries.ListPendingHashConflicts(c.Request().Context()) + if err != nil { + return renderErrorPage(c, "Error loading hash conflicts", "conflicts_load_error") + } + + conflicts := make([]templates.HashConflictData, 0, len(pending)) + for _, p := range pending { + conflict := templates.HashConflictData{ + ID: uuid.UUID(p.ID.Bytes).String(), + LibraryName: p.LibraryName, + SHA256: p.FileSha256, + SHAShort: p.FileSha256[:16] + "…", + CreatedAt: p.CreatedAt.Time.Format("Jan 2, 2006"), + Items: []templates.HashConflictItemData{}, + } + + items, err := cfg.Queries.ListMediaItemsBySHA256AndLibrary(c.Request().Context(), database.ListMediaItemsBySHA256AndLibraryParams{ + FileSha256: pgtype.Text{String: p.FileSha256, Valid: true}, + LibraryID: p.LibraryID, + }) + if err != nil { + continue + } + + for _, mi := range items { + counts, err := cfg.Queries.GetMediaItemUsageCounts(c.Request().Context(), mi.ID) + if err != nil { + counts = database.GetMediaItemUsageCountsRow{} + } + totalData := counts.ProgressCount + counts.HighlightsCount + counts.BookmarksCount + counts.NotesCount + counts.CollectionsCount + conflict.Items = append(conflict.Items, templates.HashConflictItemData{ + ID: uuid.UUID(mi.ID.Bytes).String(), + Title: mi.Title, + Author: mi.Author.String, + FilePath: mi.FilePath, + FileSize: mi.FileSize.Int64, + UsageSummary: fmt.Sprintf("%d progress, %d highlights, %d bookmarks, %d notes, %d collections", + counts.ProgressCount, counts.HighlightsCount, counts.BookmarksCount, counts.NotesCount, counts.CollectionsCount), + HasReadingData: totalData > 0, + }) + } + + conflicts = append(conflicts, conflict) + } + + var buf bytes.Buffer + err = templates.AdminHashConflicts(user, conflicts).Render(c.Request().Context(), &buf) + if err != nil { + return err + } + return c.HTML(http.StatusOK, buf.String()) + })) + // Admin users page frontendProtected.GET("/admin/users", handlers.AdminMiddleware(func(c *echo.Context) error { user, err := getTemplateUserWithTheme(c, cfg) diff --git a/internal/router/router.go b/internal/router/router.go index 318abfa..b894a95 100644 --- a/internal/router/router.go +++ b/internal/router/router.go @@ -47,6 +47,7 @@ type Config struct { MediaHandler *handlers.MediaHandler MatchingHandler *handlers.MatchingHandler ProcessingIssuesHandler *handlers.ProcessingIssuesHandler + HashConflictsHandler *handlers.HashConflictsHandler KOReaderHandler *handlers.KOReaderHandler WSHandler *handlers.WSHandler ConflictHandler *handlers.ConflictHandler @@ -294,6 +295,17 @@ func RegisterRoutes(cfg *Config) *handlers.Handler { } }() + // One-time hash backfill: compute and store SHA-256 for media items + // imported before hashing existed, then flag any content-duplicate groups + // for admin review on the Hash Conflicts page. Runs independently of + // auto-scan (it is a one-shot self-heal, not a recurring scan) and is a + // no-op once every item is hashed. Delayed so it does not compete with + // startup scans for disk I/O. + go func() { + time.Sleep(30 * time.Second) + services.NewHashBackfillService(cfg.Queries).Run(context.Background()) + }() + // Register progress routes with actual handler registerProgressRoutes(cfg, scannerHandler) @@ -301,5 +313,9 @@ func RegisterRoutes(cfg *Config) *handlers.Handler { admin := protected.Group("", handlers.AdminMiddleware) registerScannerRoutes(admin, scannerHandler) + // Hash conflict routes (admin only) + admin.GET("/api/admin/hash-conflicts", cfg.HashConflictsHandler.ListHashConflicts) + admin.POST("/api/admin/hash-conflicts/:id/resolve", cfg.HashConflictsHandler.ResolveHashConflict) + return scannerHandler } diff --git a/internal/services/hash_backfill.go b/internal/services/hash_backfill.go new file mode 100644 index 0000000..31e80ee --- /dev/null +++ b/internal/services/hash_backfill.go @@ -0,0 +1,119 @@ +package services + +import ( + "bookhoard/internal/database" + "context" + "log" + "time" + + "github.com/jackc/pgx/v5/pgtype" +) + +// HashBackfillService is a one-time self-heal pass that computes and stores the +// SHA-256 for media items imported before hashing existed (file_sha256 IS +// NULL). It runs once shortly after startup, independently of auto-scan, and +// also performs a final conflict sweep that flags any content-duplicate groups +// (same library + SHA-256 at different paths) on the admin Hash Conflicts page. +// +// The sweep runs after the per-item pass because during the pass only one side +// of a preexisting duplicate pair may be hashed at a time - the group only +// becomes visible once every item has its hash. +type HashBackfillService struct { + db *database.Queries + libSvc *LibraryService +} + +// NewHashBackfillService creates a backfill service. +func NewHashBackfillService(db *database.Queries) *HashBackfillService { + return &HashBackfillService{db: db, libSvc: NewLibraryService(db)} +} + +// Run performs the backfill pass followed by the conflict sweep. It logs +// progress and never returns an error - failures on individual items are +// skipped so one unreadable file cannot block the rest. +func (s *HashBackfillService) Run(ctx context.Context) { + items, err := s.db.ListMediaItemsMissingHash(ctx) + if err != nil { + log.Printf("[HASH-BACKFILL] failed to list items missing hash: %v", err) + return + } + if len(items) == 0 { + log.Printf("[HASH-BACKFILL] all media items already hashed, nothing to do") + s.sweepConflicts(ctx) + return + } + + log.Printf("[HASH-BACKFILL] computing SHA-256 for %d unhashed media items", len(items)) + started := time.Now() + hashed, failed := 0, 0 + + for _, item := range items { + if ctx.Err() != nil { + log.Printf("[HASH-BACKFILL] cancelled after %d items", hashed) + return + } + + path, err := s.libSvc.ResolveMediaPath(ctx, item.LibraryID, item.FilePath) + if err != nil { + log.Printf("[HASH-BACKFILL] could not resolve path for %q: %v", item.FilePath, err) + failed++ + continue + } + + sha, err := computeFileSHA256(path) + if err != nil { + log.Printf("[HASH-BACKFILL] could not hash %q: %v", path, err) + failed++ + continue + } + + _, err = s.db.UpdateMediaItemIdentifiers(ctx, database.UpdateMediaItemIdentifiersParams{ + ID: item.ID, + FileSha256: pgtype.Text{String: sha, Valid: true}, + HashConfidence: pgtype.Text{String: "sha256_full", Valid: true}, + }) + if err != nil { + log.Printf("[HASH-BACKFILL] could not store hash for %q: %v", item.FilePath, err) + failed++ + continue + } + hashed++ + + if hashed%25 == 0 { + log.Printf("[HASH-BACKFILL] progress: %d/%d hashed", hashed, len(items)) + } + } + + log.Printf("[HASH-BACKFILL] done in %s: %d hashed, %d failed (of %d)", + time.Since(started).Round(time.Second), hashed, failed, len(items)) + + s.sweepConflicts(ctx) +} + +// sweepConflicts flags every content-duplicate group (same library + SHA-256, +// more than one item) as a pending hash conflict. The upsert is a no-op for +// groups that are already tracked or resolved, so admins who chose "keep both" +// are never re-prompted. +func (s *HashBackfillService) sweepConflicts(ctx context.Context) { + groups, err := s.db.FindHashConflictGroups(ctx) + if err != nil { + log.Printf("[HASH-BACKFILL] conflict sweep failed: %v", err) + return + } + if len(groups) == 0 { + return + } + + flagged := 0 + for _, g := range groups { + if err := s.db.CreateHashConflict(ctx, database.CreateHashConflictParams{ + LibraryID: g.LibraryID, + FileSha256: g.FileSha256.String, + }); err != nil { + log.Printf("[HASH-BACKFILL] could not record conflict group: %v", err) + continue + } + flagged++ + } + log.Printf("[HASH-BACKFILL] flagged %d content-duplicate group(s) for admin review", flagged) +}