Bookhoard/bookhoard
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 12 Wiki Activity

feat(db): hash_conflicts table and backfill/conflict queries

Browse Source 
Content duplicates (same library + file_sha256 at different paths,
e.g. the same book imported twice under two names on a preexisting
database) cannot be auto-collapsed the way path duplicates were:
keeping both copies may be intentional. Surface them for an explicit
admin decision instead.

Schema:
- new hash_conflicts table keyed (library_id, file_sha256) with a
  status/resolution lifecycle: 'pending' until an admin resolves via
  'keep_all' or 'kept:<uuid>' (which copy was kept after merging)
- resolution is VARCHAR(50) - 'kept:<uuid>' is 41 chars; include a
  widening ALTER for databases created with the initial 30-char width
- resolution/resolved_by/resolved_at record who decided what and when

Queries:
- ListMediaItemsMissingHash: items imported before hashing existed
  (file_sha256 IS NULL), ordered oldest-first for the backfill pass
- FindHashConflictGroups: the content-duplicate group detection
  (GROUP BY library_id, file_sha256 HAVING COUNT(*) > 1)
- ListMediaItemsBySHA256AndLibrary: full membership of one group
- CreateHashConflict: upsert with DO NOTHING so already-tracked groups
  are untouched - critical behavior: a group an admin resolved as
  'keep both' is never re-flagged by later sweeps
- ListPendingHashConflicts: admin listing with library name and live
  item counts (items may have been deleted since flagging)
- GetHashConflict / ResolveHashConflict: lifecycle
- GetMediaItemUsageCounts: per-item progress/highlight/bookmark/note/
  collection counts so the admin can make an informed keep choice
- ReparentMediaItemChildren: sqlc binding for the existing
  reparent_media_item_children() migration function, used to merge a
  losing copy's child rows into the kept copy
This commit is contained in:
john-okeefe
2026-08-14 08:52:05 -04:00
parent 8599e5c250
commit 0c39e04e4a
5 changed files with 485 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/database/queries.sql.go
+368
View File
@@ -567,6 +567,26 @@ func (q *Queries) CreateDictionaryEntry(ctx context.Context, arg CreateDictionar
return i, err
}
const CreateHashConflict = `-- name: CreateHashConflict :exec
INSERT INTO hash_conflicts (library_id, file_sha256)
VALUES ($1, $2)
ON CONFLICT (library_id, file_sha256) DO NOTHING
`
type CreateHashConflictParams struct {
LibraryID pgtype.UUID `db:"library_id" json:"library_id"`
FileSha256 string `db:"file_sha256" json:"file_sha256"`
}
// HASH CONFLICTS QUERIES
// Record a pending hash conflict (no-op if the group is already tracked, so
// resolved groups stay resolved and are never re-flagged)
func (q *Queries) CreateHashConflict(ctx context.Context, arg CreateHashConflictParams) error {
_, err := q.db.Exec(ctx, CreateHashConflict, arg.LibraryID, arg.FileSha256)
return err
}
const CreateLibrary = `-- name: CreateLibrary :one
INSERT INTO libraries (name, description, library_type_id, created_by_admin_id)
VALUES ($1, $2, $3, $4)
@@ -2085,6 +2105,41 @@ func (q *Queries) DeleteUserSystemCollection(ctx context.Context, arg DeleteUser
return err
}
const FindHashConflictGroups = `-- name: FindHashConflictGroups :many
SELECT library_id, file_sha256, COUNT(*) AS dup_count
FROM media_items
WHERE file_sha256 IS NOT NULL
GROUP BY library_id, file_sha256
HAVING COUNT(*) > 1
`
type FindHashConflictGroupsRow struct {
LibraryID pgtype.UUID `db:"library_id" json:"library_id"`
FileSha256 pgtype.Text `db:"file_sha256" json:"file_sha256"`
DupCount int64 `db:"dup_count" json:"dup_count"`
}
// Find content-duplicate groups (same library + SHA-256, more than one row)
func (q *Queries) FindHashConflictGroups(ctx context.Context) ([]FindHashConflictGroupsRow, error) {
rows, err := q.db.Query(ctx, FindHashConflictGroups)
if err != nil {
return nil, err
}
defer rows.Close()
items := []FindHashConflictGroupsRow{}
for rows.Next() {
var i FindHashConflictGroupsRow
if err := rows.Scan(&i.LibraryID, &i.FileSha256, &i.DupCount); err != nil {
return nil, err
}
items = append(items, i)
}
if err := rows.Err(); err != nil {
return nil, err
}
return items, nil
}
const GenerateKoboEntitlementId = `-- name: GenerateKoboEntitlementId :one
SELECT 'kobo_' || uuid_generate_v4()::TEXT as entitlement_id
`
@@ -3711,6 +3766,26 @@ func (q *Queries) GetFirstAdminExclude(ctx context.Context, id pgtype.UUID) (Get
return i, err
}
const GetHashConflict = `-- name: GetHashConflict :one
SELECT id, library_id, file_sha256, status, resolution, resolved_by, created_at, resolved_at FROM hash_conflicts WHERE id = $1
`
func (q *Queries) GetHashConflict(ctx context.Context, id pgtype.UUID) (HashConflicts, error) {
row := q.db.QueryRow(ctx, GetHashConflict, id)
var i HashConflicts
err := row.Scan(
&i.ID,
&i.LibraryID,
&i.FileSha256,
&i.Status,
&i.Resolution,
&i.ResolvedBy,
&i.CreatedAt,
&i.ResolvedAt,
)
return i, err
}
const GetKoboEntitlementByContentId = `-- name: GetKoboEntitlementByContentId :one
SELECT ke.id, ke.device_id, ke.media_item_id, ke.entitlement_id, ke.content_id, ke.revision_number, ke.purchase_date, ke.accession_date, ke.book_status, ke.sync_status, ke.kobo_metadata, ke.created_at, ke.updated_at, mi.title, mi.author, mi.file_path
FROM kobo_entitlements ke
@@ -5481,6 +5556,37 @@ func (q *Queries) GetMediaItemFormats(ctx context.Context, mediaItemID pgtype.UU
return items, nil
}
const GetMediaItemUsageCounts = `-- name: GetMediaItemUsageCounts :one
SELECT
(SELECT COUNT(*) FROM reading_progress rp WHERE rp.media_item_id = $1) AS progress_count,
(SELECT COUNT(*) FROM media_highlights mh WHERE mh.media_item_id = $1) AS highlights_count,
(SELECT COUNT(*) FROM media_bookmarks mb WHERE mb.media_item_id = $1) AS bookmarks_count,
(SELECT COUNT(*) FROM media_notes mn WHERE mn.media_item_id = $1) AS notes_count,
(SELECT COUNT(*) FROM collection_items ci WHERE ci.media_item_id = $1) AS collections_count
`
type GetMediaItemUsageCountsRow struct {
ProgressCount int64 `db:"progress_count" json:"progress_count"`
HighlightsCount int64 `db:"highlights_count" json:"highlights_count"`
BookmarksCount int64 `db:"bookmarks_count" json:"bookmarks_count"`
NotesCount int64 `db:"notes_count" json:"notes_count"`
CollectionsCount int64 `db:"collections_count" json:"collections_count"`
}
// Per-item user-data counts, used when choosing which duplicate copy to keep
func (q *Queries) GetMediaItemUsageCounts(ctx context.Context, mediaItemID pgtype.UUID) (GetMediaItemUsageCountsRow, error) {
row := q.db.QueryRow(ctx, GetMediaItemUsageCounts, mediaItemID)
var i GetMediaItemUsageCountsRow
err := row.Scan(
&i.ProgressCount,
&i.HighlightsCount,
&i.BookmarksCount,
&i.NotesCount,
&i.CollectionsCount,
)
return i, err
}
const GetMediaNote = `-- name: GetMediaNote :one
SELECT id, media_item_id, user_id, content, position, created_at, updated_at, percentage_location, character_start, character_end, epubcfi_location, chapter_reference, paragraph_reference, device_sync_data, dedup_key, last_modified_at, last_modified_source, deleted, deleted_at FROM media_notes WHERE id = $1
`
@@ -8450,6 +8556,185 @@ func (q *Queries) ListMediaItemsByLibrary(ctx context.Context, libraryID pgtype.
return items, nil
}
const ListMediaItemsBySHA256AndLibrary = `-- name: ListMediaItemsBySHA256AndLibrary :many
SELECT id, library_id, title, author, isbn, description, file_path, file_size, mime_type, cover_image_path, series, series_number, tags, asin, date_published, publisher, contributors, language, edition, page_count, genre, copyright_year, goodreads_id, openlibrary_id, google_books_id, added_by_admin_id, created_at, imported_at, updated_at, format_group, format_mimetype, is_reflowable, has_fixed_layout, total_characters, chapter_count, entitlement_id, revision_number, kobo_content_id, kobo_metadata, manga_type, reading_direction, series_count, volume, imprint, age_rating, web_url, story_arc, is_black_and_white, metadata_notes, community_rating, alternate_info, scan_information, summary, chapter_metadata, library_type_name, tags_search, contributors_search, file_sha256, opf_identifier, opf_uuid, hash_confidence FROM media_items WHERE file_sha256 = $1 AND library_id = $2 ORDER BY file_path
`
type ListMediaItemsBySHA256AndLibraryParams struct {
FileSha256 pgtype.Text `db:"file_sha256" json:"file_sha256"`
LibraryID pgtype.UUID `db:"library_id" json:"library_id"`
}
// List all media items sharing a SHA-256 hash within a library (hash conflict group)
func (q *Queries) ListMediaItemsBySHA256AndLibrary(ctx context.Context, arg ListMediaItemsBySHA256AndLibraryParams) ([]MediaItems, error) {
rows, err := q.db.Query(ctx, ListMediaItemsBySHA256AndLibrary, arg.FileSha256, arg.LibraryID)
if err != nil {
return nil, err
}
defer rows.Close()
items := []MediaItems{}
for rows.Next() {
var i MediaItems
if err := rows.Scan(
&i.ID,
&i.LibraryID,
&i.Title,
&i.Author,
&i.Isbn,
&i.Description,
&i.FilePath,
&i.FileSize,
&i.MimeType,
&i.CoverImagePath,
&i.Series,
&i.SeriesNumber,
&i.Tags,
&i.Asin,
&i.DatePublished,
&i.Publisher,
&i.Contributors,
&i.Language,
&i.Edition,
&i.PageCount,
&i.Genre,
&i.CopyrightYear,
&i.GoodreadsID,
&i.OpenlibraryID,
&i.GoogleBooksID,
&i.AddedByAdminID,
&i.CreatedAt,
&i.ImportedAt,
&i.UpdatedAt,
&i.FormatGroup,
&i.FormatMimetype,
&i.IsReflowable,
&i.HasFixedLayout,
&i.TotalCharacters,
&i.ChapterCount,
&i.EntitlementID,
&i.RevisionNumber,
&i.KoboContentID,
&i.KoboMetadata,
&i.MangaType,
&i.ReadingDirection,
&i.SeriesCount,
&i.Volume,
&i.Imprint,
&i.AgeRating,
&i.WebUrl,
&i.StoryArc,
&i.IsBlackAndWhite,
&i.MetadataNotes,
&i.CommunityRating,
&i.AlternateInfo,
&i.ScanInformation,
&i.Summary,
&i.ChapterMetadata,
&i.LibraryTypeName,
&i.TagsSearch,
&i.ContributorsSearch,
&i.FileSha256,
&i.OpfIdentifier,
&i.OpfUuid,
&i.HashConfidence,
); err != nil {
return nil, err
}
items = append(items, i)
}
if err := rows.Err(); err != nil {
return nil, err
}
return items, nil
}
const ListMediaItemsMissingHash = `-- name: ListMediaItemsMissingHash :many
SELECT id, library_id, title, author, isbn, description, file_path, file_size, mime_type, cover_image_path, series, series_number, tags, asin, date_published, publisher, contributors, language, edition, page_count, genre, copyright_year, goodreads_id, openlibrary_id, google_books_id, added_by_admin_id, created_at, imported_at, updated_at, format_group, format_mimetype, is_reflowable, has_fixed_layout, total_characters, chapter_count, entitlement_id, revision_number, kobo_content_id, kobo_metadata, manga_type, reading_direction, series_count, volume, imprint, age_rating, web_url, story_arc, is_black_and_white, metadata_notes, community_rating, alternate_info, scan_information, summary, chapter_metadata, library_type_name, tags_search, contributors_search, file_sha256, opf_identifier, opf_uuid, hash_confidence FROM media_items WHERE file_sha256 IS NULL ORDER BY created_at
`
// List media items that have no stored SHA-256 (imported before hashing existed)
func (q *Queries) ListMediaItemsMissingHash(ctx context.Context) ([]MediaItems, error) {
rows, err := q.db.Query(ctx, ListMediaItemsMissingHash)
if err != nil {
return nil, err
}
defer rows.Close()
items := []MediaItems{}
for rows.Next() {
var i MediaItems
if err := rows.Scan(
&i.ID,
&i.LibraryID,
&i.Title,
&i.Author,
&i.Isbn,
&i.Description,
&i.FilePath,
&i.FileSize,
&i.MimeType,
&i.CoverImagePath,
&i.Series,
&i.SeriesNumber,
&i.Tags,
&i.Asin,
&i.DatePublished,
&i.Publisher,
&i.Contributors,
&i.Language,
&i.Edition,
&i.PageCount,
&i.Genre,
&i.CopyrightYear,
&i.GoodreadsID,
&i.OpenlibraryID,
&i.GoogleBooksID,
&i.AddedByAdminID,
&i.CreatedAt,
&i.ImportedAt,
&i.UpdatedAt,
&i.FormatGroup,
&i.FormatMimetype,
&i.IsReflowable,
&i.HasFixedLayout,
&i.TotalCharacters,
&i.ChapterCount,
&i.EntitlementID,
&i.RevisionNumber,
&i.KoboContentID,
&i.KoboMetadata,
&i.MangaType,
&i.ReadingDirection,
&i.SeriesCount,
&i.Volume,
&i.Imprint,
&i.AgeRating,
&i.WebUrl,
&i.StoryArc,
&i.IsBlackAndWhite,
&i.MetadataNotes,
&i.CommunityRating,
&i.AlternateInfo,
&i.ScanInformation,
&i.Summary,
&i.ChapterMetadata,
&i.LibraryTypeName,
&i.TagsSearch,
&i.ContributorsSearch,
&i.FileSha256,
&i.OpfIdentifier,
&i.OpfUuid,
&i.HashConfidence,
); err != nil {
return nil, err
}
items = append(items, i)
}
if err := rows.Err(); err != nil {
return nil, err
}
return items, nil
}
const ListMediaItemsSorted = `-- name: ListMediaItemsSorted :many
SELECT mi.id, mi.library_id, mi.title, mi.author, mi.isbn, mi.description, mi.file_path, mi.file_size, mi.mime_type, mi.cover_image_path, mi.series, mi.series_number, mi.tags, mi.asin, mi.date_published, mi.publisher, mi.contributors, mi.language, mi.edition, mi.page_count, mi.genre, mi.copyright_year, mi.goodreads_id, mi.openlibrary_id, mi.google_books_id, mi.added_by_admin_id, mi.created_at, mi.imported_at, mi.updated_at, mi.format_group, mi.format_mimetype, mi.is_reflowable, mi.has_fixed_layout, mi.total_characters, mi.chapter_count, mi.entitlement_id, mi.revision_number, mi.kobo_content_id, mi.kobo_metadata, mi.manga_type, mi.reading_direction, mi.series_count, mi.volume, mi.imprint, mi.age_rating, mi.web_url, mi.story_arc, mi.is_black_and_white, mi.metadata_notes, mi.community_rating, mi.alternate_info, mi.scan_information, mi.summary, mi.chapter_metadata, mi.library_type_name, mi.tags_search, mi.contributors_search, mi.file_sha256, mi.opf_identifier, mi.opf_uuid, mi.hash_confidence, l.name as library_name, lt.name as library_type_name
FROM media_items mi
@@ -8687,6 +8972,54 @@ func (q *Queries) ListMediaItemsSorted(ctx context.Context, arg ListMediaItemsSo
return items, nil
}
const ListPendingHashConflicts = `-- name: ListPendingHashConflicts :many
SELECT hc.id, hc.library_id, hc.file_sha256, hc.created_at,
l.name AS library_name,
COUNT(mi.id) AS item_count
FROM hash_conflicts hc
JOIN libraries l ON l.id = hc.library_id
LEFT JOIN media_items mi ON mi.library_id = hc.library_id AND mi.file_sha256 = hc.file_sha256
WHERE hc.status = 'pending'
GROUP BY hc.id, hc.library_id, hc.file_sha256, hc.created_at, l.name
ORDER BY hc.created_at
`
type ListPendingHashConflictsRow struct {
ID pgtype.UUID `db:"id" json:"id"`
LibraryID pgtype.UUID `db:"library_id" json:"library_id"`
FileSha256 string `db:"file_sha256" json:"file_sha256"`
CreatedAt pgtype.Timestamptz `db:"created_at" json:"created_at"`
LibraryName string `db:"library_name" json:"library_name"`
ItemCount int64 `db:"item_count" json:"item_count"`
}
func (q *Queries) ListPendingHashConflicts(ctx context.Context) ([]ListPendingHashConflictsRow, error) {
rows, err := q.db.Query(ctx, ListPendingHashConflicts)
if err != nil {
return nil, err
}
defer rows.Close()
items := []ListPendingHashConflictsRow{}
for rows.Next() {
var i ListPendingHashConflictsRow
if err := rows.Scan(
&i.ID,
&i.LibraryID,
&i.FileSha256,
&i.CreatedAt,
&i.LibraryName,
&i.ItemCount,
); err != nil {
return nil, err
}
items = append(items, i)
}
if err := rows.Err(); err != nil {
return nil, err
}
return items, nil
}
const ListPendingSyncQueueItems = `-- name: ListPendingSyncQueueItems :many
SELECT id, device_id, media_item_id, sync_type, sync_data, priority, attempts, max_attempts, status, error_message, created_at, processed_at FROM sync_queue
WHERE device_id = $1 AND status = 'pending'
@@ -9221,6 +9554,21 @@ func (q *Queries) RemoveBookFromKoboShelf(ctx context.Context, arg RemoveBookFro
return err
}
const ReparentMediaItemChildren = `-- name: ReparentMediaItemChildren :exec
SELECT reparent_media_item_children($1::uuid, $2::uuid)
`
type ReparentMediaItemChildrenParams struct {
Column1 pgtype.UUID `db:"column_1" json:"column_1"`
Column2 pgtype.UUID `db:"column_2" json:"column_2"`
}
// Re-parent all child rows of p_source onto p_target (defined in schema.sql)
func (q *Queries) ReparentMediaItemChildren(ctx context.Context, arg ReparentMediaItemChildrenParams) error {
_, err := q.db.Exec(ctx, ReparentMediaItemChildren, arg.Column1, arg.Column2)
return err
}
const ResetSystemCollectionMetadata = `-- name: ResetSystemCollectionMetadata :exec
UPDATE collections
SET description = $3,
@@ -9253,6 +9601,26 @@ func (q *Queries) ResetSystemCollectionMetadata(ctx context.Context, arg ResetSy
return err
}
const ResolveHashConflict = `-- name: ResolveHashConflict :exec
UPDATE hash_conflicts
SET status = 'resolved',
resolution = $2,
resolved_by = $3,
resolved_at = NOW()
WHERE id = $1
`
type ResolveHashConflictParams struct {
ID pgtype.UUID `db:"id" json:"id"`
Resolution pgtype.Text `db:"resolution" json:"resolution"`
ResolvedBy pgtype.UUID `db:"resolved_by" json:"resolved_by"`
}
func (q *Queries) ResolveHashConflict(ctx context.Context, arg ResolveHashConflictParams) error {
_, err := q.db.Exec(ctx, ResolveHashConflict, arg.ID, arg.Resolution, arg.ResolvedBy)
return err
}
const ResolveProcessingIssue = `-- name: ResolveProcessingIssue :one
UPDATE processing_issues
SET resolved = true,