From eea08630cecc0a612f423fd9e4b7718220fa5cdb Mon Sep 17 00:00:00 2001 From: John O'Keefe Date: Sun, 8 Feb 2026 11:03:40 -0500 Subject: [PATCH] feat: implement punctuation-aware tag and contributor normalization MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add comprehensive normalization functions with dual-field support: - NormalizeTags: Titlecase, preserve hyphens/apostrophes, prefer punctuated versions - NormalizeTagsSearch: Lowercase, remove punctuation for search - NormalizeContributors: Preserve case/punctuation, prefer punctuated versions - NormalizeContributorsSearch: Lowercase, remove punctuation for search Key features: - Case-insensitive deduplication using punctuation-free keys - Punctuation preference: keeps "ACME CORP." over "acme corp" - Handles hyphens as spaces ("non-fiction" → "non fiction" for search) - Preserves original casing for contributors (CAPSLOCK companies) Relates to Tags & Contributors Migration Phase 3 --- internal/utils/tags.go | 157 +++++++++++++++++++++++++++++++++-------- 1 file changed, 128 insertions(+), 29 deletions(-) diff --git a/internal/utils/tags.go b/internal/utils/tags.go index 5066a73..216481d 100644 --- a/internal/utils/tags.go +++ b/internal/utils/tags.go @@ -1,6 +1,7 @@ package utils import ( + "sort" "strings" "unicode" @@ -9,35 +10,81 @@ import ( ) // titleCaser is a global caser for titlecase conversion -var titleCaser = cases.Title(language.Und, cases.NoLower) +var titleCaser = cases.Title(language.Und) -// titlecase converts a string to title case while preserving hyphenation -// Example: "science fiction" → "Science Fiction", "non-fiction" → "Non-Fiction" +// titlecase converts a string to title case while preserving hyphenation and apostrophes +// Example: "science fiction" → "Science Fiction", "non-fiction" → "Non-Fiction", "o'reilly" → "O'Reilly" func titlecase(s string) string { - return titleCaser.String(s) + words := strings.Fields(s) + for i, word := range words { + // Handle words with hyphens or apostrophes + if strings.ContainsAny(word, "-'") { + var result strings.Builder + capitalizeNext := true + for _, r := range word { + if r == '-' || r == '\'' { + result.WriteRune(r) + capitalizeNext = true + } else if capitalizeNext { + result.WriteRune(unicode.ToUpper(r)) + capitalizeNext = false + } else { + result.WriteRune(unicode.ToLower(r)) + } + } + words[i] = result.String() + } else { + words[i] = titleCaser.String(word) + } + } + return strings.Join(words, " ") +} + +// hasPunctuation returns true if string contains any punctuation character +func hasPunctuation(s string) bool { + for _, r := range s { + if unicode.IsPunct(r) { + return true + } + } + return false } // removePunctuation removes all punctuation characters from a string -// Used for search field normalization only +// Replaces hyphens with spaces for search field normalization func removePunctuation(s string) string { - return strings.Map(func(r rune) rune { + var result strings.Builder + for _, r := range s { if unicode.IsPunct(r) { - return -1 + // Replace hyphens with spaces, remove other punctuation + if r == '-' { + result.WriteRune(' ') + } + } else { + result.WriteRune(r) } - return r - }, s) + } + + // Clean up multiple spaces + return strings.Join(strings.Fields(result.String()), " ") +} + +// candidate represents a potential tag/contributor value with metadata +type candidate struct { + value string + hasPunct bool + firstIndex int } // NormalizeTags normalizes an array of tags for display: // 1. Trim whitespace // 2. Titlecase (preserves hyphenation) -// 3. Case-insensitive deduplication +// 3. Case-insensitive deduplication with punctuation preference // 4. Remove empty strings func NormalizeTags(tags []string) []string { - seen := make(map[string]struct{}) - var normalized []string + candidates := make(map[string]candidate) - for _, tag := range tags { + for i, tag := range tags { // Trim whitespace tag = strings.TrimSpace(tag) @@ -47,19 +94,49 @@ func NormalizeTags(tags []string) []string { } // Titlecase for display (preserves hyphenation: "Non-Fiction") - tag = titlecase(tag) + titlecasedTag := titlecase(tag) - // Case-insensitive deduplication - key := strings.ToLower(tag) - if _, exists := seen[key]; !exists { - seen[key] = struct{}{} - normalized = append(normalized, tag) + // Case-insensitive deduplication with punctuation preference + // Remove punctuation and spaces for comparison to handle "nonfiction" == "non-fiction" + key := strings.ReplaceAll(removePunctuation(strings.ToLower(titlecasedTag)), " ", "") + if existing, exists := candidates[key]; !exists { + // First occurrence - store it + candidates[key] = candidate{value: titlecasedTag, hasPunct: hasPunctuation(titlecasedTag), firstIndex: i} + } else { + // Duplicate - upgrade if new version has punctuation and existing doesn't + if hasPunctuation(titlecasedTag) && !existing.hasPunct { + candidates[key] = candidate{value: titlecasedTag, hasPunct: true, firstIndex: existing.firstIndex} + } + // Otherwise keep first occurrence } } + // Extract values in order of first appearance + result := make([]pair, 0, len(candidates)) + for _, c := range candidates { + result = append(result, pair{value: c.value, index: c.firstIndex}) + } + + // Sort by first appearance + sort.Slice(result, func(i, j int) bool { + return result[i].index < result[j].index + }) + + // Extract sorted values + normalized := make([]string, len(result)) + for i, p := range result { + normalized[i] = p.value + } + return normalized } +// pair is a helper for sorting values by their original index +type pair struct { + value string + index int +} + // NormalizeTagsSearch normalizes an array of tags for searching: // 1. Trim whitespace // 2. Remove punctuation @@ -68,7 +145,7 @@ func NormalizeTags(tags []string) []string { // 5. Remove empty strings func NormalizeTagsSearch(tags []string) []string { seen := make(map[string]struct{}) - var normalized []string + normalized := make([]string, 0) for _, tag := range tags { // Trim whitespace @@ -104,13 +181,12 @@ func NormalizeTagsSearch(tags []string) []string { // 1. Trim whitespace // 2. Preserve original casing (including CAPSLOCK companies) // 3. Preserve original punctuation for display -// 4. Case-insensitive deduplication (removes punctuation for comparison only) +// 4. Case-insensitive deduplication with punctuation preference (removes punctuation for comparison only) // 5. Remove empty strings func NormalizeContributors(contributors []string) []string { - seen := make(map[string]struct{}) - var normalized []string + candidates := make(map[string]candidate) - for _, contributor := range contributors { + for i, contributor := range contributors { // Trim whitespace contributor = strings.TrimSpace(contributor) @@ -122,13 +198,36 @@ func NormalizeContributors(contributors []string) []string { // Case-insensitive deduplication (remove punctuation for dedup check only) dedupKey := removePunctuation(strings.ToLower(contributor)) - // Keep original case and punctuation for display - if _, exists := seen[dedupKey]; !exists { - _seen[dedupKey] = struct{}{} - normalized = append(normalized, contributor) + // Keep original case and punctuation for display with punctuation preference + if existing, exists := candidates[dedupKey]; !exists { + // First occurrence - store it + candidates[dedupKey] = candidate{value: contributor, hasPunct: hasPunctuation(contributor), firstIndex: i} + } else { + // Duplicate - upgrade if new version has punctuation and existing doesn't + if hasPunctuation(contributor) && !existing.hasPunct { + candidates[dedupKey] = candidate{value: contributor, hasPunct: true, firstIndex: existing.firstIndex} + } + // Otherwise keep first occurrence } } + // Extract values in order of first appearance + result := make([]pair, 0, len(candidates)) + for _, c := range candidates { + result = append(result, pair{value: c.value, index: c.firstIndex}) + } + + // Sort by first appearance + sort.Slice(result, func(i, j int) bool { + return result[i].index < result[j].index + }) + + // Extract sorted values + normalized := make([]string, len(result)) + for i, p := range result { + normalized[i] = p.value + } + return normalized } @@ -140,7 +239,7 @@ func NormalizeContributors(contributors []string) []string { // 5. Remove empty strings func NormalizeContributorsSearch(contributors []string) []string { seen := make(map[string]struct{}) - var normalized []string + normalized := make([]string, 0) for _, contributor := range contributors { // Trim whitespace