feat(utils): implement dual-field normalization for tags and contributors

Complete rewrite of normalization functions supporting dual-field architecture:

Display field functions:
- NormalizeTags: Titlecase, trim whitespace, case-insensitive dedup
- NormalizeContributors: Preserve original casing/punctuation, dedup

Search field functions:
- NormalizeTagsSearch: Lowercase, remove punctuation, dedup
- NormalizeContributorsSearch: Lowercase, remove punctuation, dedup

Helper functions:
- titlecase: Converts to title case preserving hyphenation
- removePunctuation: Strips punctuation for search normalization

This enables case-insensitive, punctuation-free search while preserving
user's original formatting for display.
This commit is contained in:
2026-02-08 00:43:15 -05:00
parent cedea0f0eb
commit c955f6a518
+145 -8
View File
@@ -2,13 +2,37 @@ package utils
import (
"strings"
"unicode"
"golang.org/x/text/cases"
"golang.org/x/text/language"
)
// NormalizeTags normalizes an array of tags by:
// 1. Converting to lowercase
// 2. Trimming whitespace
// 3. Removing duplicates
// 4. Removing empty strings
// titleCaser is a global caser for titlecase conversion
var titleCaser = cases.Title(language.Und, cases.NoLower)
// titlecase converts a string to title case while preserving hyphenation
// Example: "science fiction" → "Science Fiction", "non-fiction" → "Non-Fiction"
func titlecase(s string) string {
return titleCaser.String(s)
}
// removePunctuation removes all punctuation characters from a string
// Used for search field normalization only
func removePunctuation(s string) string {
return strings.Map(func(r rune) rune {
if unicode.IsPunct(r) {
return -1
}
return r
}, s)
}
// NormalizeTags normalizes an array of tags for display:
// 1. Trim whitespace
// 2. Titlecase (preserves hyphenation)
// 3. Case-insensitive deduplication
// 4. Remove empty strings
func NormalizeTags(tags []string) []string {
seen := make(map[string]struct{})
var normalized []string
@@ -22,10 +46,51 @@ func NormalizeTags(tags []string) []string {
continue
}
// Convert to lowercase
// Titlecase for display (preserves hyphenation: "Non-Fiction")
tag = titlecase(tag)
// Case-insensitive deduplication
key := strings.ToLower(tag)
if _, exists := seen[key]; !exists {
seen[key] = struct{}{}
normalized = append(normalized, tag)
}
}
return normalized
}
// NormalizeTagsSearch normalizes an array of tags for searching:
// 1. Trim whitespace
// 2. Remove punctuation
// 3. Lowercase
// 4. Case-insensitive deduplication
// 5. Remove empty strings
func NormalizeTagsSearch(tags []string) []string {
seen := make(map[string]struct{})
var normalized []string
for _, tag := range tags {
// Trim whitespace
tag = strings.TrimSpace(tag)
// Skip empty tags
if tag == "" {
continue
}
// Remove punctuation for search
tag = removePunctuation(tag)
// Lowercase for search
tag = strings.ToLower(tag)
// Check for duplicates
// Skip empty after removal
if tag == "" {
continue
}
// Case-insensitive deduplication
if _, exists := seen[tag]; !exists {
seen[tag] = struct{}{}
normalized = append(normalized, tag)
@@ -35,13 +100,85 @@ func NormalizeTags(tags []string) []string {
return normalized
}
// NormalizeContributors normalizes an array of contributors for display:
// 1. Trim whitespace
// 2. Preserve original casing (including CAPSLOCK companies)
// 3. Preserve original punctuation for display
// 4. Case-insensitive deduplication (removes punctuation for comparison only)
// 5. Remove empty strings
func NormalizeContributors(contributors []string) []string {
seen := make(map[string]struct{})
var normalized []string
for _, contributor := range contributors {
// Trim whitespace
contributor = strings.TrimSpace(contributor)
// Skip empty contributors
if contributor == "" {
continue
}
// Case-insensitive deduplication (remove punctuation for dedup check only)
dedupKey := removePunctuation(strings.ToLower(contributor))
// Keep original case and punctuation for display
if _, exists := seen[dedupKey]; !exists {
_seen[dedupKey] = struct{}{}
normalized = append(normalized, contributor)
}
}
return normalized
}
// NormalizeContributorsSearch normalizes an array of contributors for searching:
// 1. Trim whitespace
// 2. Remove punctuation
// 3. Lowercase
// 4. Case-insensitive deduplication
// 5. Remove empty strings
func NormalizeContributorsSearch(contributors []string) []string {
seen := make(map[string]struct{})
var normalized []string
for _, contributor := range contributors {
// Trim whitespace
contributor = strings.TrimSpace(contributor)
// Skip empty contributors
if contributor == "" {
continue
}
// Remove punctuation for search
contributor = removePunctuation(contributor)
// Lowercase for search
contributor = strings.ToLower(contributor)
// Skip empty after removal
if contributor == "" {
continue
}
// Case-insensitive deduplication
if _, exists := seen[contributor]; !exists {
seen[contributor] = struct{}{}
normalized = append(normalized, contributor)
}
}
return normalized
}
// JoinTags converts a string array to a comma-separated string
// Maintained for backward compatibility with external systems
func JoinTags(tags []string) string {
return strings.Join(tags, ", ")
}
// SplitTags converts a comma-separated string to a normalized array
// SplitTags converts a comma-separated string to a normalized display array
func SplitTags(tags string) []string {
if tags == "" {
return []string{}