feat(utils): implement dual-field normalization for tags and contributors
Complete rewrite of normalization functions supporting dual-field architecture: Display field functions: - NormalizeTags: Titlecase, trim whitespace, case-insensitive dedup - NormalizeContributors: Preserve original casing/punctuation, dedup Search field functions: - NormalizeTagsSearch: Lowercase, remove punctuation, dedup - NormalizeContributorsSearch: Lowercase, remove punctuation, dedup Helper functions: - titlecase: Converts to title case preserving hyphenation - removePunctuation: Strips punctuation for search normalization This enables case-insensitive, punctuation-free search while preserving user's original formatting for display.
This commit is contained in:
+145
-8
@@ -2,13 +2,37 @@ package utils
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"unicode"
|
||||
|
||||
"golang.org/x/text/cases"
|
||||
"golang.org/x/text/language"
|
||||
)
|
||||
|
||||
// NormalizeTags normalizes an array of tags by:
|
||||
// 1. Converting to lowercase
|
||||
// 2. Trimming whitespace
|
||||
// 3. Removing duplicates
|
||||
// 4. Removing empty strings
|
||||
// titleCaser is a global caser for titlecase conversion
|
||||
var titleCaser = cases.Title(language.Und, cases.NoLower)
|
||||
|
||||
// titlecase converts a string to title case while preserving hyphenation
|
||||
// Example: "science fiction" → "Science Fiction", "non-fiction" → "Non-Fiction"
|
||||
func titlecase(s string) string {
|
||||
return titleCaser.String(s)
|
||||
}
|
||||
|
||||
// removePunctuation removes all punctuation characters from a string
|
||||
// Used for search field normalization only
|
||||
func removePunctuation(s string) string {
|
||||
return strings.Map(func(r rune) rune {
|
||||
if unicode.IsPunct(r) {
|
||||
return -1
|
||||
}
|
||||
return r
|
||||
}, s)
|
||||
}
|
||||
|
||||
// NormalizeTags normalizes an array of tags for display:
|
||||
// 1. Trim whitespace
|
||||
// 2. Titlecase (preserves hyphenation)
|
||||
// 3. Case-insensitive deduplication
|
||||
// 4. Remove empty strings
|
||||
func NormalizeTags(tags []string) []string {
|
||||
seen := make(map[string]struct{})
|
||||
var normalized []string
|
||||
@@ -22,10 +46,51 @@ func NormalizeTags(tags []string) []string {
|
||||
continue
|
||||
}
|
||||
|
||||
// Convert to lowercase
|
||||
// Titlecase for display (preserves hyphenation: "Non-Fiction")
|
||||
tag = titlecase(tag)
|
||||
|
||||
// Case-insensitive deduplication
|
||||
key := strings.ToLower(tag)
|
||||
if _, exists := seen[key]; !exists {
|
||||
seen[key] = struct{}{}
|
||||
normalized = append(normalized, tag)
|
||||
}
|
||||
}
|
||||
|
||||
return normalized
|
||||
}
|
||||
|
||||
// NormalizeTagsSearch normalizes an array of tags for searching:
|
||||
// 1. Trim whitespace
|
||||
// 2. Remove punctuation
|
||||
// 3. Lowercase
|
||||
// 4. Case-insensitive deduplication
|
||||
// 5. Remove empty strings
|
||||
func NormalizeTagsSearch(tags []string) []string {
|
||||
seen := make(map[string]struct{})
|
||||
var normalized []string
|
||||
|
||||
for _, tag := range tags {
|
||||
// Trim whitespace
|
||||
tag = strings.TrimSpace(tag)
|
||||
|
||||
// Skip empty tags
|
||||
if tag == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
// Remove punctuation for search
|
||||
tag = removePunctuation(tag)
|
||||
|
||||
// Lowercase for search
|
||||
tag = strings.ToLower(tag)
|
||||
|
||||
// Check for duplicates
|
||||
// Skip empty after removal
|
||||
if tag == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
// Case-insensitive deduplication
|
||||
if _, exists := seen[tag]; !exists {
|
||||
seen[tag] = struct{}{}
|
||||
normalized = append(normalized, tag)
|
||||
@@ -35,13 +100,85 @@ func NormalizeTags(tags []string) []string {
|
||||
return normalized
|
||||
}
|
||||
|
||||
// NormalizeContributors normalizes an array of contributors for display:
|
||||
// 1. Trim whitespace
|
||||
// 2. Preserve original casing (including CAPSLOCK companies)
|
||||
// 3. Preserve original punctuation for display
|
||||
// 4. Case-insensitive deduplication (removes punctuation for comparison only)
|
||||
// 5. Remove empty strings
|
||||
func NormalizeContributors(contributors []string) []string {
|
||||
seen := make(map[string]struct{})
|
||||
var normalized []string
|
||||
|
||||
for _, contributor := range contributors {
|
||||
// Trim whitespace
|
||||
contributor = strings.TrimSpace(contributor)
|
||||
|
||||
// Skip empty contributors
|
||||
if contributor == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
// Case-insensitive deduplication (remove punctuation for dedup check only)
|
||||
dedupKey := removePunctuation(strings.ToLower(contributor))
|
||||
|
||||
// Keep original case and punctuation for display
|
||||
if _, exists := seen[dedupKey]; !exists {
|
||||
_seen[dedupKey] = struct{}{}
|
||||
normalized = append(normalized, contributor)
|
||||
}
|
||||
}
|
||||
|
||||
return normalized
|
||||
}
|
||||
|
||||
// NormalizeContributorsSearch normalizes an array of contributors for searching:
|
||||
// 1. Trim whitespace
|
||||
// 2. Remove punctuation
|
||||
// 3. Lowercase
|
||||
// 4. Case-insensitive deduplication
|
||||
// 5. Remove empty strings
|
||||
func NormalizeContributorsSearch(contributors []string) []string {
|
||||
seen := make(map[string]struct{})
|
||||
var normalized []string
|
||||
|
||||
for _, contributor := range contributors {
|
||||
// Trim whitespace
|
||||
contributor = strings.TrimSpace(contributor)
|
||||
|
||||
// Skip empty contributors
|
||||
if contributor == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
// Remove punctuation for search
|
||||
contributor = removePunctuation(contributor)
|
||||
|
||||
// Lowercase for search
|
||||
contributor = strings.ToLower(contributor)
|
||||
|
||||
// Skip empty after removal
|
||||
if contributor == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
// Case-insensitive deduplication
|
||||
if _, exists := seen[contributor]; !exists {
|
||||
seen[contributor] = struct{}{}
|
||||
normalized = append(normalized, contributor)
|
||||
}
|
||||
}
|
||||
|
||||
return normalized
|
||||
}
|
||||
|
||||
// JoinTags converts a string array to a comma-separated string
|
||||
// Maintained for backward compatibility with external systems
|
||||
func JoinTags(tags []string) string {
|
||||
return strings.Join(tags, ", ")
|
||||
}
|
||||
|
||||
// SplitTags converts a comma-separated string to a normalized array
|
||||
// SplitTags converts a comma-separated string to a normalized display array
|
||||
func SplitTags(tags string) []string {
|
||||
if tags == "" {
|
||||
return []string{}
|
||||
|
||||
Reference in New Issue
Block a user