feat(utils): implement dual-field normalization for tags and contributors
Complete rewrite of normalization functions supporting dual-field architecture: Display field functions: - NormalizeTags: Titlecase, trim whitespace, case-insensitive dedup - NormalizeContributors: Preserve original casing/punctuation, dedup Search field functions: - NormalizeTagsSearch: Lowercase, remove punctuation, dedup - NormalizeContributorsSearch: Lowercase, remove punctuation, dedup Helper functions: - titlecase: Converts to title case preserving hyphenation - removePunctuation: Strips punctuation for search normalization This enables case-insensitive, punctuation-free search while preserving user's original formatting for display.
This commit is contained in:
+145
-8
@@ -2,13 +2,37 @@ package utils
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"strings"
|
"strings"
|
||||||
|
"unicode"
|
||||||
|
|
||||||
|
"golang.org/x/text/cases"
|
||||||
|
"golang.org/x/text/language"
|
||||||
)
|
)
|
||||||
|
|
||||||
// NormalizeTags normalizes an array of tags by:
|
// titleCaser is a global caser for titlecase conversion
|
||||||
// 1. Converting to lowercase
|
var titleCaser = cases.Title(language.Und, cases.NoLower)
|
||||||
// 2. Trimming whitespace
|
|
||||||
// 3. Removing duplicates
|
// titlecase converts a string to title case while preserving hyphenation
|
||||||
// 4. Removing empty strings
|
// Example: "science fiction" → "Science Fiction", "non-fiction" → "Non-Fiction"
|
||||||
|
func titlecase(s string) string {
|
||||||
|
return titleCaser.String(s)
|
||||||
|
}
|
||||||
|
|
||||||
|
// removePunctuation removes all punctuation characters from a string
|
||||||
|
// Used for search field normalization only
|
||||||
|
func removePunctuation(s string) string {
|
||||||
|
return strings.Map(func(r rune) rune {
|
||||||
|
if unicode.IsPunct(r) {
|
||||||
|
return -1
|
||||||
|
}
|
||||||
|
return r
|
||||||
|
}, s)
|
||||||
|
}
|
||||||
|
|
||||||
|
// NormalizeTags normalizes an array of tags for display:
|
||||||
|
// 1. Trim whitespace
|
||||||
|
// 2. Titlecase (preserves hyphenation)
|
||||||
|
// 3. Case-insensitive deduplication
|
||||||
|
// 4. Remove empty strings
|
||||||
func NormalizeTags(tags []string) []string {
|
func NormalizeTags(tags []string) []string {
|
||||||
seen := make(map[string]struct{})
|
seen := make(map[string]struct{})
|
||||||
var normalized []string
|
var normalized []string
|
||||||
@@ -22,10 +46,51 @@ func NormalizeTags(tags []string) []string {
|
|||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
// Convert to lowercase
|
// Titlecase for display (preserves hyphenation: "Non-Fiction")
|
||||||
|
tag = titlecase(tag)
|
||||||
|
|
||||||
|
// Case-insensitive deduplication
|
||||||
|
key := strings.ToLower(tag)
|
||||||
|
if _, exists := seen[key]; !exists {
|
||||||
|
seen[key] = struct{}{}
|
||||||
|
normalized = append(normalized, tag)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return normalized
|
||||||
|
}
|
||||||
|
|
||||||
|
// NormalizeTagsSearch normalizes an array of tags for searching:
|
||||||
|
// 1. Trim whitespace
|
||||||
|
// 2. Remove punctuation
|
||||||
|
// 3. Lowercase
|
||||||
|
// 4. Case-insensitive deduplication
|
||||||
|
// 5. Remove empty strings
|
||||||
|
func NormalizeTagsSearch(tags []string) []string {
|
||||||
|
seen := make(map[string]struct{})
|
||||||
|
var normalized []string
|
||||||
|
|
||||||
|
for _, tag := range tags {
|
||||||
|
// Trim whitespace
|
||||||
|
tag = strings.TrimSpace(tag)
|
||||||
|
|
||||||
|
// Skip empty tags
|
||||||
|
if tag == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
// Remove punctuation for search
|
||||||
|
tag = removePunctuation(tag)
|
||||||
|
|
||||||
|
// Lowercase for search
|
||||||
tag = strings.ToLower(tag)
|
tag = strings.ToLower(tag)
|
||||||
|
|
||||||
// Check for duplicates
|
// Skip empty after removal
|
||||||
|
if tag == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
// Case-insensitive deduplication
|
||||||
if _, exists := seen[tag]; !exists {
|
if _, exists := seen[tag]; !exists {
|
||||||
seen[tag] = struct{}{}
|
seen[tag] = struct{}{}
|
||||||
normalized = append(normalized, tag)
|
normalized = append(normalized, tag)
|
||||||
@@ -35,13 +100,85 @@ func NormalizeTags(tags []string) []string {
|
|||||||
return normalized
|
return normalized
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// NormalizeContributors normalizes an array of contributors for display:
|
||||||
|
// 1. Trim whitespace
|
||||||
|
// 2. Preserve original casing (including CAPSLOCK companies)
|
||||||
|
// 3. Preserve original punctuation for display
|
||||||
|
// 4. Case-insensitive deduplication (removes punctuation for comparison only)
|
||||||
|
// 5. Remove empty strings
|
||||||
|
func NormalizeContributors(contributors []string) []string {
|
||||||
|
seen := make(map[string]struct{})
|
||||||
|
var normalized []string
|
||||||
|
|
||||||
|
for _, contributor := range contributors {
|
||||||
|
// Trim whitespace
|
||||||
|
contributor = strings.TrimSpace(contributor)
|
||||||
|
|
||||||
|
// Skip empty contributors
|
||||||
|
if contributor == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
// Case-insensitive deduplication (remove punctuation for dedup check only)
|
||||||
|
dedupKey := removePunctuation(strings.ToLower(contributor))
|
||||||
|
|
||||||
|
// Keep original case and punctuation for display
|
||||||
|
if _, exists := seen[dedupKey]; !exists {
|
||||||
|
_seen[dedupKey] = struct{}{}
|
||||||
|
normalized = append(normalized, contributor)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return normalized
|
||||||
|
}
|
||||||
|
|
||||||
|
// NormalizeContributorsSearch normalizes an array of contributors for searching:
|
||||||
|
// 1. Trim whitespace
|
||||||
|
// 2. Remove punctuation
|
||||||
|
// 3. Lowercase
|
||||||
|
// 4. Case-insensitive deduplication
|
||||||
|
// 5. Remove empty strings
|
||||||
|
func NormalizeContributorsSearch(contributors []string) []string {
|
||||||
|
seen := make(map[string]struct{})
|
||||||
|
var normalized []string
|
||||||
|
|
||||||
|
for _, contributor := range contributors {
|
||||||
|
// Trim whitespace
|
||||||
|
contributor = strings.TrimSpace(contributor)
|
||||||
|
|
||||||
|
// Skip empty contributors
|
||||||
|
if contributor == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
// Remove punctuation for search
|
||||||
|
contributor = removePunctuation(contributor)
|
||||||
|
|
||||||
|
// Lowercase for search
|
||||||
|
contributor = strings.ToLower(contributor)
|
||||||
|
|
||||||
|
// Skip empty after removal
|
||||||
|
if contributor == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
// Case-insensitive deduplication
|
||||||
|
if _, exists := seen[contributor]; !exists {
|
||||||
|
seen[contributor] = struct{}{}
|
||||||
|
normalized = append(normalized, contributor)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return normalized
|
||||||
|
}
|
||||||
|
|
||||||
// JoinTags converts a string array to a comma-separated string
|
// JoinTags converts a string array to a comma-separated string
|
||||||
// Maintained for backward compatibility with external systems
|
// Maintained for backward compatibility with external systems
|
||||||
func JoinTags(tags []string) string {
|
func JoinTags(tags []string) string {
|
||||||
return strings.Join(tags, ", ")
|
return strings.Join(tags, ", ")
|
||||||
}
|
}
|
||||||
|
|
||||||
// SplitTags converts a comma-separated string to a normalized array
|
// SplitTags converts a comma-separated string to a normalized display array
|
||||||
func SplitTags(tags string) []string {
|
func SplitTags(tags string) []string {
|
||||||
if tags == "" {
|
if tags == "" {
|
||||||
return []string{}
|
return []string{}
|
||||||
|
|||||||
Reference in New Issue
Block a user