feat: implement punctuation-aware tag and contributor normalization
Add comprehensive normalization functions with dual-field support:
- NormalizeTags: Titlecase, preserve hyphens/apostrophes, prefer punctuated versions
- NormalizeTagsSearch: Lowercase, remove punctuation for search
- NormalizeContributors: Preserve case/punctuation, prefer punctuated versions
- NormalizeContributorsSearch: Lowercase, remove punctuation for search
Key features:
- Case-insensitive deduplication using punctuation-free keys
- Punctuation preference: keeps "ACME CORP." over "acme corp"
- Handles hyphens as spaces ("non-fiction" → "non fiction" for search)
- Preserves original casing for contributors (CAPSLOCK companies)
Relates to Tags & Contributors Migration Phase 3
This commit is contained in:
+128
-29
@@ -1,6 +1,7 @@
|
||||
package utils
|
||||
|
||||
import (
|
||||
"sort"
|
||||
"strings"
|
||||
"unicode"
|
||||
|
||||
@@ -9,35 +10,81 @@ import (
|
||||
)
|
||||
|
||||
// titleCaser is a global caser for titlecase conversion
|
||||
var titleCaser = cases.Title(language.Und, cases.NoLower)
|
||||
var titleCaser = cases.Title(language.Und)
|
||||
|
||||
// titlecase converts a string to title case while preserving hyphenation
|
||||
// Example: "science fiction" → "Science Fiction", "non-fiction" → "Non-Fiction"
|
||||
// titlecase converts a string to title case while preserving hyphenation and apostrophes
|
||||
// Example: "science fiction" → "Science Fiction", "non-fiction" → "Non-Fiction", "o'reilly" → "O'Reilly"
|
||||
func titlecase(s string) string {
|
||||
return titleCaser.String(s)
|
||||
words := strings.Fields(s)
|
||||
for i, word := range words {
|
||||
// Handle words with hyphens or apostrophes
|
||||
if strings.ContainsAny(word, "-'") {
|
||||
var result strings.Builder
|
||||
capitalizeNext := true
|
||||
for _, r := range word {
|
||||
if r == '-' || r == '\'' {
|
||||
result.WriteRune(r)
|
||||
capitalizeNext = true
|
||||
} else if capitalizeNext {
|
||||
result.WriteRune(unicode.ToUpper(r))
|
||||
capitalizeNext = false
|
||||
} else {
|
||||
result.WriteRune(unicode.ToLower(r))
|
||||
}
|
||||
}
|
||||
words[i] = result.String()
|
||||
} else {
|
||||
words[i] = titleCaser.String(word)
|
||||
}
|
||||
}
|
||||
return strings.Join(words, " ")
|
||||
}
|
||||
|
||||
// hasPunctuation returns true if string contains any punctuation character
|
||||
func hasPunctuation(s string) bool {
|
||||
for _, r := range s {
|
||||
if unicode.IsPunct(r) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// removePunctuation removes all punctuation characters from a string
|
||||
// Used for search field normalization only
|
||||
// Replaces hyphens with spaces for search field normalization
|
||||
func removePunctuation(s string) string {
|
||||
return strings.Map(func(r rune) rune {
|
||||
var result strings.Builder
|
||||
for _, r := range s {
|
||||
if unicode.IsPunct(r) {
|
||||
return -1
|
||||
// Replace hyphens with spaces, remove other punctuation
|
||||
if r == '-' {
|
||||
result.WriteRune(' ')
|
||||
}
|
||||
} else {
|
||||
result.WriteRune(r)
|
||||
}
|
||||
return r
|
||||
}, s)
|
||||
}
|
||||
|
||||
// Clean up multiple spaces
|
||||
return strings.Join(strings.Fields(result.String()), " ")
|
||||
}
|
||||
|
||||
// candidate represents a potential tag/contributor value with metadata
|
||||
type candidate struct {
|
||||
value string
|
||||
hasPunct bool
|
||||
firstIndex int
|
||||
}
|
||||
|
||||
// NormalizeTags normalizes an array of tags for display:
|
||||
// 1. Trim whitespace
|
||||
// 2. Titlecase (preserves hyphenation)
|
||||
// 3. Case-insensitive deduplication
|
||||
// 3. Case-insensitive deduplication with punctuation preference
|
||||
// 4. Remove empty strings
|
||||
func NormalizeTags(tags []string) []string {
|
||||
seen := make(map[string]struct{})
|
||||
var normalized []string
|
||||
candidates := make(map[string]candidate)
|
||||
|
||||
for _, tag := range tags {
|
||||
for i, tag := range tags {
|
||||
// Trim whitespace
|
||||
tag = strings.TrimSpace(tag)
|
||||
|
||||
@@ -47,19 +94,49 @@ func NormalizeTags(tags []string) []string {
|
||||
}
|
||||
|
||||
// Titlecase for display (preserves hyphenation: "Non-Fiction")
|
||||
tag = titlecase(tag)
|
||||
titlecasedTag := titlecase(tag)
|
||||
|
||||
// Case-insensitive deduplication
|
||||
key := strings.ToLower(tag)
|
||||
if _, exists := seen[key]; !exists {
|
||||
seen[key] = struct{}{}
|
||||
normalized = append(normalized, tag)
|
||||
// Case-insensitive deduplication with punctuation preference
|
||||
// Remove punctuation and spaces for comparison to handle "nonfiction" == "non-fiction"
|
||||
key := strings.ReplaceAll(removePunctuation(strings.ToLower(titlecasedTag)), " ", "")
|
||||
if existing, exists := candidates[key]; !exists {
|
||||
// First occurrence - store it
|
||||
candidates[key] = candidate{value: titlecasedTag, hasPunct: hasPunctuation(titlecasedTag), firstIndex: i}
|
||||
} else {
|
||||
// Duplicate - upgrade if new version has punctuation and existing doesn't
|
||||
if hasPunctuation(titlecasedTag) && !existing.hasPunct {
|
||||
candidates[key] = candidate{value: titlecasedTag, hasPunct: true, firstIndex: existing.firstIndex}
|
||||
}
|
||||
// Otherwise keep first occurrence
|
||||
}
|
||||
}
|
||||
|
||||
// Extract values in order of first appearance
|
||||
result := make([]pair, 0, len(candidates))
|
||||
for _, c := range candidates {
|
||||
result = append(result, pair{value: c.value, index: c.firstIndex})
|
||||
}
|
||||
|
||||
// Sort by first appearance
|
||||
sort.Slice(result, func(i, j int) bool {
|
||||
return result[i].index < result[j].index
|
||||
})
|
||||
|
||||
// Extract sorted values
|
||||
normalized := make([]string, len(result))
|
||||
for i, p := range result {
|
||||
normalized[i] = p.value
|
||||
}
|
||||
|
||||
return normalized
|
||||
}
|
||||
|
||||
// pair is a helper for sorting values by their original index
|
||||
type pair struct {
|
||||
value string
|
||||
index int
|
||||
}
|
||||
|
||||
// NormalizeTagsSearch normalizes an array of tags for searching:
|
||||
// 1. Trim whitespace
|
||||
// 2. Remove punctuation
|
||||
@@ -68,7 +145,7 @@ func NormalizeTags(tags []string) []string {
|
||||
// 5. Remove empty strings
|
||||
func NormalizeTagsSearch(tags []string) []string {
|
||||
seen := make(map[string]struct{})
|
||||
var normalized []string
|
||||
normalized := make([]string, 0)
|
||||
|
||||
for _, tag := range tags {
|
||||
// Trim whitespace
|
||||
@@ -104,13 +181,12 @@ func NormalizeTagsSearch(tags []string) []string {
|
||||
// 1. Trim whitespace
|
||||
// 2. Preserve original casing (including CAPSLOCK companies)
|
||||
// 3. Preserve original punctuation for display
|
||||
// 4. Case-insensitive deduplication (removes punctuation for comparison only)
|
||||
// 4. Case-insensitive deduplication with punctuation preference (removes punctuation for comparison only)
|
||||
// 5. Remove empty strings
|
||||
func NormalizeContributors(contributors []string) []string {
|
||||
seen := make(map[string]struct{})
|
||||
var normalized []string
|
||||
candidates := make(map[string]candidate)
|
||||
|
||||
for _, contributor := range contributors {
|
||||
for i, contributor := range contributors {
|
||||
// Trim whitespace
|
||||
contributor = strings.TrimSpace(contributor)
|
||||
|
||||
@@ -122,13 +198,36 @@ func NormalizeContributors(contributors []string) []string {
|
||||
// Case-insensitive deduplication (remove punctuation for dedup check only)
|
||||
dedupKey := removePunctuation(strings.ToLower(contributor))
|
||||
|
||||
// Keep original case and punctuation for display
|
||||
if _, exists := seen[dedupKey]; !exists {
|
||||
_seen[dedupKey] = struct{}{}
|
||||
normalized = append(normalized, contributor)
|
||||
// Keep original case and punctuation for display with punctuation preference
|
||||
if existing, exists := candidates[dedupKey]; !exists {
|
||||
// First occurrence - store it
|
||||
candidates[dedupKey] = candidate{value: contributor, hasPunct: hasPunctuation(contributor), firstIndex: i}
|
||||
} else {
|
||||
// Duplicate - upgrade if new version has punctuation and existing doesn't
|
||||
if hasPunctuation(contributor) && !existing.hasPunct {
|
||||
candidates[dedupKey] = candidate{value: contributor, hasPunct: true, firstIndex: existing.firstIndex}
|
||||
}
|
||||
// Otherwise keep first occurrence
|
||||
}
|
||||
}
|
||||
|
||||
// Extract values in order of first appearance
|
||||
result := make([]pair, 0, len(candidates))
|
||||
for _, c := range candidates {
|
||||
result = append(result, pair{value: c.value, index: c.firstIndex})
|
||||
}
|
||||
|
||||
// Sort by first appearance
|
||||
sort.Slice(result, func(i, j int) bool {
|
||||
return result[i].index < result[j].index
|
||||
})
|
||||
|
||||
// Extract sorted values
|
||||
normalized := make([]string, len(result))
|
||||
for i, p := range result {
|
||||
normalized[i] = p.value
|
||||
}
|
||||
|
||||
return normalized
|
||||
}
|
||||
|
||||
@@ -140,7 +239,7 @@ func NormalizeContributors(contributors []string) []string {
|
||||
// 5. Remove empty strings
|
||||
func NormalizeContributorsSearch(contributors []string) []string {
|
||||
seen := make(map[string]struct{})
|
||||
var normalized []string
|
||||
normalized := make([]string, 0)
|
||||
|
||||
for _, contributor := range contributors {
|
||||
// Trim whitespace
|
||||
|
||||
Reference in New Issue
Block a user