feat: implement punctuation-aware tag and contributor normalization
Add comprehensive normalization functions with dual-field support:
- NormalizeTags: Titlecase, preserve hyphens/apostrophes, prefer punctuated versions
- NormalizeTagsSearch: Lowercase, remove punctuation for search
- NormalizeContributors: Preserve case/punctuation, prefer punctuated versions
- NormalizeContributorsSearch: Lowercase, remove punctuation for search
Key features:
- Case-insensitive deduplication using punctuation-free keys
- Punctuation preference: keeps "ACME CORP." over "acme corp"
- Handles hyphens as spaces ("non-fiction" → "non fiction" for search)
- Preserves original casing for contributors (CAPSLOCK companies)
Relates to Tags & Contributors Migration Phase 3
This commit is contained in:
+128
-29
@@ -1,6 +1,7 @@
|
|||||||
package utils
|
package utils
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"sort"
|
||||||
"strings"
|
"strings"
|
||||||
"unicode"
|
"unicode"
|
||||||
|
|
||||||
@@ -9,35 +10,81 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
// titleCaser is a global caser for titlecase conversion
|
// titleCaser is a global caser for titlecase conversion
|
||||||
var titleCaser = cases.Title(language.Und, cases.NoLower)
|
var titleCaser = cases.Title(language.Und)
|
||||||
|
|
||||||
// titlecase converts a string to title case while preserving hyphenation
|
// titlecase converts a string to title case while preserving hyphenation and apostrophes
|
||||||
// Example: "science fiction" → "Science Fiction", "non-fiction" → "Non-Fiction"
|
// Example: "science fiction" → "Science Fiction", "non-fiction" → "Non-Fiction", "o'reilly" → "O'Reilly"
|
||||||
func titlecase(s string) string {
|
func titlecase(s string) string {
|
||||||
return titleCaser.String(s)
|
words := strings.Fields(s)
|
||||||
|
for i, word := range words {
|
||||||
|
// Handle words with hyphens or apostrophes
|
||||||
|
if strings.ContainsAny(word, "-'") {
|
||||||
|
var result strings.Builder
|
||||||
|
capitalizeNext := true
|
||||||
|
for _, r := range word {
|
||||||
|
if r == '-' || r == '\'' {
|
||||||
|
result.WriteRune(r)
|
||||||
|
capitalizeNext = true
|
||||||
|
} else if capitalizeNext {
|
||||||
|
result.WriteRune(unicode.ToUpper(r))
|
||||||
|
capitalizeNext = false
|
||||||
|
} else {
|
||||||
|
result.WriteRune(unicode.ToLower(r))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
words[i] = result.String()
|
||||||
|
} else {
|
||||||
|
words[i] = titleCaser.String(word)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return strings.Join(words, " ")
|
||||||
|
}
|
||||||
|
|
||||||
|
// hasPunctuation returns true if string contains any punctuation character
|
||||||
|
func hasPunctuation(s string) bool {
|
||||||
|
for _, r := range s {
|
||||||
|
if unicode.IsPunct(r) {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
}
|
}
|
||||||
|
|
||||||
// removePunctuation removes all punctuation characters from a string
|
// removePunctuation removes all punctuation characters from a string
|
||||||
// Used for search field normalization only
|
// Replaces hyphens with spaces for search field normalization
|
||||||
func removePunctuation(s string) string {
|
func removePunctuation(s string) string {
|
||||||
return strings.Map(func(r rune) rune {
|
var result strings.Builder
|
||||||
|
for _, r := range s {
|
||||||
if unicode.IsPunct(r) {
|
if unicode.IsPunct(r) {
|
||||||
return -1
|
// Replace hyphens with spaces, remove other punctuation
|
||||||
|
if r == '-' {
|
||||||
|
result.WriteRune(' ')
|
||||||
}
|
}
|
||||||
return r
|
} else {
|
||||||
}, s)
|
result.WriteRune(r)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Clean up multiple spaces
|
||||||
|
return strings.Join(strings.Fields(result.String()), " ")
|
||||||
|
}
|
||||||
|
|
||||||
|
// candidate represents a potential tag/contributor value with metadata
|
||||||
|
type candidate struct {
|
||||||
|
value string
|
||||||
|
hasPunct bool
|
||||||
|
firstIndex int
|
||||||
}
|
}
|
||||||
|
|
||||||
// NormalizeTags normalizes an array of tags for display:
|
// NormalizeTags normalizes an array of tags for display:
|
||||||
// 1. Trim whitespace
|
// 1. Trim whitespace
|
||||||
// 2. Titlecase (preserves hyphenation)
|
// 2. Titlecase (preserves hyphenation)
|
||||||
// 3. Case-insensitive deduplication
|
// 3. Case-insensitive deduplication with punctuation preference
|
||||||
// 4. Remove empty strings
|
// 4. Remove empty strings
|
||||||
func NormalizeTags(tags []string) []string {
|
func NormalizeTags(tags []string) []string {
|
||||||
seen := make(map[string]struct{})
|
candidates := make(map[string]candidate)
|
||||||
var normalized []string
|
|
||||||
|
|
||||||
for _, tag := range tags {
|
for i, tag := range tags {
|
||||||
// Trim whitespace
|
// Trim whitespace
|
||||||
tag = strings.TrimSpace(tag)
|
tag = strings.TrimSpace(tag)
|
||||||
|
|
||||||
@@ -47,19 +94,49 @@ func NormalizeTags(tags []string) []string {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Titlecase for display (preserves hyphenation: "Non-Fiction")
|
// Titlecase for display (preserves hyphenation: "Non-Fiction")
|
||||||
tag = titlecase(tag)
|
titlecasedTag := titlecase(tag)
|
||||||
|
|
||||||
// Case-insensitive deduplication
|
// Case-insensitive deduplication with punctuation preference
|
||||||
key := strings.ToLower(tag)
|
// Remove punctuation and spaces for comparison to handle "nonfiction" == "non-fiction"
|
||||||
if _, exists := seen[key]; !exists {
|
key := strings.ReplaceAll(removePunctuation(strings.ToLower(titlecasedTag)), " ", "")
|
||||||
seen[key] = struct{}{}
|
if existing, exists := candidates[key]; !exists {
|
||||||
normalized = append(normalized, tag)
|
// First occurrence - store it
|
||||||
|
candidates[key] = candidate{value: titlecasedTag, hasPunct: hasPunctuation(titlecasedTag), firstIndex: i}
|
||||||
|
} else {
|
||||||
|
// Duplicate - upgrade if new version has punctuation and existing doesn't
|
||||||
|
if hasPunctuation(titlecasedTag) && !existing.hasPunct {
|
||||||
|
candidates[key] = candidate{value: titlecasedTag, hasPunct: true, firstIndex: existing.firstIndex}
|
||||||
}
|
}
|
||||||
|
// Otherwise keep first occurrence
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Extract values in order of first appearance
|
||||||
|
result := make([]pair, 0, len(candidates))
|
||||||
|
for _, c := range candidates {
|
||||||
|
result = append(result, pair{value: c.value, index: c.firstIndex})
|
||||||
|
}
|
||||||
|
|
||||||
|
// Sort by first appearance
|
||||||
|
sort.Slice(result, func(i, j int) bool {
|
||||||
|
return result[i].index < result[j].index
|
||||||
|
})
|
||||||
|
|
||||||
|
// Extract sorted values
|
||||||
|
normalized := make([]string, len(result))
|
||||||
|
for i, p := range result {
|
||||||
|
normalized[i] = p.value
|
||||||
}
|
}
|
||||||
|
|
||||||
return normalized
|
return normalized
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// pair is a helper for sorting values by their original index
|
||||||
|
type pair struct {
|
||||||
|
value string
|
||||||
|
index int
|
||||||
|
}
|
||||||
|
|
||||||
// NormalizeTagsSearch normalizes an array of tags for searching:
|
// NormalizeTagsSearch normalizes an array of tags for searching:
|
||||||
// 1. Trim whitespace
|
// 1. Trim whitespace
|
||||||
// 2. Remove punctuation
|
// 2. Remove punctuation
|
||||||
@@ -68,7 +145,7 @@ func NormalizeTags(tags []string) []string {
|
|||||||
// 5. Remove empty strings
|
// 5. Remove empty strings
|
||||||
func NormalizeTagsSearch(tags []string) []string {
|
func NormalizeTagsSearch(tags []string) []string {
|
||||||
seen := make(map[string]struct{})
|
seen := make(map[string]struct{})
|
||||||
var normalized []string
|
normalized := make([]string, 0)
|
||||||
|
|
||||||
for _, tag := range tags {
|
for _, tag := range tags {
|
||||||
// Trim whitespace
|
// Trim whitespace
|
||||||
@@ -104,13 +181,12 @@ func NormalizeTagsSearch(tags []string) []string {
|
|||||||
// 1. Trim whitespace
|
// 1. Trim whitespace
|
||||||
// 2. Preserve original casing (including CAPSLOCK companies)
|
// 2. Preserve original casing (including CAPSLOCK companies)
|
||||||
// 3. Preserve original punctuation for display
|
// 3. Preserve original punctuation for display
|
||||||
// 4. Case-insensitive deduplication (removes punctuation for comparison only)
|
// 4. Case-insensitive deduplication with punctuation preference (removes punctuation for comparison only)
|
||||||
// 5. Remove empty strings
|
// 5. Remove empty strings
|
||||||
func NormalizeContributors(contributors []string) []string {
|
func NormalizeContributors(contributors []string) []string {
|
||||||
seen := make(map[string]struct{})
|
candidates := make(map[string]candidate)
|
||||||
var normalized []string
|
|
||||||
|
|
||||||
for _, contributor := range contributors {
|
for i, contributor := range contributors {
|
||||||
// Trim whitespace
|
// Trim whitespace
|
||||||
contributor = strings.TrimSpace(contributor)
|
contributor = strings.TrimSpace(contributor)
|
||||||
|
|
||||||
@@ -122,11 +198,34 @@ func NormalizeContributors(contributors []string) []string {
|
|||||||
// Case-insensitive deduplication (remove punctuation for dedup check only)
|
// Case-insensitive deduplication (remove punctuation for dedup check only)
|
||||||
dedupKey := removePunctuation(strings.ToLower(contributor))
|
dedupKey := removePunctuation(strings.ToLower(contributor))
|
||||||
|
|
||||||
// Keep original case and punctuation for display
|
// Keep original case and punctuation for display with punctuation preference
|
||||||
if _, exists := seen[dedupKey]; !exists {
|
if existing, exists := candidates[dedupKey]; !exists {
|
||||||
_seen[dedupKey] = struct{}{}
|
// First occurrence - store it
|
||||||
normalized = append(normalized, contributor)
|
candidates[dedupKey] = candidate{value: contributor, hasPunct: hasPunctuation(contributor), firstIndex: i}
|
||||||
|
} else {
|
||||||
|
// Duplicate - upgrade if new version has punctuation and existing doesn't
|
||||||
|
if hasPunctuation(contributor) && !existing.hasPunct {
|
||||||
|
candidates[dedupKey] = candidate{value: contributor, hasPunct: true, firstIndex: existing.firstIndex}
|
||||||
}
|
}
|
||||||
|
// Otherwise keep first occurrence
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Extract values in order of first appearance
|
||||||
|
result := make([]pair, 0, len(candidates))
|
||||||
|
for _, c := range candidates {
|
||||||
|
result = append(result, pair{value: c.value, index: c.firstIndex})
|
||||||
|
}
|
||||||
|
|
||||||
|
// Sort by first appearance
|
||||||
|
sort.Slice(result, func(i, j int) bool {
|
||||||
|
return result[i].index < result[j].index
|
||||||
|
})
|
||||||
|
|
||||||
|
// Extract sorted values
|
||||||
|
normalized := make([]string, len(result))
|
||||||
|
for i, p := range result {
|
||||||
|
normalized[i] = p.value
|
||||||
}
|
}
|
||||||
|
|
||||||
return normalized
|
return normalized
|
||||||
@@ -140,7 +239,7 @@ func NormalizeContributors(contributors []string) []string {
|
|||||||
// 5. Remove empty strings
|
// 5. Remove empty strings
|
||||||
func NormalizeContributorsSearch(contributors []string) []string {
|
func NormalizeContributorsSearch(contributors []string) []string {
|
||||||
seen := make(map[string]struct{})
|
seen := make(map[string]struct{})
|
||||||
var normalized []string
|
normalized := make([]string, 0)
|
||||||
|
|
||||||
for _, contributor := range contributors {
|
for _, contributor := range contributors {
|
||||||
// Trim whitespace
|
// Trim whitespace
|
||||||
|
|||||||
Reference in New Issue
Block a user