package sensitive import ( "strings" "unicode" "unicode/utf8" ) const maxScanRunes = 8192 // Index maps normalized rune position to original UTF-8 byte offsets [from, to). type Index struct { Norm []rune OrigFrom []int OrigTo []int OrigBytes []byte } // FoldWord normalizes a dictionary entry (same rules as FoldAndIndex on short input). func FoldWord(s string) string { idx := FoldAndIndex(s) if len(idx.Norm) == 0 { return "" } return string(idx.Norm) } // FoldAndIndex builds the normalized rune stream and byte mapping into the original text. func FoldAndIndex(text string) Index { raw := []byte(text) var norm []rune var from, to []int runes := 0 for i := 0; i < len(raw); { r, w := decodeRune(raw, i) if w == 0 { break } start := i end := i + w i = end if dropRune(r) { continue } r = foldRune(r) if r == 0 { continue } runes++ if runes > maxScanRunes { break } norm = append(norm, r) from = append(from, start) to = append(to, end) } return Index{Norm: norm, OrigFrom: from, OrigTo: to, OrigBytes: raw} } func dropRune(r rune) bool { if r <= unicode.MaxASCII { if unicode.IsSpace(r) || isASCIIPunct(r) { return true } } if unicode.In(r, unicode.Cf, unicode.Zs, unicode.Zl, unicode.Zp) { return true } if isCJKPunct(r) { return true } return false } func isASCIIPunct(r rune) bool { return strings.ContainsRune(`!"#$%&'()*+,-./:;<=>?@[\]^_`+"`"+`{|}~`, r) } func isCJKPunct(r rune) bool { return strings.ContainsRune(",。!?、;:…·「」『』【】()", r) } func foldRune(r rune) rune { if r >= 0xFF01 && r <= 0xFF5E { return r - 0xFEE0 } return unicode.ToLower(r) } func decodeRune(b []byte, i int) (rune, int) { if i >= len(b) { return 0, 0 } r, w := utf8.DecodeRune(b[i:]) if w == 0 { return utf8.RuneError, 1 } return r, w }