100 lines
1.8 KiB
Go
100 lines
1.8 KiB
Go
package sensitive
|
|
|
|
import (
|
|
"strings"
|
|
"unicode"
|
|
"unicode/utf8"
|
|
)
|
|
|
|
const maxScanRunes = 8192
|
|
|
|
// Index maps normalized rune position to original UTF-8 byte offsets [from, to).
|
|
type Index struct {
|
|
Norm []rune
|
|
OrigFrom []int
|
|
OrigTo []int
|
|
OrigBytes []byte
|
|
}
|
|
|
|
// FoldWord normalizes a dictionary entry (same rules as FoldAndIndex on short input).
|
|
func FoldWord(s string) string {
|
|
idx := FoldAndIndex(s)
|
|
if len(idx.Norm) == 0 {
|
|
return ""
|
|
}
|
|
return string(idx.Norm)
|
|
}
|
|
|
|
// FoldAndIndex builds the normalized rune stream and byte mapping into the original text.
|
|
func FoldAndIndex(text string) Index {
|
|
raw := []byte(text)
|
|
var norm []rune
|
|
var from, to []int
|
|
runes := 0
|
|
for i := 0; i < len(raw); {
|
|
r, w := decodeRune(raw, i)
|
|
if w == 0 {
|
|
break
|
|
}
|
|
start := i
|
|
end := i + w
|
|
i = end
|
|
if dropRune(r) {
|
|
continue
|
|
}
|
|
r = foldRune(r)
|
|
if r == 0 {
|
|
continue
|
|
}
|
|
runes++
|
|
if runes > maxScanRunes {
|
|
break
|
|
}
|
|
norm = append(norm, r)
|
|
from = append(from, start)
|
|
to = append(to, end)
|
|
}
|
|
return Index{Norm: norm, OrigFrom: from, OrigTo: to, OrigBytes: raw}
|
|
}
|
|
|
|
func dropRune(r rune) bool {
|
|
if r <= unicode.MaxASCII {
|
|
if unicode.IsSpace(r) || isASCIIPunct(r) {
|
|
return true
|
|
}
|
|
}
|
|
if unicode.In(r, unicode.Cf, unicode.Zs, unicode.Zl, unicode.Zp) {
|
|
return true
|
|
}
|
|
if isCJKPunct(r) {
|
|
return true
|
|
}
|
|
return false
|
|
}
|
|
|
|
func isASCIIPunct(r rune) bool {
|
|
return strings.ContainsRune(`!"#$%&'()*+,-./:;<=>?@[\]^_`+"`"+`{|}~`, r)
|
|
}
|
|
|
|
func isCJKPunct(r rune) bool {
|
|
return strings.ContainsRune("，。！？、；：…·「」『』【】（）", r)
|
|
}
|
|
|
|
func foldRune(r rune) rune {
|
|
if r >= 0xFF01 && r <= 0xFF5E {
|
|
return r - 0xFEE0
|
|
}
|
|
return unicode.ToLower(r)
|
|
}
|
|
|
|
func decodeRune(b []byte, i int) (rune, int) {
|
|
if i >= len(b) {
|
|
return 0, 0
|
|
}
|
|
r, w := utf8.DecodeRune(b[i:])
|
|
if w == 0 {
|
|
return utf8.RuneError, 1
|
|
}
|
|
return r, w
|
|
}
|