package textproc
|
|
import (
|
"regexp"
|
"strings"
|
"unicode"
|
)
|
|
// tagRE matches SenseVoice special tokens like <|zh|>, <|NEUTRAL|>, <|Speech|>.
|
var tagRE = regexp.MustCompile(`<\|[^|]*\|>`)
|
|
// English fillers: whole-word match, case-insensitive
|
var enFillerRE = regexp.MustCompile(`(?i)\b(?:um|uh|hmm|hm|eh|ah|er|erm|uhm)\b`)
|
|
// Chinese multi-char fillers: always safe to remove
|
var cnFillerMultiRE = regexp.MustCompile(`(?:嗯嗯|啊啊|呃呃|哦哦|嗯啊|嗯呢)`)
|
|
// Chinese single-char fillers at start of text (followed by optional punctuation)
|
var cnFillerStartRE = regexp.MustCompile(`^[嗯呃额哦唔][，,、\s]*`)
|
|
// Chinese single-char fillers between punctuation: ，嗯，→ ，
|
var cnFillerMidRE = regexp.MustCompile(`([，。！？、,!?\s])[嗯呃额哦唔]([，。！？、,!?\s])`)
|
|
// isCJK returns true for CJK ideographs (Chinese, Japanese kanji, Korean hanja)
|
// but NOT CJK punctuation, so we don't insert spaces around punctuation.
|
func isCJK(r rune) bool {
|
return (r >= 0x4E00 && r <= 0x9FFF) || // CJK Unified Ideographs
|
(r >= 0x3400 && r <= 0x4DBF) || // CJK Extension A
|
(r >= 0x3040 && r <= 0x309F) || // Hiragana
|
(r >= 0x30A0 && r <= 0x30FF) || // Katakana
|
(r >= 0xF900 && r <= 0xFAFF) || // CJK Compatibility Ideographs
|
(r >= 0x20000 && r <= 0x2A6DF) // CJK Extension B
|
}
|
|
func isASCIIAlNum(r rune) bool {
|
return (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9')
|
}
|
|
// removeFillers strips common speech filler words (语气词) from text.
|
func removeFillers(text string) string {
|
// English fillers
|
text = enFillerRE.ReplaceAllString(text, "")
|
|
// Chinese multi-char fillers (always safe)
|
text = cnFillerMultiRE.ReplaceAllString(text, "")
|
|
// Chinese single-char at start
|
text = cnFillerStartRE.ReplaceAllString(text, "")
|
|
// Chinese single-char between punctuation (may need multiple passes)
|
for i := 0; i < 3; i++ {
|
newText := cnFillerMidRE.ReplaceAllString(text, "$1")
|
if newText == text {
|
break
|
}
|
text = newText
|
}
|
|
return strings.TrimSpace(text)
|
}
|
|
func removeSpacesBetweenCJK(text string) string {
|
runes := []rune(text)
|
buf := make([]rune, 0, len(runes))
|
|
for i := 0; i < len(runes); i++ {
|
r := runes[i]
|
if unicode.IsSpace(r) {
|
j := i
|
for j < len(runes) && unicode.IsSpace(runes[j]) {
|
j++
|
}
|
if len(buf) > 0 && j < len(runes) && isCJK(buf[len(buf)-1]) && isCJK(runes[j]) {
|
i = j - 1
|
continue
|
}
|
}
|
buf = append(buf, r)
|
}
|
|
return string(buf)
|
}
|
|
// PostProcess cleans up mixed Chinese-English text from SenseVoice:
|
// - Strips special tokens (<|zh|>, <|NEUTRAL|>, etc.)
|
// - Removes model-added spaces between CJK characters
|
// - Inserts spaces between CJK characters and ASCII letters/digits
|
// - Capitalizes the first letter and letters after sentence-ending punctuation
|
// - Collapses multiple spaces
|
func PostProcess(text string) string {
|
// Strip SenseVoice special tokens
|
text = tagRE.ReplaceAllString(text, "")
|
text = strings.TrimSpace(text)
|
if text == "" {
|
return text
|
}
|
|
// Remove filler words
|
text = removeFillers(text)
|
if text == "" {
|
return text
|
}
|
|
text = removeSpacesBetweenCJK(text)
|
|
// Insert spaces at CJK ↔ ASCII alphanumeric boundaries
|
runes := []rune(text)
|
buf := make([]rune, 0, len(runes)+16)
|
for i, r := range runes {
|
if i > 0 {
|
prev := runes[i-1]
|
if (isCJK(prev) && isASCIIAlNum(r)) || (isASCIIAlNum(prev) && isCJK(r)) {
|
buf = append(buf, ' ')
|
}
|
}
|
buf = append(buf, r)
|
}
|
|
// Capitalize first letter and after sentence-ending punctuation
|
capNext := true
|
for i, r := range buf {
|
if capNext && unicode.IsLetter(r) {
|
buf[i] = unicode.ToUpper(r)
|
capNext = false
|
}
|
if r == '。' || r == '！' || r == '？' || r == '.' || r == '!' || r == '?' {
|
capNext = true
|
}
|
}
|
|
// Collapse multiple consecutive spaces
|
result := string(buf)
|
for strings.Contains(result, " ") {
|
result = strings.ReplaceAll(result, " ", " ")
|
}
|
|
return strings.TrimSpace(result)
|
}
|
