| | |
| | | return strings.TrimSpace(text) |
| | | } |
| | | |
| | | func removeSpacesBetweenCJK(text string) string { |
| | | runes := []rune(text) |
| | | buf := make([]rune, 0, len(runes)) |
| | | |
| | | for i := 0; i < len(runes); i++ { |
| | | r := runes[i] |
| | | if unicode.IsSpace(r) { |
| | | j := i |
| | | for j < len(runes) && unicode.IsSpace(runes[j]) { |
| | | j++ |
| | | } |
| | | if len(buf) > 0 && j < len(runes) && isCJK(buf[len(buf)-1]) && isCJK(runes[j]) { |
| | | i = j - 1 |
| | | continue |
| | | } |
| | | } |
| | | buf = append(buf, r) |
| | | } |
| | | |
| | | return string(buf) |
| | | } |
| | | |
| | | // PostProcess cleans up mixed Chinese-English text from SenseVoice: |
| | | // - Strips special tokens (<|zh|>, <|NEUTRAL|>, etc.) |
| | | // - Removes model-added spaces between CJK characters |
| | | // - Inserts spaces between CJK characters and ASCII letters/digits |
| | | // - Capitalizes the first letter and letters after sentence-ending punctuation |
| | | // - Collapses multiple spaces |
| | |
| | | return text |
| | | } |
| | | |
| | | text = removeSpacesBetweenCJK(text) |
| | | |
| | | // Insert spaces at CJK ↔ ASCII alphanumeric boundaries |
| | | runes := []rune(text) |
| | | buf := make([]rune, 0, len(runes)+16) |