package correctioncsv
|
|
import (
|
"bytes"
|
"encoding/csv"
|
"fmt"
|
"io"
|
"sort"
|
"strconv"
|
"strings"
|
"time"
|
"unicode/utf8"
|
"voicesnap/internal/history"
|
"voicesnap/internal/userdict"
|
)
|
|
const (
|
DefaultMaxNewRules = 500
|
DefaultMaxLength = 120
|
DefaultDetailLimit = 20
|
|
StatusNew = "new"
|
StatusCSVDuplicate = "csv_duplicate"
|
StatusDictDuplicate = "dict_duplicate"
|
StatusConflict = "conflict"
|
StatusInvalid = "invalid"
|
StatusEmpty = "empty"
|
StatusLimit = "limit"
|
)
|
|
var header = []string{
|
"记录ID",
|
"识别时间",
|
"识别历史文本",
|
"错误词1",
|
"正确词1",
|
"错误词2",
|
"正确词2",
|
"错误词3",
|
"正确词3",
|
"错误词4",
|
"正确词4",
|
"错误词5",
|
"正确词5",
|
}
|
|
// Options controls CSV import classification limits.
|
type Options struct {
|
MaxNewRules int
|
MaxLength int
|
DetailLimit int
|
}
|
|
// Rule is a replacement candidate that can be appended to the user dictionary.
|
type Rule struct {
|
From string `json:"from"`
|
To string `json:"to"`
|
}
|
|
// Summary is the complete classification count for all scanned CSV slots.
|
type Summary struct {
|
New int `json:"new"`
|
CSVDuplicate int `json:"csvDuplicate"`
|
DictDuplicate int `json:"dictDuplicate"`
|
Conflict int `json:"conflict"`
|
Invalid int `json:"invalid"`
|
Empty int `json:"empty"`
|
Limit int `json:"limit"`
|
}
|
|
// PreviewItem is one visible classification row in the import preview.
|
type PreviewItem struct {
|
Row int `json:"row"`
|
Slot int `json:"slot"`
|
From string `json:"from"`
|
To string `json:"to"`
|
Status string `json:"status"`
|
}
|
|
// Preview contains a complete import summary and the first visible details.
|
type Preview struct {
|
Summary Summary `json:"summary"`
|
Items []PreviewItem `json:"items"`
|
TotalSlots int `json:"totalSlots"`
|
VisibleLimit int `json:"visibleLimit"`
|
DetailsTruncated bool `json:"detailsTruncated"`
|
NewRules []Rule `json:"-"`
|
}
|
|
// BuildHistoryCSV creates the Phase 1 correction CSV for current retained history entries.
|
func BuildHistoryCSV(entries []history.Entry, loc *time.Location) ([]byte, error) {
|
if loc == nil {
|
loc = time.Local
|
}
|
|
var buf bytes.Buffer
|
buf.Write([]byte{0xEF, 0xBB, 0xBF})
|
|
writer := csv.NewWriter(&buf)
|
if err := writer.Write(header); err != nil {
|
return nil, err
|
}
|
if err := writer.Write(exampleRow()); err != nil {
|
return nil, err
|
}
|
|
for _, entry := range entries {
|
row := []string{
|
strconv.FormatInt(entry.Timestamp, 10),
|
time.UnixMilli(entry.Timestamp).In(loc).Format("2006-01-02 15:04:05"),
|
entry.Text,
|
"", "", "", "", "", "", "", "", "", "",
|
}
|
if err := writer.Write(row); err != nil {
|
return nil, err
|
}
|
}
|
|
writer.Flush()
|
if err := writer.Error(); err != nil {
|
return nil, err
|
}
|
return buf.Bytes(), nil
|
}
|
|
// ParseAndPreviewCSV parses a correction CSV and classifies all correction slots.
|
func ParseAndPreviewCSV(data []byte, existing []userdict.Entry, opts Options) (Preview, error) {
|
opts = normalizeOptions(opts)
|
data = trimUTF8BOM(data)
|
if !utf8.Valid(data) {
|
return Preview{}, fmt.Errorf("file is not valid UTF-8")
|
}
|
|
reader := csv.NewReader(bytes.NewReader(data))
|
reader.FieldsPerRecord = -1
|
records, err := reader.ReadAll()
|
if err != nil {
|
if err == io.EOF {
|
return Preview{}, fmt.Errorf("CSV is empty")
|
}
|
return Preview{}, err
|
}
|
if len(records) == 0 {
|
return Preview{}, fmt.Errorf("CSV is empty")
|
}
|
|
columns := mapHeaders(records[0])
|
if err := validateColumns(columns); err != nil {
|
return Preview{}, err
|
}
|
|
existingPairs := map[string]struct{}{}
|
existingFrom := map[string]string{}
|
for _, entry := range existing {
|
from := normalizeCell(entry.From)
|
to := normalizeCell(entry.To)
|
if from == "" {
|
continue
|
}
|
existingPairs[pairKey(from, to)] = struct{}{}
|
if _, ok := existingFrom[from]; !ok {
|
existingFrom[from] = to
|
}
|
}
|
|
seenPairs := map[string]struct{}{}
|
seenFrom := map[string]string{}
|
preview := Preview{
|
VisibleLimit: opts.DetailLimit,
|
}
|
allItems := make([]PreviewItem, 0)
|
|
for recordIndex, row := range records[1:] {
|
rowNumber := recordIndex + 2
|
if isExampleRow(row, columns) {
|
continue
|
}
|
|
for slot := 1; slot <= 5; slot++ {
|
preview.TotalSlots++
|
from := normalizeCell(get(row, columns, fmt.Sprintf("error_%d", slot)))
|
to := normalizeCell(get(row, columns, fmt.Sprintf("corrected_%d", slot)))
|
status := classify(from, to, opts, seenPairs, seenFrom, existingPairs, existingFrom, len(preview.NewRules))
|
|
if from != "" && to != "" && from != to && len([]rune(from)) <= opts.MaxLength && len([]rune(to)) <= opts.MaxLength {
|
key := pairKey(from, to)
|
if _, duplicate := seenPairs[key]; !duplicate {
|
seenPairs[key] = struct{}{}
|
}
|
if _, exists := seenFrom[from]; !exists {
|
seenFrom[from] = to
|
}
|
}
|
|
preview.addSummary(status)
|
allItems = append(allItems, PreviewItem{
|
Row: rowNumber,
|
Slot: slot,
|
From: from,
|
To: to,
|
Status: status,
|
})
|
|
if status == StatusNew {
|
preview.NewRules = append(preview.NewRules, Rule{From: from, To: to})
|
}
|
}
|
}
|
|
preview.Items, preview.DetailsTruncated = prioritizePreviewItems(allItems, opts.DetailLimit)
|
return preview, nil
|
}
|
|
func exampleRow() []string {
|
return []string{
|
"示例",
|
"示例，可删除或保留",
|
"正确的格式是点 CSV",
|
"点 CSV",
|
".csv",
|
"", "", "", "", "", "", "", "",
|
}
|
}
|
|
func normalizeOptions(opts Options) Options {
|
if opts.MaxNewRules <= 0 {
|
opts.MaxNewRules = DefaultMaxNewRules
|
}
|
if opts.MaxLength <= 0 {
|
opts.MaxLength = DefaultMaxLength
|
}
|
if opts.DetailLimit <= 0 {
|
opts.DetailLimit = DefaultDetailLimit
|
}
|
return opts
|
}
|
|
func trimUTF8BOM(data []byte) []byte {
|
return bytes.TrimPrefix(data, []byte{0xEF, 0xBB, 0xBF})
|
}
|
|
func mapHeaders(row []string) map[string]int {
|
columns := map[string]int{}
|
for i, name := range row {
|
canonical := canonicalHeaderName(name)
|
if canonical == "" {
|
continue
|
}
|
if _, exists := columns[canonical]; !exists {
|
columns[canonical] = i
|
}
|
}
|
return columns
|
}
|
|
func canonicalHeaderName(name string) string {
|
normalized := strings.ToLower(strings.ReplaceAll(strings.TrimSpace(name), " ", ""))
|
switch normalized {
|
case "timestamp", "记录id", "记录编号", "时间戳":
|
return "timestamp"
|
case "datetime_local", "datetimelocal", "识别时间", "本地时间", "记录时间":
|
return "datetime_local"
|
case "text", "识别历史文本", "历史文本", "识别文本", "原文":
|
return "text"
|
}
|
for slot := 1; slot <= 5; slot++ {
|
if normalized == fmt.Sprintf("error_%d", slot) ||
|
normalized == fmt.Sprintf("error%d", slot) ||
|
normalized == fmt.Sprintf("错误词%d", slot) ||
|
normalized == fmt.Sprintf("错误词_%d", slot) {
|
return fmt.Sprintf("error_%d", slot)
|
}
|
if normalized == fmt.Sprintf("corrected_%d", slot) ||
|
normalized == fmt.Sprintf("corrected%d", slot) ||
|
normalized == fmt.Sprintf("正确词%d", slot) ||
|
normalized == fmt.Sprintf("正确词_%d", slot) ||
|
normalized == fmt.Sprintf("改正词%d", slot) ||
|
normalized == fmt.Sprintf("改正词_%d", slot) {
|
return fmt.Sprintf("corrected_%d", slot)
|
}
|
}
|
return ""
|
}
|
|
func validateColumns(columns map[string]int) error {
|
required := []string{"timestamp", "datetime_local", "text"}
|
for slot := 1; slot <= 5; slot++ {
|
required = append(required, fmt.Sprintf("error_%d", slot), fmt.Sprintf("corrected_%d", slot))
|
}
|
for _, name := range required {
|
if _, ok := columns[name]; !ok {
|
return fmt.Errorf("missing required CSV column: %s", name)
|
}
|
}
|
return nil
|
}
|
|
func isExampleRow(row []string, columns map[string]int) bool {
|
marker := strings.TrimSpace(get(row, columns, "timestamp"))
|
return marker == "__example__" || marker == "示例" || strings.EqualFold(marker, "example")
|
}
|
|
func get(row []string, columns map[string]int, name string) string {
|
i, ok := columns[name]
|
if !ok || i >= len(row) {
|
return ""
|
}
|
return row[i]
|
}
|
|
func normalizeCell(value string) string {
|
value = strings.ReplaceAll(value, "\r\n", " ")
|
value = strings.ReplaceAll(value, "\r", " ")
|
value = strings.ReplaceAll(value, "\n", " ")
|
return strings.TrimSpace(value)
|
}
|
|
func classify(
|
from string,
|
to string,
|
opts Options,
|
seenPairs map[string]struct{},
|
seenFrom map[string]string,
|
existingPairs map[string]struct{},
|
existingFrom map[string]string,
|
newCount int,
|
) string {
|
if from == "" || to == "" {
|
return StatusEmpty
|
}
|
if from == to {
|
return StatusInvalid
|
}
|
if len([]rune(from)) > opts.MaxLength || len([]rune(to)) > opts.MaxLength {
|
return StatusInvalid
|
}
|
|
if _, ok := seenPairs[pairKey(from, to)]; ok {
|
return StatusCSVDuplicate
|
}
|
if existingTo, ok := seenFrom[from]; ok && existingTo != to {
|
return StatusConflict
|
}
|
|
if _, ok := existingPairs[pairKey(from, to)]; ok {
|
return StatusDictDuplicate
|
}
|
if existingTo, ok := existingFrom[from]; ok && existingTo != to {
|
return StatusConflict
|
}
|
|
if newCount >= opts.MaxNewRules {
|
return StatusLimit
|
}
|
return StatusNew
|
}
|
|
func (p *Preview) addSummary(status string) {
|
switch status {
|
case StatusNew:
|
p.Summary.New++
|
case StatusCSVDuplicate:
|
p.Summary.CSVDuplicate++
|
case StatusDictDuplicate:
|
p.Summary.DictDuplicate++
|
case StatusConflict:
|
p.Summary.Conflict++
|
case StatusInvalid:
|
p.Summary.Invalid++
|
case StatusEmpty:
|
p.Summary.Empty++
|
case StatusLimit:
|
p.Summary.Limit++
|
}
|
}
|
|
func prioritizePreviewItems(items []PreviewItem, detailLimit int) ([]PreviewItem, bool) {
|
if detailLimit <= 0 {
|
return nil, len(items) > 0
|
}
|
|
sorted := make([]PreviewItem, len(items))
|
copy(sorted, items)
|
sort.SliceStable(sorted, func(i, j int) bool {
|
return previewStatusRank(sorted[i].Status) < previewStatusRank(sorted[j].Status)
|
})
|
|
truncated := len(sorted) > detailLimit
|
if truncated {
|
sorted = sorted[:detailLimit]
|
}
|
return sorted, truncated
|
}
|
|
func previewStatusRank(status string) int {
|
switch status {
|
case StatusNew:
|
return 0
|
case StatusConflict:
|
return 1
|
case StatusInvalid, StatusLimit, StatusDictDuplicate, StatusCSVDuplicate:
|
return 2
|
case StatusEmpty:
|
return 3
|
default:
|
return 4
|
}
|
}
|
|
func pairKey(from, to string) string {
|
return from + "\x00" + to
|
}
|
