package correctioncsv import ( "bytes" "encoding/csv" "fmt" "io" "sort" "strconv" "strings" "time" "unicode/utf8" "voicesnap/internal/history" "voicesnap/internal/userdict" ) const ( DefaultMaxNewRules = 500 DefaultMaxLength = 120 DefaultDetailLimit = 20 StatusNew = "new" StatusCSVDuplicate = "csv_duplicate" StatusDictDuplicate = "dict_duplicate" StatusConflict = "conflict" StatusInvalid = "invalid" StatusEmpty = "empty" StatusLimit = "limit" ) var header = []string{ "记录ID", "识别时间", "识别历史文本", "错误词1", "正确词1", "错误词2", "正确词2", "错误词3", "正确词3", "错误词4", "正确词4", "错误词5", "正确词5", } // Options controls CSV import classification limits. type Options struct { MaxNewRules int MaxLength int DetailLimit int } // Rule is a replacement candidate that can be appended to the user dictionary. type Rule struct { From string `json:"from"` To string `json:"to"` } // Summary is the complete classification count for all scanned CSV slots. type Summary struct { New int `json:"new"` CSVDuplicate int `json:"csvDuplicate"` DictDuplicate int `json:"dictDuplicate"` Conflict int `json:"conflict"` Invalid int `json:"invalid"` Empty int `json:"empty"` Limit int `json:"limit"` } // PreviewItem is one visible classification row in the import preview. type PreviewItem struct { Row int `json:"row"` Slot int `json:"slot"` From string `json:"from"` To string `json:"to"` Status string `json:"status"` } // Preview contains a complete import summary and the first visible details. type Preview struct { Summary Summary `json:"summary"` Items []PreviewItem `json:"items"` TotalSlots int `json:"totalSlots"` VisibleLimit int `json:"visibleLimit"` DetailsTruncated bool `json:"detailsTruncated"` NewRules []Rule `json:"-"` } // BuildHistoryCSV creates the Phase 1 correction CSV for current retained history entries. func BuildHistoryCSV(entries []history.Entry, loc *time.Location) ([]byte, error) { if loc == nil { loc = time.Local } var buf bytes.Buffer buf.Write([]byte{0xEF, 0xBB, 0xBF}) writer := csv.NewWriter(&buf) if err := writer.Write(header); err != nil { return nil, err } if err := writer.Write(exampleRow()); err != nil { return nil, err } for _, entry := range entries { row := []string{ strconv.FormatInt(entry.Timestamp, 10), time.UnixMilli(entry.Timestamp).In(loc).Format("2006-01-02 15:04:05"), entry.Text, "", "", "", "", "", "", "", "", "", "", } if err := writer.Write(row); err != nil { return nil, err } } writer.Flush() if err := writer.Error(); err != nil { return nil, err } return buf.Bytes(), nil } // ParseAndPreviewCSV parses a correction CSV and classifies all correction slots. func ParseAndPreviewCSV(data []byte, existing []userdict.Entry, opts Options) (Preview, error) { opts = normalizeOptions(opts) data = trimUTF8BOM(data) if !utf8.Valid(data) { return Preview{}, fmt.Errorf("file is not valid UTF-8") } reader := csv.NewReader(bytes.NewReader(data)) reader.FieldsPerRecord = -1 records, err := reader.ReadAll() if err != nil { if err == io.EOF { return Preview{}, fmt.Errorf("CSV is empty") } return Preview{}, err } if len(records) == 0 { return Preview{}, fmt.Errorf("CSV is empty") } columns := mapHeaders(records[0]) if err := validateColumns(columns); err != nil { return Preview{}, err } existingPairs := map[string]struct{}{} existingFrom := map[string]string{} for _, entry := range existing { from := normalizeCell(entry.From) to := normalizeCell(entry.To) if from == "" { continue } existingPairs[pairKey(from, to)] = struct{}{} if _, ok := existingFrom[from]; !ok { existingFrom[from] = to } } seenPairs := map[string]struct{}{} seenFrom := map[string]string{} preview := Preview{ VisibleLimit: opts.DetailLimit, } allItems := make([]PreviewItem, 0) for recordIndex, row := range records[1:] { rowNumber := recordIndex + 2 if isExampleRow(row, columns) { continue } for slot := 1; slot <= 5; slot++ { preview.TotalSlots++ from := normalizeCell(get(row, columns, fmt.Sprintf("error_%d", slot))) to := normalizeCell(get(row, columns, fmt.Sprintf("corrected_%d", slot))) status := classify(from, to, opts, seenPairs, seenFrom, existingPairs, existingFrom, len(preview.NewRules)) if from != "" && to != "" && from != to && len([]rune(from)) <= opts.MaxLength && len([]rune(to)) <= opts.MaxLength { key := pairKey(from, to) if _, duplicate := seenPairs[key]; !duplicate { seenPairs[key] = struct{}{} } if _, exists := seenFrom[from]; !exists { seenFrom[from] = to } } preview.addSummary(status) allItems = append(allItems, PreviewItem{ Row: rowNumber, Slot: slot, From: from, To: to, Status: status, }) if status == StatusNew { preview.NewRules = append(preview.NewRules, Rule{From: from, To: to}) } } } preview.Items, preview.DetailsTruncated = prioritizePreviewItems(allItems, opts.DetailLimit) return preview, nil } func exampleRow() []string { return []string{ "示例", "示例,可删除或保留", "正确的格式是点 CSV", "点 CSV", ".csv", "", "", "", "", "", "", "", "", } } func normalizeOptions(opts Options) Options { if opts.MaxNewRules <= 0 { opts.MaxNewRules = DefaultMaxNewRules } if opts.MaxLength <= 0 { opts.MaxLength = DefaultMaxLength } if opts.DetailLimit <= 0 { opts.DetailLimit = DefaultDetailLimit } return opts } func trimUTF8BOM(data []byte) []byte { return bytes.TrimPrefix(data, []byte{0xEF, 0xBB, 0xBF}) } func mapHeaders(row []string) map[string]int { columns := map[string]int{} for i, name := range row { canonical := canonicalHeaderName(name) if canonical == "" { continue } if _, exists := columns[canonical]; !exists { columns[canonical] = i } } return columns } func canonicalHeaderName(name string) string { normalized := strings.ToLower(strings.ReplaceAll(strings.TrimSpace(name), " ", "")) switch normalized { case "timestamp", "记录id", "记录编号", "时间戳": return "timestamp" case "datetime_local", "datetimelocal", "识别时间", "本地时间", "记录时间": return "datetime_local" case "text", "识别历史文本", "历史文本", "识别文本", "原文": return "text" } for slot := 1; slot <= 5; slot++ { if normalized == fmt.Sprintf("error_%d", slot) || normalized == fmt.Sprintf("error%d", slot) || normalized == fmt.Sprintf("错误词%d", slot) || normalized == fmt.Sprintf("错误词_%d", slot) { return fmt.Sprintf("error_%d", slot) } if normalized == fmt.Sprintf("corrected_%d", slot) || normalized == fmt.Sprintf("corrected%d", slot) || normalized == fmt.Sprintf("正确词%d", slot) || normalized == fmt.Sprintf("正确词_%d", slot) || normalized == fmt.Sprintf("改正词%d", slot) || normalized == fmt.Sprintf("改正词_%d", slot) { return fmt.Sprintf("corrected_%d", slot) } } return "" } func validateColumns(columns map[string]int) error { required := []string{"timestamp", "datetime_local", "text"} for slot := 1; slot <= 5; slot++ { required = append(required, fmt.Sprintf("error_%d", slot), fmt.Sprintf("corrected_%d", slot)) } for _, name := range required { if _, ok := columns[name]; !ok { return fmt.Errorf("missing required CSV column: %s", name) } } return nil } func isExampleRow(row []string, columns map[string]int) bool { marker := strings.TrimSpace(get(row, columns, "timestamp")) return marker == "__example__" || marker == "示例" || strings.EqualFold(marker, "example") } func get(row []string, columns map[string]int, name string) string { i, ok := columns[name] if !ok || i >= len(row) { return "" } return row[i] } func normalizeCell(value string) string { value = strings.ReplaceAll(value, "\r\n", " ") value = strings.ReplaceAll(value, "\r", " ") value = strings.ReplaceAll(value, "\n", " ") return strings.TrimSpace(value) } func classify( from string, to string, opts Options, seenPairs map[string]struct{}, seenFrom map[string]string, existingPairs map[string]struct{}, existingFrom map[string]string, newCount int, ) string { if from == "" || to == "" { return StatusEmpty } if from == to { return StatusInvalid } if len([]rune(from)) > opts.MaxLength || len([]rune(to)) > opts.MaxLength { return StatusInvalid } if _, ok := seenPairs[pairKey(from, to)]; ok { return StatusCSVDuplicate } if existingTo, ok := seenFrom[from]; ok && existingTo != to { return StatusConflict } if _, ok := existingPairs[pairKey(from, to)]; ok { return StatusDictDuplicate } if existingTo, ok := existingFrom[from]; ok && existingTo != to { return StatusConflict } if newCount >= opts.MaxNewRules { return StatusLimit } return StatusNew } func (p *Preview) addSummary(status string) { switch status { case StatusNew: p.Summary.New++ case StatusCSVDuplicate: p.Summary.CSVDuplicate++ case StatusDictDuplicate: p.Summary.DictDuplicate++ case StatusConflict: p.Summary.Conflict++ case StatusInvalid: p.Summary.Invalid++ case StatusEmpty: p.Summary.Empty++ case StatusLimit: p.Summary.Limit++ } } func prioritizePreviewItems(items []PreviewItem, detailLimit int) ([]PreviewItem, bool) { if detailLimit <= 0 { return nil, len(items) > 0 } sorted := make([]PreviewItem, len(items)) copy(sorted, items) sort.SliceStable(sorted, func(i, j int) bool { return previewStatusRank(sorted[i].Status) < previewStatusRank(sorted[j].Status) }) truncated := len(sorted) > detailLimit if truncated { sorted = sorted[:detailLimit] } return sorted, truncated } func previewStatusRank(status string) int { switch status { case StatusNew: return 0 case StatusConflict: return 1 case StatusInvalid, StatusLimit, StatusDictDuplicate, StatusCSVDuplicate: return 2 case StatusEmpty: return 3 default: return 4 } } func pairKey(from, to string) string { return from + "\x00" + to }