package correctioncsv import ( "bytes" "encoding/csv" "strings" "testing" "time" "voicesnap/internal/history" "voicesnap/internal/userdict" ) func TestBuildHistoryCSVIncludesBOMHeaderExampleAndEscapedRows(t *testing.T) { entries := []history.Entry{ { Timestamp: time.Date(2026, 5, 30, 13, 9, 8, 0, time.UTC).UnixMilli(), Text: "中文,a \"quote\"\nnext", }, } data, err := BuildHistoryCSV(entries, time.UTC) if err != nil { t.Fatal(err) } if !bytes.HasPrefix(data, []byte{0xEF, 0xBB, 0xBF}) { t.Fatal("expected UTF-8 BOM") } reader := csv.NewReader(bytes.NewReader(trimUTF8BOM(data))) records, err := reader.ReadAll() if err != nil { t.Fatal(err) } if len(records) != 3 { t.Fatalf("expected header, example, and one data row, got %d rows", len(records)) } if got := strings.Join(records[0], ","); got != strings.Join(header, ",") { t.Fatalf("unexpected header: %s", got) } if records[1][0] != "示例" { t.Fatalf("expected example row, got marker %q", records[1][0]) } if records[1][2] != "正确的格式是点 CSV" { t.Fatalf("unexpected example text: %q", records[1][2]) } if records[2][2] != entries[0].Text { t.Fatalf("expected text to round-trip, got %q", records[2][2]) } } func TestParseAndPreviewClassifiesRows(t *testing.T) { data := csvBytes(t, []string{"corrected_1", "error_1", "text", "datetime_local", "timestamp", "error_2", "corrected_2", "error_3", "corrected_3", "error_4", "corrected_4", "error_5", "corrected_5", "notes"}, []string{".csv", "点 CSV", "sample", "now", "__example__", "", "", "", "", "", "", "", "", "ignored"}, []string{".csv", " 点 CSV ", "text", "now", "1", "麦克", "Mike", "空", "", "same", "same", strings.Repeat("长", 121), "x", "ignored"}, ) preview, err := ParseAndPreviewCSV( append([]byte{0xEF, 0xBB, 0xBF}, data...), []userdict.Entry{{From: "麦克", To: "Mac"}}, Options{}, ) if err != nil { t.Fatal(err) } if preview.TotalSlots != 5 { t.Fatalf("expected 5 scanned slots, got %d", preview.TotalSlots) } if preview.Summary.New != 1 { t.Fatalf("expected 1 new rule, got %+v", preview.Summary) } if preview.Summary.Conflict != 1 { t.Fatalf("expected 1 conflict, got %+v", preview.Summary) } if preview.Summary.Empty != 1 { t.Fatalf("expected 1 empty slot, got %+v", preview.Summary) } if preview.Summary.Invalid != 2 { t.Fatalf("expected 2 invalid slots, got %+v", preview.Summary) } if len(preview.NewRules) != 1 || preview.NewRules[0].From != "点 CSV" || preview.NewRules[0].To != ".csv" { t.Fatalf("unexpected new rules: %+v", preview.NewRules) } } func TestParseAndPreviewAcceptsChineseHeadersAndSkipsChineseExampleRow(t *testing.T) { data := csvBytes(t, []string{"正确词1", "错误词1", "识别历史文本", "识别时间", "记录ID", "错误词2", "正确词2", "错误词3", "正确词3", "错误词4", "正确词4", "错误词5", "正确词5", "备注"}, []string{".csv", "点 CSV", "正确的格式是点 CSV", "示例,可删除或保留", "示例", "", "", "", "", "", "", "", "", "ignored"}, []string{".pdf", "点 PDF", "text", "2026-05-30 13:00:00", "1", "", "", "", "", "", "", "", "", "ignored"}, ) preview, err := ParseAndPreviewCSV(data, nil, Options{}) if err != nil { t.Fatal(err) } if preview.TotalSlots != 5 { t.Fatalf("expected 5 scanned slots, got %d", preview.TotalSlots) } if preview.Summary.New != 1 { t.Fatalf("expected only the real data row to create 1 new rule, got %+v", preview.Summary) } if len(preview.NewRules) != 1 || preview.NewRules[0].From != "点 PDF" || preview.NewRules[0].To != ".pdf" { t.Fatalf("unexpected new rules: %+v", preview.NewRules) } } func TestParseAndPreviewDetectsCSVDuplicateConflictLimitAndTruncatesDetails(t *testing.T) { data := csvBytes(t, header, []string{"1", "now", "text", "A", "B", "A", "B", "A", "C", "D", "E", "F", "G"}, ) preview, err := ParseAndPreviewCSV(data, nil, Options{MaxNewRules: 2, DetailLimit: 3}) if err != nil { t.Fatal(err) } if preview.Summary.New != 2 || preview.Summary.CSVDuplicate != 1 || preview.Summary.Conflict != 1 || preview.Summary.Limit != 1 { t.Fatalf("unexpected summary: %+v", preview.Summary) } if len(preview.Items) != 3 { t.Fatalf("expected 3 visible details, got %d", len(preview.Items)) } if !preview.DetailsTruncated { t.Fatal("expected details to be truncated") } if preview.VisibleLimit != 3 { t.Fatalf("expected visible limit 3, got %d", preview.VisibleLimit) } } func TestParseAndPreviewPrioritizesUsefulDetailsOverEmptySlots(t *testing.T) { data := csvBytes(t, header, []string{"1", "now", "empty row", "", "", "", "", "", "", "", "", "", ""}, []string{"2", "now", "useful row", "A", "B", "A", "C", "", "", "", "", "", ""}, ) preview, err := ParseAndPreviewCSV(data, nil, Options{DetailLimit: 2}) if err != nil { t.Fatal(err) } if len(preview.Items) != 2 { t.Fatalf("expected 2 visible details, got %d", len(preview.Items)) } if preview.Items[0].Status != StatusNew || preview.Items[1].Status != StatusConflict { t.Fatalf("expected new and conflict to be shown before empty slots, got %+v", preview.Items) } if !preview.DetailsTruncated { t.Fatal("expected details to be truncated") } if preview.Summary.Empty != 8 || preview.Summary.New != 1 || preview.Summary.Conflict != 1 { t.Fatalf("unexpected summary: %+v", preview.Summary) } } func TestParseAndPreviewRejectsInvalidInput(t *testing.T) { if _, err := ParseAndPreviewCSV([]byte{0xff, 0xfe}, nil, Options{}); err == nil { t.Fatal("expected invalid UTF-8 to fail") } data := csvBytes(t, []string{"timestamp"}, []string{"1"}) if _, err := ParseAndPreviewCSV(data, nil, Options{}); err == nil { t.Fatal("expected missing required columns to fail") } } func csvBytes(t *testing.T, rows ...[]string) []byte { t.Helper() var buf bytes.Buffer writer := csv.NewWriter(&buf) for _, row := range rows { if err := writer.Write(row); err != nil { t.Fatal(err) } } writer.Flush() if err := writer.Error(); err != nil { t.Fatal(err) } return buf.Bytes() }