package correctioncsv
|
|
import (
|
"bytes"
|
"encoding/csv"
|
"strings"
|
"testing"
|
"time"
|
"voicesnap/internal/history"
|
"voicesnap/internal/userdict"
|
)
|
|
func TestBuildHistoryCSVIncludesBOMHeaderExampleAndEscapedRows(t *testing.T) {
|
entries := []history.Entry{
|
{
|
Timestamp: time.Date(2026, 5, 30, 13, 9, 8, 0, time.UTC).UnixMilli(),
|
Text: "中文,a \"quote\"\nnext",
|
},
|
}
|
|
data, err := BuildHistoryCSV(entries, time.UTC)
|
if err != nil {
|
t.Fatal(err)
|
}
|
if !bytes.HasPrefix(data, []byte{0xEF, 0xBB, 0xBF}) {
|
t.Fatal("expected UTF-8 BOM")
|
}
|
|
reader := csv.NewReader(bytes.NewReader(trimUTF8BOM(data)))
|
records, err := reader.ReadAll()
|
if err != nil {
|
t.Fatal(err)
|
}
|
if len(records) != 3 {
|
t.Fatalf("expected header, example, and one data row, got %d rows", len(records))
|
}
|
if got := strings.Join(records[0], ","); got != strings.Join(header, ",") {
|
t.Fatalf("unexpected header: %s", got)
|
}
|
if records[1][0] != "示例" {
|
t.Fatalf("expected example row, got marker %q", records[1][0])
|
}
|
if records[1][2] != "正确的格式是点 CSV" {
|
t.Fatalf("unexpected example text: %q", records[1][2])
|
}
|
if records[2][2] != entries[0].Text {
|
t.Fatalf("expected text to round-trip, got %q", records[2][2])
|
}
|
}
|
|
func TestParseAndPreviewClassifiesRows(t *testing.T) {
|
data := csvBytes(t,
|
[]string{"corrected_1", "error_1", "text", "datetime_local", "timestamp", "error_2", "corrected_2", "error_3", "corrected_3", "error_4", "corrected_4", "error_5", "corrected_5", "notes"},
|
[]string{".csv", "点 CSV", "sample", "now", "__example__", "", "", "", "", "", "", "", "", "ignored"},
|
[]string{".csv", " 点 CSV ", "text", "now", "1", "麦克", "Mike", "空", "", "same", "same", strings.Repeat("长", 121), "x", "ignored"},
|
)
|
|
preview, err := ParseAndPreviewCSV(
|
append([]byte{0xEF, 0xBB, 0xBF}, data...),
|
[]userdict.Entry{{From: "麦克", To: "Mac"}},
|
Options{},
|
)
|
if err != nil {
|
t.Fatal(err)
|
}
|
|
if preview.TotalSlots != 5 {
|
t.Fatalf("expected 5 scanned slots, got %d", preview.TotalSlots)
|
}
|
if preview.Summary.New != 1 {
|
t.Fatalf("expected 1 new rule, got %+v", preview.Summary)
|
}
|
if preview.Summary.Conflict != 1 {
|
t.Fatalf("expected 1 conflict, got %+v", preview.Summary)
|
}
|
if preview.Summary.Empty != 1 {
|
t.Fatalf("expected 1 empty slot, got %+v", preview.Summary)
|
}
|
if preview.Summary.Invalid != 2 {
|
t.Fatalf("expected 2 invalid slots, got %+v", preview.Summary)
|
}
|
if len(preview.NewRules) != 1 || preview.NewRules[0].From != "点 CSV" || preview.NewRules[0].To != ".csv" {
|
t.Fatalf("unexpected new rules: %+v", preview.NewRules)
|
}
|
}
|
|
func TestParseAndPreviewAcceptsChineseHeadersAndSkipsChineseExampleRow(t *testing.T) {
|
data := csvBytes(t,
|
[]string{"正确词1", "错误词1", "识别历史文本", "识别时间", "记录ID", "错误词2", "正确词2", "错误词3", "正确词3", "错误词4", "正确词4", "错误词5", "正确词5", "备注"},
|
[]string{".csv", "点 CSV", "正确的格式是点 CSV", "示例，可删除或保留", "示例", "", "", "", "", "", "", "", "", "ignored"},
|
[]string{".pdf", "点 PDF", "text", "2026-05-30 13:00:00", "1", "", "", "", "", "", "", "", "", "ignored"},
|
)
|
|
preview, err := ParseAndPreviewCSV(data, nil, Options{})
|
if err != nil {
|
t.Fatal(err)
|
}
|
|
if preview.TotalSlots != 5 {
|
t.Fatalf("expected 5 scanned slots, got %d", preview.TotalSlots)
|
}
|
if preview.Summary.New != 1 {
|
t.Fatalf("expected only the real data row to create 1 new rule, got %+v", preview.Summary)
|
}
|
if len(preview.NewRules) != 1 || preview.NewRules[0].From != "点 PDF" || preview.NewRules[0].To != ".pdf" {
|
t.Fatalf("unexpected new rules: %+v", preview.NewRules)
|
}
|
}
|
|
func TestParseAndPreviewDetectsCSVDuplicateConflictLimitAndTruncatesDetails(t *testing.T) {
|
data := csvBytes(t,
|
header,
|
[]string{"1", "now", "text", "A", "B", "A", "B", "A", "C", "D", "E", "F", "G"},
|
)
|
|
preview, err := ParseAndPreviewCSV(data, nil, Options{MaxNewRules: 2, DetailLimit: 3})
|
if err != nil {
|
t.Fatal(err)
|
}
|
|
if preview.Summary.New != 2 || preview.Summary.CSVDuplicate != 1 || preview.Summary.Conflict != 1 || preview.Summary.Limit != 1 {
|
t.Fatalf("unexpected summary: %+v", preview.Summary)
|
}
|
if len(preview.Items) != 3 {
|
t.Fatalf("expected 3 visible details, got %d", len(preview.Items))
|
}
|
if !preview.DetailsTruncated {
|
t.Fatal("expected details to be truncated")
|
}
|
if preview.VisibleLimit != 3 {
|
t.Fatalf("expected visible limit 3, got %d", preview.VisibleLimit)
|
}
|
}
|
|
func TestParseAndPreviewPrioritizesUsefulDetailsOverEmptySlots(t *testing.T) {
|
data := csvBytes(t,
|
header,
|
[]string{"1", "now", "empty row", "", "", "", "", "", "", "", "", "", ""},
|
[]string{"2", "now", "useful row", "A", "B", "A", "C", "", "", "", "", "", ""},
|
)
|
|
preview, err := ParseAndPreviewCSV(data, nil, Options{DetailLimit: 2})
|
if err != nil {
|
t.Fatal(err)
|
}
|
|
if len(preview.Items) != 2 {
|
t.Fatalf("expected 2 visible details, got %d", len(preview.Items))
|
}
|
if preview.Items[0].Status != StatusNew || preview.Items[1].Status != StatusConflict {
|
t.Fatalf("expected new and conflict to be shown before empty slots, got %+v", preview.Items)
|
}
|
if !preview.DetailsTruncated {
|
t.Fatal("expected details to be truncated")
|
}
|
if preview.Summary.Empty != 8 || preview.Summary.New != 1 || preview.Summary.Conflict != 1 {
|
t.Fatalf("unexpected summary: %+v", preview.Summary)
|
}
|
}
|
|
func TestParseAndPreviewRejectsInvalidInput(t *testing.T) {
|
if _, err := ParseAndPreviewCSV([]byte{0xff, 0xfe}, nil, Options{}); err == nil {
|
t.Fatal("expected invalid UTF-8 to fail")
|
}
|
|
data := csvBytes(t, []string{"timestamp"}, []string{"1"})
|
if _, err := ParseAndPreviewCSV(data, nil, Options{}); err == nil {
|
t.Fatal("expected missing required columns to fail")
|
}
|
}
|
|
func csvBytes(t *testing.T, rows ...[]string) []byte {
|
t.Helper()
|
var buf bytes.Buffer
|
writer := csv.NewWriter(&buf)
|
for _, row := range rows {
|
if err := writer.Write(row); err != nil {
|
t.Fatal(err)
|
}
|
}
|
writer.Flush()
|
if err := writer.Error(); err != nil {
|
t.Fatal(err)
|
}
|
return buf.Bytes()
|
}
|
