Add Cantonese language model support

Ariver

2026-07-02 0ffbf1935c9d091cce22a5583275ac0902f7a693

 privatevoice.src/internal/model/registry.go

@@ -8,8 +8,24 @@
const (
   BackendSenseVoice     = "sensevoice"
   BackendMoonshine      = "moonshine"
   BackendTransducer     = "transducer"
   BackendNemoTransducer = "nemo_transducer"
   BackendQwen3ASR       = "qwen3_asr"
   BackendXASRStreaming  = "x_asr_streaming"
)

var languageProfileOrder = []string{
   DefaultLanguageID,
   CantoneseLanguageID,
   EnglishLanguageID,
   FrenchLanguageID,
   GermanLanguageID,
   SpanishLanguageID,
   ItalianLanguageID,
   PortugueseLanguageID,
   JapaneseLanguageID,
   KoreanLanguageID,
}

var modelProfiles = map[string]ModelProfile{
   DefaultModelID: {
@@ -130,16 +146,264 @@
      ProviderOrder: []string{"cpu"},
      NumThreads:    4,
   },
   ParakeetV3ModelID: {
      ID:          ParakeetV3ModelID,
      DisplayName: "Parakeet TDT v3",
      BackendKind: BackendNemoTransducer,
      Tier:        "default",
      SupportedLanguageIDs: []string{
         FrenchLanguageID,
         GermanLanguageID,
         SpanishLanguageID,
         ItalianLanguageID,
         PortugueseLanguageID,
      },
      RecommendedFor: []string{
         FrenchLanguageID,
         GermanLanguageID,
         SpanishLanguageID,
         ItalianLanguageID,
         PortugueseLanguageID,
      },
      Description: "欧洲多语种离线模型，适合法语、德语、西班牙语、意大利语和葡萄牙语听写。",
      ApproxSize:  "约 640 MiB 解压后模型目录",
      DownloadURLs: []string{
         "https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-nemo-parakeet-tdt-0.6b-v3-int8.tar.bz2",
      },
      InstallDirName: "parakeet-tdt-0.6b-v3",
      RequiredFiles: []RequiredFileRule{
         {
            Role:     "encoder",
            AllOf:    []string{"encoder.int8.onnx"},
            Required: true,
         },
         {
            Role:     "decoder",
            AllOf:    []string{"decoder.int8.onnx"},
            Required: true,
         },
         {
            Role:     "joiner",
            AllOf:    []string{"joiner.int8.onnx"},
            Required: true,
         },
         {
            Role:     "tokens",
            AllOf:    []string{"tokens.txt"},
            Required: true,
         },
      },
      ProviderOrder: []string{"cpu"},
      NumThreads:    2,
   },
   JapaneseModelID: {
      ID:                   JapaneseModelID,
      DisplayName:          "Zipformer Japanese",
      BackendKind:          BackendTransducer,
      Tier:                 "default",
      SupportedLanguageIDs: []string{JapaneseLanguageID},
      RecommendedFor:       []string{JapaneseLanguageID},
      Description:          "日语专项离线模型，基于 ReazonSpeech 日语语料。",
      ApproxSize:           "约 165 MiB int8 关键模型文件，解压目录包含 fp32 对照文件",
      DownloadURLs: []string{
         "https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-zipformer-ja-reazonspeech-2024-08-01.tar.bz2",
      },
      InstallDirName: "zipformer-ja-reazonspeech",
      RequiredFiles: []RequiredFileRule{
         {
            Role:     "encoder",
            AllOf:    []string{"encoder-epoch-99-avg-1.int8.onnx"},
            Required: true,
         },
         {
            Role:     "decoder",
            AllOf:    []string{"decoder-epoch-99-avg-1.onnx"},
            Required: true,
         },
         {
            Role:     "joiner",
            AllOf:    []string{"joiner-epoch-99-avg-1.int8.onnx"},
            Required: true,
         },
         {
            Role:     "tokens",
            AllOf:    []string{"tokens.txt"},
            Required: true,
         },
      },
      ProviderOrder: []string{"cpu"},
      NumThreads:    1,
   },
   KoreanModelID: {
      ID:                   KoreanModelID,
      DisplayName:          "Zipformer Korean",
      BackendKind:          BackendTransducer,
      Tier:                 "default",
      SupportedLanguageIDs: []string{KoreanLanguageID},
      RecommendedFor:       []string{KoreanLanguageID},
      Description:          "韩语专项离线模型，适合韩语听写。",
      ApproxSize:           "约 73 MiB int8 关键模型文件，解压目录包含 fp32 对照文件",
      DownloadURLs: []string{
         "https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-zipformer-korean-2024-06-24.tar.bz2",
      },
      InstallDirName: "zipformer-ko",
      RequiredFiles: []RequiredFileRule{
         {
            Role:     "encoder",
            AllOf:    []string{"encoder-epoch-99-avg-1.int8.onnx"},
            Required: true,
         },
         {
            Role:     "decoder",
            AllOf:    []string{"decoder-epoch-99-avg-1.onnx"},
            Required: true,
         },
         {
            Role:     "joiner",
            AllOf:    []string{"joiner-epoch-99-avg-1.int8.onnx"},
            Required: true,
         },
         {
            Role:     "tokens",
            AllOf:    []string{"tokens.txt"},
            Required: true,
         },
      },
      ProviderOrder: []string{"cpu"},
      NumThreads:    2,
   },
   Qwen3ASRModelID: {
      ID:                   Qwen3ASRModelID,
      DisplayName:          "Qwen3-ASR",
      BackendKind:          BackendQwen3ASR,
      Tier:                 "advanced",
      SupportedLanguageIDs: []string{DefaultLanguageID, "zh-Hans", "zh-Hant", "yue"},
      RecommendedFor:       []string{DefaultLanguageID},
      Description:          "高质量中文离线模型，适合中文长句、技术词和中英混输准确率需求。",
      ApproxSize:           "约 838 MiB 下载包，解压后约 1.9 GiB",
      DownloadURLs: []string{
         "https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-qwen3-asr-0.6B-int8-2026-03-25.tar.bz2",
      },
      InstallDirName: "qwen3-asr-0.6b",
      RequiredFiles: []RequiredFileRule{
         {
            Role:     "conv_frontend",
            AllOf:    []string{"conv_frontend.onnx"},
            Required: true,
         },
         {
            Role:     "encoder",
            AllOf:    []string{"encoder.int8.onnx"},
            Required: true,
         },
         {
            Role:     "decoder",
            AllOf:    []string{"decoder.int8.onnx"},
            Required: true,
         },
         {
            Role:      "tokenizer",
            AllOf:     []string{"tokenizer"},
            Required:  true,
            Directory: true,
         },
         {
            Role:     "tokenizer_merges",
            AllOf:    []string{"tokenizer/merges.txt"},
            Required: true,
         },
         {
            Role:     "tokenizer_vocab",
            AllOf:    []string{"tokenizer/vocab.json"},
            Required: true,
         },
      },
      ProviderOrder: []string{"cpu"},
      NumThreads:    4,
   },
   XASRZhEn960ModelID: {
      ID:                   XASRZhEn960ModelID,
      DisplayName:          "X-ASR zh-en 960ms",
      BackendKind:          BackendXASRStreaming,
      Tier:                 "experimental",
      SupportedLanguageIDs: []string{DefaultLanguageID, "zh-Hans", "zh-Hant", "yue", EnglishLanguageID},
      RecommendedFor:       []string{DefaultLanguageID, EnglishLanguageID},
      Description:          "实验性中英流式模型，适合评估中英混输和未来边说边出字体验。",
      ApproxSize:           "单个 960 ms 模型目录约 586 MiB",
      InstallDirName:       "x-asr-zh-en-960ms",
      RequiredFiles: []RequiredFileRule{
         {
            Role:     "encoder",
            AllOf:    []string{"encoder-960ms.onnx"},
            Required: true,
         },
         {
            Role:     "decoder",
            AllOf:    []string{"decoder-960ms.onnx"},
            Required: true,
         },
         {
            Role:     "joiner",
            AllOf:    []string{"joiner-960ms.onnx"},
            Required: true,
         },
         {
            Role:     "tokens",
            AllOf:    []string{"tokens.txt"},
            Required: true,
         },
      },
      ProviderOrder: []string{"cpu"},
      NumThreads:    1,
   },
   SenseVoiceYueModelID: {
      ID:                   SenseVoiceYueModelID,
      DisplayName:          "SenseVoice Yue",
      BackendKind:          BackendSenseVoice,
      Tier:                 "default",
      SupportedLanguageIDs: []string{CantoneseLanguageID},
      RecommendedFor:       []string{CantoneseLanguageID},
      Description:          "粤语专项离线模型，基于 SenseVoice 并使用粤语数据增强。",
      ApproxSize:           "约 226 MiB 模型文件；下载包大小以实际 HTTP HEAD 为准",
      LanguageParam:        "yue",
      DownloadURLs: []string{
         "https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-sense-voice-zh-en-ja-ko-yue-int8-2025-09-09.tar.bz2",
      },
      InstallDirName: "sensevoice-yue-2025-09-09",
      RequiredFiles: []RequiredFileRule{
         {
            Role:     "model",
            AllOf:    []string{"model.int8.onnx"},
            Required: true,
         },
         {
            Role:     "tokens",
            AllOf:    []string{"tokens.txt"},
            Required: true,
         },
      },
      ProviderOrder: []string{"coreml", "cpu"},
      NumThreads:    4,
   },
}

var languageProfiles = map[string]LanguageProfile{
   DefaultLanguageID: {
      ID:             DefaultLanguageID,
      DisplayName:    "Chinese",
      NativeName:     "中文",
      ID:              DefaultLanguageID,
      DisplayName:     "Chinese",
      NativeName:      "中文",
      UILocale:        "zh",
      SystemMatchers:  []string{"zh", "zh-CN", "zh-Hans", "zh-Hant"},
      DefaultModelID:  DefaultModelID,
      UpgradeModelIDs: []string{Qwen3ASRModelID, XASRZhEn960ModelID},
   },
   CantoneseLanguageID: {
      ID:             CantoneseLanguageID,
      DisplayName:    "Cantonese",
      NativeName:     "粤语",
      UILocale:       "zh",
      SystemMatchers: []string{"zh", "zh-CN", "zh-Hans", "zh-Hant"},
      DefaultModelID: DefaultModelID,
      SystemMatchers: []string{"yue", "yue-HK"},
      DefaultModelID: SenseVoiceYueModelID,
   },
   EnglishLanguageID: {
      ID:              EnglishLanguageID,
@@ -148,7 +412,63 @@
      UILocale:        "en",
      SystemMatchers:  []string{"en", "en-US", "en-GB", "en-AU", "en-CA"},
      DefaultModelID:  MoonshineModelID,
      UpgradeModelIDs: []string{ParakeetModelID},
      UpgradeModelIDs: []string{ParakeetModelID, XASRZhEn960ModelID},
   },
   FrenchLanguageID: {
      ID:             FrenchLanguageID,
      DisplayName:    "French",
      NativeName:     "Français",
      UILocale:       "fr",
      SystemMatchers: []string{"fr", "fr-FR", "fr-BE", "fr-CA", "fr-CH"},
      DefaultModelID: ParakeetV3ModelID,
   },
   GermanLanguageID: {
      ID:             GermanLanguageID,
      DisplayName:    "German",
      NativeName:     "Deutsch",
      UILocale:       "de",
      SystemMatchers: []string{"de", "de-DE", "de-AT", "de-CH"},
      DefaultModelID: ParakeetV3ModelID,
   },
   SpanishLanguageID: {
      ID:             SpanishLanguageID,
      DisplayName:    "Spanish",
      NativeName:     "Español",
      UILocale:       "es",
      SystemMatchers: []string{"es", "es-ES", "es-MX", "es-US", "es-AR"},
      DefaultModelID: ParakeetV3ModelID,
   },
   ItalianLanguageID: {
      ID:             ItalianLanguageID,
      DisplayName:    "Italian",
      NativeName:     "Italiano",
      UILocale:       "it",
      SystemMatchers: []string{"it", "it-IT", "it-CH"},
      DefaultModelID: ParakeetV3ModelID,
   },
   PortugueseLanguageID: {
      ID:             PortugueseLanguageID,
      DisplayName:    "Portuguese",
      NativeName:     "Português",
      UILocale:       "pt",
      SystemMatchers: []string{"pt", "pt-PT", "pt-BR"},
      DefaultModelID: ParakeetV3ModelID,
   },
   JapaneseLanguageID: {
      ID:             JapaneseLanguageID,
      DisplayName:    "Japanese",
      NativeName:     "日本語",
      UILocale:       "ja",
      SystemMatchers: []string{"ja", "ja-JP"},
      DefaultModelID: JapaneseModelID,
   },
   KoreanLanguageID: {
      ID:             KoreanLanguageID,
      DisplayName:    "Korean",
      NativeName:     "한국어",
      UILocale:       "ko",
      SystemMatchers: []string{"ko", "ko-KR"},
      DefaultModelID: KoreanModelID,
   },
}

@@ -192,22 +512,42 @@

func ListLanguageProfiles() []LanguageProfile {
   profiles := make([]LanguageProfile, 0, len(languageProfiles))
   for _, profile := range languageProfiles {
      profiles = append(profiles, profile)
   seen := make(map[string]bool, len(languageProfiles))
   for _, id := range languageProfileOrder {
      if profile, ok := languageProfiles[id]; ok {
         profiles = append(profiles, profile)
         seen[id] = true
      }
   }
   for id, profile := range languageProfiles {
      if !seen[id] {
         profiles = append(profiles, profile)
      }
   }
   sort.Slice(profiles, func(i, j int) bool {
      if profiles[i].ID == DefaultLanguageID {
         return true
      }
      if profiles[j].ID == DefaultLanguageID {
         return false
      left := languageOrderIndex(profiles[i].ID)
      right := languageOrderIndex(profiles[j].ID)
      if left != right {
         return left < right
      }
      return profiles[i].ID < profiles[j].ID
   })
   return profiles
}

func languageOrderIndex(id string) int {
   for i, orderedID := range languageProfileOrder {
      if id == orderedID {
         return i
      }
   }
   return len(languageProfileOrder)
}

func NormalizeLanguageID(id string) string {
   if id == "yue" {
      return CantoneseLanguageID
   }
   if _, err := GetLanguageProfile(id); err != nil {
      return DefaultLanguageID
   }