| | |
| | | |
| | | import ( |
| | | "fmt" |
| | | "strings" |
| | | "sync" |
| | | "time" |
| | | "voicesnap/internal/logger" |
| | | "voicesnap/internal/model" |
| | | |
| | |
| | | sherpa "github.com/k2-fsa/sherpa-onnx-go/sherpa_onnx" |
| | | ) |
| | | |
| | | const ( |
| | | asrSampleRate = 16000 |
| | | xasrHeadPaddingSamples = asrSampleRate / 4 |
| | | xasrTailPaddingSamples = asrSampleRate + asrSampleRate/2 |
| | | xasrReleaseTailDelay = 300 * time.Millisecond |
| | | ) |
| | | |
| | | var ( |
| | | xasrHeadPadding = make([]float32, xasrHeadPaddingSamples) |
| | | xasrTailPadding = make([]float32, xasrTailPaddingSamples) |
| | | ) |
| | | |
| | | type sherpaEngine struct { |
| | | recognizer *sherpa.OfflineRecognizer |
| | | hwInfo string |
| | | } |
| | | |
| | | type xasrStreamingEngine struct { |
| | | recognizer *sherpa.OnlineRecognizer |
| | | hwInfo string |
| | | mu sync.Mutex |
| | | } |
| | | |
| | | type xasrStreamingSession struct { |
| | | engine *xasrStreamingEngine |
| | | stream *sherpa.OnlineStream |
| | | lastText string |
| | | hasAcceptedSamples bool |
| | | finished bool |
| | | } |
| | | |
| | | func newPlatformEngine(resolved model.ResolvedModel) (Engine, error) { |
| | | if !isSupportedBackend(resolved.BackendKind) { |
| | | return nil, fmt.Errorf("unsupported backend kind: %s", resolved.BackendKind) |
| | | } |
| | | if resolved.BackendKind == model.BackendXASRStreaming { |
| | | return newXASRStreamingEngine(resolved) |
| | | } |
| | | |
| | | providers := darwinProviders(resolved.ProviderOrder) |
| | | |
| | | for _, p := range providers { |
| | | config := sherpa.OfflineRecognizerConfig{} |
| | | config.FeatConfig.SampleRate = 16000 |
| | | config.FeatConfig.FeatureDim = 80 |
| | | config.ModelConfig.SenseVoice.Model = resolved.Files["model"] |
| | | config.ModelConfig.SenseVoice.UseInverseTextNormalization = 1 |
| | | config.ModelConfig.Tokens = resolved.Files["tokens"] |
| | | config.ModelConfig.NumThreads = resolved.Profile.NumThreads |
| | | config.ModelConfig.Provider = p.provider |
| | | config.DecodingMethod = "greedy_search" |
| | | config, err := offlineConfigForResolvedModel(resolved, p.provider) |
| | | if err != nil { |
| | | return nil, err |
| | | } |
| | | |
| | | recognizer := sherpa.NewOfflineRecognizer(&config) |
| | | if recognizer != nil { |
| | | info := fmt.Sprintf("SenseVoice · %s", p.name) |
| | | info := fmt.Sprintf("%s · %s", resolved.Profile.DisplayName, p.name) |
| | | logger.Info("Engine initialized: %s", info) |
| | | return &sherpaEngine{ |
| | | recognizer: recognizer, |
| | |
| | | } |
| | | |
| | | return nil, fmt.Errorf("failed to initialize sherpa-onnx with any provider") |
| | | } |
| | | |
| | | func newXASRStreamingEngine(resolved model.ResolvedModel) (Engine, error) { |
| | | providers := darwinProviders(resolved.ProviderOrder) |
| | | |
| | | for _, p := range providers { |
| | | config, err := onlineConfigForResolvedModel(resolved, p.provider) |
| | | if err != nil { |
| | | return nil, err |
| | | } |
| | | |
| | | recognizer := sherpa.NewOnlineRecognizer(&config) |
| | | if recognizer != nil { |
| | | info := fmt.Sprintf("%s · %s", resolved.Profile.DisplayName, p.name) |
| | | logger.Info("Engine initialized: %s", info) |
| | | return &xasrStreamingEngine{ |
| | | recognizer: recognizer, |
| | | hwInfo: info, |
| | | }, nil |
| | | } |
| | | logger.Info("Failed to init X-ASR with %s, trying next provider", p.name) |
| | | } |
| | | |
| | | return nil, fmt.Errorf("failed to initialize X-ASR streaming recognizer with any provider") |
| | | } |
| | | |
| | | func offlineConfigForResolvedModel(resolved model.ResolvedModel, provider string) (sherpa.OfflineRecognizerConfig, error) { |
| | | config := sherpa.OfflineRecognizerConfig{} |
| | | config.FeatConfig.SampleRate = 16000 |
| | | config.FeatConfig.FeatureDim = 80 |
| | | config.ModelConfig.Tokens = resolved.Files["tokens"] |
| | | config.ModelConfig.NumThreads = resolved.Profile.NumThreads |
| | | config.ModelConfig.Provider = provider |
| | | config.DecodingMethod = "greedy_search" |
| | | |
| | | switch resolved.BackendKind { |
| | | case model.BackendSenseVoice: |
| | | config.ModelConfig.SenseVoice.Model = resolved.Files["model"] |
| | | config.ModelConfig.SenseVoice.UseInverseTextNormalization = 1 |
| | | case model.BackendMoonshine: |
| | | config.ModelConfig.Moonshine.Preprocessor = resolved.Files["preprocessor"] |
| | | config.ModelConfig.Moonshine.Encoder = resolved.Files["encoder"] |
| | | config.ModelConfig.Moonshine.UncachedDecoder = resolved.Files["uncached_decoder"] |
| | | config.ModelConfig.Moonshine.CachedDecoder = resolved.Files["cached_decoder"] |
| | | case model.BackendTransducer: |
| | | config.ModelConfig.Transducer.Encoder = resolved.Files["encoder"] |
| | | config.ModelConfig.Transducer.Decoder = resolved.Files["decoder"] |
| | | config.ModelConfig.Transducer.Joiner = resolved.Files["joiner"] |
| | | case model.BackendNemoTransducer: |
| | | config.ModelConfig.Transducer.Encoder = resolved.Files["encoder"] |
| | | config.ModelConfig.Transducer.Decoder = resolved.Files["decoder"] |
| | | config.ModelConfig.Transducer.Joiner = resolved.Files["joiner"] |
| | | config.ModelConfig.ModelType = model.BackendNemoTransducer |
| | | case model.BackendQwen3ASR: |
| | | config.ModelConfig.Qwen3ASR.ConvFrontend = resolved.Files["conv_frontend"] |
| | | config.ModelConfig.Qwen3ASR.Encoder = resolved.Files["encoder"] |
| | | config.ModelConfig.Qwen3ASR.Decoder = resolved.Files["decoder"] |
| | | config.ModelConfig.Qwen3ASR.Tokenizer = resolved.Files["tokenizer"] |
| | | config.ModelConfig.Qwen3ASR.MaxTotalLen = 1024 |
| | | config.ModelConfig.Qwen3ASR.MaxNewTokens = 256 |
| | | config.ModelConfig.Qwen3ASR.Temperature = 0.0 |
| | | config.ModelConfig.Qwen3ASR.TopP = 0.9 |
| | | config.ModelConfig.Qwen3ASR.Seed = 0 |
| | | default: |
| | | return sherpa.OfflineRecognizerConfig{}, fmt.Errorf("unsupported backend kind: %s", resolved.BackendKind) |
| | | } |
| | | |
| | | return config, nil |
| | | } |
| | | |
| | | func onlineConfigForResolvedModel(resolved model.ResolvedModel, provider string) (sherpa.OnlineRecognizerConfig, error) { |
| | | config := sherpa.OnlineRecognizerConfig{} |
| | | config.FeatConfig.SampleRate = 16000 |
| | | config.FeatConfig.FeatureDim = 80 |
| | | config.ModelConfig.Tokens = resolved.Files["tokens"] |
| | | config.ModelConfig.NumThreads = resolved.Profile.NumThreads |
| | | config.ModelConfig.Provider = provider |
| | | config.ModelConfig.ModelType = "zipformer2" |
| | | config.DecodingMethod = "greedy_search" |
| | | config.EnableEndpoint = 0 |
| | | |
| | | switch resolved.BackendKind { |
| | | case model.BackendXASRStreaming: |
| | | config.ModelConfig.Transducer.Encoder = resolved.Files["encoder"] |
| | | config.ModelConfig.Transducer.Decoder = resolved.Files["decoder"] |
| | | config.ModelConfig.Transducer.Joiner = resolved.Files["joiner"] |
| | | default: |
| | | return sherpa.OnlineRecognizerConfig{}, fmt.Errorf("unsupported online backend kind: %s", resolved.BackendKind) |
| | | } |
| | | |
| | | return config, nil |
| | | } |
| | | |
| | | func darwinProviders(providerOrder []string) []struct { |
| | |
| | | stream := sherpa.NewOfflineStream(e.recognizer) |
| | | defer sherpa.DeleteOfflineStream(stream) |
| | | |
| | | stream.AcceptWaveform(16000, samples) |
| | | stream.AcceptWaveform(asrSampleRate, samples) |
| | | |
| | | e.recognizer.Decode(stream) |
| | | result := stream.GetResult() |
| | |
| | | return result.Text, nil |
| | | } |
| | | |
| | | func (e *xasrStreamingEngine) Recognize(samples []float32) (string, error) { |
| | | session, err := e.NewStreamingSession() |
| | | if err != nil { |
| | | return "", err |
| | | } |
| | | defer session.Close() |
| | | |
| | | if _, err := session.Accept(samples); err != nil { |
| | | return "", err |
| | | } |
| | | return session.Finish() |
| | | } |
| | | |
| | | func (e *xasrStreamingEngine) NewStreamingSession() (StreamingSession, error) { |
| | | if e.recognizer == nil { |
| | | return nil, fmt.Errorf("X-ASR recognizer is not initialized") |
| | | } |
| | | stream := sherpa.NewOnlineStream(e.recognizer) |
| | | if stream == nil { |
| | | return nil, fmt.Errorf("failed to create X-ASR streaming session") |
| | | } |
| | | return &xasrStreamingSession{ |
| | | engine: e, |
| | | stream: stream, |
| | | }, nil |
| | | } |
| | | |
| | | func (s *xasrStreamingSession) Accept(samples []float32) (string, error) { |
| | | if len(samples) == 0 { |
| | | return s.lastText, nil |
| | | } |
| | | if s.stream == nil || s.finished { |
| | | return s.lastText, nil |
| | | } |
| | | if !s.hasAcceptedSamples { |
| | | samples = xasrSamplesWithHeadPadding(samples) |
| | | s.hasAcceptedSamples = true |
| | | } |
| | | |
| | | s.engine.mu.Lock() |
| | | defer s.engine.mu.Unlock() |
| | | |
| | | s.stream.AcceptWaveform(asrSampleRate, samples) |
| | | s.lastText = s.engine.decodeReadyLocked(s.stream, s.lastText) |
| | | return s.lastText, nil |
| | | } |
| | | |
| | | func (s *xasrStreamingSession) Finish() (string, error) { |
| | | if s.stream == nil || s.finished { |
| | | return s.lastText, nil |
| | | } |
| | | |
| | | s.engine.mu.Lock() |
| | | defer s.engine.mu.Unlock() |
| | | |
| | | s.stream.AcceptWaveform(asrSampleRate, xasrTailPadding) |
| | | s.lastText = s.engine.decodeReadyLocked(s.stream, s.lastText) |
| | | s.stream.InputFinished() |
| | | s.lastText = s.engine.decodeReadyLocked(s.stream, s.lastText) |
| | | s.finished = true |
| | | return s.lastText, nil |
| | | } |
| | | |
| | | func (s *xasrStreamingSession) Close() { |
| | | if s.stream == nil { |
| | | return |
| | | } |
| | | |
| | | s.engine.mu.Lock() |
| | | defer s.engine.mu.Unlock() |
| | | |
| | | sherpa.DeleteOnlineStream(s.stream) |
| | | s.stream = nil |
| | | } |
| | | |
| | | func (e *xasrStreamingEngine) decodeReadyLocked(stream *sherpa.OnlineStream, lastText string) string { |
| | | for e.recognizer.IsReady(stream) { |
| | | e.recognizer.Decode(stream) |
| | | lastText = rememberNonEmptyText(lastText, onlineResultText(e.recognizer, stream)) |
| | | } |
| | | return rememberNonEmptyText(lastText, onlineResultText(e.recognizer, stream)) |
| | | } |
| | | |
| | | func onlineResultText(recognizer *sherpa.OnlineRecognizer, stream *sherpa.OnlineStream) string { |
| | | result := recognizer.GetResult(stream) |
| | | if result == nil { |
| | | return "" |
| | | } |
| | | return result.Text |
| | | } |
| | | |
| | | func rememberNonEmptyText(lastText, nextText string) string { |
| | | if strings.TrimSpace(nextText) == "" { |
| | | return lastText |
| | | } |
| | | return nextText |
| | | } |
| | | |
| | | func xasrSamplesWithHeadPadding(samples []float32) []float32 { |
| | | padded := make([]float32, 0, len(xasrHeadPadding)+len(samples)) |
| | | padded = append(padded, xasrHeadPadding...) |
| | | padded = append(padded, samples...) |
| | | return padded |
| | | } |
| | | |
| | | func (e *sherpaEngine) HardwareInfo() string { |
| | | return e.hwInfo |
| | | } |
| | | |
| | | func (e *xasrStreamingEngine) HardwareInfo() string { |
| | | return e.hwInfo |
| | | } |
| | | |
| | | func (e *xasrStreamingEngine) ReleaseTailCaptureDelay() time.Duration { |
| | | return xasrReleaseTailDelay |
| | | } |
| | | |
| | | func (e *xasrStreamingEngine) HoldPreCaptureEnabled() bool { |
| | | return true |
| | | } |
| | | |
| | | func (e *sherpaEngine) Close() { |
| | |
| | | e.recognizer = nil |
| | | } |
| | | } |
| | | |
| | | func (e *xasrStreamingEngine) Close() { |
| | | e.mu.Lock() |
| | | defer e.mu.Unlock() |
| | | |
| | | if e.recognizer != nil { |
| | | sherpa.DeleteOnlineRecognizer(e.recognizer) |
| | | e.recognizer = nil |
| | | } |
| | | } |