From 2776a3104f076b1dbf33452ad88cb60036c39702 Mon Sep 17 00:00:00 2001
From: cai <cai@nbcai.cc>
Date: Fri, 10 Jul 2026 18:29:06 +0800
Subject: [PATCH] fix: decouple livekit audio drain from turn processing
---
src/main.rs | 101 ++++++++++++++++++++++++++++++++++++++++++++++++--
README.md | 2 +
2 files changed, 98 insertions(+), 5 deletions(-)
diff --git a/README.md b/README.md
index e9ba507..6e38c05 100644
--- a/README.md
+++ b/README.md
@@ -103,6 +103,8 @@
当 Java 在 `sessions/start.runtime` 下发 `asrRealtimeEnabled=true`、`asrRealtimeUrl` 与 `asrRealtimeChunkDurationMs=200` 时，worker 会从 VAD speech start 起接收 20ms 用户音频帧，聚合成 `16kHz / mono / 200ms` NDJSON chunk 持续上传给 Java；最后一块允许短于 200ms。helper 只负责音频传输和 fallback 编排，ASR provider session、partial/final 归一化、activity 和 `asrResultRef` 仍由 Java 管理；realtime 失败时只回退一次既有 final-only ASR stream。
+用户音轨由独立 drain task 持续读取 `NativeAudioStream`，并通过容量 `100` 的原始 PCM frame 队列交给 VAD / realtime ASR 消费；ASR final、turn bridge、LLM/TTS 或控制面等待不得阻塞 LiveKit 原生音频队列。drain 队列满时丢最新 frame，并低频记录累计接收/丢弃计数；VAD 使用 frame 捕获时的单调时间戳，ASR 始终消费原始 PCM，并保留 speech start 确认窗口中的 prefix frame。该结构参考 `cb-sdk/combrabo-platform origin/stag@a81f17c` 的音频排空与 raw PCM 修复经验。
+
鉴权口径：
- Java 调 helper 控制面使用 `Authorization: Bearer {helperAuthToken}`。
diff --git a/src/main.rs b/src/main.rs
index 62107a9..02f9fdb 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -3,6 +3,7 @@
mod service;
use std::{
+ borrow::Cow,
env, fs,
path::{Path, PathBuf},
sync::{
@@ -32,12 +33,18 @@
use reqwest::Client;
use serde::{Deserialize, Serialize};
use serde_json::json;
-use tokio::time::{sleep, sleep_until};
-use tokio::{sync::mpsc::UnboundedReceiver, task::JoinHandle};
+use tokio::time::{sleep, sleep_until, timeout};
+use tokio::{
+ sync::{mpsc, mpsc::UnboundedReceiver, watch},
+ task::JoinHandle,
+};
use tracing::{info, warn};
const USER_AUDIO_SAMPLE_RATE_HZ: u32 = 48_000;
const USER_AUDIO_NUM_CHANNELS: u16 = 1;
+const INBOUND_AUDIO_QUEUE_CAPACITY: usize = 100;
+const INBOUND_AUDIO_DROP_LOG_INTERVAL: u64 = 100;
+const AUDIO_DRAIN_STOP_GRACE: Duration = Duration::from_secs(2);
const DEFAULT_BOT_AUDIO_PROFILE: &str = "pcm-16k";
const LIVEKIT_48K_SAMPLE_RATE_HZ: u32 = 48_000;
const PCM_16K_SAMPLE_RATE_HZ: u32 = 16_000;
@@ -2469,6 +2476,68 @@
i32::from(USER_AUDIO_NUM_CHANNELS),
);
let started_at = Instant::now();
+ let (frame_tx, mut frame_rx) =
+ mpsc::channel::<DrainedUserAudioFrame>(INBOUND_AUDIO_QUEUE_CAPACITY);
+ let (drain_shutdown_tx, mut drain_shutdown_rx) = watch::channel(false);
+ let drain_call_id = call_id.clone();
+ let drain_trace_id = trace_id.clone();
+ let drain_task = tokio::spawn(async move {
+ let mut received_frame_count: u64 = 0;
+ let mut dropped_frame_count: u64 = 0;
+ loop {
+ tokio::select! {
+ changed = drain_shutdown_rx.changed() => {
+ match changed {
+ Ok(()) if *drain_shutdown_rx.borrow() => break,
+ Ok(()) => {}
+ Err(_) => break,
+ }
+ }
+ maybe_frame = stream.next() => {
+ let Some(frame) = maybe_frame else {
+ break;
+ };
+ received_frame_count = received_frame_count.saturating_add(1);
+ let drained = DrainedUserAudioFrame {
+ frame_index: received_frame_count,
+ captured_elapsed_ms: started_at.elapsed().as_millis() as u64,
+ frame: AudioFrame {
+ data: Cow::Owned(frame.data.as_ref().to_vec()),
+ sample_rate: frame.sample_rate,
+ num_channels: frame.num_channels,
+ samples_per_channel: frame.samples_per_channel,
+ },
+ };
+ match frame_tx.try_send(drained) {
+ Ok(()) => {}
+ Err(mpsc::error::TrySendError::Full(_)) => {
+ dropped_frame_count = dropped_frame_count.saturating_add(1);
+ if dropped_frame_count % INBOUND_AUDIO_DROP_LOG_INTERVAL == 1 {
+ warn!(
+ call_id = %drain_call_id,
+ trace_id = %drain_trace_id,
+ dropped_frame_count,
+ received_frame_count,
+ queue_capacity = INBOUND_AUDIO_QUEUE_CAPACITY,
+ "runtime helper inbound_audio_queue_full_dropping_newest"
+ );
+ }
+ }
+ Err(mpsc::error::TrySendError::Closed(_)) => break,
+ }
+ }
+ }
+ }
+ stream.close();
+ info!(
+ call_id = %drain_call_id,
+ trace_id = %drain_trace_id,
+ received_frame_count,
+ dropped_frame_count,
+ queue_capacity = INBOUND_AUDIO_QUEUE_CAPACITY,
+ "runtime helper user_audio_drain_ended"
+ );
+ });
let mut frame_count: u64 = 0;
let mut sample_count: u64 = 0;
let mut simple_vad = if simple_vad_enabled {
@@ -2478,10 +2547,11 @@
};
let mut realtime_asr_upload: Option<RealtimeAsrUpload> = None;
- while let Some(frame) = stream.next().await {
- frame_count += 1;
+ while let Some(drained) = frame_rx.recv().await {
+ let frame = drained.frame;
+ frame_count = drained.frame_index;
sample_count += u64::from(frame.samples_per_channel) * u64::from(frame.num_channels);
- let elapsed_ms = started_at.elapsed().as_millis() as u64;
+ let elapsed_ms = drained.captured_elapsed_ms;
if frame_count == 1 {
info!(
call_id = %call_id,
@@ -2641,6 +2711,21 @@
if let Some(upload) = realtime_asr_upload.take() {
upload.cancel("stream_end").await;
}
+ let _ = drain_shutdown_tx.send(true);
+ let mut drain_task = drain_task;
+ if timeout(AUDIO_DRAIN_STOP_GRACE, &mut drain_task)
+ .await
+ .is_err()
+ {
+ drain_task.abort();
+ let _ = drain_task.await;
+ warn!(
+ call_id = %call_id,
+ trace_id = %trace_id,
+ stop_grace_ms = AUDIO_DRAIN_STOP_GRACE.as_millis() as u64,
+ "runtime helper user_audio_drain_stop_timeout"
+ );
+ }
info!(
call_id = %call_id,
@@ -2655,6 +2740,12 @@
})
}
+struct DrainedUserAudioFrame {
+ frame_index: u64,
+ captured_elapsed_ms: u64,
+ frame: AudioFrame<'static>,
+}
+
async fn finish_realtime_asr_upload(
upload: RealtimeAsrUpload,
call_id: &str,
--
Gitblit v1.9.3