From e1612a0151ef28c4e02c0f1843dd725a92d4c268 Mon Sep 17 00:00:00 2001
From: cai <cai@nbcai.cc>
Date: Wed, 08 Jul 2026 12:35:21 +0800
Subject: [PATCH] feat: default combrabo voice bot audio to 16k
---
docs/runtime-contract.md | 57 +++++++++++
src/audio.rs | 32 ++++++
tools/validate-turn-stream-fixture.mjs | 5
src/main.rs | 128 ++++++++++++++++++++-----
fixtures/turn-stream-happy.ndjson | 2
README.md | 30 +++++
6 files changed, 223 insertions(+), 31 deletions(-)
diff --git a/README.md b/README.md
index 9ea88e6..88fd05a 100644
--- a/README.md
+++ b/README.md
@@ -124,6 +124,34 @@
结构化事件后才返回 `STARTED + botTrackReady=true`；如果 worker 失败或超时，则返回结构化
`RUNTIME_START_FAILED` / `RUNTIME_START_TIMEOUT`。
+## Bot 音频输出 profile
+
+helper 的 bot 下行音频输出通过部署配置选择，不通过 `sessions/start` 临时传业务参数。本轮 Lmtest / Combrabo Voice 动态回复主链路按用户最新裁决收口为全链路 `16kHz / mono / pcm_s16le`：Java stream bridge 下发 `sampleRate=16000/channels=1`，helper 写入 LiveKit bot track 前也保持 16k，不在业务层显式转成 48k。
+
+```bash
+# 默认值和本轮目标链路：bot NativeAudioSource = 16kHz / mono。
+export CV_BOT_AUDIO_PROFILE=pcm-16k
+
+# 历史 / 紧急回退：bot NativeAudioSource = 48kHz / mono。
+# 不作为本轮 Lmtest 动态回复主链路或音质验收目标。
+export CV_BOT_AUDIO_PROFILE=livekit-48k
+
+# 调试值：显式指定采样率和声道。
+export CV_BOT_AUDIO_PROFILE=custom
+export CV_BOT_SAMPLE_RATE_HZ=16000
+export CV_BOT_NUM_CHANNELS=1
+```
+
+`bot_track_ready` activity 会回写 `audioProfile`、`sampleRate`、`numChannels`，这是 smoke 判断当前运行 profile 的准确信号。动态回复 `bot_reply_audio_write_finished` 会回写 source / target 元数据：
+
+- `sourceSampleRate` / `sourceChannels`：Java stream bridge 下发的 `reply_audio_chunk` 源格式。
+- `targetAudioProfile` / `targetSampleRate` / `targetChannels`：helper 写入 LiveKit bot track 前的目标格式。
+- `networkChunkCount`、`debugSourcePath`、`debugPcmWavPath`：用于脱敏排查 chunk/frame 对齐与听感问题。
+
+本轮验收要求动态回复 `stream-reply-{turnId}-source.wav` 与 `stream-reply-{turnId}-target.wav` 都是 `16000Hz/mono/pcm_s16le`。如果 target 仍是 `48000Hz`，视为未完成全链路 16k 目标。
+
+当前只把 bot 下行输出 profile 做成可配置。用户上行采集、VAD 和 turn artifact 首版仍保持 `48kHz / mono`，避免牵动 ASR 输入、turn bridge 和历史 smoke 口径。
+
## 本机固定问候音频
helper 当前 MVP 支持 **WAV / MP3** 音频自动识别。主动问候 `prepare` 生成的 MP3 可以直接进入 helper；本地 smoke 仍可使用 WAV fixture 作为固定兜底样本。
@@ -173,7 +201,7 @@
<callId>-greeting-target.wav
```
-其中 `source` 是原始主动问候音频，`target` 是推送给 LiveKit 前的 `48kHz/mono/16-bit` WAV。文件只用于本地回听排查，不进入 Git、不写入协作事件正文，不上传到线上环境。
+其中 `source` 是原始主动问候音频，`target` 是推送给 LiveKit 前的目标 profile WAV。默认是 `16kHz/mono/16-bit`；只有显式配置 `CV_BOT_AUDIO_PROFILE=livekit-48k` 时才会生成 `48kHz/mono/16-bit` target。文件只用于本地回听排查，不进入 Git、不写入协作事件正文，不上传到线上环境。
## 用户上行音频观测
diff --git a/docs/runtime-contract.md b/docs/runtime-contract.md
index 1ac5424..d521b07 100644
--- a/docs/runtime-contract.md
+++ b/docs/runtime-contract.md
@@ -34,7 +34,7 @@
- `reply_playback_mode_selected` appears before audio chunks.
- `reply_playback_started` appears before the first audio chunk.
- `reply_audio_chunk.audioChunk.format` is `pcm_s16le`, `mp3`, `mpeg` or `wav`.
-- `pcm_s16le` chunks are `48000Hz` mono and 16-bit aligned.
+- `pcm_s16le` chunks are `16000Hz` or `48000Hz` mono and 16-bit aligned.
- Encoded chunks such as `mp3` must have non-empty payload bytes. This validates contract shape only; decoder and LiveKit write behavior still require runtime smoke.
- A terminal event exists.
@@ -80,6 +80,61 @@
Service mode does not change the media worker boundary. It only replaces `lmrobot-app -> ProcessBuilder` with `lmrobot-app -> helper HTTP control plane`.
+## Bot audio output profile
+
+The bot output profile is a helper deployment setting, not a per-call business field. Worker mode reads it directly from `CV_*` variables. Service mode workers inherit it from the helper service process.
+
+For the current Lmtest / Combrabo Voice dynamic reply target, the business chain is fixed to `16000Hz / mono / pcm_s16le` from Java stream bridge to helper bot-track output. The helper must not explicitly convert dynamic reply output to `48000Hz` in this path. `livekit-48k` is retained only for historical paths or emergency rollback, not as this round's target profile.
+
+| Profile | NativeAudioSource | Purpose |
+| --- | --- | --- |
+| `pcm-16k` | `16000Hz / mono` | Default and current dynamic reply target profile. |
+| `livekit-48k` | `48000Hz / mono` | Historical / emergency rollback profile only. |
+| `custom` | `CV_BOT_SAMPLE_RATE_HZ / CV_BOT_NUM_CHANNELS` | Local debug only unless promoted by a separate decision. |
+
+```bash
+CV_BOT_AUDIO_PROFILE=pcm-16k
+CV_BOT_AUDIO_PROFILE=livekit-48k
+CV_BOT_AUDIO_PROFILE=custom CV_BOT_SAMPLE_RATE_HZ=16000 CV_BOT_NUM_CHANNELS=1
+```
+
+`bot_track_ready` is the authoritative runtime signal for the selected profile:
+
+```json
+{
+ "eventType": "bot_track_ready",
+ "eventPayload": {
+ "trackName": "bot-main-audio",
+ "audioProfile": "pcm-16k",
+ "sampleRate": 16000,
+ "numChannels": 1
+ }
+}
+```
+
+Dynamic streaming reply completion must expose both source and target audio metadata:
+
+```json
+{
+ "eventType": "bot_reply_audio_write_finished",
+ "eventPayload": {
+ "format": "pcm_s16le",
+ "sourceSampleRate": 16000,
+ "sourceChannels": 1,
+ "targetAudioProfile": "pcm-16k",
+ "targetSampleRate": 16000,
+ "targetChannels": 1,
+ "networkChunkCount": 68,
+ "debugSourcePath": "/tmp/.../stream-reply-turn-0001-source.wav",
+ "debugPcmWavPath": "/tmp/.../stream-reply-turn-0001-target.wav"
+ }
+}
+```
+
+This profile affects bot output only. User audio observation, VAD and turn artifacts remain `48000Hz / mono` in this phase.
+
+The current acceptance requires both `stream-reply-{turnId}-source.wav` and `stream-reply-{turnId}-target.wav` to be `16000Hz / mono / pcm_s16le`. If the target dump is `48000Hz`, the dynamic reply 16k chain is not complete.
+
## Service mode endpoints
All session endpoints require:
diff --git a/fixtures/turn-stream-happy.ndjson b/fixtures/turn-stream-happy.ndjson
index b5f48b3..d7e7c4a 100644
--- a/fixtures/turn-stream-happy.ndjson
+++ b/fixtures/turn-stream-happy.ndjson
@@ -1,5 +1,5 @@
{"event":"reply_playback_mode_selected","callId":"call_fixture","traceId":"trace_fixture","turnId":"turn_fixture_001","seq":1,"replyPlaybackMode":"streaming_tts"}
{"event":"reply_state","callId":"call_fixture","traceId":"trace_fixture","turnId":"turn_fixture_001","seq":2,"replyPlaybackMode":"streaming_tts","state":"reply_output_pending"}
{"event":"reply_state","callId":"call_fixture","traceId":"trace_fixture","turnId":"turn_fixture_001","seq":3,"replyPlaybackMode":"streaming_tts","state":"reply_playback_started"}
-{"event":"reply_audio_chunk","callId":"call_fixture","traceId":"trace_fixture","turnId":"turn_fixture_001","seq":4,"replyPlaybackMode":"streaming_tts","audioChunk":{"chunkSeq":1,"format":"pcm_s16le","sampleRate":48000,"channels":1,"payloadBase64":"AAAAAAAAAAA=","last":true}}
+{"event":"reply_audio_chunk","callId":"call_fixture","traceId":"trace_fixture","turnId":"turn_fixture_001","seq":4,"replyPlaybackMode":"streaming_tts","audioChunk":{"chunkSeq":1,"format":"pcm_s16le","sampleRate":16000,"channels":1,"payloadBase64":"AAAAAAAAAAA=","last":true}}
{"event":"turn_completed","callId":"call_fixture","traceId":"trace_fixture","turnId":"turn_fixture_001","seq":5,"replyPlaybackMode":"streaming_tts","completion":{"messageId":"msg_fixture_alias","audioChunkCount":1}}
diff --git a/src/audio.rs b/src/audio.rs
index 34415a7..46613eb 100644
--- a/src/audio.rs
+++ b/src/audio.rs
@@ -1040,4 +1040,36 @@
assert_eq!(960, second.frames[0].samples_per_channel);
assert_eq!(0, second.buffered_source_samples);
}
+
+ #[test]
+ fn pcm_s16le_stream_decoder_can_keep_16k_native_audio_profile() {
+ let mut decoder = PcmS16leStreamDecoder::new(16_000, 1, 16_000, 1).expect("stream decoder");
+ let mut pcm = Vec::new();
+ for index in 0..1_600 {
+ let sample = if index % 2 == 0 { 768_i16 } else { -768_i16 };
+ pcm.extend_from_slice(&sample.to_le_bytes());
+ }
+
+ let mut frames = Vec::new();
+ let chunk_size = 250 * 2;
+ for (index, chunk) in pcm.chunks(chunk_size).enumerate() {
+ let last = (index + 1) * chunk_size >= pcm.len();
+ let result = decoder
+ .push_bytes(chunk, 16_000, 1, last)
+ .expect("push pcm chunk");
+ frames.extend(result.frames);
+ }
+
+ assert_eq!(5, frames.len());
+ assert!(frames.iter().all(|frame| frame.sample_rate == 16_000));
+ assert!(frames.iter().all(|frame| frame.num_channels == 1));
+ assert!(frames.iter().all(|frame| frame.samples_per_channel == 320));
+ assert_eq!(
+ 1_600,
+ frames
+ .iter()
+ .map(|frame| frame.samples.len())
+ .sum::<usize>()
+ );
+ }
}
diff --git a/src/main.rs b/src/main.rs
index 64eedd4..cf7cb4f 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -34,8 +34,12 @@
use tokio::{sync::mpsc::UnboundedReceiver, task::JoinHandle};
use tracing::{info, warn};
-const TARGET_SAMPLE_RATE_HZ: u32 = 48_000;
-const TARGET_NUM_CHANNELS: u16 = 1;
+const USER_AUDIO_SAMPLE_RATE_HZ: u32 = 48_000;
+const USER_AUDIO_NUM_CHANNELS: u16 = 1;
+const DEFAULT_BOT_AUDIO_PROFILE: &str = "pcm-16k";
+const LIVEKIT_48K_SAMPLE_RATE_HZ: u32 = 48_000;
+const PCM_16K_SAMPLE_RATE_HZ: u32 = 16_000;
+const BOT_NUM_CHANNELS: u16 = 1;
const TRACK_NAME: &str = "bot-main-audio";
#[tokio::main(flavor = "multi_thread")]
@@ -58,8 +62,8 @@
&http,
config.greeting_audio_file.as_deref(),
config.greeting_audio_url.as_deref(),
- TARGET_SAMPLE_RATE_HZ,
- TARGET_NUM_CHANNELS,
+ config.bot_audio_profile.sample_rate_hz,
+ config.bot_audio_profile.num_channels,
config.audio_debug_dump_dir.as_deref(),
&config.call_id,
"greeting",
@@ -81,6 +85,9 @@
room_alias = %redact(&config.room_id),
participant_alias = %redact(&config.bot_participant_identity),
greeting_source = %config.greeting_source,
+ bot_audio_profile = %config.bot_audio_profile.profile,
+ bot_sample_rate_hz = config.bot_audio_profile.sample_rate_hz,
+ bot_num_channels = config.bot_audio_profile.num_channels,
"combrabo voice runtime helper connected"
);
emit_activity(
@@ -129,8 +136,9 @@
&config.call_id,
&config.trace_id,
TRACK_NAME,
- TARGET_SAMPLE_RATE_HZ,
- u32::from(TARGET_NUM_CHANNELS),
+ config.bot_audio_profile.profile.clone(),
+ config.bot_audio_profile.sample_rate_hz,
+ u32::from(config.bot_audio_profile.num_channels),
config.user_participant_identity.clone(),
)
.await?;
@@ -269,6 +277,54 @@
simple_vad_gate_until_greeting_done: bool,
simple_vad_post_greeting_delay_ms: u64,
simple_vad_config: SimpleVadConfig,
+ bot_audio_profile: BotAudioProfile,
+}
+
+#[derive(Clone)]
+struct BotAudioProfile {
+ profile: String,
+ sample_rate_hz: u32,
+ num_channels: u16,
+}
+
+impl BotAudioProfile {
+ fn from_env() -> Result<Self> {
+ let profile = env::var("CV_BOT_AUDIO_PROFILE")
+ .unwrap_or_else(|_| DEFAULT_BOT_AUDIO_PROFILE.to_string())
+ .trim()
+ .to_ascii_lowercase();
+ match profile.as_str() {
+ "livekit-48k" | "48k" => Ok(Self {
+ profile: "livekit-48k".to_string(),
+ sample_rate_hz: LIVEKIT_48K_SAMPLE_RATE_HZ,
+ num_channels: BOT_NUM_CHANNELS,
+ }),
+ "pcm-16k" | "16k" => Ok(Self {
+ profile: "pcm-16k".to_string(),
+ sample_rate_hz: PCM_16K_SAMPLE_RATE_HZ,
+ num_channels: BOT_NUM_CHANNELS,
+ }),
+ "custom" => {
+ let sample_rate_hz = u32_env("CV_BOT_SAMPLE_RATE_HZ", LIVEKIT_48K_SAMPLE_RATE_HZ);
+ let num_channels = u16_env("CV_BOT_NUM_CHANNELS", BOT_NUM_CHANNELS);
+ if sample_rate_hz == 0 {
+ return Err(anyhow!("CV_BOT_SAMPLE_RATE_HZ must be positive"));
+ }
+ if num_channels == 0 {
+ return Err(anyhow!("CV_BOT_NUM_CHANNELS must be positive"));
+ }
+ Ok(Self {
+ profile,
+ sample_rate_hz,
+ num_channels,
+ })
+ }
+ _ => Err(anyhow!(
+ "unsupported CV_BOT_AUDIO_PROFILE {}; expected livekit-48k, pcm-16k or custom",
+ profile
+ )),
+ }
+ }
}
#[derive(Clone)]
@@ -346,6 +402,7 @@
simple_vad_gate_until_greeting_done: bool_env("CV_VAD_GATE_UNTIL_GREETING_DONE", true),
simple_vad_post_greeting_delay_ms: u64_env("CV_VAD_POST_GREETING_DELAY_MS", 800),
simple_vad_config: SimpleVadConfig::from_env(),
+ bot_audio_profile: BotAudioProfile::from_env()?,
})
}
}
@@ -489,6 +546,13 @@
env::var(key)
.ok()
.and_then(|value| value.trim().parse::<u32>().ok())
+ .unwrap_or(default_value)
+}
+
+fn u16_env(key: &str, default_value: u16) -> u16 {
+ env::var(key)
+ .ok()
+ .and_then(|value| value.trim().parse::<u16>().ok())
.unwrap_or(default_value)
}
@@ -930,8 +994,8 @@
audio::write_pcm_wav(
&output_path,
&turn.samples,
- TARGET_SAMPLE_RATE_HZ,
- TARGET_NUM_CHANNELS,
+ USER_AUDIO_SAMPLE_RATE_HZ,
+ USER_AUDIO_NUM_CHANNELS,
)?;
let byte_size = fs::metadata(&output_path)
.context("failed to stat turn artifact")?
@@ -959,8 +1023,8 @@
artifact_type: "local_file".to_string(),
path_ref: path_ref.to_string(),
format: "wav".to_string(),
- sample_rate: TARGET_SAMPLE_RATE_HZ,
- channels: u32::from(TARGET_NUM_CHANNELS),
+ sample_rate: USER_AUDIO_SAMPLE_RATE_HZ,
+ channels: u32::from(USER_AUDIO_NUM_CHANNELS),
duration_ms: turn.duration_ms,
byte_size,
},
@@ -1343,16 +1407,14 @@
.trim()
.to_ascii_lowercase();
let frames = if format == "pcm_s16le" {
- let sample_rate = audio_chunk.sample_rate.unwrap_or(TARGET_SAMPLE_RATE_HZ);
- let channels = audio_chunk
- .channels
- .unwrap_or(u32::from(TARGET_NUM_CHANNELS));
+ let sample_rate = audio_chunk.sample_rate.unwrap_or(sink.sample_rate_hz);
+ let channels = audio_chunk.channels.unwrap_or(u32::from(sink.num_channels));
if state.pcm_stream_decoder.is_none() {
state.pcm_stream_decoder = Some(audio::PcmS16leStreamDecoder::new(
sample_rate,
channels,
- TARGET_SAMPLE_RATE_HZ,
- TARGET_NUM_CHANNELS,
+ sink.sample_rate_hz,
+ sink.num_channels,
)?);
}
state.pcm_stream_network_chunk_count =
@@ -1396,8 +1458,8 @@
match audio::decode_audio_bytes_to_frames(
&state.encoded_audio_buffer,
"stream_chunk",
- TARGET_SAMPLE_RATE_HZ,
- TARGET_NUM_CHANNELS,
+ sink.sample_rate_hz,
+ sink.num_channels,
bridge_config.audio_debug_dump_dir.as_deref(),
call_id,
&format!("stream-reply-{}", turn.turn_id),
@@ -1526,8 +1588,11 @@
"debugSourcePath": debug_source_path,
"debugPcmWavPath": debug_pcm_wav_path,
"debugPcmWavSizeBytes": debug_pcm_wav_size_bytes,
- "sampleRate": audio_chunk.sample_rate,
- "channels": audio_chunk.channels,
+ "sourceSampleRate": audio_chunk.sample_rate,
+ "sourceChannels": audio_chunk.channels,
+ "targetAudioProfile": sink.profile.as_str(),
+ "targetSampleRate": sink.sample_rate_hz,
+ "targetChannels": sink.num_channels,
"replyTotalAfterVadEndMs": turn_pipeline_started_at.elapsed().as_millis() as u64,
}),
);
@@ -1566,8 +1631,8 @@
artifact_type: "local_file".to_string(),
path_ref: path_ref.to_string(),
format: "wav".to_string(),
- sample_rate: TARGET_SAMPLE_RATE_HZ,
- channels: u32::from(TARGET_NUM_CHANNELS),
+ sample_rate: USER_AUDIO_SAMPLE_RATE_HZ,
+ channels: u32::from(USER_AUDIO_NUM_CHANNELS),
duration_ms: turn.duration_ms,
byte_size,
},
@@ -1730,8 +1795,8 @@
http,
Some(&audio_path_string),
None,
- TARGET_SAMPLE_RATE_HZ,
- TARGET_NUM_CHANNELS,
+ sink.sample_rate_hz,
+ sink.num_channels,
bridge_config.audio_debug_dump_dir.as_deref(),
call_id,
&reply_debug_label,
@@ -2075,8 +2140,8 @@
tokio::spawn(async move {
let mut stream = NativeAudioStream::new(
track.rtc_track(),
- TARGET_SAMPLE_RATE_HZ as i32,
- i32::from(TARGET_NUM_CHANNELS),
+ USER_AUDIO_SAMPLE_RATE_HZ as i32,
+ i32::from(USER_AUDIO_NUM_CHANNELS),
);
let started_at = Instant::now();
let mut frame_count: u64 = 0;
@@ -2563,6 +2628,9 @@
rtc_source: NativeAudioSource,
track: LocalAudioTrack,
device_output_destination_identity: Option<String>,
+ profile: String,
+ sample_rate_hz: u32,
+ num_channels: u16,
}
impl BotAudioOutputSink {
@@ -2573,6 +2641,7 @@
call_id: &str,
trace_id: &str,
track_name: &str,
+ profile: String,
sample_rate: u32,
num_channels: u32,
device_output_destination_identity: Option<String>,
@@ -2601,11 +2670,14 @@
"failed to publish bot audio track in room {room_alias} for participant {participant_alias}: {error}"
)
})?;
+ let num_channels_u16 = u16::try_from(num_channels)
+ .map_err(|_| anyhow!("unsupported bot audio channel count {num_channels}"))?;
info!(
room_alias = %room_alias,
participant_alias = %participant_alias,
track_name = %track_name,
+ bot_audio_profile = %profile,
sample_rate,
num_channels,
"runtime helper published bot audio track"
@@ -2620,6 +2692,7 @@
None,
json!({
"trackName": track_name,
+ "audioProfile": profile,
"sampleRate": sample_rate,
"numChannels": num_channels,
}),
@@ -2630,6 +2703,9 @@
rtc_source,
track,
device_output_destination_identity,
+ profile,
+ sample_rate_hz: sample_rate,
+ num_channels: num_channels_u16,
})
}
diff --git a/tools/validate-turn-stream-fixture.mjs b/tools/validate-turn-stream-fixture.mjs
index 710365f..bcdd77f 100755
--- a/tools/validate-turn-stream-fixture.mjs
+++ b/tools/validate-turn-stream-fixture.mjs
@@ -30,6 +30,7 @@
let traceId = null
let turnId = null
const supportedAudioFormats = new Set(['pcm_s16le', 'mp3', 'mpeg', 'wav'])
+const supportedPcmSampleRates = new Set([16000, 48000])
function fail(message) {
console.error(`turn stream fixture invalid: ${message}`)
@@ -100,8 +101,8 @@
fail(`${chunk.format} payload must be non-empty`)
}
if (chunk.format === 'pcm_s16le') {
- if (chunk.sampleRate !== 48000 || chunk.channels !== 1) {
- fail('pcm_s16le chunk must be 48000Hz mono')
+ if (!supportedPcmSampleRates.has(chunk.sampleRate) || chunk.channels !== 1) {
+ fail('pcm_s16le chunk must be 16000Hz or 48000Hz mono')
}
if (payload.length % 2 !== 0) {
fail('pcm_s16le payload must be 16-bit aligned')
--
Gitblit v1.9.3