From b0a4e2ea93fafc4d65d474f3ada616a0ad5e19d5 Mon Sep 17 00:00:00 2001
From: Ariver <ar@Arm1.local>
Date: Sun, 12 Jul 2026 11:39:35 +0800
Subject: [PATCH] feat: bind helper stream timing markers
---
src/main.rs | 1424 ++++++++++++++++++++++++++++++++++++++++++++++++++++------
1 files changed, 1,263 insertions(+), 161 deletions(-)
diff --git a/src/main.rs b/src/main.rs
index 3b18943..611993a 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -1,7 +1,10 @@
+mod asr_realtime;
mod audio;
mod service;
use std::{
+ borrow::Cow,
+ collections::HashSet,
env, fs,
path::{Path, PathBuf},
sync::{
@@ -12,7 +15,8 @@
};
use anyhow::{Context, Result, anyhow};
-use audio::{AudioDiagnostics, PcmFrame, load_pre_recorded_frames};
+use asr_realtime::{RealtimeAsrConfig, RealtimeAsrOutcome, RealtimeAsrUpload};
+use audio::{AudioDiagnostics, load_pre_recorded_frames};
use base64::{Engine as _, engine::general_purpose};
use futures_util::StreamExt;
use libwebrtc::{
@@ -30,13 +34,29 @@
use reqwest::Client;
use serde::{Deserialize, Serialize};
use serde_json::json;
-use tokio::time::{sleep, sleep_until};
-use tokio::{sync::mpsc::UnboundedReceiver, task::JoinHandle};
+use sha2::{Digest, Sha256};
+use tokio::time::{sleep, sleep_until, timeout};
+use tokio::{
+ sync::{mpsc, mpsc::UnboundedReceiver, watch},
+ task::JoinHandle,
+};
use tracing::{info, warn};
-const TARGET_SAMPLE_RATE_HZ: u32 = 48_000;
-const TARGET_NUM_CHANNELS: u16 = 1;
+const USER_AUDIO_SAMPLE_RATE_HZ: u32 = 48_000;
+const USER_AUDIO_NUM_CHANNELS: u16 = 1;
+const INBOUND_AUDIO_QUEUE_CAPACITY: usize = 100;
+const INBOUND_AUDIO_DROP_LOG_INTERVAL: u64 = 100;
+const AUDIO_DRAIN_STOP_GRACE: Duration = Duration::from_secs(2);
+const DEFAULT_BOT_AUDIO_PROFILE: &str = "pcm-16k";
+const LIVEKIT_48K_SAMPLE_RATE_HZ: u32 = 48_000;
+const PCM_16K_SAMPLE_RATE_HZ: u32 = 16_000;
+const BOT_NUM_CHANNELS: u16 = 1;
const TRACK_NAME: &str = "bot-main-audio";
+const STREAM_TIMING_VERSION: u32 = 1;
+const STREAM_TIMING_FIRST_SEGMENT: u64 = 1;
+const STREAM_TIMING_FIRST_CHUNK: u64 = 1;
+const STREAM_TIMING_MAX_ELAPSED_MS: u64 = 5_000;
+const STREAM_TIMING_SOURCE: &str = "stream_anchor_monotonic";
#[tokio::main(flavor = "multi_thread")]
async fn main() -> Result<()> {
@@ -58,8 +78,8 @@
&http,
config.greeting_audio_file.as_deref(),
config.greeting_audio_url.as_deref(),
- TARGET_SAMPLE_RATE_HZ,
- TARGET_NUM_CHANNELS,
+ config.bot_audio_profile.sample_rate_hz,
+ config.bot_audio_profile.num_channels,
config.audio_debug_dump_dir.as_deref(),
&config.call_id,
"greeting",
@@ -81,6 +101,9 @@
room_alias = %redact(&config.room_id),
participant_alias = %redact(&config.bot_participant_identity),
greeting_source = %config.greeting_source,
+ bot_audio_profile = %config.bot_audio_profile.profile,
+ bot_sample_rate_hz = config.bot_audio_profile.sample_rate_hz,
+ bot_num_channels = config.bot_audio_profile.num_channels,
"combrabo voice runtime helper connected"
);
emit_activity(
@@ -129,8 +152,9 @@
&config.call_id,
&config.trace_id,
TRACK_NAME,
- TARGET_SAMPLE_RATE_HZ,
- u32::from(TARGET_NUM_CHANNELS),
+ config.bot_audio_profile.profile.clone(),
+ config.bot_audio_profile.sample_rate_hz,
+ u32::from(config.bot_audio_profile.num_channels),
config.user_participant_identity.clone(),
)
.await?;
@@ -262,6 +286,11 @@
runtime_turn_bridge_url: Option<String>,
runtime_turn_bridge_token: Option<String>,
runtime_turn_bridge_mode: String,
+ runtime_asr_stream_enabled: bool,
+ runtime_asr_stream_url: Option<String>,
+ runtime_asr_realtime_enabled: bool,
+ runtime_asr_realtime_url: Option<String>,
+ runtime_asr_realtime_chunk_duration_ms: u64,
runtime_turn_artifact_dir: Option<String>,
runtime_session_nonce: Option<String>,
user_audio_observer_enabled: bool,
@@ -269,6 +298,54 @@
simple_vad_gate_until_greeting_done: bool,
simple_vad_post_greeting_delay_ms: u64,
simple_vad_config: SimpleVadConfig,
+ bot_audio_profile: BotAudioProfile,
+}
+
+#[derive(Clone)]
+struct BotAudioProfile {
+ profile: String,
+ sample_rate_hz: u32,
+ num_channels: u16,
+}
+
+impl BotAudioProfile {
+ fn from_env() -> Result<Self> {
+ let profile = env::var("CV_BOT_AUDIO_PROFILE")
+ .unwrap_or_else(|_| DEFAULT_BOT_AUDIO_PROFILE.to_string())
+ .trim()
+ .to_ascii_lowercase();
+ match profile.as_str() {
+ "livekit-48k" | "48k" => Ok(Self {
+ profile: "livekit-48k".to_string(),
+ sample_rate_hz: LIVEKIT_48K_SAMPLE_RATE_HZ,
+ num_channels: BOT_NUM_CHANNELS,
+ }),
+ "pcm-16k" | "16k" => Ok(Self {
+ profile: "pcm-16k".to_string(),
+ sample_rate_hz: PCM_16K_SAMPLE_RATE_HZ,
+ num_channels: BOT_NUM_CHANNELS,
+ }),
+ "custom" => {
+ let sample_rate_hz = u32_env("CV_BOT_SAMPLE_RATE_HZ", LIVEKIT_48K_SAMPLE_RATE_HZ);
+ let num_channels = u16_env("CV_BOT_NUM_CHANNELS", BOT_NUM_CHANNELS);
+ if sample_rate_hz == 0 {
+ return Err(anyhow!("CV_BOT_SAMPLE_RATE_HZ must be positive"));
+ }
+ if num_channels == 0 {
+ return Err(anyhow!("CV_BOT_NUM_CHANNELS must be positive"));
+ }
+ Ok(Self {
+ profile,
+ sample_rate_hz,
+ num_channels,
+ })
+ }
+ _ => Err(anyhow!(
+ "unsupported CV_BOT_AUDIO_PROFILE {}; expected livekit-48k, pcm-16k or custom",
+ profile
+ )),
+ }
+ }
}
#[derive(Clone)]
@@ -276,6 +353,11 @@
bridge_url: Option<String>,
bridge_token: Option<String>,
bridge_mode: String,
+ asr_stream_enabled: bool,
+ asr_stream_url: Option<String>,
+ asr_realtime_enabled: bool,
+ asr_realtime_url: Option<String>,
+ asr_realtime_chunk_duration_ms: u64,
artifact_dir: Option<String>,
runtime_session_nonce: Option<String>,
audio_debug_dump_dir: Option<String>,
@@ -287,6 +369,11 @@
bridge_url: config.runtime_turn_bridge_url.clone(),
bridge_token: config.runtime_turn_bridge_token.clone(),
bridge_mode: config.runtime_turn_bridge_mode.clone(),
+ asr_stream_enabled: config.runtime_asr_stream_enabled,
+ asr_stream_url: config.runtime_asr_stream_url.clone(),
+ asr_realtime_enabled: config.runtime_asr_realtime_enabled,
+ asr_realtime_url: config.runtime_asr_realtime_url.clone(),
+ asr_realtime_chunk_duration_ms: config.runtime_asr_realtime_chunk_duration_ms,
artifact_dir: config.runtime_turn_artifact_dir.clone(),
runtime_session_nonce: config.runtime_session_nonce.clone(),
audio_debug_dump_dir: config.audio_debug_dump_dir.clone(),
@@ -318,6 +405,32 @@
.as_deref()
.is_some_and(|value| value.trim_end_matches('/').ends_with("/stream"))
}
+
+ fn is_asr_stream_ready(&self) -> bool {
+ self.asr_stream_enabled
+ && self
+ .asr_stream_url
+ .as_ref()
+ .is_some_and(|value| !value.is_empty())
+ && self
+ .bridge_token
+ .as_ref()
+ .is_some_and(|value| !value.is_empty())
+ && self
+ .runtime_session_nonce
+ .as_ref()
+ .is_some_and(|value| !value.is_empty())
+ }
+
+ fn realtime_asr_config(&self) -> RealtimeAsrConfig {
+ RealtimeAsrConfig {
+ enabled: self.asr_realtime_enabled,
+ url: self.asr_realtime_url.clone(),
+ runtime_token: self.bridge_token.clone(),
+ runtime_session_nonce: self.runtime_session_nonce.clone(),
+ chunk_duration_ms: self.asr_realtime_chunk_duration_ms,
+ }
+ }
}
impl Config {
@@ -339,6 +452,14 @@
runtime_turn_bridge_token: optional_env("CV_RUNTIME_TURN_BRIDGE_TOKEN"),
runtime_turn_bridge_mode: env::var("CV_RUNTIME_TURN_BRIDGE_MODE")
.unwrap_or_else(|_| "json".to_string()),
+ runtime_asr_stream_enabled: bool_env("CV_RUNTIME_ASR_STREAM_ENABLED", false),
+ runtime_asr_stream_url: optional_env("CV_RUNTIME_ASR_STREAM_URL"),
+ runtime_asr_realtime_enabled: bool_env("CV_RUNTIME_ASR_REALTIME_ENABLED", false),
+ runtime_asr_realtime_url: optional_env("CV_RUNTIME_ASR_REALTIME_URL"),
+ runtime_asr_realtime_chunk_duration_ms: u64_env(
+ "CV_RUNTIME_ASR_REALTIME_CHUNK_DURATION_MS",
+ 200,
+ ),
runtime_turn_artifact_dir: optional_env("CV_RUNTIME_TURN_ARTIFACT_DIR"),
runtime_session_nonce: optional_env("CV_RUNTIME_SESSION_NONCE"),
user_audio_observer_enabled: bool_env("CV_ENABLE_USER_AUDIO_OBSERVER", true),
@@ -346,6 +467,7 @@
simple_vad_gate_until_greeting_done: bool_env("CV_VAD_GATE_UNTIL_GREETING_DONE", true),
simple_vad_post_greeting_delay_ms: u64_env("CV_VAD_POST_GREETING_DELAY_MS", 800),
simple_vad_config: SimpleVadConfig::from_env(),
+ bot_audio_profile: BotAudioProfile::from_env()?,
})
}
}
@@ -492,6 +614,13 @@
.unwrap_or(default_value)
}
+fn u16_env(key: &str, default_value: u16) -> u16 {
+ env::var(key)
+ .ok()
+ .and_then(|value| value.trim().parse::<u16>().ok())
+ .unwrap_or(default_value)
+}
+
fn redact(value: &str) -> String {
if value.len() <= 8 {
return "redacted".to_string();
@@ -532,10 +661,36 @@
"traceId": trace_id,
"turnId": turn_id,
"eventName": event_name,
+ "eventWallTimeMs": current_time_millis(),
"result": result,
"reasonCode": reason_code,
"retryable": retryable,
"extension": extension,
+ });
+ println!("{payload}");
+}
+
+fn emit_anchored_activity(
+ call_id: &str,
+ trace_id: &str,
+ turn_id: &str,
+ event_name: &str,
+ marker: &RuntimeTurnStreamTimingMarker,
+ extension: serde_json::Value,
+) {
+ let payload = json!({
+ "type": "cv_activity",
+ "callId": call_id,
+ "traceId": trace_id,
+ "turnId": turn_id,
+ "eventName": event_name,
+ "eventWallTimeMs": current_time_millis(),
+ "serverDeltaMs": marker.server_delta_ms,
+ "serverDeltaSource": STREAM_TIMING_SOURCE,
+ "result": "ok",
+ "reasonCode": null,
+ "retryable": null,
+ "extension": marker.extension_with(extension),
});
println!("{payload}");
}
@@ -686,6 +841,7 @@
call_id: &str,
trace_id: &str,
turn: FinishedSpeechTurn,
+ realtime_asr_result_ref: Option<String>,
) {
let turn_pipeline_started_at = Instant::now();
if !bridge_config.is_ready() {
@@ -742,9 +898,13 @@
"turnArtifactBytes": byte_size,
"frameCount": turn.frame_count,
"sampleCount": turn.sample_count,
- "endReason": turn.end_reason,
+ "endReason": turn.end_reason.as_str(),
}),
);
+ let asr_result_ref = match realtime_asr_result_ref {
+ Some(value) => Some(value),
+ None => request_asr_result_ref(http, bridge_config, call_id, trace_id, &turn).await,
+ };
if bridge_config.is_stream_mode() {
match request_turn_bridge_stream(
http,
@@ -755,6 +915,7 @@
&turn,
&path_ref,
byte_size,
+ asr_result_ref.as_deref(),
turn_pipeline_started_at,
)
.await
@@ -818,6 +979,7 @@
&turn,
&path_ref,
byte_size,
+ asr_result_ref.as_deref(),
turn_pipeline_started_at,
)
.await
@@ -930,13 +1092,258 @@
audio::write_pcm_wav(
&output_path,
&turn.samples,
- TARGET_SAMPLE_RATE_HZ,
- TARGET_NUM_CHANNELS,
+ USER_AUDIO_SAMPLE_RATE_HZ,
+ USER_AUDIO_NUM_CHANNELS,
)?;
let byte_size = fs::metadata(&output_path)
.context("failed to stat turn artifact")?
.len();
Ok((path_ref, byte_size))
+}
+
+async fn request_asr_result_ref(
+ http: &Client,
+ bridge_config: &TurnBridgeConfig,
+ call_id: &str,
+ trace_id: &str,
+ turn: &FinishedSpeechTurn,
+) -> Option<String> {
+ if !bridge_config.is_asr_stream_ready() {
+ return None;
+ }
+ match request_asr_stream(http, bridge_config, call_id, trace_id, turn).await {
+ Ok(Some(asr_result_ref)) => {
+ emit_activity(
+ call_id,
+ trace_id,
+ Some(&turn.turn_id),
+ "asr_stream_ref_ready",
+ "ok",
+ None,
+ None,
+ json!({
+ "asrResultRefPresent": true,
+ "format": "pcm_s16le",
+ "sampleRate": 16000,
+ "channels": 1,
+ }),
+ );
+ Some(asr_result_ref)
+ }
+ Ok(None) => None,
+ Err(error) => {
+ warn!(
+ call_id = %call_id,
+ trace_id = %trace_id,
+ turn_id = %turn.turn_id,
+ error = %safe_error(&error.to_string()),
+ "runtime helper asr_stream_failed_fallback"
+ );
+ emit_activity(
+ call_id,
+ trace_id,
+ Some(&turn.turn_id),
+ "asr_stream_fallback",
+ "skipped",
+ Some("ASR_STREAM_INTERRUPTED"),
+ Some(true),
+ json!({
+ "fallbackReason": "asr_stream_request_failed",
+ "fallbackStage": "asr_stream",
+ }),
+ );
+ None
+ }
+ }
+}
+
+async fn request_asr_stream(
+ http: &Client,
+ bridge_config: &TurnBridgeConfig,
+ call_id: &str,
+ trace_id: &str,
+ turn: &FinishedSpeechTurn,
+) -> Result<Option<String>> {
+ let started_at = Instant::now();
+ let ndjson = build_asr_stream_ndjson(call_id, trace_id, turn, bridge_config)?;
+ let response = http
+ .post(bridge_config.asr_stream_url.as_deref().unwrap_or_default())
+ .header("Content-Type", "application/x-ndjson")
+ .header(
+ "X-CV-Runtime-Token",
+ bridge_config.bridge_token.as_deref().unwrap_or_default(),
+ )
+ .header("X-CV-Call-Id", call_id)
+ .header("X-CV-Trace-Id", trace_id)
+ .header(
+ "X-CV-Runtime-Session-Nonce",
+ bridge_config
+ .runtime_session_nonce
+ .as_deref()
+ .unwrap_or_default(),
+ )
+ .body(ndjson)
+ .send()
+ .await
+ .context("failed to post asr stream")?;
+ let status = response.status();
+ if !status.is_success() {
+ let body_len = response.text().await.map(|body| body.len()).unwrap_or(0);
+ warn!(
+ call_id = %call_id,
+ trace_id = %trace_id,
+ turn_id = %turn.turn_id,
+ http_status = status.as_u16(),
+ body_len,
+ "runtime helper asr_stream_http_failed"
+ );
+ return Ok(None);
+ }
+ let body: RuntimeTurnCommonResult<RuntimeAsrStreamResp> = response
+ .json()
+ .await
+ .context("failed to decode asr stream response")?;
+ if body.code != 0 {
+ warn!(
+ call_id = %call_id,
+ trace_id = %trace_id,
+ turn_id = %turn.turn_id,
+ code = body.code,
+ msg_len = body.msg.as_deref().unwrap_or_default().len(),
+ "runtime helper asr_stream_common_result_failed"
+ );
+ return Ok(None);
+ }
+ let Some(data) = body.data else {
+ return Ok(None);
+ };
+ if data.status.as_deref() == Some("final") {
+ info!(
+ call_id = %call_id,
+ trace_id = %trace_id,
+ turn_id = %turn.turn_id,
+ chunk_count = data.chunk_count.unwrap_or_default(),
+ audio_bytes = data.audio_bytes.unwrap_or_default(),
+ asr_duration_ms = data.asr_duration_ms.unwrap_or_default(),
+ wall_ms = started_at.elapsed().as_millis() as u64,
+ provider = %data.provider_alias.as_deref().unwrap_or("unknown"),
+ text_len = data.text_len.unwrap_or_default(),
+ "runtime helper asr_stream_final"
+ );
+ return Ok(data.asr_result_ref);
+ }
+ emit_activity(
+ call_id,
+ trace_id,
+ Some(&turn.turn_id),
+ "asr_stream_fallback",
+ "skipped",
+ None,
+ Some(true),
+ json!({
+ "fallbackReason": data.fallback_reason,
+ "fallbackStage": data.fallback_stage,
+ "status": data.status,
+ }),
+ );
+ Ok(None)
+}
+
+fn build_asr_stream_ndjson(
+ call_id: &str,
+ trace_id: &str,
+ turn: &FinishedSpeechTurn,
+ bridge_config: &TurnBridgeConfig,
+) -> Result<String> {
+ let chunks = asr_pcm_16k_chunks(turn)?;
+ let mut seq = 1u64;
+ let mut lines = Vec::with_capacity(chunks.len() + 3);
+ lines.push(serde_json::to_string(&json!({
+ "event": "asr_stream_started",
+ "seq": seq,
+ "callId": call_id,
+ "traceId": trace_id,
+ "turnId": turn.turn_id.as_str(),
+ "tsMs": current_time_millis(),
+ "payload": {
+ "format": "pcm_s16le",
+ "sampleRate": 16000,
+ "channels": 1,
+ "runtimeSessionNonce": bridge_config.runtime_session_nonce.as_deref().unwrap_or_default(),
+ "providerHint": "volcengine",
+ }
+ }))?);
+ for (index, samples) in chunks.iter().enumerate() {
+ seq += 1;
+ let bytes = pcm_i16_to_le_bytes(samples);
+ let duration_ms = ((samples.len() as u64) * 1000 / 16_000).max(1);
+ lines.push(serde_json::to_string(&json!({
+ "event": "asr_audio_chunk",
+ "seq": seq,
+ "callId": call_id,
+ "traceId": trace_id,
+ "turnId": turn.turn_id.as_str(),
+ "tsMs": current_time_millis(),
+ "payload": {
+ "chunkSeq": index + 1,
+ "format": "pcm_s16le",
+ "sampleRate": 16000,
+ "channels": 1,
+ "durationMs": duration_ms,
+ "payloadBase64": general_purpose::STANDARD.encode(bytes),
+ }
+ }))?);
+ }
+ seq += 1;
+ lines.push(serde_json::to_string(&json!({
+ "event": "vad_speech_end",
+ "seq": seq,
+ "callId": call_id,
+ "traceId": trace_id,
+ "turnId": turn.turn_id.as_str(),
+ "tsMs": current_time_millis(),
+ "payload": {
+ "endReason": turn.end_reason.as_str(),
+ "speechDurationMs": turn.duration_ms,
+ }
+ }))?);
+ seq += 1;
+ lines.push(serde_json::to_string(&json!({
+ "event": "asr_stream_finish",
+ "seq": seq,
+ "callId": call_id,
+ "traceId": trace_id,
+ "turnId": turn.turn_id.as_str(),
+ "tsMs": current_time_millis(),
+ "payload": {
+ "finalChunkSeq": chunks.len(),
+ "audioDurationMs": turn.duration_ms,
+ }
+ }))?);
+ Ok(lines.join("\n") + "\n")
+}
+
+fn asr_pcm_16k_chunks(turn: &FinishedSpeechTurn) -> Result<Vec<Vec<i16>>> {
+ if turn.samples.is_empty() {
+ return Err(anyhow!("empty turn samples"));
+ }
+ let samples_16k: Vec<i16> = turn.samples.iter().step_by(3).copied().collect();
+ if samples_16k.is_empty() {
+ return Err(anyhow!("empty 16k asr samples"));
+ }
+ let samples_per_chunk = 320usize;
+ Ok(samples_16k
+ .chunks(samples_per_chunk)
+ .map(|chunk| chunk.to_vec())
+ .collect())
+}
+
+fn pcm_i16_to_le_bytes(samples: &[i16]) -> Vec<u8> {
+ let mut bytes = Vec::with_capacity(samples.len() * 2);
+ for sample in samples {
+ bytes.extend_from_slice(&sample.to_le_bytes());
+ }
+ bytes
}
async fn request_turn_bridge_stream(
@@ -948,6 +1355,7 @@
turn: &FinishedSpeechTurn,
path_ref: &str,
byte_size: u64,
+ asr_result_ref: Option<&str>,
turn_pipeline_started_at: Instant,
) -> Result<RuntimeTurnStreamOutcome> {
let bridge_started_at = Instant::now();
@@ -955,15 +1363,16 @@
call_id: call_id.to_string(),
trace_id: trace_id.to_string(),
turn_id: turn.turn_id.clone(),
- audio_artifact: RuntimeTurnAudioArtifact {
+ audio_artifact: Some(RuntimeTurnAudioArtifact {
artifact_type: "local_file".to_string(),
path_ref: path_ref.to_string(),
format: "wav".to_string(),
- sample_rate: TARGET_SAMPLE_RATE_HZ,
- channels: u32::from(TARGET_NUM_CHANNELS),
+ sample_rate: USER_AUDIO_SAMPLE_RATE_HZ,
+ channels: u32::from(USER_AUDIO_NUM_CHANNELS),
duration_ms: turn.duration_ms,
byte_size,
- },
+ }),
+ asr_result_ref: asr_result_ref.map(str::to_string),
};
let response = http
.post(bridge_config.bridge_url.as_deref().unwrap_or_default())
@@ -1042,6 +1451,7 @@
.await?;
}
}
+ state.close_timing();
if !state.completed {
warn!(
call_id = %call_id,
@@ -1171,6 +1581,7 @@
call_id,
trace_id,
turn,
+ &event,
audio_chunk,
state,
turn_pipeline_started_at,
@@ -1189,6 +1600,7 @@
}
Some("turn_completed") => {
state.completed = true;
+ state.close_timing();
info!(
call_id = %call_id,
trace_id = %trace_id,
@@ -1199,6 +1611,7 @@
);
}
Some("turn_failed") => {
+ state.close_timing();
let error = event.error.as_ref();
let reason_code = error
.and_then(|value| value.reason_code.as_deref())
@@ -1233,6 +1646,7 @@
}
Some("turn_cancelled") => {
state.completed = true;
+ state.close_timing();
info!(
call_id = %call_id,
trace_id = %trace_id,
@@ -1265,6 +1679,19 @@
}
Some("activity") => {
if let Some(activity) = event.activity.as_ref() {
+ if activity.event_type.as_deref() == Some("tts_first_audio_chunk_ready") {
+ if let Some(runtime_session_nonce) =
+ bridge_config.runtime_session_nonce.as_deref()
+ {
+ let _ = state.arm_timing_anchor(
+ call_id,
+ trace_id,
+ &turn.turn_id,
+ runtime_session_nonce,
+ &event,
+ );
+ }
+ }
info!(
call_id = %call_id,
trace_id = %trace_id,
@@ -1323,6 +1750,7 @@
call_id: &str,
trace_id: &str,
turn: &FinishedSpeechTurn,
+ event: &RuntimeTurnStreamEvent,
audio_chunk: &RuntimeTurnStreamAudioChunk,
state: &mut RuntimeTurnStreamState,
turn_pipeline_started_at: Instant,
@@ -1342,53 +1770,112 @@
.unwrap_or("pcm_s16le")
.trim()
.to_ascii_lowercase();
+ if !matches!(format.as_str(), "pcm_s16le" | "mp3" | "mpeg" | "wav") {
+ return Err(anyhow!("unsupported reply_audio_chunk format {format}"));
+ }
+ match state.reply_chunk_markers.observe(audio_chunk.segment_seq) {
+ ReplyChunkMarker::FirstReply => {
+ let extension = json!({
+ "segmentSeq": audio_chunk.segment_seq,
+ "chunkSeq": audio_chunk.chunk_seq,
+ "format": format.as_str(),
+ "bytes": payload.len(),
+ });
+ if let Some(marker) =
+ state.record_m6(call_id, trace_id, &turn.turn_id, event, audio_chunk)
+ {
+ emit_anchored_activity(
+ call_id,
+ trace_id,
+ &turn.turn_id,
+ "helper_first_reply_audio_chunk_received",
+ &marker,
+ extension,
+ );
+ } else {
+ emit_activity(
+ call_id,
+ trace_id,
+ Some(&turn.turn_id),
+ "helper_first_reply_audio_chunk_received",
+ "ok",
+ None,
+ None,
+ extension,
+ );
+ }
+ }
+ ReplyChunkMarker::SegmentFirst => emit_activity(
+ call_id,
+ trace_id,
+ Some(&turn.turn_id),
+ "helper_segment_first_audio_chunk_received",
+ "ok",
+ None,
+ None,
+ json!({
+ "segmentSeq": audio_chunk.segment_seq,
+ "chunkSeq": audio_chunk.chunk_seq,
+ "format": format.as_str(),
+ "bytes": payload.len(),
+ }),
+ ),
+ ReplyChunkMarker::None => {}
+ }
let frames = if format == "pcm_s16le" {
- let frame_alignment_bytes = pcm_s16le_frame_alignment_bytes(
- audio_chunk
- .channels
- .unwrap_or(u32::from(TARGET_NUM_CHANNELS)),
- )?;
- let (aligned_payload, dropped_tail_bytes) = take_aligned_pcm_payload(
- &mut state.pcm_audio_buffer,
- &payload,
- frame_alignment_bytes,
- audio_chunk.last.unwrap_or(false),
- );
- if dropped_tail_bytes > 0 {
+ let sample_rate = audio_chunk.sample_rate.unwrap_or(sink.sample_rate_hz);
+ let channels = audio_chunk.channels.unwrap_or(u32::from(sink.num_channels));
+ if state.pcm_stream_decoder.is_none() {
+ state.pcm_stream_decoder = Some(audio::PcmS16leStreamDecoder::new(
+ sample_rate,
+ channels,
+ sink.sample_rate_hz,
+ sink.num_channels,
+ )?);
+ }
+ state.pcm_stream_network_chunk_count =
+ state.pcm_stream_network_chunk_count.saturating_add(1);
+ let stream_result = state
+ .pcm_stream_decoder
+ .as_mut()
+ .expect("pcm stream decoder initialized")
+ .push_bytes(
+ &payload,
+ sample_rate,
+ channels,
+ audio_chunk.last.unwrap_or(false),
+ )?;
+ if stream_result.dropped_tail_bytes > 0 {
warn!(
call_id = %call_id,
trace_id = %trace_id,
turn_id = %turn.turn_id,
chunk_seq = audio_chunk.chunk_seq.unwrap_or_default(),
- dropped_tail_bytes,
+ dropped_tail_bytes = stream_result.dropped_tail_bytes,
"runtime helper stream_audio_pcm_unaligned_tail_dropped"
);
}
- if aligned_payload.is_empty() {
+ if stream_result.frames.is_empty() {
warn!(
call_id = %call_id,
trace_id = %trace_id,
turn_id = %turn.turn_id,
chunk_seq = audio_chunk.chunk_seq.unwrap_or_default(),
- buffered_bytes = state.pcm_audio_buffer.len(),
- "runtime helper stream_audio_pcm_waiting_for_sample_boundary"
+ buffered_source_bytes = stream_result.buffered_source_bytes,
+ buffered_source_samples = stream_result.buffered_source_samples,
+ network_chunk_count = state.pcm_stream_network_chunk_count,
+ "runtime helper stream_audio_pcm_waiting_for_20ms_frame"
);
return Ok(0);
}
- pcm_s16le_payload_to_frames(
- &aligned_payload,
- audio_chunk.sample_rate.unwrap_or(TARGET_SAMPLE_RATE_HZ),
- audio_chunk
- .channels
- .unwrap_or(u32::from(TARGET_NUM_CHANNELS)),
- )?
+ stream_result.frames
} else if matches!(format.as_str(), "mp3" | "mpeg" | "wav") {
state.encoded_audio_buffer.extend_from_slice(&payload);
match audio::decode_audio_bytes_to_frames(
&state.encoded_audio_buffer,
"stream_chunk",
- TARGET_SAMPLE_RATE_HZ,
- TARGET_NUM_CHANNELS,
+ sink.sample_rate_hz,
+ sink.num_channels,
bridge_config.audio_debug_dump_dir.as_deref(),
call_id,
&format!("stream-reply-{}", turn.turn_id),
@@ -1426,7 +1913,7 @@
Err(error) => return Err(error).context("failed to decode final stream audio chunk"),
}
} else {
- return Err(anyhow!("unsupported reply_audio_chunk format {format}"));
+ unreachable!("supported encoded format checked above")
};
if frames.is_empty() {
return Ok(0);
@@ -1454,25 +1941,65 @@
sink.write_pcm_frame(frame).await?;
if !state.first_audio_frame_written {
state.first_audio_frame_written = true;
- emit_activity(
- call_id,
- trace_id,
- Some(&turn.turn_id),
- "bot_reply_first_audio_frame_written",
- "ok",
- None,
- None,
- json!({
- "replyPlaybackMode": state.reply_playback_mode.as_str(),
- "format": format.as_str(),
- "chunkSeq": audio_chunk.chunk_seq,
- "replyTotalAfterVadEndMs": turn_pipeline_started_at.elapsed().as_millis() as u64,
- }),
- );
+ let extension = json!({
+ "replyPlaybackMode": state.reply_playback_mode.as_str(),
+ "format": format.as_str(),
+ "chunkSeq": audio_chunk.chunk_seq,
+ "replyTotalAfterVadEndMs": turn_pipeline_started_at.elapsed().as_millis() as u64,
+ });
+ if let Some(marker) = state.record_m7() {
+ emit_anchored_activity(
+ call_id,
+ trace_id,
+ &turn.turn_id,
+ "bot_reply_first_audio_frame_written",
+ &marker,
+ extension,
+ );
+ } else {
+ emit_activity(
+ call_id,
+ trace_id,
+ Some(&turn.turn_id),
+ "bot_reply_first_audio_frame_written",
+ "ok",
+ None,
+ None,
+ extension,
+ );
+ }
}
sleep_until(pacing_started_at + Duration::from_millis(((index + 1) as u64) * 20)).await;
}
if audio_chunk.last.unwrap_or(false) {
+ let mut debug_source_path = None;
+ let mut debug_pcm_wav_path = None;
+ let mut debug_pcm_wav_size_bytes = None;
+ if format == "pcm_s16le" {
+ if let (Some(debug_dump_dir), Some(decoder)) = (
+ bridge_config.audio_debug_dump_dir.as_deref(),
+ state.pcm_stream_decoder.as_ref(),
+ ) {
+ match decoder.write_debug_dump(
+ debug_dump_dir,
+ call_id,
+ &format!("stream-reply-{}", turn.turn_id),
+ ) {
+ Ok(debug_dump) => {
+ debug_source_path = debug_dump.debug_source_path;
+ debug_pcm_wav_path = debug_dump.debug_pcm_wav_path;
+ debug_pcm_wav_size_bytes = debug_dump.debug_pcm_wav_size_bytes;
+ }
+ Err(error) => warn!(
+ call_id = %call_id,
+ trace_id = %trace_id,
+ turn_id = %turn.turn_id,
+ error = %safe_error(&error.to_string()),
+ "runtime helper stream_audio_pcm_debug_dump_failed"
+ ),
+ }
+ }
+ }
emit_activity(
call_id,
trace_id,
@@ -1485,94 +2012,20 @@
"replyPlaybackMode": state.reply_playback_mode.as_str(),
"format": format.as_str(),
"chunkSeq": audio_chunk.chunk_seq,
+ "networkChunkCount": state.pcm_stream_network_chunk_count,
+ "debugSourcePath": debug_source_path,
+ "debugPcmWavPath": debug_pcm_wav_path,
+ "debugPcmWavSizeBytes": debug_pcm_wav_size_bytes,
+ "sourceSampleRate": audio_chunk.sample_rate,
+ "sourceChannels": audio_chunk.channels,
+ "targetAudioProfile": sink.profile.as_str(),
+ "targetSampleRate": sink.sample_rate_hz,
+ "targetChannels": sink.num_channels,
"replyTotalAfterVadEndMs": turn_pipeline_started_at.elapsed().as_millis() as u64,
}),
);
}
Ok(frames.len())
-}
-
-fn pcm_s16le_payload_to_frames(
- payload: &[u8],
- sample_rate: u32,
- channels: u32,
-) -> Result<Vec<PcmFrame>> {
- audio::pcm_s16le_bytes_to_frames(
- payload,
- sample_rate,
- channels,
- TARGET_SAMPLE_RATE_HZ,
- TARGET_NUM_CHANNELS,
- )
-}
-
-fn pcm_s16le_frame_alignment_bytes(channels: u32) -> Result<usize> {
- if channels == 0 {
- return Err(anyhow!("pcm_s16le channel count is zero"));
- }
- let channel_count = usize::try_from(channels)
- .map_err(|_| anyhow!("unsupported pcm_s16le channel count {channels}"))?;
- Ok(2 * channel_count)
-}
-
-fn take_aligned_pcm_payload(
- buffer: &mut Vec<u8>,
- payload: &[u8],
- frame_alignment_bytes: usize,
- last: bool,
-) -> (Vec<u8>, usize) {
- let alignment = frame_alignment_bytes.max(2);
- buffer.extend_from_slice(payload);
- let aligned_len = buffer.len() - buffer.len() % alignment;
- let aligned_payload = if aligned_len == 0 {
- Vec::new()
- } else {
- buffer.drain(..aligned_len).collect()
- };
- let dropped_tail_bytes = if last && !buffer.is_empty() {
- let dropped = buffer.len();
- buffer.clear();
- dropped
- } else {
- 0
- };
- (aligned_payload, dropped_tail_bytes)
-}
-
-#[cfg(test)]
-mod pcm_stream_tests {
- use super::take_aligned_pcm_payload;
-
- #[test]
- fn take_aligned_pcm_payload_buffers_split_sample_bytes() {
- let mut buffer = Vec::new();
-
- let (first, dropped) = take_aligned_pcm_payload(&mut buffer, &[0x01], 2, false);
- assert!(first.is_empty());
- assert_eq!(0, dropped);
- assert_eq!(vec![0x01], buffer);
-
- let (second, dropped) = take_aligned_pcm_payload(&mut buffer, &[0x02, 0x03], 2, false);
- assert_eq!(vec![0x01, 0x02], second);
- assert_eq!(0, dropped);
- assert_eq!(vec![0x03], buffer);
-
- let (third, dropped) = take_aligned_pcm_payload(&mut buffer, &[0x04], 2, true);
- assert_eq!(vec![0x03, 0x04], third);
- assert_eq!(0, dropped);
- assert!(buffer.is_empty());
- }
-
- #[test]
- fn take_aligned_pcm_payload_drops_final_half_sample() {
- let mut buffer = Vec::new();
-
- let (payload, dropped) =
- take_aligned_pcm_payload(&mut buffer, &[0x01, 0x02, 0x03], 2, true);
- assert_eq!(vec![0x01, 0x02], payload);
- assert_eq!(1, dropped);
- assert!(buffer.is_empty());
- }
}
fn trim_ascii_whitespace(value: &[u8]) -> &[u8] {
@@ -1595,6 +2048,7 @@
turn: &FinishedSpeechTurn,
path_ref: &str,
byte_size: u64,
+ asr_result_ref: Option<&str>,
turn_pipeline_started_at: Instant,
) -> Result<RuntimeTurnBridgeOutcome> {
let bridge_started_at = Instant::now();
@@ -1602,15 +2056,16 @@
call_id: call_id.to_string(),
trace_id: trace_id.to_string(),
turn_id: turn.turn_id.clone(),
- audio_artifact: RuntimeTurnAudioArtifact {
+ audio_artifact: Some(RuntimeTurnAudioArtifact {
artifact_type: "local_file".to_string(),
path_ref: path_ref.to_string(),
format: "wav".to_string(),
- sample_rate: TARGET_SAMPLE_RATE_HZ,
- channels: u32::from(TARGET_NUM_CHANNELS),
+ sample_rate: USER_AUDIO_SAMPLE_RATE_HZ,
+ channels: u32::from(USER_AUDIO_NUM_CHANNELS),
duration_ms: turn.duration_ms,
byte_size,
- },
+ }),
+ asr_result_ref: asr_result_ref.map(str::to_string),
};
let response = http
.post(bridge_config.bridge_url.as_deref().unwrap_or_default())
@@ -1770,8 +2225,8 @@
http,
Some(&audio_path_string),
None,
- TARGET_SAMPLE_RATE_HZ,
- TARGET_NUM_CHANNELS,
+ sink.sample_rate_hz,
+ sink.num_channels,
bridge_config.audio_debug_dump_dir.as_deref(),
call_id,
&reply_debug_label,
@@ -1901,7 +2356,10 @@
#[serde(rename = "turnId")]
turn_id: String,
#[serde(rename = "audioArtifact")]
- audio_artifact: RuntimeTurnAudioArtifact,
+ #[serde(skip_serializing_if = "Option::is_none")]
+ audio_artifact: Option<RuntimeTurnAudioArtifact>,
+ #[serde(rename = "asrResultRef", skip_serializing_if = "Option::is_none")]
+ asr_result_ref: Option<String>,
}
#[derive(Serialize)]
@@ -1929,6 +2387,20 @@
retryable: Option<bool>,
}
+#[derive(Deserialize)]
+#[serde(rename_all = "camelCase")]
+struct RuntimeAsrStreamResp {
+ status: Option<String>,
+ asr_result_ref: Option<String>,
+ chunk_count: Option<u64>,
+ audio_bytes: Option<u64>,
+ asr_duration_ms: Option<u64>,
+ provider_alias: Option<String>,
+ text_len: Option<u64>,
+ fallback_reason: Option<String>,
+ fallback_stage: Option<String>,
+}
+
#[derive(Default)]
struct RuntimeTurnBridgeOutcome {
reply_audio_artifact: Option<RuntimeTurnReplyAudioArtifact>,
@@ -1950,7 +2422,10 @@
audio_chunk_count: u64,
device_output_count: u64,
encoded_audio_buffer: Vec<u8>,
- pcm_audio_buffer: Vec<u8>,
+ pcm_stream_decoder: Option<audio::PcmS16leStreamDecoder>,
+ pcm_stream_network_chunk_count: u64,
+ reply_chunk_markers: ReplyChunkMarkerState,
+ timing: RuntimeTurnStreamTimingState,
}
impl Default for RuntimeTurnStreamState {
@@ -1964,8 +2439,264 @@
audio_chunk_count: 0,
device_output_count: 0,
encoded_audio_buffer: Vec::new(),
- pcm_audio_buffer: Vec::new(),
+ pcm_stream_decoder: None,
+ pcm_stream_network_chunk_count: 0,
+ reply_chunk_markers: ReplyChunkMarkerState::default(),
+ timing: RuntimeTurnStreamTimingState::default(),
}
+ }
+}
+
+impl RuntimeTurnStreamState {
+ fn arm_timing_anchor(
+ &mut self,
+ call_id: &str,
+ trace_id: &str,
+ turn_id: &str,
+ runtime_session_nonce: &str,
+ event: &RuntimeTurnStreamEvent,
+ ) -> bool {
+ if self.timing.phase != RuntimeTurnStreamTimingPhase::Empty
+ || event.call_id.as_deref() != Some(call_id)
+ || event.trace_id.as_deref() != Some(trace_id)
+ || event.turn_id.as_deref() != Some(turn_id)
+ {
+ return false;
+ }
+ let Some(activity) = event.activity.as_ref() else {
+ return false;
+ };
+ if activity.event_type.as_deref() != Some("tts_first_audio_chunk_ready") {
+ return false;
+ }
+ let Some(extension) = activity.extension.as_ref() else {
+ return false;
+ };
+ let Some(anchor_id) = extension.stream_anchor_id.as_deref() else {
+ return false;
+ };
+ let valid_anchor_id = (16..=64).contains(&anchor_id.len()) && anchor_id.is_ascii();
+ let expected_nonce_hash = runtime_session_nonce_hash(runtime_session_nonce);
+ if extension.stream_timing_version != Some(STREAM_TIMING_VERSION)
+ || !valid_anchor_id
+ || extension.runtime_session_nonce_hash.as_deref() != Some(expected_nonce_hash.as_str())
+ || extension.segment_seq != Some(STREAM_TIMING_FIRST_SEGMENT)
+ || extension.stream_timing_validation.as_deref() != Some("bound")
+ {
+ return false;
+ }
+ let Some(server_delta_ms) = extension.stream_anchor_server_delta_ms else {
+ return false;
+ };
+ self.timing.anchor = Some(RuntimeTurnStreamTimingAnchor {
+ call_id: call_id.to_string(),
+ trace_id: trace_id.to_string(),
+ turn_id: turn_id.to_string(),
+ anchor_id: anchor_id.to_string(),
+ server_delta_ms,
+ runtime_session_nonce_hash: expected_nonce_hash,
+ segment_seq: STREAM_TIMING_FIRST_SEGMENT,
+ received_at: Instant::now(),
+ });
+ self.timing.phase = RuntimeTurnStreamTimingPhase::Armed;
+ true
+ }
+
+ fn record_m6(
+ &mut self,
+ call_id: &str,
+ trace_id: &str,
+ turn_id: &str,
+ event: &RuntimeTurnStreamEvent,
+ audio_chunk: &RuntimeTurnStreamAudioChunk,
+ ) -> Option<RuntimeTurnStreamTimingMarker> {
+ if self.timing.phase != RuntimeTurnStreamTimingPhase::Armed {
+ return None;
+ }
+ let anchor = self.timing.anchor.as_ref()?;
+ if anchor.call_id != call_id
+ || anchor.trace_id != trace_id
+ || anchor.turn_id != turn_id
+ || event.call_id.as_deref() != Some(call_id)
+ || event.trace_id.as_deref() != Some(trace_id)
+ || event.turn_id.as_deref() != Some(turn_id)
+ || audio_chunk.segment_seq != Some(anchor.segment_seq)
+ || audio_chunk.chunk_seq != Some(STREAM_TIMING_FIRST_CHUNK)
+ || audio_chunk.stream_timing_version != Some(STREAM_TIMING_VERSION)
+ || audio_chunk.stream_anchor_id.as_deref() != Some(anchor.anchor_id.as_str())
+ {
+ return None;
+ }
+ let Some(marker) = RuntimeTurnStreamTimingMarker::from_anchor(
+ anchor,
+ audio_chunk.chunk_seq.unwrap_or(STREAM_TIMING_FIRST_CHUNK),
+ ) else {
+ self.close_timing();
+ return None;
+ };
+ self.timing.chunk_seq = Some(marker.chunk_seq);
+ self.timing.phase = RuntimeTurnStreamTimingPhase::M6Recorded;
+ Some(marker)
+ }
+
+ fn record_m7(&mut self) -> Option<RuntimeTurnStreamTimingMarker> {
+ if self.timing.phase != RuntimeTurnStreamTimingPhase::M6Recorded {
+ return None;
+ }
+ let anchor = self.timing.anchor.as_ref()?;
+ let Some(marker) = RuntimeTurnStreamTimingMarker::from_anchor(
+ anchor,
+ self.timing.chunk_seq.unwrap_or(STREAM_TIMING_FIRST_CHUNK),
+ ) else {
+ self.close_timing();
+ return None;
+ };
+ self.timing.phase = RuntimeTurnStreamTimingPhase::M7Recorded;
+ Some(marker)
+ }
+
+ fn close_timing(&mut self) {
+ self.timing.close();
+ }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+enum RuntimeTurnStreamTimingPhase {
+ Empty,
+ Armed,
+ M6Recorded,
+ M7Recorded,
+ Closed,
+}
+
+struct RuntimeTurnStreamTimingState {
+ phase: RuntimeTurnStreamTimingPhase,
+ anchor: Option<RuntimeTurnStreamTimingAnchor>,
+ chunk_seq: Option<u64>,
+}
+
+impl Default for RuntimeTurnStreamTimingState {
+ fn default() -> Self {
+ Self {
+ phase: RuntimeTurnStreamTimingPhase::Empty,
+ anchor: None,
+ chunk_seq: None,
+ }
+ }
+}
+
+impl RuntimeTurnStreamTimingState {
+ fn close(&mut self) {
+ self.anchor = None;
+ self.chunk_seq = None;
+ self.phase = RuntimeTurnStreamTimingPhase::Closed;
+ }
+}
+
+impl Drop for RuntimeTurnStreamTimingState {
+ fn drop(&mut self) {
+ self.anchor = None;
+ self.chunk_seq = None;
+ }
+}
+
+struct RuntimeTurnStreamTimingAnchor {
+ call_id: String,
+ trace_id: String,
+ turn_id: String,
+ anchor_id: String,
+ server_delta_ms: u64,
+ runtime_session_nonce_hash: String,
+ segment_seq: u64,
+ received_at: Instant,
+}
+
+struct RuntimeTurnStreamTimingMarker {
+ version: u32,
+ anchor_id: String,
+ anchor_server_delta_ms: u64,
+ anchor_elapsed_ms: u64,
+ server_delta_ms: u64,
+ runtime_session_nonce_hash: String,
+ segment_seq: u64,
+ chunk_seq: u64,
+}
+
+impl RuntimeTurnStreamTimingMarker {
+ fn from_anchor(anchor: &RuntimeTurnStreamTimingAnchor, chunk_seq: u64) -> Option<Self> {
+ let elapsed_ms = u64::try_from(anchor.received_at.elapsed().as_millis()).ok()?;
+ if elapsed_ms > STREAM_TIMING_MAX_ELAPSED_MS {
+ return None;
+ }
+ Some(Self {
+ version: STREAM_TIMING_VERSION,
+ anchor_id: anchor.anchor_id.clone(),
+ anchor_server_delta_ms: anchor.server_delta_ms,
+ anchor_elapsed_ms: elapsed_ms,
+ server_delta_ms: anchor.server_delta_ms.checked_add(elapsed_ms)?,
+ runtime_session_nonce_hash: anchor.runtime_session_nonce_hash.clone(),
+ segment_seq: anchor.segment_seq,
+ chunk_seq,
+ })
+ }
+
+ fn extension_with(&self, extra: serde_json::Value) -> serde_json::Value {
+ let mut extension = match extra {
+ serde_json::Value::Object(value) => value,
+ _ => serde_json::Map::new(),
+ };
+ extension.insert("streamTimingVersion".to_string(), json!(self.version));
+ extension.insert("streamAnchorId".to_string(), json!(self.anchor_id));
+ extension.insert(
+ "streamAnchorServerDeltaMs".to_string(),
+ json!(self.anchor_server_delta_ms),
+ );
+ extension.insert("anchorElapsedMs".to_string(), json!(self.anchor_elapsed_ms));
+ extension.insert(
+ "runtimeSessionNonceHash".to_string(),
+ json!(self.runtime_session_nonce_hash),
+ );
+ extension.insert("segmentSeq".to_string(), json!(self.segment_seq));
+ extension.insert("chunkSeq".to_string(), json!(self.chunk_seq));
+ extension.insert("streamTimingValidation".to_string(), json!("bound"));
+ serde_json::Value::Object(extension)
+ }
+}
+
+fn runtime_session_nonce_hash(value: &str) -> String {
+ let digest = Sha256::digest(value.as_bytes());
+ digest[..6]
+ .iter()
+ .map(|byte| format!("{byte:02x}"))
+ .collect()
+}
+
+#[derive(Debug, PartialEq, Eq)]
+enum ReplyChunkMarker {
+ FirstReply,
+ SegmentFirst,
+ None,
+}
+
+#[derive(Default)]
+struct ReplyChunkMarkerState {
+ first_reply_seen: bool,
+ seen_segments: HashSet<u64>,
+}
+
+impl ReplyChunkMarkerState {
+ fn observe(&mut self, segment_seq: Option<u64>) -> ReplyChunkMarker {
+ let first_for_segment = segment_seq
+ .map(|value| self.seen_segments.insert(value))
+ .unwrap_or(false);
+ if !self.first_reply_seen {
+ self.first_reply_seen = true;
+ return ReplyChunkMarker::FirstReply;
+ }
+ if first_for_segment {
+ return ReplyChunkMarker::SegmentFirst;
+ }
+ ReplyChunkMarker::None
}
}
@@ -1985,6 +2716,12 @@
struct RuntimeTurnStreamEvent {
#[serde(rename = "type", alias = "event")]
event_type: Option<String>,
+ #[serde(rename = "callId")]
+ call_id: Option<String>,
+ #[serde(rename = "traceId")]
+ trace_id: Option<String>,
+ #[serde(rename = "turnId")]
+ turn_id: Option<String>,
#[serde(rename = "seq")]
seq: Option<u64>,
#[serde(rename = "replyPlaybackMode")]
@@ -2020,6 +2757,12 @@
struct RuntimeTurnStreamAudioChunk {
#[serde(rename = "chunkSeq", alias = "seq")]
chunk_seq: Option<u64>,
+ #[serde(rename = "segmentSeq")]
+ segment_seq: Option<u64>,
+ #[serde(rename = "streamTimingVersion")]
+ stream_timing_version: Option<u32>,
+ #[serde(rename = "streamAnchorId")]
+ stream_anchor_id: Option<String>,
format: Option<String>,
#[serde(rename = "sampleRate")]
sample_rate: Option<u32>,
@@ -2036,6 +2779,23 @@
stage: Option<String>,
#[serde(rename = "reasonCode")]
reason_code: Option<String>,
+ extension: Option<RuntimeTurnStreamTimingExtension>,
+}
+
+#[derive(Clone, Deserialize)]
+struct RuntimeTurnStreamTimingExtension {
+ #[serde(rename = "streamTimingVersion")]
+ stream_timing_version: Option<u32>,
+ #[serde(rename = "streamAnchorId")]
+ stream_anchor_id: Option<String>,
+ #[serde(rename = "streamAnchorServerDeltaMs")]
+ stream_anchor_server_delta_ms: Option<u64>,
+ #[serde(rename = "runtimeSessionNonceHash")]
+ runtime_session_nonce_hash: Option<String>,
+ #[serde(rename = "segmentSeq")]
+ segment_seq: Option<u64>,
+ #[serde(rename = "streamTimingValidation")]
+ stream_timing_validation: Option<String>,
}
#[derive(Deserialize)]
@@ -2113,10 +2873,72 @@
tokio::spawn(async move {
let mut stream = NativeAudioStream::new(
track.rtc_track(),
- TARGET_SAMPLE_RATE_HZ as i32,
- i32::from(TARGET_NUM_CHANNELS),
+ USER_AUDIO_SAMPLE_RATE_HZ as i32,
+ i32::from(USER_AUDIO_NUM_CHANNELS),
);
let started_at = Instant::now();
+ let (frame_tx, mut frame_rx) =
+ mpsc::channel::<DrainedUserAudioFrame>(INBOUND_AUDIO_QUEUE_CAPACITY);
+ let (drain_shutdown_tx, mut drain_shutdown_rx) = watch::channel(false);
+ let drain_call_id = call_id.clone();
+ let drain_trace_id = trace_id.clone();
+ let drain_task = tokio::spawn(async move {
+ let mut received_frame_count: u64 = 0;
+ let mut dropped_frame_count: u64 = 0;
+ loop {
+ tokio::select! {
+ changed = drain_shutdown_rx.changed() => {
+ match changed {
+ Ok(()) if *drain_shutdown_rx.borrow() => break,
+ Ok(()) => {}
+ Err(_) => break,
+ }
+ }
+ maybe_frame = stream.next() => {
+ let Some(frame) = maybe_frame else {
+ break;
+ };
+ received_frame_count = received_frame_count.saturating_add(1);
+ let drained = DrainedUserAudioFrame {
+ frame_index: received_frame_count,
+ captured_elapsed_ms: started_at.elapsed().as_millis() as u64,
+ frame: AudioFrame {
+ data: Cow::Owned(frame.data.as_ref().to_vec()),
+ sample_rate: frame.sample_rate,
+ num_channels: frame.num_channels,
+ samples_per_channel: frame.samples_per_channel,
+ },
+ };
+ match frame_tx.try_send(drained) {
+ Ok(()) => {}
+ Err(mpsc::error::TrySendError::Full(_)) => {
+ dropped_frame_count = dropped_frame_count.saturating_add(1);
+ if dropped_frame_count % INBOUND_AUDIO_DROP_LOG_INTERVAL == 1 {
+ warn!(
+ call_id = %drain_call_id,
+ trace_id = %drain_trace_id,
+ dropped_frame_count,
+ received_frame_count,
+ queue_capacity = INBOUND_AUDIO_QUEUE_CAPACITY,
+ "runtime helper inbound_audio_queue_full_dropping_newest"
+ );
+ }
+ }
+ Err(mpsc::error::TrySendError::Closed(_)) => break,
+ }
+ }
+ }
+ }
+ stream.close();
+ info!(
+ call_id = %drain_call_id,
+ trace_id = %drain_trace_id,
+ received_frame_count,
+ dropped_frame_count,
+ queue_capacity = INBOUND_AUDIO_QUEUE_CAPACITY,
+ "runtime helper user_audio_drain_ended"
+ );
+ });
let mut frame_count: u64 = 0;
let mut sample_count: u64 = 0;
let mut simple_vad = if simple_vad_enabled {
@@ -2124,11 +2946,13 @@
} else {
None
};
+ let mut realtime_asr_upload: Option<RealtimeAsrUpload> = None;
- while let Some(frame) = stream.next().await {
- frame_count += 1;
+ while let Some(drained) = frame_rx.recv().await {
+ let frame = drained.frame;
+ frame_count = drained.frame_index;
sample_count += u64::from(frame.samples_per_channel) * u64::from(frame.num_channels);
- let elapsed_ms = started_at.elapsed().as_millis() as u64;
+ let elapsed_ms = drained.captured_elapsed_ms;
if frame_count == 1 {
info!(
call_id = %call_id,
@@ -2156,7 +2980,8 @@
if let Some(vad) = simple_vad.as_mut() {
if vad_enabled_gate.load(Ordering::Acquire) {
- if let Some(turn) = vad.observe_frame(
+ let was_in_speech = vad.in_speech;
+ let turn = vad.observe_frame(
&call_id,
&trace_id,
&participant_alias,
@@ -2164,7 +2989,65 @@
frame_count,
elapsed_ms,
&frame,
- ) {
+ );
+ let is_in_speech = vad.in_speech;
+
+ if !was_in_speech && is_in_speech {
+ let turn_id = format!("turn-{:04}", vad.turn_index);
+ match RealtimeAsrUpload::start(
+ http.clone(),
+ turn_bridge_config.realtime_asr_config(),
+ &call_id,
+ &trace_id,
+ &turn_id,
+ &vad.speech_samples,
+ ) {
+ Ok(upload) => {
+ info!(
+ call_id = %call_id,
+ trace_id = %trace_id,
+ turn_id = %turn_id,
+ "runtime helper asr_realtime_session_started"
+ );
+ realtime_asr_upload = Some(upload);
+ }
+ Err(error) if turn_bridge_config.asr_realtime_enabled => {
+ warn!(
+ call_id = %call_id,
+ trace_id = %trace_id,
+ turn_id = %turn_id,
+ error = %safe_error(&error.to_string()),
+ "runtime helper asr_realtime_start_failed_fallback"
+ );
+ }
+ Err(_) => {}
+ }
+ } else if was_in_speech {
+ let push_failed = realtime_asr_upload
+ .as_mut()
+ .and_then(|upload| upload.push_48k_samples(frame.data.as_ref()).err());
+ if let Some(error) = push_failed {
+ warn!(
+ call_id = %call_id,
+ trace_id = %trace_id,
+ error = %safe_error(&error.to_string()),
+ "runtime helper asr_realtime_upload_failed_fallback"
+ );
+ if let Some(upload) = realtime_asr_upload.take() {
+ tokio::spawn(async move {
+ upload.cancel("upload_backpressure").await;
+ });
+ }
+ }
+ }
+
+ if let Some(turn) = turn {
+ let realtime_asr_result_ref = match realtime_asr_upload.take() {
+ Some(upload) => {
+ finish_realtime_asr_upload(upload, &call_id, &trace_id, &turn).await
+ }
+ None => None,
+ };
handle_finished_turn(
&http,
&turn_bridge_config,
@@ -2172,10 +3055,22 @@
&call_id,
&trace_id,
turn,
+ realtime_asr_result_ref,
)
.await;
+ } else if was_in_speech && !is_in_speech {
+ if let Some(upload) = realtime_asr_upload.take() {
+ tokio::spawn(async move {
+ upload.cancel("speech_too_short").await;
+ });
+ }
}
} else {
+ if let Some(upload) = realtime_asr_upload.take() {
+ tokio::spawn(async move {
+ upload.cancel("vad_disabled").await;
+ });
+ }
vad.observe_disabled_frame(
&call_id,
&trace_id,
@@ -2196,9 +3091,41 @@
&track_sid_alias,
started_at.elapsed().as_millis() as u64,
) {
- handle_finished_turn(&http, &turn_bridge_config, &sink, &call_id, &trace_id, turn)
- .await;
+ let realtime_asr_result_ref = match realtime_asr_upload.take() {
+ Some(upload) => {
+ finish_realtime_asr_upload(upload, &call_id, &trace_id, &turn).await
+ }
+ None => None,
+ };
+ handle_finished_turn(
+ &http,
+ &turn_bridge_config,
+ &sink,
+ &call_id,
+ &trace_id,
+ turn,
+ realtime_asr_result_ref,
+ )
+ .await;
}
+ }
+ if let Some(upload) = realtime_asr_upload.take() {
+ upload.cancel("stream_end").await;
+ }
+ let _ = drain_shutdown_tx.send(true);
+ let mut drain_task = drain_task;
+ if timeout(AUDIO_DRAIN_STOP_GRACE, &mut drain_task)
+ .await
+ .is_err()
+ {
+ drain_task.abort();
+ let _ = drain_task.await;
+ warn!(
+ call_id = %call_id,
+ trace_id = %trace_id,
+ stop_grace_ms = AUDIO_DRAIN_STOP_GRACE.as_millis() as u64,
+ "runtime helper user_audio_drain_stop_timeout"
+ );
}
info!(
@@ -2212,6 +3139,64 @@
"runtime helper user_audio_stream_ended"
);
})
+}
+
+struct DrainedUserAudioFrame {
+ frame_index: u64,
+ captured_elapsed_ms: u64,
+ frame: AudioFrame<'static>,
+}
+
+async fn finish_realtime_asr_upload(
+ upload: RealtimeAsrUpload,
+ call_id: &str,
+ trace_id: &str,
+ turn: &FinishedSpeechTurn,
+) -> Option<String> {
+ match upload.finish(turn.duration_ms, &turn.end_reason).await {
+ Ok(RealtimeAsrOutcome {
+ status,
+ asr_result_ref,
+ provider_alias,
+ partial_count,
+ fallback_reason,
+ fallback_stage,
+ chunk_count,
+ audio_bytes,
+ wall_ms,
+ }) => {
+ info!(
+ call_id = %call_id,
+ trace_id = %trace_id,
+ turn_id = %turn.turn_id,
+ status = %status,
+ provider_alias = ?provider_alias,
+ partial_count,
+ chunk_count,
+ audio_bytes,
+ wall_ms,
+ asr_result_ref_present = asr_result_ref.is_some(),
+ fallback_reason = ?fallback_reason,
+ fallback_stage = ?fallback_stage,
+ "runtime helper asr_realtime_finished"
+ );
+ if status == "final" {
+ asr_result_ref
+ } else {
+ None
+ }
+ }
+ Err(error) => {
+ warn!(
+ call_id = %call_id,
+ trace_id = %trace_id,
+ turn_id = %turn.turn_id,
+ error = %safe_error(&error.to_string()),
+ "runtime helper asr_realtime_failed_fallback"
+ );
+ None
+ }
+ }
}
#[derive(Clone)]
@@ -2231,8 +3216,8 @@
rms_threshold: f64_env("CV_VAD_RMS_THRESHOLD", 0.012),
peak_threshold: f64_env("CV_VAD_PEAK_THRESHOLD", 0.08),
start_frames: u32_env("CV_VAD_START_FRAMES", 5).max(1),
- end_silence_ms: u64_env("CV_VAD_END_SILENCE_MS", 700).max(100),
- min_speech_ms: u64_env("CV_VAD_MIN_SPEECH_MS", 300).max(1),
+ end_silence_ms: u64_env("CV_VAD_END_SILENCE_MS", 400).max(100),
+ min_speech_ms: u64_env("CV_VAD_MIN_SPEECH_MS", 250).max(1),
max_turn_ms: u64_env("CV_VAD_MAX_TURN_MS", 10_000).max(1_000),
initial_ignore_ms: u64_env("CV_VAD_INITIAL_IGNORE_MS", 500),
}
@@ -2345,7 +3330,9 @@
return None;
}
- let voiced = rms >= self.config.rms_threshold || peak >= self.config.peak_threshold;
+ // Align with cb-sdk's energy-based segmentation: peak is diagnostic only,
+ // otherwise isolated spikes can keep a turn open until max_turn_ms.
+ let voiced = rms >= self.config.rms_threshold;
if !self.in_speech {
self.remember_pre_speech_frame(frame);
}
@@ -2601,6 +3588,9 @@
rtc_source: NativeAudioSource,
track: LocalAudioTrack,
device_output_destination_identity: Option<String>,
+ profile: String,
+ sample_rate_hz: u32,
+ num_channels: u16,
}
impl BotAudioOutputSink {
@@ -2611,6 +3601,7 @@
call_id: &str,
trace_id: &str,
track_name: &str,
+ profile: String,
sample_rate: u32,
num_channels: u32,
device_output_destination_identity: Option<String>,
@@ -2639,11 +3630,14 @@
"failed to publish bot audio track in room {room_alias} for participant {participant_alias}: {error}"
)
})?;
+ let num_channels_u16 = u16::try_from(num_channels)
+ .map_err(|_| anyhow!("unsupported bot audio channel count {num_channels}"))?;
info!(
room_alias = %room_alias,
participant_alias = %participant_alias,
track_name = %track_name,
+ bot_audio_profile = %profile,
sample_rate,
num_channels,
"runtime helper published bot audio track"
@@ -2658,6 +3652,7 @@
None,
json!({
"trackName": track_name,
+ "audioProfile": profile,
"sampleRate": sample_rate,
"numChannels": num_channels,
}),
@@ -2668,6 +3663,9 @@
rtc_source,
track,
device_output_destination_identity,
+ profile,
+ sample_rate_hz: sample_rate,
+ num_channels: num_channels_u16,
})
}
@@ -2892,3 +3890,107 @@
Ok(())
}
}
+
+#[cfg(test)]
+mod tests {
+ use super::{
+ ReplyChunkMarker, ReplyChunkMarkerState, RuntimeTurnStreamEvent, RuntimeTurnStreamState,
+ RuntimeTurnStreamTimingPhase, runtime_session_nonce_hash,
+ };
+
+ #[test]
+ fn reply_chunk_marker_state_emits_turn_first_once_and_later_segment_first_once() {
+ let mut state = ReplyChunkMarkerState::default();
+
+ assert_eq!(ReplyChunkMarker::FirstReply, state.observe(Some(1)));
+ assert_eq!(ReplyChunkMarker::None, state.observe(Some(1)));
+ assert_eq!(ReplyChunkMarker::SegmentFirst, state.observe(Some(2)));
+ assert_eq!(ReplyChunkMarker::None, state.observe(Some(2)));
+ assert_eq!(ReplyChunkMarker::SegmentFirst, state.observe(Some(3)));
+ }
+
+ #[test]
+ fn reply_chunk_marker_state_without_segment_only_emits_turn_first() {
+ let mut state = ReplyChunkMarkerState::default();
+
+ assert_eq!(ReplyChunkMarker::FirstReply, state.observe(None));
+ assert_eq!(ReplyChunkMarker::None, state.observe(None));
+ }
+
+ #[test]
+ fn runtime_turn_stream_audio_chunk_reads_segment_seq() {
+ let event: RuntimeTurnStreamEvent = serde_json::from_str(
+ r#"{"type":"reply_audio_chunk","audioChunk":{"chunkSeq":4,"segmentSeq":2,"format":"pcm_s16le","payloadBase64":"AA==","last":false}}"#,
+ )
+ .expect("turn stream event");
+
+ assert_eq!(
+ Some(2),
+ event.audio_chunk.and_then(|chunk| chunk.segment_seq)
+ );
+ }
+
+ #[test]
+ fn runtime_turn_stream_reads_frozen_m5_and_audio_chunk_timing_contract() {
+ let activity: RuntimeTurnStreamEvent = serde_json::from_str(
+ r#"{"type":"activity","callId":"call-1","traceId":"trace-1","turnId":"turn-1","activity":{"eventType":"tts_first_audio_chunk_ready","extension":{"streamTimingVersion":1,"streamAnchorId":"0123456789abcdef","streamAnchorServerDeltaMs":1200,"runtimeSessionNonceHash":"abcdef012345","segmentSeq":1,"streamTimingValidation":"bound"}}}"#,
+ )
+ .expect("m5 activity event");
+ let audio: RuntimeTurnStreamEvent = serde_json::from_str(
+ r#"{"type":"reply_audio_chunk","callId":"call-1","traceId":"trace-1","turnId":"turn-1","audioChunk":{"chunkSeq":1,"segmentSeq":1,"streamTimingVersion":1,"streamAnchorId":"0123456789abcdef","format":"pcm_s16le","payloadBase64":"AA==","last":false}}"#,
+ )
+ .expect("timed audio chunk event");
+
+ assert_eq!(Some("call-1"), activity.call_id.as_deref());
+ assert_eq!(Some("trace-1"), activity.trace_id.as_deref());
+ assert_eq!(Some("turn-1"), activity.turn_id.as_deref());
+ let extension = activity.activity.unwrap().extension.unwrap();
+ assert_eq!(Some(1), extension.stream_timing_version);
+ assert_eq!(Some(1200), extension.stream_anchor_server_delta_ms);
+ assert_eq!(
+ Some("0123456789abcdef"),
+ audio.audio_chunk.unwrap().stream_anchor_id.as_deref()
+ );
+ }
+
+ #[test]
+ fn runtime_turn_stream_timing_records_m6_and_m7_once_then_rejects_terminal_late_events() {
+ let nonce = "runtime-nonce";
+ let nonce_hash = runtime_session_nonce_hash(nonce);
+ let activity: RuntimeTurnStreamEvent = serde_json::from_str(&format!(
+ r#"{{"type":"activity","callId":"call-1","traceId":"trace-1","turnId":"turn-1","activity":{{"eventType":"tts_first_audio_chunk_ready","extension":{{"streamTimingVersion":1,"streamAnchorId":"0123456789abcdef","streamAnchorServerDeltaMs":1200,"runtimeSessionNonceHash":"{nonce_hash}","segmentSeq":1,"streamTimingValidation":"bound"}}}}}}"#,
+ ))
+ .expect("m5 activity event");
+ let audio: RuntimeTurnStreamEvent = serde_json::from_str(
+ r#"{"type":"reply_audio_chunk","callId":"call-1","traceId":"trace-1","turnId":"turn-1","audioChunk":{"chunkSeq":1,"segmentSeq":1,"streamTimingVersion":1,"streamAnchorId":"0123456789abcdef","format":"pcm_s16le","payloadBase64":"AA==","last":false}}"#,
+ )
+ .expect("timed audio chunk event");
+ let mut state = RuntimeTurnStreamState::default();
+
+ assert!(state.arm_timing_anchor("call-1", "trace-1", "turn-1", nonce, &activity));
+ let chunk = audio.audio_chunk.as_ref().expect("audio chunk");
+ assert!(
+ state
+ .record_m6("call-1", "trace-1", "turn-1", &audio, chunk)
+ .is_some()
+ );
+ assert!(
+ state
+ .record_m6("call-1", "trace-1", "turn-1", &audio, chunk)
+ .is_none()
+ );
+ assert!(state.record_m7().is_some());
+ assert!(state.record_m7().is_none());
+
+ state.close_timing();
+ assert_eq!(RuntimeTurnStreamTimingPhase::Closed, state.timing.phase);
+ assert!(state.timing.anchor.is_none());
+ assert!(!state.arm_timing_anchor("call-1", "trace-1", "turn-1", nonce, &activity));
+ assert!(
+ state
+ .record_m6("call-1", "trace-1", "turn-1", &audio, chunk)
+ .is_none()
+ );
+ assert!(state.record_m7().is_none());
+ }
+}
--
Gitblit v1.9.3