From e3d68ccc3714175a12854428f34adbb69f09474f Mon Sep 17 00:00:00 2001
From: cai <cai@nbcai.cc>
Date: Sat, 08 Aug 2026 18:42:58 +0800
Subject: [PATCH] chore: bind dynamic metadata test candidate
---
src/main.rs | 1210 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++-
1 files changed, 1,178 insertions(+), 32 deletions(-)
diff --git a/src/main.rs b/src/main.rs
index 05ba69b..d357e4a 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -1,7 +1,10 @@
+mod asr_realtime;
mod audio;
mod service;
use std::{
+ borrow::Cow,
+ collections::HashSet,
env, fs,
path::{Path, PathBuf},
sync::{
@@ -12,6 +15,9 @@
};
use anyhow::{Context, Result, anyhow};
+use asr_realtime::{
+ AudioIngressMetadata, RealtimeAsrConfig, RealtimeAsrOutcome, RealtimeAsrUpload,
+};
use audio::{AudioDiagnostics, load_pre_recorded_frames};
use base64::{Engine as _, engine::general_purpose};
use futures_util::StreamExt;
@@ -24,23 +30,35 @@
options::TrackPublishOptions,
prelude::{
DataPacket, LocalAudioTrack, LocalTrack, ParticipantIdentity, RemoteAudioTrack,
- RemoteTrack, Room, RoomEvent, RoomOptions,
+ RemoteParticipant, RemoteTrack, Room, RoomEvent, RoomOptions,
},
};
use reqwest::Client;
use serde::{Deserialize, Serialize};
use serde_json::json;
-use tokio::time::{sleep, sleep_until};
-use tokio::{sync::mpsc::UnboundedReceiver, task::JoinHandle};
+use sha2::{Digest, Sha256};
+use tokio::time::{sleep, sleep_until, timeout};
+use tokio::{
+ sync::{mpsc, mpsc::UnboundedReceiver, watch},
+ task::JoinHandle,
+};
use tracing::{info, warn};
const USER_AUDIO_SAMPLE_RATE_HZ: u32 = 48_000;
const USER_AUDIO_NUM_CHANNELS: u16 = 1;
+const INBOUND_AUDIO_QUEUE_CAPACITY: usize = 100;
+const INBOUND_AUDIO_DROP_LOG_INTERVAL: u64 = 100;
+const AUDIO_DRAIN_STOP_GRACE: Duration = Duration::from_secs(2);
const DEFAULT_BOT_AUDIO_PROFILE: &str = "pcm-16k";
const LIVEKIT_48K_SAMPLE_RATE_HZ: u32 = 48_000;
const PCM_16K_SAMPLE_RATE_HZ: u32 = 16_000;
const BOT_NUM_CHANNELS: u16 = 1;
const TRACK_NAME: &str = "bot-main-audio";
+const STREAM_TIMING_VERSION: u32 = 1;
+const STREAM_TIMING_FIRST_SEGMENT: u64 = 1;
+const STREAM_TIMING_FIRST_CHUNK: u64 = 1;
+const STREAM_TIMING_MAX_ELAPSED_MS: u64 = 5_000;
+const STREAM_TIMING_SOURCE: &str = "stream_anchor_monotonic";
#[tokio::main(flavor = "multi_thread")]
async fn main() -> Result<()> {
@@ -270,6 +288,11 @@
runtime_turn_bridge_url: Option<String>,
runtime_turn_bridge_token: Option<String>,
runtime_turn_bridge_mode: String,
+ runtime_asr_stream_enabled: bool,
+ runtime_asr_stream_url: Option<String>,
+ runtime_asr_realtime_enabled: bool,
+ runtime_asr_realtime_url: Option<String>,
+ runtime_asr_realtime_chunk_duration_ms: u64,
runtime_turn_artifact_dir: Option<String>,
runtime_session_nonce: Option<String>,
user_audio_observer_enabled: bool,
@@ -332,6 +355,11 @@
bridge_url: Option<String>,
bridge_token: Option<String>,
bridge_mode: String,
+ asr_stream_enabled: bool,
+ asr_stream_url: Option<String>,
+ asr_realtime_enabled: bool,
+ asr_realtime_url: Option<String>,
+ asr_realtime_chunk_duration_ms: u64,
artifact_dir: Option<String>,
runtime_session_nonce: Option<String>,
audio_debug_dump_dir: Option<String>,
@@ -343,6 +371,11 @@
bridge_url: config.runtime_turn_bridge_url.clone(),
bridge_token: config.runtime_turn_bridge_token.clone(),
bridge_mode: config.runtime_turn_bridge_mode.clone(),
+ asr_stream_enabled: config.runtime_asr_stream_enabled,
+ asr_stream_url: config.runtime_asr_stream_url.clone(),
+ asr_realtime_enabled: config.runtime_asr_realtime_enabled,
+ asr_realtime_url: config.runtime_asr_realtime_url.clone(),
+ asr_realtime_chunk_duration_ms: config.runtime_asr_realtime_chunk_duration_ms,
artifact_dir: config.runtime_turn_artifact_dir.clone(),
runtime_session_nonce: config.runtime_session_nonce.clone(),
audio_debug_dump_dir: config.audio_debug_dump_dir.clone(),
@@ -374,6 +407,32 @@
.as_deref()
.is_some_and(|value| value.trim_end_matches('/').ends_with("/stream"))
}
+
+ fn is_asr_stream_ready(&self) -> bool {
+ self.asr_stream_enabled
+ && self
+ .asr_stream_url
+ .as_ref()
+ .is_some_and(|value| !value.is_empty())
+ && self
+ .bridge_token
+ .as_ref()
+ .is_some_and(|value| !value.is_empty())
+ && self
+ .runtime_session_nonce
+ .as_ref()
+ .is_some_and(|value| !value.is_empty())
+ }
+
+ fn realtime_asr_config(&self) -> RealtimeAsrConfig {
+ RealtimeAsrConfig {
+ enabled: self.asr_realtime_enabled,
+ url: self.asr_realtime_url.clone(),
+ runtime_token: self.bridge_token.clone(),
+ runtime_session_nonce: self.runtime_session_nonce.clone(),
+ chunk_duration_ms: self.asr_realtime_chunk_duration_ms,
+ }
+ }
}
impl Config {
@@ -395,6 +454,14 @@
runtime_turn_bridge_token: optional_env("CV_RUNTIME_TURN_BRIDGE_TOKEN"),
runtime_turn_bridge_mode: env::var("CV_RUNTIME_TURN_BRIDGE_MODE")
.unwrap_or_else(|_| "json".to_string()),
+ runtime_asr_stream_enabled: bool_env("CV_RUNTIME_ASR_STREAM_ENABLED", false),
+ runtime_asr_stream_url: optional_env("CV_RUNTIME_ASR_STREAM_URL"),
+ runtime_asr_realtime_enabled: bool_env("CV_RUNTIME_ASR_REALTIME_ENABLED", false),
+ runtime_asr_realtime_url: optional_env("CV_RUNTIME_ASR_REALTIME_URL"),
+ runtime_asr_realtime_chunk_duration_ms: u64_env(
+ "CV_RUNTIME_ASR_REALTIME_CHUNK_DURATION_MS",
+ 200,
+ ),
runtime_turn_artifact_dir: optional_env("CV_RUNTIME_TURN_ARTIFACT_DIR"),
runtime_session_nonce: optional_env("CV_RUNTIME_SESSION_NONCE"),
user_audio_observer_enabled: bool_env("CV_ENABLE_USER_AUDIO_OBSERVER", true),
@@ -605,6 +672,31 @@
println!("{payload}");
}
+fn emit_anchored_activity(
+ call_id: &str,
+ trace_id: &str,
+ turn_id: &str,
+ event_name: &str,
+ marker: &RuntimeTurnStreamTimingMarker,
+ extension: serde_json::Value,
+) {
+ let payload = json!({
+ "type": "cv_activity",
+ "callId": call_id,
+ "traceId": trace_id,
+ "turnId": turn_id,
+ "eventName": event_name,
+ "eventWallTimeMs": current_time_millis(),
+ "serverDeltaMs": marker.server_delta_ms,
+ "serverDeltaSource": STREAM_TIMING_SOURCE,
+ "result": "ok",
+ "reasonCode": null,
+ "retryable": null,
+ "extension": marker.extension_with(extension),
+ });
+ println!("{payload}");
+}
+
fn spawn_user_audio_observer(
events: UnboundedReceiver<RoomEvent>,
config: &Config,
@@ -617,6 +709,7 @@
let enabled = config.user_audio_observer_enabled;
let simple_vad_enabled = config.simple_vad_enabled;
let simple_vad_config = config.simple_vad_config.clone();
+ let user_participant_identity = config.user_participant_identity.clone();
let turn_bridge_config = TurnBridgeConfig::from_config(config);
tokio::spawn(async move {
@@ -638,6 +731,7 @@
turn_bridge_config,
http,
sink,
+ user_participant_identity,
)
.await;
})
@@ -653,6 +747,7 @@
turn_bridge_config: TurnBridgeConfig,
http: Client,
sink: Arc<BotAudioOutputSink>,
+ user_participant_identity: Option<String>,
) {
info!(
call_id = %call_id,
@@ -676,6 +771,15 @@
publication: _,
participant,
} => {
+ if user_participant_identity
+ .as_deref()
+ .is_some_and(|expected| participant.identity().to_string() != expected)
+ {
+ warn!(call_id = %call_id, trace_id = %trace_id,
+ metadata_status = "wrong_participant",
+ "runtime helper ignored non-user audio participant");
+ continue;
+ }
let participant_alias = redact(&participant.identity().to_string());
let track_sid_alias = redact(&track.sid().to_string());
let track_name = track.name();
@@ -701,6 +805,7 @@
turn_bridge_config.clone(),
http.clone(),
sink.clone(),
+ participant,
);
}
RoomEvent::TrackSubscribed {
@@ -751,6 +856,7 @@
call_id: &str,
trace_id: &str,
turn: FinishedSpeechTurn,
+ realtime_asr_result_ref: Option<String>,
) {
let turn_pipeline_started_at = Instant::now();
if !bridge_config.is_ready() {
@@ -807,9 +913,13 @@
"turnArtifactBytes": byte_size,
"frameCount": turn.frame_count,
"sampleCount": turn.sample_count,
- "endReason": turn.end_reason,
+ "endReason": turn.end_reason.as_str(),
}),
);
+ let asr_result_ref = match realtime_asr_result_ref {
+ Some(value) => Some(value),
+ None => request_asr_result_ref(http, bridge_config, call_id, trace_id, &turn).await,
+ };
if bridge_config.is_stream_mode() {
match request_turn_bridge_stream(
http,
@@ -820,6 +930,7 @@
&turn,
&path_ref,
byte_size,
+ asr_result_ref.as_deref(),
turn_pipeline_started_at,
)
.await
@@ -883,12 +994,17 @@
&turn,
&path_ref,
byte_size,
+ asr_result_ref.as_deref(),
turn_pipeline_started_at,
)
.await
{
Ok(outcome) => {
+ let mut published_device_outputs = HashSet::new();
for output in &outcome.device_outputs {
+ if !should_publish_device_output(&mut published_device_outputs, output) {
+ continue;
+ }
if let Err(error) = sink
.publish_device_output(call_id, trace_id, &turn.turn_id, output)
.await
@@ -1004,6 +1120,251 @@
Ok((path_ref, byte_size))
}
+async fn request_asr_result_ref(
+ http: &Client,
+ bridge_config: &TurnBridgeConfig,
+ call_id: &str,
+ trace_id: &str,
+ turn: &FinishedSpeechTurn,
+) -> Option<String> {
+ if !bridge_config.is_asr_stream_ready() {
+ return None;
+ }
+ match request_asr_stream(http, bridge_config, call_id, trace_id, turn).await {
+ Ok(Some(asr_result_ref)) => {
+ emit_activity(
+ call_id,
+ trace_id,
+ Some(&turn.turn_id),
+ "asr_stream_ref_ready",
+ "ok",
+ None,
+ None,
+ json!({
+ "asrResultRefPresent": true,
+ "format": "pcm_s16le",
+ "sampleRate": 16000,
+ "channels": 1,
+ }),
+ );
+ Some(asr_result_ref)
+ }
+ Ok(None) => None,
+ Err(error) => {
+ warn!(
+ call_id = %call_id,
+ trace_id = %trace_id,
+ turn_id = %turn.turn_id,
+ error = %safe_error(&error.to_string()),
+ "runtime helper asr_stream_failed_fallback"
+ );
+ emit_activity(
+ call_id,
+ trace_id,
+ Some(&turn.turn_id),
+ "asr_stream_fallback",
+ "skipped",
+ Some("ASR_STREAM_INTERRUPTED"),
+ Some(true),
+ json!({
+ "fallbackReason": "asr_stream_request_failed",
+ "fallbackStage": "asr_stream",
+ }),
+ );
+ None
+ }
+ }
+}
+
+async fn request_asr_stream(
+ http: &Client,
+ bridge_config: &TurnBridgeConfig,
+ call_id: &str,
+ trace_id: &str,
+ turn: &FinishedSpeechTurn,
+) -> Result<Option<String>> {
+ let started_at = Instant::now();
+ let ndjson = build_asr_stream_ndjson(call_id, trace_id, turn, bridge_config)?;
+ let response = http
+ .post(bridge_config.asr_stream_url.as_deref().unwrap_or_default())
+ .header("Content-Type", "application/x-ndjson")
+ .header(
+ "X-CV-Runtime-Token",
+ bridge_config.bridge_token.as_deref().unwrap_or_default(),
+ )
+ .header("X-CV-Call-Id", call_id)
+ .header("X-CV-Trace-Id", trace_id)
+ .header(
+ "X-CV-Runtime-Session-Nonce",
+ bridge_config
+ .runtime_session_nonce
+ .as_deref()
+ .unwrap_or_default(),
+ )
+ .body(ndjson)
+ .send()
+ .await
+ .context("failed to post asr stream")?;
+ let status = response.status();
+ if !status.is_success() {
+ let body_len = response.text().await.map(|body| body.len()).unwrap_or(0);
+ warn!(
+ call_id = %call_id,
+ trace_id = %trace_id,
+ turn_id = %turn.turn_id,
+ http_status = status.as_u16(),
+ body_len,
+ "runtime helper asr_stream_http_failed"
+ );
+ return Ok(None);
+ }
+ let body: RuntimeTurnCommonResult<RuntimeAsrStreamResp> = response
+ .json()
+ .await
+ .context("failed to decode asr stream response")?;
+ if body.code != 0 {
+ warn!(
+ call_id = %call_id,
+ trace_id = %trace_id,
+ turn_id = %turn.turn_id,
+ code = body.code,
+ msg_len = body.msg.as_deref().unwrap_or_default().len(),
+ "runtime helper asr_stream_common_result_failed"
+ );
+ return Ok(None);
+ }
+ let Some(data) = body.data else {
+ return Ok(None);
+ };
+ if data.status.as_deref() == Some("final") {
+ info!(
+ call_id = %call_id,
+ trace_id = %trace_id,
+ turn_id = %turn.turn_id,
+ chunk_count = data.chunk_count.unwrap_or_default(),
+ audio_bytes = data.audio_bytes.unwrap_or_default(),
+ asr_duration_ms = data.asr_duration_ms.unwrap_or_default(),
+ wall_ms = started_at.elapsed().as_millis() as u64,
+ provider = %data.provider_alias.as_deref().unwrap_or("unknown"),
+ text_len = data.text_len.unwrap_or_default(),
+ "runtime helper asr_stream_final"
+ );
+ return Ok(data.asr_result_ref);
+ }
+ emit_activity(
+ call_id,
+ trace_id,
+ Some(&turn.turn_id),
+ "asr_stream_fallback",
+ "skipped",
+ None,
+ Some(true),
+ json!({
+ "fallbackReason": data.fallback_reason,
+ "fallbackStage": data.fallback_stage,
+ "status": data.status,
+ }),
+ );
+ Ok(None)
+}
+
+fn build_asr_stream_ndjson(
+ call_id: &str,
+ trace_id: &str,
+ turn: &FinishedSpeechTurn,
+ bridge_config: &TurnBridgeConfig,
+) -> Result<String> {
+ let chunks = asr_pcm_16k_chunks(turn)?;
+ let mut seq = 1u64;
+ let mut lines = Vec::with_capacity(chunks.len() + 3);
+ lines.push(serde_json::to_string(&json!({
+ "event": "asr_stream_started",
+ "seq": seq,
+ "callId": call_id,
+ "traceId": trace_id,
+ "turnId": turn.turn_id.as_str(),
+ "tsMs": current_time_millis(),
+ "payload": {
+ "format": "pcm_s16le",
+ "sampleRate": 16000,
+ "channels": 1,
+ "runtimeSessionNonce": bridge_config.runtime_session_nonce.as_deref().unwrap_or_default(),
+ "providerHint": "volcengine",
+ }
+ }))?);
+ for (index, samples) in chunks.iter().enumerate() {
+ seq += 1;
+ let bytes = pcm_i16_to_le_bytes(samples);
+ let duration_ms = ((samples.len() as u64) * 1000 / 16_000).max(1);
+ lines.push(serde_json::to_string(&json!({
+ "event": "asr_audio_chunk",
+ "seq": seq,
+ "callId": call_id,
+ "traceId": trace_id,
+ "turnId": turn.turn_id.as_str(),
+ "tsMs": current_time_millis(),
+ "payload": {
+ "chunkSeq": index + 1,
+ "format": "pcm_s16le",
+ "sampleRate": 16000,
+ "channels": 1,
+ "durationMs": duration_ms,
+ "payloadBase64": general_purpose::STANDARD.encode(bytes),
+ }
+ }))?);
+ }
+ seq += 1;
+ lines.push(serde_json::to_string(&json!({
+ "event": "vad_speech_end",
+ "seq": seq,
+ "callId": call_id,
+ "traceId": trace_id,
+ "turnId": turn.turn_id.as_str(),
+ "tsMs": current_time_millis(),
+ "payload": {
+ "endReason": turn.end_reason.as_str(),
+ "speechDurationMs": turn.duration_ms,
+ }
+ }))?);
+ seq += 1;
+ lines.push(serde_json::to_string(&json!({
+ "event": "asr_stream_finish",
+ "seq": seq,
+ "callId": call_id,
+ "traceId": trace_id,
+ "turnId": turn.turn_id.as_str(),
+ "tsMs": current_time_millis(),
+ "payload": {
+ "finalChunkSeq": chunks.len(),
+ "audioDurationMs": turn.duration_ms,
+ }
+ }))?);
+ Ok(lines.join("\n") + "\n")
+}
+
+fn asr_pcm_16k_chunks(turn: &FinishedSpeechTurn) -> Result<Vec<Vec<i16>>> {
+ if turn.samples.is_empty() {
+ return Err(anyhow!("empty turn samples"));
+ }
+ let samples_16k: Vec<i16> = turn.samples.iter().step_by(3).copied().collect();
+ if samples_16k.is_empty() {
+ return Err(anyhow!("empty 16k asr samples"));
+ }
+ let samples_per_chunk = 320usize;
+ Ok(samples_16k
+ .chunks(samples_per_chunk)
+ .map(|chunk| chunk.to_vec())
+ .collect())
+}
+
+fn pcm_i16_to_le_bytes(samples: &[i16]) -> Vec<u8> {
+ let mut bytes = Vec::with_capacity(samples.len() * 2);
+ for sample in samples {
+ bytes.extend_from_slice(&sample.to_le_bytes());
+ }
+ bytes
+}
+
async fn request_turn_bridge_stream(
http: &Client,
bridge_config: &TurnBridgeConfig,
@@ -1013,6 +1374,7 @@
turn: &FinishedSpeechTurn,
path_ref: &str,
byte_size: u64,
+ asr_result_ref: Option<&str>,
turn_pipeline_started_at: Instant,
) -> Result<RuntimeTurnStreamOutcome> {
let bridge_started_at = Instant::now();
@@ -1020,7 +1382,7 @@
call_id: call_id.to_string(),
trace_id: trace_id.to_string(),
turn_id: turn.turn_id.clone(),
- audio_artifact: RuntimeTurnAudioArtifact {
+ audio_artifact: Some(RuntimeTurnAudioArtifact {
artifact_type: "local_file".to_string(),
path_ref: path_ref.to_string(),
format: "wav".to_string(),
@@ -1028,7 +1390,8 @@
channels: u32::from(USER_AUDIO_NUM_CHANNELS),
duration_ms: turn.duration_ms,
byte_size,
- },
+ }),
+ asr_result_ref: asr_result_ref.map(str::to_string),
};
let response = http
.post(bridge_config.bridge_url.as_deref().unwrap_or_default())
@@ -1107,6 +1470,7 @@
.await?;
}
}
+ state.close_timing();
if !state.completed {
warn!(
call_id = %call_id,
@@ -1236,6 +1600,7 @@
call_id,
trace_id,
turn,
+ &event,
audio_chunk,
state,
turn_pipeline_started_at,
@@ -1247,6 +1612,9 @@
}
Some("device_output") => {
if let Some(output) = event.device_output.as_ref() {
+ if !should_publish_device_output(&mut state.published_device_output_ids, output) {
+ return Ok(());
+ }
sink.publish_device_output(call_id, trace_id, &turn.turn_id, output)
.await?;
state.device_output_count = state.device_output_count.saturating_add(1);
@@ -1254,6 +1622,7 @@
}
Some("turn_completed") => {
state.completed = true;
+ state.close_timing();
info!(
call_id = %call_id,
trace_id = %trace_id,
@@ -1264,6 +1633,7 @@
);
}
Some("turn_failed") => {
+ state.close_timing();
let error = event.error.as_ref();
let reason_code = error
.and_then(|value| value.reason_code.as_deref())
@@ -1298,6 +1668,7 @@
}
Some("turn_cancelled") => {
state.completed = true;
+ state.close_timing();
info!(
call_id = %call_id,
trace_id = %trace_id,
@@ -1330,6 +1701,19 @@
}
Some("activity") => {
if let Some(activity) = event.activity.as_ref() {
+ if activity.event_type.as_deref() == Some("tts_first_audio_chunk_ready") {
+ if let Some(runtime_session_nonce) =
+ bridge_config.runtime_session_nonce.as_deref()
+ {
+ let _ = state.arm_timing_anchor(
+ call_id,
+ trace_id,
+ &turn.turn_id,
+ runtime_session_nonce,
+ &event,
+ );
+ }
+ }
info!(
call_id = %call_id,
trace_id = %trace_id,
@@ -1388,6 +1772,7 @@
call_id: &str,
trace_id: &str,
turn: &FinishedSpeechTurn,
+ event: &RuntimeTurnStreamEvent,
audio_chunk: &RuntimeTurnStreamAudioChunk,
state: &mut RuntimeTurnStreamState,
turn_pipeline_started_at: Instant,
@@ -1407,6 +1792,58 @@
.unwrap_or("pcm_s16le")
.trim()
.to_ascii_lowercase();
+ if !matches!(format.as_str(), "pcm_s16le" | "mp3" | "mpeg" | "wav") {
+ return Err(anyhow!("unsupported reply_audio_chunk format {format}"));
+ }
+ match state.reply_chunk_markers.observe(audio_chunk.segment_seq) {
+ ReplyChunkMarker::FirstReply => {
+ let extension = json!({
+ "segmentSeq": audio_chunk.segment_seq,
+ "chunkSeq": audio_chunk.chunk_seq,
+ "format": format.as_str(),
+ "bytes": payload.len(),
+ });
+ if let Some(marker) =
+ state.record_m6(call_id, trace_id, &turn.turn_id, event, audio_chunk)
+ {
+ emit_anchored_activity(
+ call_id,
+ trace_id,
+ &turn.turn_id,
+ "helper_first_reply_audio_chunk_received",
+ &marker,
+ extension,
+ );
+ } else {
+ emit_activity(
+ call_id,
+ trace_id,
+ Some(&turn.turn_id),
+ "helper_first_reply_audio_chunk_received",
+ "ok",
+ None,
+ None,
+ extension,
+ );
+ }
+ }
+ ReplyChunkMarker::SegmentFirst => emit_activity(
+ call_id,
+ trace_id,
+ Some(&turn.turn_id),
+ "helper_segment_first_audio_chunk_received",
+ "ok",
+ None,
+ None,
+ json!({
+ "segmentSeq": audio_chunk.segment_seq,
+ "chunkSeq": audio_chunk.chunk_seq,
+ "format": format.as_str(),
+ "bytes": payload.len(),
+ }),
+ ),
+ ReplyChunkMarker::None => {}
+ }
let frames = if format == "pcm_s16le" {
let sample_rate = audio_chunk.sample_rate.unwrap_or(sink.sample_rate_hz);
let channels = audio_chunk.channels.unwrap_or(u32::from(sink.num_channels));
@@ -1498,7 +1935,7 @@
Err(error) => return Err(error).context("failed to decode final stream audio chunk"),
}
} else {
- return Err(anyhow!("unsupported reply_audio_chunk format {format}"));
+ unreachable!("supported encoded format checked above")
};
if frames.is_empty() {
return Ok(0);
@@ -1526,21 +1963,33 @@
sink.write_pcm_frame(frame).await?;
if !state.first_audio_frame_written {
state.first_audio_frame_written = true;
- emit_activity(
- call_id,
- trace_id,
- Some(&turn.turn_id),
- "bot_reply_first_audio_frame_written",
- "ok",
- None,
- None,
- json!({
- "replyPlaybackMode": state.reply_playback_mode.as_str(),
- "format": format.as_str(),
- "chunkSeq": audio_chunk.chunk_seq,
- "replyTotalAfterVadEndMs": turn_pipeline_started_at.elapsed().as_millis() as u64,
- }),
- );
+ let extension = json!({
+ "replyPlaybackMode": state.reply_playback_mode.as_str(),
+ "format": format.as_str(),
+ "chunkSeq": audio_chunk.chunk_seq,
+ "replyTotalAfterVadEndMs": turn_pipeline_started_at.elapsed().as_millis() as u64,
+ });
+ if let Some(marker) = state.record_m7() {
+ emit_anchored_activity(
+ call_id,
+ trace_id,
+ &turn.turn_id,
+ "bot_reply_first_audio_frame_written",
+ &marker,
+ extension,
+ );
+ } else {
+ emit_activity(
+ call_id,
+ trace_id,
+ Some(&turn.turn_id),
+ "bot_reply_first_audio_frame_written",
+ "ok",
+ None,
+ None,
+ extension,
+ );
+ }
}
sleep_until(pacing_started_at + Duration::from_millis(((index + 1) as u64) * 20)).await;
}
@@ -1621,6 +2070,7 @@
turn: &FinishedSpeechTurn,
path_ref: &str,
byte_size: u64,
+ asr_result_ref: Option<&str>,
turn_pipeline_started_at: Instant,
) -> Result<RuntimeTurnBridgeOutcome> {
let bridge_started_at = Instant::now();
@@ -1628,7 +2078,7 @@
call_id: call_id.to_string(),
trace_id: trace_id.to_string(),
turn_id: turn.turn_id.clone(),
- audio_artifact: RuntimeTurnAudioArtifact {
+ audio_artifact: Some(RuntimeTurnAudioArtifact {
artifact_type: "local_file".to_string(),
path_ref: path_ref.to_string(),
format: "wav".to_string(),
@@ -1636,7 +2086,8 @@
channels: u32::from(USER_AUDIO_NUM_CHANNELS),
duration_ms: turn.duration_ms,
byte_size,
- },
+ }),
+ asr_result_ref: asr_result_ref.map(str::to_string),
};
let response = http
.post(bridge_config.bridge_url.as_deref().unwrap_or_default())
@@ -1927,7 +2378,10 @@
#[serde(rename = "turnId")]
turn_id: String,
#[serde(rename = "audioArtifact")]
- audio_artifact: RuntimeTurnAudioArtifact,
+ #[serde(skip_serializing_if = "Option::is_none")]
+ audio_artifact: Option<RuntimeTurnAudioArtifact>,
+ #[serde(rename = "asrResultRef", skip_serializing_if = "Option::is_none")]
+ asr_result_ref: Option<String>,
}
#[derive(Serialize)]
@@ -1955,6 +2409,20 @@
retryable: Option<bool>,
}
+#[derive(Deserialize)]
+#[serde(rename_all = "camelCase")]
+struct RuntimeAsrStreamResp {
+ status: Option<String>,
+ asr_result_ref: Option<String>,
+ chunk_count: Option<u64>,
+ audio_bytes: Option<u64>,
+ asr_duration_ms: Option<u64>,
+ provider_alias: Option<String>,
+ text_len: Option<u64>,
+ fallback_reason: Option<String>,
+ fallback_stage: Option<String>,
+}
+
#[derive(Default)]
struct RuntimeTurnBridgeOutcome {
reply_audio_artifact: Option<RuntimeTurnReplyAudioArtifact>,
@@ -1975,9 +2443,12 @@
completed: bool,
audio_chunk_count: u64,
device_output_count: u64,
+ published_device_output_ids: HashSet<String>,
encoded_audio_buffer: Vec<u8>,
pcm_stream_decoder: Option<audio::PcmS16leStreamDecoder>,
pcm_stream_network_chunk_count: u64,
+ reply_chunk_markers: ReplyChunkMarkerState,
+ timing: RuntimeTurnStreamTimingState,
}
impl Default for RuntimeTurnStreamState {
@@ -1990,10 +2461,266 @@
completed: false,
audio_chunk_count: 0,
device_output_count: 0,
+ published_device_output_ids: HashSet::new(),
encoded_audio_buffer: Vec::new(),
pcm_stream_decoder: None,
pcm_stream_network_chunk_count: 0,
+ reply_chunk_markers: ReplyChunkMarkerState::default(),
+ timing: RuntimeTurnStreamTimingState::default(),
}
+ }
+}
+
+impl RuntimeTurnStreamState {
+ fn arm_timing_anchor(
+ &mut self,
+ call_id: &str,
+ trace_id: &str,
+ turn_id: &str,
+ runtime_session_nonce: &str,
+ event: &RuntimeTurnStreamEvent,
+ ) -> bool {
+ if self.timing.phase != RuntimeTurnStreamTimingPhase::Empty
+ || event.call_id.as_deref() != Some(call_id)
+ || event.trace_id.as_deref() != Some(trace_id)
+ || event.turn_id.as_deref() != Some(turn_id)
+ {
+ return false;
+ }
+ let Some(activity) = event.activity.as_ref() else {
+ return false;
+ };
+ if activity.event_type.as_deref() != Some("tts_first_audio_chunk_ready") {
+ return false;
+ }
+ let Some(extension) = activity.extension.as_ref() else {
+ return false;
+ };
+ let Some(anchor_id) = extension.stream_anchor_id.as_deref() else {
+ return false;
+ };
+ let valid_anchor_id = (16..=64).contains(&anchor_id.len()) && anchor_id.is_ascii();
+ let expected_nonce_hash = runtime_session_nonce_hash(runtime_session_nonce);
+ if extension.stream_timing_version != Some(STREAM_TIMING_VERSION)
+ || !valid_anchor_id
+ || extension.runtime_session_nonce_hash.as_deref() != Some(expected_nonce_hash.as_str())
+ || extension.segment_seq != Some(STREAM_TIMING_FIRST_SEGMENT)
+ || extension.stream_timing_validation.as_deref() != Some("bound")
+ {
+ return false;
+ }
+ let Some(server_delta_ms) = extension.stream_anchor_server_delta_ms else {
+ return false;
+ };
+ self.timing.anchor = Some(RuntimeTurnStreamTimingAnchor {
+ call_id: call_id.to_string(),
+ trace_id: trace_id.to_string(),
+ turn_id: turn_id.to_string(),
+ anchor_id: anchor_id.to_string(),
+ server_delta_ms,
+ runtime_session_nonce_hash: expected_nonce_hash,
+ segment_seq: STREAM_TIMING_FIRST_SEGMENT,
+ received_at: Instant::now(),
+ });
+ self.timing.phase = RuntimeTurnStreamTimingPhase::Armed;
+ true
+ }
+
+ fn record_m6(
+ &mut self,
+ call_id: &str,
+ trace_id: &str,
+ turn_id: &str,
+ event: &RuntimeTurnStreamEvent,
+ audio_chunk: &RuntimeTurnStreamAudioChunk,
+ ) -> Option<RuntimeTurnStreamTimingMarker> {
+ if self.timing.phase != RuntimeTurnStreamTimingPhase::Armed {
+ return None;
+ }
+ let anchor = self.timing.anchor.as_ref()?;
+ if anchor.call_id != call_id
+ || anchor.trace_id != trace_id
+ || anchor.turn_id != turn_id
+ || event.call_id.as_deref() != Some(call_id)
+ || event.trace_id.as_deref() != Some(trace_id)
+ || event.turn_id.as_deref() != Some(turn_id)
+ || audio_chunk.segment_seq != Some(anchor.segment_seq)
+ || audio_chunk.chunk_seq != Some(STREAM_TIMING_FIRST_CHUNK)
+ || audio_chunk.stream_timing_version != Some(STREAM_TIMING_VERSION)
+ || audio_chunk.stream_anchor_id.as_deref() != Some(anchor.anchor_id.as_str())
+ {
+ return None;
+ }
+ let Some(marker) = RuntimeTurnStreamTimingMarker::from_anchor(
+ anchor,
+ audio_chunk.chunk_seq.unwrap_or(STREAM_TIMING_FIRST_CHUNK),
+ ) else {
+ self.close_timing();
+ return None;
+ };
+ self.timing.chunk_seq = Some(marker.chunk_seq);
+ self.timing.phase = RuntimeTurnStreamTimingPhase::M6Recorded;
+ Some(marker)
+ }
+
+ fn record_m7(&mut self) -> Option<RuntimeTurnStreamTimingMarker> {
+ if self.timing.phase != RuntimeTurnStreamTimingPhase::M6Recorded {
+ return None;
+ }
+ let anchor = self.timing.anchor.as_ref()?;
+ let Some(marker) = RuntimeTurnStreamTimingMarker::from_anchor(
+ anchor,
+ self.timing.chunk_seq.unwrap_or(STREAM_TIMING_FIRST_CHUNK),
+ ) else {
+ self.close_timing();
+ return None;
+ };
+ self.timing.phase = RuntimeTurnStreamTimingPhase::M7Recorded;
+ Some(marker)
+ }
+
+ fn close_timing(&mut self) {
+ self.timing.close();
+ }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+enum RuntimeTurnStreamTimingPhase {
+ Empty,
+ Armed,
+ M6Recorded,
+ M7Recorded,
+ Closed,
+}
+
+struct RuntimeTurnStreamTimingState {
+ phase: RuntimeTurnStreamTimingPhase,
+ anchor: Option<RuntimeTurnStreamTimingAnchor>,
+ chunk_seq: Option<u64>,
+}
+
+impl Default for RuntimeTurnStreamTimingState {
+ fn default() -> Self {
+ Self {
+ phase: RuntimeTurnStreamTimingPhase::Empty,
+ anchor: None,
+ chunk_seq: None,
+ }
+ }
+}
+
+impl RuntimeTurnStreamTimingState {
+ fn close(&mut self) {
+ self.anchor = None;
+ self.chunk_seq = None;
+ self.phase = RuntimeTurnStreamTimingPhase::Closed;
+ }
+}
+
+impl Drop for RuntimeTurnStreamTimingState {
+ fn drop(&mut self) {
+ self.anchor = None;
+ self.chunk_seq = None;
+ }
+}
+
+struct RuntimeTurnStreamTimingAnchor {
+ call_id: String,
+ trace_id: String,
+ turn_id: String,
+ anchor_id: String,
+ server_delta_ms: u64,
+ runtime_session_nonce_hash: String,
+ segment_seq: u64,
+ received_at: Instant,
+}
+
+struct RuntimeTurnStreamTimingMarker {
+ version: u32,
+ anchor_id: String,
+ anchor_server_delta_ms: u64,
+ anchor_elapsed_ms: u64,
+ server_delta_ms: u64,
+ runtime_session_nonce_hash: String,
+ segment_seq: u64,
+ chunk_seq: u64,
+}
+
+impl RuntimeTurnStreamTimingMarker {
+ fn from_anchor(anchor: &RuntimeTurnStreamTimingAnchor, chunk_seq: u64) -> Option<Self> {
+ let elapsed_ms = u64::try_from(anchor.received_at.elapsed().as_millis()).ok()?;
+ if elapsed_ms > STREAM_TIMING_MAX_ELAPSED_MS {
+ return None;
+ }
+ Some(Self {
+ version: STREAM_TIMING_VERSION,
+ anchor_id: anchor.anchor_id.clone(),
+ anchor_server_delta_ms: anchor.server_delta_ms,
+ anchor_elapsed_ms: elapsed_ms,
+ server_delta_ms: anchor.server_delta_ms.checked_add(elapsed_ms)?,
+ runtime_session_nonce_hash: anchor.runtime_session_nonce_hash.clone(),
+ segment_seq: anchor.segment_seq,
+ chunk_seq,
+ })
+ }
+
+ fn extension_with(&self, extra: serde_json::Value) -> serde_json::Value {
+ let mut extension = match extra {
+ serde_json::Value::Object(value) => value,
+ _ => serde_json::Map::new(),
+ };
+ extension.insert("streamTimingVersion".to_string(), json!(self.version));
+ extension.insert("streamAnchorId".to_string(), json!(self.anchor_id));
+ extension.insert(
+ "streamAnchorServerDeltaMs".to_string(),
+ json!(self.anchor_server_delta_ms),
+ );
+ extension.insert("anchorElapsedMs".to_string(), json!(self.anchor_elapsed_ms));
+ extension.insert(
+ "runtimeSessionNonceHash".to_string(),
+ json!(self.runtime_session_nonce_hash),
+ );
+ extension.insert("segmentSeq".to_string(), json!(self.segment_seq));
+ extension.insert("chunkSeq".to_string(), json!(self.chunk_seq));
+ extension.insert("streamTimingValidation".to_string(), json!("bound"));
+ serde_json::Value::Object(extension)
+ }
+}
+
+fn runtime_session_nonce_hash(value: &str) -> String {
+ let digest = Sha256::digest(value.as_bytes());
+ digest[..6]
+ .iter()
+ .map(|byte| format!("{byte:02x}"))
+ .collect()
+}
+
+#[derive(Debug, PartialEq, Eq)]
+enum ReplyChunkMarker {
+ FirstReply,
+ SegmentFirst,
+ None,
+}
+
+#[derive(Default)]
+struct ReplyChunkMarkerState {
+ first_reply_seen: bool,
+ seen_segments: HashSet<u64>,
+}
+
+impl ReplyChunkMarkerState {
+ fn observe(&mut self, segment_seq: Option<u64>) -> ReplyChunkMarker {
+ let first_for_segment = segment_seq
+ .map(|value| self.seen_segments.insert(value))
+ .unwrap_or(false);
+ if !self.first_reply_seen {
+ self.first_reply_seen = true;
+ return ReplyChunkMarker::FirstReply;
+ }
+ if first_for_segment {
+ return ReplyChunkMarker::SegmentFirst;
+ }
+ ReplyChunkMarker::None
}
}
@@ -2013,6 +2740,12 @@
struct RuntimeTurnStreamEvent {
#[serde(rename = "type", alias = "event")]
event_type: Option<String>,
+ #[serde(rename = "callId")]
+ call_id: Option<String>,
+ #[serde(rename = "traceId")]
+ trace_id: Option<String>,
+ #[serde(rename = "turnId")]
+ turn_id: Option<String>,
#[serde(rename = "seq")]
seq: Option<u64>,
#[serde(rename = "replyPlaybackMode")]
@@ -2048,6 +2781,12 @@
struct RuntimeTurnStreamAudioChunk {
#[serde(rename = "chunkSeq", alias = "seq")]
chunk_seq: Option<u64>,
+ #[serde(rename = "segmentSeq")]
+ segment_seq: Option<u64>,
+ #[serde(rename = "streamTimingVersion")]
+ stream_timing_version: Option<u32>,
+ #[serde(rename = "streamAnchorId")]
+ stream_anchor_id: Option<String>,
format: Option<String>,
#[serde(rename = "sampleRate")]
sample_rate: Option<u32>,
@@ -2064,6 +2803,23 @@
stage: Option<String>,
#[serde(rename = "reasonCode")]
reason_code: Option<String>,
+ extension: Option<RuntimeTurnStreamTimingExtension>,
+}
+
+#[derive(Clone, Deserialize)]
+struct RuntimeTurnStreamTimingExtension {
+ #[serde(rename = "streamTimingVersion")]
+ stream_timing_version: Option<u32>,
+ #[serde(rename = "streamAnchorId")]
+ stream_anchor_id: Option<String>,
+ #[serde(rename = "streamAnchorServerDeltaMs")]
+ stream_anchor_server_delta_ms: Option<u64>,
+ #[serde(rename = "runtimeSessionNonceHash")]
+ runtime_session_nonce_hash: Option<String>,
+ #[serde(rename = "segmentSeq")]
+ segment_seq: Option<u64>,
+ #[serde(rename = "streamTimingValidation")]
+ stream_timing_validation: Option<String>,
}
#[derive(Deserialize)]
@@ -2096,6 +2852,30 @@
#[serde(rename = "commandCode")]
command_code: Option<String>,
params: Option<serde_json::Value>,
+}
+
+fn should_publish_device_output(
+ published_ids: &mut HashSet<String>,
+ output: &RuntimeTurnDeviceOutput,
+) -> bool {
+ let Some(command_id) = output
+ .command_id
+ .as_deref()
+ .map(str::trim)
+ .filter(|value| !value.is_empty())
+ else {
+ return false;
+ };
+ if output
+ .command_code
+ .as_deref()
+ .map(str::trim)
+ .filter(|value| !value.is_empty())
+ .is_none()
+ {
+ return false;
+ }
+ published_ids.insert(command_id.to_string())
}
fn require_safe_segment(value: &str) -> Result<()> {
@@ -2137,6 +2917,7 @@
turn_bridge_config: TurnBridgeConfig,
http: Client,
sink: Arc<BotAudioOutputSink>,
+ participant: RemoteParticipant,
) -> JoinHandle<()> {
tokio::spawn(async move {
let mut stream = NativeAudioStream::new(
@@ -2145,6 +2926,68 @@
i32::from(USER_AUDIO_NUM_CHANNELS),
);
let started_at = Instant::now();
+ let (frame_tx, mut frame_rx) =
+ mpsc::channel::<DrainedUserAudioFrame>(INBOUND_AUDIO_QUEUE_CAPACITY);
+ let (drain_shutdown_tx, mut drain_shutdown_rx) = watch::channel(false);
+ let drain_call_id = call_id.clone();
+ let drain_trace_id = trace_id.clone();
+ let drain_task = tokio::spawn(async move {
+ let mut received_frame_count: u64 = 0;
+ let mut dropped_frame_count: u64 = 0;
+ loop {
+ tokio::select! {
+ changed = drain_shutdown_rx.changed() => {
+ match changed {
+ Ok(()) if *drain_shutdown_rx.borrow() => break,
+ Ok(()) => {}
+ Err(_) => break,
+ }
+ }
+ maybe_frame = stream.next() => {
+ let Some(frame) = maybe_frame else {
+ break;
+ };
+ received_frame_count = received_frame_count.saturating_add(1);
+ let drained = DrainedUserAudioFrame {
+ frame_index: received_frame_count,
+ captured_elapsed_ms: started_at.elapsed().as_millis() as u64,
+ frame: AudioFrame {
+ data: Cow::Owned(frame.data.as_ref().to_vec()),
+ sample_rate: frame.sample_rate,
+ num_channels: frame.num_channels,
+ samples_per_channel: frame.samples_per_channel,
+ },
+ };
+ match frame_tx.try_send(drained) {
+ Ok(()) => {}
+ Err(mpsc::error::TrySendError::Full(_)) => {
+ dropped_frame_count = dropped_frame_count.saturating_add(1);
+ if dropped_frame_count % INBOUND_AUDIO_DROP_LOG_INTERVAL == 1 {
+ warn!(
+ call_id = %drain_call_id,
+ trace_id = %drain_trace_id,
+ dropped_frame_count,
+ received_frame_count,
+ queue_capacity = INBOUND_AUDIO_QUEUE_CAPACITY,
+ "runtime helper inbound_audio_queue_full_dropping_newest"
+ );
+ }
+ }
+ Err(mpsc::error::TrySendError::Closed(_)) => break,
+ }
+ }
+ }
+ }
+ stream.close();
+ info!(
+ call_id = %drain_call_id,
+ trace_id = %drain_trace_id,
+ received_frame_count,
+ dropped_frame_count,
+ queue_capacity = INBOUND_AUDIO_QUEUE_CAPACITY,
+ "runtime helper user_audio_drain_ended"
+ );
+ });
let mut frame_count: u64 = 0;
let mut sample_count: u64 = 0;
let mut simple_vad = if simple_vad_enabled {
@@ -2152,11 +2995,13 @@
} else {
None
};
+ let mut realtime_asr_upload: Option<RealtimeAsrUpload> = None;
- while let Some(frame) = stream.next().await {
- frame_count += 1;
+ while let Some(drained) = frame_rx.recv().await {
+ let frame = drained.frame;
+ frame_count = drained.frame_index;
sample_count += u64::from(frame.samples_per_channel) * u64::from(frame.num_channels);
- let elapsed_ms = started_at.elapsed().as_millis() as u64;
+ let elapsed_ms = drained.captured_elapsed_ms;
if frame_count == 1 {
info!(
call_id = %call_id,
@@ -2184,7 +3029,8 @@
if let Some(vad) = simple_vad.as_mut() {
if vad_enabled_gate.load(Ordering::Acquire) {
- if let Some(turn) = vad.observe_frame(
+ let was_in_speech = vad.in_speech;
+ let turn = vad.observe_frame(
&call_id,
&trace_id,
&participant_alias,
@@ -2192,7 +3038,66 @@
frame_count,
elapsed_ms,
&frame,
- ) {
+ );
+ let is_in_speech = vad.in_speech;
+
+ if !was_in_speech && is_in_speech {
+ let turn_id = format!("turn-{:04}", vad.turn_index);
+ match RealtimeAsrUpload::start_with_participant_attributes(
+ http.clone(),
+ turn_bridge_config.realtime_asr_config(),
+ &call_id,
+ &trace_id,
+ &turn_id,
+ &vad.speech_samples,
+ || participant.attributes(),
+ ) {
+ Ok(upload) => {
+ info!(
+ call_id = %call_id,
+ trace_id = %trace_id,
+ turn_id = %turn_id,
+ "runtime helper asr_realtime_session_started"
+ );
+ realtime_asr_upload = Some(upload);
+ }
+ Err(error) if turn_bridge_config.asr_realtime_enabled => {
+ warn!(
+ call_id = %call_id,
+ trace_id = %trace_id,
+ turn_id = %turn_id,
+ error = %safe_error(&error.to_string()),
+ "runtime helper asr_realtime_start_failed_fallback"
+ );
+ }
+ Err(_) => {}
+ }
+ } else if was_in_speech {
+ let push_failed = realtime_asr_upload
+ .as_mut()
+ .and_then(|upload| upload.push_48k_samples(frame.data.as_ref()).err());
+ if let Some(error) = push_failed {
+ warn!(
+ call_id = %call_id,
+ trace_id = %trace_id,
+ error = %safe_error(&error.to_string()),
+ "runtime helper asr_realtime_upload_failed_fallback"
+ );
+ if let Some(upload) = realtime_asr_upload.take() {
+ tokio::spawn(async move {
+ upload.cancel("upload_backpressure").await;
+ });
+ }
+ }
+ }
+
+ if let Some(turn) = turn {
+ let realtime_asr_result_ref = match realtime_asr_upload.take() {
+ Some(upload) => {
+ finish_realtime_asr_upload(upload, &call_id, &trace_id, &turn).await
+ }
+ None => None,
+ };
handle_finished_turn(
&http,
&turn_bridge_config,
@@ -2200,10 +3105,22 @@
&call_id,
&trace_id,
turn,
+ realtime_asr_result_ref,
)
.await;
+ } else if was_in_speech && !is_in_speech {
+ if let Some(upload) = realtime_asr_upload.take() {
+ tokio::spawn(async move {
+ upload.cancel("speech_too_short").await;
+ });
+ }
}
} else {
+ if let Some(upload) = realtime_asr_upload.take() {
+ tokio::spawn(async move {
+ upload.cancel("vad_disabled").await;
+ });
+ }
vad.observe_disabled_frame(
&call_id,
&trace_id,
@@ -2224,9 +3141,41 @@
&track_sid_alias,
started_at.elapsed().as_millis() as u64,
) {
- handle_finished_turn(&http, &turn_bridge_config, &sink, &call_id, &trace_id, turn)
- .await;
+ let realtime_asr_result_ref = match realtime_asr_upload.take() {
+ Some(upload) => {
+ finish_realtime_asr_upload(upload, &call_id, &trace_id, &turn).await
+ }
+ None => None,
+ };
+ handle_finished_turn(
+ &http,
+ &turn_bridge_config,
+ &sink,
+ &call_id,
+ &trace_id,
+ turn,
+ realtime_asr_result_ref,
+ )
+ .await;
}
+ }
+ if let Some(upload) = realtime_asr_upload.take() {
+ upload.cancel("stream_end").await;
+ }
+ let _ = drain_shutdown_tx.send(true);
+ let mut drain_task = drain_task;
+ if timeout(AUDIO_DRAIN_STOP_GRACE, &mut drain_task)
+ .await
+ .is_err()
+ {
+ drain_task.abort();
+ let _ = drain_task.await;
+ warn!(
+ call_id = %call_id,
+ trace_id = %trace_id,
+ stop_grace_ms = AUDIO_DRAIN_STOP_GRACE.as_millis() as u64,
+ "runtime helper user_audio_drain_stop_timeout"
+ );
}
info!(
@@ -2240,6 +3189,64 @@
"runtime helper user_audio_stream_ended"
);
})
+}
+
+struct DrainedUserAudioFrame {
+ frame_index: u64,
+ captured_elapsed_ms: u64,
+ frame: AudioFrame<'static>,
+}
+
+async fn finish_realtime_asr_upload(
+ upload: RealtimeAsrUpload,
+ call_id: &str,
+ trace_id: &str,
+ turn: &FinishedSpeechTurn,
+) -> Option<String> {
+ match upload.finish(turn.duration_ms, &turn.end_reason).await {
+ Ok(RealtimeAsrOutcome {
+ status,
+ asr_result_ref,
+ provider_alias,
+ partial_count,
+ fallback_reason,
+ fallback_stage,
+ chunk_count,
+ audio_bytes,
+ wall_ms,
+ }) => {
+ info!(
+ call_id = %call_id,
+ trace_id = %trace_id,
+ turn_id = %turn.turn_id,
+ status = %status,
+ provider_alias = ?provider_alias,
+ partial_count,
+ chunk_count,
+ audio_bytes,
+ wall_ms,
+ asr_result_ref_present = asr_result_ref.is_some(),
+ fallback_reason = ?fallback_reason,
+ fallback_stage = ?fallback_stage,
+ "runtime helper asr_realtime_finished"
+ );
+ if status == "final" {
+ asr_result_ref
+ } else {
+ None
+ }
+ }
+ Err(error) => {
+ warn!(
+ call_id = %call_id,
+ trace_id = %trace_id,
+ turn_id = %turn.turn_id,
+ error = %safe_error(&error.to_string()),
+ "runtime helper asr_realtime_failed_fallback"
+ );
+ None
+ }
+ }
}
#[derive(Clone)]
@@ -2933,3 +3940,142 @@
Ok(())
}
}
+
+#[cfg(test)]
+mod tests {
+ use super::{
+ ReplyChunkMarker, ReplyChunkMarkerState, RuntimeTurnDeviceOutput, RuntimeTurnStreamEvent,
+ RuntimeTurnStreamState, RuntimeTurnStreamTimingPhase, runtime_session_nonce_hash,
+ should_publish_device_output,
+ };
+ use std::collections::HashSet;
+
+ #[test]
+ fn reply_chunk_marker_state_emits_turn_first_once_and_later_segment_first_once() {
+ let mut state = ReplyChunkMarkerState::default();
+
+ assert_eq!(ReplyChunkMarker::FirstReply, state.observe(Some(1)));
+ assert_eq!(ReplyChunkMarker::None, state.observe(Some(1)));
+ assert_eq!(ReplyChunkMarker::SegmentFirst, state.observe(Some(2)));
+ assert_eq!(ReplyChunkMarker::None, state.observe(Some(2)));
+ assert_eq!(ReplyChunkMarker::SegmentFirst, state.observe(Some(3)));
+ }
+
+ #[test]
+ fn reply_chunk_marker_state_without_segment_only_emits_turn_first() {
+ let mut state = ReplyChunkMarkerState::default();
+
+ assert_eq!(ReplyChunkMarker::FirstReply, state.observe(None));
+ assert_eq!(ReplyChunkMarker::None, state.observe(None));
+ }
+
+ #[test]
+ fn runtime_turn_stream_audio_chunk_reads_segment_seq() {
+ let event: RuntimeTurnStreamEvent = serde_json::from_str(
+ r#"{"type":"reply_audio_chunk","audioChunk":{"chunkSeq":4,"segmentSeq":2,"format":"pcm_s16le","payloadBase64":"AA==","last":false}}"#,
+ )
+ .expect("turn stream event");
+
+ assert_eq!(
+ Some(2),
+ event.audio_chunk.and_then(|chunk| chunk.segment_seq)
+ );
+ }
+
+ #[test]
+ fn runtime_turn_stream_reads_frozen_m5_and_audio_chunk_timing_contract() {
+ let activity: RuntimeTurnStreamEvent = serde_json::from_str(
+ r#"{"type":"activity","callId":"call-1","traceId":"trace-1","turnId":"turn-1","activity":{"eventType":"tts_first_audio_chunk_ready","extension":{"streamTimingVersion":1,"streamAnchorId":"0123456789abcdef","streamAnchorServerDeltaMs":1200,"runtimeSessionNonceHash":"abcdef012345","segmentSeq":1,"streamTimingValidation":"bound"}}}"#,
+ )
+ .expect("m5 activity event");
+ let audio: RuntimeTurnStreamEvent = serde_json::from_str(
+ r#"{"type":"reply_audio_chunk","callId":"call-1","traceId":"trace-1","turnId":"turn-1","audioChunk":{"chunkSeq":1,"segmentSeq":1,"streamTimingVersion":1,"streamAnchorId":"0123456789abcdef","format":"pcm_s16le","payloadBase64":"AA==","last":false}}"#,
+ )
+ .expect("timed audio chunk event");
+
+ assert_eq!(Some("call-1"), activity.call_id.as_deref());
+ assert_eq!(Some("trace-1"), activity.trace_id.as_deref());
+ assert_eq!(Some("turn-1"), activity.turn_id.as_deref());
+ let extension = activity.activity.unwrap().extension.unwrap();
+ assert_eq!(Some(1), extension.stream_timing_version);
+ assert_eq!(Some(1200), extension.stream_anchor_server_delta_ms);
+ assert_eq!(
+ Some("0123456789abcdef"),
+ audio.audio_chunk.unwrap().stream_anchor_id.as_deref()
+ );
+ }
+
+ #[test]
+ fn runtime_turn_stream_timing_records_m6_and_m7_once_then_rejects_terminal_late_events() {
+ let nonce = "runtime-nonce";
+ let nonce_hash = runtime_session_nonce_hash(nonce);
+ let activity: RuntimeTurnStreamEvent = serde_json::from_str(&format!(
+ r#"{{"type":"activity","callId":"call-1","traceId":"trace-1","turnId":"turn-1","activity":{{"eventType":"tts_first_audio_chunk_ready","extension":{{"streamTimingVersion":1,"streamAnchorId":"0123456789abcdef","streamAnchorServerDeltaMs":1200,"runtimeSessionNonceHash":"{nonce_hash}","segmentSeq":1,"streamTimingValidation":"bound"}}}}}}"#,
+ ))
+ .expect("m5 activity event");
+ let audio: RuntimeTurnStreamEvent = serde_json::from_str(
+ r#"{"type":"reply_audio_chunk","callId":"call-1","traceId":"trace-1","turnId":"turn-1","audioChunk":{"chunkSeq":1,"segmentSeq":1,"streamTimingVersion":1,"streamAnchorId":"0123456789abcdef","format":"pcm_s16le","payloadBase64":"AA==","last":false}}"#,
+ )
+ .expect("timed audio chunk event");
+ let mut state = RuntimeTurnStreamState::default();
+
+ assert!(state.arm_timing_anchor("call-1", "trace-1", "turn-1", nonce, &activity));
+ let chunk = audio.audio_chunk.as_ref().expect("audio chunk");
+ assert!(
+ state
+ .record_m6("call-1", "trace-1", "turn-1", &audio, chunk)
+ .is_some()
+ );
+ assert!(
+ state
+ .record_m6("call-1", "trace-1", "turn-1", &audio, chunk)
+ .is_none()
+ );
+ assert!(state.record_m7().is_some());
+ assert!(state.record_m7().is_none());
+
+ state.close_timing();
+ assert_eq!(RuntimeTurnStreamTimingPhase::Closed, state.timing.phase);
+ assert!(state.timing.anchor.is_none());
+ assert!(!state.arm_timing_anchor("call-1", "trace-1", "turn-1", nonce, &activity));
+ assert!(
+ state
+ .record_m6("call-1", "trace-1", "turn-1", &audio, chunk)
+ .is_none()
+ );
+ assert!(state.record_m7().is_none());
+ }
+
+ #[test]
+ fn device_output_contract_is_reliable_and_deduplicated() {
+ let output = RuntimeTurnDeviceOutput {
+ command_id: Some("cmd-1".to_string()),
+ command_code: Some("custom.app.DeviceLevelChange".to_string()),
+ params: Some(serde_json::json!({"level": 1})),
+ };
+ let mut published = HashSet::new();
+ assert!(should_publish_device_output(&mut published, &output));
+ assert!(!should_publish_device_output(&mut published, &output));
+ assert_eq!(published.len(), 1);
+ }
+
+ #[test]
+ fn device_output_invalid_or_missing_command_is_fail_closed() {
+ for output in [
+ RuntimeTurnDeviceOutput {
+ command_id: None,
+ command_code: Some("custom.app.DeviceLevelChange".to_string()),
+ params: None,
+ },
+ RuntimeTurnDeviceOutput {
+ command_id: Some("cmd-1".to_string()),
+ command_code: None,
+ params: None,
+ },
+ ] {
+ let mut published = HashSet::new();
+ assert!(!should_publish_device_output(&mut published, &output));
+ assert!(published.is_empty());
+ }
+ }
+}
--
Gitblit v1.9.3