From 4c14ddfeb8964d09b83f14ce7629e42863fa67ce Mon Sep 17 00:00:00 2001
From: cai <cai@nbcai.cc>
Date: Wed, 12 Aug 2026 15:58:53 +0800
Subject: [PATCH] feat(helper): distinguish current participant visibility
---
src/main.rs | 3661 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++-
1 files changed, 3,548 insertions(+), 113 deletions(-)
diff --git a/src/main.rs b/src/main.rs
index 9e99d6c..af61418 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -1,8 +1,12 @@
+mod asr_realtime;
mod audio;
mod service;
use std::{
+ borrow::Cow,
+ collections::HashSet,
env, fs,
+ future::Future,
path::{Path, PathBuf},
sync::{
Arc,
@@ -12,7 +16,10 @@
};
use anyhow::{Context, Result, anyhow};
-use audio::{AudioDiagnostics, PcmFrame, load_pre_recorded_frames};
+use asr_realtime::{
+ AudioIngressMetadata, RealtimeAsrConfig, RealtimeAsrOutcome, RealtimeAsrUpload,
+};
+use audio::{AudioDiagnostics, load_pre_recorded_frames};
use base64::{Engine as _, engine::general_purpose};
use futures_util::StreamExt;
use libwebrtc::{
@@ -24,19 +31,575 @@
options::TrackPublishOptions,
prelude::{
DataPacket, LocalAudioTrack, LocalTrack, ParticipantIdentity, RemoteAudioTrack,
- RemoteTrack, Room, RoomEvent, RoomOptions,
+ RemoteParticipant, RemoteTrack, Room, RoomEvent, RoomOptions,
},
};
use reqwest::Client;
use serde::{Deserialize, Serialize};
use serde_json::json;
-use tokio::time::{sleep, sleep_until};
-use tokio::{sync::mpsc::UnboundedReceiver, task::JoinHandle};
+use sha2::{Digest, Sha256};
+use tokio::time::{sleep, sleep_until, timeout};
+use tokio::{
+ sync::{mpsc, mpsc::UnboundedReceiver, watch},
+ task::JoinHandle,
+};
use tracing::{info, warn};
-const TARGET_SAMPLE_RATE_HZ: u32 = 48_000;
-const TARGET_NUM_CHANNELS: u16 = 1;
+const USER_AUDIO_SAMPLE_RATE_HZ: u32 = 48_000;
+const USER_AUDIO_NUM_CHANNELS: u16 = 1;
+const INBOUND_AUDIO_QUEUE_CAPACITY: usize = 100;
+const INBOUND_AUDIO_DROP_LOG_INTERVAL: u64 = 100;
+const AUDIO_DRAIN_STOP_GRACE: Duration = Duration::from_secs(2);
+const DEFAULT_BOT_AUDIO_PROFILE: &str = "pcm-16k";
+const LIVEKIT_48K_SAMPLE_RATE_HZ: u32 = 48_000;
+const PCM_16K_SAMPLE_RATE_HZ: u32 = 16_000;
+const BOT_NUM_CHANNELS: u16 = 1;
const TRACK_NAME: &str = "bot-main-audio";
+const STREAM_TIMING_VERSION: u32 = 1;
+const STREAM_TIMING_FIRST_SEGMENT: u64 = 1;
+const STREAM_TIMING_FIRST_CHUNK: u64 = 1;
+const STREAM_TIMING_MAX_ELAPSED_MS: u64 = 5_000;
+const STREAM_TIMING_SOURCE: &str = "stream_anchor_monotonic";
+const CONTROLLED_FIXTURE_PROBE_TOPIC: &str = "controlled_fixture_attribute_probe";
+const CONTROLLED_FIXTURE_ACK_TOPIC: &str = "controlled_fixture_attribute_ack";
+const CONTROLLED_FIXTURE_PROTOCOL_VERSION: u64 = 1;
+const CONTROLLED_FIXTURE_GENERATION: u64 = 1;
+const CONTROLLED_FIXTURE_PROBE_RECHECK_DELAY: Duration = Duration::from_millis(25);
+const CONTROLLED_FIXTURE_PROBE_TTL: Duration = Duration::from_millis(250);
+const CONTROLLED_FIXTURE_POST_EXPIRY_WINDOW: Duration = Duration::from_millis(2_000);
+
+const CONTROLLED_FIXTURE_ACK_RESULTS: [&str; 4] =
+ ["observed", "rejected", "timeout", "publish_failed"];
+const CONTROLLED_FIXTURE_REJECT_REASONS: [&str; 10] = [
+ "missing_attributes",
+ "wrong_source",
+ "missing_sequence",
+ "wrong_sequence",
+ "wrong_participant",
+ "expired",
+ "duplicate_or_old_sequence",
+ "no_current_participant",
+ "ack_publish_failed",
+ "unknown",
+];
+
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+struct ControlledFixtureAttributeProbe {
+ #[serde(rename = "type")]
+ message_type: String,
+ protocol_version: u64,
+ call_id_hash: String,
+ call_trace_id_hash: String,
+ generation: u64,
+ client_fixture_sequence: String,
+}
+
+#[derive(Debug, Serialize)]
+#[serde(rename_all = "camelCase")]
+struct ControlledFixtureAttributeAck {
+ #[serde(rename = "type")]
+ message_type: &'static str,
+ protocol_version: u64,
+ call_id_hash: String,
+ call_trace_id_hash: String,
+ generation: u64,
+ client_fixture_sequence: String,
+ result: &'static str,
+ #[serde(skip_serializing_if = "Option::is_none")]
+ input_source_category: Option<&'static str>,
+ #[serde(skip_serializing_if = "Option::is_none")]
+ reject_reason: Option<&'static str>,
+}
+
+#[derive(Clone, Debug)]
+struct PendingControlledFixtureProbe {
+ sender: ParticipantIdentity,
+ call_id_hash: String,
+ call_trace_id_hash: String,
+ generation: u64,
+ sequence: String,
+ received_at: Instant,
+ expires_at: Instant,
+}
+
+#[derive(Debug, PartialEq, Eq)]
+struct ControlledFixtureVisibilityEvidence {
+ first_visible_bucket: &'static str,
+ visibility_source: &'static str,
+ visibility_result: &'static str,
+ binding_matched: bool,
+}
+
+#[derive(Debug, PartialEq, Eq)]
+struct ControlledFixtureAckPublishOutcome {
+ observed: bool,
+ published: bool,
+}
+
+fn sha256_hex(value: &str) -> String {
+ let mut digest = Sha256::new();
+ digest.update(value.as_bytes());
+ digest
+ .finalize()
+ .iter()
+ .map(|byte| format!("{byte:02x}"))
+ .collect()
+}
+
+fn controlled_fixture_ack_classification(
+ decision: Result<(), &'static str>,
+) -> (&'static str, Option<&'static str>, bool) {
+ match decision {
+ Ok(()) => ("observed", None, true),
+ Err("timeout") => ("timeout", Some("expired"), false),
+ Err(reason) if CONTROLLED_FIXTURE_REJECT_REASONS.contains(&reason) => {
+ ("rejected", Some(reason), false)
+ }
+ Err(_) => ("rejected", Some("unknown"), false),
+ }
+}
+
+fn record_controlled_fixture_probe_event(
+ runtime_call_id: &str,
+ runtime_trace_id: &str,
+ stage: &'static str,
+ call_id_hash: &str,
+ trace_id_hash: &str,
+ generation: u64,
+ sequence: &str,
+ observed: bool,
+ ack_result: Option<&'static str>,
+ reject_reason: Option<&'static str>,
+) {
+ debug_assert!(ack_result.is_none_or(|value| CONTROLLED_FIXTURE_ACK_RESULTS.contains(&value)));
+ debug_assert!(
+ reject_reason.is_none_or(|value| CONTROLLED_FIXTURE_REJECT_REASONS.contains(&value))
+ );
+ println!(
+ "{}",
+ controlled_fixture_probe_event(
+ runtime_call_id,
+ runtime_trace_id,
+ stage,
+ call_id_hash,
+ trace_id_hash,
+ generation,
+ sequence,
+ observed,
+ ack_result,
+ reject_reason,
+ )
+ );
+}
+
+fn controlled_fixture_probe_event(
+ runtime_call_id: &str,
+ runtime_trace_id: &str,
+ stage: &'static str,
+ call_id_hash: &str,
+ trace_id_hash: &str,
+ generation: u64,
+ sequence: &str,
+ observed: bool,
+ ack_result: Option<&'static str>,
+ reject_reason: Option<&'static str>,
+) -> serde_json::Value {
+ json!({
+ "type": "cv_activity",
+ "callId": runtime_call_id,
+ "traceId": runtime_trace_id,
+ "turnId": null,
+ "eventName": "controlled_fixture_attribute_probe",
+ "eventWallTimeMs": current_time_millis(),
+ "result": "ok",
+ "reasonCode": null,
+ "retryable": null,
+ "extension": {
+ "stage": stage,
+ "observed": observed,
+ "ack_result": ack_result,
+ "reject_reason": reject_reason,
+ "call_id_hash": call_id_hash,
+ "trace_id_hash": trace_id_hash,
+ "generation": generation,
+ "sequence_hash": sha256_hex(sequence),
+ },
+ })
+}
+
+fn record_controlled_fixture_attribute_decision(
+ decision: Result<(), &'static str>,
+ runtime_call_id: &str,
+ runtime_trace_id: &str,
+ call_id_hash: &str,
+ trace_id_hash: &str,
+ generation: u64,
+ sequence: &str,
+) -> (&'static str, Option<&'static str>, bool) {
+ let classification = controlled_fixture_ack_classification(decision);
+ record_controlled_fixture_probe_event(
+ runtime_call_id,
+ runtime_trace_id,
+ "attributes_classified",
+ call_id_hash,
+ trace_id_hash,
+ generation,
+ sequence,
+ classification.2,
+ Some(classification.0),
+ classification.1,
+ );
+ classification
+}
+
+async fn complete_controlled_fixture_ack_publish<F, E>(
+ publish: F,
+ runtime_call_id: &str,
+ runtime_trace_id: &str,
+ observed: bool,
+ ack_result: &'static str,
+ reject_reason: Option<&'static str>,
+ call_id_hash: &str,
+ trace_id_hash: &str,
+ generation: u64,
+ sequence: &str,
+ acknowledged_probe_sequences: &mut HashSet<String>,
+) -> ControlledFixtureAckPublishOutcome
+where
+ F: Future<Output = Result<(), E>>,
+{
+ if publish.await.is_ok() {
+ record_controlled_fixture_probe_event(
+ runtime_call_id,
+ runtime_trace_id,
+ "ack_publish_completed",
+ call_id_hash,
+ trace_id_hash,
+ generation,
+ sequence,
+ observed,
+ Some(ack_result),
+ reject_reason,
+ );
+ acknowledged_probe_sequences.insert(sequence.to_string());
+ ControlledFixtureAckPublishOutcome {
+ observed,
+ published: true,
+ }
+ } else {
+ record_controlled_fixture_probe_event(
+ runtime_call_id,
+ runtime_trace_id,
+ "ack_publish_completed",
+ call_id_hash,
+ trace_id_hash,
+ generation,
+ sequence,
+ false,
+ Some("publish_failed"),
+ Some("ack_publish_failed"),
+ );
+ ControlledFixtureAckPublishOutcome {
+ observed: false,
+ published: false,
+ }
+ }
+}
+
+fn controlled_fixture_ack_from_probe(
+ probe: &PendingControlledFixtureProbe,
+ observed: bool,
+ reject_reason: Option<&'static str>,
+) -> ControlledFixtureAttributeAck {
+ ControlledFixtureAttributeAck {
+ message_type: CONTROLLED_FIXTURE_ACK_TOPIC,
+ protocol_version: CONTROLLED_FIXTURE_PROTOCOL_VERSION,
+ call_id_hash: probe.call_id_hash.clone(),
+ call_trace_id_hash: probe.call_trace_id_hash.clone(),
+ generation: probe.generation,
+ client_fixture_sequence: probe.sequence.clone(),
+ result: if observed { "observed" } else { "rejected" },
+ input_source_category: observed.then_some("controlled_fixture"),
+ reject_reason,
+ }
+}
+
+fn controlled_fixture_probe(
+ payload: &[u8],
+ call_id: &str,
+ trace_id: &str,
+ sender: &ParticipantIdentity,
+ expected_participant: Option<&str>,
+) -> Option<PendingControlledFixtureProbe> {
+ if !is_bound_user_participant(sender.as_str(), expected_participant) {
+ return None;
+ }
+ let probe: ControlledFixtureAttributeProbe = serde_json::from_slice(payload).ok()?;
+ if probe.message_type != CONTROLLED_FIXTURE_PROBE_TOPIC
+ || probe.protocol_version != CONTROLLED_FIXTURE_PROTOCOL_VERSION
+ || probe.generation != CONTROLLED_FIXTURE_GENERATION
+ || probe.call_id_hash != sha256_hex(call_id)
+ || probe.call_trace_id_hash != sha256_hex(trace_id)
+ || probe.client_fixture_sequence.trim().is_empty()
+ {
+ return None;
+ }
+ let received_at = Instant::now();
+ Some(PendingControlledFixtureProbe {
+ sender: sender.clone(),
+ call_id_hash: probe.call_id_hash,
+ call_trace_id_hash: probe.call_trace_id_hash,
+ generation: probe.generation,
+ sequence: probe.client_fixture_sequence,
+ received_at,
+ expires_at: received_at + CONTROLLED_FIXTURE_PROBE_TTL,
+ })
+}
+
+fn controlled_fixture_visibility_bucket(elapsed: Duration) -> &'static str {
+ if elapsed <= Duration::from_millis(250) {
+ "lte_250ms"
+ } else if elapsed <= Duration::from_millis(500) {
+ "250_500ms"
+ } else if elapsed <= Duration::from_millis(1_000) {
+ "500_1000ms"
+ } else {
+ "1000_2000ms"
+ }
+}
+
+async fn observe_controlled_fixture_post_expiry_views<F, G>(
+ received_at: Instant,
+ observation_deadline: Instant,
+ held_participant: &str,
+ expected_participant: Option<&str>,
+ requested_sequence: &str,
+ lifecycle_active: Arc<AtomicBool>,
+ mut read_held_attributes: F,
+ mut read_current_participant: G,
+) -> Option<ControlledFixtureVisibilityEvidence>
+where
+ F: FnMut() -> std::collections::HashMap<String, String>,
+ G: FnMut() -> Option<(String, std::collections::HashMap<String, String>)>,
+{
+ loop {
+ if !lifecycle_active.load(Ordering::Acquire) {
+ return None;
+ }
+ let now = Instant::now();
+ let held_decision = classify_controlled_fixture_attributes(
+ held_participant,
+ expected_participant,
+ &read_held_attributes(),
+ requested_sequence,
+ );
+ match held_decision {
+ Ok(()) => {
+ return Some(ControlledFixtureVisibilityEvidence {
+ first_visible_bucket: controlled_fixture_visibility_bucket(
+ now.saturating_duration_since(received_at),
+ ),
+ visibility_source: "participant_attributes_poll",
+ visibility_result: "held_visible",
+ binding_matched: true,
+ });
+ }
+ Err("missing_attributes") => {}
+ Err(_) => return None,
+ }
+ let Some((current_identity, current_attributes)) = read_current_participant() else {
+ return None;
+ };
+ match classify_controlled_fixture_attributes(
+ ¤t_identity,
+ expected_participant,
+ ¤t_attributes,
+ requested_sequence,
+ ) {
+ Ok(()) => {
+ return Some(ControlledFixtureVisibilityEvidence {
+ first_visible_bucket: controlled_fixture_visibility_bucket(
+ now.saturating_duration_since(received_at),
+ ),
+ visibility_source: "current_room_lookup",
+ visibility_result: "held_stale_current_visible",
+ binding_matched: true,
+ });
+ }
+ Err("missing_attributes") => {}
+ Err(_) => return None,
+ }
+ if now >= observation_deadline {
+ return Some(ControlledFixtureVisibilityEvidence {
+ first_visible_bucket: "never_visible_within_observation_window",
+ visibility_source: "held_and_current_room_lookup",
+ visibility_result: "unavailable_both",
+ binding_matched: true,
+ });
+ }
+ sleep(CONTROLLED_FIXTURE_PROBE_RECHECK_DELAY).await;
+ }
+}
+
+fn controlled_fixture_visibility_event(
+ runtime_call_id: &str,
+ runtime_trace_id: &str,
+ probe: &PendingControlledFixtureProbe,
+ evidence: &ControlledFixtureVisibilityEvidence,
+) -> serde_json::Value {
+ json!({
+ "type": "cv_activity",
+ "callId": runtime_call_id,
+ "traceId": runtime_trace_id,
+ "turnId": null,
+ "eventName": "controlled_fixture_attribute_probe",
+ "eventWallTimeMs": current_time_millis(),
+ "result": "ok",
+ "reasonCode": null,
+ "retryable": null,
+ "extension": {
+ "stage": "post_expiry_visibility",
+ "first_visible_bucket": evidence.first_visible_bucket,
+ "visibility_source": evidence.visibility_source,
+ "visibility_result": evidence.visibility_result,
+ "binding_matched": evidence.binding_matched,
+ "call_id_hash": probe.call_id_hash,
+ "trace_id_hash": probe.call_trace_id_hash,
+ "generation": probe.generation,
+ "sequence_hash": sha256_hex(&probe.sequence),
+ "evidence_count": 1,
+ },
+ })
+}
+
+fn spawn_controlled_fixture_post_expiry_observation(
+ probe: PendingControlledFixtureProbe,
+ participant: RemoteParticipant,
+ room: Arc<Room>,
+ expected_participant: Option<String>,
+ lifecycle_active: Arc<AtomicBool>,
+ runtime_call_id: String,
+ runtime_trace_id: String,
+) {
+ tokio::spawn(async move {
+ let participant_identity = participant.identity().to_string();
+ let evidence = observe_controlled_fixture_post_expiry_views(
+ probe.received_at,
+ probe.received_at + CONTROLLED_FIXTURE_POST_EXPIRY_WINDOW,
+ &participant_identity,
+ expected_participant.as_deref(),
+ &probe.sequence,
+ lifecycle_active.clone(),
+ || participant.attributes(),
+ || {
+ room.remote_participants()
+ .get(&probe.sender)
+ .map(|current| (current.identity().to_string(), current.attributes()))
+ },
+ )
+ .await;
+ if lifecycle_active.load(Ordering::Acquire) {
+ if let Some(evidence) = evidence {
+ println!(
+ "{}",
+ controlled_fixture_visibility_event(
+ &runtime_call_id,
+ &runtime_trace_id,
+ &probe,
+ &evidence,
+ )
+ );
+ }
+ }
+ });
+}
+
+fn classify_controlled_fixture_attributes(
+ actual_participant: &str,
+ expected_participant: Option<&str>,
+ attributes: &std::collections::HashMap<String, String>,
+ requested_sequence: &str,
+) -> Result<(), &'static str> {
+ if !is_bound_user_participant(actual_participant, expected_participant) {
+ return Err("wrong_participant");
+ }
+ let source = attributes
+ .get("inputSourceCategory")
+ .map(String::as_str)
+ .unwrap_or("");
+ if source != "controlled_fixture" {
+ return Err(if source.is_empty() {
+ "missing_attributes"
+ } else {
+ "wrong_source"
+ });
+ }
+ let sequence = attributes
+ .get("clientFixtureSequence")
+ .map(String::as_str)
+ .unwrap_or("");
+ if sequence.trim().is_empty() {
+ return Err("missing_sequence");
+ }
+ if sequence != requested_sequence {
+ return Err("wrong_sequence");
+ }
+ Ok(())
+}
+
+async fn observe_controlled_fixture_attributes<F>(
+ expires_at: Instant,
+ actual_participant: &str,
+ expected_participant: Option<&str>,
+ requested_sequence: &str,
+ mut read_attributes: F,
+) -> Result<(), &'static str>
+where
+ F: FnMut() -> std::collections::HashMap<String, String>,
+{
+ loop {
+ if Instant::now() > expires_at {
+ return Err("timeout");
+ }
+ let decision = classify_controlled_fixture_attributes(
+ actual_participant,
+ expected_participant,
+ &read_attributes(),
+ requested_sequence,
+ );
+ if decision.is_ok() || !matches!(decision, Err("missing_attributes")) {
+ return decision;
+ }
+ let Some(next_check) = Instant::now().checked_add(CONTROLLED_FIXTURE_PROBE_RECHECK_DELAY)
+ else {
+ return Err("timeout");
+ };
+ if next_check > expires_at {
+ return Err("timeout");
+ }
+ sleep(CONTROLLED_FIXTURE_PROBE_RECHECK_DELAY).await;
+ }
+}
+
+fn controlled_fixture_observer_gate(pending_probe: bool, probe_result: Option<bool>) -> bool {
+ !pending_probe || probe_result == Some(true)
+}
+
+fn start_observer_after_controlled_fixture_probe<F>(
+ pending_probe: bool,
+ probe_result: Option<bool>,
+ spawn: F,
+) -> bool
+where
+ F: FnOnce(),
+{
+ if !controlled_fixture_observer_gate(pending_probe, probe_result) {
+ return false;
+ }
+ spawn();
+ true
+}
#[tokio::main(flavor = "multi_thread")]
async fn main() -> Result<()> {
@@ -58,8 +621,8 @@
&http,
config.greeting_audio_file.as_deref(),
config.greeting_audio_url.as_deref(),
- TARGET_SAMPLE_RATE_HZ,
- TARGET_NUM_CHANNELS,
+ config.bot_audio_profile.sample_rate_hz,
+ config.bot_audio_profile.num_channels,
config.audio_debug_dump_dir.as_deref(),
&config.call_id,
"greeting",
@@ -81,6 +644,9 @@
room_alias = %redact(&config.room_id),
participant_alias = %redact(&config.bot_participant_identity),
greeting_source = %config.greeting_source,
+ bot_audio_profile = %config.bot_audio_profile.profile,
+ bot_sample_rate_hz = config.bot_audio_profile.sample_rate_hz,
+ bot_num_channels = config.bot_audio_profile.num_channels,
"combrabo voice runtime helper connected"
);
emit_activity(
@@ -129,8 +695,9 @@
&config.call_id,
&config.trace_id,
TRACK_NAME,
- TARGET_SAMPLE_RATE_HZ,
- u32::from(TARGET_NUM_CHANNELS),
+ config.bot_audio_profile.profile.clone(),
+ config.bot_audio_profile.sample_rate_hz,
+ u32::from(config.bot_audio_profile.num_channels),
config.user_participant_identity.clone(),
)
.await?;
@@ -262,6 +829,11 @@
runtime_turn_bridge_url: Option<String>,
runtime_turn_bridge_token: Option<String>,
runtime_turn_bridge_mode: String,
+ runtime_asr_stream_enabled: bool,
+ runtime_asr_stream_url: Option<String>,
+ runtime_asr_realtime_enabled: bool,
+ runtime_asr_realtime_url: Option<String>,
+ runtime_asr_realtime_chunk_duration_ms: u64,
runtime_turn_artifact_dir: Option<String>,
runtime_session_nonce: Option<String>,
user_audio_observer_enabled: bool,
@@ -269,6 +841,54 @@
simple_vad_gate_until_greeting_done: bool,
simple_vad_post_greeting_delay_ms: u64,
simple_vad_config: SimpleVadConfig,
+ bot_audio_profile: BotAudioProfile,
+}
+
+#[derive(Clone)]
+struct BotAudioProfile {
+ profile: String,
+ sample_rate_hz: u32,
+ num_channels: u16,
+}
+
+impl BotAudioProfile {
+ fn from_env() -> Result<Self> {
+ let profile = env::var("CV_BOT_AUDIO_PROFILE")
+ .unwrap_or_else(|_| DEFAULT_BOT_AUDIO_PROFILE.to_string())
+ .trim()
+ .to_ascii_lowercase();
+ match profile.as_str() {
+ "livekit-48k" | "48k" => Ok(Self {
+ profile: "livekit-48k".to_string(),
+ sample_rate_hz: LIVEKIT_48K_SAMPLE_RATE_HZ,
+ num_channels: BOT_NUM_CHANNELS,
+ }),
+ "pcm-16k" | "16k" => Ok(Self {
+ profile: "pcm-16k".to_string(),
+ sample_rate_hz: PCM_16K_SAMPLE_RATE_HZ,
+ num_channels: BOT_NUM_CHANNELS,
+ }),
+ "custom" => {
+ let sample_rate_hz = u32_env("CV_BOT_SAMPLE_RATE_HZ", LIVEKIT_48K_SAMPLE_RATE_HZ);
+ let num_channels = u16_env("CV_BOT_NUM_CHANNELS", BOT_NUM_CHANNELS);
+ if sample_rate_hz == 0 {
+ return Err(anyhow!("CV_BOT_SAMPLE_RATE_HZ must be positive"));
+ }
+ if num_channels == 0 {
+ return Err(anyhow!("CV_BOT_NUM_CHANNELS must be positive"));
+ }
+ Ok(Self {
+ profile,
+ sample_rate_hz,
+ num_channels,
+ })
+ }
+ _ => Err(anyhow!(
+ "unsupported CV_BOT_AUDIO_PROFILE {}; expected livekit-48k, pcm-16k or custom",
+ profile
+ )),
+ }
+ }
}
#[derive(Clone)]
@@ -276,6 +896,11 @@
bridge_url: Option<String>,
bridge_token: Option<String>,
bridge_mode: String,
+ asr_stream_enabled: bool,
+ asr_stream_url: Option<String>,
+ asr_realtime_enabled: bool,
+ asr_realtime_url: Option<String>,
+ asr_realtime_chunk_duration_ms: u64,
artifact_dir: Option<String>,
runtime_session_nonce: Option<String>,
audio_debug_dump_dir: Option<String>,
@@ -287,6 +912,11 @@
bridge_url: config.runtime_turn_bridge_url.clone(),
bridge_token: config.runtime_turn_bridge_token.clone(),
bridge_mode: config.runtime_turn_bridge_mode.clone(),
+ asr_stream_enabled: config.runtime_asr_stream_enabled,
+ asr_stream_url: config.runtime_asr_stream_url.clone(),
+ asr_realtime_enabled: config.runtime_asr_realtime_enabled,
+ asr_realtime_url: config.runtime_asr_realtime_url.clone(),
+ asr_realtime_chunk_duration_ms: config.runtime_asr_realtime_chunk_duration_ms,
artifact_dir: config.runtime_turn_artifact_dir.clone(),
runtime_session_nonce: config.runtime_session_nonce.clone(),
audio_debug_dump_dir: config.audio_debug_dump_dir.clone(),
@@ -318,6 +948,32 @@
.as_deref()
.is_some_and(|value| value.trim_end_matches('/').ends_with("/stream"))
}
+
+ fn is_asr_stream_ready(&self) -> bool {
+ self.asr_stream_enabled
+ && self
+ .asr_stream_url
+ .as_ref()
+ .is_some_and(|value| !value.is_empty())
+ && self
+ .bridge_token
+ .as_ref()
+ .is_some_and(|value| !value.is_empty())
+ && self
+ .runtime_session_nonce
+ .as_ref()
+ .is_some_and(|value| !value.is_empty())
+ }
+
+ fn realtime_asr_config(&self) -> RealtimeAsrConfig {
+ RealtimeAsrConfig {
+ enabled: self.asr_realtime_enabled,
+ url: self.asr_realtime_url.clone(),
+ runtime_token: self.bridge_token.clone(),
+ runtime_session_nonce: self.runtime_session_nonce.clone(),
+ chunk_duration_ms: self.asr_realtime_chunk_duration_ms,
+ }
+ }
}
impl Config {
@@ -339,6 +995,14 @@
runtime_turn_bridge_token: optional_env("CV_RUNTIME_TURN_BRIDGE_TOKEN"),
runtime_turn_bridge_mode: env::var("CV_RUNTIME_TURN_BRIDGE_MODE")
.unwrap_or_else(|_| "json".to_string()),
+ runtime_asr_stream_enabled: bool_env("CV_RUNTIME_ASR_STREAM_ENABLED", false),
+ runtime_asr_stream_url: optional_env("CV_RUNTIME_ASR_STREAM_URL"),
+ runtime_asr_realtime_enabled: bool_env("CV_RUNTIME_ASR_REALTIME_ENABLED", false),
+ runtime_asr_realtime_url: optional_env("CV_RUNTIME_ASR_REALTIME_URL"),
+ runtime_asr_realtime_chunk_duration_ms: u64_env(
+ "CV_RUNTIME_ASR_REALTIME_CHUNK_DURATION_MS",
+ 200,
+ ),
runtime_turn_artifact_dir: optional_env("CV_RUNTIME_TURN_ARTIFACT_DIR"),
runtime_session_nonce: optional_env("CV_RUNTIME_SESSION_NONCE"),
user_audio_observer_enabled: bool_env("CV_ENABLE_USER_AUDIO_OBSERVER", true),
@@ -346,6 +1010,7 @@
simple_vad_gate_until_greeting_done: bool_env("CV_VAD_GATE_UNTIL_GREETING_DONE", true),
simple_vad_post_greeting_delay_ms: u64_env("CV_VAD_POST_GREETING_DELAY_MS", 800),
simple_vad_config: SimpleVadConfig::from_env(),
+ bot_audio_profile: BotAudioProfile::from_env()?,
})
}
}
@@ -492,6 +1157,13 @@
.unwrap_or(default_value)
}
+fn u16_env(key: &str, default_value: u16) -> u16 {
+ env::var(key)
+ .ok()
+ .and_then(|value| value.trim().parse::<u16>().ok())
+ .unwrap_or(default_value)
+}
+
fn redact(value: &str) -> String {
if value.len() <= 8 {
return "redacted".to_string();
@@ -532,10 +1204,36 @@
"traceId": trace_id,
"turnId": turn_id,
"eventName": event_name,
+ "eventWallTimeMs": current_time_millis(),
"result": result,
"reasonCode": reason_code,
"retryable": retryable,
"extension": extension,
+ });
+ println!("{payload}");
+}
+
+fn emit_anchored_activity(
+ call_id: &str,
+ trace_id: &str,
+ turn_id: &str,
+ event_name: &str,
+ marker: &RuntimeTurnStreamTimingMarker,
+ extension: serde_json::Value,
+) {
+ let payload = json!({
+ "type": "cv_activity",
+ "callId": call_id,
+ "traceId": trace_id,
+ "turnId": turn_id,
+ "eventName": event_name,
+ "eventWallTimeMs": current_time_millis(),
+ "serverDeltaMs": marker.server_delta_ms,
+ "serverDeltaSource": STREAM_TIMING_SOURCE,
+ "result": "ok",
+ "reasonCode": null,
+ "retryable": null,
+ "extension": marker.extension_with(extension),
});
println!("{payload}");
}
@@ -552,6 +1250,7 @@
let enabled = config.user_audio_observer_enabled;
let simple_vad_enabled = config.simple_vad_enabled;
let simple_vad_config = config.simple_vad_config.clone();
+ let user_participant_identity = config.user_participant_identity.clone();
let turn_bridge_config = TurnBridgeConfig::from_config(config);
tokio::spawn(async move {
@@ -573,9 +1272,14 @@
turn_bridge_config,
http,
sink,
+ user_participant_identity,
)
.await;
})
+}
+
+fn is_bound_user_participant(identity: &str, expected: Option<&str>) -> bool {
+ expected.is_none_or(|value| identity == value)
}
async fn observe_user_audio_events(
@@ -588,6 +1292,7 @@
turn_bridge_config: TurnBridgeConfig,
http: Client,
sink: Arc<BotAudioOutputSink>,
+ user_participant_identity: Option<String>,
) {
info!(
call_id = %call_id,
@@ -604,6 +1309,11 @@
"runtime helper user_track_subscribe_requested"
);
+ let mut current_user_participant: Option<RemoteParticipant> = None;
+ let mut pending_probe: Option<PendingControlledFixtureProbe> = None;
+ let mut acknowledged_probe_sequences = HashSet::new();
+ let controlled_fixture_lifecycle_active = Arc::new(AtomicBool::new(true));
+
while let Some(event) = events.recv().await {
match event {
RoomEvent::TrackSubscribed {
@@ -611,6 +1321,15 @@
publication: _,
participant,
} => {
+ if !is_bound_user_participant(
+ &participant.identity().to_string(),
+ user_participant_identity.as_deref(),
+ ) {
+ warn!(call_id = %call_id, trace_id = %trace_id,
+ metadata_status = "wrong_participant",
+ "runtime helper ignored non-user audio participant");
+ continue;
+ }
let participant_alias = redact(&participant.identity().to_string());
let track_sid_alias = redact(&track.sid().to_string());
let track_name = track.name();
@@ -624,19 +1343,130 @@
track_source = %track_source,
"runtime helper user_track_subscribed"
);
- spawn_user_audio_frame_observer(
- track,
- call_id.clone(),
- trace_id.clone(),
- participant_alias,
- track_sid_alias,
- simple_vad_enabled,
- simple_vad_config.clone(),
- vad_enabled_gate.clone(),
- turn_bridge_config.clone(),
- http.clone(),
- sink.clone(),
+ let pending_binding = pending_probe.clone();
+ let participant_for_probe = participant.clone();
+ let probe_result = process_controlled_fixture_probe(
+ &mut pending_probe,
+ &mut acknowledged_probe_sequences,
+ Some(&participant_for_probe),
+ &sink,
+ user_participant_identity.as_deref(),
+ &call_id,
+ &trace_id,
+ controlled_fixture_lifecycle_active.clone(),
+ )
+ .await;
+ let observer_started = start_observer_after_controlled_fixture_probe(
+ pending_binding.is_some(),
+ probe_result,
+ || {
+ spawn_user_audio_frame_observer(
+ track,
+ call_id.clone(),
+ trace_id.clone(),
+ participant_alias,
+ track_sid_alias,
+ simple_vad_enabled,
+ simple_vad_config.clone(),
+ vad_enabled_gate.clone(),
+ turn_bridge_config.clone(),
+ http.clone(),
+ sink.clone(),
+ user_participant_identity.clone(),
+ participant,
+ );
+ },
);
+ if let Some(probe) = pending_binding.as_ref() {
+ let (ack_result, reject_reason, observed) = match probe_result {
+ Some(true) => (Some("observed"), None, true),
+ Some(false) => (Some("rejected"), Some("unknown"), false),
+ None => (None, Some("no_current_participant"), false),
+ };
+ record_controlled_fixture_probe_event(
+ &call_id,
+ &trace_id,
+ if observer_started {
+ "audio_observer_allowed"
+ } else {
+ "audio_observer_blocked"
+ },
+ &probe.call_id_hash,
+ &probe.call_trace_id_hash,
+ probe.generation,
+ &probe.sequence,
+ observed,
+ ack_result,
+ reject_reason,
+ );
+ }
+ if !observer_started {
+ warn!(
+ call_id = %call_id,
+ trace_id = %trace_id,
+ "runtime helper withheld audio observer until controlled fixture ACK"
+ );
+ continue;
+ }
+ current_user_participant = Some(participant_for_probe);
+ }
+ RoomEvent::DataReceived {
+ payload,
+ topic: Some(topic),
+ participant: Some(sender),
+ ..
+ } if topic == CONTROLLED_FIXTURE_PROBE_TOPIC => {
+ if let Ok(probe) =
+ serde_json::from_slice::<ControlledFixtureAttributeProbe>(&payload)
+ {
+ if acknowledged_probe_sequences.contains(&probe.client_fixture_sequence) {
+ record_controlled_fixture_probe_event(
+ &call_id,
+ &trace_id,
+ "data_received",
+ &probe.call_id_hash,
+ &probe.call_trace_id_hash,
+ probe.generation,
+ &probe.client_fixture_sequence,
+ false,
+ Some("rejected"),
+ Some("duplicate_or_old_sequence"),
+ );
+ continue;
+ }
+ }
+ pending_probe = controlled_fixture_probe(
+ &payload,
+ &call_id,
+ &trace_id,
+ &sender.identity(),
+ user_participant_identity.as_deref(),
+ );
+ if let Some(probe) = pending_probe.as_ref() {
+ record_controlled_fixture_probe_event(
+ &call_id,
+ &trace_id,
+ "data_received",
+ &probe.call_id_hash,
+ &probe.call_trace_id_hash,
+ probe.generation,
+ &probe.sequence,
+ false,
+ None,
+ None,
+ );
+ }
+ process_controlled_fixture_probe(
+ &mut pending_probe,
+ &mut acknowledged_probe_sequences,
+ current_user_participant.as_ref(),
+ &sink,
+ user_participant_identity.as_deref(),
+ &call_id,
+ &trace_id,
+ controlled_fixture_lifecycle_active.clone(),
+ )
+ .await;
}
RoomEvent::TrackSubscribed {
track: RemoteTrack::Video(track),
@@ -677,6 +1507,150 @@
_ => {}
}
}
+ controlled_fixture_lifecycle_active.store(false, Ordering::Release);
+}
+
+async fn process_controlled_fixture_probe(
+ pending_probe: &mut Option<PendingControlledFixtureProbe>,
+ acknowledged_probe_sequences: &mut HashSet<String>,
+ participant: Option<&RemoteParticipant>,
+ sink: &BotAudioOutputSink,
+ expected_participant: Option<&str>,
+ runtime_call_id: &str,
+ runtime_trace_id: &str,
+ lifecycle_active: Arc<AtomicBool>,
+) -> Option<bool> {
+ let Some(probe) = pending_probe.take() else {
+ return None;
+ };
+ if acknowledged_probe_sequences.contains(&probe.sequence) {
+ record_controlled_fixture_probe_event(
+ runtime_call_id,
+ runtime_trace_id,
+ "attributes_classified",
+ &probe.call_id_hash,
+ &probe.call_trace_id_hash,
+ probe.generation,
+ &probe.sequence,
+ false,
+ Some("rejected"),
+ Some("duplicate_or_old_sequence"),
+ );
+ return Some(false);
+ }
+ if Instant::now() > probe.expires_at {
+ record_controlled_fixture_probe_event(
+ runtime_call_id,
+ runtime_trace_id,
+ "attributes_classified",
+ &probe.call_id_hash,
+ &probe.call_trace_id_hash,
+ probe.generation,
+ &probe.sequence,
+ false,
+ Some("timeout"),
+ Some("expired"),
+ );
+ return Some(false);
+ }
+ let Some(participant) = participant else {
+ record_controlled_fixture_probe_event(
+ runtime_call_id,
+ runtime_trace_id,
+ "attributes_classified",
+ &probe.call_id_hash,
+ &probe.call_trace_id_hash,
+ probe.generation,
+ &probe.sequence,
+ false,
+ None,
+ Some("no_current_participant"),
+ );
+ *pending_probe = Some(probe);
+ return None;
+ };
+ if participant.identity() != probe.sender {
+ record_controlled_fixture_probe_event(
+ runtime_call_id,
+ runtime_trace_id,
+ "attributes_classified",
+ &probe.call_id_hash,
+ &probe.call_trace_id_hash,
+ probe.generation,
+ &probe.sequence,
+ false,
+ Some("rejected"),
+ Some("wrong_participant"),
+ );
+ return Some(false);
+ }
+ let decision = observe_controlled_fixture_attributes(
+ probe.expires_at,
+ &participant.identity().to_string(),
+ expected_participant,
+ &probe.sequence,
+ || participant.attributes(),
+ )
+ .await;
+ let (ack_result, reject_reason, observed) = record_controlled_fixture_attribute_decision(
+ decision,
+ runtime_call_id,
+ runtime_trace_id,
+ &probe.call_id_hash,
+ &probe.call_trace_id_hash,
+ probe.generation,
+ &probe.sequence,
+ );
+ let ack = controlled_fixture_ack_from_probe(&probe, observed, reject_reason);
+ let payload = match serde_json::to_vec(&ack) {
+ Ok(payload) => payload,
+ Err(_) => return Some(false),
+ };
+ let local_participant = sink.room.local_participant();
+ record_controlled_fixture_probe_event(
+ runtime_call_id,
+ runtime_trace_id,
+ "ack_publish_started",
+ &probe.call_id_hash,
+ &probe.call_trace_id_hash,
+ probe.generation,
+ &probe.sequence,
+ observed,
+ Some(ack_result),
+ reject_reason,
+ );
+ let publish = local_participant.publish_data(DataPacket {
+ payload,
+ topic: Some(CONTROLLED_FIXTURE_ACK_TOPIC.to_string()),
+ reliable: true,
+ destination_identities: vec![probe.sender.clone()],
+ });
+ let outcome = complete_controlled_fixture_ack_publish(
+ publish,
+ runtime_call_id,
+ runtime_trace_id,
+ observed,
+ ack_result,
+ reject_reason,
+ &probe.call_id_hash,
+ &probe.call_trace_id_hash,
+ probe.generation,
+ &probe.sequence,
+ acknowledged_probe_sequences,
+ )
+ .await;
+ if outcome.published && ack_result == "timeout" && reject_reason == Some("expired") {
+ spawn_controlled_fixture_post_expiry_observation(
+ probe,
+ participant.clone(),
+ sink.room.clone(),
+ expected_participant.map(str::to_string),
+ lifecycle_active,
+ runtime_call_id.to_string(),
+ runtime_trace_id.to_string(),
+ );
+ }
+ Some(outcome.observed)
}
async fn handle_finished_turn(
@@ -686,6 +1660,7 @@
call_id: &str,
trace_id: &str,
turn: FinishedSpeechTurn,
+ realtime_asr_result_ref: Option<String>,
) {
let turn_pipeline_started_at = Instant::now();
if !bridge_config.is_ready() {
@@ -742,9 +1717,13 @@
"turnArtifactBytes": byte_size,
"frameCount": turn.frame_count,
"sampleCount": turn.sample_count,
- "endReason": turn.end_reason,
+ "endReason": turn.end_reason.as_str(),
}),
);
+ let asr_result_ref = match realtime_asr_result_ref {
+ Some(value) => Some(value),
+ None => request_asr_result_ref(http, bridge_config, call_id, trace_id, &turn).await,
+ };
if bridge_config.is_stream_mode() {
match request_turn_bridge_stream(
http,
@@ -755,6 +1734,7 @@
&turn,
&path_ref,
byte_size,
+ asr_result_ref.as_deref(),
turn_pipeline_started_at,
)
.await
@@ -818,12 +1798,17 @@
&turn,
&path_ref,
byte_size,
+ asr_result_ref.as_deref(),
turn_pipeline_started_at,
)
.await
{
Ok(outcome) => {
+ let mut published_device_outputs = HashSet::new();
for output in &outcome.device_outputs {
+ if !should_publish_device_output(&mut published_device_outputs, output) {
+ continue;
+ }
if let Err(error) = sink
.publish_device_output(call_id, trace_id, &turn.turn_id, output)
.await
@@ -930,13 +1915,258 @@
audio::write_pcm_wav(
&output_path,
&turn.samples,
- TARGET_SAMPLE_RATE_HZ,
- TARGET_NUM_CHANNELS,
+ USER_AUDIO_SAMPLE_RATE_HZ,
+ USER_AUDIO_NUM_CHANNELS,
)?;
let byte_size = fs::metadata(&output_path)
.context("failed to stat turn artifact")?
.len();
Ok((path_ref, byte_size))
+}
+
+async fn request_asr_result_ref(
+ http: &Client,
+ bridge_config: &TurnBridgeConfig,
+ call_id: &str,
+ trace_id: &str,
+ turn: &FinishedSpeechTurn,
+) -> Option<String> {
+ if !bridge_config.is_asr_stream_ready() {
+ return None;
+ }
+ match request_asr_stream(http, bridge_config, call_id, trace_id, turn).await {
+ Ok(Some(asr_result_ref)) => {
+ emit_activity(
+ call_id,
+ trace_id,
+ Some(&turn.turn_id),
+ "asr_stream_ref_ready",
+ "ok",
+ None,
+ None,
+ json!({
+ "asrResultRefPresent": true,
+ "format": "pcm_s16le",
+ "sampleRate": 16000,
+ "channels": 1,
+ }),
+ );
+ Some(asr_result_ref)
+ }
+ Ok(None) => None,
+ Err(error) => {
+ warn!(
+ call_id = %call_id,
+ trace_id = %trace_id,
+ turn_id = %turn.turn_id,
+ error = %safe_error(&error.to_string()),
+ "runtime helper asr_stream_failed_fallback"
+ );
+ emit_activity(
+ call_id,
+ trace_id,
+ Some(&turn.turn_id),
+ "asr_stream_fallback",
+ "skipped",
+ Some("ASR_STREAM_INTERRUPTED"),
+ Some(true),
+ json!({
+ "fallbackReason": "asr_stream_request_failed",
+ "fallbackStage": "asr_stream",
+ }),
+ );
+ None
+ }
+ }
+}
+
+async fn request_asr_stream(
+ http: &Client,
+ bridge_config: &TurnBridgeConfig,
+ call_id: &str,
+ trace_id: &str,
+ turn: &FinishedSpeechTurn,
+) -> Result<Option<String>> {
+ let started_at = Instant::now();
+ let ndjson = build_asr_stream_ndjson(call_id, trace_id, turn, bridge_config)?;
+ let response = http
+ .post(bridge_config.asr_stream_url.as_deref().unwrap_or_default())
+ .header("Content-Type", "application/x-ndjson")
+ .header(
+ "X-CV-Runtime-Token",
+ bridge_config.bridge_token.as_deref().unwrap_or_default(),
+ )
+ .header("X-CV-Call-Id", call_id)
+ .header("X-CV-Trace-Id", trace_id)
+ .header(
+ "X-CV-Runtime-Session-Nonce",
+ bridge_config
+ .runtime_session_nonce
+ .as_deref()
+ .unwrap_or_default(),
+ )
+ .body(ndjson)
+ .send()
+ .await
+ .context("failed to post asr stream")?;
+ let status = response.status();
+ if !status.is_success() {
+ let body_len = response.text().await.map(|body| body.len()).unwrap_or(0);
+ warn!(
+ call_id = %call_id,
+ trace_id = %trace_id,
+ turn_id = %turn.turn_id,
+ http_status = status.as_u16(),
+ body_len,
+ "runtime helper asr_stream_http_failed"
+ );
+ return Ok(None);
+ }
+ let body: RuntimeTurnCommonResult<RuntimeAsrStreamResp> = response
+ .json()
+ .await
+ .context("failed to decode asr stream response")?;
+ if body.code != 0 {
+ warn!(
+ call_id = %call_id,
+ trace_id = %trace_id,
+ turn_id = %turn.turn_id,
+ code = body.code,
+ msg_len = body.msg.as_deref().unwrap_or_default().len(),
+ "runtime helper asr_stream_common_result_failed"
+ );
+ return Ok(None);
+ }
+ let Some(data) = body.data else {
+ return Ok(None);
+ };
+ if data.status.as_deref() == Some("final") {
+ info!(
+ call_id = %call_id,
+ trace_id = %trace_id,
+ turn_id = %turn.turn_id,
+ chunk_count = data.chunk_count.unwrap_or_default(),
+ audio_bytes = data.audio_bytes.unwrap_or_default(),
+ asr_duration_ms = data.asr_duration_ms.unwrap_or_default(),
+ wall_ms = started_at.elapsed().as_millis() as u64,
+ provider = %data.provider_alias.as_deref().unwrap_or("unknown"),
+ text_len = data.text_len.unwrap_or_default(),
+ "runtime helper asr_stream_final"
+ );
+ return Ok(data.asr_result_ref);
+ }
+ emit_activity(
+ call_id,
+ trace_id,
+ Some(&turn.turn_id),
+ "asr_stream_fallback",
+ "skipped",
+ None,
+ Some(true),
+ json!({
+ "fallbackReason": data.fallback_reason,
+ "fallbackStage": data.fallback_stage,
+ "status": data.status,
+ }),
+ );
+ Ok(None)
+}
+
+fn build_asr_stream_ndjson(
+ call_id: &str,
+ trace_id: &str,
+ turn: &FinishedSpeechTurn,
+ bridge_config: &TurnBridgeConfig,
+) -> Result<String> {
+ let chunks = asr_pcm_16k_chunks(turn)?;
+ let mut seq = 1u64;
+ let mut lines = Vec::with_capacity(chunks.len() + 3);
+ lines.push(serde_json::to_string(&json!({
+ "event": "asr_stream_started",
+ "seq": seq,
+ "callId": call_id,
+ "traceId": trace_id,
+ "turnId": turn.turn_id.as_str(),
+ "tsMs": current_time_millis(),
+ "payload": {
+ "format": "pcm_s16le",
+ "sampleRate": 16000,
+ "channels": 1,
+ "runtimeSessionNonce": bridge_config.runtime_session_nonce.as_deref().unwrap_or_default(),
+ "providerHint": "volcengine",
+ }
+ }))?);
+ for (index, samples) in chunks.iter().enumerate() {
+ seq += 1;
+ let bytes = pcm_i16_to_le_bytes(samples);
+ let duration_ms = ((samples.len() as u64) * 1000 / 16_000).max(1);
+ lines.push(serde_json::to_string(&json!({
+ "event": "asr_audio_chunk",
+ "seq": seq,
+ "callId": call_id,
+ "traceId": trace_id,
+ "turnId": turn.turn_id.as_str(),
+ "tsMs": current_time_millis(),
+ "payload": {
+ "chunkSeq": index + 1,
+ "format": "pcm_s16le",
+ "sampleRate": 16000,
+ "channels": 1,
+ "durationMs": duration_ms,
+ "payloadBase64": general_purpose::STANDARD.encode(bytes),
+ }
+ }))?);
+ }
+ seq += 1;
+ lines.push(serde_json::to_string(&json!({
+ "event": "vad_speech_end",
+ "seq": seq,
+ "callId": call_id,
+ "traceId": trace_id,
+ "turnId": turn.turn_id.as_str(),
+ "tsMs": current_time_millis(),
+ "payload": {
+ "endReason": turn.end_reason.as_str(),
+ "speechDurationMs": turn.duration_ms,
+ }
+ }))?);
+ seq += 1;
+ lines.push(serde_json::to_string(&json!({
+ "event": "asr_stream_finish",
+ "seq": seq,
+ "callId": call_id,
+ "traceId": trace_id,
+ "turnId": turn.turn_id.as_str(),
+ "tsMs": current_time_millis(),
+ "payload": {
+ "finalChunkSeq": chunks.len(),
+ "audioDurationMs": turn.duration_ms,
+ }
+ }))?);
+ Ok(lines.join("\n") + "\n")
+}
+
+fn asr_pcm_16k_chunks(turn: &FinishedSpeechTurn) -> Result<Vec<Vec<i16>>> {
+ if turn.samples.is_empty() {
+ return Err(anyhow!("empty turn samples"));
+ }
+ let samples_16k: Vec<i16> = turn.samples.iter().step_by(3).copied().collect();
+ if samples_16k.is_empty() {
+ return Err(anyhow!("empty 16k asr samples"));
+ }
+ let samples_per_chunk = 320usize;
+ Ok(samples_16k
+ .chunks(samples_per_chunk)
+ .map(|chunk| chunk.to_vec())
+ .collect())
+}
+
+fn pcm_i16_to_le_bytes(samples: &[i16]) -> Vec<u8> {
+ let mut bytes = Vec::with_capacity(samples.len() * 2);
+ for sample in samples {
+ bytes.extend_from_slice(&sample.to_le_bytes());
+ }
+ bytes
}
async fn request_turn_bridge_stream(
@@ -948,6 +2178,7 @@
turn: &FinishedSpeechTurn,
path_ref: &str,
byte_size: u64,
+ asr_result_ref: Option<&str>,
turn_pipeline_started_at: Instant,
) -> Result<RuntimeTurnStreamOutcome> {
let bridge_started_at = Instant::now();
@@ -955,15 +2186,16 @@
call_id: call_id.to_string(),
trace_id: trace_id.to_string(),
turn_id: turn.turn_id.clone(),
- audio_artifact: RuntimeTurnAudioArtifact {
+ audio_artifact: Some(RuntimeTurnAudioArtifact {
artifact_type: "local_file".to_string(),
path_ref: path_ref.to_string(),
format: "wav".to_string(),
- sample_rate: TARGET_SAMPLE_RATE_HZ,
- channels: u32::from(TARGET_NUM_CHANNELS),
+ sample_rate: USER_AUDIO_SAMPLE_RATE_HZ,
+ channels: u32::from(USER_AUDIO_NUM_CHANNELS),
duration_ms: turn.duration_ms,
byte_size,
- },
+ }),
+ asr_result_ref: asr_result_ref.map(str::to_string),
};
let response = http
.post(bridge_config.bridge_url.as_deref().unwrap_or_default())
@@ -1042,6 +2274,7 @@
.await?;
}
}
+ state.close_timing();
if !state.completed {
warn!(
call_id = %call_id,
@@ -1171,6 +2404,7 @@
call_id,
trace_id,
turn,
+ &event,
audio_chunk,
state,
turn_pipeline_started_at,
@@ -1182,6 +2416,9 @@
}
Some("device_output") => {
if let Some(output) = event.device_output.as_ref() {
+ if !should_publish_device_output(&mut state.published_device_output_ids, output) {
+ return Ok(());
+ }
sink.publish_device_output(call_id, trace_id, &turn.turn_id, output)
.await?;
state.device_output_count = state.device_output_count.saturating_add(1);
@@ -1189,6 +2426,7 @@
}
Some("turn_completed") => {
state.completed = true;
+ state.close_timing();
info!(
call_id = %call_id,
trace_id = %trace_id,
@@ -1199,6 +2437,7 @@
);
}
Some("turn_failed") => {
+ state.close_timing();
let error = event.error.as_ref();
let reason_code = error
.and_then(|value| value.reason_code.as_deref())
@@ -1233,6 +2472,7 @@
}
Some("turn_cancelled") => {
state.completed = true;
+ state.close_timing();
info!(
call_id = %call_id,
trace_id = %trace_id,
@@ -1265,6 +2505,19 @@
}
Some("activity") => {
if let Some(activity) = event.activity.as_ref() {
+ if activity.event_type.as_deref() == Some("tts_first_audio_chunk_ready") {
+ if let Some(runtime_session_nonce) =
+ bridge_config.runtime_session_nonce.as_deref()
+ {
+ let _ = state.arm_timing_anchor(
+ call_id,
+ trace_id,
+ &turn.turn_id,
+ runtime_session_nonce,
+ &event,
+ );
+ }
+ }
info!(
call_id = %call_id,
trace_id = %trace_id,
@@ -1323,6 +2576,7 @@
call_id: &str,
trace_id: &str,
turn: &FinishedSpeechTurn,
+ event: &RuntimeTurnStreamEvent,
audio_chunk: &RuntimeTurnStreamAudioChunk,
state: &mut RuntimeTurnStreamState,
turn_pipeline_started_at: Instant,
@@ -1342,21 +2596,112 @@
.unwrap_or("pcm_s16le")
.trim()
.to_ascii_lowercase();
+ if !matches!(format.as_str(), "pcm_s16le" | "mp3" | "mpeg" | "wav") {
+ return Err(anyhow!("unsupported reply_audio_chunk format {format}"));
+ }
+ match state.reply_chunk_markers.observe(audio_chunk.segment_seq) {
+ ReplyChunkMarker::FirstReply => {
+ let extension = json!({
+ "segmentSeq": audio_chunk.segment_seq,
+ "chunkSeq": audio_chunk.chunk_seq,
+ "format": format.as_str(),
+ "bytes": payload.len(),
+ });
+ if let Some(marker) =
+ state.record_m6(call_id, trace_id, &turn.turn_id, event, audio_chunk)
+ {
+ emit_anchored_activity(
+ call_id,
+ trace_id,
+ &turn.turn_id,
+ "helper_first_reply_audio_chunk_received",
+ &marker,
+ extension,
+ );
+ } else {
+ emit_activity(
+ call_id,
+ trace_id,
+ Some(&turn.turn_id),
+ "helper_first_reply_audio_chunk_received",
+ "ok",
+ None,
+ None,
+ extension,
+ );
+ }
+ }
+ ReplyChunkMarker::SegmentFirst => emit_activity(
+ call_id,
+ trace_id,
+ Some(&turn.turn_id),
+ "helper_segment_first_audio_chunk_received",
+ "ok",
+ None,
+ None,
+ json!({
+ "segmentSeq": audio_chunk.segment_seq,
+ "chunkSeq": audio_chunk.chunk_seq,
+ "format": format.as_str(),
+ "bytes": payload.len(),
+ }),
+ ),
+ ReplyChunkMarker::None => {}
+ }
let frames = if format == "pcm_s16le" {
- pcm_s16le_payload_to_frames(
- &payload,
- audio_chunk.sample_rate.unwrap_or(TARGET_SAMPLE_RATE_HZ),
- audio_chunk
- .channels
- .unwrap_or(u32::from(TARGET_NUM_CHANNELS)),
- )?
+ let sample_rate = audio_chunk.sample_rate.unwrap_or(sink.sample_rate_hz);
+ let channels = audio_chunk.channels.unwrap_or(u32::from(sink.num_channels));
+ if state.pcm_stream_decoder.is_none() {
+ state.pcm_stream_decoder = Some(audio::PcmS16leStreamDecoder::new(
+ sample_rate,
+ channels,
+ sink.sample_rate_hz,
+ sink.num_channels,
+ )?);
+ }
+ state.pcm_stream_network_chunk_count =
+ state.pcm_stream_network_chunk_count.saturating_add(1);
+ let stream_result = state
+ .pcm_stream_decoder
+ .as_mut()
+ .expect("pcm stream decoder initialized")
+ .push_bytes(
+ &payload,
+ sample_rate,
+ channels,
+ audio_chunk.last.unwrap_or(false),
+ )?;
+ if stream_result.dropped_tail_bytes > 0 {
+ warn!(
+ call_id = %call_id,
+ trace_id = %trace_id,
+ turn_id = %turn.turn_id,
+ chunk_seq = audio_chunk.chunk_seq.unwrap_or_default(),
+ dropped_tail_bytes = stream_result.dropped_tail_bytes,
+ "runtime helper stream_audio_pcm_unaligned_tail_dropped"
+ );
+ }
+ if stream_result.frames.is_empty() {
+ warn!(
+ call_id = %call_id,
+ trace_id = %trace_id,
+ turn_id = %turn.turn_id,
+ chunk_seq = audio_chunk.chunk_seq.unwrap_or_default(),
+ buffered_source_bytes = stream_result.buffered_source_bytes,
+ buffered_source_samples = stream_result.buffered_source_samples,
+ network_chunk_count = state.pcm_stream_network_chunk_count,
+ "runtime helper stream_audio_pcm_waiting_for_20ms_frame"
+ );
+ return Ok(0);
+ }
+ stream_result.frames
} else if matches!(format.as_str(), "mp3" | "mpeg" | "wav") {
state.encoded_audio_buffer.extend_from_slice(&payload);
match audio::decode_audio_bytes_to_frames(
&state.encoded_audio_buffer,
"stream_chunk",
- TARGET_SAMPLE_RATE_HZ,
- TARGET_NUM_CHANNELS,
+ sink.sample_rate_hz,
+ sink.num_channels,
bridge_config.audio_debug_dump_dir.as_deref(),
call_id,
&format!("stream-reply-{}", turn.turn_id),
@@ -1377,10 +2722,24 @@
);
return Ok(0);
}
+ Err(error) if state.first_audio_frame_written => {
+ warn!(
+ call_id = %call_id,
+ trace_id = %trace_id,
+ turn_id = %turn.turn_id,
+ chunk_seq = audio_chunk.chunk_seq.unwrap_or_default(),
+ format = %format,
+ buffered_bytes = state.encoded_audio_buffer.len(),
+ error = %safe_error(&error.to_string()),
+ "runtime helper stream_audio_final_chunk_decode_ignored"
+ );
+ state.encoded_audio_buffer.clear();
+ return Ok(0);
+ }
Err(error) => return Err(error).context("failed to decode final stream audio chunk"),
}
} else {
- return Err(anyhow!("unsupported reply_audio_chunk format {format}"));
+ unreachable!("supported encoded format checked above")
};
if frames.is_empty() {
return Ok(0);
@@ -1408,25 +2767,65 @@
sink.write_pcm_frame(frame).await?;
if !state.first_audio_frame_written {
state.first_audio_frame_written = true;
- emit_activity(
- call_id,
- trace_id,
- Some(&turn.turn_id),
- "bot_reply_first_audio_frame_written",
- "ok",
- None,
- None,
- json!({
- "replyPlaybackMode": state.reply_playback_mode.as_str(),
- "format": format.as_str(),
- "chunkSeq": audio_chunk.chunk_seq,
- "replyTotalAfterVadEndMs": turn_pipeline_started_at.elapsed().as_millis() as u64,
- }),
- );
+ let extension = json!({
+ "replyPlaybackMode": state.reply_playback_mode.as_str(),
+ "format": format.as_str(),
+ "chunkSeq": audio_chunk.chunk_seq,
+ "replyTotalAfterVadEndMs": turn_pipeline_started_at.elapsed().as_millis() as u64,
+ });
+ if let Some(marker) = state.record_m7() {
+ emit_anchored_activity(
+ call_id,
+ trace_id,
+ &turn.turn_id,
+ "bot_reply_first_audio_frame_written",
+ &marker,
+ extension,
+ );
+ } else {
+ emit_activity(
+ call_id,
+ trace_id,
+ Some(&turn.turn_id),
+ "bot_reply_first_audio_frame_written",
+ "ok",
+ None,
+ None,
+ extension,
+ );
+ }
}
sleep_until(pacing_started_at + Duration::from_millis(((index + 1) as u64) * 20)).await;
}
if audio_chunk.last.unwrap_or(false) {
+ let mut debug_source_path = None;
+ let mut debug_pcm_wav_path = None;
+ let mut debug_pcm_wav_size_bytes = None;
+ if format == "pcm_s16le" {
+ if let (Some(debug_dump_dir), Some(decoder)) = (
+ bridge_config.audio_debug_dump_dir.as_deref(),
+ state.pcm_stream_decoder.as_ref(),
+ ) {
+ match decoder.write_debug_dump(
+ debug_dump_dir,
+ call_id,
+ &format!("stream-reply-{}", turn.turn_id),
+ ) {
+ Ok(debug_dump) => {
+ debug_source_path = debug_dump.debug_source_path;
+ debug_pcm_wav_path = debug_dump.debug_pcm_wav_path;
+ debug_pcm_wav_size_bytes = debug_dump.debug_pcm_wav_size_bytes;
+ }
+ Err(error) => warn!(
+ call_id = %call_id,
+ trace_id = %trace_id,
+ turn_id = %turn.turn_id,
+ error = %safe_error(&error.to_string()),
+ "runtime helper stream_audio_pcm_debug_dump_failed"
+ ),
+ }
+ }
+ }
emit_activity(
call_id,
trace_id,
@@ -1439,51 +2838,20 @@
"replyPlaybackMode": state.reply_playback_mode.as_str(),
"format": format.as_str(),
"chunkSeq": audio_chunk.chunk_seq,
+ "networkChunkCount": state.pcm_stream_network_chunk_count,
+ "debugSourcePath": debug_source_path,
+ "debugPcmWavPath": debug_pcm_wav_path,
+ "debugPcmWavSizeBytes": debug_pcm_wav_size_bytes,
+ "sourceSampleRate": audio_chunk.sample_rate,
+ "sourceChannels": audio_chunk.channels,
+ "targetAudioProfile": sink.profile.as_str(),
+ "targetSampleRate": sink.sample_rate_hz,
+ "targetChannels": sink.num_channels,
"replyTotalAfterVadEndMs": turn_pipeline_started_at.elapsed().as_millis() as u64,
}),
);
}
Ok(frames.len())
-}
-
-fn pcm_s16le_payload_to_frames(
- payload: &[u8],
- sample_rate: u32,
- channels: u32,
-) -> Result<Vec<PcmFrame>> {
- if sample_rate != TARGET_SAMPLE_RATE_HZ || channels != u32::from(TARGET_NUM_CHANNELS) {
- return Err(anyhow!(
- "unsupported pcm_s16le stream format: sample_rate={}, channels={}",
- sample_rate,
- channels
- ));
- }
- if payload.len() % 2 != 0 {
- return Err(anyhow!("pcm_s16le payload has odd byte length"));
- }
- let samples: Vec<i16> = payload
- .chunks_exact(2)
- .map(|chunk| i16::from_le_bytes([chunk[0], chunk[1]]))
- .collect();
- if samples.is_empty() {
- return Ok(Vec::new());
- }
- let samples_per_channel = (sample_rate / 50).max(1);
- let frame_sample_count = samples_per_channel as usize * channels as usize;
- let mut frames = Vec::new();
- for chunk in samples.chunks(frame_sample_count) {
- let chunk_samples_per_channel = (chunk.len() / channels as usize) as u32;
- if chunk_samples_per_channel == 0 {
- continue;
- }
- frames.push(PcmFrame::new(
- chunk.to_vec(),
- sample_rate,
- channels,
- chunk_samples_per_channel,
- ));
- }
- Ok(frames)
}
fn trim_ascii_whitespace(value: &[u8]) -> &[u8] {
@@ -1506,6 +2874,7 @@
turn: &FinishedSpeechTurn,
path_ref: &str,
byte_size: u64,
+ asr_result_ref: Option<&str>,
turn_pipeline_started_at: Instant,
) -> Result<RuntimeTurnBridgeOutcome> {
let bridge_started_at = Instant::now();
@@ -1513,15 +2882,16 @@
call_id: call_id.to_string(),
trace_id: trace_id.to_string(),
turn_id: turn.turn_id.clone(),
- audio_artifact: RuntimeTurnAudioArtifact {
+ audio_artifact: Some(RuntimeTurnAudioArtifact {
artifact_type: "local_file".to_string(),
path_ref: path_ref.to_string(),
format: "wav".to_string(),
- sample_rate: TARGET_SAMPLE_RATE_HZ,
- channels: u32::from(TARGET_NUM_CHANNELS),
+ sample_rate: USER_AUDIO_SAMPLE_RATE_HZ,
+ channels: u32::from(USER_AUDIO_NUM_CHANNELS),
duration_ms: turn.duration_ms,
byte_size,
- },
+ }),
+ asr_result_ref: asr_result_ref.map(str::to_string),
};
let response = http
.post(bridge_config.bridge_url.as_deref().unwrap_or_default())
@@ -1681,8 +3051,8 @@
http,
Some(&audio_path_string),
None,
- TARGET_SAMPLE_RATE_HZ,
- TARGET_NUM_CHANNELS,
+ sink.sample_rate_hz,
+ sink.num_channels,
bridge_config.audio_debug_dump_dir.as_deref(),
call_id,
&reply_debug_label,
@@ -1812,7 +3182,10 @@
#[serde(rename = "turnId")]
turn_id: String,
#[serde(rename = "audioArtifact")]
- audio_artifact: RuntimeTurnAudioArtifact,
+ #[serde(skip_serializing_if = "Option::is_none")]
+ audio_artifact: Option<RuntimeTurnAudioArtifact>,
+ #[serde(rename = "asrResultRef", skip_serializing_if = "Option::is_none")]
+ asr_result_ref: Option<String>,
}
#[derive(Serialize)]
@@ -1840,6 +3213,20 @@
retryable: Option<bool>,
}
+#[derive(Deserialize)]
+#[serde(rename_all = "camelCase")]
+struct RuntimeAsrStreamResp {
+ status: Option<String>,
+ asr_result_ref: Option<String>,
+ chunk_count: Option<u64>,
+ audio_bytes: Option<u64>,
+ asr_duration_ms: Option<u64>,
+ provider_alias: Option<String>,
+ text_len: Option<u64>,
+ fallback_reason: Option<String>,
+ fallback_stage: Option<String>,
+}
+
#[derive(Default)]
struct RuntimeTurnBridgeOutcome {
reply_audio_artifact: Option<RuntimeTurnReplyAudioArtifact>,
@@ -1860,7 +3247,12 @@
completed: bool,
audio_chunk_count: u64,
device_output_count: u64,
+ published_device_output_ids: HashSet<String>,
encoded_audio_buffer: Vec<u8>,
+ pcm_stream_decoder: Option<audio::PcmS16leStreamDecoder>,
+ pcm_stream_network_chunk_count: u64,
+ reply_chunk_markers: ReplyChunkMarkerState,
+ timing: RuntimeTurnStreamTimingState,
}
impl Default for RuntimeTurnStreamState {
@@ -1873,8 +3265,266 @@
completed: false,
audio_chunk_count: 0,
device_output_count: 0,
+ published_device_output_ids: HashSet::new(),
encoded_audio_buffer: Vec::new(),
+ pcm_stream_decoder: None,
+ pcm_stream_network_chunk_count: 0,
+ reply_chunk_markers: ReplyChunkMarkerState::default(),
+ timing: RuntimeTurnStreamTimingState::default(),
}
+ }
+}
+
+impl RuntimeTurnStreamState {
+ fn arm_timing_anchor(
+ &mut self,
+ call_id: &str,
+ trace_id: &str,
+ turn_id: &str,
+ runtime_session_nonce: &str,
+ event: &RuntimeTurnStreamEvent,
+ ) -> bool {
+ if self.timing.phase != RuntimeTurnStreamTimingPhase::Empty
+ || event.call_id.as_deref() != Some(call_id)
+ || event.trace_id.as_deref() != Some(trace_id)
+ || event.turn_id.as_deref() != Some(turn_id)
+ {
+ return false;
+ }
+ let Some(activity) = event.activity.as_ref() else {
+ return false;
+ };
+ if activity.event_type.as_deref() != Some("tts_first_audio_chunk_ready") {
+ return false;
+ }
+ let Some(extension) = activity.extension.as_ref() else {
+ return false;
+ };
+ let Some(anchor_id) = extension.stream_anchor_id.as_deref() else {
+ return false;
+ };
+ let valid_anchor_id = (16..=64).contains(&anchor_id.len()) && anchor_id.is_ascii();
+ let expected_nonce_hash = runtime_session_nonce_hash(runtime_session_nonce);
+ if extension.stream_timing_version != Some(STREAM_TIMING_VERSION)
+ || !valid_anchor_id
+ || extension.runtime_session_nonce_hash.as_deref() != Some(expected_nonce_hash.as_str())
+ || extension.segment_seq != Some(STREAM_TIMING_FIRST_SEGMENT)
+ || extension.stream_timing_validation.as_deref() != Some("bound")
+ {
+ return false;
+ }
+ let Some(server_delta_ms) = extension.stream_anchor_server_delta_ms else {
+ return false;
+ };
+ self.timing.anchor = Some(RuntimeTurnStreamTimingAnchor {
+ call_id: call_id.to_string(),
+ trace_id: trace_id.to_string(),
+ turn_id: turn_id.to_string(),
+ anchor_id: anchor_id.to_string(),
+ server_delta_ms,
+ runtime_session_nonce_hash: expected_nonce_hash,
+ segment_seq: STREAM_TIMING_FIRST_SEGMENT,
+ received_at: Instant::now(),
+ });
+ self.timing.phase = RuntimeTurnStreamTimingPhase::Armed;
+ true
+ }
+
+ fn record_m6(
+ &mut self,
+ call_id: &str,
+ trace_id: &str,
+ turn_id: &str,
+ event: &RuntimeTurnStreamEvent,
+ audio_chunk: &RuntimeTurnStreamAudioChunk,
+ ) -> Option<RuntimeTurnStreamTimingMarker> {
+ if self.timing.phase != RuntimeTurnStreamTimingPhase::Armed {
+ return None;
+ }
+ let anchor = self.timing.anchor.as_ref()?;
+ if anchor.call_id != call_id
+ || anchor.trace_id != trace_id
+ || anchor.turn_id != turn_id
+ || event.call_id.as_deref() != Some(call_id)
+ || event.trace_id.as_deref() != Some(trace_id)
+ || event.turn_id.as_deref() != Some(turn_id)
+ || audio_chunk.segment_seq != Some(anchor.segment_seq)
+ || audio_chunk.chunk_seq != Some(STREAM_TIMING_FIRST_CHUNK)
+ || audio_chunk.stream_timing_version != Some(STREAM_TIMING_VERSION)
+ || audio_chunk.stream_anchor_id.as_deref() != Some(anchor.anchor_id.as_str())
+ {
+ return None;
+ }
+ let Some(marker) = RuntimeTurnStreamTimingMarker::from_anchor(
+ anchor,
+ audio_chunk.chunk_seq.unwrap_or(STREAM_TIMING_FIRST_CHUNK),
+ ) else {
+ self.close_timing();
+ return None;
+ };
+ self.timing.chunk_seq = Some(marker.chunk_seq);
+ self.timing.phase = RuntimeTurnStreamTimingPhase::M6Recorded;
+ Some(marker)
+ }
+
+ fn record_m7(&mut self) -> Option<RuntimeTurnStreamTimingMarker> {
+ if self.timing.phase != RuntimeTurnStreamTimingPhase::M6Recorded {
+ return None;
+ }
+ let anchor = self.timing.anchor.as_ref()?;
+ let Some(marker) = RuntimeTurnStreamTimingMarker::from_anchor(
+ anchor,
+ self.timing.chunk_seq.unwrap_or(STREAM_TIMING_FIRST_CHUNK),
+ ) else {
+ self.close_timing();
+ return None;
+ };
+ self.timing.phase = RuntimeTurnStreamTimingPhase::M7Recorded;
+ Some(marker)
+ }
+
+ fn close_timing(&mut self) {
+ self.timing.close();
+ }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+enum RuntimeTurnStreamTimingPhase {
+ Empty,
+ Armed,
+ M6Recorded,
+ M7Recorded,
+ Closed,
+}
+
+struct RuntimeTurnStreamTimingState {
+ phase: RuntimeTurnStreamTimingPhase,
+ anchor: Option<RuntimeTurnStreamTimingAnchor>,
+ chunk_seq: Option<u64>,
+}
+
+impl Default for RuntimeTurnStreamTimingState {
+ fn default() -> Self {
+ Self {
+ phase: RuntimeTurnStreamTimingPhase::Empty,
+ anchor: None,
+ chunk_seq: None,
+ }
+ }
+}
+
+impl RuntimeTurnStreamTimingState {
+ fn close(&mut self) {
+ self.anchor = None;
+ self.chunk_seq = None;
+ self.phase = RuntimeTurnStreamTimingPhase::Closed;
+ }
+}
+
+impl Drop for RuntimeTurnStreamTimingState {
+ fn drop(&mut self) {
+ self.anchor = None;
+ self.chunk_seq = None;
+ }
+}
+
+struct RuntimeTurnStreamTimingAnchor {
+ call_id: String,
+ trace_id: String,
+ turn_id: String,
+ anchor_id: String,
+ server_delta_ms: u64,
+ runtime_session_nonce_hash: String,
+ segment_seq: u64,
+ received_at: Instant,
+}
+
+struct RuntimeTurnStreamTimingMarker {
+ version: u32,
+ anchor_id: String,
+ anchor_server_delta_ms: u64,
+ anchor_elapsed_ms: u64,
+ server_delta_ms: u64,
+ runtime_session_nonce_hash: String,
+ segment_seq: u64,
+ chunk_seq: u64,
+}
+
+impl RuntimeTurnStreamTimingMarker {
+ fn from_anchor(anchor: &RuntimeTurnStreamTimingAnchor, chunk_seq: u64) -> Option<Self> {
+ let elapsed_ms = u64::try_from(anchor.received_at.elapsed().as_millis()).ok()?;
+ if elapsed_ms > STREAM_TIMING_MAX_ELAPSED_MS {
+ return None;
+ }
+ Some(Self {
+ version: STREAM_TIMING_VERSION,
+ anchor_id: anchor.anchor_id.clone(),
+ anchor_server_delta_ms: anchor.server_delta_ms,
+ anchor_elapsed_ms: elapsed_ms,
+ server_delta_ms: anchor.server_delta_ms.checked_add(elapsed_ms)?,
+ runtime_session_nonce_hash: anchor.runtime_session_nonce_hash.clone(),
+ segment_seq: anchor.segment_seq,
+ chunk_seq,
+ })
+ }
+
+ fn extension_with(&self, extra: serde_json::Value) -> serde_json::Value {
+ let mut extension = match extra {
+ serde_json::Value::Object(value) => value,
+ _ => serde_json::Map::new(),
+ };
+ extension.insert("streamTimingVersion".to_string(), json!(self.version));
+ extension.insert("streamAnchorId".to_string(), json!(self.anchor_id));
+ extension.insert(
+ "streamAnchorServerDeltaMs".to_string(),
+ json!(self.anchor_server_delta_ms),
+ );
+ extension.insert("anchorElapsedMs".to_string(), json!(self.anchor_elapsed_ms));
+ extension.insert(
+ "runtimeSessionNonceHash".to_string(),
+ json!(self.runtime_session_nonce_hash),
+ );
+ extension.insert("segmentSeq".to_string(), json!(self.segment_seq));
+ extension.insert("chunkSeq".to_string(), json!(self.chunk_seq));
+ extension.insert("streamTimingValidation".to_string(), json!("bound"));
+ serde_json::Value::Object(extension)
+ }
+}
+
+fn runtime_session_nonce_hash(value: &str) -> String {
+ let digest = Sha256::digest(value.as_bytes());
+ digest[..6]
+ .iter()
+ .map(|byte| format!("{byte:02x}"))
+ .collect()
+}
+
+#[derive(Debug, PartialEq, Eq)]
+enum ReplyChunkMarker {
+ FirstReply,
+ SegmentFirst,
+ None,
+}
+
+#[derive(Default)]
+struct ReplyChunkMarkerState {
+ first_reply_seen: bool,
+ seen_segments: HashSet<u64>,
+}
+
+impl ReplyChunkMarkerState {
+ fn observe(&mut self, segment_seq: Option<u64>) -> ReplyChunkMarker {
+ let first_for_segment = segment_seq
+ .map(|value| self.seen_segments.insert(value))
+ .unwrap_or(false);
+ if !self.first_reply_seen {
+ self.first_reply_seen = true;
+ return ReplyChunkMarker::FirstReply;
+ }
+ if first_for_segment {
+ return ReplyChunkMarker::SegmentFirst;
+ }
+ ReplyChunkMarker::None
}
}
@@ -1894,6 +3544,12 @@
struct RuntimeTurnStreamEvent {
#[serde(rename = "type", alias = "event")]
event_type: Option<String>,
+ #[serde(rename = "callId")]
+ call_id: Option<String>,
+ #[serde(rename = "traceId")]
+ trace_id: Option<String>,
+ #[serde(rename = "turnId")]
+ turn_id: Option<String>,
#[serde(rename = "seq")]
seq: Option<u64>,
#[serde(rename = "replyPlaybackMode")]
@@ -1929,6 +3585,12 @@
struct RuntimeTurnStreamAudioChunk {
#[serde(rename = "chunkSeq", alias = "seq")]
chunk_seq: Option<u64>,
+ #[serde(rename = "segmentSeq")]
+ segment_seq: Option<u64>,
+ #[serde(rename = "streamTimingVersion")]
+ stream_timing_version: Option<u32>,
+ #[serde(rename = "streamAnchorId")]
+ stream_anchor_id: Option<String>,
format: Option<String>,
#[serde(rename = "sampleRate")]
sample_rate: Option<u32>,
@@ -1945,6 +3607,23 @@
stage: Option<String>,
#[serde(rename = "reasonCode")]
reason_code: Option<String>,
+ extension: Option<RuntimeTurnStreamTimingExtension>,
+}
+
+#[derive(Clone, Deserialize)]
+struct RuntimeTurnStreamTimingExtension {
+ #[serde(rename = "streamTimingVersion")]
+ stream_timing_version: Option<u32>,
+ #[serde(rename = "streamAnchorId")]
+ stream_anchor_id: Option<String>,
+ #[serde(rename = "streamAnchorServerDeltaMs")]
+ stream_anchor_server_delta_ms: Option<u64>,
+ #[serde(rename = "runtimeSessionNonceHash")]
+ runtime_session_nonce_hash: Option<String>,
+ #[serde(rename = "segmentSeq")]
+ segment_seq: Option<u64>,
+ #[serde(rename = "streamTimingValidation")]
+ stream_timing_validation: Option<String>,
}
#[derive(Deserialize)]
@@ -1977,6 +3656,30 @@
#[serde(rename = "commandCode")]
command_code: Option<String>,
params: Option<serde_json::Value>,
+}
+
+fn should_publish_device_output(
+ published_ids: &mut HashSet<String>,
+ output: &RuntimeTurnDeviceOutput,
+) -> bool {
+ let Some(command_id) = output
+ .command_id
+ .as_deref()
+ .map(str::trim)
+ .filter(|value| !value.is_empty())
+ else {
+ return false;
+ };
+ if output
+ .command_code
+ .as_deref()
+ .map(str::trim)
+ .filter(|value| !value.is_empty())
+ .is_none()
+ {
+ return false;
+ }
+ published_ids.insert(command_id.to_string())
}
fn require_safe_segment(value: &str) -> Result<()> {
@@ -2018,14 +3721,78 @@
turn_bridge_config: TurnBridgeConfig,
http: Client,
sink: Arc<BotAudioOutputSink>,
+ expected_participant_identity: Option<String>,
+ participant: RemoteParticipant,
) -> JoinHandle<()> {
tokio::spawn(async move {
let mut stream = NativeAudioStream::new(
track.rtc_track(),
- TARGET_SAMPLE_RATE_HZ as i32,
- i32::from(TARGET_NUM_CHANNELS),
+ USER_AUDIO_SAMPLE_RATE_HZ as i32,
+ i32::from(USER_AUDIO_NUM_CHANNELS),
);
let started_at = Instant::now();
+ let (frame_tx, mut frame_rx) =
+ mpsc::channel::<DrainedUserAudioFrame>(INBOUND_AUDIO_QUEUE_CAPACITY);
+ let (drain_shutdown_tx, mut drain_shutdown_rx) = watch::channel(false);
+ let drain_call_id = call_id.clone();
+ let drain_trace_id = trace_id.clone();
+ let drain_task = tokio::spawn(async move {
+ let mut received_frame_count: u64 = 0;
+ let mut dropped_frame_count: u64 = 0;
+ loop {
+ tokio::select! {
+ changed = drain_shutdown_rx.changed() => {
+ match changed {
+ Ok(()) if *drain_shutdown_rx.borrow() => break,
+ Ok(()) => {}
+ Err(_) => break,
+ }
+ }
+ maybe_frame = stream.next() => {
+ let Some(frame) = maybe_frame else {
+ break;
+ };
+ received_frame_count = received_frame_count.saturating_add(1);
+ let drained = DrainedUserAudioFrame {
+ frame_index: received_frame_count,
+ captured_elapsed_ms: started_at.elapsed().as_millis() as u64,
+ frame: AudioFrame {
+ data: Cow::Owned(frame.data.as_ref().to_vec()),
+ sample_rate: frame.sample_rate,
+ num_channels: frame.num_channels,
+ samples_per_channel: frame.samples_per_channel,
+ },
+ };
+ match frame_tx.try_send(drained) {
+ Ok(()) => {}
+ Err(mpsc::error::TrySendError::Full(_)) => {
+ dropped_frame_count = dropped_frame_count.saturating_add(1);
+ if dropped_frame_count % INBOUND_AUDIO_DROP_LOG_INTERVAL == 1 {
+ warn!(
+ call_id = %drain_call_id,
+ trace_id = %drain_trace_id,
+ dropped_frame_count,
+ received_frame_count,
+ queue_capacity = INBOUND_AUDIO_QUEUE_CAPACITY,
+ "runtime helper inbound_audio_queue_full_dropping_newest"
+ );
+ }
+ }
+ Err(mpsc::error::TrySendError::Closed(_)) => break,
+ }
+ }
+ }
+ }
+ stream.close();
+ info!(
+ call_id = %drain_call_id,
+ trace_id = %drain_trace_id,
+ received_frame_count,
+ dropped_frame_count,
+ queue_capacity = INBOUND_AUDIO_QUEUE_CAPACITY,
+ "runtime helper user_audio_drain_ended"
+ );
+ });
let mut frame_count: u64 = 0;
let mut sample_count: u64 = 0;
let mut simple_vad = if simple_vad_enabled {
@@ -2033,11 +3800,14 @@
} else {
None
};
+ let mut realtime_asr_upload: Option<RealtimeAsrUpload> = None;
+ let mut last_fixture_sequence: Option<String> = None;
- while let Some(frame) = stream.next().await {
- frame_count += 1;
+ while let Some(drained) = frame_rx.recv().await {
+ let frame = drained.frame;
+ frame_count = drained.frame_index;
sample_count += u64::from(frame.samples_per_channel) * u64::from(frame.num_channels);
- let elapsed_ms = started_at.elapsed().as_millis() as u64;
+ let elapsed_ms = drained.captured_elapsed_ms;
if frame_count == 1 {
info!(
call_id = %call_id,
@@ -2065,7 +3835,11 @@
if let Some(vad) = simple_vad.as_mut() {
if vad_enabled_gate.load(Ordering::Acquire) {
- if let Some(turn) = vad.observe_frame(
+ let participant_identity = participant.identity().to_string();
+ let (was_in_speech, is_in_speech, turn) = observe_bound_participant_frame(
+ &participant_identity,
+ expected_participant_identity.as_deref(),
+ vad,
&call_id,
&trace_id,
&participant_alias,
@@ -2073,7 +3847,40 @@
frame_count,
elapsed_ms,
&frame,
- ) {
+ http.clone(),
+ turn_bridge_config.realtime_asr_config(),
+ || participant.attributes(),
+ &mut realtime_asr_upload,
+ &mut last_fixture_sequence,
+ turn_bridge_config.asr_realtime_enabled,
+ );
+
+ if was_in_speech {
+ let push_failed = realtime_asr_upload
+ .as_mut()
+ .and_then(|upload| upload.push_48k_samples(frame.data.as_ref()).err());
+ if let Some(error) = push_failed {
+ warn!(
+ call_id = %call_id,
+ trace_id = %trace_id,
+ error = %safe_error(&error.to_string()),
+ "runtime helper asr_realtime_upload_failed_fallback"
+ );
+ if let Some(upload) = realtime_asr_upload.take() {
+ tokio::spawn(async move {
+ upload.cancel("upload_backpressure").await;
+ });
+ }
+ }
+ }
+
+ if let Some(turn) = turn {
+ let realtime_asr_result_ref = match realtime_asr_upload.take() {
+ Some(upload) => {
+ finish_realtime_asr_upload(upload, &call_id, &trace_id, &turn).await
+ }
+ None => None,
+ };
handle_finished_turn(
&http,
&turn_bridge_config,
@@ -2081,10 +3888,22 @@
&call_id,
&trace_id,
turn,
+ realtime_asr_result_ref,
)
.await;
+ } else if was_in_speech && !is_in_speech {
+ if let Some(upload) = realtime_asr_upload.take() {
+ tokio::spawn(async move {
+ upload.cancel("speech_too_short").await;
+ });
+ }
}
} else {
+ if let Some(upload) = realtime_asr_upload.take() {
+ tokio::spawn(async move {
+ upload.cancel("vad_disabled").await;
+ });
+ }
vad.observe_disabled_frame(
&call_id,
&trace_id,
@@ -2105,9 +3924,41 @@
&track_sid_alias,
started_at.elapsed().as_millis() as u64,
) {
- handle_finished_turn(&http, &turn_bridge_config, &sink, &call_id, &trace_id, turn)
- .await;
+ let realtime_asr_result_ref = match realtime_asr_upload.take() {
+ Some(upload) => {
+ finish_realtime_asr_upload(upload, &call_id, &trace_id, &turn).await
+ }
+ None => None,
+ };
+ handle_finished_turn(
+ &http,
+ &turn_bridge_config,
+ &sink,
+ &call_id,
+ &trace_id,
+ turn,
+ realtime_asr_result_ref,
+ )
+ .await;
}
+ }
+ if let Some(upload) = realtime_asr_upload.take() {
+ upload.cancel("stream_end").await;
+ }
+ let _ = drain_shutdown_tx.send(true);
+ let mut drain_task = drain_task;
+ if timeout(AUDIO_DRAIN_STOP_GRACE, &mut drain_task)
+ .await
+ .is_err()
+ {
+ drain_task.abort();
+ let _ = drain_task.await;
+ warn!(
+ call_id = %call_id,
+ trace_id = %trace_id,
+ stop_grace_ms = AUDIO_DRAIN_STOP_GRACE.as_millis() as u64,
+ "runtime helper user_audio_drain_stop_timeout"
+ );
}
info!(
@@ -2121,6 +3972,234 @@
"runtime helper user_audio_stream_ended"
);
})
+}
+
+fn observe_frame_and_start_session<F>(
+ vad: &mut SimpleVad,
+ call_id: &str,
+ trace_id: &str,
+ participant_alias: &str,
+ track_sid_alias: &str,
+ frame_count: u64,
+ elapsed_ms: u64,
+ frame: &AudioFrame<'_>,
+ http: Client,
+ config: RealtimeAsrConfig,
+ read_attributes: F,
+ upload_slot: &mut Option<RealtimeAsrUpload>,
+ last_fixture_sequence: &mut Option<String>,
+ realtime_enabled: bool,
+) -> (bool, bool, Option<FinishedSpeechTurn>)
+where
+ F: FnOnce() -> std::collections::HashMap<String, String>,
+{
+ let was_in_speech = vad.in_speech;
+ let turn = vad.observe_frame(
+ call_id,
+ trace_id,
+ participant_alias,
+ track_sid_alias,
+ frame_count,
+ elapsed_ms,
+ frame,
+ );
+ let is_in_speech = vad.in_speech;
+ if !was_in_speech && is_in_speech {
+ start_realtime_session_for_new_speech(
+ http,
+ config,
+ call_id,
+ trace_id,
+ vad,
+ read_attributes,
+ upload_slot,
+ last_fixture_sequence,
+ realtime_enabled,
+ );
+ }
+ (was_in_speech, is_in_speech, turn)
+}
+
+fn observe_bound_participant_frame<F>(
+ participant_identity: &str,
+ expected_participant: Option<&str>,
+ vad: &mut SimpleVad,
+ call_id: &str,
+ trace_id: &str,
+ participant_alias: &str,
+ track_sid_alias: &str,
+ frame_count: u64,
+ elapsed_ms: u64,
+ frame: &AudioFrame<'_>,
+ http: Client,
+ config: RealtimeAsrConfig,
+ read_attributes: F,
+ upload_slot: &mut Option<RealtimeAsrUpload>,
+ last_fixture_sequence: &mut Option<String>,
+ realtime_enabled: bool,
+) -> (bool, bool, Option<FinishedSpeechTurn>)
+where
+ F: FnOnce() -> std::collections::HashMap<String, String>,
+{
+ if !is_bound_user_participant(participant_identity, expected_participant) {
+ warn!(
+ "audioIngressOriginStatus" = "wrong_participant_or_track",
+ "runtime helper rejected audio participant before VAD/session"
+ );
+ return (vad.in_speech, vad.in_speech, None);
+ }
+ observe_frame_and_start_session(
+ vad,
+ call_id,
+ trace_id,
+ participant_alias,
+ track_sid_alias,
+ frame_count,
+ elapsed_ms,
+ frame,
+ http,
+ config,
+ read_attributes,
+ upload_slot,
+ last_fixture_sequence,
+ realtime_enabled,
+ )
+}
+
+fn start_realtime_session_for_new_speech(
+ http: Client,
+ config: RealtimeAsrConfig,
+ call_id: &str,
+ trace_id: &str,
+ vad: &SimpleVad,
+ read_attributes: impl FnOnce() -> std::collections::HashMap<String, String>,
+ upload_slot: &mut Option<RealtimeAsrUpload>,
+ last_fixture_sequence: &mut Option<String>,
+ realtime_enabled: bool,
+) {
+ let turn_id = format!("turn-{:04}", vad.turn_index);
+ let attributes = read_attributes();
+ let origin_status = AudioIngressMetadata::origin_status(&attributes);
+ let metadata = match AudioIngressMetadata::from_participant(&attributes) {
+ Ok(metadata) => metadata,
+ Err(reason) => {
+ warn!(call_id = %call_id, trace_id = %trace_id, turn_id = %turn_id,
+ reason, audioIngressOriginStatus = %AudioIngressMetadata::rejected_origin_status(
+ reason, &attributes
+ ),
+ "runtime helper asr_realtime_metadata_rejected");
+ return;
+ }
+ };
+ if let Some(metadata) = metadata.as_ref() {
+ if !fixture_sequence_is_new(
+ last_fixture_sequence.as_deref(),
+ &metadata.client_fixture_sequence,
+ ) {
+ warn!(call_id = %call_id, trace_id = %trace_id, turn_id = %turn_id,
+ audioIngressOriginStatus = "sequence_replayed_or_regressed",
+ "runtime helper asr_realtime_metadata_sequence_rejected");
+ return;
+ }
+ }
+ match RealtimeAsrUpload::start(
+ http,
+ config,
+ call_id,
+ trace_id,
+ &turn_id,
+ &vad.speech_samples,
+ metadata.as_ref(),
+ ) {
+ Ok(upload) => {
+ if let Some(metadata) = metadata {
+ *last_fixture_sequence = Some(metadata.client_fixture_sequence);
+ }
+ info!(call_id = %call_id, trace_id = %trace_id, turn_id = %turn_id,
+ origin_status, "runtime helper asr_realtime_session_started");
+ *upload_slot = Some(upload);
+ }
+ Err(error) if realtime_enabled => {
+ warn!(call_id = %call_id, trace_id = %trace_id, turn_id = %turn_id,
+ error = %safe_error(&error.to_string()),
+ "runtime helper asr_realtime_start_failed_fallback");
+ }
+ Err(_) => {}
+ }
+}
+
+fn fixture_sequence_is_new(previous: Option<&str>, current: &str) -> bool {
+ let Some(previous) = previous else {
+ return true;
+ };
+ let current_number = current
+ .rsplit_once('-')
+ .and_then(|(_, value)| value.parse::<u64>().ok());
+ let previous_number = previous
+ .rsplit_once('-')
+ .and_then(|(_, value)| value.parse::<u64>().ok());
+ match (previous_number, current_number) {
+ (Some(previous), Some(current)) => current > previous,
+ _ => previous != current,
+ }
+}
+
+struct DrainedUserAudioFrame {
+ frame_index: u64,
+ captured_elapsed_ms: u64,
+ frame: AudioFrame<'static>,
+}
+
+async fn finish_realtime_asr_upload(
+ upload: RealtimeAsrUpload,
+ call_id: &str,
+ trace_id: &str,
+ turn: &FinishedSpeechTurn,
+) -> Option<String> {
+ match upload.finish(turn.duration_ms, &turn.end_reason).await {
+ Ok(RealtimeAsrOutcome {
+ status,
+ asr_result_ref,
+ provider_alias,
+ partial_count,
+ fallback_reason,
+ fallback_stage,
+ chunk_count,
+ audio_bytes,
+ wall_ms,
+ }) => {
+ info!(
+ call_id = %call_id,
+ trace_id = %trace_id,
+ turn_id = %turn.turn_id,
+ status = %status,
+ provider_alias = ?provider_alias,
+ partial_count,
+ chunk_count,
+ audio_bytes,
+ wall_ms,
+ asr_result_ref_present = asr_result_ref.is_some(),
+ fallback_reason = ?fallback_reason,
+ fallback_stage = ?fallback_stage,
+ "runtime helper asr_realtime_finished"
+ );
+ if status == "final" {
+ asr_result_ref
+ } else {
+ None
+ }
+ }
+ Err(error) => {
+ warn!(
+ call_id = %call_id,
+ trace_id = %trace_id,
+ turn_id = %turn.turn_id,
+ error = %safe_error(&error.to_string()),
+ "runtime helper asr_realtime_failed_fallback"
+ );
+ None
+ }
+ }
}
#[derive(Clone)]
@@ -2140,8 +4219,8 @@
rms_threshold: f64_env("CV_VAD_RMS_THRESHOLD", 0.012),
peak_threshold: f64_env("CV_VAD_PEAK_THRESHOLD", 0.08),
start_frames: u32_env("CV_VAD_START_FRAMES", 5).max(1),
- end_silence_ms: u64_env("CV_VAD_END_SILENCE_MS", 700).max(100),
- min_speech_ms: u64_env("CV_VAD_MIN_SPEECH_MS", 300).max(1),
+ end_silence_ms: u64_env("CV_VAD_END_SILENCE_MS", 400).max(100),
+ min_speech_ms: u64_env("CV_VAD_MIN_SPEECH_MS", 250).max(1),
max_turn_ms: u64_env("CV_VAD_MAX_TURN_MS", 10_000).max(1_000),
initial_ignore_ms: u64_env("CV_VAD_INITIAL_IGNORE_MS", 500),
}
@@ -2254,7 +4333,9 @@
return None;
}
- let voiced = rms >= self.config.rms_threshold || peak >= self.config.peak_threshold;
+ // Align with cb-sdk's energy-based segmentation: peak is diagnostic only,
+ // otherwise isolated spikes can keep a turn open until max_turn_ms.
+ let voiced = rms >= self.config.rms_threshold;
if !self.in_speech {
self.remember_pre_speech_frame(frame);
}
@@ -2510,6 +4591,9 @@
rtc_source: NativeAudioSource,
track: LocalAudioTrack,
device_output_destination_identity: Option<String>,
+ profile: String,
+ sample_rate_hz: u32,
+ num_channels: u16,
}
impl BotAudioOutputSink {
@@ -2520,6 +4604,7 @@
call_id: &str,
trace_id: &str,
track_name: &str,
+ profile: String,
sample_rate: u32,
num_channels: u32,
device_output_destination_identity: Option<String>,
@@ -2548,11 +4633,14 @@
"failed to publish bot audio track in room {room_alias} for participant {participant_alias}: {error}"
)
})?;
+ let num_channels_u16 = u16::try_from(num_channels)
+ .map_err(|_| anyhow!("unsupported bot audio channel count {num_channels}"))?;
info!(
room_alias = %room_alias,
participant_alias = %participant_alias,
track_name = %track_name,
+ bot_audio_profile = %profile,
sample_rate,
num_channels,
"runtime helper published bot audio track"
@@ -2567,6 +4655,7 @@
None,
json!({
"trackName": track_name,
+ "audioProfile": profile,
"sampleRate": sample_rate,
"numChannels": num_channels,
}),
@@ -2577,6 +4666,9 @@
rtc_source,
track,
device_output_destination_identity,
+ profile,
+ sample_rate_hz: sample_rate,
+ num_channels: num_channels_u16,
})
}
@@ -2801,3 +4893,1346 @@
Ok(())
}
}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use std::{
+ collections::{HashMap, HashSet},
+ io::{Read, Write},
+ net::TcpListener,
+ sync::{
+ Arc,
+ atomic::{AtomicUsize, Ordering},
+ mpsc,
+ },
+ thread,
+ time::Duration,
+ };
+
+ #[derive(Debug)]
+ enum PreAudioOrderEvent {
+ DataReceived {
+ sender: String,
+ sequence: String,
+ },
+ TrackSubscribed {
+ participant: String,
+ attributes: HashMap<String, String>,
+ },
+ }
+
+ fn drive_pre_audio_order_test_seam(events: &[PreAudioOrderEvent]) -> Vec<&'static str> {
+ let mut pending_sequence = None;
+ let mut effects = Vec::new();
+ for event in events {
+ match event {
+ PreAudioOrderEvent::DataReceived { sender, sequence } if sender == "user-1" => {
+ pending_sequence = Some(sequence.as_str());
+ }
+ PreAudioOrderEvent::TrackSubscribed {
+ participant,
+ attributes,
+ } => {
+ let pending = pending_sequence.is_some();
+ let probe_result = pending_sequence.map(|sequence| {
+ classify_controlled_fixture_attributes(
+ participant,
+ Some("user-1"),
+ attributes,
+ sequence,
+ )
+ .is_ok()
+ });
+ let mut ack_observed = false;
+ let mut observer_started = false;
+ start_observer_after_controlled_fixture_probe(pending, probe_result, || {
+ if pending && probe_result == Some(true) {
+ ack_observed = true;
+ }
+ observer_started = true;
+ });
+ if ack_observed {
+ effects.push("ack_observed");
+ }
+ if observer_started {
+ effects.push("observer_started");
+ }
+ pending_sequence = None;
+ }
+ PreAudioOrderEvent::DataReceived { .. } => {}
+ }
+ }
+ effects
+ }
+
+ #[test]
+ fn production_vad_session_boundary_reads_updated_attributes() {
+ let config = SimpleVadConfig {
+ rms_threshold: 0.001,
+ peak_threshold: 0.01,
+ start_frames: 2,
+ end_silence_ms: 100,
+ min_speech_ms: 1,
+ max_turn_ms: 1_000,
+ initial_ignore_ms: 0,
+ };
+ let mut vad = SimpleVad::new(config);
+ let samples = vec![1_000i16; 160];
+ let frame = AudioFrame {
+ data: samples.as_slice().into(),
+ sample_rate: 16_000,
+ num_channels: 1,
+ samples_per_channel: 160,
+ };
+ let mut attributes = std::collections::HashMap::from([
+ (
+ "inputSourceCategory".to_string(),
+ "controlled_fixture".to_string(),
+ ),
+ (
+ "clientFixtureSequence".to_string(),
+ "fixture-01".to_string(),
+ ),
+ ]);
+ let mut starts = Vec::new();
+ for (session_index, sequence) in [(1, "fixture-01"), (2, "fixture-02")] {
+ let was_in_speech = vad.in_speech;
+ vad.observe_frame(
+ "call-001",
+ "trace-001",
+ "participant",
+ "track",
+ session_index * 2 - 1,
+ 1_000 * session_index,
+ &frame,
+ );
+ vad.observe_frame(
+ "call-001",
+ "trace-001",
+ "participant",
+ "track",
+ session_index * 2,
+ 1_000 * session_index + 10,
+ &frame,
+ );
+ let is_in_speech = vad.in_speech;
+ assert!(!was_in_speech && is_in_speech);
+ attributes.insert("clientFixtureSequence".to_string(), sequence.to_string());
+ let metadata = AudioIngressMetadata::from_participant(&attributes)
+ .expect("valid participant attributes")
+ .expect("controlled fixture metadata");
+ let session_line = asr_realtime::session_start_line(
+ "call-001",
+ "trace-001",
+ &format!("turn-{session_index:04}"),
+ "nonce-001",
+ Some(&metadata),
+ )
+ .expect("session start line");
+ let session_json: serde_json::Value =
+ serde_json::from_slice(&session_line).expect("session start json");
+ assert_eq!(sequence, session_json["clientFixtureSequence"]);
+ starts.push(metadata.client_fixture_sequence);
+ vad.reset_current_turn();
+ }
+ assert_eq!(vec!["fixture-01", "fixture-02"], starts);
+ attributes.insert("inputSourceCategory".to_string(), "other".to_string());
+ assert!(AudioIngressMetadata::from_participant(&attributes).is_err());
+ assert!(
+ AudioIngressMetadata::from_participant(&std::collections::HashMap::new())
+ .expect("missing attributes is absent")
+ .is_none()
+ );
+ attributes.insert(
+ "clientFixtureSequence".to_string(),
+ "fixture-01".to_string(),
+ );
+ assert!(AudioIngressMetadata::from_participant(&attributes).is_err());
+ }
+
+ #[test]
+ fn production_observer_rejects_wrong_participant_before_vad_session() {
+ assert!(!is_bound_user_participant(
+ "participant-other",
+ Some("participant-user")
+ ));
+ assert!(is_bound_user_participant(
+ "participant-user",
+ Some("participant-user")
+ ));
+ assert!(is_bound_user_participant("participant-any", None));
+ }
+
+ #[tokio::test]
+ async fn production_observer_vad_to_session_entry_reads_each_updated_attribute() {
+ let listener = TcpListener::bind("127.0.0.1:0").expect("bind local ASR fixture");
+ let address = listener.local_addr().expect("fixture address");
+ let (request_tx, request_rx) = mpsc::channel::<String>();
+ let captured_count = Arc::new(AtomicUsize::new(0));
+ let captured_count_for_server = Arc::clone(&captured_count);
+ let server = thread::spawn(move || {
+ for _ in 0..2 {
+ let (mut stream, _) = listener.accept().expect("accept ASR session");
+ stream
+ .set_read_timeout(Some(Duration::from_secs(2)))
+ .expect("set fixture timeout");
+ let mut bytes = Vec::new();
+ let mut buffer = [0_u8; 4096];
+ loop {
+ match stream.read(&mut buffer) {
+ Ok(0) => break,
+ Ok(size) => {
+ bytes.extend_from_slice(&buffer[..size]);
+ if bytes.windows(7).any(|window| window == b"0\r\n\r\n") {
+ break;
+ }
+ }
+ Err(_) => break,
+ }
+ }
+ request_tx
+ .send(String::from_utf8_lossy(&bytes).into_owned())
+ .expect("capture ASR request");
+ captured_count_for_server.fetch_add(1, Ordering::SeqCst);
+ stream
+ .write_all(b"HTTP/1.1 200 OK\r\ncontent-type: application/json\r\ncontent-length: 39\r\nconnection: close\r\n\r\n{\"code\":0,\"data\":{\"status\":\"ok\"}}")
+ .expect("write fixture response");
+ }
+ });
+ let mut vad = SimpleVad::new(SimpleVadConfig {
+ rms_threshold: 0.001,
+ peak_threshold: 0.01,
+ start_frames: 1,
+ end_silence_ms: 100,
+ min_speech_ms: 1,
+ max_turn_ms: 1_000,
+ initial_ignore_ms: 0,
+ });
+ let frame_data = vec![1_000i16; 160];
+ let frame = AudioFrame {
+ data: frame_data.as_slice().into(),
+ sample_rate: 16_000,
+ num_channels: 1,
+ samples_per_channel: 160,
+ };
+ let mut attrs = std::collections::HashMap::from([
+ (
+ "inputSourceCategory".to_string(),
+ "controlled_fixture".to_string(),
+ ),
+ (
+ "clientFixtureSequence".to_string(),
+ "fixture-01".to_string(),
+ ),
+ ]);
+ let mut upload = None;
+ let mut last_fixture_sequence = None;
+ let config = RealtimeAsrConfig {
+ enabled: true,
+ url: Some(format!("http://{address}/runtime/asr/realtime")),
+ runtime_token: Some("test".to_string()),
+ runtime_session_nonce: Some("test".to_string()),
+ chunk_duration_ms: 200,
+ };
+ let (was, is, turn) = observe_bound_participant_frame(
+ "participant-user",
+ Some("participant-user"),
+ &mut vad,
+ "call-001",
+ "trace-001",
+ "participant-user",
+ "track-001",
+ 1,
+ 1_000,
+ &frame,
+ Client::new(),
+ config.clone(),
+ || attrs.clone(),
+ &mut upload,
+ &mut last_fixture_sequence,
+ true,
+ );
+ assert!(!was && is && turn.is_none());
+ assert!(upload.is_some());
+ upload.take().unwrap().cancel("test").await;
+ vad.reset_current_turn();
+ attrs.insert(
+ "clientFixtureSequence".to_string(),
+ "fixture-02".to_string(),
+ );
+ let (was, is, turn) = observe_bound_participant_frame(
+ "participant-user",
+ Some("participant-user"),
+ &mut vad,
+ "call-001",
+ "trace-001",
+ "participant-user",
+ "track-001",
+ 2,
+ 2_000,
+ &frame,
+ Client::new(),
+ config,
+ || attrs.clone(),
+ &mut upload,
+ &mut last_fixture_sequence,
+ true,
+ );
+ assert!(!was && is && turn.is_none());
+ assert!(upload.is_some());
+ upload.take().unwrap().cancel("test").await;
+
+ let first_request = request_rx
+ .recv_timeout(Duration::from_secs(2))
+ .expect("first session request");
+ let second_request = request_rx
+ .recv_timeout(Duration::from_secs(2))
+ .expect("second session request");
+ assert!(first_request.contains("\"clientFixtureSequence\":\"fixture-01\""));
+ assert!(second_request.contains("\"clientFixtureSequence\":\"fixture-02\""));
+ assert!(
+ first_request.contains("\"audioIngressOriginStatus\":\"controlled_fixture_bound\"")
+ );
+ assert!(
+ second_request.contains("\"audioIngressOriginStatus\":\"controlled_fixture_bound\"")
+ );
+ // The same production boundary rejects a wrong participant before VAD/session creation.
+ assert!(!is_bound_user_participant(
+ "participant-other",
+ Some("participant-user")
+ ));
+ assert_eq!(0, request_rx.try_iter().count());
+ vad.reset_current_turn();
+ attrs.insert(
+ "clientFixtureSequence".to_string(),
+ "fixture-03".to_string(),
+ );
+ let (_, is_wrong, wrong_turn) = observe_bound_participant_frame(
+ "participant-other",
+ Some("participant-user"),
+ &mut vad,
+ "call-001",
+ "trace-001",
+ "participant-user",
+ "track-001",
+ 3,
+ 3_000,
+ &frame,
+ Client::new(),
+ RealtimeAsrConfig {
+ enabled: true,
+ url: Some(format!("http://{address}/runtime/asr/realtime")),
+ runtime_token: Some("test".to_string()),
+ runtime_session_nonce: Some("test".to_string()),
+ chunk_duration_ms: 200,
+ },
+ || attrs.clone(),
+ &mut upload,
+ &mut last_fixture_sequence,
+ true,
+ );
+ assert!(!is_wrong && wrong_turn.is_none() && upload.is_none());
+ vad.reset_current_turn();
+ attrs.insert(
+ "clientFixtureSequence".to_string(),
+ "fixture-01".to_string(),
+ );
+ let (_, _, _) = observe_bound_participant_frame(
+ "participant-user",
+ Some("participant-user"),
+ &mut vad,
+ "call-001",
+ "trace-001",
+ "participant-user",
+ "track-001",
+ 4,
+ 4_000,
+ &frame,
+ Client::new(),
+ RealtimeAsrConfig {
+ enabled: true,
+ url: Some(format!("http://{address}/runtime/asr/realtime")),
+ runtime_token: Some("test".to_string()),
+ runtime_session_nonce: Some("test".to_string()),
+ chunk_duration_ms: 200,
+ },
+ || attrs.clone(),
+ &mut upload,
+ &mut last_fixture_sequence,
+ true,
+ );
+ assert!(upload.is_none());
+ vad.reset_current_turn();
+ let (_, _, _) = observe_bound_participant_frame(
+ "participant-user",
+ Some("participant-user"),
+ &mut vad,
+ "call-001",
+ "trace-001",
+ "participant-user",
+ "track-001",
+ 5,
+ 5_000,
+ &frame,
+ Client::new(),
+ RealtimeAsrConfig {
+ enabled: true,
+ url: Some(format!("http://{address}/runtime/asr/realtime")),
+ runtime_token: Some("test".to_string()),
+ runtime_session_nonce: Some("test".to_string()),
+ chunk_duration_ms: 200,
+ },
+ || attrs.clone(),
+ &mut upload,
+ &mut last_fixture_sequence,
+ true,
+ );
+ assert!(upload.is_none());
+ vad.reset_current_turn();
+ attrs.remove("clientFixtureSequence");
+ let (_, _, missing_sequence_turn) = observe_bound_participant_frame(
+ "participant-user",
+ Some("participant-user"),
+ &mut vad,
+ "call-001",
+ "trace-001",
+ "participant-user",
+ "track-001",
+ 6,
+ 6_000,
+ &frame,
+ Client::new(),
+ RealtimeAsrConfig {
+ enabled: true,
+ url: Some(format!("http://{address}/runtime/asr/realtime")),
+ runtime_token: Some("test".to_string()),
+ runtime_session_nonce: Some("test".to_string()),
+ chunk_duration_ms: 200,
+ },
+ || attrs.clone(),
+ &mut upload,
+ &mut last_fixture_sequence,
+ true,
+ );
+ assert!(missing_sequence_turn.is_none() && upload.is_none());
+ assert_eq!(
+ "sequence_absent",
+ AudioIngressMetadata::rejected_origin_status("incomplete_metadata", &attrs)
+ );
+ assert_eq!(2, captured_count.load(Ordering::SeqCst));
+ vad.reset_current_turn();
+ attrs.insert("inputSourceCategory".to_string(), "other".to_string());
+ let mut invalid_upload = None;
+ let (_, _, invalid_turn) = observe_bound_participant_frame(
+ "participant-user",
+ Some("participant-user"),
+ &mut vad,
+ "call-001",
+ "trace-001",
+ "participant-other",
+ "track-001",
+ 3,
+ 3_000,
+ &frame,
+ Client::new(),
+ RealtimeAsrConfig {
+ enabled: true,
+ url: Some("http://127.0.0.1:9".to_string()),
+ runtime_token: Some("test".to_string()),
+ runtime_session_nonce: Some("test".to_string()),
+ chunk_duration_ms: 200,
+ },
+ || attrs,
+ &mut invalid_upload,
+ &mut last_fixture_sequence,
+ true,
+ );
+ assert!(invalid_turn.is_none());
+ assert!(invalid_upload.is_none());
+ assert_eq!(2, captured_count.load(Ordering::SeqCst));
+ assert_eq!(0, request_rx.try_iter().count());
+ server.join().expect("fixture server");
+ }
+
+ #[test]
+ fn reply_chunk_marker_state_emits_turn_first_once_and_later_segment_first_once() {
+ let mut state = ReplyChunkMarkerState::default();
+
+ assert_eq!(ReplyChunkMarker::FirstReply, state.observe(Some(1)));
+ assert_eq!(ReplyChunkMarker::None, state.observe(Some(1)));
+ assert_eq!(ReplyChunkMarker::SegmentFirst, state.observe(Some(2)));
+ assert_eq!(ReplyChunkMarker::None, state.observe(Some(2)));
+ assert_eq!(ReplyChunkMarker::SegmentFirst, state.observe(Some(3)));
+ }
+
+ #[test]
+ fn reply_chunk_marker_state_without_segment_only_emits_turn_first() {
+ let mut state = ReplyChunkMarkerState::default();
+
+ assert_eq!(ReplyChunkMarker::FirstReply, state.observe(None));
+ assert_eq!(ReplyChunkMarker::None, state.observe(None));
+ }
+
+ #[test]
+ fn runtime_turn_stream_audio_chunk_reads_segment_seq() {
+ let event: RuntimeTurnStreamEvent = serde_json::from_str(
+ r#"{"type":"reply_audio_chunk","audioChunk":{"chunkSeq":4,"segmentSeq":2,"format":"pcm_s16le","payloadBase64":"AA==","last":false}}"#,
+ )
+ .expect("turn stream event");
+
+ assert_eq!(
+ Some(2),
+ event.audio_chunk.and_then(|chunk| chunk.segment_seq)
+ );
+ }
+
+ #[test]
+ fn runtime_turn_stream_reads_frozen_m5_and_audio_chunk_timing_contract() {
+ let activity: RuntimeTurnStreamEvent = serde_json::from_str(
+ r#"{"type":"activity","callId":"call-1","traceId":"trace-1","turnId":"turn-1","activity":{"eventType":"tts_first_audio_chunk_ready","extension":{"streamTimingVersion":1,"streamAnchorId":"0123456789abcdef","streamAnchorServerDeltaMs":1200,"runtimeSessionNonceHash":"abcdef012345","segmentSeq":1,"streamTimingValidation":"bound"}}}"#,
+ )
+ .expect("m5 activity event");
+ let audio: RuntimeTurnStreamEvent = serde_json::from_str(
+ r#"{"type":"reply_audio_chunk","callId":"call-1","traceId":"trace-1","turnId":"turn-1","audioChunk":{"chunkSeq":1,"segmentSeq":1,"streamTimingVersion":1,"streamAnchorId":"0123456789abcdef","format":"pcm_s16le","payloadBase64":"AA==","last":false}}"#,
+ )
+ .expect("timed audio chunk event");
+
+ assert_eq!(Some("call-1"), activity.call_id.as_deref());
+ assert_eq!(Some("trace-1"), activity.trace_id.as_deref());
+ assert_eq!(Some("turn-1"), activity.turn_id.as_deref());
+ let extension = activity.activity.unwrap().extension.unwrap();
+ assert_eq!(Some(1), extension.stream_timing_version);
+ assert_eq!(Some(1200), extension.stream_anchor_server_delta_ms);
+ assert_eq!(
+ Some("0123456789abcdef"),
+ audio.audio_chunk.unwrap().stream_anchor_id.as_deref()
+ );
+ }
+
+ #[test]
+ fn runtime_turn_stream_timing_records_m6_and_m7_once_then_rejects_terminal_late_events() {
+ let nonce = "runtime-nonce";
+ let nonce_hash = runtime_session_nonce_hash(nonce);
+ let activity: RuntimeTurnStreamEvent = serde_json::from_str(&format!(
+ r#"{{"type":"activity","callId":"call-1","traceId":"trace-1","turnId":"turn-1","activity":{{"eventType":"tts_first_audio_chunk_ready","extension":{{"streamTimingVersion":1,"streamAnchorId":"0123456789abcdef","streamAnchorServerDeltaMs":1200,"runtimeSessionNonceHash":"{nonce_hash}","segmentSeq":1,"streamTimingValidation":"bound"}}}}}}"#,
+ ))
+ .expect("m5 activity event");
+ let audio: RuntimeTurnStreamEvent = serde_json::from_str(
+ r#"{"type":"reply_audio_chunk","callId":"call-1","traceId":"trace-1","turnId":"turn-1","audioChunk":{"chunkSeq":1,"segmentSeq":1,"streamTimingVersion":1,"streamAnchorId":"0123456789abcdef","format":"pcm_s16le","payloadBase64":"AA==","last":false}}"#,
+ )
+ .expect("timed audio chunk event");
+ let mut state = RuntimeTurnStreamState::default();
+
+ assert!(state.arm_timing_anchor("call-1", "trace-1", "turn-1", nonce, &activity));
+ let chunk = audio.audio_chunk.as_ref().expect("audio chunk");
+ assert!(
+ state
+ .record_m6("call-1", "trace-1", "turn-1", &audio, chunk)
+ .is_some()
+ );
+ assert!(
+ state
+ .record_m6("call-1", "trace-1", "turn-1", &audio, chunk)
+ .is_none()
+ );
+ assert!(state.record_m7().is_some());
+ assert!(state.record_m7().is_none());
+
+ state.close_timing();
+ assert_eq!(RuntimeTurnStreamTimingPhase::Closed, state.timing.phase);
+ assert!(state.timing.anchor.is_none());
+ assert!(!state.arm_timing_anchor("call-1", "trace-1", "turn-1", nonce, &activity));
+ assert!(
+ state
+ .record_m6("call-1", "trace-1", "turn-1", &audio, chunk)
+ .is_none()
+ );
+ assert!(state.record_m7().is_none());
+ }
+
+ #[test]
+ fn device_output_contract_is_reliable_and_deduplicated() {
+ let output = RuntimeTurnDeviceOutput {
+ command_id: Some("cmd-1".to_string()),
+ command_code: Some("custom.app.DeviceLevelChange".to_string()),
+ params: Some(serde_json::json!({"level": 1})),
+ };
+ let mut published = HashSet::new();
+ assert!(should_publish_device_output(&mut published, &output));
+ assert!(!should_publish_device_output(&mut published, &output));
+ assert_eq!(published.len(), 1);
+ }
+
+ #[test]
+ fn device_output_invalid_or_missing_command_is_fail_closed() {
+ for output in [
+ RuntimeTurnDeviceOutput {
+ command_id: None,
+ command_code: Some("custom.app.DeviceLevelChange".to_string()),
+ params: None,
+ },
+ RuntimeTurnDeviceOutput {
+ command_id: Some("cmd-1".to_string()),
+ command_code: None,
+ params: None,
+ },
+ ] {
+ let mut published = HashSet::new();
+ assert!(!should_publish_device_output(&mut published, &output));
+ assert!(published.is_empty());
+ }
+ }
+
+ #[test]
+ fn controlled_fixture_probe_requires_bound_hashes_and_protocol() {
+ let call_id = "call-ack-1";
+ let trace_id = "trace-ack-1";
+ let payload = serde_json::to_vec(&json!({
+ "type": CONTROLLED_FIXTURE_PROBE_TOPIC,
+ "protocolVersion": CONTROLLED_FIXTURE_PROTOCOL_VERSION,
+ "callIdHash": sha256_hex(call_id),
+ "callTraceIdHash": sha256_hex(trace_id),
+ "generation": CONTROLLED_FIXTURE_GENERATION,
+ "clientFixtureSequence": "fixture-01"
+ }))
+ .unwrap();
+ let sender = ParticipantIdentity("user-1".to_string());
+ let pending =
+ controlled_fixture_probe(&payload, call_id, trace_id, &sender, Some("user-1"))
+ .expect("valid probe");
+ assert_eq!(pending.sequence, "fixture-01");
+ assert_eq!(pending.call_id_hash, sha256_hex(call_id));
+ assert_eq!(pending.call_trace_id_hash, sha256_hex(trace_id));
+ assert_eq!(pending.generation, CONTROLLED_FIXTURE_GENERATION);
+ assert!(
+ controlled_fixture_probe(&payload, call_id, "other-trace", &sender, Some("user-1"),)
+ .is_none()
+ );
+ assert!(
+ controlled_fixture_probe(
+ &payload,
+ call_id,
+ trace_id,
+ &ParticipantIdentity("other-user".to_string()),
+ Some("user-1"),
+ )
+ .is_none()
+ );
+ }
+
+ #[test]
+ fn controlled_fixture_attributes_ack_only_on_exact_current_sequence() {
+ let mut attributes = std::collections::HashMap::from([
+ (
+ "inputSourceCategory".to_string(),
+ "controlled_fixture".to_string(),
+ ),
+ (
+ "clientFixtureSequence".to_string(),
+ "fixture-01".to_string(),
+ ),
+ ]);
+ assert!(
+ classify_controlled_fixture_attributes(
+ "user-1",
+ Some("user-1"),
+ &attributes,
+ "fixture-01"
+ )
+ .is_ok()
+ );
+ attributes.insert(
+ "clientFixtureSequence".to_string(),
+ "fixture-02".to_string(),
+ );
+ assert_eq!(
+ classify_controlled_fixture_attributes(
+ "user-1",
+ Some("user-1"),
+ &attributes,
+ "fixture-01"
+ ),
+ Err("wrong_sequence")
+ );
+ attributes.remove("clientFixtureSequence");
+ assert_eq!(
+ classify_controlled_fixture_attributes(
+ "user-1",
+ Some("user-1"),
+ &attributes,
+ "fixture-01"
+ ),
+ Err("missing_sequence")
+ );
+ assert_eq!(
+ classify_controlled_fixture_attributes(
+ "other-user",
+ Some("user-1"),
+ &attributes,
+ "fixture-01"
+ ),
+ Err("wrong_participant")
+ );
+ }
+
+ #[test]
+ fn controlled_fixture_probe_runtime_projection_binds_request_hashes_and_audio_gate() {
+ let call_id = "private-call-value";
+ let trace_id = "private-trace-value";
+ let sequence = "private-sequence-value";
+ let call_id_hash = sha256_hex(call_id);
+ let trace_id_hash = sha256_hex(trace_id);
+ let mut observer_starts = 0;
+ let (ack_result, reject_reason, observed) =
+ controlled_fixture_ack_classification(Err("wrong_source"));
+ let stages = [
+ ("data_received", None, None),
+ ("attributes_classified", Some(ack_result), reject_reason),
+ ("ack_publish_started", Some(ack_result), reject_reason),
+ ("ack_publish_completed", Some(ack_result), reject_reason),
+ ("audio_observer_blocked", Some(ack_result), reject_reason),
+ ];
+ for (stage, result, reason) in stages {
+ let event = controlled_fixture_probe_event(
+ call_id,
+ trace_id,
+ stage,
+ &call_id_hash,
+ &trace_id_hash,
+ CONTROLLED_FIXTURE_GENERATION,
+ sequence,
+ observed,
+ result,
+ reason,
+ );
+ assert_eq!(event["type"], "cv_activity");
+ assert_eq!(event["eventName"], "controlled_fixture_attribute_probe");
+ assert_eq!(event["extension"]["call_id_hash"], call_id_hash);
+ assert_eq!(event["extension"]["trace_id_hash"], trace_id_hash);
+ assert_eq!(event["extension"]["stage"], stage);
+ let output = event.to_string();
+ assert!(!output.contains(sequence));
+ }
+ assert!(!start_observer_after_controlled_fixture_probe(
+ true,
+ Some(observed),
+ || observer_starts += 1,
+ ));
+ assert_eq!(observer_starts, 0);
+
+ let (ack_result, reject_reason, observed) = controlled_fixture_ack_classification(Ok(()));
+ let allowed = controlled_fixture_probe_event(
+ call_id,
+ trace_id,
+ "audio_observer_allowed",
+ &call_id_hash,
+ &trace_id_hash,
+ CONTROLLED_FIXTURE_GENERATION,
+ sequence,
+ observed,
+ Some(ack_result),
+ reject_reason,
+ );
+ assert_eq!(allowed["extension"]["trace_id_hash"], trace_id_hash);
+ assert!(start_observer_after_controlled_fixture_probe(
+ true,
+ Some(observed),
+ || observer_starts += 1,
+ ));
+ assert_eq!(observer_starts, 1);
+ }
+
+ #[test]
+ fn controlled_fixture_probe_observed_and_failure_enums_are_stable() {
+ assert_eq!(
+ controlled_fixture_ack_classification(Ok(())),
+ ("observed", None, true)
+ );
+ assert_eq!(
+ controlled_fixture_ack_classification(Err("timeout")),
+ ("timeout", Some("expired"), false)
+ );
+ for reason in [
+ "missing_attributes",
+ "wrong_source",
+ "missing_sequence",
+ "wrong_sequence",
+ "wrong_participant",
+ ] {
+ assert_eq!(
+ controlled_fixture_ack_classification(Err(reason)),
+ ("rejected", Some(reason), false)
+ );
+ }
+ assert_eq!(
+ controlled_fixture_ack_classification(Err("unclassified")),
+ ("rejected", Some("unknown"), false)
+ );
+ }
+
+ #[test]
+ fn production_probe_binding_drives_rejected_and_observed_ack_without_local_rehash() {
+ let request_call_hash = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa";
+ let request_trace_hash = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb";
+ let probe = PendingControlledFixtureProbe {
+ sender: ParticipantIdentity("user-1".to_string()),
+ call_id_hash: request_call_hash.to_string(),
+ call_trace_id_hash: request_trace_hash.to_string(),
+ generation: CONTROLLED_FIXTURE_GENERATION,
+ sequence: "fixture-01".to_string(),
+ received_at: Instant::now(),
+ expires_at: Instant::now() + CONTROLLED_FIXTURE_PROBE_TTL,
+ };
+
+ let (ack_result, reject_reason, observed) =
+ controlled_fixture_ack_classification(Err("wrong_sequence"));
+ let rejected_ack = controlled_fixture_ack_from_probe(&probe, observed, reject_reason);
+ let rejected_json = serde_json::to_value(&rejected_ack).unwrap();
+ let mut rejected_observer_starts = 0;
+ assert_eq!(ack_result, "rejected");
+ assert_eq!(rejected_json["callIdHash"], request_call_hash);
+ assert_eq!(rejected_json["callTraceIdHash"], request_trace_hash);
+ assert_eq!(rejected_json["rejectReason"], "wrong_sequence");
+ assert!(!start_observer_after_controlled_fixture_probe(
+ true,
+ Some(observed),
+ || rejected_observer_starts += 1,
+ ));
+ assert_eq!(rejected_observer_starts, 0);
+
+ let (ack_result, reject_reason, observed) = controlled_fixture_ack_classification(Ok(()));
+ let observed_ack = controlled_fixture_ack_from_probe(&probe, observed, reject_reason);
+ let observed_json = serde_json::to_value(&observed_ack).unwrap();
+ let mut observed_observer_starts = 0;
+ assert_eq!(ack_result, "observed");
+ assert_eq!(observed_json["callIdHash"], request_call_hash);
+ assert_eq!(observed_json["callTraceIdHash"], request_trace_hash);
+ assert!(observed_json.get("rejectReason").is_none());
+ assert!(start_observer_after_controlled_fixture_probe(
+ true,
+ Some(observed),
+ || observed_observer_starts += 1,
+ ));
+ assert_eq!(observed_observer_starts, 1);
+ }
+
+ #[tokio::test]
+ async fn production_ack_publish_failure_keeps_observer_session_and_audio_closed() {
+ let mut acknowledged = HashSet::new();
+ let probe_result = complete_controlled_fixture_ack_publish(
+ async { Err::<(), ()>(()) },
+ "runtime-call-publish-failure",
+ "runtime-trace-publish-failure",
+ true,
+ "observed",
+ None,
+ "call-publish-failure",
+ "trace-publish-failure",
+ CONTROLLED_FIXTURE_GENERATION,
+ "fixture-01",
+ &mut acknowledged,
+ )
+ .await;
+ let mut observer_starts = 0;
+ let mut session_starts = 0;
+ let mut audio_starts = 0;
+ let mut speaking_starts = 0;
+ assert!(!start_observer_after_controlled_fixture_probe(
+ true,
+ Some(probe_result.observed),
+ || {
+ observer_starts += 1;
+ session_starts += 1;
+ audio_starts += 1;
+ speaking_starts += 1;
+ },
+ ));
+ assert_eq!(
+ probe_result,
+ ControlledFixtureAckPublishOutcome {
+ observed: false,
+ published: false,
+ }
+ );
+ assert!(acknowledged.is_empty());
+ assert_eq!(observer_starts, 0);
+ assert_eq!(session_starts, 0);
+ assert_eq!(audio_starts, 0);
+ assert_eq!(speaking_starts, 0);
+
+ let observed_result = complete_controlled_fixture_ack_publish(
+ async { Ok::<(), ()>(()) },
+ "runtime-call-publish-success",
+ "runtime-trace-publish-success",
+ true,
+ "observed",
+ None,
+ "call-publish-success",
+ "trace-publish-success",
+ CONTROLLED_FIXTURE_GENERATION,
+ "fixture-02",
+ &mut acknowledged,
+ )
+ .await;
+ let mut successful_observer_starts = 0;
+ assert!(start_observer_after_controlled_fixture_probe(
+ true,
+ Some(observed_result.observed),
+ || successful_observer_starts += 1,
+ ));
+ assert_eq!(
+ observed_result,
+ ControlledFixtureAckPublishOutcome {
+ observed: true,
+ published: true,
+ }
+ );
+ assert!(acknowledged.contains("fixture-02"));
+ assert_eq!(successful_observer_starts, 1);
+ }
+
+ #[tokio::test]
+ async fn production_attribute_observation_accepts_server_visibility_within_probe_ttl() {
+ let expected = HashMap::from([
+ (
+ "inputSourceCategory".to_string(),
+ "controlled_fixture".to_string(),
+ ),
+ (
+ "clientFixtureSequence".to_string(),
+ "fixture-01".to_string(),
+ ),
+ ]);
+ let mut reads = 0;
+ let result = observe_controlled_fixture_attributes(
+ Instant::now() + CONTROLLED_FIXTURE_PROBE_TTL,
+ "user-1",
+ Some("user-1"),
+ "fixture-01",
+ || {
+ reads += 1;
+ if reads <= 3 {
+ HashMap::new()
+ } else {
+ expected.clone()
+ }
+ },
+ )
+ .await;
+ assert_eq!(result, Ok(()));
+ assert_eq!(reads, 4);
+ }
+
+ #[tokio::test]
+ async fn production_attribute_observation_rejects_wrong_sequence_without_audio_effect() {
+ let attributes = HashMap::from([
+ (
+ "inputSourceCategory".to_string(),
+ "controlled_fixture".to_string(),
+ ),
+ (
+ "clientFixtureSequence".to_string(),
+ "fixture-02".to_string(),
+ ),
+ ]);
+ let result = observe_controlled_fixture_attributes(
+ Instant::now() + CONTROLLED_FIXTURE_PROBE_TTL,
+ "user-1",
+ Some("user-1"),
+ "fixture-01",
+ || attributes.clone(),
+ )
+ .await;
+ let mut observer_starts = 0;
+ assert_eq!(result, Err("wrong_sequence"));
+ assert!(!start_observer_after_controlled_fixture_probe(
+ true,
+ Some(result.is_ok()),
+ || observer_starts += 1,
+ ));
+ assert_eq!(observer_starts, 0);
+ }
+
+ #[tokio::test]
+ async fn production_post_expiry_observation_records_bounded_visibility_without_second_ack() {
+ let started_at = Instant::now();
+ let active = Arc::new(AtomicBool::new(true));
+ let expected = HashMap::from([
+ (
+ "inputSourceCategory".to_string(),
+ "controlled_fixture".to_string(),
+ ),
+ (
+ "clientFixtureSequence".to_string(),
+ "fixture-01".to_string(),
+ ),
+ ]);
+ let mut acknowledged = HashSet::new();
+ let expired_ack = complete_controlled_fixture_ack_publish(
+ async { Ok::<(), ()>(()) },
+ "runtime-call-post-expiry",
+ "runtime-trace-post-expiry",
+ false,
+ "timeout",
+ Some("expired"),
+ "call-post-expiry",
+ "trace-post-expiry",
+ CONTROLLED_FIXTURE_GENERATION,
+ "fixture-01",
+ &mut acknowledged,
+ )
+ .await;
+ assert_eq!(
+ expired_ack,
+ ControlledFixtureAckPublishOutcome {
+ observed: false,
+ published: true,
+ }
+ );
+ let evidence = observe_controlled_fixture_post_expiry_views(
+ started_at,
+ started_at + CONTROLLED_FIXTURE_POST_EXPIRY_WINDOW,
+ "user-1",
+ Some("user-1"),
+ "fixture-01",
+ active,
+ || {
+ if started_at.elapsed() >= Duration::from_millis(300) {
+ expected.clone()
+ } else {
+ HashMap::new()
+ }
+ },
+ || Some(("user-1".to_string(), HashMap::new())),
+ )
+ .await;
+ assert_eq!(acknowledged.len(), 1);
+ assert_eq!(
+ evidence,
+ Some(ControlledFixtureVisibilityEvidence {
+ first_visible_bucket: "250_500ms",
+ visibility_source: "participant_attributes_poll",
+ visibility_result: "held_visible",
+ binding_matched: true,
+ })
+ );
+
+ let never_started_at = Instant::now();
+ assert_eq!(
+ observe_controlled_fixture_post_expiry_views(
+ never_started_at,
+ never_started_at + Duration::from_millis(40),
+ "user-1",
+ Some("user-1"),
+ "fixture-01",
+ Arc::new(AtomicBool::new(true)),
+ HashMap::new,
+ || Some(("user-1".to_string(), HashMap::new())),
+ )
+ .await,
+ Some(ControlledFixtureVisibilityEvidence {
+ first_visible_bucket: "never_visible_within_observation_window",
+ visibility_source: "held_and_current_room_lookup",
+ visibility_result: "unavailable_both",
+ binding_matched: true,
+ })
+ );
+
+ assert_eq!(
+ observe_controlled_fixture_post_expiry_views(
+ Instant::now(),
+ Instant::now() + Duration::from_millis(50),
+ "cross-call-user",
+ Some("user-1"),
+ "fixture-01",
+ Arc::new(AtomicBool::new(true)),
+ || expected.clone(),
+ || Some(("user-1".to_string(), expected.clone())),
+ )
+ .await,
+ None
+ );
+
+ let inactive = Arc::new(AtomicBool::new(false));
+ assert_eq!(
+ observe_controlled_fixture_post_expiry_views(
+ Instant::now(),
+ Instant::now() + Duration::from_millis(50),
+ "user-1",
+ Some("user-1"),
+ "fixture-01",
+ inactive,
+ HashMap::new,
+ || Some(("user-1".to_string(), HashMap::new())),
+ )
+ .await,
+ None
+ );
+ assert_eq!(acknowledged.len(), 1);
+
+ let probe = PendingControlledFixtureProbe {
+ sender: ParticipantIdentity("user-1".to_string()),
+ call_id_hash: "call-post-expiry".to_string(),
+ call_trace_id_hash: "trace-post-expiry".to_string(),
+ generation: CONTROLLED_FIXTURE_GENERATION,
+ sequence: "fixture-01".to_string(),
+ received_at: started_at,
+ expires_at: started_at + CONTROLLED_FIXTURE_PROBE_TTL,
+ };
+ let event = controlled_fixture_visibility_event(
+ "runtime-call-post-expiry",
+ "runtime-trace-post-expiry",
+ &probe,
+ &evidence.unwrap(),
+ );
+ let extension = event["extension"].as_object().unwrap();
+ let mut keys = extension.keys().map(String::as_str).collect::<Vec<_>>();
+ keys.sort_unstable();
+ assert_eq!(
+ keys,
+ vec![
+ "binding_matched",
+ "call_id_hash",
+ "evidence_count",
+ "first_visible_bucket",
+ "generation",
+ "sequence_hash",
+ "stage",
+ "trace_id_hash",
+ "visibility_result",
+ "visibility_source",
+ ]
+ );
+ let encoded = event.to_string();
+ for forbidden in [
+ "\"participant\":",
+ "\"room\":",
+ "\"track\":",
+ "\"payload\":",
+ "\"audio\":",
+ ] {
+ assert!(!encoded.contains(forbidden));
+ }
+ }
+
+ #[tokio::test]
+ async fn production_post_expiry_observation_distinguishes_held_stale_from_current_room_view() {
+ let started_at = Instant::now();
+ let current_attributes = HashMap::from([
+ (
+ "inputSourceCategory".to_string(),
+ "controlled_fixture".to_string(),
+ ),
+ (
+ "clientFixtureSequence".to_string(),
+ "fixture-01".to_string(),
+ ),
+ ]);
+ let mut acknowledged = HashSet::new();
+ let expired_ack = complete_controlled_fixture_ack_publish(
+ async { Ok::<(), ()>(()) },
+ "runtime-call-current-view",
+ "runtime-trace-current-view",
+ false,
+ "timeout",
+ Some("expired"),
+ "call-current-view",
+ "trace-current-view",
+ CONTROLLED_FIXTURE_GENERATION,
+ "fixture-01",
+ &mut acknowledged,
+ )
+ .await;
+ let evidence = observe_controlled_fixture_post_expiry_views(
+ started_at,
+ started_at + Duration::from_millis(100),
+ "user-1",
+ Some("user-1"),
+ "fixture-01",
+ Arc::new(AtomicBool::new(true)),
+ HashMap::new,
+ || Some(("user-1".to_string(), current_attributes.clone())),
+ )
+ .await;
+
+ assert_eq!(
+ expired_ack,
+ ControlledFixtureAckPublishOutcome {
+ observed: false,
+ published: true,
+ }
+ );
+ assert_eq!(acknowledged.len(), 1);
+ assert_eq!(
+ evidence,
+ Some(ControlledFixtureVisibilityEvidence {
+ first_visible_bucket: "lte_250ms",
+ visibility_source: "current_room_lookup",
+ visibility_result: "held_stale_current_visible",
+ binding_matched: true,
+ })
+ );
+ let mut observer_starts = 0;
+ assert!(!start_observer_after_controlled_fixture_probe(
+ true,
+ Some(expired_ack.observed),
+ || observer_starts += 1,
+ ));
+ assert_eq!(observer_starts, 0);
+
+ for current_view in [
+ None,
+ Some(("cross-call-user".to_string(), current_attributes.clone())),
+ Some((
+ "user-1".to_string(),
+ HashMap::from([
+ (
+ "inputSourceCategory".to_string(),
+ "controlled_fixture".to_string(),
+ ),
+ (
+ "clientFixtureSequence".to_string(),
+ "fixture-old".to_string(),
+ ),
+ ]),
+ )),
+ ] {
+ assert_eq!(
+ observe_controlled_fixture_post_expiry_views(
+ Instant::now(),
+ Instant::now() + Duration::from_millis(20),
+ "user-1",
+ Some("user-1"),
+ "fixture-01",
+ Arc::new(AtomicBool::new(true)),
+ HashMap::new,
+ || current_view.clone(),
+ )
+ .await,
+ None
+ );
+ }
+ }
+
+ #[test]
+ fn controlled_fixture_ack_payload_is_reliable_and_redacted() {
+ let ack = ControlledFixtureAttributeAck {
+ message_type: CONTROLLED_FIXTURE_ACK_TOPIC,
+ protocol_version: CONTROLLED_FIXTURE_PROTOCOL_VERSION,
+ call_id_hash: sha256_hex("call-1"),
+ call_trace_id_hash: sha256_hex("trace-1"),
+ generation: CONTROLLED_FIXTURE_GENERATION,
+ client_fixture_sequence: "fixture-01".to_string(),
+ result: "observed",
+ input_source_category: Some("controlled_fixture"),
+ reject_reason: None,
+ };
+ let encoded = serde_json::to_vec(&ack).unwrap();
+ let decoded: serde_json::Value = serde_json::from_slice(&encoded).unwrap();
+ assert_eq!(decoded["type"], CONTROLLED_FIXTURE_ACK_TOPIC);
+ assert_eq!(
+ decoded["protocolVersion"],
+ CONTROLLED_FIXTURE_PROTOCOL_VERSION
+ );
+ assert_eq!(decoded["result"], "observed");
+ assert!(decoded.get("callId").is_none());
+ assert!(decoded.get("traceId").is_none());
+ assert!(decoded.get("participantIdentity").is_none());
+ assert!(decoded.get("audio").is_none());
+ }
+
+ #[test]
+ fn controlled_fixture_probe_does_not_create_session_or_audio_side_effects() {
+ let attributes = std::collections::HashMap::new();
+ assert_eq!(
+ classify_controlled_fixture_attributes(
+ "user-1",
+ Some("user-1"),
+ &attributes,
+ "fixture-01"
+ ),
+ Err("missing_attributes")
+ );
+ assert_eq!(
+ CONTROLLED_FIXTURE_PROBE_TOPIC,
+ "controlled_fixture_attribute_probe"
+ );
+ assert_eq!(
+ CONTROLLED_FIXTURE_ACK_TOPIC,
+ "controlled_fixture_attribute_ack"
+ );
+ }
+
+ #[test]
+ fn controlled_fixture_probe_must_be_observed_before_audio_observer() {
+ assert!(controlled_fixture_observer_gate(false, None));
+ assert!(controlled_fixture_observer_gate(true, Some(true)));
+ assert!(!controlled_fixture_observer_gate(true, Some(false)));
+ assert!(!controlled_fixture_observer_gate(true, None));
+ }
+
+ #[test]
+ fn production_event_order_probe_then_track_publishes_ack_before_observer() {
+ let attributes = HashMap::from([
+ (
+ "inputSourceCategory".to_string(),
+ "controlled_fixture".to_string(),
+ ),
+ (
+ "clientFixtureSequence".to_string(),
+ "fixture-01".to_string(),
+ ),
+ ]);
+ let effects = drive_pre_audio_order_test_seam(&[
+ PreAudioOrderEvent::DataReceived {
+ sender: "user-1".to_string(),
+ sequence: "fixture-01".to_string(),
+ },
+ PreAudioOrderEvent::TrackSubscribed {
+ participant: "user-1".to_string(),
+ attributes,
+ },
+ ]);
+ assert_eq!(effects, ["ack_observed", "observer_started"]);
+ }
+
+ #[test]
+ fn production_event_order_negative_probe_has_no_observer_or_session_effect() {
+ let mut invalid = HashMap::new();
+ invalid.insert(
+ "inputSourceCategory".to_string(),
+ "ordinary_mic".to_string(),
+ );
+ let effects = drive_pre_audio_order_test_seam(&[
+ PreAudioOrderEvent::DataReceived {
+ sender: "user-1".to_string(),
+ sequence: "fixture-01".to_string(),
+ },
+ PreAudioOrderEvent::TrackSubscribed {
+ participant: "user-1".to_string(),
+ attributes: invalid,
+ },
+ ]);
+ assert!(effects.is_empty());
+ }
+
+ #[test]
+ fn production_audio_branch_orders_probe_before_spawn_callsite() {
+ let source = include_str!("main.rs");
+ let branch = source
+ .find("RoomEvent::TrackSubscribed {\n track: RemoteTrack::Audio")
+ .expect("audio TrackSubscribed production branch");
+ let branch_source = &source[branch..];
+ let probe = branch_source
+ .find("let probe_result = process_controlled_fixture_probe")
+ .expect("probe must be processed in audio branch");
+ let spawn = branch_source
+ .find("start_observer_after_controlled_fixture_probe")
+ .expect("spawn must use shared order entry");
+ assert!(
+ probe < spawn,
+ "probe must precede shared observer spawn entry"
+ );
+ }
+}
--
Gitblit v1.9.3