From 9eca12eae4372c79eb35cef972f1ab4756c0a5e0 Mon Sep 17 00:00:00 2001
From: cai <cai@nbcai.cc>
Date: Wed, 12 Aug 2026 11:55:49 +0800
Subject: [PATCH] Add controlled fixture ACK observability
---
src/main.rs | 2704 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++
1 files changed, 2,660 insertions(+), 44 deletions(-)
diff --git a/src/main.rs b/src/main.rs
index 05ba69b..6a0ca0e 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -1,7 +1,10 @@
+mod asr_realtime;
mod audio;
mod service;
use std::{
+ borrow::Cow,
+ collections::HashSet,
env, fs,
path::{Path, PathBuf},
sync::{
@@ -12,6 +15,9 @@
};
use anyhow::{Context, Result, anyhow};
+use asr_realtime::{
+ AudioIngressMetadata, RealtimeAsrConfig, RealtimeAsrOutcome, RealtimeAsrUpload,
+};
use audio::{AudioDiagnostics, load_pre_recorded_frames};
use base64::{Engine as _, engine::general_purpose};
use futures_util::StreamExt;
@@ -24,23 +30,299 @@
options::TrackPublishOptions,
prelude::{
DataPacket, LocalAudioTrack, LocalTrack, ParticipantIdentity, RemoteAudioTrack,
- RemoteTrack, Room, RoomEvent, RoomOptions,
+ RemoteParticipant, RemoteTrack, Room, RoomEvent, RoomOptions,
},
};
use reqwest::Client;
use serde::{Deserialize, Serialize};
use serde_json::json;
-use tokio::time::{sleep, sleep_until};
-use tokio::{sync::mpsc::UnboundedReceiver, task::JoinHandle};
+use sha2::{Digest, Sha256};
+use tokio::time::{sleep, sleep_until, timeout};
+use tokio::{
+ sync::{mpsc, mpsc::UnboundedReceiver, watch},
+ task::JoinHandle,
+};
use tracing::{info, warn};
const USER_AUDIO_SAMPLE_RATE_HZ: u32 = 48_000;
const USER_AUDIO_NUM_CHANNELS: u16 = 1;
+const INBOUND_AUDIO_QUEUE_CAPACITY: usize = 100;
+const INBOUND_AUDIO_DROP_LOG_INTERVAL: u64 = 100;
+const AUDIO_DRAIN_STOP_GRACE: Duration = Duration::from_secs(2);
const DEFAULT_BOT_AUDIO_PROFILE: &str = "pcm-16k";
const LIVEKIT_48K_SAMPLE_RATE_HZ: u32 = 48_000;
const PCM_16K_SAMPLE_RATE_HZ: u32 = 16_000;
const BOT_NUM_CHANNELS: u16 = 1;
const TRACK_NAME: &str = "bot-main-audio";
+const STREAM_TIMING_VERSION: u32 = 1;
+const STREAM_TIMING_FIRST_SEGMENT: u64 = 1;
+const STREAM_TIMING_FIRST_CHUNK: u64 = 1;
+const STREAM_TIMING_MAX_ELAPSED_MS: u64 = 5_000;
+const STREAM_TIMING_SOURCE: &str = "stream_anchor_monotonic";
+const CONTROLLED_FIXTURE_PROBE_TOPIC: &str = "controlled_fixture_attribute_probe";
+const CONTROLLED_FIXTURE_ACK_TOPIC: &str = "controlled_fixture_attribute_ack";
+const CONTROLLED_FIXTURE_PROTOCOL_VERSION: u64 = 1;
+const CONTROLLED_FIXTURE_GENERATION: u64 = 1;
+const CONTROLLED_FIXTURE_PROBE_RECHECK_DELAY: Duration = Duration::from_millis(25);
+const CONTROLLED_FIXTURE_PROBE_TTL: Duration = Duration::from_millis(250);
+
+const CONTROLLED_FIXTURE_ACK_RESULTS: [&str; 4] =
+ ["observed", "rejected", "timeout", "publish_failed"];
+const CONTROLLED_FIXTURE_REJECT_REASONS: [&str; 10] = [
+ "missing_attributes",
+ "wrong_source",
+ "missing_sequence",
+ "wrong_sequence",
+ "wrong_participant",
+ "expired",
+ "duplicate_or_old_sequence",
+ "no_current_participant",
+ "ack_publish_failed",
+ "unknown",
+];
+
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+struct ControlledFixtureAttributeProbe {
+ #[serde(rename = "type")]
+ message_type: String,
+ protocol_version: u64,
+ call_id_hash: String,
+ call_trace_id_hash: String,
+ generation: u64,
+ client_fixture_sequence: String,
+}
+
+#[derive(Debug, Serialize)]
+#[serde(rename_all = "camelCase")]
+struct ControlledFixtureAttributeAck {
+ #[serde(rename = "type")]
+ message_type: &'static str,
+ protocol_version: u64,
+ call_id_hash: String,
+ call_trace_id_hash: String,
+ generation: u64,
+ client_fixture_sequence: String,
+ result: &'static str,
+ #[serde(skip_serializing_if = "Option::is_none")]
+ input_source_category: Option<&'static str>,
+ #[serde(skip_serializing_if = "Option::is_none")]
+ reject_reason: Option<&'static str>,
+}
+
+#[derive(Debug)]
+struct PendingControlledFixtureProbe {
+ sender: ParticipantIdentity,
+ sequence: String,
+ expires_at: Instant,
+}
+
+fn sha256_hex(value: &str) -> String {
+ let mut digest = Sha256::new();
+ digest.update(value.as_bytes());
+ digest
+ .finalize()
+ .iter()
+ .map(|byte| format!("{byte:02x}"))
+ .collect()
+}
+
+fn controlled_fixture_ack_classification(
+ decision: Result<(), &'static str>,
+) -> (&'static str, Option<&'static str>, bool) {
+ match decision {
+ Ok(()) => ("observed", None, true),
+ Err("timeout") => ("timeout", Some("expired"), false),
+ Err(reason) if CONTROLLED_FIXTURE_REJECT_REASONS.contains(&reason) => {
+ ("rejected", Some(reason), false)
+ }
+ Err(_) => ("rejected", Some("unknown"), false),
+ }
+}
+
+fn record_controlled_fixture_probe_event(
+ stage: &'static str,
+ call_id: &str,
+ trace_id: &str,
+ sequence: &str,
+ observed: bool,
+ ack_result: Option<&'static str>,
+ reject_reason: Option<&'static str>,
+) {
+ debug_assert!(ack_result.is_none_or(|value| CONTROLLED_FIXTURE_ACK_RESULTS.contains(&value)));
+ debug_assert!(
+ reject_reason.is_none_or(|value| CONTROLLED_FIXTURE_REJECT_REASONS.contains(&value))
+ );
+ if let Some(ack_result) = ack_result {
+ info!(
+ event = "controlled_fixture_attribute_probe",
+ stage,
+ observed,
+ ack_result,
+ reject_reason = reject_reason.unwrap_or("unknown"),
+ call_id_hash = %sha256_hex(call_id),
+ trace_id_hash = %sha256_hex(trace_id),
+ generation = CONTROLLED_FIXTURE_GENERATION,
+ sequence_hash = %sha256_hex(sequence),
+ "runtime helper controlled fixture probe state"
+ );
+ } else if let Some(reject_reason) = reject_reason {
+ info!(
+ event = "controlled_fixture_attribute_probe",
+ stage,
+ observed,
+ reject_reason,
+ call_id_hash = %sha256_hex(call_id),
+ trace_id_hash = %sha256_hex(trace_id),
+ generation = CONTROLLED_FIXTURE_GENERATION,
+ sequence_hash = %sha256_hex(sequence),
+ "runtime helper controlled fixture probe state"
+ );
+ } else {
+ info!(
+ event = "controlled_fixture_attribute_probe",
+ stage,
+ observed,
+ call_id_hash = %sha256_hex(call_id),
+ trace_id_hash = %sha256_hex(trace_id),
+ generation = CONTROLLED_FIXTURE_GENERATION,
+ sequence_hash = %sha256_hex(sequence),
+ "runtime helper controlled fixture probe state"
+ );
+ }
+}
+
+fn record_controlled_fixture_attribute_decision(
+ decision: Result<(), &'static str>,
+ call_id: &str,
+ trace_id: &str,
+ sequence: &str,
+) -> (&'static str, Option<&'static str>, bool) {
+ let classification = controlled_fixture_ack_classification(decision);
+ record_controlled_fixture_probe_event(
+ "attributes_classified",
+ call_id,
+ trace_id,
+ sequence,
+ classification.2,
+ Some(classification.0),
+ classification.1,
+ );
+ classification
+}
+
+fn controlled_fixture_probe(
+ payload: &[u8],
+ call_id: &str,
+ trace_id: &str,
+ sender: &ParticipantIdentity,
+ expected_participant: Option<&str>,
+) -> Option<PendingControlledFixtureProbe> {
+ if !is_bound_user_participant(sender.as_str(), expected_participant) {
+ return None;
+ }
+ let probe: ControlledFixtureAttributeProbe = serde_json::from_slice(payload).ok()?;
+ if probe.message_type != CONTROLLED_FIXTURE_PROBE_TOPIC
+ || probe.protocol_version != CONTROLLED_FIXTURE_PROTOCOL_VERSION
+ || probe.generation != CONTROLLED_FIXTURE_GENERATION
+ || probe.call_id_hash != sha256_hex(call_id)
+ || probe.call_trace_id_hash != sha256_hex(trace_id)
+ || probe.client_fixture_sequence.trim().is_empty()
+ {
+ return None;
+ }
+ Some(PendingControlledFixtureProbe {
+ sender: sender.clone(),
+ sequence: probe.client_fixture_sequence,
+ expires_at: Instant::now() + CONTROLLED_FIXTURE_PROBE_TTL,
+ })
+}
+
+fn classify_controlled_fixture_attributes(
+ actual_participant: &str,
+ expected_participant: Option<&str>,
+ attributes: &std::collections::HashMap<String, String>,
+ requested_sequence: &str,
+) -> Result<(), &'static str> {
+ if !is_bound_user_participant(actual_participant, expected_participant) {
+ return Err("wrong_participant");
+ }
+ let source = attributes
+ .get("inputSourceCategory")
+ .map(String::as_str)
+ .unwrap_or("");
+ if source != "controlled_fixture" {
+ return Err(if source.is_empty() {
+ "missing_attributes"
+ } else {
+ "wrong_source"
+ });
+ }
+ let sequence = attributes
+ .get("clientFixtureSequence")
+ .map(String::as_str)
+ .unwrap_or("");
+ if sequence.trim().is_empty() {
+ return Err("missing_sequence");
+ }
+ if sequence != requested_sequence {
+ return Err("wrong_sequence");
+ }
+ Ok(())
+}
+
+async fn observe_controlled_fixture_attributes<F>(
+ expires_at: Instant,
+ actual_participant: &str,
+ expected_participant: Option<&str>,
+ requested_sequence: &str,
+ mut read_attributes: F,
+) -> Result<(), &'static str>
+where
+ F: FnMut() -> std::collections::HashMap<String, String>,
+{
+ loop {
+ if Instant::now() > expires_at {
+ return Err("timeout");
+ }
+ let decision = classify_controlled_fixture_attributes(
+ actual_participant,
+ expected_participant,
+ &read_attributes(),
+ requested_sequence,
+ );
+ if decision.is_ok() || !matches!(decision, Err("missing_attributes")) {
+ return decision;
+ }
+ let Some(next_check) = Instant::now().checked_add(CONTROLLED_FIXTURE_PROBE_RECHECK_DELAY)
+ else {
+ return Err("timeout");
+ };
+ if next_check > expires_at {
+ return Err("timeout");
+ }
+ sleep(CONTROLLED_FIXTURE_PROBE_RECHECK_DELAY).await;
+ }
+}
+
+fn controlled_fixture_observer_gate(pending_probe: bool, probe_result: Option<bool>) -> bool {
+ !pending_probe || probe_result == Some(true)
+}
+
+fn start_observer_after_controlled_fixture_probe<F>(
+ pending_probe: bool,
+ probe_result: Option<bool>,
+ spawn: F,
+) -> bool
+where
+ F: FnOnce(),
+{
+ if !controlled_fixture_observer_gate(pending_probe, probe_result) {
+ return false;
+ }
+ spawn();
+ true
+}
#[tokio::main(flavor = "multi_thread")]
async fn main() -> Result<()> {
@@ -270,6 +552,11 @@
runtime_turn_bridge_url: Option<String>,
runtime_turn_bridge_token: Option<String>,
runtime_turn_bridge_mode: String,
+ runtime_asr_stream_enabled: bool,
+ runtime_asr_stream_url: Option<String>,
+ runtime_asr_realtime_enabled: bool,
+ runtime_asr_realtime_url: Option<String>,
+ runtime_asr_realtime_chunk_duration_ms: u64,
runtime_turn_artifact_dir: Option<String>,
runtime_session_nonce: Option<String>,
user_audio_observer_enabled: bool,
@@ -332,6 +619,11 @@
bridge_url: Option<String>,
bridge_token: Option<String>,
bridge_mode: String,
+ asr_stream_enabled: bool,
+ asr_stream_url: Option<String>,
+ asr_realtime_enabled: bool,
+ asr_realtime_url: Option<String>,
+ asr_realtime_chunk_duration_ms: u64,
artifact_dir: Option<String>,
runtime_session_nonce: Option<String>,
audio_debug_dump_dir: Option<String>,
@@ -343,6 +635,11 @@
bridge_url: config.runtime_turn_bridge_url.clone(),
bridge_token: config.runtime_turn_bridge_token.clone(),
bridge_mode: config.runtime_turn_bridge_mode.clone(),
+ asr_stream_enabled: config.runtime_asr_stream_enabled,
+ asr_stream_url: config.runtime_asr_stream_url.clone(),
+ asr_realtime_enabled: config.runtime_asr_realtime_enabled,
+ asr_realtime_url: config.runtime_asr_realtime_url.clone(),
+ asr_realtime_chunk_duration_ms: config.runtime_asr_realtime_chunk_duration_ms,
artifact_dir: config.runtime_turn_artifact_dir.clone(),
runtime_session_nonce: config.runtime_session_nonce.clone(),
audio_debug_dump_dir: config.audio_debug_dump_dir.clone(),
@@ -374,6 +671,32 @@
.as_deref()
.is_some_and(|value| value.trim_end_matches('/').ends_with("/stream"))
}
+
+ fn is_asr_stream_ready(&self) -> bool {
+ self.asr_stream_enabled
+ && self
+ .asr_stream_url
+ .as_ref()
+ .is_some_and(|value| !value.is_empty())
+ && self
+ .bridge_token
+ .as_ref()
+ .is_some_and(|value| !value.is_empty())
+ && self
+ .runtime_session_nonce
+ .as_ref()
+ .is_some_and(|value| !value.is_empty())
+ }
+
+ fn realtime_asr_config(&self) -> RealtimeAsrConfig {
+ RealtimeAsrConfig {
+ enabled: self.asr_realtime_enabled,
+ url: self.asr_realtime_url.clone(),
+ runtime_token: self.bridge_token.clone(),
+ runtime_session_nonce: self.runtime_session_nonce.clone(),
+ chunk_duration_ms: self.asr_realtime_chunk_duration_ms,
+ }
+ }
}
impl Config {
@@ -395,6 +718,14 @@
runtime_turn_bridge_token: optional_env("CV_RUNTIME_TURN_BRIDGE_TOKEN"),
runtime_turn_bridge_mode: env::var("CV_RUNTIME_TURN_BRIDGE_MODE")
.unwrap_or_else(|_| "json".to_string()),
+ runtime_asr_stream_enabled: bool_env("CV_RUNTIME_ASR_STREAM_ENABLED", false),
+ runtime_asr_stream_url: optional_env("CV_RUNTIME_ASR_STREAM_URL"),
+ runtime_asr_realtime_enabled: bool_env("CV_RUNTIME_ASR_REALTIME_ENABLED", false),
+ runtime_asr_realtime_url: optional_env("CV_RUNTIME_ASR_REALTIME_URL"),
+ runtime_asr_realtime_chunk_duration_ms: u64_env(
+ "CV_RUNTIME_ASR_REALTIME_CHUNK_DURATION_MS",
+ 200,
+ ),
runtime_turn_artifact_dir: optional_env("CV_RUNTIME_TURN_ARTIFACT_DIR"),
runtime_session_nonce: optional_env("CV_RUNTIME_SESSION_NONCE"),
user_audio_observer_enabled: bool_env("CV_ENABLE_USER_AUDIO_OBSERVER", true),
@@ -605,6 +936,31 @@
println!("{payload}");
}
+fn emit_anchored_activity(
+ call_id: &str,
+ trace_id: &str,
+ turn_id: &str,
+ event_name: &str,
+ marker: &RuntimeTurnStreamTimingMarker,
+ extension: serde_json::Value,
+) {
+ let payload = json!({
+ "type": "cv_activity",
+ "callId": call_id,
+ "traceId": trace_id,
+ "turnId": turn_id,
+ "eventName": event_name,
+ "eventWallTimeMs": current_time_millis(),
+ "serverDeltaMs": marker.server_delta_ms,
+ "serverDeltaSource": STREAM_TIMING_SOURCE,
+ "result": "ok",
+ "reasonCode": null,
+ "retryable": null,
+ "extension": marker.extension_with(extension),
+ });
+ println!("{payload}");
+}
+
fn spawn_user_audio_observer(
events: UnboundedReceiver<RoomEvent>,
config: &Config,
@@ -617,6 +973,7 @@
let enabled = config.user_audio_observer_enabled;
let simple_vad_enabled = config.simple_vad_enabled;
let simple_vad_config = config.simple_vad_config.clone();
+ let user_participant_identity = config.user_participant_identity.clone();
let turn_bridge_config = TurnBridgeConfig::from_config(config);
tokio::spawn(async move {
@@ -638,9 +995,14 @@
turn_bridge_config,
http,
sink,
+ user_participant_identity,
)
.await;
})
+}
+
+fn is_bound_user_participant(identity: &str, expected: Option<&str>) -> bool {
+ expected.is_none_or(|value| identity == value)
}
async fn observe_user_audio_events(
@@ -653,6 +1015,7 @@
turn_bridge_config: TurnBridgeConfig,
http: Client,
sink: Arc<BotAudioOutputSink>,
+ user_participant_identity: Option<String>,
) {
info!(
call_id = %call_id,
@@ -669,6 +1032,10 @@
"runtime helper user_track_subscribe_requested"
);
+ let mut current_user_participant: Option<RemoteParticipant> = None;
+ let mut pending_probe: Option<PendingControlledFixtureProbe> = None;
+ let mut acknowledged_probe_sequences = HashSet::new();
+
while let Some(event) = events.recv().await {
match event {
RoomEvent::TrackSubscribed {
@@ -676,6 +1043,15 @@
publication: _,
participant,
} => {
+ if !is_bound_user_participant(
+ &participant.identity().to_string(),
+ user_participant_identity.as_deref(),
+ ) {
+ warn!(call_id = %call_id, trace_id = %trace_id,
+ metadata_status = "wrong_participant",
+ "runtime helper ignored non-user audio participant");
+ continue;
+ }
let participant_alias = redact(&participant.identity().to_string());
let track_sid_alias = redact(&track.sid().to_string());
let track_name = track.name();
@@ -689,19 +1065,119 @@
track_source = %track_source,
"runtime helper user_track_subscribed"
);
- spawn_user_audio_frame_observer(
- track,
- call_id.clone(),
- trace_id.clone(),
- participant_alias,
- track_sid_alias,
- simple_vad_enabled,
- simple_vad_config.clone(),
- vad_enabled_gate.clone(),
- turn_bridge_config.clone(),
- http.clone(),
- sink.clone(),
+ let pending_sequence = pending_probe.as_ref().map(|probe| probe.sequence.clone());
+ let participant_for_probe = participant.clone();
+ let probe_result = process_controlled_fixture_probe(
+ &mut pending_probe,
+ &mut acknowledged_probe_sequences,
+ Some(&participant_for_probe),
+ &sink,
+ &call_id,
+ &trace_id,
+ user_participant_identity.as_deref(),
+ )
+ .await;
+ let observer_started = start_observer_after_controlled_fixture_probe(
+ pending_sequence.is_some(),
+ probe_result,
+ || {
+ spawn_user_audio_frame_observer(
+ track,
+ call_id.clone(),
+ trace_id.clone(),
+ participant_alias,
+ track_sid_alias,
+ simple_vad_enabled,
+ simple_vad_config.clone(),
+ vad_enabled_gate.clone(),
+ turn_bridge_config.clone(),
+ http.clone(),
+ sink.clone(),
+ user_participant_identity.clone(),
+ participant,
+ );
+ },
);
+ if let Some(sequence) = pending_sequence.as_deref() {
+ let (ack_result, reject_reason, observed) = match probe_result {
+ Some(true) => (Some("observed"), None, true),
+ Some(false) => (Some("rejected"), Some("unknown"), false),
+ None => (None, Some("no_current_participant"), false),
+ };
+ record_controlled_fixture_probe_event(
+ if observer_started {
+ "audio_observer_allowed"
+ } else {
+ "audio_observer_blocked"
+ },
+ &call_id,
+ &trace_id,
+ sequence,
+ observed,
+ ack_result,
+ reject_reason,
+ );
+ }
+ if !observer_started {
+ warn!(
+ call_id = %call_id,
+ trace_id = %trace_id,
+ "runtime helper withheld audio observer until controlled fixture ACK"
+ );
+ continue;
+ }
+ current_user_participant = Some(participant_for_probe);
+ }
+ RoomEvent::DataReceived {
+ payload,
+ topic: Some(topic),
+ participant: Some(sender),
+ ..
+ } if topic == CONTROLLED_FIXTURE_PROBE_TOPIC => {
+ if let Ok(probe) =
+ serde_json::from_slice::<ControlledFixtureAttributeProbe>(&payload)
+ {
+ if acknowledged_probe_sequences.contains(&probe.client_fixture_sequence) {
+ record_controlled_fixture_probe_event(
+ "data_received",
+ &call_id,
+ &trace_id,
+ &probe.client_fixture_sequence,
+ false,
+ Some("rejected"),
+ Some("duplicate_or_old_sequence"),
+ );
+ continue;
+ }
+ }
+ pending_probe = controlled_fixture_probe(
+ &payload,
+ &call_id,
+ &trace_id,
+ &sender.identity(),
+ user_participant_identity.as_deref(),
+ );
+ if let Some(probe) = pending_probe.as_ref() {
+ record_controlled_fixture_probe_event(
+ "data_received",
+ &call_id,
+ &trace_id,
+ &probe.sequence,
+ false,
+ None,
+ None,
+ );
+ }
+ process_controlled_fixture_probe(
+ &mut pending_probe,
+ &mut acknowledged_probe_sequences,
+ current_user_participant.as_ref(),
+ &sink,
+ &call_id,
+ &trace_id,
+ user_participant_identity.as_deref(),
+ )
+ .await;
}
RoomEvent::TrackSubscribed {
track: RemoteTrack::Video(track),
@@ -744,6 +1220,136 @@
}
}
+async fn process_controlled_fixture_probe(
+ pending_probe: &mut Option<PendingControlledFixtureProbe>,
+ acknowledged_probe_sequences: &mut HashSet<String>,
+ participant: Option<&RemoteParticipant>,
+ sink: &BotAudioOutputSink,
+ call_id: &str,
+ trace_id: &str,
+ expected_participant: Option<&str>,
+) -> Option<bool> {
+ let Some(probe) = pending_probe.take() else {
+ return None;
+ };
+ if acknowledged_probe_sequences.contains(&probe.sequence) {
+ record_controlled_fixture_probe_event(
+ "attributes_classified",
+ call_id,
+ trace_id,
+ &probe.sequence,
+ false,
+ Some("rejected"),
+ Some("duplicate_or_old_sequence"),
+ );
+ return Some(false);
+ }
+ if Instant::now() > probe.expires_at {
+ record_controlled_fixture_probe_event(
+ "attributes_classified",
+ call_id,
+ trace_id,
+ &probe.sequence,
+ false,
+ Some("timeout"),
+ Some("expired"),
+ );
+ return Some(false);
+ }
+ let Some(participant) = participant else {
+ record_controlled_fixture_probe_event(
+ "attributes_classified",
+ call_id,
+ trace_id,
+ &probe.sequence,
+ false,
+ None,
+ Some("no_current_participant"),
+ );
+ *pending_probe = Some(probe);
+ return None;
+ };
+ if participant.identity() != probe.sender {
+ record_controlled_fixture_probe_event(
+ "attributes_classified",
+ call_id,
+ trace_id,
+ &probe.sequence,
+ false,
+ Some("rejected"),
+ Some("wrong_participant"),
+ );
+ return Some(false);
+ }
+ let decision = observe_controlled_fixture_attributes(
+ probe.expires_at,
+ &participant.identity().to_string(),
+ expected_participant,
+ &probe.sequence,
+ || participant.attributes(),
+ )
+ .await;
+ let protocol_reject_reason = decision.err();
+ let (ack_result, reject_reason, observed) =
+ record_controlled_fixture_attribute_decision(decision, call_id, trace_id, &probe.sequence);
+ let result = if observed { "observed" } else { "rejected" };
+ let input_source_category = observed.then_some("controlled_fixture");
+ let ack = ControlledFixtureAttributeAck {
+ message_type: CONTROLLED_FIXTURE_ACK_TOPIC,
+ protocol_version: CONTROLLED_FIXTURE_PROTOCOL_VERSION,
+ call_id_hash: sha256_hex(call_id),
+ call_trace_id_hash: sha256_hex(trace_id),
+ generation: CONTROLLED_FIXTURE_GENERATION,
+ client_fixture_sequence: probe.sequence.clone(),
+ result,
+ input_source_category,
+ reject_reason: protocol_reject_reason,
+ };
+ let payload = match serde_json::to_vec(&ack) {
+ Ok(payload) => payload,
+ Err(_) => return Some(false),
+ };
+ let local_participant = sink.room.local_participant();
+ record_controlled_fixture_probe_event(
+ "ack_publish_started",
+ call_id,
+ trace_id,
+ &probe.sequence,
+ observed,
+ Some(ack_result),
+ reject_reason,
+ );
+ let publish = local_participant.publish_data(DataPacket {
+ payload,
+ topic: Some(CONTROLLED_FIXTURE_ACK_TOPIC.to_string()),
+ reliable: true,
+ destination_identities: vec![probe.sender],
+ });
+ if publish.await.is_ok() {
+ record_controlled_fixture_probe_event(
+ "ack_publish_completed",
+ call_id,
+ trace_id,
+ &probe.sequence,
+ observed,
+ Some(ack_result),
+ reject_reason,
+ );
+ acknowledged_probe_sequences.insert(probe.sequence);
+ } else {
+ record_controlled_fixture_probe_event(
+ "ack_publish_completed",
+ call_id,
+ trace_id,
+ &probe.sequence,
+ false,
+ Some("publish_failed"),
+ Some("ack_publish_failed"),
+ );
+ }
+ Some(result == "observed")
+}
+
async fn handle_finished_turn(
http: &Client,
bridge_config: &TurnBridgeConfig,
@@ -751,6 +1357,7 @@
call_id: &str,
trace_id: &str,
turn: FinishedSpeechTurn,
+ realtime_asr_result_ref: Option<String>,
) {
let turn_pipeline_started_at = Instant::now();
if !bridge_config.is_ready() {
@@ -807,9 +1414,13 @@
"turnArtifactBytes": byte_size,
"frameCount": turn.frame_count,
"sampleCount": turn.sample_count,
- "endReason": turn.end_reason,
+ "endReason": turn.end_reason.as_str(),
}),
);
+ let asr_result_ref = match realtime_asr_result_ref {
+ Some(value) => Some(value),
+ None => request_asr_result_ref(http, bridge_config, call_id, trace_id, &turn).await,
+ };
if bridge_config.is_stream_mode() {
match request_turn_bridge_stream(
http,
@@ -820,6 +1431,7 @@
&turn,
&path_ref,
byte_size,
+ asr_result_ref.as_deref(),
turn_pipeline_started_at,
)
.await
@@ -883,12 +1495,17 @@
&turn,
&path_ref,
byte_size,
+ asr_result_ref.as_deref(),
turn_pipeline_started_at,
)
.await
{
Ok(outcome) => {
+ let mut published_device_outputs = HashSet::new();
for output in &outcome.device_outputs {
+ if !should_publish_device_output(&mut published_device_outputs, output) {
+ continue;
+ }
if let Err(error) = sink
.publish_device_output(call_id, trace_id, &turn.turn_id, output)
.await
@@ -1004,6 +1621,251 @@
Ok((path_ref, byte_size))
}
+async fn request_asr_result_ref(
+ http: &Client,
+ bridge_config: &TurnBridgeConfig,
+ call_id: &str,
+ trace_id: &str,
+ turn: &FinishedSpeechTurn,
+) -> Option<String> {
+ if !bridge_config.is_asr_stream_ready() {
+ return None;
+ }
+ match request_asr_stream(http, bridge_config, call_id, trace_id, turn).await {
+ Ok(Some(asr_result_ref)) => {
+ emit_activity(
+ call_id,
+ trace_id,
+ Some(&turn.turn_id),
+ "asr_stream_ref_ready",
+ "ok",
+ None,
+ None,
+ json!({
+ "asrResultRefPresent": true,
+ "format": "pcm_s16le",
+ "sampleRate": 16000,
+ "channels": 1,
+ }),
+ );
+ Some(asr_result_ref)
+ }
+ Ok(None) => None,
+ Err(error) => {
+ warn!(
+ call_id = %call_id,
+ trace_id = %trace_id,
+ turn_id = %turn.turn_id,
+ error = %safe_error(&error.to_string()),
+ "runtime helper asr_stream_failed_fallback"
+ );
+ emit_activity(
+ call_id,
+ trace_id,
+ Some(&turn.turn_id),
+ "asr_stream_fallback",
+ "skipped",
+ Some("ASR_STREAM_INTERRUPTED"),
+ Some(true),
+ json!({
+ "fallbackReason": "asr_stream_request_failed",
+ "fallbackStage": "asr_stream",
+ }),
+ );
+ None
+ }
+ }
+}
+
+async fn request_asr_stream(
+ http: &Client,
+ bridge_config: &TurnBridgeConfig,
+ call_id: &str,
+ trace_id: &str,
+ turn: &FinishedSpeechTurn,
+) -> Result<Option<String>> {
+ let started_at = Instant::now();
+ let ndjson = build_asr_stream_ndjson(call_id, trace_id, turn, bridge_config)?;
+ let response = http
+ .post(bridge_config.asr_stream_url.as_deref().unwrap_or_default())
+ .header("Content-Type", "application/x-ndjson")
+ .header(
+ "X-CV-Runtime-Token",
+ bridge_config.bridge_token.as_deref().unwrap_or_default(),
+ )
+ .header("X-CV-Call-Id", call_id)
+ .header("X-CV-Trace-Id", trace_id)
+ .header(
+ "X-CV-Runtime-Session-Nonce",
+ bridge_config
+ .runtime_session_nonce
+ .as_deref()
+ .unwrap_or_default(),
+ )
+ .body(ndjson)
+ .send()
+ .await
+ .context("failed to post asr stream")?;
+ let status = response.status();
+ if !status.is_success() {
+ let body_len = response.text().await.map(|body| body.len()).unwrap_or(0);
+ warn!(
+ call_id = %call_id,
+ trace_id = %trace_id,
+ turn_id = %turn.turn_id,
+ http_status = status.as_u16(),
+ body_len,
+ "runtime helper asr_stream_http_failed"
+ );
+ return Ok(None);
+ }
+ let body: RuntimeTurnCommonResult<RuntimeAsrStreamResp> = response
+ .json()
+ .await
+ .context("failed to decode asr stream response")?;
+ if body.code != 0 {
+ warn!(
+ call_id = %call_id,
+ trace_id = %trace_id,
+ turn_id = %turn.turn_id,
+ code = body.code,
+ msg_len = body.msg.as_deref().unwrap_or_default().len(),
+ "runtime helper asr_stream_common_result_failed"
+ );
+ return Ok(None);
+ }
+ let Some(data) = body.data else {
+ return Ok(None);
+ };
+ if data.status.as_deref() == Some("final") {
+ info!(
+ call_id = %call_id,
+ trace_id = %trace_id,
+ turn_id = %turn.turn_id,
+ chunk_count = data.chunk_count.unwrap_or_default(),
+ audio_bytes = data.audio_bytes.unwrap_or_default(),
+ asr_duration_ms = data.asr_duration_ms.unwrap_or_default(),
+ wall_ms = started_at.elapsed().as_millis() as u64,
+ provider = %data.provider_alias.as_deref().unwrap_or("unknown"),
+ text_len = data.text_len.unwrap_or_default(),
+ "runtime helper asr_stream_final"
+ );
+ return Ok(data.asr_result_ref);
+ }
+ emit_activity(
+ call_id,
+ trace_id,
+ Some(&turn.turn_id),
+ "asr_stream_fallback",
+ "skipped",
+ None,
+ Some(true),
+ json!({
+ "fallbackReason": data.fallback_reason,
+ "fallbackStage": data.fallback_stage,
+ "status": data.status,
+ }),
+ );
+ Ok(None)
+}
+
+fn build_asr_stream_ndjson(
+ call_id: &str,
+ trace_id: &str,
+ turn: &FinishedSpeechTurn,
+ bridge_config: &TurnBridgeConfig,
+) -> Result<String> {
+ let chunks = asr_pcm_16k_chunks(turn)?;
+ let mut seq = 1u64;
+ let mut lines = Vec::with_capacity(chunks.len() + 3);
+ lines.push(serde_json::to_string(&json!({
+ "event": "asr_stream_started",
+ "seq": seq,
+ "callId": call_id,
+ "traceId": trace_id,
+ "turnId": turn.turn_id.as_str(),
+ "tsMs": current_time_millis(),
+ "payload": {
+ "format": "pcm_s16le",
+ "sampleRate": 16000,
+ "channels": 1,
+ "runtimeSessionNonce": bridge_config.runtime_session_nonce.as_deref().unwrap_or_default(),
+ "providerHint": "volcengine",
+ }
+ }))?);
+ for (index, samples) in chunks.iter().enumerate() {
+ seq += 1;
+ let bytes = pcm_i16_to_le_bytes(samples);
+ let duration_ms = ((samples.len() as u64) * 1000 / 16_000).max(1);
+ lines.push(serde_json::to_string(&json!({
+ "event": "asr_audio_chunk",
+ "seq": seq,
+ "callId": call_id,
+ "traceId": trace_id,
+ "turnId": turn.turn_id.as_str(),
+ "tsMs": current_time_millis(),
+ "payload": {
+ "chunkSeq": index + 1,
+ "format": "pcm_s16le",
+ "sampleRate": 16000,
+ "channels": 1,
+ "durationMs": duration_ms,
+ "payloadBase64": general_purpose::STANDARD.encode(bytes),
+ }
+ }))?);
+ }
+ seq += 1;
+ lines.push(serde_json::to_string(&json!({
+ "event": "vad_speech_end",
+ "seq": seq,
+ "callId": call_id,
+ "traceId": trace_id,
+ "turnId": turn.turn_id.as_str(),
+ "tsMs": current_time_millis(),
+ "payload": {
+ "endReason": turn.end_reason.as_str(),
+ "speechDurationMs": turn.duration_ms,
+ }
+ }))?);
+ seq += 1;
+ lines.push(serde_json::to_string(&json!({
+ "event": "asr_stream_finish",
+ "seq": seq,
+ "callId": call_id,
+ "traceId": trace_id,
+ "turnId": turn.turn_id.as_str(),
+ "tsMs": current_time_millis(),
+ "payload": {
+ "finalChunkSeq": chunks.len(),
+ "audioDurationMs": turn.duration_ms,
+ }
+ }))?);
+ Ok(lines.join("\n") + "\n")
+}
+
+fn asr_pcm_16k_chunks(turn: &FinishedSpeechTurn) -> Result<Vec<Vec<i16>>> {
+ if turn.samples.is_empty() {
+ return Err(anyhow!("empty turn samples"));
+ }
+ let samples_16k: Vec<i16> = turn.samples.iter().step_by(3).copied().collect();
+ if samples_16k.is_empty() {
+ return Err(anyhow!("empty 16k asr samples"));
+ }
+ let samples_per_chunk = 320usize;
+ Ok(samples_16k
+ .chunks(samples_per_chunk)
+ .map(|chunk| chunk.to_vec())
+ .collect())
+}
+
+fn pcm_i16_to_le_bytes(samples: &[i16]) -> Vec<u8> {
+ let mut bytes = Vec::with_capacity(samples.len() * 2);
+ for sample in samples {
+ bytes.extend_from_slice(&sample.to_le_bytes());
+ }
+ bytes
+}
+
async fn request_turn_bridge_stream(
http: &Client,
bridge_config: &TurnBridgeConfig,
@@ -1013,6 +1875,7 @@
turn: &FinishedSpeechTurn,
path_ref: &str,
byte_size: u64,
+ asr_result_ref: Option<&str>,
turn_pipeline_started_at: Instant,
) -> Result<RuntimeTurnStreamOutcome> {
let bridge_started_at = Instant::now();
@@ -1020,7 +1883,7 @@
call_id: call_id.to_string(),
trace_id: trace_id.to_string(),
turn_id: turn.turn_id.clone(),
- audio_artifact: RuntimeTurnAudioArtifact {
+ audio_artifact: Some(RuntimeTurnAudioArtifact {
artifact_type: "local_file".to_string(),
path_ref: path_ref.to_string(),
format: "wav".to_string(),
@@ -1028,7 +1891,8 @@
channels: u32::from(USER_AUDIO_NUM_CHANNELS),
duration_ms: turn.duration_ms,
byte_size,
- },
+ }),
+ asr_result_ref: asr_result_ref.map(str::to_string),
};
let response = http
.post(bridge_config.bridge_url.as_deref().unwrap_or_default())
@@ -1107,6 +1971,7 @@
.await?;
}
}
+ state.close_timing();
if !state.completed {
warn!(
call_id = %call_id,
@@ -1236,6 +2101,7 @@
call_id,
trace_id,
turn,
+ &event,
audio_chunk,
state,
turn_pipeline_started_at,
@@ -1247,6 +2113,9 @@
}
Some("device_output") => {
if let Some(output) = event.device_output.as_ref() {
+ if !should_publish_device_output(&mut state.published_device_output_ids, output) {
+ return Ok(());
+ }
sink.publish_device_output(call_id, trace_id, &turn.turn_id, output)
.await?;
state.device_output_count = state.device_output_count.saturating_add(1);
@@ -1254,6 +2123,7 @@
}
Some("turn_completed") => {
state.completed = true;
+ state.close_timing();
info!(
call_id = %call_id,
trace_id = %trace_id,
@@ -1264,6 +2134,7 @@
);
}
Some("turn_failed") => {
+ state.close_timing();
let error = event.error.as_ref();
let reason_code = error
.and_then(|value| value.reason_code.as_deref())
@@ -1298,6 +2169,7 @@
}
Some("turn_cancelled") => {
state.completed = true;
+ state.close_timing();
info!(
call_id = %call_id,
trace_id = %trace_id,
@@ -1330,6 +2202,19 @@
}
Some("activity") => {
if let Some(activity) = event.activity.as_ref() {
+ if activity.event_type.as_deref() == Some("tts_first_audio_chunk_ready") {
+ if let Some(runtime_session_nonce) =
+ bridge_config.runtime_session_nonce.as_deref()
+ {
+ let _ = state.arm_timing_anchor(
+ call_id,
+ trace_id,
+ &turn.turn_id,
+ runtime_session_nonce,
+ &event,
+ );
+ }
+ }
info!(
call_id = %call_id,
trace_id = %trace_id,
@@ -1388,6 +2273,7 @@
call_id: &str,
trace_id: &str,
turn: &FinishedSpeechTurn,
+ event: &RuntimeTurnStreamEvent,
audio_chunk: &RuntimeTurnStreamAudioChunk,
state: &mut RuntimeTurnStreamState,
turn_pipeline_started_at: Instant,
@@ -1407,6 +2293,58 @@
.unwrap_or("pcm_s16le")
.trim()
.to_ascii_lowercase();
+ if !matches!(format.as_str(), "pcm_s16le" | "mp3" | "mpeg" | "wav") {
+ return Err(anyhow!("unsupported reply_audio_chunk format {format}"));
+ }
+ match state.reply_chunk_markers.observe(audio_chunk.segment_seq) {
+ ReplyChunkMarker::FirstReply => {
+ let extension = json!({
+ "segmentSeq": audio_chunk.segment_seq,
+ "chunkSeq": audio_chunk.chunk_seq,
+ "format": format.as_str(),
+ "bytes": payload.len(),
+ });
+ if let Some(marker) =
+ state.record_m6(call_id, trace_id, &turn.turn_id, event, audio_chunk)
+ {
+ emit_anchored_activity(
+ call_id,
+ trace_id,
+ &turn.turn_id,
+ "helper_first_reply_audio_chunk_received",
+ &marker,
+ extension,
+ );
+ } else {
+ emit_activity(
+ call_id,
+ trace_id,
+ Some(&turn.turn_id),
+ "helper_first_reply_audio_chunk_received",
+ "ok",
+ None,
+ None,
+ extension,
+ );
+ }
+ }
+ ReplyChunkMarker::SegmentFirst => emit_activity(
+ call_id,
+ trace_id,
+ Some(&turn.turn_id),
+ "helper_segment_first_audio_chunk_received",
+ "ok",
+ None,
+ None,
+ json!({
+ "segmentSeq": audio_chunk.segment_seq,
+ "chunkSeq": audio_chunk.chunk_seq,
+ "format": format.as_str(),
+ "bytes": payload.len(),
+ }),
+ ),
+ ReplyChunkMarker::None => {}
+ }
let frames = if format == "pcm_s16le" {
let sample_rate = audio_chunk.sample_rate.unwrap_or(sink.sample_rate_hz);
let channels = audio_chunk.channels.unwrap_or(u32::from(sink.num_channels));
@@ -1498,7 +2436,7 @@
Err(error) => return Err(error).context("failed to decode final stream audio chunk"),
}
} else {
- return Err(anyhow!("unsupported reply_audio_chunk format {format}"));
+ unreachable!("supported encoded format checked above")
};
if frames.is_empty() {
return Ok(0);
@@ -1526,21 +2464,33 @@
sink.write_pcm_frame(frame).await?;
if !state.first_audio_frame_written {
state.first_audio_frame_written = true;
- emit_activity(
- call_id,
- trace_id,
- Some(&turn.turn_id),
- "bot_reply_first_audio_frame_written",
- "ok",
- None,
- None,
- json!({
- "replyPlaybackMode": state.reply_playback_mode.as_str(),
- "format": format.as_str(),
- "chunkSeq": audio_chunk.chunk_seq,
- "replyTotalAfterVadEndMs": turn_pipeline_started_at.elapsed().as_millis() as u64,
- }),
- );
+ let extension = json!({
+ "replyPlaybackMode": state.reply_playback_mode.as_str(),
+ "format": format.as_str(),
+ "chunkSeq": audio_chunk.chunk_seq,
+ "replyTotalAfterVadEndMs": turn_pipeline_started_at.elapsed().as_millis() as u64,
+ });
+ if let Some(marker) = state.record_m7() {
+ emit_anchored_activity(
+ call_id,
+ trace_id,
+ &turn.turn_id,
+ "bot_reply_first_audio_frame_written",
+ &marker,
+ extension,
+ );
+ } else {
+ emit_activity(
+ call_id,
+ trace_id,
+ Some(&turn.turn_id),
+ "bot_reply_first_audio_frame_written",
+ "ok",
+ None,
+ None,
+ extension,
+ );
+ }
}
sleep_until(pacing_started_at + Duration::from_millis(((index + 1) as u64) * 20)).await;
}
@@ -1621,6 +2571,7 @@
turn: &FinishedSpeechTurn,
path_ref: &str,
byte_size: u64,
+ asr_result_ref: Option<&str>,
turn_pipeline_started_at: Instant,
) -> Result<RuntimeTurnBridgeOutcome> {
let bridge_started_at = Instant::now();
@@ -1628,7 +2579,7 @@
call_id: call_id.to_string(),
trace_id: trace_id.to_string(),
turn_id: turn.turn_id.clone(),
- audio_artifact: RuntimeTurnAudioArtifact {
+ audio_artifact: Some(RuntimeTurnAudioArtifact {
artifact_type: "local_file".to_string(),
path_ref: path_ref.to_string(),
format: "wav".to_string(),
@@ -1636,7 +2587,8 @@
channels: u32::from(USER_AUDIO_NUM_CHANNELS),
duration_ms: turn.duration_ms,
byte_size,
- },
+ }),
+ asr_result_ref: asr_result_ref.map(str::to_string),
};
let response = http
.post(bridge_config.bridge_url.as_deref().unwrap_or_default())
@@ -1927,7 +2879,10 @@
#[serde(rename = "turnId")]
turn_id: String,
#[serde(rename = "audioArtifact")]
- audio_artifact: RuntimeTurnAudioArtifact,
+ #[serde(skip_serializing_if = "Option::is_none")]
+ audio_artifact: Option<RuntimeTurnAudioArtifact>,
+ #[serde(rename = "asrResultRef", skip_serializing_if = "Option::is_none")]
+ asr_result_ref: Option<String>,
}
#[derive(Serialize)]
@@ -1955,6 +2910,20 @@
retryable: Option<bool>,
}
+#[derive(Deserialize)]
+#[serde(rename_all = "camelCase")]
+struct RuntimeAsrStreamResp {
+ status: Option<String>,
+ asr_result_ref: Option<String>,
+ chunk_count: Option<u64>,
+ audio_bytes: Option<u64>,
+ asr_duration_ms: Option<u64>,
+ provider_alias: Option<String>,
+ text_len: Option<u64>,
+ fallback_reason: Option<String>,
+ fallback_stage: Option<String>,
+}
+
#[derive(Default)]
struct RuntimeTurnBridgeOutcome {
reply_audio_artifact: Option<RuntimeTurnReplyAudioArtifact>,
@@ -1975,9 +2944,12 @@
completed: bool,
audio_chunk_count: u64,
device_output_count: u64,
+ published_device_output_ids: HashSet<String>,
encoded_audio_buffer: Vec<u8>,
pcm_stream_decoder: Option<audio::PcmS16leStreamDecoder>,
pcm_stream_network_chunk_count: u64,
+ reply_chunk_markers: ReplyChunkMarkerState,
+ timing: RuntimeTurnStreamTimingState,
}
impl Default for RuntimeTurnStreamState {
@@ -1990,10 +2962,266 @@
completed: false,
audio_chunk_count: 0,
device_output_count: 0,
+ published_device_output_ids: HashSet::new(),
encoded_audio_buffer: Vec::new(),
pcm_stream_decoder: None,
pcm_stream_network_chunk_count: 0,
+ reply_chunk_markers: ReplyChunkMarkerState::default(),
+ timing: RuntimeTurnStreamTimingState::default(),
}
+ }
+}
+
+impl RuntimeTurnStreamState {
+ fn arm_timing_anchor(
+ &mut self,
+ call_id: &str,
+ trace_id: &str,
+ turn_id: &str,
+ runtime_session_nonce: &str,
+ event: &RuntimeTurnStreamEvent,
+ ) -> bool {
+ if self.timing.phase != RuntimeTurnStreamTimingPhase::Empty
+ || event.call_id.as_deref() != Some(call_id)
+ || event.trace_id.as_deref() != Some(trace_id)
+ || event.turn_id.as_deref() != Some(turn_id)
+ {
+ return false;
+ }
+ let Some(activity) = event.activity.as_ref() else {
+ return false;
+ };
+ if activity.event_type.as_deref() != Some("tts_first_audio_chunk_ready") {
+ return false;
+ }
+ let Some(extension) = activity.extension.as_ref() else {
+ return false;
+ };
+ let Some(anchor_id) = extension.stream_anchor_id.as_deref() else {
+ return false;
+ };
+ let valid_anchor_id = (16..=64).contains(&anchor_id.len()) && anchor_id.is_ascii();
+ let expected_nonce_hash = runtime_session_nonce_hash(runtime_session_nonce);
+ if extension.stream_timing_version != Some(STREAM_TIMING_VERSION)
+ || !valid_anchor_id
+ || extension.runtime_session_nonce_hash.as_deref() != Some(expected_nonce_hash.as_str())
+ || extension.segment_seq != Some(STREAM_TIMING_FIRST_SEGMENT)
+ || extension.stream_timing_validation.as_deref() != Some("bound")
+ {
+ return false;
+ }
+ let Some(server_delta_ms) = extension.stream_anchor_server_delta_ms else {
+ return false;
+ };
+ self.timing.anchor = Some(RuntimeTurnStreamTimingAnchor {
+ call_id: call_id.to_string(),
+ trace_id: trace_id.to_string(),
+ turn_id: turn_id.to_string(),
+ anchor_id: anchor_id.to_string(),
+ server_delta_ms,
+ runtime_session_nonce_hash: expected_nonce_hash,
+ segment_seq: STREAM_TIMING_FIRST_SEGMENT,
+ received_at: Instant::now(),
+ });
+ self.timing.phase = RuntimeTurnStreamTimingPhase::Armed;
+ true
+ }
+
+ fn record_m6(
+ &mut self,
+ call_id: &str,
+ trace_id: &str,
+ turn_id: &str,
+ event: &RuntimeTurnStreamEvent,
+ audio_chunk: &RuntimeTurnStreamAudioChunk,
+ ) -> Option<RuntimeTurnStreamTimingMarker> {
+ if self.timing.phase != RuntimeTurnStreamTimingPhase::Armed {
+ return None;
+ }
+ let anchor = self.timing.anchor.as_ref()?;
+ if anchor.call_id != call_id
+ || anchor.trace_id != trace_id
+ || anchor.turn_id != turn_id
+ || event.call_id.as_deref() != Some(call_id)
+ || event.trace_id.as_deref() != Some(trace_id)
+ || event.turn_id.as_deref() != Some(turn_id)
+ || audio_chunk.segment_seq != Some(anchor.segment_seq)
+ || audio_chunk.chunk_seq != Some(STREAM_TIMING_FIRST_CHUNK)
+ || audio_chunk.stream_timing_version != Some(STREAM_TIMING_VERSION)
+ || audio_chunk.stream_anchor_id.as_deref() != Some(anchor.anchor_id.as_str())
+ {
+ return None;
+ }
+ let Some(marker) = RuntimeTurnStreamTimingMarker::from_anchor(
+ anchor,
+ audio_chunk.chunk_seq.unwrap_or(STREAM_TIMING_FIRST_CHUNK),
+ ) else {
+ self.close_timing();
+ return None;
+ };
+ self.timing.chunk_seq = Some(marker.chunk_seq);
+ self.timing.phase = RuntimeTurnStreamTimingPhase::M6Recorded;
+ Some(marker)
+ }
+
+ fn record_m7(&mut self) -> Option<RuntimeTurnStreamTimingMarker> {
+ if self.timing.phase != RuntimeTurnStreamTimingPhase::M6Recorded {
+ return None;
+ }
+ let anchor = self.timing.anchor.as_ref()?;
+ let Some(marker) = RuntimeTurnStreamTimingMarker::from_anchor(
+ anchor,
+ self.timing.chunk_seq.unwrap_or(STREAM_TIMING_FIRST_CHUNK),
+ ) else {
+ self.close_timing();
+ return None;
+ };
+ self.timing.phase = RuntimeTurnStreamTimingPhase::M7Recorded;
+ Some(marker)
+ }
+
+ fn close_timing(&mut self) {
+ self.timing.close();
+ }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+enum RuntimeTurnStreamTimingPhase {
+ Empty,
+ Armed,
+ M6Recorded,
+ M7Recorded,
+ Closed,
+}
+
+struct RuntimeTurnStreamTimingState {
+ phase: RuntimeTurnStreamTimingPhase,
+ anchor: Option<RuntimeTurnStreamTimingAnchor>,
+ chunk_seq: Option<u64>,
+}
+
+impl Default for RuntimeTurnStreamTimingState {
+ fn default() -> Self {
+ Self {
+ phase: RuntimeTurnStreamTimingPhase::Empty,
+ anchor: None,
+ chunk_seq: None,
+ }
+ }
+}
+
+impl RuntimeTurnStreamTimingState {
+ fn close(&mut self) {
+ self.anchor = None;
+ self.chunk_seq = None;
+ self.phase = RuntimeTurnStreamTimingPhase::Closed;
+ }
+}
+
+impl Drop for RuntimeTurnStreamTimingState {
+ fn drop(&mut self) {
+ self.anchor = None;
+ self.chunk_seq = None;
+ }
+}
+
+struct RuntimeTurnStreamTimingAnchor {
+ call_id: String,
+ trace_id: String,
+ turn_id: String,
+ anchor_id: String,
+ server_delta_ms: u64,
+ runtime_session_nonce_hash: String,
+ segment_seq: u64,
+ received_at: Instant,
+}
+
+struct RuntimeTurnStreamTimingMarker {
+ version: u32,
+ anchor_id: String,
+ anchor_server_delta_ms: u64,
+ anchor_elapsed_ms: u64,
+ server_delta_ms: u64,
+ runtime_session_nonce_hash: String,
+ segment_seq: u64,
+ chunk_seq: u64,
+}
+
+impl RuntimeTurnStreamTimingMarker {
+ fn from_anchor(anchor: &RuntimeTurnStreamTimingAnchor, chunk_seq: u64) -> Option<Self> {
+ let elapsed_ms = u64::try_from(anchor.received_at.elapsed().as_millis()).ok()?;
+ if elapsed_ms > STREAM_TIMING_MAX_ELAPSED_MS {
+ return None;
+ }
+ Some(Self {
+ version: STREAM_TIMING_VERSION,
+ anchor_id: anchor.anchor_id.clone(),
+ anchor_server_delta_ms: anchor.server_delta_ms,
+ anchor_elapsed_ms: elapsed_ms,
+ server_delta_ms: anchor.server_delta_ms.checked_add(elapsed_ms)?,
+ runtime_session_nonce_hash: anchor.runtime_session_nonce_hash.clone(),
+ segment_seq: anchor.segment_seq,
+ chunk_seq,
+ })
+ }
+
+ fn extension_with(&self, extra: serde_json::Value) -> serde_json::Value {
+ let mut extension = match extra {
+ serde_json::Value::Object(value) => value,
+ _ => serde_json::Map::new(),
+ };
+ extension.insert("streamTimingVersion".to_string(), json!(self.version));
+ extension.insert("streamAnchorId".to_string(), json!(self.anchor_id));
+ extension.insert(
+ "streamAnchorServerDeltaMs".to_string(),
+ json!(self.anchor_server_delta_ms),
+ );
+ extension.insert("anchorElapsedMs".to_string(), json!(self.anchor_elapsed_ms));
+ extension.insert(
+ "runtimeSessionNonceHash".to_string(),
+ json!(self.runtime_session_nonce_hash),
+ );
+ extension.insert("segmentSeq".to_string(), json!(self.segment_seq));
+ extension.insert("chunkSeq".to_string(), json!(self.chunk_seq));
+ extension.insert("streamTimingValidation".to_string(), json!("bound"));
+ serde_json::Value::Object(extension)
+ }
+}
+
+fn runtime_session_nonce_hash(value: &str) -> String {
+ let digest = Sha256::digest(value.as_bytes());
+ digest[..6]
+ .iter()
+ .map(|byte| format!("{byte:02x}"))
+ .collect()
+}
+
+#[derive(Debug, PartialEq, Eq)]
+enum ReplyChunkMarker {
+ FirstReply,
+ SegmentFirst,
+ None,
+}
+
+#[derive(Default)]
+struct ReplyChunkMarkerState {
+ first_reply_seen: bool,
+ seen_segments: HashSet<u64>,
+}
+
+impl ReplyChunkMarkerState {
+ fn observe(&mut self, segment_seq: Option<u64>) -> ReplyChunkMarker {
+ let first_for_segment = segment_seq
+ .map(|value| self.seen_segments.insert(value))
+ .unwrap_or(false);
+ if !self.first_reply_seen {
+ self.first_reply_seen = true;
+ return ReplyChunkMarker::FirstReply;
+ }
+ if first_for_segment {
+ return ReplyChunkMarker::SegmentFirst;
+ }
+ ReplyChunkMarker::None
}
}
@@ -2013,6 +3241,12 @@
struct RuntimeTurnStreamEvent {
#[serde(rename = "type", alias = "event")]
event_type: Option<String>,
+ #[serde(rename = "callId")]
+ call_id: Option<String>,
+ #[serde(rename = "traceId")]
+ trace_id: Option<String>,
+ #[serde(rename = "turnId")]
+ turn_id: Option<String>,
#[serde(rename = "seq")]
seq: Option<u64>,
#[serde(rename = "replyPlaybackMode")]
@@ -2048,6 +3282,12 @@
struct RuntimeTurnStreamAudioChunk {
#[serde(rename = "chunkSeq", alias = "seq")]
chunk_seq: Option<u64>,
+ #[serde(rename = "segmentSeq")]
+ segment_seq: Option<u64>,
+ #[serde(rename = "streamTimingVersion")]
+ stream_timing_version: Option<u32>,
+ #[serde(rename = "streamAnchorId")]
+ stream_anchor_id: Option<String>,
format: Option<String>,
#[serde(rename = "sampleRate")]
sample_rate: Option<u32>,
@@ -2064,6 +3304,23 @@
stage: Option<String>,
#[serde(rename = "reasonCode")]
reason_code: Option<String>,
+ extension: Option<RuntimeTurnStreamTimingExtension>,
+}
+
+#[derive(Clone, Deserialize)]
+struct RuntimeTurnStreamTimingExtension {
+ #[serde(rename = "streamTimingVersion")]
+ stream_timing_version: Option<u32>,
+ #[serde(rename = "streamAnchorId")]
+ stream_anchor_id: Option<String>,
+ #[serde(rename = "streamAnchorServerDeltaMs")]
+ stream_anchor_server_delta_ms: Option<u64>,
+ #[serde(rename = "runtimeSessionNonceHash")]
+ runtime_session_nonce_hash: Option<String>,
+ #[serde(rename = "segmentSeq")]
+ segment_seq: Option<u64>,
+ #[serde(rename = "streamTimingValidation")]
+ stream_timing_validation: Option<String>,
}
#[derive(Deserialize)]
@@ -2096,6 +3353,30 @@
#[serde(rename = "commandCode")]
command_code: Option<String>,
params: Option<serde_json::Value>,
+}
+
+fn should_publish_device_output(
+ published_ids: &mut HashSet<String>,
+ output: &RuntimeTurnDeviceOutput,
+) -> bool {
+ let Some(command_id) = output
+ .command_id
+ .as_deref()
+ .map(str::trim)
+ .filter(|value| !value.is_empty())
+ else {
+ return false;
+ };
+ if output
+ .command_code
+ .as_deref()
+ .map(str::trim)
+ .filter(|value| !value.is_empty())
+ .is_none()
+ {
+ return false;
+ }
+ published_ids.insert(command_id.to_string())
}
fn require_safe_segment(value: &str) -> Result<()> {
@@ -2137,6 +3418,8 @@
turn_bridge_config: TurnBridgeConfig,
http: Client,
sink: Arc<BotAudioOutputSink>,
+ expected_participant_identity: Option<String>,
+ participant: RemoteParticipant,
) -> JoinHandle<()> {
tokio::spawn(async move {
let mut stream = NativeAudioStream::new(
@@ -2145,6 +3428,68 @@
i32::from(USER_AUDIO_NUM_CHANNELS),
);
let started_at = Instant::now();
+ let (frame_tx, mut frame_rx) =
+ mpsc::channel::<DrainedUserAudioFrame>(INBOUND_AUDIO_QUEUE_CAPACITY);
+ let (drain_shutdown_tx, mut drain_shutdown_rx) = watch::channel(false);
+ let drain_call_id = call_id.clone();
+ let drain_trace_id = trace_id.clone();
+ let drain_task = tokio::spawn(async move {
+ let mut received_frame_count: u64 = 0;
+ let mut dropped_frame_count: u64 = 0;
+ loop {
+ tokio::select! {
+ changed = drain_shutdown_rx.changed() => {
+ match changed {
+ Ok(()) if *drain_shutdown_rx.borrow() => break,
+ Ok(()) => {}
+ Err(_) => break,
+ }
+ }
+ maybe_frame = stream.next() => {
+ let Some(frame) = maybe_frame else {
+ break;
+ };
+ received_frame_count = received_frame_count.saturating_add(1);
+ let drained = DrainedUserAudioFrame {
+ frame_index: received_frame_count,
+ captured_elapsed_ms: started_at.elapsed().as_millis() as u64,
+ frame: AudioFrame {
+ data: Cow::Owned(frame.data.as_ref().to_vec()),
+ sample_rate: frame.sample_rate,
+ num_channels: frame.num_channels,
+ samples_per_channel: frame.samples_per_channel,
+ },
+ };
+ match frame_tx.try_send(drained) {
+ Ok(()) => {}
+ Err(mpsc::error::TrySendError::Full(_)) => {
+ dropped_frame_count = dropped_frame_count.saturating_add(1);
+ if dropped_frame_count % INBOUND_AUDIO_DROP_LOG_INTERVAL == 1 {
+ warn!(
+ call_id = %drain_call_id,
+ trace_id = %drain_trace_id,
+ dropped_frame_count,
+ received_frame_count,
+ queue_capacity = INBOUND_AUDIO_QUEUE_CAPACITY,
+ "runtime helper inbound_audio_queue_full_dropping_newest"
+ );
+ }
+ }
+ Err(mpsc::error::TrySendError::Closed(_)) => break,
+ }
+ }
+ }
+ }
+ stream.close();
+ info!(
+ call_id = %drain_call_id,
+ trace_id = %drain_trace_id,
+ received_frame_count,
+ dropped_frame_count,
+ queue_capacity = INBOUND_AUDIO_QUEUE_CAPACITY,
+ "runtime helper user_audio_drain_ended"
+ );
+ });
let mut frame_count: u64 = 0;
let mut sample_count: u64 = 0;
let mut simple_vad = if simple_vad_enabled {
@@ -2152,11 +3497,14 @@
} else {
None
};
+ let mut realtime_asr_upload: Option<RealtimeAsrUpload> = None;
+ let mut last_fixture_sequence: Option<String> = None;
- while let Some(frame) = stream.next().await {
- frame_count += 1;
+ while let Some(drained) = frame_rx.recv().await {
+ let frame = drained.frame;
+ frame_count = drained.frame_index;
sample_count += u64::from(frame.samples_per_channel) * u64::from(frame.num_channels);
- let elapsed_ms = started_at.elapsed().as_millis() as u64;
+ let elapsed_ms = drained.captured_elapsed_ms;
if frame_count == 1 {
info!(
call_id = %call_id,
@@ -2184,7 +3532,11 @@
if let Some(vad) = simple_vad.as_mut() {
if vad_enabled_gate.load(Ordering::Acquire) {
- if let Some(turn) = vad.observe_frame(
+ let participant_identity = participant.identity().to_string();
+ let (was_in_speech, is_in_speech, turn) = observe_bound_participant_frame(
+ &participant_identity,
+ expected_participant_identity.as_deref(),
+ vad,
&call_id,
&trace_id,
&participant_alias,
@@ -2192,7 +3544,40 @@
frame_count,
elapsed_ms,
&frame,
- ) {
+ http.clone(),
+ turn_bridge_config.realtime_asr_config(),
+ || participant.attributes(),
+ &mut realtime_asr_upload,
+ &mut last_fixture_sequence,
+ turn_bridge_config.asr_realtime_enabled,
+ );
+
+ if was_in_speech {
+ let push_failed = realtime_asr_upload
+ .as_mut()
+ .and_then(|upload| upload.push_48k_samples(frame.data.as_ref()).err());
+ if let Some(error) = push_failed {
+ warn!(
+ call_id = %call_id,
+ trace_id = %trace_id,
+ error = %safe_error(&error.to_string()),
+ "runtime helper asr_realtime_upload_failed_fallback"
+ );
+ if let Some(upload) = realtime_asr_upload.take() {
+ tokio::spawn(async move {
+ upload.cancel("upload_backpressure").await;
+ });
+ }
+ }
+ }
+
+ if let Some(turn) = turn {
+ let realtime_asr_result_ref = match realtime_asr_upload.take() {
+ Some(upload) => {
+ finish_realtime_asr_upload(upload, &call_id, &trace_id, &turn).await
+ }
+ None => None,
+ };
handle_finished_turn(
&http,
&turn_bridge_config,
@@ -2200,10 +3585,22 @@
&call_id,
&trace_id,
turn,
+ realtime_asr_result_ref,
)
.await;
+ } else if was_in_speech && !is_in_speech {
+ if let Some(upload) = realtime_asr_upload.take() {
+ tokio::spawn(async move {
+ upload.cancel("speech_too_short").await;
+ });
+ }
}
} else {
+ if let Some(upload) = realtime_asr_upload.take() {
+ tokio::spawn(async move {
+ upload.cancel("vad_disabled").await;
+ });
+ }
vad.observe_disabled_frame(
&call_id,
&trace_id,
@@ -2224,9 +3621,41 @@
&track_sid_alias,
started_at.elapsed().as_millis() as u64,
) {
- handle_finished_turn(&http, &turn_bridge_config, &sink, &call_id, &trace_id, turn)
- .await;
+ let realtime_asr_result_ref = match realtime_asr_upload.take() {
+ Some(upload) => {
+ finish_realtime_asr_upload(upload, &call_id, &trace_id, &turn).await
+ }
+ None => None,
+ };
+ handle_finished_turn(
+ &http,
+ &turn_bridge_config,
+ &sink,
+ &call_id,
+ &trace_id,
+ turn,
+ realtime_asr_result_ref,
+ )
+ .await;
}
+ }
+ if let Some(upload) = realtime_asr_upload.take() {
+ upload.cancel("stream_end").await;
+ }
+ let _ = drain_shutdown_tx.send(true);
+ let mut drain_task = drain_task;
+ if timeout(AUDIO_DRAIN_STOP_GRACE, &mut drain_task)
+ .await
+ .is_err()
+ {
+ drain_task.abort();
+ let _ = drain_task.await;
+ warn!(
+ call_id = %call_id,
+ trace_id = %trace_id,
+ stop_grace_ms = AUDIO_DRAIN_STOP_GRACE.as_millis() as u64,
+ "runtime helper user_audio_drain_stop_timeout"
+ );
}
info!(
@@ -2240,6 +3669,234 @@
"runtime helper user_audio_stream_ended"
);
})
+}
+
+fn observe_frame_and_start_session<F>(
+ vad: &mut SimpleVad,
+ call_id: &str,
+ trace_id: &str,
+ participant_alias: &str,
+ track_sid_alias: &str,
+ frame_count: u64,
+ elapsed_ms: u64,
+ frame: &AudioFrame<'_>,
+ http: Client,
+ config: RealtimeAsrConfig,
+ read_attributes: F,
+ upload_slot: &mut Option<RealtimeAsrUpload>,
+ last_fixture_sequence: &mut Option<String>,
+ realtime_enabled: bool,
+) -> (bool, bool, Option<FinishedSpeechTurn>)
+where
+ F: FnOnce() -> std::collections::HashMap<String, String>,
+{
+ let was_in_speech = vad.in_speech;
+ let turn = vad.observe_frame(
+ call_id,
+ trace_id,
+ participant_alias,
+ track_sid_alias,
+ frame_count,
+ elapsed_ms,
+ frame,
+ );
+ let is_in_speech = vad.in_speech;
+ if !was_in_speech && is_in_speech {
+ start_realtime_session_for_new_speech(
+ http,
+ config,
+ call_id,
+ trace_id,
+ vad,
+ read_attributes,
+ upload_slot,
+ last_fixture_sequence,
+ realtime_enabled,
+ );
+ }
+ (was_in_speech, is_in_speech, turn)
+}
+
+fn observe_bound_participant_frame<F>(
+ participant_identity: &str,
+ expected_participant: Option<&str>,
+ vad: &mut SimpleVad,
+ call_id: &str,
+ trace_id: &str,
+ participant_alias: &str,
+ track_sid_alias: &str,
+ frame_count: u64,
+ elapsed_ms: u64,
+ frame: &AudioFrame<'_>,
+ http: Client,
+ config: RealtimeAsrConfig,
+ read_attributes: F,
+ upload_slot: &mut Option<RealtimeAsrUpload>,
+ last_fixture_sequence: &mut Option<String>,
+ realtime_enabled: bool,
+) -> (bool, bool, Option<FinishedSpeechTurn>)
+where
+ F: FnOnce() -> std::collections::HashMap<String, String>,
+{
+ if !is_bound_user_participant(participant_identity, expected_participant) {
+ warn!(
+ "audioIngressOriginStatus" = "wrong_participant_or_track",
+ "runtime helper rejected audio participant before VAD/session"
+ );
+ return (vad.in_speech, vad.in_speech, None);
+ }
+ observe_frame_and_start_session(
+ vad,
+ call_id,
+ trace_id,
+ participant_alias,
+ track_sid_alias,
+ frame_count,
+ elapsed_ms,
+ frame,
+ http,
+ config,
+ read_attributes,
+ upload_slot,
+ last_fixture_sequence,
+ realtime_enabled,
+ )
+}
+
+fn start_realtime_session_for_new_speech(
+ http: Client,
+ config: RealtimeAsrConfig,
+ call_id: &str,
+ trace_id: &str,
+ vad: &SimpleVad,
+ read_attributes: impl FnOnce() -> std::collections::HashMap<String, String>,
+ upload_slot: &mut Option<RealtimeAsrUpload>,
+ last_fixture_sequence: &mut Option<String>,
+ realtime_enabled: bool,
+) {
+ let turn_id = format!("turn-{:04}", vad.turn_index);
+ let attributes = read_attributes();
+ let origin_status = AudioIngressMetadata::origin_status(&attributes);
+ let metadata = match AudioIngressMetadata::from_participant(&attributes) {
+ Ok(metadata) => metadata,
+ Err(reason) => {
+ warn!(call_id = %call_id, trace_id = %trace_id, turn_id = %turn_id,
+ reason, audioIngressOriginStatus = %AudioIngressMetadata::rejected_origin_status(
+ reason, &attributes
+ ),
+ "runtime helper asr_realtime_metadata_rejected");
+ return;
+ }
+ };
+ if let Some(metadata) = metadata.as_ref() {
+ if !fixture_sequence_is_new(
+ last_fixture_sequence.as_deref(),
+ &metadata.client_fixture_sequence,
+ ) {
+ warn!(call_id = %call_id, trace_id = %trace_id, turn_id = %turn_id,
+ audioIngressOriginStatus = "sequence_replayed_or_regressed",
+ "runtime helper asr_realtime_metadata_sequence_rejected");
+ return;
+ }
+ }
+ match RealtimeAsrUpload::start(
+ http,
+ config,
+ call_id,
+ trace_id,
+ &turn_id,
+ &vad.speech_samples,
+ metadata.as_ref(),
+ ) {
+ Ok(upload) => {
+ if let Some(metadata) = metadata {
+ *last_fixture_sequence = Some(metadata.client_fixture_sequence);
+ }
+ info!(call_id = %call_id, trace_id = %trace_id, turn_id = %turn_id,
+ origin_status, "runtime helper asr_realtime_session_started");
+ *upload_slot = Some(upload);
+ }
+ Err(error) if realtime_enabled => {
+ warn!(call_id = %call_id, trace_id = %trace_id, turn_id = %turn_id,
+ error = %safe_error(&error.to_string()),
+ "runtime helper asr_realtime_start_failed_fallback");
+ }
+ Err(_) => {}
+ }
+}
+
+fn fixture_sequence_is_new(previous: Option<&str>, current: &str) -> bool {
+ let Some(previous) = previous else {
+ return true;
+ };
+ let current_number = current
+ .rsplit_once('-')
+ .and_then(|(_, value)| value.parse::<u64>().ok());
+ let previous_number = previous
+ .rsplit_once('-')
+ .and_then(|(_, value)| value.parse::<u64>().ok());
+ match (previous_number, current_number) {
+ (Some(previous), Some(current)) => current > previous,
+ _ => previous != current,
+ }
+}
+
+struct DrainedUserAudioFrame {
+ frame_index: u64,
+ captured_elapsed_ms: u64,
+ frame: AudioFrame<'static>,
+}
+
+async fn finish_realtime_asr_upload(
+ upload: RealtimeAsrUpload,
+ call_id: &str,
+ trace_id: &str,
+ turn: &FinishedSpeechTurn,
+) -> Option<String> {
+ match upload.finish(turn.duration_ms, &turn.end_reason).await {
+ Ok(RealtimeAsrOutcome {
+ status,
+ asr_result_ref,
+ provider_alias,
+ partial_count,
+ fallback_reason,
+ fallback_stage,
+ chunk_count,
+ audio_bytes,
+ wall_ms,
+ }) => {
+ info!(
+ call_id = %call_id,
+ trace_id = %trace_id,
+ turn_id = %turn.turn_id,
+ status = %status,
+ provider_alias = ?provider_alias,
+ partial_count,
+ chunk_count,
+ audio_bytes,
+ wall_ms,
+ asr_result_ref_present = asr_result_ref.is_some(),
+ fallback_reason = ?fallback_reason,
+ fallback_stage = ?fallback_stage,
+ "runtime helper asr_realtime_finished"
+ );
+ if status == "final" {
+ asr_result_ref
+ } else {
+ None
+ }
+ }
+ Err(error) => {
+ warn!(
+ call_id = %call_id,
+ trace_id = %trace_id,
+ turn_id = %turn.turn_id,
+ error = %safe_error(&error.to_string()),
+ "runtime helper asr_realtime_failed_fallback"
+ );
+ None
+ }
+ }
}
#[derive(Clone)]
@@ -2933,3 +4590,962 @@
Ok(())
}
}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use std::{
+ collections::{HashMap, HashSet},
+ io::{Read, Write},
+ net::TcpListener,
+ sync::{
+ Arc, Mutex,
+ atomic::{AtomicUsize, Ordering},
+ mpsc,
+ },
+ thread,
+ time::Duration,
+ };
+
+ #[derive(Debug)]
+ enum PreAudioOrderEvent {
+ DataReceived {
+ sender: String,
+ sequence: String,
+ },
+ TrackSubscribed {
+ participant: String,
+ attributes: HashMap<String, String>,
+ },
+ }
+
+ #[derive(Clone, Default)]
+ struct CapturedLogs(Arc<Mutex<Vec<u8>>>);
+
+ struct CapturedLogWriter(Arc<Mutex<Vec<u8>>>);
+
+ impl Write for CapturedLogWriter {
+ fn write(&mut self, bytes: &[u8]) -> std::io::Result<usize> {
+ self.0.lock().unwrap().extend_from_slice(bytes);
+ Ok(bytes.len())
+ }
+
+ fn flush(&mut self) -> std::io::Result<()> {
+ Ok(())
+ }
+ }
+
+ impl<'a> tracing_subscriber::fmt::MakeWriter<'a> for CapturedLogs {
+ type Writer = CapturedLogWriter;
+
+ fn make_writer(&'a self) -> Self::Writer {
+ CapturedLogWriter(self.0.clone())
+ }
+ }
+
+ fn drive_pre_audio_order_test_seam(events: &[PreAudioOrderEvent]) -> Vec<&'static str> {
+ let mut pending_sequence = None;
+ let mut effects = Vec::new();
+ for event in events {
+ match event {
+ PreAudioOrderEvent::DataReceived { sender, sequence } if sender == "user-1" => {
+ pending_sequence = Some(sequence.as_str());
+ }
+ PreAudioOrderEvent::TrackSubscribed {
+ participant,
+ attributes,
+ } => {
+ let pending = pending_sequence.is_some();
+ let probe_result = pending_sequence.map(|sequence| {
+ classify_controlled_fixture_attributes(
+ participant,
+ Some("user-1"),
+ attributes,
+ sequence,
+ )
+ .is_ok()
+ });
+ let mut ack_observed = false;
+ let mut observer_started = false;
+ start_observer_after_controlled_fixture_probe(pending, probe_result, || {
+ if pending && probe_result == Some(true) {
+ ack_observed = true;
+ }
+ observer_started = true;
+ });
+ if ack_observed {
+ effects.push("ack_observed");
+ }
+ if observer_started {
+ effects.push("observer_started");
+ }
+ pending_sequence = None;
+ }
+ PreAudioOrderEvent::DataReceived { .. } => {}
+ }
+ }
+ effects
+ }
+
+ #[test]
+ fn production_vad_session_boundary_reads_updated_attributes() {
+ let config = SimpleVadConfig {
+ rms_threshold: 0.001,
+ peak_threshold: 0.01,
+ start_frames: 2,
+ end_silence_ms: 100,
+ min_speech_ms: 1,
+ max_turn_ms: 1_000,
+ initial_ignore_ms: 0,
+ };
+ let mut vad = SimpleVad::new(config);
+ let samples = vec![1_000i16; 160];
+ let frame = AudioFrame {
+ data: samples.as_slice().into(),
+ sample_rate: 16_000,
+ num_channels: 1,
+ samples_per_channel: 160,
+ };
+ let mut attributes = std::collections::HashMap::from([
+ (
+ "inputSourceCategory".to_string(),
+ "controlled_fixture".to_string(),
+ ),
+ (
+ "clientFixtureSequence".to_string(),
+ "fixture-01".to_string(),
+ ),
+ ]);
+ let mut starts = Vec::new();
+ for (session_index, sequence) in [(1, "fixture-01"), (2, "fixture-02")] {
+ let was_in_speech = vad.in_speech;
+ vad.observe_frame(
+ "call-001",
+ "trace-001",
+ "participant",
+ "track",
+ session_index * 2 - 1,
+ 1_000 * session_index,
+ &frame,
+ );
+ vad.observe_frame(
+ "call-001",
+ "trace-001",
+ "participant",
+ "track",
+ session_index * 2,
+ 1_000 * session_index + 10,
+ &frame,
+ );
+ let is_in_speech = vad.in_speech;
+ assert!(!was_in_speech && is_in_speech);
+ attributes.insert("clientFixtureSequence".to_string(), sequence.to_string());
+ let metadata = AudioIngressMetadata::from_participant(&attributes)
+ .expect("valid participant attributes")
+ .expect("controlled fixture metadata");
+ let session_line = asr_realtime::session_start_line(
+ "call-001",
+ "trace-001",
+ &format!("turn-{session_index:04}"),
+ "nonce-001",
+ Some(&metadata),
+ )
+ .expect("session start line");
+ let session_json: serde_json::Value =
+ serde_json::from_slice(&session_line).expect("session start json");
+ assert_eq!(sequence, session_json["clientFixtureSequence"]);
+ starts.push(metadata.client_fixture_sequence);
+ vad.reset_current_turn();
+ }
+ assert_eq!(vec!["fixture-01", "fixture-02"], starts);
+ attributes.insert("inputSourceCategory".to_string(), "other".to_string());
+ assert!(AudioIngressMetadata::from_participant(&attributes).is_err());
+ assert!(
+ AudioIngressMetadata::from_participant(&std::collections::HashMap::new())
+ .expect("missing attributes is absent")
+ .is_none()
+ );
+ attributes.insert(
+ "clientFixtureSequence".to_string(),
+ "fixture-01".to_string(),
+ );
+ assert!(AudioIngressMetadata::from_participant(&attributes).is_err());
+ }
+
+ #[test]
+ fn production_observer_rejects_wrong_participant_before_vad_session() {
+ assert!(!is_bound_user_participant(
+ "participant-other",
+ Some("participant-user")
+ ));
+ assert!(is_bound_user_participant(
+ "participant-user",
+ Some("participant-user")
+ ));
+ assert!(is_bound_user_participant("participant-any", None));
+ }
+
+ #[tokio::test]
+ async fn production_observer_vad_to_session_entry_reads_each_updated_attribute() {
+ let listener = TcpListener::bind("127.0.0.1:0").expect("bind local ASR fixture");
+ let address = listener.local_addr().expect("fixture address");
+ let (request_tx, request_rx) = mpsc::channel::<String>();
+ let captured_count = Arc::new(AtomicUsize::new(0));
+ let captured_count_for_server = Arc::clone(&captured_count);
+ let server = thread::spawn(move || {
+ for _ in 0..2 {
+ let (mut stream, _) = listener.accept().expect("accept ASR session");
+ stream
+ .set_read_timeout(Some(Duration::from_secs(2)))
+ .expect("set fixture timeout");
+ let mut bytes = Vec::new();
+ let mut buffer = [0_u8; 4096];
+ loop {
+ match stream.read(&mut buffer) {
+ Ok(0) => break,
+ Ok(size) => {
+ bytes.extend_from_slice(&buffer[..size]);
+ if bytes.windows(7).any(|window| window == b"0\r\n\r\n") {
+ break;
+ }
+ }
+ Err(_) => break,
+ }
+ }
+ request_tx
+ .send(String::from_utf8_lossy(&bytes).into_owned())
+ .expect("capture ASR request");
+ captured_count_for_server.fetch_add(1, Ordering::SeqCst);
+ stream
+ .write_all(b"HTTP/1.1 200 OK\r\ncontent-type: application/json\r\ncontent-length: 39\r\nconnection: close\r\n\r\n{\"code\":0,\"data\":{\"status\":\"ok\"}}")
+ .expect("write fixture response");
+ }
+ });
+ let mut vad = SimpleVad::new(SimpleVadConfig {
+ rms_threshold: 0.001,
+ peak_threshold: 0.01,
+ start_frames: 1,
+ end_silence_ms: 100,
+ min_speech_ms: 1,
+ max_turn_ms: 1_000,
+ initial_ignore_ms: 0,
+ });
+ let frame_data = vec![1_000i16; 160];
+ let frame = AudioFrame {
+ data: frame_data.as_slice().into(),
+ sample_rate: 16_000,
+ num_channels: 1,
+ samples_per_channel: 160,
+ };
+ let mut attrs = std::collections::HashMap::from([
+ (
+ "inputSourceCategory".to_string(),
+ "controlled_fixture".to_string(),
+ ),
+ (
+ "clientFixtureSequence".to_string(),
+ "fixture-01".to_string(),
+ ),
+ ]);
+ let mut upload = None;
+ let mut last_fixture_sequence = None;
+ let config = RealtimeAsrConfig {
+ enabled: true,
+ url: Some(format!("http://{address}/runtime/asr/realtime")),
+ runtime_token: Some("test".to_string()),
+ runtime_session_nonce: Some("test".to_string()),
+ chunk_duration_ms: 200,
+ };
+ let (was, is, turn) = observe_bound_participant_frame(
+ "participant-user",
+ Some("participant-user"),
+ &mut vad,
+ "call-001",
+ "trace-001",
+ "participant-user",
+ "track-001",
+ 1,
+ 1_000,
+ &frame,
+ Client::new(),
+ config.clone(),
+ || attrs.clone(),
+ &mut upload,
+ &mut last_fixture_sequence,
+ true,
+ );
+ assert!(!was && is && turn.is_none());
+ assert!(upload.is_some());
+ upload.take().unwrap().cancel("test").await;
+ vad.reset_current_turn();
+ attrs.insert(
+ "clientFixtureSequence".to_string(),
+ "fixture-02".to_string(),
+ );
+ let (was, is, turn) = observe_bound_participant_frame(
+ "participant-user",
+ Some("participant-user"),
+ &mut vad,
+ "call-001",
+ "trace-001",
+ "participant-user",
+ "track-001",
+ 2,
+ 2_000,
+ &frame,
+ Client::new(),
+ config,
+ || attrs.clone(),
+ &mut upload,
+ &mut last_fixture_sequence,
+ true,
+ );
+ assert!(!was && is && turn.is_none());
+ assert!(upload.is_some());
+ upload.take().unwrap().cancel("test").await;
+
+ let first_request = request_rx
+ .recv_timeout(Duration::from_secs(2))
+ .expect("first session request");
+ let second_request = request_rx
+ .recv_timeout(Duration::from_secs(2))
+ .expect("second session request");
+ assert!(first_request.contains("\"clientFixtureSequence\":\"fixture-01\""));
+ assert!(second_request.contains("\"clientFixtureSequence\":\"fixture-02\""));
+ assert!(
+ first_request.contains("\"audioIngressOriginStatus\":\"controlled_fixture_bound\"")
+ );
+ assert!(
+ second_request.contains("\"audioIngressOriginStatus\":\"controlled_fixture_bound\"")
+ );
+ // The same production boundary rejects a wrong participant before VAD/session creation.
+ assert!(!is_bound_user_participant(
+ "participant-other",
+ Some("participant-user")
+ ));
+ assert_eq!(0, request_rx.try_iter().count());
+ vad.reset_current_turn();
+ attrs.insert(
+ "clientFixtureSequence".to_string(),
+ "fixture-03".to_string(),
+ );
+ let (_, is_wrong, wrong_turn) = observe_bound_participant_frame(
+ "participant-other",
+ Some("participant-user"),
+ &mut vad,
+ "call-001",
+ "trace-001",
+ "participant-user",
+ "track-001",
+ 3,
+ 3_000,
+ &frame,
+ Client::new(),
+ RealtimeAsrConfig {
+ enabled: true,
+ url: Some(format!("http://{address}/runtime/asr/realtime")),
+ runtime_token: Some("test".to_string()),
+ runtime_session_nonce: Some("test".to_string()),
+ chunk_duration_ms: 200,
+ },
+ || attrs.clone(),
+ &mut upload,
+ &mut last_fixture_sequence,
+ true,
+ );
+ assert!(!is_wrong && wrong_turn.is_none() && upload.is_none());
+ vad.reset_current_turn();
+ attrs.insert(
+ "clientFixtureSequence".to_string(),
+ "fixture-01".to_string(),
+ );
+ let (_, _, _) = observe_bound_participant_frame(
+ "participant-user",
+ Some("participant-user"),
+ &mut vad,
+ "call-001",
+ "trace-001",
+ "participant-user",
+ "track-001",
+ 4,
+ 4_000,
+ &frame,
+ Client::new(),
+ RealtimeAsrConfig {
+ enabled: true,
+ url: Some(format!("http://{address}/runtime/asr/realtime")),
+ runtime_token: Some("test".to_string()),
+ runtime_session_nonce: Some("test".to_string()),
+ chunk_duration_ms: 200,
+ },
+ || attrs.clone(),
+ &mut upload,
+ &mut last_fixture_sequence,
+ true,
+ );
+ assert!(upload.is_none());
+ vad.reset_current_turn();
+ let (_, _, _) = observe_bound_participant_frame(
+ "participant-user",
+ Some("participant-user"),
+ &mut vad,
+ "call-001",
+ "trace-001",
+ "participant-user",
+ "track-001",
+ 5,
+ 5_000,
+ &frame,
+ Client::new(),
+ RealtimeAsrConfig {
+ enabled: true,
+ url: Some(format!("http://{address}/runtime/asr/realtime")),
+ runtime_token: Some("test".to_string()),
+ runtime_session_nonce: Some("test".to_string()),
+ chunk_duration_ms: 200,
+ },
+ || attrs.clone(),
+ &mut upload,
+ &mut last_fixture_sequence,
+ true,
+ );
+ assert!(upload.is_none());
+ vad.reset_current_turn();
+ attrs.remove("clientFixtureSequence");
+ let (_, _, missing_sequence_turn) = observe_bound_participant_frame(
+ "participant-user",
+ Some("participant-user"),
+ &mut vad,
+ "call-001",
+ "trace-001",
+ "participant-user",
+ "track-001",
+ 6,
+ 6_000,
+ &frame,
+ Client::new(),
+ RealtimeAsrConfig {
+ enabled: true,
+ url: Some(format!("http://{address}/runtime/asr/realtime")),
+ runtime_token: Some("test".to_string()),
+ runtime_session_nonce: Some("test".to_string()),
+ chunk_duration_ms: 200,
+ },
+ || attrs.clone(),
+ &mut upload,
+ &mut last_fixture_sequence,
+ true,
+ );
+ assert!(missing_sequence_turn.is_none() && upload.is_none());
+ assert_eq!(
+ "sequence_absent",
+ AudioIngressMetadata::rejected_origin_status("incomplete_metadata", &attrs)
+ );
+ assert_eq!(2, captured_count.load(Ordering::SeqCst));
+ vad.reset_current_turn();
+ attrs.insert("inputSourceCategory".to_string(), "other".to_string());
+ let mut invalid_upload = None;
+ let (_, _, invalid_turn) = observe_bound_participant_frame(
+ "participant-user",
+ Some("participant-user"),
+ &mut vad,
+ "call-001",
+ "trace-001",
+ "participant-other",
+ "track-001",
+ 3,
+ 3_000,
+ &frame,
+ Client::new(),
+ RealtimeAsrConfig {
+ enabled: true,
+ url: Some("http://127.0.0.1:9".to_string()),
+ runtime_token: Some("test".to_string()),
+ runtime_session_nonce: Some("test".to_string()),
+ chunk_duration_ms: 200,
+ },
+ || attrs,
+ &mut invalid_upload,
+ &mut last_fixture_sequence,
+ true,
+ );
+ assert!(invalid_turn.is_none());
+ assert!(invalid_upload.is_none());
+ assert_eq!(2, captured_count.load(Ordering::SeqCst));
+ assert_eq!(0, request_rx.try_iter().count());
+ server.join().expect("fixture server");
+ }
+
+ #[test]
+ fn reply_chunk_marker_state_emits_turn_first_once_and_later_segment_first_once() {
+ let mut state = ReplyChunkMarkerState::default();
+
+ assert_eq!(ReplyChunkMarker::FirstReply, state.observe(Some(1)));
+ assert_eq!(ReplyChunkMarker::None, state.observe(Some(1)));
+ assert_eq!(ReplyChunkMarker::SegmentFirst, state.observe(Some(2)));
+ assert_eq!(ReplyChunkMarker::None, state.observe(Some(2)));
+ assert_eq!(ReplyChunkMarker::SegmentFirst, state.observe(Some(3)));
+ }
+
+ #[test]
+ fn reply_chunk_marker_state_without_segment_only_emits_turn_first() {
+ let mut state = ReplyChunkMarkerState::default();
+
+ assert_eq!(ReplyChunkMarker::FirstReply, state.observe(None));
+ assert_eq!(ReplyChunkMarker::None, state.observe(None));
+ }
+
+ #[test]
+ fn runtime_turn_stream_audio_chunk_reads_segment_seq() {
+ let event: RuntimeTurnStreamEvent = serde_json::from_str(
+ r#"{"type":"reply_audio_chunk","audioChunk":{"chunkSeq":4,"segmentSeq":2,"format":"pcm_s16le","payloadBase64":"AA==","last":false}}"#,
+ )
+ .expect("turn stream event");
+
+ assert_eq!(
+ Some(2),
+ event.audio_chunk.and_then(|chunk| chunk.segment_seq)
+ );
+ }
+
+ #[test]
+ fn runtime_turn_stream_reads_frozen_m5_and_audio_chunk_timing_contract() {
+ let activity: RuntimeTurnStreamEvent = serde_json::from_str(
+ r#"{"type":"activity","callId":"call-1","traceId":"trace-1","turnId":"turn-1","activity":{"eventType":"tts_first_audio_chunk_ready","extension":{"streamTimingVersion":1,"streamAnchorId":"0123456789abcdef","streamAnchorServerDeltaMs":1200,"runtimeSessionNonceHash":"abcdef012345","segmentSeq":1,"streamTimingValidation":"bound"}}}"#,
+ )
+ .expect("m5 activity event");
+ let audio: RuntimeTurnStreamEvent = serde_json::from_str(
+ r#"{"type":"reply_audio_chunk","callId":"call-1","traceId":"trace-1","turnId":"turn-1","audioChunk":{"chunkSeq":1,"segmentSeq":1,"streamTimingVersion":1,"streamAnchorId":"0123456789abcdef","format":"pcm_s16le","payloadBase64":"AA==","last":false}}"#,
+ )
+ .expect("timed audio chunk event");
+
+ assert_eq!(Some("call-1"), activity.call_id.as_deref());
+ assert_eq!(Some("trace-1"), activity.trace_id.as_deref());
+ assert_eq!(Some("turn-1"), activity.turn_id.as_deref());
+ let extension = activity.activity.unwrap().extension.unwrap();
+ assert_eq!(Some(1), extension.stream_timing_version);
+ assert_eq!(Some(1200), extension.stream_anchor_server_delta_ms);
+ assert_eq!(
+ Some("0123456789abcdef"),
+ audio.audio_chunk.unwrap().stream_anchor_id.as_deref()
+ );
+ }
+
+ #[test]
+ fn runtime_turn_stream_timing_records_m6_and_m7_once_then_rejects_terminal_late_events() {
+ let nonce = "runtime-nonce";
+ let nonce_hash = runtime_session_nonce_hash(nonce);
+ let activity: RuntimeTurnStreamEvent = serde_json::from_str(&format!(
+ r#"{{"type":"activity","callId":"call-1","traceId":"trace-1","turnId":"turn-1","activity":{{"eventType":"tts_first_audio_chunk_ready","extension":{{"streamTimingVersion":1,"streamAnchorId":"0123456789abcdef","streamAnchorServerDeltaMs":1200,"runtimeSessionNonceHash":"{nonce_hash}","segmentSeq":1,"streamTimingValidation":"bound"}}}}}}"#,
+ ))
+ .expect("m5 activity event");
+ let audio: RuntimeTurnStreamEvent = serde_json::from_str(
+ r#"{"type":"reply_audio_chunk","callId":"call-1","traceId":"trace-1","turnId":"turn-1","audioChunk":{"chunkSeq":1,"segmentSeq":1,"streamTimingVersion":1,"streamAnchorId":"0123456789abcdef","format":"pcm_s16le","payloadBase64":"AA==","last":false}}"#,
+ )
+ .expect("timed audio chunk event");
+ let mut state = RuntimeTurnStreamState::default();
+
+ assert!(state.arm_timing_anchor("call-1", "trace-1", "turn-1", nonce, &activity));
+ let chunk = audio.audio_chunk.as_ref().expect("audio chunk");
+ assert!(
+ state
+ .record_m6("call-1", "trace-1", "turn-1", &audio, chunk)
+ .is_some()
+ );
+ assert!(
+ state
+ .record_m6("call-1", "trace-1", "turn-1", &audio, chunk)
+ .is_none()
+ );
+ assert!(state.record_m7().is_some());
+ assert!(state.record_m7().is_none());
+
+ state.close_timing();
+ assert_eq!(RuntimeTurnStreamTimingPhase::Closed, state.timing.phase);
+ assert!(state.timing.anchor.is_none());
+ assert!(!state.arm_timing_anchor("call-1", "trace-1", "turn-1", nonce, &activity));
+ assert!(
+ state
+ .record_m6("call-1", "trace-1", "turn-1", &audio, chunk)
+ .is_none()
+ );
+ assert!(state.record_m7().is_none());
+ }
+
+ #[test]
+ fn device_output_contract_is_reliable_and_deduplicated() {
+ let output = RuntimeTurnDeviceOutput {
+ command_id: Some("cmd-1".to_string()),
+ command_code: Some("custom.app.DeviceLevelChange".to_string()),
+ params: Some(serde_json::json!({"level": 1})),
+ };
+ let mut published = HashSet::new();
+ assert!(should_publish_device_output(&mut published, &output));
+ assert!(!should_publish_device_output(&mut published, &output));
+ assert_eq!(published.len(), 1);
+ }
+
+ #[test]
+ fn device_output_invalid_or_missing_command_is_fail_closed() {
+ for output in [
+ RuntimeTurnDeviceOutput {
+ command_id: None,
+ command_code: Some("custom.app.DeviceLevelChange".to_string()),
+ params: None,
+ },
+ RuntimeTurnDeviceOutput {
+ command_id: Some("cmd-1".to_string()),
+ command_code: None,
+ params: None,
+ },
+ ] {
+ let mut published = HashSet::new();
+ assert!(!should_publish_device_output(&mut published, &output));
+ assert!(published.is_empty());
+ }
+ }
+
+ #[test]
+ fn controlled_fixture_probe_requires_bound_hashes_and_protocol() {
+ let call_id = "call-ack-1";
+ let trace_id = "trace-ack-1";
+ let payload = serde_json::to_vec(&json!({
+ "type": CONTROLLED_FIXTURE_PROBE_TOPIC,
+ "protocolVersion": CONTROLLED_FIXTURE_PROTOCOL_VERSION,
+ "callIdHash": sha256_hex(call_id),
+ "callTraceIdHash": sha256_hex(trace_id),
+ "generation": CONTROLLED_FIXTURE_GENERATION,
+ "clientFixtureSequence": "fixture-01"
+ }))
+ .unwrap();
+ let sender = ParticipantIdentity("user-1".to_string());
+ let pending =
+ controlled_fixture_probe(&payload, call_id, trace_id, &sender, Some("user-1"))
+ .expect("valid probe");
+ assert_eq!(pending.sequence, "fixture-01");
+ assert!(
+ controlled_fixture_probe(&payload, call_id, "other-trace", &sender, Some("user-1"),)
+ .is_none()
+ );
+ assert!(
+ controlled_fixture_probe(
+ &payload,
+ call_id,
+ trace_id,
+ &ParticipantIdentity("other-user".to_string()),
+ Some("user-1"),
+ )
+ .is_none()
+ );
+ }
+
+ #[test]
+ fn controlled_fixture_attributes_ack_only_on_exact_current_sequence() {
+ let mut attributes = std::collections::HashMap::from([
+ (
+ "inputSourceCategory".to_string(),
+ "controlled_fixture".to_string(),
+ ),
+ (
+ "clientFixtureSequence".to_string(),
+ "fixture-01".to_string(),
+ ),
+ ]);
+ assert!(
+ classify_controlled_fixture_attributes(
+ "user-1",
+ Some("user-1"),
+ &attributes,
+ "fixture-01"
+ )
+ .is_ok()
+ );
+ attributes.insert(
+ "clientFixtureSequence".to_string(),
+ "fixture-02".to_string(),
+ );
+ assert_eq!(
+ classify_controlled_fixture_attributes(
+ "user-1",
+ Some("user-1"),
+ &attributes,
+ "fixture-01"
+ ),
+ Err("wrong_sequence")
+ );
+ attributes.remove("clientFixtureSequence");
+ assert_eq!(
+ classify_controlled_fixture_attributes(
+ "user-1",
+ Some("user-1"),
+ &attributes,
+ "fixture-01"
+ ),
+ Err("missing_sequence")
+ );
+ assert_eq!(
+ classify_controlled_fixture_attributes(
+ "other-user",
+ Some("user-1"),
+ &attributes,
+ "fixture-01"
+ ),
+ Err("wrong_participant")
+ );
+ }
+
+ #[test]
+ fn controlled_fixture_probe_failure_log_is_fixed_redacted_and_has_no_audio_effect() {
+ let logs = CapturedLogs::default();
+ let subscriber = tracing_subscriber::fmt()
+ .without_time()
+ .with_ansi(false)
+ .with_writer(logs.clone())
+ .finish();
+ let call_id = "private-call-value";
+ let trace_id = "private-trace-value";
+ let sequence = "private-sequence-value";
+ let mut observer_starts = 0;
+ tracing::subscriber::with_default(subscriber, || {
+ let decision = Err("wrong_source");
+ let (ack_result, reject_reason, observed) =
+ record_controlled_fixture_attribute_decision(decision, call_id, trace_id, sequence);
+ record_controlled_fixture_probe_event(
+ "audio_observer_blocked",
+ call_id,
+ trace_id,
+ sequence,
+ observed,
+ Some(ack_result),
+ reject_reason,
+ );
+ assert!(!start_observer_after_controlled_fixture_probe(
+ true,
+ Some(observed),
+ || observer_starts += 1,
+ ));
+ });
+ let output = String::from_utf8(logs.0.lock().unwrap().clone()).unwrap();
+ assert!(output.contains("ack_result=\"rejected\""));
+ assert!(output.contains("reject_reason=\"wrong_source\""));
+ assert!(output.contains("observed=false"));
+ assert!(output.contains(&sha256_hex(call_id)));
+ assert!(output.contains(&sha256_hex(trace_id)));
+ assert!(output.contains(&sha256_hex(sequence)));
+ assert!(!output.contains(call_id));
+ assert!(!output.contains(trace_id));
+ assert!(!output.contains(sequence));
+ assert_eq!(observer_starts, 0);
+ }
+
+ #[test]
+ fn controlled_fixture_probe_observed_and_failure_enums_are_stable() {
+ assert_eq!(
+ controlled_fixture_ack_classification(Ok(())),
+ ("observed", None, true)
+ );
+ assert_eq!(
+ controlled_fixture_ack_classification(Err("timeout")),
+ ("timeout", Some("expired"), false)
+ );
+ for reason in [
+ "missing_attributes",
+ "wrong_source",
+ "missing_sequence",
+ "wrong_sequence",
+ "wrong_participant",
+ ] {
+ assert_eq!(
+ controlled_fixture_ack_classification(Err(reason)),
+ ("rejected", Some(reason), false)
+ );
+ }
+ assert_eq!(
+ controlled_fixture_ack_classification(Err("unclassified")),
+ ("rejected", Some("unknown"), false)
+ );
+ }
+
+ #[tokio::test]
+ async fn production_attribute_observation_accepts_server_visibility_within_probe_ttl() {
+ let expected = HashMap::from([
+ (
+ "inputSourceCategory".to_string(),
+ "controlled_fixture".to_string(),
+ ),
+ (
+ "clientFixtureSequence".to_string(),
+ "fixture-01".to_string(),
+ ),
+ ]);
+ let mut reads = 0;
+ let result = observe_controlled_fixture_attributes(
+ Instant::now() + CONTROLLED_FIXTURE_PROBE_TTL,
+ "user-1",
+ Some("user-1"),
+ "fixture-01",
+ || {
+ reads += 1;
+ if reads <= 3 {
+ HashMap::new()
+ } else {
+ expected.clone()
+ }
+ },
+ )
+ .await;
+ assert_eq!(result, Ok(()));
+ assert_eq!(reads, 4);
+ }
+
+ #[tokio::test]
+ async fn production_attribute_observation_rejects_wrong_sequence_without_audio_effect() {
+ let attributes = HashMap::from([
+ (
+ "inputSourceCategory".to_string(),
+ "controlled_fixture".to_string(),
+ ),
+ (
+ "clientFixtureSequence".to_string(),
+ "fixture-02".to_string(),
+ ),
+ ]);
+ let result = observe_controlled_fixture_attributes(
+ Instant::now() + CONTROLLED_FIXTURE_PROBE_TTL,
+ "user-1",
+ Some("user-1"),
+ "fixture-01",
+ || attributes.clone(),
+ )
+ .await;
+ let mut observer_starts = 0;
+ assert_eq!(result, Err("wrong_sequence"));
+ assert!(!start_observer_after_controlled_fixture_probe(
+ true,
+ Some(result.is_ok()),
+ || observer_starts += 1,
+ ));
+ assert_eq!(observer_starts, 0);
+ }
+
+ #[test]
+ fn controlled_fixture_ack_payload_is_reliable_and_redacted() {
+ let ack = ControlledFixtureAttributeAck {
+ message_type: CONTROLLED_FIXTURE_ACK_TOPIC,
+ protocol_version: CONTROLLED_FIXTURE_PROTOCOL_VERSION,
+ call_id_hash: sha256_hex("call-1"),
+ call_trace_id_hash: sha256_hex("trace-1"),
+ generation: CONTROLLED_FIXTURE_GENERATION,
+ client_fixture_sequence: "fixture-01".to_string(),
+ result: "observed",
+ input_source_category: Some("controlled_fixture"),
+ reject_reason: None,
+ };
+ let encoded = serde_json::to_vec(&ack).unwrap();
+ let decoded: serde_json::Value = serde_json::from_slice(&encoded).unwrap();
+ assert_eq!(decoded["type"], CONTROLLED_FIXTURE_ACK_TOPIC);
+ assert_eq!(
+ decoded["protocolVersion"],
+ CONTROLLED_FIXTURE_PROTOCOL_VERSION
+ );
+ assert_eq!(decoded["result"], "observed");
+ assert!(decoded.get("callId").is_none());
+ assert!(decoded.get("traceId").is_none());
+ assert!(decoded.get("participantIdentity").is_none());
+ assert!(decoded.get("audio").is_none());
+ }
+
+ #[test]
+ fn controlled_fixture_probe_does_not_create_session_or_audio_side_effects() {
+ let attributes = std::collections::HashMap::new();
+ assert_eq!(
+ classify_controlled_fixture_attributes(
+ "user-1",
+ Some("user-1"),
+ &attributes,
+ "fixture-01"
+ ),
+ Err("missing_attributes")
+ );
+ assert_eq!(
+ CONTROLLED_FIXTURE_PROBE_TOPIC,
+ "controlled_fixture_attribute_probe"
+ );
+ assert_eq!(
+ CONTROLLED_FIXTURE_ACK_TOPIC,
+ "controlled_fixture_attribute_ack"
+ );
+ }
+
+ #[test]
+ fn controlled_fixture_probe_must_be_observed_before_audio_observer() {
+ assert!(controlled_fixture_observer_gate(false, None));
+ assert!(controlled_fixture_observer_gate(true, Some(true)));
+ assert!(!controlled_fixture_observer_gate(true, Some(false)));
+ assert!(!controlled_fixture_observer_gate(true, None));
+ }
+
+ #[test]
+ fn production_event_order_probe_then_track_publishes_ack_before_observer() {
+ let attributes = HashMap::from([
+ (
+ "inputSourceCategory".to_string(),
+ "controlled_fixture".to_string(),
+ ),
+ (
+ "clientFixtureSequence".to_string(),
+ "fixture-01".to_string(),
+ ),
+ ]);
+ let effects = drive_pre_audio_order_test_seam(&[
+ PreAudioOrderEvent::DataReceived {
+ sender: "user-1".to_string(),
+ sequence: "fixture-01".to_string(),
+ },
+ PreAudioOrderEvent::TrackSubscribed {
+ participant: "user-1".to_string(),
+ attributes,
+ },
+ ]);
+ assert_eq!(effects, ["ack_observed", "observer_started"]);
+ }
+
+ #[test]
+ fn production_event_order_negative_probe_has_no_observer_or_session_effect() {
+ let mut invalid = HashMap::new();
+ invalid.insert(
+ "inputSourceCategory".to_string(),
+ "ordinary_mic".to_string(),
+ );
+ let effects = drive_pre_audio_order_test_seam(&[
+ PreAudioOrderEvent::DataReceived {
+ sender: "user-1".to_string(),
+ sequence: "fixture-01".to_string(),
+ },
+ PreAudioOrderEvent::TrackSubscribed {
+ participant: "user-1".to_string(),
+ attributes: invalid,
+ },
+ ]);
+ assert!(effects.is_empty());
+ }
+
+ #[test]
+ fn production_audio_branch_orders_probe_before_spawn_callsite() {
+ let source = include_str!("main.rs");
+ let branch = source
+ .find("RoomEvent::TrackSubscribed {\n track: RemoteTrack::Audio")
+ .expect("audio TrackSubscribed production branch");
+ let branch_source = &source[branch..];
+ let probe = branch_source
+ .find("let probe_result = process_controlled_fixture_probe")
+ .expect("probe must be processed in audio branch");
+ let spawn = branch_source
+ .find("start_observer_after_controlled_fixture_probe")
+ .expect("spawn must use shared order entry");
+ assert!(
+ probe < spawn,
+ "probe must precede shared observer spawn entry"
+ );
+ }
+}
--
Gitblit v1.9.3