| | |
| | | mod asr_realtime; |
| | | mod audio; |
| | | mod service; |
| | | |
| | | use std::{ |
| | | borrow::Cow, |
| | | collections::HashSet, |
| | | env, fs, |
| | | future::Future, |
| | | path::{Path, PathBuf}, |
| | | sync::{ |
| | | Arc, |
| | |
| | | }; |
| | | |
| | | use anyhow::{Context, Result, anyhow}; |
| | | use asr_realtime::{ |
| | | AudioIngressMetadata, RealtimeAsrConfig, RealtimeAsrOutcome, RealtimeAsrUpload, |
| | | }; |
| | | use audio::{AudioDiagnostics, load_pre_recorded_frames}; |
| | | use base64::{Engine as _, engine::general_purpose}; |
| | | use futures_util::StreamExt; |
| | |
| | | options::TrackPublishOptions, |
| | | prelude::{ |
| | | DataPacket, LocalAudioTrack, LocalTrack, ParticipantIdentity, RemoteAudioTrack, |
| | | RemoteTrack, Room, RoomEvent, RoomOptions, |
| | | RemoteParticipant, RemoteTrack, Room, RoomEvent, RoomOptions, |
| | | }, |
| | | }; |
| | | use reqwest::Client; |
| | | use serde::{Deserialize, Serialize}; |
| | | use serde_json::json; |
| | | use tokio::time::{sleep, sleep_until}; |
| | | use tokio::{sync::mpsc::UnboundedReceiver, task::JoinHandle}; |
| | | use sha2::{Digest, Sha256}; |
| | | use tokio::time::{sleep, sleep_until, timeout}; |
| | | use tokio::{ |
| | | sync::{mpsc, mpsc::UnboundedReceiver, watch}, |
| | | task::JoinHandle, |
| | | }; |
| | | use tracing::{info, warn}; |
| | | |
| | | const USER_AUDIO_SAMPLE_RATE_HZ: u32 = 48_000; |
| | | const USER_AUDIO_NUM_CHANNELS: u16 = 1; |
| | | const INBOUND_AUDIO_QUEUE_CAPACITY: usize = 100; |
| | | const INBOUND_AUDIO_DROP_LOG_INTERVAL: u64 = 100; |
| | | const AUDIO_DRAIN_STOP_GRACE: Duration = Duration::from_secs(2); |
| | | const DEFAULT_BOT_AUDIO_PROFILE: &str = "pcm-16k"; |
| | | const LIVEKIT_48K_SAMPLE_RATE_HZ: u32 = 48_000; |
| | | const PCM_16K_SAMPLE_RATE_HZ: u32 = 16_000; |
| | | const BOT_NUM_CHANNELS: u16 = 1; |
| | | const TRACK_NAME: &str = "bot-main-audio"; |
| | | const STREAM_TIMING_VERSION: u32 = 1; |
| | | const STREAM_TIMING_FIRST_SEGMENT: u64 = 1; |
| | | const STREAM_TIMING_FIRST_CHUNK: u64 = 1; |
| | | const STREAM_TIMING_MAX_ELAPSED_MS: u64 = 5_000; |
| | | const STREAM_TIMING_SOURCE: &str = "stream_anchor_monotonic"; |
| | | const CONTROLLED_FIXTURE_PROBE_TOPIC: &str = "controlled_fixture_attribute_probe"; |
| | | const CONTROLLED_FIXTURE_ACK_TOPIC: &str = "controlled_fixture_attribute_ack"; |
| | | const CONTROLLED_FIXTURE_PROTOCOL_VERSION: u64 = 1; |
| | | const CONTROLLED_FIXTURE_GENERATION: u64 = 1; |
| | | const CONTROLLED_FIXTURE_PROBE_RECHECK_DELAY: Duration = Duration::from_millis(25); |
| | | const CONTROLLED_FIXTURE_PROBE_TTL: Duration = Duration::from_millis(250); |
| | | const CONTROLLED_FIXTURE_POST_EXPIRY_WINDOW: Duration = Duration::from_millis(2_000); |
| | | |
| | | const CONTROLLED_FIXTURE_ACK_RESULTS: [&str; 4] = |
| | | ["observed", "rejected", "timeout", "publish_failed"]; |
| | | const CONTROLLED_FIXTURE_REJECT_REASONS: [&str; 10] = [ |
| | | "missing_attributes", |
| | | "wrong_source", |
| | | "missing_sequence", |
| | | "wrong_sequence", |
| | | "wrong_participant", |
| | | "expired", |
| | | "duplicate_or_old_sequence", |
| | | "no_current_participant", |
| | | "ack_publish_failed", |
| | | "unknown", |
| | | ]; |
| | | |
| | | #[derive(Debug, Deserialize)] |
| | | #[serde(rename_all = "camelCase")] |
| | | struct ControlledFixtureAttributeProbe { |
| | | #[serde(rename = "type")] |
| | | message_type: String, |
| | | protocol_version: u64, |
| | | call_id_hash: String, |
| | | call_trace_id_hash: String, |
| | | generation: u64, |
| | | client_fixture_sequence: String, |
| | | } |
| | | |
| | | #[derive(Debug, Serialize)] |
| | | #[serde(rename_all = "camelCase")] |
| | | struct ControlledFixtureAttributeAck { |
| | | #[serde(rename = "type")] |
| | | message_type: &'static str, |
| | | protocol_version: u64, |
| | | call_id_hash: String, |
| | | call_trace_id_hash: String, |
| | | generation: u64, |
| | | client_fixture_sequence: String, |
| | | result: &'static str, |
| | | #[serde(skip_serializing_if = "Option::is_none")] |
| | | input_source_category: Option<&'static str>, |
| | | #[serde(skip_serializing_if = "Option::is_none")] |
| | | reject_reason: Option<&'static str>, |
| | | } |
| | | |
| | | #[derive(Clone, Debug)] |
| | | struct PendingControlledFixtureProbe { |
| | | sender: ParticipantIdentity, |
| | | call_id_hash: String, |
| | | call_trace_id_hash: String, |
| | | generation: u64, |
| | | sequence: String, |
| | | received_at: Instant, |
| | | expires_at: Instant, |
| | | } |
| | | |
| | | #[derive(Debug, PartialEq, Eq)] |
| | | struct ControlledFixtureVisibilityEvidence { |
| | | first_visible_bucket: &'static str, |
| | | visibility_source: &'static str, |
| | | binding_matched: bool, |
| | | } |
| | | |
| | | #[derive(Debug, PartialEq, Eq)] |
| | | struct ControlledFixtureAckPublishOutcome { |
| | | observed: bool, |
| | | published: bool, |
| | | } |
| | | |
| | | fn sha256_hex(value: &str) -> String { |
| | | let mut digest = Sha256::new(); |
| | | digest.update(value.as_bytes()); |
| | | digest |
| | | .finalize() |
| | | .iter() |
| | | .map(|byte| format!("{byte:02x}")) |
| | | .collect() |
| | | } |
| | | |
| | | fn controlled_fixture_ack_classification( |
| | | decision: Result<(), &'static str>, |
| | | ) -> (&'static str, Option<&'static str>, bool) { |
| | | match decision { |
| | | Ok(()) => ("observed", None, true), |
| | | Err("timeout") => ("timeout", Some("expired"), false), |
| | | Err(reason) if CONTROLLED_FIXTURE_REJECT_REASONS.contains(&reason) => { |
| | | ("rejected", Some(reason), false) |
| | | } |
| | | Err(_) => ("rejected", Some("unknown"), false), |
| | | } |
| | | } |
| | | |
| | | fn record_controlled_fixture_probe_event( |
| | | runtime_call_id: &str, |
| | | runtime_trace_id: &str, |
| | | stage: &'static str, |
| | | call_id_hash: &str, |
| | | trace_id_hash: &str, |
| | | generation: u64, |
| | | sequence: &str, |
| | | observed: bool, |
| | | ack_result: Option<&'static str>, |
| | | reject_reason: Option<&'static str>, |
| | | ) { |
| | | debug_assert!(ack_result.is_none_or(|value| CONTROLLED_FIXTURE_ACK_RESULTS.contains(&value))); |
| | | debug_assert!( |
| | | reject_reason.is_none_or(|value| CONTROLLED_FIXTURE_REJECT_REASONS.contains(&value)) |
| | | ); |
| | | println!( |
| | | "{}", |
| | | controlled_fixture_probe_event( |
| | | runtime_call_id, |
| | | runtime_trace_id, |
| | | stage, |
| | | call_id_hash, |
| | | trace_id_hash, |
| | | generation, |
| | | sequence, |
| | | observed, |
| | | ack_result, |
| | | reject_reason, |
| | | ) |
| | | ); |
| | | } |
| | | |
| | | fn controlled_fixture_probe_event( |
| | | runtime_call_id: &str, |
| | | runtime_trace_id: &str, |
| | | stage: &'static str, |
| | | call_id_hash: &str, |
| | | trace_id_hash: &str, |
| | | generation: u64, |
| | | sequence: &str, |
| | | observed: bool, |
| | | ack_result: Option<&'static str>, |
| | | reject_reason: Option<&'static str>, |
| | | ) -> serde_json::Value { |
| | | json!({ |
| | | "type": "cv_activity", |
| | | "callId": runtime_call_id, |
| | | "traceId": runtime_trace_id, |
| | | "turnId": null, |
| | | "eventName": "controlled_fixture_attribute_probe", |
| | | "eventWallTimeMs": current_time_millis(), |
| | | "result": "ok", |
| | | "reasonCode": null, |
| | | "retryable": null, |
| | | "extension": { |
| | | "stage": stage, |
| | | "observed": observed, |
| | | "ack_result": ack_result, |
| | | "reject_reason": reject_reason, |
| | | "call_id_hash": call_id_hash, |
| | | "trace_id_hash": trace_id_hash, |
| | | "generation": generation, |
| | | "sequence_hash": sha256_hex(sequence), |
| | | }, |
| | | }) |
| | | } |
| | | |
| | | fn record_controlled_fixture_attribute_decision( |
| | | decision: Result<(), &'static str>, |
| | | runtime_call_id: &str, |
| | | runtime_trace_id: &str, |
| | | call_id_hash: &str, |
| | | trace_id_hash: &str, |
| | | generation: u64, |
| | | sequence: &str, |
| | | ) -> (&'static str, Option<&'static str>, bool) { |
| | | let classification = controlled_fixture_ack_classification(decision); |
| | | record_controlled_fixture_probe_event( |
| | | runtime_call_id, |
| | | runtime_trace_id, |
| | | "attributes_classified", |
| | | call_id_hash, |
| | | trace_id_hash, |
| | | generation, |
| | | sequence, |
| | | classification.2, |
| | | Some(classification.0), |
| | | classification.1, |
| | | ); |
| | | classification |
| | | } |
| | | |
| | | async fn complete_controlled_fixture_ack_publish<F, E>( |
| | | publish: F, |
| | | runtime_call_id: &str, |
| | | runtime_trace_id: &str, |
| | | observed: bool, |
| | | ack_result: &'static str, |
| | | reject_reason: Option<&'static str>, |
| | | call_id_hash: &str, |
| | | trace_id_hash: &str, |
| | | generation: u64, |
| | | sequence: &str, |
| | | acknowledged_probe_sequences: &mut HashSet<String>, |
| | | ) -> ControlledFixtureAckPublishOutcome |
| | | where |
| | | F: Future<Output = Result<(), E>>, |
| | | { |
| | | if publish.await.is_ok() { |
| | | record_controlled_fixture_probe_event( |
| | | runtime_call_id, |
| | | runtime_trace_id, |
| | | "ack_publish_completed", |
| | | call_id_hash, |
| | | trace_id_hash, |
| | | generation, |
| | | sequence, |
| | | observed, |
| | | Some(ack_result), |
| | | reject_reason, |
| | | ); |
| | | acknowledged_probe_sequences.insert(sequence.to_string()); |
| | | ControlledFixtureAckPublishOutcome { |
| | | observed, |
| | | published: true, |
| | | } |
| | | } else { |
| | | record_controlled_fixture_probe_event( |
| | | runtime_call_id, |
| | | runtime_trace_id, |
| | | "ack_publish_completed", |
| | | call_id_hash, |
| | | trace_id_hash, |
| | | generation, |
| | | sequence, |
| | | false, |
| | | Some("publish_failed"), |
| | | Some("ack_publish_failed"), |
| | | ); |
| | | ControlledFixtureAckPublishOutcome { |
| | | observed: false, |
| | | published: false, |
| | | } |
| | | } |
| | | } |
| | | |
| | | fn controlled_fixture_ack_from_probe( |
| | | probe: &PendingControlledFixtureProbe, |
| | | observed: bool, |
| | | reject_reason: Option<&'static str>, |
| | | ) -> ControlledFixtureAttributeAck { |
| | | ControlledFixtureAttributeAck { |
| | | message_type: CONTROLLED_FIXTURE_ACK_TOPIC, |
| | | protocol_version: CONTROLLED_FIXTURE_PROTOCOL_VERSION, |
| | | call_id_hash: probe.call_id_hash.clone(), |
| | | call_trace_id_hash: probe.call_trace_id_hash.clone(), |
| | | generation: probe.generation, |
| | | client_fixture_sequence: probe.sequence.clone(), |
| | | result: if observed { "observed" } else { "rejected" }, |
| | | input_source_category: observed.then_some("controlled_fixture"), |
| | | reject_reason, |
| | | } |
| | | } |
| | | |
| | | fn controlled_fixture_probe( |
| | | payload: &[u8], |
| | | call_id: &str, |
| | | trace_id: &str, |
| | | sender: &ParticipantIdentity, |
| | | expected_participant: Option<&str>, |
| | | ) -> Option<PendingControlledFixtureProbe> { |
| | | if !is_bound_user_participant(sender.as_str(), expected_participant) { |
| | | return None; |
| | | } |
| | | let probe: ControlledFixtureAttributeProbe = serde_json::from_slice(payload).ok()?; |
| | | if probe.message_type != CONTROLLED_FIXTURE_PROBE_TOPIC |
| | | || probe.protocol_version != CONTROLLED_FIXTURE_PROTOCOL_VERSION |
| | | || probe.generation != CONTROLLED_FIXTURE_GENERATION |
| | | || probe.call_id_hash != sha256_hex(call_id) |
| | | || probe.call_trace_id_hash != sha256_hex(trace_id) |
| | | || probe.client_fixture_sequence.trim().is_empty() |
| | | { |
| | | return None; |
| | | } |
| | | let received_at = Instant::now(); |
| | | Some(PendingControlledFixtureProbe { |
| | | sender: sender.clone(), |
| | | call_id_hash: probe.call_id_hash, |
| | | call_trace_id_hash: probe.call_trace_id_hash, |
| | | generation: probe.generation, |
| | | sequence: probe.client_fixture_sequence, |
| | | received_at, |
| | | expires_at: received_at + CONTROLLED_FIXTURE_PROBE_TTL, |
| | | }) |
| | | } |
| | | |
| | | fn controlled_fixture_visibility_bucket(elapsed: Duration) -> &'static str { |
| | | if elapsed <= Duration::from_millis(250) { |
| | | "lte_250ms" |
| | | } else if elapsed <= Duration::from_millis(500) { |
| | | "250_500ms" |
| | | } else if elapsed <= Duration::from_millis(1_000) { |
| | | "500_1000ms" |
| | | } else { |
| | | "1000_2000ms" |
| | | } |
| | | } |
| | | |
| | | async fn observe_controlled_fixture_post_expiry<F>( |
| | | received_at: Instant, |
| | | observation_deadline: Instant, |
| | | actual_participant: &str, |
| | | expected_participant: Option<&str>, |
| | | requested_sequence: &str, |
| | | lifecycle_active: Arc<AtomicBool>, |
| | | mut read_attributes: F, |
| | | ) -> Option<ControlledFixtureVisibilityEvidence> |
| | | where |
| | | F: FnMut() -> std::collections::HashMap<String, String>, |
| | | { |
| | | loop { |
| | | if !lifecycle_active.load(Ordering::Acquire) { |
| | | return None; |
| | | } |
| | | let now = Instant::now(); |
| | | let decision = classify_controlled_fixture_attributes( |
| | | actual_participant, |
| | | expected_participant, |
| | | &read_attributes(), |
| | | requested_sequence, |
| | | ); |
| | | match decision { |
| | | Ok(()) => { |
| | | return Some(ControlledFixtureVisibilityEvidence { |
| | | first_visible_bucket: controlled_fixture_visibility_bucket( |
| | | now.saturating_duration_since(received_at), |
| | | ), |
| | | visibility_source: "participant_attributes_poll", |
| | | binding_matched: true, |
| | | }); |
| | | } |
| | | Err("missing_attributes") if now < observation_deadline => {} |
| | | Err("missing_attributes") => { |
| | | return Some(ControlledFixtureVisibilityEvidence { |
| | | first_visible_bucket: "never_visible_within_observation_window", |
| | | visibility_source: "participant_attributes_poll", |
| | | binding_matched: true, |
| | | }); |
| | | } |
| | | Err(_) => return None, |
| | | } |
| | | sleep(CONTROLLED_FIXTURE_PROBE_RECHECK_DELAY).await; |
| | | } |
| | | } |
| | | |
| | | fn controlled_fixture_visibility_event( |
| | | runtime_call_id: &str, |
| | | runtime_trace_id: &str, |
| | | probe: &PendingControlledFixtureProbe, |
| | | evidence: &ControlledFixtureVisibilityEvidence, |
| | | ) -> serde_json::Value { |
| | | json!({ |
| | | "type": "cv_activity", |
| | | "callId": runtime_call_id, |
| | | "traceId": runtime_trace_id, |
| | | "turnId": null, |
| | | "eventName": "controlled_fixture_attribute_probe", |
| | | "eventWallTimeMs": current_time_millis(), |
| | | "result": "ok", |
| | | "reasonCode": null, |
| | | "retryable": null, |
| | | "extension": { |
| | | "stage": "post_expiry_visibility", |
| | | "first_visible_bucket": evidence.first_visible_bucket, |
| | | "visibility_source": evidence.visibility_source, |
| | | "binding_matched": evidence.binding_matched, |
| | | "call_id_hash": probe.call_id_hash, |
| | | "trace_id_hash": probe.call_trace_id_hash, |
| | | "generation": probe.generation, |
| | | "sequence_hash": sha256_hex(&probe.sequence), |
| | | "evidence_count": 1, |
| | | }, |
| | | }) |
| | | } |
| | | |
| | | fn spawn_controlled_fixture_post_expiry_observation( |
| | | probe: PendingControlledFixtureProbe, |
| | | participant: RemoteParticipant, |
| | | expected_participant: Option<String>, |
| | | lifecycle_active: Arc<AtomicBool>, |
| | | runtime_call_id: String, |
| | | runtime_trace_id: String, |
| | | ) { |
| | | tokio::spawn(async move { |
| | | let participant_identity = participant.identity().to_string(); |
| | | let evidence = observe_controlled_fixture_post_expiry( |
| | | probe.received_at, |
| | | probe.received_at + CONTROLLED_FIXTURE_POST_EXPIRY_WINDOW, |
| | | &participant_identity, |
| | | expected_participant.as_deref(), |
| | | &probe.sequence, |
| | | lifecycle_active.clone(), |
| | | || participant.attributes(), |
| | | ) |
| | | .await; |
| | | if lifecycle_active.load(Ordering::Acquire) { |
| | | if let Some(evidence) = evidence { |
| | | println!( |
| | | "{}", |
| | | controlled_fixture_visibility_event( |
| | | &runtime_call_id, |
| | | &runtime_trace_id, |
| | | &probe, |
| | | &evidence, |
| | | ) |
| | | ); |
| | | } |
| | | } |
| | | }); |
| | | } |
| | | |
| | | fn classify_controlled_fixture_attributes( |
| | | actual_participant: &str, |
| | | expected_participant: Option<&str>, |
| | | attributes: &std::collections::HashMap<String, String>, |
| | | requested_sequence: &str, |
| | | ) -> Result<(), &'static str> { |
| | | if !is_bound_user_participant(actual_participant, expected_participant) { |
| | | return Err("wrong_participant"); |
| | | } |
| | | let source = attributes |
| | | .get("inputSourceCategory") |
| | | .map(String::as_str) |
| | | .unwrap_or(""); |
| | | if source != "controlled_fixture" { |
| | | return Err(if source.is_empty() { |
| | | "missing_attributes" |
| | | } else { |
| | | "wrong_source" |
| | | }); |
| | | } |
| | | let sequence = attributes |
| | | .get("clientFixtureSequence") |
| | | .map(String::as_str) |
| | | .unwrap_or(""); |
| | | if sequence.trim().is_empty() { |
| | | return Err("missing_sequence"); |
| | | } |
| | | if sequence != requested_sequence { |
| | | return Err("wrong_sequence"); |
| | | } |
| | | Ok(()) |
| | | } |
| | | |
| | | async fn observe_controlled_fixture_attributes<F>( |
| | | expires_at: Instant, |
| | | actual_participant: &str, |
| | | expected_participant: Option<&str>, |
| | | requested_sequence: &str, |
| | | mut read_attributes: F, |
| | | ) -> Result<(), &'static str> |
| | | where |
| | | F: FnMut() -> std::collections::HashMap<String, String>, |
| | | { |
| | | loop { |
| | | if Instant::now() > expires_at { |
| | | return Err("timeout"); |
| | | } |
| | | let decision = classify_controlled_fixture_attributes( |
| | | actual_participant, |
| | | expected_participant, |
| | | &read_attributes(), |
| | | requested_sequence, |
| | | ); |
| | | if decision.is_ok() || !matches!(decision, Err("missing_attributes")) { |
| | | return decision; |
| | | } |
| | | let Some(next_check) = Instant::now().checked_add(CONTROLLED_FIXTURE_PROBE_RECHECK_DELAY) |
| | | else { |
| | | return Err("timeout"); |
| | | }; |
| | | if next_check > expires_at { |
| | | return Err("timeout"); |
| | | } |
| | | sleep(CONTROLLED_FIXTURE_PROBE_RECHECK_DELAY).await; |
| | | } |
| | | } |
| | | |
| | | fn controlled_fixture_observer_gate(pending_probe: bool, probe_result: Option<bool>) -> bool { |
| | | !pending_probe || probe_result == Some(true) |
| | | } |
| | | |
| | | fn start_observer_after_controlled_fixture_probe<F>( |
| | | pending_probe: bool, |
| | | probe_result: Option<bool>, |
| | | spawn: F, |
| | | ) -> bool |
| | | where |
| | | F: FnOnce(), |
| | | { |
| | | if !controlled_fixture_observer_gate(pending_probe, probe_result) { |
| | | return false; |
| | | } |
| | | spawn(); |
| | | true |
| | | } |
| | | |
| | | #[tokio::main(flavor = "multi_thread")] |
| | | async fn main() -> Result<()> { |
| | |
| | | runtime_turn_bridge_url: Option<String>, |
| | | runtime_turn_bridge_token: Option<String>, |
| | | runtime_turn_bridge_mode: String, |
| | | runtime_asr_stream_enabled: bool, |
| | | runtime_asr_stream_url: Option<String>, |
| | | runtime_asr_realtime_enabled: bool, |
| | | runtime_asr_realtime_url: Option<String>, |
| | | runtime_asr_realtime_chunk_duration_ms: u64, |
| | | runtime_turn_artifact_dir: Option<String>, |
| | | runtime_session_nonce: Option<String>, |
| | | user_audio_observer_enabled: bool, |
| | |
| | | bridge_url: Option<String>, |
| | | bridge_token: Option<String>, |
| | | bridge_mode: String, |
| | | asr_stream_enabled: bool, |
| | | asr_stream_url: Option<String>, |
| | | asr_realtime_enabled: bool, |
| | | asr_realtime_url: Option<String>, |
| | | asr_realtime_chunk_duration_ms: u64, |
| | | artifact_dir: Option<String>, |
| | | runtime_session_nonce: Option<String>, |
| | | audio_debug_dump_dir: Option<String>, |
| | |
| | | bridge_url: config.runtime_turn_bridge_url.clone(), |
| | | bridge_token: config.runtime_turn_bridge_token.clone(), |
| | | bridge_mode: config.runtime_turn_bridge_mode.clone(), |
| | | asr_stream_enabled: config.runtime_asr_stream_enabled, |
| | | asr_stream_url: config.runtime_asr_stream_url.clone(), |
| | | asr_realtime_enabled: config.runtime_asr_realtime_enabled, |
| | | asr_realtime_url: config.runtime_asr_realtime_url.clone(), |
| | | asr_realtime_chunk_duration_ms: config.runtime_asr_realtime_chunk_duration_ms, |
| | | artifact_dir: config.runtime_turn_artifact_dir.clone(), |
| | | runtime_session_nonce: config.runtime_session_nonce.clone(), |
| | | audio_debug_dump_dir: config.audio_debug_dump_dir.clone(), |
| | |
| | | .as_deref() |
| | | .is_some_and(|value| value.trim_end_matches('/').ends_with("/stream")) |
| | | } |
| | | |
| | | fn is_asr_stream_ready(&self) -> bool { |
| | | self.asr_stream_enabled |
| | | && self |
| | | .asr_stream_url |
| | | .as_ref() |
| | | .is_some_and(|value| !value.is_empty()) |
| | | && self |
| | | .bridge_token |
| | | .as_ref() |
| | | .is_some_and(|value| !value.is_empty()) |
| | | && self |
| | | .runtime_session_nonce |
| | | .as_ref() |
| | | .is_some_and(|value| !value.is_empty()) |
| | | } |
| | | |
| | | fn realtime_asr_config(&self) -> RealtimeAsrConfig { |
| | | RealtimeAsrConfig { |
| | | enabled: self.asr_realtime_enabled, |
| | | url: self.asr_realtime_url.clone(), |
| | | runtime_token: self.bridge_token.clone(), |
| | | runtime_session_nonce: self.runtime_session_nonce.clone(), |
| | | chunk_duration_ms: self.asr_realtime_chunk_duration_ms, |
| | | } |
| | | } |
| | | } |
| | | |
| | | impl Config { |
| | |
| | | runtime_turn_bridge_token: optional_env("CV_RUNTIME_TURN_BRIDGE_TOKEN"), |
| | | runtime_turn_bridge_mode: env::var("CV_RUNTIME_TURN_BRIDGE_MODE") |
| | | .unwrap_or_else(|_| "json".to_string()), |
| | | runtime_asr_stream_enabled: bool_env("CV_RUNTIME_ASR_STREAM_ENABLED", false), |
| | | runtime_asr_stream_url: optional_env("CV_RUNTIME_ASR_STREAM_URL"), |
| | | runtime_asr_realtime_enabled: bool_env("CV_RUNTIME_ASR_REALTIME_ENABLED", false), |
| | | runtime_asr_realtime_url: optional_env("CV_RUNTIME_ASR_REALTIME_URL"), |
| | | runtime_asr_realtime_chunk_duration_ms: u64_env( |
| | | "CV_RUNTIME_ASR_REALTIME_CHUNK_DURATION_MS", |
| | | 200, |
| | | ), |
| | | runtime_turn_artifact_dir: optional_env("CV_RUNTIME_TURN_ARTIFACT_DIR"), |
| | | runtime_session_nonce: optional_env("CV_RUNTIME_SESSION_NONCE"), |
| | | user_audio_observer_enabled: bool_env("CV_ENABLE_USER_AUDIO_OBSERVER", true), |
| | |
| | | "traceId": trace_id, |
| | | "turnId": turn_id, |
| | | "eventName": event_name, |
| | | "eventWallTimeMs": current_time_millis(), |
| | | "result": result, |
| | | "reasonCode": reason_code, |
| | | "retryable": retryable, |
| | | "extension": extension, |
| | | }); |
| | | println!("{payload}"); |
| | | } |
| | | |
| | | fn emit_anchored_activity( |
| | | call_id: &str, |
| | | trace_id: &str, |
| | | turn_id: &str, |
| | | event_name: &str, |
| | | marker: &RuntimeTurnStreamTimingMarker, |
| | | extension: serde_json::Value, |
| | | ) { |
| | | let payload = json!({ |
| | | "type": "cv_activity", |
| | | "callId": call_id, |
| | | "traceId": trace_id, |
| | | "turnId": turn_id, |
| | | "eventName": event_name, |
| | | "eventWallTimeMs": current_time_millis(), |
| | | "serverDeltaMs": marker.server_delta_ms, |
| | | "serverDeltaSource": STREAM_TIMING_SOURCE, |
| | | "result": "ok", |
| | | "reasonCode": null, |
| | | "retryable": null, |
| | | "extension": marker.extension_with(extension), |
| | | }); |
| | | println!("{payload}"); |
| | | } |
| | |
| | | let enabled = config.user_audio_observer_enabled; |
| | | let simple_vad_enabled = config.simple_vad_enabled; |
| | | let simple_vad_config = config.simple_vad_config.clone(); |
| | | let user_participant_identity = config.user_participant_identity.clone(); |
| | | let turn_bridge_config = TurnBridgeConfig::from_config(config); |
| | | |
| | | tokio::spawn(async move { |
| | |
| | | turn_bridge_config, |
| | | http, |
| | | sink, |
| | | user_participant_identity, |
| | | ) |
| | | .await; |
| | | }) |
| | | } |
| | | |
| | | fn is_bound_user_participant(identity: &str, expected: Option<&str>) -> bool { |
| | | expected.is_none_or(|value| identity == value) |
| | | } |
| | | |
| | | async fn observe_user_audio_events( |
| | |
| | | turn_bridge_config: TurnBridgeConfig, |
| | | http: Client, |
| | | sink: Arc<BotAudioOutputSink>, |
| | | user_participant_identity: Option<String>, |
| | | ) { |
| | | info!( |
| | | call_id = %call_id, |
| | |
| | | "runtime helper user_track_subscribe_requested" |
| | | ); |
| | | |
| | | let mut current_user_participant: Option<RemoteParticipant> = None; |
| | | let mut pending_probe: Option<PendingControlledFixtureProbe> = None; |
| | | let mut acknowledged_probe_sequences = HashSet::new(); |
| | | let controlled_fixture_lifecycle_active = Arc::new(AtomicBool::new(true)); |
| | | |
| | | while let Some(event) = events.recv().await { |
| | | match event { |
| | | RoomEvent::TrackSubscribed { |
| | |
| | | publication: _, |
| | | participant, |
| | | } => { |
| | | if !is_bound_user_participant( |
| | | &participant.identity().to_string(), |
| | | user_participant_identity.as_deref(), |
| | | ) { |
| | | warn!(call_id = %call_id, trace_id = %trace_id, |
| | | metadata_status = "wrong_participant", |
| | | "runtime helper ignored non-user audio participant"); |
| | | continue; |
| | | } |
| | | let participant_alias = redact(&participant.identity().to_string()); |
| | | let track_sid_alias = redact(&track.sid().to_string()); |
| | | let track_name = track.name(); |
| | |
| | | track_source = %track_source, |
| | | "runtime helper user_track_subscribed" |
| | | ); |
| | | spawn_user_audio_frame_observer( |
| | | track, |
| | | call_id.clone(), |
| | | trace_id.clone(), |
| | | participant_alias, |
| | | track_sid_alias, |
| | | simple_vad_enabled, |
| | | simple_vad_config.clone(), |
| | | vad_enabled_gate.clone(), |
| | | turn_bridge_config.clone(), |
| | | http.clone(), |
| | | sink.clone(), |
| | | let pending_binding = pending_probe.clone(); |
| | | let participant_for_probe = participant.clone(); |
| | | let probe_result = process_controlled_fixture_probe( |
| | | &mut pending_probe, |
| | | &mut acknowledged_probe_sequences, |
| | | Some(&participant_for_probe), |
| | | &sink, |
| | | user_participant_identity.as_deref(), |
| | | &call_id, |
| | | &trace_id, |
| | | controlled_fixture_lifecycle_active.clone(), |
| | | ) |
| | | .await; |
| | | let observer_started = start_observer_after_controlled_fixture_probe( |
| | | pending_binding.is_some(), |
| | | probe_result, |
| | | || { |
| | | spawn_user_audio_frame_observer( |
| | | track, |
| | | call_id.clone(), |
| | | trace_id.clone(), |
| | | participant_alias, |
| | | track_sid_alias, |
| | | simple_vad_enabled, |
| | | simple_vad_config.clone(), |
| | | vad_enabled_gate.clone(), |
| | | turn_bridge_config.clone(), |
| | | http.clone(), |
| | | sink.clone(), |
| | | user_participant_identity.clone(), |
| | | participant, |
| | | ); |
| | | }, |
| | | ); |
| | | if let Some(probe) = pending_binding.as_ref() { |
| | | let (ack_result, reject_reason, observed) = match probe_result { |
| | | Some(true) => (Some("observed"), None, true), |
| | | Some(false) => (Some("rejected"), Some("unknown"), false), |
| | | None => (None, Some("no_current_participant"), false), |
| | | }; |
| | | record_controlled_fixture_probe_event( |
| | | &call_id, |
| | | &trace_id, |
| | | if observer_started { |
| | | "audio_observer_allowed" |
| | | } else { |
| | | "audio_observer_blocked" |
| | | }, |
| | | &probe.call_id_hash, |
| | | &probe.call_trace_id_hash, |
| | | probe.generation, |
| | | &probe.sequence, |
| | | observed, |
| | | ack_result, |
| | | reject_reason, |
| | | ); |
| | | } |
| | | if !observer_started { |
| | | warn!( |
| | | call_id = %call_id, |
| | | trace_id = %trace_id, |
| | | "runtime helper withheld audio observer until controlled fixture ACK" |
| | | ); |
| | | continue; |
| | | } |
| | | current_user_participant = Some(participant_for_probe); |
| | | } |
| | | RoomEvent::DataReceived { |
| | | payload, |
| | | topic: Some(topic), |
| | | participant: Some(sender), |
| | | .. |
| | | } if topic == CONTROLLED_FIXTURE_PROBE_TOPIC => { |
| | | if let Ok(probe) = |
| | | serde_json::from_slice::<ControlledFixtureAttributeProbe>(&payload) |
| | | { |
| | | if acknowledged_probe_sequences.contains(&probe.client_fixture_sequence) { |
| | | record_controlled_fixture_probe_event( |
| | | &call_id, |
| | | &trace_id, |
| | | "data_received", |
| | | &probe.call_id_hash, |
| | | &probe.call_trace_id_hash, |
| | | probe.generation, |
| | | &probe.client_fixture_sequence, |
| | | false, |
| | | Some("rejected"), |
| | | Some("duplicate_or_old_sequence"), |
| | | ); |
| | | continue; |
| | | } |
| | | } |
| | | pending_probe = controlled_fixture_probe( |
| | | &payload, |
| | | &call_id, |
| | | &trace_id, |
| | | &sender.identity(), |
| | | user_participant_identity.as_deref(), |
| | | ); |
| | | if let Some(probe) = pending_probe.as_ref() { |
| | | record_controlled_fixture_probe_event( |
| | | &call_id, |
| | | &trace_id, |
| | | "data_received", |
| | | &probe.call_id_hash, |
| | | &probe.call_trace_id_hash, |
| | | probe.generation, |
| | | &probe.sequence, |
| | | false, |
| | | None, |
| | | None, |
| | | ); |
| | | } |
| | | process_controlled_fixture_probe( |
| | | &mut pending_probe, |
| | | &mut acknowledged_probe_sequences, |
| | | current_user_participant.as_ref(), |
| | | &sink, |
| | | user_participant_identity.as_deref(), |
| | | &call_id, |
| | | &trace_id, |
| | | controlled_fixture_lifecycle_active.clone(), |
| | | ) |
| | | .await; |
| | | } |
| | | RoomEvent::TrackSubscribed { |
| | | track: RemoteTrack::Video(track), |
| | |
| | | _ => {} |
| | | } |
| | | } |
| | | controlled_fixture_lifecycle_active.store(false, Ordering::Release); |
| | | } |
| | | |
| | | async fn process_controlled_fixture_probe( |
| | | pending_probe: &mut Option<PendingControlledFixtureProbe>, |
| | | acknowledged_probe_sequences: &mut HashSet<String>, |
| | | participant: Option<&RemoteParticipant>, |
| | | sink: &BotAudioOutputSink, |
| | | expected_participant: Option<&str>, |
| | | runtime_call_id: &str, |
| | | runtime_trace_id: &str, |
| | | lifecycle_active: Arc<AtomicBool>, |
| | | ) -> Option<bool> { |
| | | let Some(probe) = pending_probe.take() else { |
| | | return None; |
| | | }; |
| | | if acknowledged_probe_sequences.contains(&probe.sequence) { |
| | | record_controlled_fixture_probe_event( |
| | | runtime_call_id, |
| | | runtime_trace_id, |
| | | "attributes_classified", |
| | | &probe.call_id_hash, |
| | | &probe.call_trace_id_hash, |
| | | probe.generation, |
| | | &probe.sequence, |
| | | false, |
| | | Some("rejected"), |
| | | Some("duplicate_or_old_sequence"), |
| | | ); |
| | | return Some(false); |
| | | } |
| | | if Instant::now() > probe.expires_at { |
| | | record_controlled_fixture_probe_event( |
| | | runtime_call_id, |
| | | runtime_trace_id, |
| | | "attributes_classified", |
| | | &probe.call_id_hash, |
| | | &probe.call_trace_id_hash, |
| | | probe.generation, |
| | | &probe.sequence, |
| | | false, |
| | | Some("timeout"), |
| | | Some("expired"), |
| | | ); |
| | | return Some(false); |
| | | } |
| | | let Some(participant) = participant else { |
| | | record_controlled_fixture_probe_event( |
| | | runtime_call_id, |
| | | runtime_trace_id, |
| | | "attributes_classified", |
| | | &probe.call_id_hash, |
| | | &probe.call_trace_id_hash, |
| | | probe.generation, |
| | | &probe.sequence, |
| | | false, |
| | | None, |
| | | Some("no_current_participant"), |
| | | ); |
| | | *pending_probe = Some(probe); |
| | | return None; |
| | | }; |
| | | if participant.identity() != probe.sender { |
| | | record_controlled_fixture_probe_event( |
| | | runtime_call_id, |
| | | runtime_trace_id, |
| | | "attributes_classified", |
| | | &probe.call_id_hash, |
| | | &probe.call_trace_id_hash, |
| | | probe.generation, |
| | | &probe.sequence, |
| | | false, |
| | | Some("rejected"), |
| | | Some("wrong_participant"), |
| | | ); |
| | | return Some(false); |
| | | } |
| | | let decision = observe_controlled_fixture_attributes( |
| | | probe.expires_at, |
| | | &participant.identity().to_string(), |
| | | expected_participant, |
| | | &probe.sequence, |
| | | || participant.attributes(), |
| | | ) |
| | | .await; |
| | | let (ack_result, reject_reason, observed) = record_controlled_fixture_attribute_decision( |
| | | decision, |
| | | runtime_call_id, |
| | | runtime_trace_id, |
| | | &probe.call_id_hash, |
| | | &probe.call_trace_id_hash, |
| | | probe.generation, |
| | | &probe.sequence, |
| | | ); |
| | | let ack = controlled_fixture_ack_from_probe(&probe, observed, reject_reason); |
| | | let payload = match serde_json::to_vec(&ack) { |
| | | Ok(payload) => payload, |
| | | Err(_) => return Some(false), |
| | | }; |
| | | let local_participant = sink.room.local_participant(); |
| | | record_controlled_fixture_probe_event( |
| | | runtime_call_id, |
| | | runtime_trace_id, |
| | | "ack_publish_started", |
| | | &probe.call_id_hash, |
| | | &probe.call_trace_id_hash, |
| | | probe.generation, |
| | | &probe.sequence, |
| | | observed, |
| | | Some(ack_result), |
| | | reject_reason, |
| | | ); |
| | | let publish = local_participant.publish_data(DataPacket { |
| | | payload, |
| | | topic: Some(CONTROLLED_FIXTURE_ACK_TOPIC.to_string()), |
| | | reliable: true, |
| | | destination_identities: vec![probe.sender.clone()], |
| | | }); |
| | | let outcome = complete_controlled_fixture_ack_publish( |
| | | publish, |
| | | runtime_call_id, |
| | | runtime_trace_id, |
| | | observed, |
| | | ack_result, |
| | | reject_reason, |
| | | &probe.call_id_hash, |
| | | &probe.call_trace_id_hash, |
| | | probe.generation, |
| | | &probe.sequence, |
| | | acknowledged_probe_sequences, |
| | | ) |
| | | .await; |
| | | if outcome.published && ack_result == "timeout" && reject_reason == Some("expired") { |
| | | spawn_controlled_fixture_post_expiry_observation( |
| | | probe, |
| | | participant.clone(), |
| | | expected_participant.map(str::to_string), |
| | | lifecycle_active, |
| | | runtime_call_id.to_string(), |
| | | runtime_trace_id.to_string(), |
| | | ); |
| | | } |
| | | Some(outcome.observed) |
| | | } |
| | | |
| | | async fn handle_finished_turn( |
| | |
| | | call_id: &str, |
| | | trace_id: &str, |
| | | turn: FinishedSpeechTurn, |
| | | realtime_asr_result_ref: Option<String>, |
| | | ) { |
| | | let turn_pipeline_started_at = Instant::now(); |
| | | if !bridge_config.is_ready() { |
| | |
| | | "turnArtifactBytes": byte_size, |
| | | "frameCount": turn.frame_count, |
| | | "sampleCount": turn.sample_count, |
| | | "endReason": turn.end_reason, |
| | | "endReason": turn.end_reason.as_str(), |
| | | }), |
| | | ); |
| | | let asr_result_ref = match realtime_asr_result_ref { |
| | | Some(value) => Some(value), |
| | | None => request_asr_result_ref(http, bridge_config, call_id, trace_id, &turn).await, |
| | | }; |
| | | if bridge_config.is_stream_mode() { |
| | | match request_turn_bridge_stream( |
| | | http, |
| | |
| | | &turn, |
| | | &path_ref, |
| | | byte_size, |
| | | asr_result_ref.as_deref(), |
| | | turn_pipeline_started_at, |
| | | ) |
| | | .await |
| | |
| | | &turn, |
| | | &path_ref, |
| | | byte_size, |
| | | asr_result_ref.as_deref(), |
| | | turn_pipeline_started_at, |
| | | ) |
| | | .await |
| | | { |
| | | Ok(outcome) => { |
| | | let mut published_device_outputs = HashSet::new(); |
| | | for output in &outcome.device_outputs { |
| | | if !should_publish_device_output(&mut published_device_outputs, output) { |
| | | continue; |
| | | } |
| | | if let Err(error) = sink |
| | | .publish_device_output(call_id, trace_id, &turn.turn_id, output) |
| | | .await |
| | |
| | | Ok((path_ref, byte_size)) |
| | | } |
| | | |
| | | async fn request_asr_result_ref( |
| | | http: &Client, |
| | | bridge_config: &TurnBridgeConfig, |
| | | call_id: &str, |
| | | trace_id: &str, |
| | | turn: &FinishedSpeechTurn, |
| | | ) -> Option<String> { |
| | | if !bridge_config.is_asr_stream_ready() { |
| | | return None; |
| | | } |
| | | match request_asr_stream(http, bridge_config, call_id, trace_id, turn).await { |
| | | Ok(Some(asr_result_ref)) => { |
| | | emit_activity( |
| | | call_id, |
| | | trace_id, |
| | | Some(&turn.turn_id), |
| | | "asr_stream_ref_ready", |
| | | "ok", |
| | | None, |
| | | None, |
| | | json!({ |
| | | "asrResultRefPresent": true, |
| | | "format": "pcm_s16le", |
| | | "sampleRate": 16000, |
| | | "channels": 1, |
| | | }), |
| | | ); |
| | | Some(asr_result_ref) |
| | | } |
| | | Ok(None) => None, |
| | | Err(error) => { |
| | | warn!( |
| | | call_id = %call_id, |
| | | trace_id = %trace_id, |
| | | turn_id = %turn.turn_id, |
| | | error = %safe_error(&error.to_string()), |
| | | "runtime helper asr_stream_failed_fallback" |
| | | ); |
| | | emit_activity( |
| | | call_id, |
| | | trace_id, |
| | | Some(&turn.turn_id), |
| | | "asr_stream_fallback", |
| | | "skipped", |
| | | Some("ASR_STREAM_INTERRUPTED"), |
| | | Some(true), |
| | | json!({ |
| | | "fallbackReason": "asr_stream_request_failed", |
| | | "fallbackStage": "asr_stream", |
| | | }), |
| | | ); |
| | | None |
| | | } |
| | | } |
| | | } |
| | | |
| | | async fn request_asr_stream( |
| | | http: &Client, |
| | | bridge_config: &TurnBridgeConfig, |
| | | call_id: &str, |
| | | trace_id: &str, |
| | | turn: &FinishedSpeechTurn, |
| | | ) -> Result<Option<String>> { |
| | | let started_at = Instant::now(); |
| | | let ndjson = build_asr_stream_ndjson(call_id, trace_id, turn, bridge_config)?; |
| | | let response = http |
| | | .post(bridge_config.asr_stream_url.as_deref().unwrap_or_default()) |
| | | .header("Content-Type", "application/x-ndjson") |
| | | .header( |
| | | "X-CV-Runtime-Token", |
| | | bridge_config.bridge_token.as_deref().unwrap_or_default(), |
| | | ) |
| | | .header("X-CV-Call-Id", call_id) |
| | | .header("X-CV-Trace-Id", trace_id) |
| | | .header( |
| | | "X-CV-Runtime-Session-Nonce", |
| | | bridge_config |
| | | .runtime_session_nonce |
| | | .as_deref() |
| | | .unwrap_or_default(), |
| | | ) |
| | | .body(ndjson) |
| | | .send() |
| | | .await |
| | | .context("failed to post asr stream")?; |
| | | let status = response.status(); |
| | | if !status.is_success() { |
| | | let body_len = response.text().await.map(|body| body.len()).unwrap_or(0); |
| | | warn!( |
| | | call_id = %call_id, |
| | | trace_id = %trace_id, |
| | | turn_id = %turn.turn_id, |
| | | http_status = status.as_u16(), |
| | | body_len, |
| | | "runtime helper asr_stream_http_failed" |
| | | ); |
| | | return Ok(None); |
| | | } |
| | | let body: RuntimeTurnCommonResult<RuntimeAsrStreamResp> = response |
| | | .json() |
| | | .await |
| | | .context("failed to decode asr stream response")?; |
| | | if body.code != 0 { |
| | | warn!( |
| | | call_id = %call_id, |
| | | trace_id = %trace_id, |
| | | turn_id = %turn.turn_id, |
| | | code = body.code, |
| | | msg_len = body.msg.as_deref().unwrap_or_default().len(), |
| | | "runtime helper asr_stream_common_result_failed" |
| | | ); |
| | | return Ok(None); |
| | | } |
| | | let Some(data) = body.data else { |
| | | return Ok(None); |
| | | }; |
| | | if data.status.as_deref() == Some("final") { |
| | | info!( |
| | | call_id = %call_id, |
| | | trace_id = %trace_id, |
| | | turn_id = %turn.turn_id, |
| | | chunk_count = data.chunk_count.unwrap_or_default(), |
| | | audio_bytes = data.audio_bytes.unwrap_or_default(), |
| | | asr_duration_ms = data.asr_duration_ms.unwrap_or_default(), |
| | | wall_ms = started_at.elapsed().as_millis() as u64, |
| | | provider = %data.provider_alias.as_deref().unwrap_or("unknown"), |
| | | text_len = data.text_len.unwrap_or_default(), |
| | | "runtime helper asr_stream_final" |
| | | ); |
| | | return Ok(data.asr_result_ref); |
| | | } |
| | | emit_activity( |
| | | call_id, |
| | | trace_id, |
| | | Some(&turn.turn_id), |
| | | "asr_stream_fallback", |
| | | "skipped", |
| | | None, |
| | | Some(true), |
| | | json!({ |
| | | "fallbackReason": data.fallback_reason, |
| | | "fallbackStage": data.fallback_stage, |
| | | "status": data.status, |
| | | }), |
| | | ); |
| | | Ok(None) |
| | | } |
| | | |
| | | fn build_asr_stream_ndjson( |
| | | call_id: &str, |
| | | trace_id: &str, |
| | | turn: &FinishedSpeechTurn, |
| | | bridge_config: &TurnBridgeConfig, |
| | | ) -> Result<String> { |
| | | let chunks = asr_pcm_16k_chunks(turn)?; |
| | | let mut seq = 1u64; |
| | | let mut lines = Vec::with_capacity(chunks.len() + 3); |
| | | lines.push(serde_json::to_string(&json!({ |
| | | "event": "asr_stream_started", |
| | | "seq": seq, |
| | | "callId": call_id, |
| | | "traceId": trace_id, |
| | | "turnId": turn.turn_id.as_str(), |
| | | "tsMs": current_time_millis(), |
| | | "payload": { |
| | | "format": "pcm_s16le", |
| | | "sampleRate": 16000, |
| | | "channels": 1, |
| | | "runtimeSessionNonce": bridge_config.runtime_session_nonce.as_deref().unwrap_or_default(), |
| | | "providerHint": "volcengine", |
| | | } |
| | | }))?); |
| | | for (index, samples) in chunks.iter().enumerate() { |
| | | seq += 1; |
| | | let bytes = pcm_i16_to_le_bytes(samples); |
| | | let duration_ms = ((samples.len() as u64) * 1000 / 16_000).max(1); |
| | | lines.push(serde_json::to_string(&json!({ |
| | | "event": "asr_audio_chunk", |
| | | "seq": seq, |
| | | "callId": call_id, |
| | | "traceId": trace_id, |
| | | "turnId": turn.turn_id.as_str(), |
| | | "tsMs": current_time_millis(), |
| | | "payload": { |
| | | "chunkSeq": index + 1, |
| | | "format": "pcm_s16le", |
| | | "sampleRate": 16000, |
| | | "channels": 1, |
| | | "durationMs": duration_ms, |
| | | "payloadBase64": general_purpose::STANDARD.encode(bytes), |
| | | } |
| | | }))?); |
| | | } |
| | | seq += 1; |
| | | lines.push(serde_json::to_string(&json!({ |
| | | "event": "vad_speech_end", |
| | | "seq": seq, |
| | | "callId": call_id, |
| | | "traceId": trace_id, |
| | | "turnId": turn.turn_id.as_str(), |
| | | "tsMs": current_time_millis(), |
| | | "payload": { |
| | | "endReason": turn.end_reason.as_str(), |
| | | "speechDurationMs": turn.duration_ms, |
| | | } |
| | | }))?); |
| | | seq += 1; |
| | | lines.push(serde_json::to_string(&json!({ |
| | | "event": "asr_stream_finish", |
| | | "seq": seq, |
| | | "callId": call_id, |
| | | "traceId": trace_id, |
| | | "turnId": turn.turn_id.as_str(), |
| | | "tsMs": current_time_millis(), |
| | | "payload": { |
| | | "finalChunkSeq": chunks.len(), |
| | | "audioDurationMs": turn.duration_ms, |
| | | } |
| | | }))?); |
| | | Ok(lines.join("\n") + "\n") |
| | | } |
| | | |
| | | fn asr_pcm_16k_chunks(turn: &FinishedSpeechTurn) -> Result<Vec<Vec<i16>>> { |
| | | if turn.samples.is_empty() { |
| | | return Err(anyhow!("empty turn samples")); |
| | | } |
| | | let samples_16k: Vec<i16> = turn.samples.iter().step_by(3).copied().collect(); |
| | | if samples_16k.is_empty() { |
| | | return Err(anyhow!("empty 16k asr samples")); |
| | | } |
| | | let samples_per_chunk = 320usize; |
| | | Ok(samples_16k |
| | | .chunks(samples_per_chunk) |
| | | .map(|chunk| chunk.to_vec()) |
| | | .collect()) |
| | | } |
| | | |
| | | fn pcm_i16_to_le_bytes(samples: &[i16]) -> Vec<u8> { |
| | | let mut bytes = Vec::with_capacity(samples.len() * 2); |
| | | for sample in samples { |
| | | bytes.extend_from_slice(&sample.to_le_bytes()); |
| | | } |
| | | bytes |
| | | } |
| | | |
| | | async fn request_turn_bridge_stream( |
| | | http: &Client, |
| | | bridge_config: &TurnBridgeConfig, |
| | |
| | | turn: &FinishedSpeechTurn, |
| | | path_ref: &str, |
| | | byte_size: u64, |
| | | asr_result_ref: Option<&str>, |
| | | turn_pipeline_started_at: Instant, |
| | | ) -> Result<RuntimeTurnStreamOutcome> { |
| | | let bridge_started_at = Instant::now(); |
| | |
| | | call_id: call_id.to_string(), |
| | | trace_id: trace_id.to_string(), |
| | | turn_id: turn.turn_id.clone(), |
| | | audio_artifact: RuntimeTurnAudioArtifact { |
| | | audio_artifact: Some(RuntimeTurnAudioArtifact { |
| | | artifact_type: "local_file".to_string(), |
| | | path_ref: path_ref.to_string(), |
| | | format: "wav".to_string(), |
| | |
| | | channels: u32::from(USER_AUDIO_NUM_CHANNELS), |
| | | duration_ms: turn.duration_ms, |
| | | byte_size, |
| | | }, |
| | | }), |
| | | asr_result_ref: asr_result_ref.map(str::to_string), |
| | | }; |
| | | let response = http |
| | | .post(bridge_config.bridge_url.as_deref().unwrap_or_default()) |
| | |
| | | .await?; |
| | | } |
| | | } |
| | | state.close_timing(); |
| | | if !state.completed { |
| | | warn!( |
| | | call_id = %call_id, |
| | |
| | | call_id, |
| | | trace_id, |
| | | turn, |
| | | &event, |
| | | audio_chunk, |
| | | state, |
| | | turn_pipeline_started_at, |
| | |
| | | } |
| | | Some("device_output") => { |
| | | if let Some(output) = event.device_output.as_ref() { |
| | | if !should_publish_device_output(&mut state.published_device_output_ids, output) { |
| | | return Ok(()); |
| | | } |
| | | sink.publish_device_output(call_id, trace_id, &turn.turn_id, output) |
| | | .await?; |
| | | state.device_output_count = state.device_output_count.saturating_add(1); |
| | |
| | | } |
| | | Some("turn_completed") => { |
| | | state.completed = true; |
| | | state.close_timing(); |
| | | info!( |
| | | call_id = %call_id, |
| | | trace_id = %trace_id, |
| | |
| | | ); |
| | | } |
| | | Some("turn_failed") => { |
| | | state.close_timing(); |
| | | let error = event.error.as_ref(); |
| | | let reason_code = error |
| | | .and_then(|value| value.reason_code.as_deref()) |
| | |
| | | } |
| | | Some("turn_cancelled") => { |
| | | state.completed = true; |
| | | state.close_timing(); |
| | | info!( |
| | | call_id = %call_id, |
| | | trace_id = %trace_id, |
| | |
| | | } |
| | | Some("activity") => { |
| | | if let Some(activity) = event.activity.as_ref() { |
| | | if activity.event_type.as_deref() == Some("tts_first_audio_chunk_ready") { |
| | | if let Some(runtime_session_nonce) = |
| | | bridge_config.runtime_session_nonce.as_deref() |
| | | { |
| | | let _ = state.arm_timing_anchor( |
| | | call_id, |
| | | trace_id, |
| | | &turn.turn_id, |
| | | runtime_session_nonce, |
| | | &event, |
| | | ); |
| | | } |
| | | } |
| | | info!( |
| | | call_id = %call_id, |
| | | trace_id = %trace_id, |
| | |
| | | call_id: &str, |
| | | trace_id: &str, |
| | | turn: &FinishedSpeechTurn, |
| | | event: &RuntimeTurnStreamEvent, |
| | | audio_chunk: &RuntimeTurnStreamAudioChunk, |
| | | state: &mut RuntimeTurnStreamState, |
| | | turn_pipeline_started_at: Instant, |
| | |
| | | .unwrap_or("pcm_s16le") |
| | | .trim() |
| | | .to_ascii_lowercase(); |
| | | if !matches!(format.as_str(), "pcm_s16le" | "mp3" | "mpeg" | "wav") { |
| | | return Err(anyhow!("unsupported reply_audio_chunk format {format}")); |
| | | } |
| | | match state.reply_chunk_markers.observe(audio_chunk.segment_seq) { |
| | | ReplyChunkMarker::FirstReply => { |
| | | let extension = json!({ |
| | | "segmentSeq": audio_chunk.segment_seq, |
| | | "chunkSeq": audio_chunk.chunk_seq, |
| | | "format": format.as_str(), |
| | | "bytes": payload.len(), |
| | | }); |
| | | if let Some(marker) = |
| | | state.record_m6(call_id, trace_id, &turn.turn_id, event, audio_chunk) |
| | | { |
| | | emit_anchored_activity( |
| | | call_id, |
| | | trace_id, |
| | | &turn.turn_id, |
| | | "helper_first_reply_audio_chunk_received", |
| | | &marker, |
| | | extension, |
| | | ); |
| | | } else { |
| | | emit_activity( |
| | | call_id, |
| | | trace_id, |
| | | Some(&turn.turn_id), |
| | | "helper_first_reply_audio_chunk_received", |
| | | "ok", |
| | | None, |
| | | None, |
| | | extension, |
| | | ); |
| | | } |
| | | } |
| | | ReplyChunkMarker::SegmentFirst => emit_activity( |
| | | call_id, |
| | | trace_id, |
| | | Some(&turn.turn_id), |
| | | "helper_segment_first_audio_chunk_received", |
| | | "ok", |
| | | None, |
| | | None, |
| | | json!({ |
| | | "segmentSeq": audio_chunk.segment_seq, |
| | | "chunkSeq": audio_chunk.chunk_seq, |
| | | "format": format.as_str(), |
| | | "bytes": payload.len(), |
| | | }), |
| | | ), |
| | | ReplyChunkMarker::None => {} |
| | | } |
| | | let frames = if format == "pcm_s16le" { |
| | | let sample_rate = audio_chunk.sample_rate.unwrap_or(sink.sample_rate_hz); |
| | | let channels = audio_chunk.channels.unwrap_or(u32::from(sink.num_channels)); |
| | |
| | | Err(error) => return Err(error).context("failed to decode final stream audio chunk"), |
| | | } |
| | | } else { |
| | | return Err(anyhow!("unsupported reply_audio_chunk format {format}")); |
| | | unreachable!("supported encoded format checked above") |
| | | }; |
| | | if frames.is_empty() { |
| | | return Ok(0); |
| | |
| | | sink.write_pcm_frame(frame).await?; |
| | | if !state.first_audio_frame_written { |
| | | state.first_audio_frame_written = true; |
| | | emit_activity( |
| | | call_id, |
| | | trace_id, |
| | | Some(&turn.turn_id), |
| | | "bot_reply_first_audio_frame_written", |
| | | "ok", |
| | | None, |
| | | None, |
| | | json!({ |
| | | "replyPlaybackMode": state.reply_playback_mode.as_str(), |
| | | "format": format.as_str(), |
| | | "chunkSeq": audio_chunk.chunk_seq, |
| | | "replyTotalAfterVadEndMs": turn_pipeline_started_at.elapsed().as_millis() as u64, |
| | | }), |
| | | ); |
| | | let extension = json!({ |
| | | "replyPlaybackMode": state.reply_playback_mode.as_str(), |
| | | "format": format.as_str(), |
| | | "chunkSeq": audio_chunk.chunk_seq, |
| | | "replyTotalAfterVadEndMs": turn_pipeline_started_at.elapsed().as_millis() as u64, |
| | | }); |
| | | if let Some(marker) = state.record_m7() { |
| | | emit_anchored_activity( |
| | | call_id, |
| | | trace_id, |
| | | &turn.turn_id, |
| | | "bot_reply_first_audio_frame_written", |
| | | &marker, |
| | | extension, |
| | | ); |
| | | } else { |
| | | emit_activity( |
| | | call_id, |
| | | trace_id, |
| | | Some(&turn.turn_id), |
| | | "bot_reply_first_audio_frame_written", |
| | | "ok", |
| | | None, |
| | | None, |
| | | extension, |
| | | ); |
| | | } |
| | | } |
| | | sleep_until(pacing_started_at + Duration::from_millis(((index + 1) as u64) * 20)).await; |
| | | } |
| | |
| | | turn: &FinishedSpeechTurn, |
| | | path_ref: &str, |
| | | byte_size: u64, |
| | | asr_result_ref: Option<&str>, |
| | | turn_pipeline_started_at: Instant, |
| | | ) -> Result<RuntimeTurnBridgeOutcome> { |
| | | let bridge_started_at = Instant::now(); |
| | |
| | | call_id: call_id.to_string(), |
| | | trace_id: trace_id.to_string(), |
| | | turn_id: turn.turn_id.clone(), |
| | | audio_artifact: RuntimeTurnAudioArtifact { |
| | | audio_artifact: Some(RuntimeTurnAudioArtifact { |
| | | artifact_type: "local_file".to_string(), |
| | | path_ref: path_ref.to_string(), |
| | | format: "wav".to_string(), |
| | |
| | | channels: u32::from(USER_AUDIO_NUM_CHANNELS), |
| | | duration_ms: turn.duration_ms, |
| | | byte_size, |
| | | }, |
| | | }), |
| | | asr_result_ref: asr_result_ref.map(str::to_string), |
| | | }; |
| | | let response = http |
| | | .post(bridge_config.bridge_url.as_deref().unwrap_or_default()) |
| | |
| | | #[serde(rename = "turnId")] |
| | | turn_id: String, |
| | | #[serde(rename = "audioArtifact")] |
| | | audio_artifact: RuntimeTurnAudioArtifact, |
| | | #[serde(skip_serializing_if = "Option::is_none")] |
| | | audio_artifact: Option<RuntimeTurnAudioArtifact>, |
| | | #[serde(rename = "asrResultRef", skip_serializing_if = "Option::is_none")] |
| | | asr_result_ref: Option<String>, |
| | | } |
| | | |
| | | #[derive(Serialize)] |
| | |
| | | retryable: Option<bool>, |
| | | } |
| | | |
| | | #[derive(Deserialize)] |
| | | #[serde(rename_all = "camelCase")] |
| | | struct RuntimeAsrStreamResp { |
| | | status: Option<String>, |
| | | asr_result_ref: Option<String>, |
| | | chunk_count: Option<u64>, |
| | | audio_bytes: Option<u64>, |
| | | asr_duration_ms: Option<u64>, |
| | | provider_alias: Option<String>, |
| | | text_len: Option<u64>, |
| | | fallback_reason: Option<String>, |
| | | fallback_stage: Option<String>, |
| | | } |
| | | |
| | | #[derive(Default)] |
| | | struct RuntimeTurnBridgeOutcome { |
| | | reply_audio_artifact: Option<RuntimeTurnReplyAudioArtifact>, |
| | |
| | | completed: bool, |
| | | audio_chunk_count: u64, |
| | | device_output_count: u64, |
| | | published_device_output_ids: HashSet<String>, |
| | | encoded_audio_buffer: Vec<u8>, |
| | | pcm_stream_decoder: Option<audio::PcmS16leStreamDecoder>, |
| | | pcm_stream_network_chunk_count: u64, |
| | | reply_chunk_markers: ReplyChunkMarkerState, |
| | | timing: RuntimeTurnStreamTimingState, |
| | | } |
| | | |
| | | impl Default for RuntimeTurnStreamState { |
| | |
| | | completed: false, |
| | | audio_chunk_count: 0, |
| | | device_output_count: 0, |
| | | published_device_output_ids: HashSet::new(), |
| | | encoded_audio_buffer: Vec::new(), |
| | | pcm_stream_decoder: None, |
| | | pcm_stream_network_chunk_count: 0, |
| | | reply_chunk_markers: ReplyChunkMarkerState::default(), |
| | | timing: RuntimeTurnStreamTimingState::default(), |
| | | } |
| | | } |
| | | } |
| | | |
| | | impl RuntimeTurnStreamState { |
| | | fn arm_timing_anchor( |
| | | &mut self, |
| | | call_id: &str, |
| | | trace_id: &str, |
| | | turn_id: &str, |
| | | runtime_session_nonce: &str, |
| | | event: &RuntimeTurnStreamEvent, |
| | | ) -> bool { |
| | | if self.timing.phase != RuntimeTurnStreamTimingPhase::Empty |
| | | || event.call_id.as_deref() != Some(call_id) |
| | | || event.trace_id.as_deref() != Some(trace_id) |
| | | || event.turn_id.as_deref() != Some(turn_id) |
| | | { |
| | | return false; |
| | | } |
| | | let Some(activity) = event.activity.as_ref() else { |
| | | return false; |
| | | }; |
| | | if activity.event_type.as_deref() != Some("tts_first_audio_chunk_ready") { |
| | | return false; |
| | | } |
| | | let Some(extension) = activity.extension.as_ref() else { |
| | | return false; |
| | | }; |
| | | let Some(anchor_id) = extension.stream_anchor_id.as_deref() else { |
| | | return false; |
| | | }; |
| | | let valid_anchor_id = (16..=64).contains(&anchor_id.len()) && anchor_id.is_ascii(); |
| | | let expected_nonce_hash = runtime_session_nonce_hash(runtime_session_nonce); |
| | | if extension.stream_timing_version != Some(STREAM_TIMING_VERSION) |
| | | || !valid_anchor_id |
| | | || extension.runtime_session_nonce_hash.as_deref() != Some(expected_nonce_hash.as_str()) |
| | | || extension.segment_seq != Some(STREAM_TIMING_FIRST_SEGMENT) |
| | | || extension.stream_timing_validation.as_deref() != Some("bound") |
| | | { |
| | | return false; |
| | | } |
| | | let Some(server_delta_ms) = extension.stream_anchor_server_delta_ms else { |
| | | return false; |
| | | }; |
| | | self.timing.anchor = Some(RuntimeTurnStreamTimingAnchor { |
| | | call_id: call_id.to_string(), |
| | | trace_id: trace_id.to_string(), |
| | | turn_id: turn_id.to_string(), |
| | | anchor_id: anchor_id.to_string(), |
| | | server_delta_ms, |
| | | runtime_session_nonce_hash: expected_nonce_hash, |
| | | segment_seq: STREAM_TIMING_FIRST_SEGMENT, |
| | | received_at: Instant::now(), |
| | | }); |
| | | self.timing.phase = RuntimeTurnStreamTimingPhase::Armed; |
| | | true |
| | | } |
| | | |
| | | fn record_m6( |
| | | &mut self, |
| | | call_id: &str, |
| | | trace_id: &str, |
| | | turn_id: &str, |
| | | event: &RuntimeTurnStreamEvent, |
| | | audio_chunk: &RuntimeTurnStreamAudioChunk, |
| | | ) -> Option<RuntimeTurnStreamTimingMarker> { |
| | | if self.timing.phase != RuntimeTurnStreamTimingPhase::Armed { |
| | | return None; |
| | | } |
| | | let anchor = self.timing.anchor.as_ref()?; |
| | | if anchor.call_id != call_id |
| | | || anchor.trace_id != trace_id |
| | | || anchor.turn_id != turn_id |
| | | || event.call_id.as_deref() != Some(call_id) |
| | | || event.trace_id.as_deref() != Some(trace_id) |
| | | || event.turn_id.as_deref() != Some(turn_id) |
| | | || audio_chunk.segment_seq != Some(anchor.segment_seq) |
| | | || audio_chunk.chunk_seq != Some(STREAM_TIMING_FIRST_CHUNK) |
| | | || audio_chunk.stream_timing_version != Some(STREAM_TIMING_VERSION) |
| | | || audio_chunk.stream_anchor_id.as_deref() != Some(anchor.anchor_id.as_str()) |
| | | { |
| | | return None; |
| | | } |
| | | let Some(marker) = RuntimeTurnStreamTimingMarker::from_anchor( |
| | | anchor, |
| | | audio_chunk.chunk_seq.unwrap_or(STREAM_TIMING_FIRST_CHUNK), |
| | | ) else { |
| | | self.close_timing(); |
| | | return None; |
| | | }; |
| | | self.timing.chunk_seq = Some(marker.chunk_seq); |
| | | self.timing.phase = RuntimeTurnStreamTimingPhase::M6Recorded; |
| | | Some(marker) |
| | | } |
| | | |
| | | fn record_m7(&mut self) -> Option<RuntimeTurnStreamTimingMarker> { |
| | | if self.timing.phase != RuntimeTurnStreamTimingPhase::M6Recorded { |
| | | return None; |
| | | } |
| | | let anchor = self.timing.anchor.as_ref()?; |
| | | let Some(marker) = RuntimeTurnStreamTimingMarker::from_anchor( |
| | | anchor, |
| | | self.timing.chunk_seq.unwrap_or(STREAM_TIMING_FIRST_CHUNK), |
| | | ) else { |
| | | self.close_timing(); |
| | | return None; |
| | | }; |
| | | self.timing.phase = RuntimeTurnStreamTimingPhase::M7Recorded; |
| | | Some(marker) |
| | | } |
| | | |
| | | fn close_timing(&mut self) { |
| | | self.timing.close(); |
| | | } |
| | | } |
| | | |
| | | #[derive(Debug, Clone, Copy, PartialEq, Eq)] |
| | | enum RuntimeTurnStreamTimingPhase { |
| | | Empty, |
| | | Armed, |
| | | M6Recorded, |
| | | M7Recorded, |
| | | Closed, |
| | | } |
| | | |
| | | struct RuntimeTurnStreamTimingState { |
| | | phase: RuntimeTurnStreamTimingPhase, |
| | | anchor: Option<RuntimeTurnStreamTimingAnchor>, |
| | | chunk_seq: Option<u64>, |
| | | } |
| | | |
| | | impl Default for RuntimeTurnStreamTimingState { |
| | | fn default() -> Self { |
| | | Self { |
| | | phase: RuntimeTurnStreamTimingPhase::Empty, |
| | | anchor: None, |
| | | chunk_seq: None, |
| | | } |
| | | } |
| | | } |
| | | |
| | | impl RuntimeTurnStreamTimingState { |
| | | fn close(&mut self) { |
| | | self.anchor = None; |
| | | self.chunk_seq = None; |
| | | self.phase = RuntimeTurnStreamTimingPhase::Closed; |
| | | } |
| | | } |
| | | |
| | | impl Drop for RuntimeTurnStreamTimingState { |
| | | fn drop(&mut self) { |
| | | self.anchor = None; |
| | | self.chunk_seq = None; |
| | | } |
| | | } |
| | | |
| | | struct RuntimeTurnStreamTimingAnchor { |
| | | call_id: String, |
| | | trace_id: String, |
| | | turn_id: String, |
| | | anchor_id: String, |
| | | server_delta_ms: u64, |
| | | runtime_session_nonce_hash: String, |
| | | segment_seq: u64, |
| | | received_at: Instant, |
| | | } |
| | | |
| | | struct RuntimeTurnStreamTimingMarker { |
| | | version: u32, |
| | | anchor_id: String, |
| | | anchor_server_delta_ms: u64, |
| | | anchor_elapsed_ms: u64, |
| | | server_delta_ms: u64, |
| | | runtime_session_nonce_hash: String, |
| | | segment_seq: u64, |
| | | chunk_seq: u64, |
| | | } |
| | | |
| | | impl RuntimeTurnStreamTimingMarker { |
| | | fn from_anchor(anchor: &RuntimeTurnStreamTimingAnchor, chunk_seq: u64) -> Option<Self> { |
| | | let elapsed_ms = u64::try_from(anchor.received_at.elapsed().as_millis()).ok()?; |
| | | if elapsed_ms > STREAM_TIMING_MAX_ELAPSED_MS { |
| | | return None; |
| | | } |
| | | Some(Self { |
| | | version: STREAM_TIMING_VERSION, |
| | | anchor_id: anchor.anchor_id.clone(), |
| | | anchor_server_delta_ms: anchor.server_delta_ms, |
| | | anchor_elapsed_ms: elapsed_ms, |
| | | server_delta_ms: anchor.server_delta_ms.checked_add(elapsed_ms)?, |
| | | runtime_session_nonce_hash: anchor.runtime_session_nonce_hash.clone(), |
| | | segment_seq: anchor.segment_seq, |
| | | chunk_seq, |
| | | }) |
| | | } |
| | | |
| | | fn extension_with(&self, extra: serde_json::Value) -> serde_json::Value { |
| | | let mut extension = match extra { |
| | | serde_json::Value::Object(value) => value, |
| | | _ => serde_json::Map::new(), |
| | | }; |
| | | extension.insert("streamTimingVersion".to_string(), json!(self.version)); |
| | | extension.insert("streamAnchorId".to_string(), json!(self.anchor_id)); |
| | | extension.insert( |
| | | "streamAnchorServerDeltaMs".to_string(), |
| | | json!(self.anchor_server_delta_ms), |
| | | ); |
| | | extension.insert("anchorElapsedMs".to_string(), json!(self.anchor_elapsed_ms)); |
| | | extension.insert( |
| | | "runtimeSessionNonceHash".to_string(), |
| | | json!(self.runtime_session_nonce_hash), |
| | | ); |
| | | extension.insert("segmentSeq".to_string(), json!(self.segment_seq)); |
| | | extension.insert("chunkSeq".to_string(), json!(self.chunk_seq)); |
| | | extension.insert("streamTimingValidation".to_string(), json!("bound")); |
| | | serde_json::Value::Object(extension) |
| | | } |
| | | } |
| | | |
| | | fn runtime_session_nonce_hash(value: &str) -> String { |
| | | let digest = Sha256::digest(value.as_bytes()); |
| | | digest[..6] |
| | | .iter() |
| | | .map(|byte| format!("{byte:02x}")) |
| | | .collect() |
| | | } |
| | | |
| | | #[derive(Debug, PartialEq, Eq)] |
| | | enum ReplyChunkMarker { |
| | | FirstReply, |
| | | SegmentFirst, |
| | | None, |
| | | } |
| | | |
| | | #[derive(Default)] |
| | | struct ReplyChunkMarkerState { |
| | | first_reply_seen: bool, |
| | | seen_segments: HashSet<u64>, |
| | | } |
| | | |
| | | impl ReplyChunkMarkerState { |
| | | fn observe(&mut self, segment_seq: Option<u64>) -> ReplyChunkMarker { |
| | | let first_for_segment = segment_seq |
| | | .map(|value| self.seen_segments.insert(value)) |
| | | .unwrap_or(false); |
| | | if !self.first_reply_seen { |
| | | self.first_reply_seen = true; |
| | | return ReplyChunkMarker::FirstReply; |
| | | } |
| | | if first_for_segment { |
| | | return ReplyChunkMarker::SegmentFirst; |
| | | } |
| | | ReplyChunkMarker::None |
| | | } |
| | | } |
| | | |
| | |
| | | struct RuntimeTurnStreamEvent { |
| | | #[serde(rename = "type", alias = "event")] |
| | | event_type: Option<String>, |
| | | #[serde(rename = "callId")] |
| | | call_id: Option<String>, |
| | | #[serde(rename = "traceId")] |
| | | trace_id: Option<String>, |
| | | #[serde(rename = "turnId")] |
| | | turn_id: Option<String>, |
| | | #[serde(rename = "seq")] |
| | | seq: Option<u64>, |
| | | #[serde(rename = "replyPlaybackMode")] |
| | |
| | | struct RuntimeTurnStreamAudioChunk { |
| | | #[serde(rename = "chunkSeq", alias = "seq")] |
| | | chunk_seq: Option<u64>, |
| | | #[serde(rename = "segmentSeq")] |
| | | segment_seq: Option<u64>, |
| | | #[serde(rename = "streamTimingVersion")] |
| | | stream_timing_version: Option<u32>, |
| | | #[serde(rename = "streamAnchorId")] |
| | | stream_anchor_id: Option<String>, |
| | | format: Option<String>, |
| | | #[serde(rename = "sampleRate")] |
| | | sample_rate: Option<u32>, |
| | |
| | | stage: Option<String>, |
| | | #[serde(rename = "reasonCode")] |
| | | reason_code: Option<String>, |
| | | extension: Option<RuntimeTurnStreamTimingExtension>, |
| | | } |
| | | |
| | | #[derive(Clone, Deserialize)] |
| | | struct RuntimeTurnStreamTimingExtension { |
| | | #[serde(rename = "streamTimingVersion")] |
| | | stream_timing_version: Option<u32>, |
| | | #[serde(rename = "streamAnchorId")] |
| | | stream_anchor_id: Option<String>, |
| | | #[serde(rename = "streamAnchorServerDeltaMs")] |
| | | stream_anchor_server_delta_ms: Option<u64>, |
| | | #[serde(rename = "runtimeSessionNonceHash")] |
| | | runtime_session_nonce_hash: Option<String>, |
| | | #[serde(rename = "segmentSeq")] |
| | | segment_seq: Option<u64>, |
| | | #[serde(rename = "streamTimingValidation")] |
| | | stream_timing_validation: Option<String>, |
| | | } |
| | | |
| | | #[derive(Deserialize)] |
| | |
| | | #[serde(rename = "commandCode")] |
| | | command_code: Option<String>, |
| | | params: Option<serde_json::Value>, |
| | | } |
| | | |
| | | fn should_publish_device_output( |
| | | published_ids: &mut HashSet<String>, |
| | | output: &RuntimeTurnDeviceOutput, |
| | | ) -> bool { |
| | | let Some(command_id) = output |
| | | .command_id |
| | | .as_deref() |
| | | .map(str::trim) |
| | | .filter(|value| !value.is_empty()) |
| | | else { |
| | | return false; |
| | | }; |
| | | if output |
| | | .command_code |
| | | .as_deref() |
| | | .map(str::trim) |
| | | .filter(|value| !value.is_empty()) |
| | | .is_none() |
| | | { |
| | | return false; |
| | | } |
| | | published_ids.insert(command_id.to_string()) |
| | | } |
| | | |
| | | fn require_safe_segment(value: &str) -> Result<()> { |
| | |
| | | turn_bridge_config: TurnBridgeConfig, |
| | | http: Client, |
| | | sink: Arc<BotAudioOutputSink>, |
| | | expected_participant_identity: Option<String>, |
| | | participant: RemoteParticipant, |
| | | ) -> JoinHandle<()> { |
| | | tokio::spawn(async move { |
| | | let mut stream = NativeAudioStream::new( |
| | |
| | | i32::from(USER_AUDIO_NUM_CHANNELS), |
| | | ); |
| | | let started_at = Instant::now(); |
| | | let (frame_tx, mut frame_rx) = |
| | | mpsc::channel::<DrainedUserAudioFrame>(INBOUND_AUDIO_QUEUE_CAPACITY); |
| | | let (drain_shutdown_tx, mut drain_shutdown_rx) = watch::channel(false); |
| | | let drain_call_id = call_id.clone(); |
| | | let drain_trace_id = trace_id.clone(); |
| | | let drain_task = tokio::spawn(async move { |
| | | let mut received_frame_count: u64 = 0; |
| | | let mut dropped_frame_count: u64 = 0; |
| | | loop { |
| | | tokio::select! { |
| | | changed = drain_shutdown_rx.changed() => { |
| | | match changed { |
| | | Ok(()) if *drain_shutdown_rx.borrow() => break, |
| | | Ok(()) => {} |
| | | Err(_) => break, |
| | | } |
| | | } |
| | | maybe_frame = stream.next() => { |
| | | let Some(frame) = maybe_frame else { |
| | | break; |
| | | }; |
| | | received_frame_count = received_frame_count.saturating_add(1); |
| | | let drained = DrainedUserAudioFrame { |
| | | frame_index: received_frame_count, |
| | | captured_elapsed_ms: started_at.elapsed().as_millis() as u64, |
| | | frame: AudioFrame { |
| | | data: Cow::Owned(frame.data.as_ref().to_vec()), |
| | | sample_rate: frame.sample_rate, |
| | | num_channels: frame.num_channels, |
| | | samples_per_channel: frame.samples_per_channel, |
| | | }, |
| | | }; |
| | | match frame_tx.try_send(drained) { |
| | | Ok(()) => {} |
| | | Err(mpsc::error::TrySendError::Full(_)) => { |
| | | dropped_frame_count = dropped_frame_count.saturating_add(1); |
| | | if dropped_frame_count % INBOUND_AUDIO_DROP_LOG_INTERVAL == 1 { |
| | | warn!( |
| | | call_id = %drain_call_id, |
| | | trace_id = %drain_trace_id, |
| | | dropped_frame_count, |
| | | received_frame_count, |
| | | queue_capacity = INBOUND_AUDIO_QUEUE_CAPACITY, |
| | | "runtime helper inbound_audio_queue_full_dropping_newest" |
| | | ); |
| | | } |
| | | } |
| | | Err(mpsc::error::TrySendError::Closed(_)) => break, |
| | | } |
| | | } |
| | | } |
| | | } |
| | | stream.close(); |
| | | info!( |
| | | call_id = %drain_call_id, |
| | | trace_id = %drain_trace_id, |
| | | received_frame_count, |
| | | dropped_frame_count, |
| | | queue_capacity = INBOUND_AUDIO_QUEUE_CAPACITY, |
| | | "runtime helper user_audio_drain_ended" |
| | | ); |
| | | }); |
| | | let mut frame_count: u64 = 0; |
| | | let mut sample_count: u64 = 0; |
| | | let mut simple_vad = if simple_vad_enabled { |
| | |
| | | } else { |
| | | None |
| | | }; |
| | | let mut realtime_asr_upload: Option<RealtimeAsrUpload> = None; |
| | | let mut last_fixture_sequence: Option<String> = None; |
| | | |
| | | while let Some(frame) = stream.next().await { |
| | | frame_count += 1; |
| | | while let Some(drained) = frame_rx.recv().await { |
| | | let frame = drained.frame; |
| | | frame_count = drained.frame_index; |
| | | sample_count += u64::from(frame.samples_per_channel) * u64::from(frame.num_channels); |
| | | let elapsed_ms = started_at.elapsed().as_millis() as u64; |
| | | let elapsed_ms = drained.captured_elapsed_ms; |
| | | if frame_count == 1 { |
| | | info!( |
| | | call_id = %call_id, |
| | |
| | | |
| | | if let Some(vad) = simple_vad.as_mut() { |
| | | if vad_enabled_gate.load(Ordering::Acquire) { |
| | | if let Some(turn) = vad.observe_frame( |
| | | let participant_identity = participant.identity().to_string(); |
| | | let (was_in_speech, is_in_speech, turn) = observe_bound_participant_frame( |
| | | &participant_identity, |
| | | expected_participant_identity.as_deref(), |
| | | vad, |
| | | &call_id, |
| | | &trace_id, |
| | | &participant_alias, |
| | |
| | | frame_count, |
| | | elapsed_ms, |
| | | &frame, |
| | | ) { |
| | | http.clone(), |
| | | turn_bridge_config.realtime_asr_config(), |
| | | || participant.attributes(), |
| | | &mut realtime_asr_upload, |
| | | &mut last_fixture_sequence, |
| | | turn_bridge_config.asr_realtime_enabled, |
| | | ); |
| | | |
| | | if was_in_speech { |
| | | let push_failed = realtime_asr_upload |
| | | .as_mut() |
| | | .and_then(|upload| upload.push_48k_samples(frame.data.as_ref()).err()); |
| | | if let Some(error) = push_failed { |
| | | warn!( |
| | | call_id = %call_id, |
| | | trace_id = %trace_id, |
| | | error = %safe_error(&error.to_string()), |
| | | "runtime helper asr_realtime_upload_failed_fallback" |
| | | ); |
| | | if let Some(upload) = realtime_asr_upload.take() { |
| | | tokio::spawn(async move { |
| | | upload.cancel("upload_backpressure").await; |
| | | }); |
| | | } |
| | | } |
| | | } |
| | | |
| | | if let Some(turn) = turn { |
| | | let realtime_asr_result_ref = match realtime_asr_upload.take() { |
| | | Some(upload) => { |
| | | finish_realtime_asr_upload(upload, &call_id, &trace_id, &turn).await |
| | | } |
| | | None => None, |
| | | }; |
| | | handle_finished_turn( |
| | | &http, |
| | | &turn_bridge_config, |
| | |
| | | &call_id, |
| | | &trace_id, |
| | | turn, |
| | | realtime_asr_result_ref, |
| | | ) |
| | | .await; |
| | | } else if was_in_speech && !is_in_speech { |
| | | if let Some(upload) = realtime_asr_upload.take() { |
| | | tokio::spawn(async move { |
| | | upload.cancel("speech_too_short").await; |
| | | }); |
| | | } |
| | | } |
| | | } else { |
| | | if let Some(upload) = realtime_asr_upload.take() { |
| | | tokio::spawn(async move { |
| | | upload.cancel("vad_disabled").await; |
| | | }); |
| | | } |
| | | vad.observe_disabled_frame( |
| | | &call_id, |
| | | &trace_id, |
| | |
| | | &track_sid_alias, |
| | | started_at.elapsed().as_millis() as u64, |
| | | ) { |
| | | handle_finished_turn(&http, &turn_bridge_config, &sink, &call_id, &trace_id, turn) |
| | | .await; |
| | | let realtime_asr_result_ref = match realtime_asr_upload.take() { |
| | | Some(upload) => { |
| | | finish_realtime_asr_upload(upload, &call_id, &trace_id, &turn).await |
| | | } |
| | | None => None, |
| | | }; |
| | | handle_finished_turn( |
| | | &http, |
| | | &turn_bridge_config, |
| | | &sink, |
| | | &call_id, |
| | | &trace_id, |
| | | turn, |
| | | realtime_asr_result_ref, |
| | | ) |
| | | .await; |
| | | } |
| | | } |
| | | if let Some(upload) = realtime_asr_upload.take() { |
| | | upload.cancel("stream_end").await; |
| | | } |
| | | let _ = drain_shutdown_tx.send(true); |
| | | let mut drain_task = drain_task; |
| | | if timeout(AUDIO_DRAIN_STOP_GRACE, &mut drain_task) |
| | | .await |
| | | .is_err() |
| | | { |
| | | drain_task.abort(); |
| | | let _ = drain_task.await; |
| | | warn!( |
| | | call_id = %call_id, |
| | | trace_id = %trace_id, |
| | | stop_grace_ms = AUDIO_DRAIN_STOP_GRACE.as_millis() as u64, |
| | | "runtime helper user_audio_drain_stop_timeout" |
| | | ); |
| | | } |
| | | |
| | | info!( |
| | |
| | | "runtime helper user_audio_stream_ended" |
| | | ); |
| | | }) |
| | | } |
| | | |
| | | fn observe_frame_and_start_session<F>( |
| | | vad: &mut SimpleVad, |
| | | call_id: &str, |
| | | trace_id: &str, |
| | | participant_alias: &str, |
| | | track_sid_alias: &str, |
| | | frame_count: u64, |
| | | elapsed_ms: u64, |
| | | frame: &AudioFrame<'_>, |
| | | http: Client, |
| | | config: RealtimeAsrConfig, |
| | | read_attributes: F, |
| | | upload_slot: &mut Option<RealtimeAsrUpload>, |
| | | last_fixture_sequence: &mut Option<String>, |
| | | realtime_enabled: bool, |
| | | ) -> (bool, bool, Option<FinishedSpeechTurn>) |
| | | where |
| | | F: FnOnce() -> std::collections::HashMap<String, String>, |
| | | { |
| | | let was_in_speech = vad.in_speech; |
| | | let turn = vad.observe_frame( |
| | | call_id, |
| | | trace_id, |
| | | participant_alias, |
| | | track_sid_alias, |
| | | frame_count, |
| | | elapsed_ms, |
| | | frame, |
| | | ); |
| | | let is_in_speech = vad.in_speech; |
| | | if !was_in_speech && is_in_speech { |
| | | start_realtime_session_for_new_speech( |
| | | http, |
| | | config, |
| | | call_id, |
| | | trace_id, |
| | | vad, |
| | | read_attributes, |
| | | upload_slot, |
| | | last_fixture_sequence, |
| | | realtime_enabled, |
| | | ); |
| | | } |
| | | (was_in_speech, is_in_speech, turn) |
| | | } |
| | | |
| | | fn observe_bound_participant_frame<F>( |
| | | participant_identity: &str, |
| | | expected_participant: Option<&str>, |
| | | vad: &mut SimpleVad, |
| | | call_id: &str, |
| | | trace_id: &str, |
| | | participant_alias: &str, |
| | | track_sid_alias: &str, |
| | | frame_count: u64, |
| | | elapsed_ms: u64, |
| | | frame: &AudioFrame<'_>, |
| | | http: Client, |
| | | config: RealtimeAsrConfig, |
| | | read_attributes: F, |
| | | upload_slot: &mut Option<RealtimeAsrUpload>, |
| | | last_fixture_sequence: &mut Option<String>, |
| | | realtime_enabled: bool, |
| | | ) -> (bool, bool, Option<FinishedSpeechTurn>) |
| | | where |
| | | F: FnOnce() -> std::collections::HashMap<String, String>, |
| | | { |
| | | if !is_bound_user_participant(participant_identity, expected_participant) { |
| | | warn!( |
| | | "audioIngressOriginStatus" = "wrong_participant_or_track", |
| | | "runtime helper rejected audio participant before VAD/session" |
| | | ); |
| | | return (vad.in_speech, vad.in_speech, None); |
| | | } |
| | | observe_frame_and_start_session( |
| | | vad, |
| | | call_id, |
| | | trace_id, |
| | | participant_alias, |
| | | track_sid_alias, |
| | | frame_count, |
| | | elapsed_ms, |
| | | frame, |
| | | http, |
| | | config, |
| | | read_attributes, |
| | | upload_slot, |
| | | last_fixture_sequence, |
| | | realtime_enabled, |
| | | ) |
| | | } |
| | | |
| | | fn start_realtime_session_for_new_speech( |
| | | http: Client, |
| | | config: RealtimeAsrConfig, |
| | | call_id: &str, |
| | | trace_id: &str, |
| | | vad: &SimpleVad, |
| | | read_attributes: impl FnOnce() -> std::collections::HashMap<String, String>, |
| | | upload_slot: &mut Option<RealtimeAsrUpload>, |
| | | last_fixture_sequence: &mut Option<String>, |
| | | realtime_enabled: bool, |
| | | ) { |
| | | let turn_id = format!("turn-{:04}", vad.turn_index); |
| | | let attributes = read_attributes(); |
| | | let origin_status = AudioIngressMetadata::origin_status(&attributes); |
| | | let metadata = match AudioIngressMetadata::from_participant(&attributes) { |
| | | Ok(metadata) => metadata, |
| | | Err(reason) => { |
| | | warn!(call_id = %call_id, trace_id = %trace_id, turn_id = %turn_id, |
| | | reason, audioIngressOriginStatus = %AudioIngressMetadata::rejected_origin_status( |
| | | reason, &attributes |
| | | ), |
| | | "runtime helper asr_realtime_metadata_rejected"); |
| | | return; |
| | | } |
| | | }; |
| | | if let Some(metadata) = metadata.as_ref() { |
| | | if !fixture_sequence_is_new( |
| | | last_fixture_sequence.as_deref(), |
| | | &metadata.client_fixture_sequence, |
| | | ) { |
| | | warn!(call_id = %call_id, trace_id = %trace_id, turn_id = %turn_id, |
| | | audioIngressOriginStatus = "sequence_replayed_or_regressed", |
| | | "runtime helper asr_realtime_metadata_sequence_rejected"); |
| | | return; |
| | | } |
| | | } |
| | | match RealtimeAsrUpload::start( |
| | | http, |
| | | config, |
| | | call_id, |
| | | trace_id, |
| | | &turn_id, |
| | | &vad.speech_samples, |
| | | metadata.as_ref(), |
| | | ) { |
| | | Ok(upload) => { |
| | | if let Some(metadata) = metadata { |
| | | *last_fixture_sequence = Some(metadata.client_fixture_sequence); |
| | | } |
| | | info!(call_id = %call_id, trace_id = %trace_id, turn_id = %turn_id, |
| | | origin_status, "runtime helper asr_realtime_session_started"); |
| | | *upload_slot = Some(upload); |
| | | } |
| | | Err(error) if realtime_enabled => { |
| | | warn!(call_id = %call_id, trace_id = %trace_id, turn_id = %turn_id, |
| | | error = %safe_error(&error.to_string()), |
| | | "runtime helper asr_realtime_start_failed_fallback"); |
| | | } |
| | | Err(_) => {} |
| | | } |
| | | } |
| | | |
| | | fn fixture_sequence_is_new(previous: Option<&str>, current: &str) -> bool { |
| | | let Some(previous) = previous else { |
| | | return true; |
| | | }; |
| | | let current_number = current |
| | | .rsplit_once('-') |
| | | .and_then(|(_, value)| value.parse::<u64>().ok()); |
| | | let previous_number = previous |
| | | .rsplit_once('-') |
| | | .and_then(|(_, value)| value.parse::<u64>().ok()); |
| | | match (previous_number, current_number) { |
| | | (Some(previous), Some(current)) => current > previous, |
| | | _ => previous != current, |
| | | } |
| | | } |
| | | |
| | | struct DrainedUserAudioFrame { |
| | | frame_index: u64, |
| | | captured_elapsed_ms: u64, |
| | | frame: AudioFrame<'static>, |
| | | } |
| | | |
| | | async fn finish_realtime_asr_upload( |
| | | upload: RealtimeAsrUpload, |
| | | call_id: &str, |
| | | trace_id: &str, |
| | | turn: &FinishedSpeechTurn, |
| | | ) -> Option<String> { |
| | | match upload.finish(turn.duration_ms, &turn.end_reason).await { |
| | | Ok(RealtimeAsrOutcome { |
| | | status, |
| | | asr_result_ref, |
| | | provider_alias, |
| | | partial_count, |
| | | fallback_reason, |
| | | fallback_stage, |
| | | chunk_count, |
| | | audio_bytes, |
| | | wall_ms, |
| | | }) => { |
| | | info!( |
| | | call_id = %call_id, |
| | | trace_id = %trace_id, |
| | | turn_id = %turn.turn_id, |
| | | status = %status, |
| | | provider_alias = ?provider_alias, |
| | | partial_count, |
| | | chunk_count, |
| | | audio_bytes, |
| | | wall_ms, |
| | | asr_result_ref_present = asr_result_ref.is_some(), |
| | | fallback_reason = ?fallback_reason, |
| | | fallback_stage = ?fallback_stage, |
| | | "runtime helper asr_realtime_finished" |
| | | ); |
| | | if status == "final" { |
| | | asr_result_ref |
| | | } else { |
| | | None |
| | | } |
| | | } |
| | | Err(error) => { |
| | | warn!( |
| | | call_id = %call_id, |
| | | trace_id = %trace_id, |
| | | turn_id = %turn.turn_id, |
| | | error = %safe_error(&error.to_string()), |
| | | "runtime helper asr_realtime_failed_fallback" |
| | | ); |
| | | None |
| | | } |
| | | } |
| | | } |
| | | |
| | | #[derive(Clone)] |
| | |
| | | rms_threshold: f64_env("CV_VAD_RMS_THRESHOLD", 0.012), |
| | | peak_threshold: f64_env("CV_VAD_PEAK_THRESHOLD", 0.08), |
| | | start_frames: u32_env("CV_VAD_START_FRAMES", 5).max(1), |
| | | end_silence_ms: u64_env("CV_VAD_END_SILENCE_MS", 700).max(100), |
| | | min_speech_ms: u64_env("CV_VAD_MIN_SPEECH_MS", 300).max(1), |
| | | end_silence_ms: u64_env("CV_VAD_END_SILENCE_MS", 400).max(100), |
| | | min_speech_ms: u64_env("CV_VAD_MIN_SPEECH_MS", 250).max(1), |
| | | max_turn_ms: u64_env("CV_VAD_MAX_TURN_MS", 10_000).max(1_000), |
| | | initial_ignore_ms: u64_env("CV_VAD_INITIAL_IGNORE_MS", 500), |
| | | } |
| | |
| | | return None; |
| | | } |
| | | |
| | | let voiced = rms >= self.config.rms_threshold || peak >= self.config.peak_threshold; |
| | | // Align with cb-sdk's energy-based segmentation: peak is diagnostic only, |
| | | // otherwise isolated spikes can keep a turn open until max_turn_ms. |
| | | let voiced = rms >= self.config.rms_threshold; |
| | | if !self.in_speech { |
| | | self.remember_pre_speech_frame(frame); |
| | | } |
| | |
| | | Ok(()) |
| | | } |
| | | } |
| | | |
| | | #[cfg(test)] |
| | | mod tests { |
| | | use super::*; |
| | | use std::{ |
| | | collections::{HashMap, HashSet}, |
| | | io::{Read, Write}, |
| | | net::TcpListener, |
| | | sync::{ |
| | | Arc, |
| | | atomic::{AtomicUsize, Ordering}, |
| | | mpsc, |
| | | }, |
| | | thread, |
| | | time::Duration, |
| | | }; |
| | | |
| | | #[derive(Debug)] |
| | | enum PreAudioOrderEvent { |
| | | DataReceived { |
| | | sender: String, |
| | | sequence: String, |
| | | }, |
| | | TrackSubscribed { |
| | | participant: String, |
| | | attributes: HashMap<String, String>, |
| | | }, |
| | | } |
| | | |
| | | fn drive_pre_audio_order_test_seam(events: &[PreAudioOrderEvent]) -> Vec<&'static str> { |
| | | let mut pending_sequence = None; |
| | | let mut effects = Vec::new(); |
| | | for event in events { |
| | | match event { |
| | | PreAudioOrderEvent::DataReceived { sender, sequence } if sender == "user-1" => { |
| | | pending_sequence = Some(sequence.as_str()); |
| | | } |
| | | PreAudioOrderEvent::TrackSubscribed { |
| | | participant, |
| | | attributes, |
| | | } => { |
| | | let pending = pending_sequence.is_some(); |
| | | let probe_result = pending_sequence.map(|sequence| { |
| | | classify_controlled_fixture_attributes( |
| | | participant, |
| | | Some("user-1"), |
| | | attributes, |
| | | sequence, |
| | | ) |
| | | .is_ok() |
| | | }); |
| | | let mut ack_observed = false; |
| | | let mut observer_started = false; |
| | | start_observer_after_controlled_fixture_probe(pending, probe_result, || { |
| | | if pending && probe_result == Some(true) { |
| | | ack_observed = true; |
| | | } |
| | | observer_started = true; |
| | | }); |
| | | if ack_observed { |
| | | effects.push("ack_observed"); |
| | | } |
| | | if observer_started { |
| | | effects.push("observer_started"); |
| | | } |
| | | pending_sequence = None; |
| | | } |
| | | PreAudioOrderEvent::DataReceived { .. } => {} |
| | | } |
| | | } |
| | | effects |
| | | } |
| | | |
| | | #[test] |
| | | fn production_vad_session_boundary_reads_updated_attributes() { |
| | | let config = SimpleVadConfig { |
| | | rms_threshold: 0.001, |
| | | peak_threshold: 0.01, |
| | | start_frames: 2, |
| | | end_silence_ms: 100, |
| | | min_speech_ms: 1, |
| | | max_turn_ms: 1_000, |
| | | initial_ignore_ms: 0, |
| | | }; |
| | | let mut vad = SimpleVad::new(config); |
| | | let samples = vec![1_000i16; 160]; |
| | | let frame = AudioFrame { |
| | | data: samples.as_slice().into(), |
| | | sample_rate: 16_000, |
| | | num_channels: 1, |
| | | samples_per_channel: 160, |
| | | }; |
| | | let mut attributes = std::collections::HashMap::from([ |
| | | ( |
| | | "inputSourceCategory".to_string(), |
| | | "controlled_fixture".to_string(), |
| | | ), |
| | | ( |
| | | "clientFixtureSequence".to_string(), |
| | | "fixture-01".to_string(), |
| | | ), |
| | | ]); |
| | | let mut starts = Vec::new(); |
| | | for (session_index, sequence) in [(1, "fixture-01"), (2, "fixture-02")] { |
| | | let was_in_speech = vad.in_speech; |
| | | vad.observe_frame( |
| | | "call-001", |
| | | "trace-001", |
| | | "participant", |
| | | "track", |
| | | session_index * 2 - 1, |
| | | 1_000 * session_index, |
| | | &frame, |
| | | ); |
| | | vad.observe_frame( |
| | | "call-001", |
| | | "trace-001", |
| | | "participant", |
| | | "track", |
| | | session_index * 2, |
| | | 1_000 * session_index + 10, |
| | | &frame, |
| | | ); |
| | | let is_in_speech = vad.in_speech; |
| | | assert!(!was_in_speech && is_in_speech); |
| | | attributes.insert("clientFixtureSequence".to_string(), sequence.to_string()); |
| | | let metadata = AudioIngressMetadata::from_participant(&attributes) |
| | | .expect("valid participant attributes") |
| | | .expect("controlled fixture metadata"); |
| | | let session_line = asr_realtime::session_start_line( |
| | | "call-001", |
| | | "trace-001", |
| | | &format!("turn-{session_index:04}"), |
| | | "nonce-001", |
| | | Some(&metadata), |
| | | ) |
| | | .expect("session start line"); |
| | | let session_json: serde_json::Value = |
| | | serde_json::from_slice(&session_line).expect("session start json"); |
| | | assert_eq!(sequence, session_json["clientFixtureSequence"]); |
| | | starts.push(metadata.client_fixture_sequence); |
| | | vad.reset_current_turn(); |
| | | } |
| | | assert_eq!(vec!["fixture-01", "fixture-02"], starts); |
| | | attributes.insert("inputSourceCategory".to_string(), "other".to_string()); |
| | | assert!(AudioIngressMetadata::from_participant(&attributes).is_err()); |
| | | assert!( |
| | | AudioIngressMetadata::from_participant(&std::collections::HashMap::new()) |
| | | .expect("missing attributes is absent") |
| | | .is_none() |
| | | ); |
| | | attributes.insert( |
| | | "clientFixtureSequence".to_string(), |
| | | "fixture-01".to_string(), |
| | | ); |
| | | assert!(AudioIngressMetadata::from_participant(&attributes).is_err()); |
| | | } |
| | | |
| | | #[test] |
| | | fn production_observer_rejects_wrong_participant_before_vad_session() { |
| | | assert!(!is_bound_user_participant( |
| | | "participant-other", |
| | | Some("participant-user") |
| | | )); |
| | | assert!(is_bound_user_participant( |
| | | "participant-user", |
| | | Some("participant-user") |
| | | )); |
| | | assert!(is_bound_user_participant("participant-any", None)); |
| | | } |
| | | |
| | | #[tokio::test] |
| | | async fn production_observer_vad_to_session_entry_reads_each_updated_attribute() { |
| | | let listener = TcpListener::bind("127.0.0.1:0").expect("bind local ASR fixture"); |
| | | let address = listener.local_addr().expect("fixture address"); |
| | | let (request_tx, request_rx) = mpsc::channel::<String>(); |
| | | let captured_count = Arc::new(AtomicUsize::new(0)); |
| | | let captured_count_for_server = Arc::clone(&captured_count); |
| | | let server = thread::spawn(move || { |
| | | for _ in 0..2 { |
| | | let (mut stream, _) = listener.accept().expect("accept ASR session"); |
| | | stream |
| | | .set_read_timeout(Some(Duration::from_secs(2))) |
| | | .expect("set fixture timeout"); |
| | | let mut bytes = Vec::new(); |
| | | let mut buffer = [0_u8; 4096]; |
| | | loop { |
| | | match stream.read(&mut buffer) { |
| | | Ok(0) => break, |
| | | Ok(size) => { |
| | | bytes.extend_from_slice(&buffer[..size]); |
| | | if bytes.windows(7).any(|window| window == b"0\r\n\r\n") { |
| | | break; |
| | | } |
| | | } |
| | | Err(_) => break, |
| | | } |
| | | } |
| | | request_tx |
| | | .send(String::from_utf8_lossy(&bytes).into_owned()) |
| | | .expect("capture ASR request"); |
| | | captured_count_for_server.fetch_add(1, Ordering::SeqCst); |
| | | stream |
| | | .write_all(b"HTTP/1.1 200 OK\r\ncontent-type: application/json\r\ncontent-length: 39\r\nconnection: close\r\n\r\n{\"code\":0,\"data\":{\"status\":\"ok\"}}") |
| | | .expect("write fixture response"); |
| | | } |
| | | }); |
| | | let mut vad = SimpleVad::new(SimpleVadConfig { |
| | | rms_threshold: 0.001, |
| | | peak_threshold: 0.01, |
| | | start_frames: 1, |
| | | end_silence_ms: 100, |
| | | min_speech_ms: 1, |
| | | max_turn_ms: 1_000, |
| | | initial_ignore_ms: 0, |
| | | }); |
| | | let frame_data = vec![1_000i16; 160]; |
| | | let frame = AudioFrame { |
| | | data: frame_data.as_slice().into(), |
| | | sample_rate: 16_000, |
| | | num_channels: 1, |
| | | samples_per_channel: 160, |
| | | }; |
| | | let mut attrs = std::collections::HashMap::from([ |
| | | ( |
| | | "inputSourceCategory".to_string(), |
| | | "controlled_fixture".to_string(), |
| | | ), |
| | | ( |
| | | "clientFixtureSequence".to_string(), |
| | | "fixture-01".to_string(), |
| | | ), |
| | | ]); |
| | | let mut upload = None; |
| | | let mut last_fixture_sequence = None; |
| | | let config = RealtimeAsrConfig { |
| | | enabled: true, |
| | | url: Some(format!("http://{address}/runtime/asr/realtime")), |
| | | runtime_token: Some("test".to_string()), |
| | | runtime_session_nonce: Some("test".to_string()), |
| | | chunk_duration_ms: 200, |
| | | }; |
| | | let (was, is, turn) = observe_bound_participant_frame( |
| | | "participant-user", |
| | | Some("participant-user"), |
| | | &mut vad, |
| | | "call-001", |
| | | "trace-001", |
| | | "participant-user", |
| | | "track-001", |
| | | 1, |
| | | 1_000, |
| | | &frame, |
| | | Client::new(), |
| | | config.clone(), |
| | | || attrs.clone(), |
| | | &mut upload, |
| | | &mut last_fixture_sequence, |
| | | true, |
| | | ); |
| | | assert!(!was && is && turn.is_none()); |
| | | assert!(upload.is_some()); |
| | | upload.take().unwrap().cancel("test").await; |
| | | vad.reset_current_turn(); |
| | | attrs.insert( |
| | | "clientFixtureSequence".to_string(), |
| | | "fixture-02".to_string(), |
| | | ); |
| | | let (was, is, turn) = observe_bound_participant_frame( |
| | | "participant-user", |
| | | Some("participant-user"), |
| | | &mut vad, |
| | | "call-001", |
| | | "trace-001", |
| | | "participant-user", |
| | | "track-001", |
| | | 2, |
| | | 2_000, |
| | | &frame, |
| | | Client::new(), |
| | | config, |
| | | || attrs.clone(), |
| | | &mut upload, |
| | | &mut last_fixture_sequence, |
| | | true, |
| | | ); |
| | | assert!(!was && is && turn.is_none()); |
| | | assert!(upload.is_some()); |
| | | upload.take().unwrap().cancel("test").await; |
| | | |
| | | let first_request = request_rx |
| | | .recv_timeout(Duration::from_secs(2)) |
| | | .expect("first session request"); |
| | | let second_request = request_rx |
| | | .recv_timeout(Duration::from_secs(2)) |
| | | .expect("second session request"); |
| | | assert!(first_request.contains("\"clientFixtureSequence\":\"fixture-01\"")); |
| | | assert!(second_request.contains("\"clientFixtureSequence\":\"fixture-02\"")); |
| | | assert!( |
| | | first_request.contains("\"audioIngressOriginStatus\":\"controlled_fixture_bound\"") |
| | | ); |
| | | assert!( |
| | | second_request.contains("\"audioIngressOriginStatus\":\"controlled_fixture_bound\"") |
| | | ); |
| | | // The same production boundary rejects a wrong participant before VAD/session creation. |
| | | assert!(!is_bound_user_participant( |
| | | "participant-other", |
| | | Some("participant-user") |
| | | )); |
| | | assert_eq!(0, request_rx.try_iter().count()); |
| | | vad.reset_current_turn(); |
| | | attrs.insert( |
| | | "clientFixtureSequence".to_string(), |
| | | "fixture-03".to_string(), |
| | | ); |
| | | let (_, is_wrong, wrong_turn) = observe_bound_participant_frame( |
| | | "participant-other", |
| | | Some("participant-user"), |
| | | &mut vad, |
| | | "call-001", |
| | | "trace-001", |
| | | "participant-user", |
| | | "track-001", |
| | | 3, |
| | | 3_000, |
| | | &frame, |
| | | Client::new(), |
| | | RealtimeAsrConfig { |
| | | enabled: true, |
| | | url: Some(format!("http://{address}/runtime/asr/realtime")), |
| | | runtime_token: Some("test".to_string()), |
| | | runtime_session_nonce: Some("test".to_string()), |
| | | chunk_duration_ms: 200, |
| | | }, |
| | | || attrs.clone(), |
| | | &mut upload, |
| | | &mut last_fixture_sequence, |
| | | true, |
| | | ); |
| | | assert!(!is_wrong && wrong_turn.is_none() && upload.is_none()); |
| | | vad.reset_current_turn(); |
| | | attrs.insert( |
| | | "clientFixtureSequence".to_string(), |
| | | "fixture-01".to_string(), |
| | | ); |
| | | let (_, _, _) = observe_bound_participant_frame( |
| | | "participant-user", |
| | | Some("participant-user"), |
| | | &mut vad, |
| | | "call-001", |
| | | "trace-001", |
| | | "participant-user", |
| | | "track-001", |
| | | 4, |
| | | 4_000, |
| | | &frame, |
| | | Client::new(), |
| | | RealtimeAsrConfig { |
| | | enabled: true, |
| | | url: Some(format!("http://{address}/runtime/asr/realtime")), |
| | | runtime_token: Some("test".to_string()), |
| | | runtime_session_nonce: Some("test".to_string()), |
| | | chunk_duration_ms: 200, |
| | | }, |
| | | || attrs.clone(), |
| | | &mut upload, |
| | | &mut last_fixture_sequence, |
| | | true, |
| | | ); |
| | | assert!(upload.is_none()); |
| | | vad.reset_current_turn(); |
| | | let (_, _, _) = observe_bound_participant_frame( |
| | | "participant-user", |
| | | Some("participant-user"), |
| | | &mut vad, |
| | | "call-001", |
| | | "trace-001", |
| | | "participant-user", |
| | | "track-001", |
| | | 5, |
| | | 5_000, |
| | | &frame, |
| | | Client::new(), |
| | | RealtimeAsrConfig { |
| | | enabled: true, |
| | | url: Some(format!("http://{address}/runtime/asr/realtime")), |
| | | runtime_token: Some("test".to_string()), |
| | | runtime_session_nonce: Some("test".to_string()), |
| | | chunk_duration_ms: 200, |
| | | }, |
| | | || attrs.clone(), |
| | | &mut upload, |
| | | &mut last_fixture_sequence, |
| | | true, |
| | | ); |
| | | assert!(upload.is_none()); |
| | | vad.reset_current_turn(); |
| | | attrs.remove("clientFixtureSequence"); |
| | | let (_, _, missing_sequence_turn) = observe_bound_participant_frame( |
| | | "participant-user", |
| | | Some("participant-user"), |
| | | &mut vad, |
| | | "call-001", |
| | | "trace-001", |
| | | "participant-user", |
| | | "track-001", |
| | | 6, |
| | | 6_000, |
| | | &frame, |
| | | Client::new(), |
| | | RealtimeAsrConfig { |
| | | enabled: true, |
| | | url: Some(format!("http://{address}/runtime/asr/realtime")), |
| | | runtime_token: Some("test".to_string()), |
| | | runtime_session_nonce: Some("test".to_string()), |
| | | chunk_duration_ms: 200, |
| | | }, |
| | | || attrs.clone(), |
| | | &mut upload, |
| | | &mut last_fixture_sequence, |
| | | true, |
| | | ); |
| | | assert!(missing_sequence_turn.is_none() && upload.is_none()); |
| | | assert_eq!( |
| | | "sequence_absent", |
| | | AudioIngressMetadata::rejected_origin_status("incomplete_metadata", &attrs) |
| | | ); |
| | | assert_eq!(2, captured_count.load(Ordering::SeqCst)); |
| | | vad.reset_current_turn(); |
| | | attrs.insert("inputSourceCategory".to_string(), "other".to_string()); |
| | | let mut invalid_upload = None; |
| | | let (_, _, invalid_turn) = observe_bound_participant_frame( |
| | | "participant-user", |
| | | Some("participant-user"), |
| | | &mut vad, |
| | | "call-001", |
| | | "trace-001", |
| | | "participant-other", |
| | | "track-001", |
| | | 3, |
| | | 3_000, |
| | | &frame, |
| | | Client::new(), |
| | | RealtimeAsrConfig { |
| | | enabled: true, |
| | | url: Some("http://127.0.0.1:9".to_string()), |
| | | runtime_token: Some("test".to_string()), |
| | | runtime_session_nonce: Some("test".to_string()), |
| | | chunk_duration_ms: 200, |
| | | }, |
| | | || attrs, |
| | | &mut invalid_upload, |
| | | &mut last_fixture_sequence, |
| | | true, |
| | | ); |
| | | assert!(invalid_turn.is_none()); |
| | | assert!(invalid_upload.is_none()); |
| | | assert_eq!(2, captured_count.load(Ordering::SeqCst)); |
| | | assert_eq!(0, request_rx.try_iter().count()); |
| | | server.join().expect("fixture server"); |
| | | } |
| | | |
| | | #[test] |
| | | fn reply_chunk_marker_state_emits_turn_first_once_and_later_segment_first_once() { |
| | | let mut state = ReplyChunkMarkerState::default(); |
| | | |
| | | assert_eq!(ReplyChunkMarker::FirstReply, state.observe(Some(1))); |
| | | assert_eq!(ReplyChunkMarker::None, state.observe(Some(1))); |
| | | assert_eq!(ReplyChunkMarker::SegmentFirst, state.observe(Some(2))); |
| | | assert_eq!(ReplyChunkMarker::None, state.observe(Some(2))); |
| | | assert_eq!(ReplyChunkMarker::SegmentFirst, state.observe(Some(3))); |
| | | } |
| | | |
| | | #[test] |
| | | fn reply_chunk_marker_state_without_segment_only_emits_turn_first() { |
| | | let mut state = ReplyChunkMarkerState::default(); |
| | | |
| | | assert_eq!(ReplyChunkMarker::FirstReply, state.observe(None)); |
| | | assert_eq!(ReplyChunkMarker::None, state.observe(None)); |
| | | } |
| | | |
| | | #[test] |
| | | fn runtime_turn_stream_audio_chunk_reads_segment_seq() { |
| | | let event: RuntimeTurnStreamEvent = serde_json::from_str( |
| | | r#"{"type":"reply_audio_chunk","audioChunk":{"chunkSeq":4,"segmentSeq":2,"format":"pcm_s16le","payloadBase64":"AA==","last":false}}"#, |
| | | ) |
| | | .expect("turn stream event"); |
| | | |
| | | assert_eq!( |
| | | Some(2), |
| | | event.audio_chunk.and_then(|chunk| chunk.segment_seq) |
| | | ); |
| | | } |
| | | |
| | | #[test] |
| | | fn runtime_turn_stream_reads_frozen_m5_and_audio_chunk_timing_contract() { |
| | | let activity: RuntimeTurnStreamEvent = serde_json::from_str( |
| | | r#"{"type":"activity","callId":"call-1","traceId":"trace-1","turnId":"turn-1","activity":{"eventType":"tts_first_audio_chunk_ready","extension":{"streamTimingVersion":1,"streamAnchorId":"0123456789abcdef","streamAnchorServerDeltaMs":1200,"runtimeSessionNonceHash":"abcdef012345","segmentSeq":1,"streamTimingValidation":"bound"}}}"#, |
| | | ) |
| | | .expect("m5 activity event"); |
| | | let audio: RuntimeTurnStreamEvent = serde_json::from_str( |
| | | r#"{"type":"reply_audio_chunk","callId":"call-1","traceId":"trace-1","turnId":"turn-1","audioChunk":{"chunkSeq":1,"segmentSeq":1,"streamTimingVersion":1,"streamAnchorId":"0123456789abcdef","format":"pcm_s16le","payloadBase64":"AA==","last":false}}"#, |
| | | ) |
| | | .expect("timed audio chunk event"); |
| | | |
| | | assert_eq!(Some("call-1"), activity.call_id.as_deref()); |
| | | assert_eq!(Some("trace-1"), activity.trace_id.as_deref()); |
| | | assert_eq!(Some("turn-1"), activity.turn_id.as_deref()); |
| | | let extension = activity.activity.unwrap().extension.unwrap(); |
| | | assert_eq!(Some(1), extension.stream_timing_version); |
| | | assert_eq!(Some(1200), extension.stream_anchor_server_delta_ms); |
| | | assert_eq!( |
| | | Some("0123456789abcdef"), |
| | | audio.audio_chunk.unwrap().stream_anchor_id.as_deref() |
| | | ); |
| | | } |
| | | |
| | | #[test] |
| | | fn runtime_turn_stream_timing_records_m6_and_m7_once_then_rejects_terminal_late_events() { |
| | | let nonce = "runtime-nonce"; |
| | | let nonce_hash = runtime_session_nonce_hash(nonce); |
| | | let activity: RuntimeTurnStreamEvent = serde_json::from_str(&format!( |
| | | r#"{{"type":"activity","callId":"call-1","traceId":"trace-1","turnId":"turn-1","activity":{{"eventType":"tts_first_audio_chunk_ready","extension":{{"streamTimingVersion":1,"streamAnchorId":"0123456789abcdef","streamAnchorServerDeltaMs":1200,"runtimeSessionNonceHash":"{nonce_hash}","segmentSeq":1,"streamTimingValidation":"bound"}}}}}}"#, |
| | | )) |
| | | .expect("m5 activity event"); |
| | | let audio: RuntimeTurnStreamEvent = serde_json::from_str( |
| | | r#"{"type":"reply_audio_chunk","callId":"call-1","traceId":"trace-1","turnId":"turn-1","audioChunk":{"chunkSeq":1,"segmentSeq":1,"streamTimingVersion":1,"streamAnchorId":"0123456789abcdef","format":"pcm_s16le","payloadBase64":"AA==","last":false}}"#, |
| | | ) |
| | | .expect("timed audio chunk event"); |
| | | let mut state = RuntimeTurnStreamState::default(); |
| | | |
| | | assert!(state.arm_timing_anchor("call-1", "trace-1", "turn-1", nonce, &activity)); |
| | | let chunk = audio.audio_chunk.as_ref().expect("audio chunk"); |
| | | assert!( |
| | | state |
| | | .record_m6("call-1", "trace-1", "turn-1", &audio, chunk) |
| | | .is_some() |
| | | ); |
| | | assert!( |
| | | state |
| | | .record_m6("call-1", "trace-1", "turn-1", &audio, chunk) |
| | | .is_none() |
| | | ); |
| | | assert!(state.record_m7().is_some()); |
| | | assert!(state.record_m7().is_none()); |
| | | |
| | | state.close_timing(); |
| | | assert_eq!(RuntimeTurnStreamTimingPhase::Closed, state.timing.phase); |
| | | assert!(state.timing.anchor.is_none()); |
| | | assert!(!state.arm_timing_anchor("call-1", "trace-1", "turn-1", nonce, &activity)); |
| | | assert!( |
| | | state |
| | | .record_m6("call-1", "trace-1", "turn-1", &audio, chunk) |
| | | .is_none() |
| | | ); |
| | | assert!(state.record_m7().is_none()); |
| | | } |
| | | |
| | | #[test] |
| | | fn device_output_contract_is_reliable_and_deduplicated() { |
| | | let output = RuntimeTurnDeviceOutput { |
| | | command_id: Some("cmd-1".to_string()), |
| | | command_code: Some("custom.app.DeviceLevelChange".to_string()), |
| | | params: Some(serde_json::json!({"level": 1})), |
| | | }; |
| | | let mut published = HashSet::new(); |
| | | assert!(should_publish_device_output(&mut published, &output)); |
| | | assert!(!should_publish_device_output(&mut published, &output)); |
| | | assert_eq!(published.len(), 1); |
| | | } |
| | | |
| | | #[test] |
| | | fn device_output_invalid_or_missing_command_is_fail_closed() { |
| | | for output in [ |
| | | RuntimeTurnDeviceOutput { |
| | | command_id: None, |
| | | command_code: Some("custom.app.DeviceLevelChange".to_string()), |
| | | params: None, |
| | | }, |
| | | RuntimeTurnDeviceOutput { |
| | | command_id: Some("cmd-1".to_string()), |
| | | command_code: None, |
| | | params: None, |
| | | }, |
| | | ] { |
| | | let mut published = HashSet::new(); |
| | | assert!(!should_publish_device_output(&mut published, &output)); |
| | | assert!(published.is_empty()); |
| | | } |
| | | } |
| | | |
| | | #[test] |
| | | fn controlled_fixture_probe_requires_bound_hashes_and_protocol() { |
| | | let call_id = "call-ack-1"; |
| | | let trace_id = "trace-ack-1"; |
| | | let payload = serde_json::to_vec(&json!({ |
| | | "type": CONTROLLED_FIXTURE_PROBE_TOPIC, |
| | | "protocolVersion": CONTROLLED_FIXTURE_PROTOCOL_VERSION, |
| | | "callIdHash": sha256_hex(call_id), |
| | | "callTraceIdHash": sha256_hex(trace_id), |
| | | "generation": CONTROLLED_FIXTURE_GENERATION, |
| | | "clientFixtureSequence": "fixture-01" |
| | | })) |
| | | .unwrap(); |
| | | let sender = ParticipantIdentity("user-1".to_string()); |
| | | let pending = |
| | | controlled_fixture_probe(&payload, call_id, trace_id, &sender, Some("user-1")) |
| | | .expect("valid probe"); |
| | | assert_eq!(pending.sequence, "fixture-01"); |
| | | assert_eq!(pending.call_id_hash, sha256_hex(call_id)); |
| | | assert_eq!(pending.call_trace_id_hash, sha256_hex(trace_id)); |
| | | assert_eq!(pending.generation, CONTROLLED_FIXTURE_GENERATION); |
| | | assert!( |
| | | controlled_fixture_probe(&payload, call_id, "other-trace", &sender, Some("user-1"),) |
| | | .is_none() |
| | | ); |
| | | assert!( |
| | | controlled_fixture_probe( |
| | | &payload, |
| | | call_id, |
| | | trace_id, |
| | | &ParticipantIdentity("other-user".to_string()), |
| | | Some("user-1"), |
| | | ) |
| | | .is_none() |
| | | ); |
| | | } |
| | | |
| | | #[test] |
| | | fn controlled_fixture_attributes_ack_only_on_exact_current_sequence() { |
| | | let mut attributes = std::collections::HashMap::from([ |
| | | ( |
| | | "inputSourceCategory".to_string(), |
| | | "controlled_fixture".to_string(), |
| | | ), |
| | | ( |
| | | "clientFixtureSequence".to_string(), |
| | | "fixture-01".to_string(), |
| | | ), |
| | | ]); |
| | | assert!( |
| | | classify_controlled_fixture_attributes( |
| | | "user-1", |
| | | Some("user-1"), |
| | | &attributes, |
| | | "fixture-01" |
| | | ) |
| | | .is_ok() |
| | | ); |
| | | attributes.insert( |
| | | "clientFixtureSequence".to_string(), |
| | | "fixture-02".to_string(), |
| | | ); |
| | | assert_eq!( |
| | | classify_controlled_fixture_attributes( |
| | | "user-1", |
| | | Some("user-1"), |
| | | &attributes, |
| | | "fixture-01" |
| | | ), |
| | | Err("wrong_sequence") |
| | | ); |
| | | attributes.remove("clientFixtureSequence"); |
| | | assert_eq!( |
| | | classify_controlled_fixture_attributes( |
| | | "user-1", |
| | | Some("user-1"), |
| | | &attributes, |
| | | "fixture-01" |
| | | ), |
| | | Err("missing_sequence") |
| | | ); |
| | | assert_eq!( |
| | | classify_controlled_fixture_attributes( |
| | | "other-user", |
| | | Some("user-1"), |
| | | &attributes, |
| | | "fixture-01" |
| | | ), |
| | | Err("wrong_participant") |
| | | ); |
| | | } |
| | | |
| | | #[test] |
| | | fn controlled_fixture_probe_runtime_projection_binds_request_hashes_and_audio_gate() { |
| | | let call_id = "private-call-value"; |
| | | let trace_id = "private-trace-value"; |
| | | let sequence = "private-sequence-value"; |
| | | let call_id_hash = sha256_hex(call_id); |
| | | let trace_id_hash = sha256_hex(trace_id); |
| | | let mut observer_starts = 0; |
| | | let (ack_result, reject_reason, observed) = |
| | | controlled_fixture_ack_classification(Err("wrong_source")); |
| | | let stages = [ |
| | | ("data_received", None, None), |
| | | ("attributes_classified", Some(ack_result), reject_reason), |
| | | ("ack_publish_started", Some(ack_result), reject_reason), |
| | | ("ack_publish_completed", Some(ack_result), reject_reason), |
| | | ("audio_observer_blocked", Some(ack_result), reject_reason), |
| | | ]; |
| | | for (stage, result, reason) in stages { |
| | | let event = controlled_fixture_probe_event( |
| | | call_id, |
| | | trace_id, |
| | | stage, |
| | | &call_id_hash, |
| | | &trace_id_hash, |
| | | CONTROLLED_FIXTURE_GENERATION, |
| | | sequence, |
| | | observed, |
| | | result, |
| | | reason, |
| | | ); |
| | | assert_eq!(event["type"], "cv_activity"); |
| | | assert_eq!(event["eventName"], "controlled_fixture_attribute_probe"); |
| | | assert_eq!(event["extension"]["call_id_hash"], call_id_hash); |
| | | assert_eq!(event["extension"]["trace_id_hash"], trace_id_hash); |
| | | assert_eq!(event["extension"]["stage"], stage); |
| | | let output = event.to_string(); |
| | | assert!(!output.contains(sequence)); |
| | | } |
| | | assert!(!start_observer_after_controlled_fixture_probe( |
| | | true, |
| | | Some(observed), |
| | | || observer_starts += 1, |
| | | )); |
| | | assert_eq!(observer_starts, 0); |
| | | |
| | | let (ack_result, reject_reason, observed) = controlled_fixture_ack_classification(Ok(())); |
| | | let allowed = controlled_fixture_probe_event( |
| | | call_id, |
| | | trace_id, |
| | | "audio_observer_allowed", |
| | | &call_id_hash, |
| | | &trace_id_hash, |
| | | CONTROLLED_FIXTURE_GENERATION, |
| | | sequence, |
| | | observed, |
| | | Some(ack_result), |
| | | reject_reason, |
| | | ); |
| | | assert_eq!(allowed["extension"]["trace_id_hash"], trace_id_hash); |
| | | assert!(start_observer_after_controlled_fixture_probe( |
| | | true, |
| | | Some(observed), |
| | | || observer_starts += 1, |
| | | )); |
| | | assert_eq!(observer_starts, 1); |
| | | } |
| | | |
| | | #[test] |
| | | fn controlled_fixture_probe_observed_and_failure_enums_are_stable() { |
| | | assert_eq!( |
| | | controlled_fixture_ack_classification(Ok(())), |
| | | ("observed", None, true) |
| | | ); |
| | | assert_eq!( |
| | | controlled_fixture_ack_classification(Err("timeout")), |
| | | ("timeout", Some("expired"), false) |
| | | ); |
| | | for reason in [ |
| | | "missing_attributes", |
| | | "wrong_source", |
| | | "missing_sequence", |
| | | "wrong_sequence", |
| | | "wrong_participant", |
| | | ] { |
| | | assert_eq!( |
| | | controlled_fixture_ack_classification(Err(reason)), |
| | | ("rejected", Some(reason), false) |
| | | ); |
| | | } |
| | | assert_eq!( |
| | | controlled_fixture_ack_classification(Err("unclassified")), |
| | | ("rejected", Some("unknown"), false) |
| | | ); |
| | | } |
| | | |
| | | #[test] |
| | | fn production_probe_binding_drives_rejected_and_observed_ack_without_local_rehash() { |
| | | let request_call_hash = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; |
| | | let request_trace_hash = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"; |
| | | let probe = PendingControlledFixtureProbe { |
| | | sender: ParticipantIdentity("user-1".to_string()), |
| | | call_id_hash: request_call_hash.to_string(), |
| | | call_trace_id_hash: request_trace_hash.to_string(), |
| | | generation: CONTROLLED_FIXTURE_GENERATION, |
| | | sequence: "fixture-01".to_string(), |
| | | received_at: Instant::now(), |
| | | expires_at: Instant::now() + CONTROLLED_FIXTURE_PROBE_TTL, |
| | | }; |
| | | |
| | | let (ack_result, reject_reason, observed) = |
| | | controlled_fixture_ack_classification(Err("wrong_sequence")); |
| | | let rejected_ack = controlled_fixture_ack_from_probe(&probe, observed, reject_reason); |
| | | let rejected_json = serde_json::to_value(&rejected_ack).unwrap(); |
| | | let mut rejected_observer_starts = 0; |
| | | assert_eq!(ack_result, "rejected"); |
| | | assert_eq!(rejected_json["callIdHash"], request_call_hash); |
| | | assert_eq!(rejected_json["callTraceIdHash"], request_trace_hash); |
| | | assert_eq!(rejected_json["rejectReason"], "wrong_sequence"); |
| | | assert!(!start_observer_after_controlled_fixture_probe( |
| | | true, |
| | | Some(observed), |
| | | || rejected_observer_starts += 1, |
| | | )); |
| | | assert_eq!(rejected_observer_starts, 0); |
| | | |
| | | let (ack_result, reject_reason, observed) = controlled_fixture_ack_classification(Ok(())); |
| | | let observed_ack = controlled_fixture_ack_from_probe(&probe, observed, reject_reason); |
| | | let observed_json = serde_json::to_value(&observed_ack).unwrap(); |
| | | let mut observed_observer_starts = 0; |
| | | assert_eq!(ack_result, "observed"); |
| | | assert_eq!(observed_json["callIdHash"], request_call_hash); |
| | | assert_eq!(observed_json["callTraceIdHash"], request_trace_hash); |
| | | assert!(observed_json.get("rejectReason").is_none()); |
| | | assert!(start_observer_after_controlled_fixture_probe( |
| | | true, |
| | | Some(observed), |
| | | || observed_observer_starts += 1, |
| | | )); |
| | | assert_eq!(observed_observer_starts, 1); |
| | | } |
| | | |
| | | #[tokio::test] |
| | | async fn production_ack_publish_failure_keeps_observer_session_and_audio_closed() { |
| | | let mut acknowledged = HashSet::new(); |
| | | let probe_result = complete_controlled_fixture_ack_publish( |
| | | async { Err::<(), ()>(()) }, |
| | | "runtime-call-publish-failure", |
| | | "runtime-trace-publish-failure", |
| | | true, |
| | | "observed", |
| | | None, |
| | | "call-publish-failure", |
| | | "trace-publish-failure", |
| | | CONTROLLED_FIXTURE_GENERATION, |
| | | "fixture-01", |
| | | &mut acknowledged, |
| | | ) |
| | | .await; |
| | | let mut observer_starts = 0; |
| | | let mut session_starts = 0; |
| | | let mut audio_starts = 0; |
| | | let mut speaking_starts = 0; |
| | | assert!(!start_observer_after_controlled_fixture_probe( |
| | | true, |
| | | Some(probe_result.observed), |
| | | || { |
| | | observer_starts += 1; |
| | | session_starts += 1; |
| | | audio_starts += 1; |
| | | speaking_starts += 1; |
| | | }, |
| | | )); |
| | | assert_eq!( |
| | | probe_result, |
| | | ControlledFixtureAckPublishOutcome { |
| | | observed: false, |
| | | published: false, |
| | | } |
| | | ); |
| | | assert!(acknowledged.is_empty()); |
| | | assert_eq!(observer_starts, 0); |
| | | assert_eq!(session_starts, 0); |
| | | assert_eq!(audio_starts, 0); |
| | | assert_eq!(speaking_starts, 0); |
| | | |
| | | let observed_result = complete_controlled_fixture_ack_publish( |
| | | async { Ok::<(), ()>(()) }, |
| | | "runtime-call-publish-success", |
| | | "runtime-trace-publish-success", |
| | | true, |
| | | "observed", |
| | | None, |
| | | "call-publish-success", |
| | | "trace-publish-success", |
| | | CONTROLLED_FIXTURE_GENERATION, |
| | | "fixture-02", |
| | | &mut acknowledged, |
| | | ) |
| | | .await; |
| | | let mut successful_observer_starts = 0; |
| | | assert!(start_observer_after_controlled_fixture_probe( |
| | | true, |
| | | Some(observed_result.observed), |
| | | || successful_observer_starts += 1, |
| | | )); |
| | | assert_eq!( |
| | | observed_result, |
| | | ControlledFixtureAckPublishOutcome { |
| | | observed: true, |
| | | published: true, |
| | | } |
| | | ); |
| | | assert!(acknowledged.contains("fixture-02")); |
| | | assert_eq!(successful_observer_starts, 1); |
| | | } |
| | | |
| | | #[tokio::test] |
| | | async fn production_attribute_observation_accepts_server_visibility_within_probe_ttl() { |
| | | let expected = HashMap::from([ |
| | | ( |
| | | "inputSourceCategory".to_string(), |
| | | "controlled_fixture".to_string(), |
| | | ), |
| | | ( |
| | | "clientFixtureSequence".to_string(), |
| | | "fixture-01".to_string(), |
| | | ), |
| | | ]); |
| | | let mut reads = 0; |
| | | let result = observe_controlled_fixture_attributes( |
| | | Instant::now() + CONTROLLED_FIXTURE_PROBE_TTL, |
| | | "user-1", |
| | | Some("user-1"), |
| | | "fixture-01", |
| | | || { |
| | | reads += 1; |
| | | if reads <= 3 { |
| | | HashMap::new() |
| | | } else { |
| | | expected.clone() |
| | | } |
| | | }, |
| | | ) |
| | | .await; |
| | | assert_eq!(result, Ok(())); |
| | | assert_eq!(reads, 4); |
| | | } |
| | | |
| | | #[tokio::test] |
| | | async fn production_attribute_observation_rejects_wrong_sequence_without_audio_effect() { |
| | | let attributes = HashMap::from([ |
| | | ( |
| | | "inputSourceCategory".to_string(), |
| | | "controlled_fixture".to_string(), |
| | | ), |
| | | ( |
| | | "clientFixtureSequence".to_string(), |
| | | "fixture-02".to_string(), |
| | | ), |
| | | ]); |
| | | let result = observe_controlled_fixture_attributes( |
| | | Instant::now() + CONTROLLED_FIXTURE_PROBE_TTL, |
| | | "user-1", |
| | | Some("user-1"), |
| | | "fixture-01", |
| | | || attributes.clone(), |
| | | ) |
| | | .await; |
| | | let mut observer_starts = 0; |
| | | assert_eq!(result, Err("wrong_sequence")); |
| | | assert!(!start_observer_after_controlled_fixture_probe( |
| | | true, |
| | | Some(result.is_ok()), |
| | | || observer_starts += 1, |
| | | )); |
| | | assert_eq!(observer_starts, 0); |
| | | } |
| | | |
| | | #[tokio::test] |
| | | async fn production_post_expiry_observation_records_bounded_visibility_without_second_ack() { |
| | | let started_at = Instant::now(); |
| | | let active = Arc::new(AtomicBool::new(true)); |
| | | let expected = HashMap::from([ |
| | | ( |
| | | "inputSourceCategory".to_string(), |
| | | "controlled_fixture".to_string(), |
| | | ), |
| | | ( |
| | | "clientFixtureSequence".to_string(), |
| | | "fixture-01".to_string(), |
| | | ), |
| | | ]); |
| | | let mut acknowledged = HashSet::new(); |
| | | let expired_ack = complete_controlled_fixture_ack_publish( |
| | | async { Ok::<(), ()>(()) }, |
| | | "runtime-call-post-expiry", |
| | | "runtime-trace-post-expiry", |
| | | false, |
| | | "timeout", |
| | | Some("expired"), |
| | | "call-post-expiry", |
| | | "trace-post-expiry", |
| | | CONTROLLED_FIXTURE_GENERATION, |
| | | "fixture-01", |
| | | &mut acknowledged, |
| | | ) |
| | | .await; |
| | | assert_eq!( |
| | | expired_ack, |
| | | ControlledFixtureAckPublishOutcome { |
| | | observed: false, |
| | | published: true, |
| | | } |
| | | ); |
| | | let evidence = observe_controlled_fixture_post_expiry( |
| | | started_at, |
| | | started_at + CONTROLLED_FIXTURE_POST_EXPIRY_WINDOW, |
| | | "user-1", |
| | | Some("user-1"), |
| | | "fixture-01", |
| | | active, |
| | | || { |
| | | if started_at.elapsed() >= Duration::from_millis(300) { |
| | | expected.clone() |
| | | } else { |
| | | HashMap::new() |
| | | } |
| | | }, |
| | | ) |
| | | .await; |
| | | assert_eq!(acknowledged.len(), 1); |
| | | assert_eq!( |
| | | evidence, |
| | | Some(ControlledFixtureVisibilityEvidence { |
| | | first_visible_bucket: "250_500ms", |
| | | visibility_source: "participant_attributes_poll", |
| | | binding_matched: true, |
| | | }) |
| | | ); |
| | | |
| | | let never_started_at = Instant::now(); |
| | | assert_eq!( |
| | | observe_controlled_fixture_post_expiry( |
| | | never_started_at, |
| | | never_started_at + Duration::from_millis(40), |
| | | "user-1", |
| | | Some("user-1"), |
| | | "fixture-01", |
| | | Arc::new(AtomicBool::new(true)), |
| | | HashMap::new, |
| | | ) |
| | | .await, |
| | | Some(ControlledFixtureVisibilityEvidence { |
| | | first_visible_bucket: "never_visible_within_observation_window", |
| | | visibility_source: "participant_attributes_poll", |
| | | binding_matched: true, |
| | | }) |
| | | ); |
| | | |
| | | assert_eq!( |
| | | observe_controlled_fixture_post_expiry( |
| | | Instant::now(), |
| | | Instant::now() + Duration::from_millis(50), |
| | | "cross-call-user", |
| | | Some("user-1"), |
| | | "fixture-01", |
| | | Arc::new(AtomicBool::new(true)), |
| | | || expected.clone(), |
| | | ) |
| | | .await, |
| | | None |
| | | ); |
| | | |
| | | let inactive = Arc::new(AtomicBool::new(false)); |
| | | assert_eq!( |
| | | observe_controlled_fixture_post_expiry( |
| | | Instant::now(), |
| | | Instant::now() + Duration::from_millis(50), |
| | | "user-1", |
| | | Some("user-1"), |
| | | "fixture-01", |
| | | inactive, |
| | | HashMap::new, |
| | | ) |
| | | .await, |
| | | None |
| | | ); |
| | | assert_eq!(acknowledged.len(), 1); |
| | | |
| | | let probe = PendingControlledFixtureProbe { |
| | | sender: ParticipantIdentity("user-1".to_string()), |
| | | call_id_hash: "call-post-expiry".to_string(), |
| | | call_trace_id_hash: "trace-post-expiry".to_string(), |
| | | generation: CONTROLLED_FIXTURE_GENERATION, |
| | | sequence: "fixture-01".to_string(), |
| | | received_at: started_at, |
| | | expires_at: started_at + CONTROLLED_FIXTURE_PROBE_TTL, |
| | | }; |
| | | let event = controlled_fixture_visibility_event( |
| | | "runtime-call-post-expiry", |
| | | "runtime-trace-post-expiry", |
| | | &probe, |
| | | &evidence.unwrap(), |
| | | ); |
| | | let extension = event["extension"].as_object().unwrap(); |
| | | let mut keys = extension.keys().map(String::as_str).collect::<Vec<_>>(); |
| | | keys.sort_unstable(); |
| | | assert_eq!( |
| | | keys, |
| | | vec![ |
| | | "binding_matched", |
| | | "call_id_hash", |
| | | "evidence_count", |
| | | "first_visible_bucket", |
| | | "generation", |
| | | "sequence_hash", |
| | | "stage", |
| | | "trace_id_hash", |
| | | "visibility_source", |
| | | ] |
| | | ); |
| | | let encoded = event.to_string(); |
| | | for forbidden in [ |
| | | "\"participant\":", |
| | | "\"room\":", |
| | | "\"track\":", |
| | | "\"payload\":", |
| | | "\"audio\":", |
| | | ] { |
| | | assert!(!encoded.contains(forbidden)); |
| | | } |
| | | } |
| | | |
| | | #[test] |
| | | fn controlled_fixture_ack_payload_is_reliable_and_redacted() { |
| | | let ack = ControlledFixtureAttributeAck { |
| | | message_type: CONTROLLED_FIXTURE_ACK_TOPIC, |
| | | protocol_version: CONTROLLED_FIXTURE_PROTOCOL_VERSION, |
| | | call_id_hash: sha256_hex("call-1"), |
| | | call_trace_id_hash: sha256_hex("trace-1"), |
| | | generation: CONTROLLED_FIXTURE_GENERATION, |
| | | client_fixture_sequence: "fixture-01".to_string(), |
| | | result: "observed", |
| | | input_source_category: Some("controlled_fixture"), |
| | | reject_reason: None, |
| | | }; |
| | | let encoded = serde_json::to_vec(&ack).unwrap(); |
| | | let decoded: serde_json::Value = serde_json::from_slice(&encoded).unwrap(); |
| | | assert_eq!(decoded["type"], CONTROLLED_FIXTURE_ACK_TOPIC); |
| | | assert_eq!( |
| | | decoded["protocolVersion"], |
| | | CONTROLLED_FIXTURE_PROTOCOL_VERSION |
| | | ); |
| | | assert_eq!(decoded["result"], "observed"); |
| | | assert!(decoded.get("callId").is_none()); |
| | | assert!(decoded.get("traceId").is_none()); |
| | | assert!(decoded.get("participantIdentity").is_none()); |
| | | assert!(decoded.get("audio").is_none()); |
| | | } |
| | | |
| | | #[test] |
| | | fn controlled_fixture_probe_does_not_create_session_or_audio_side_effects() { |
| | | let attributes = std::collections::HashMap::new(); |
| | | assert_eq!( |
| | | classify_controlled_fixture_attributes( |
| | | "user-1", |
| | | Some("user-1"), |
| | | &attributes, |
| | | "fixture-01" |
| | | ), |
| | | Err("missing_attributes") |
| | | ); |
| | | assert_eq!( |
| | | CONTROLLED_FIXTURE_PROBE_TOPIC, |
| | | "controlled_fixture_attribute_probe" |
| | | ); |
| | | assert_eq!( |
| | | CONTROLLED_FIXTURE_ACK_TOPIC, |
| | | "controlled_fixture_attribute_ack" |
| | | ); |
| | | } |
| | | |
| | | #[test] |
| | | fn controlled_fixture_probe_must_be_observed_before_audio_observer() { |
| | | assert!(controlled_fixture_observer_gate(false, None)); |
| | | assert!(controlled_fixture_observer_gate(true, Some(true))); |
| | | assert!(!controlled_fixture_observer_gate(true, Some(false))); |
| | | assert!(!controlled_fixture_observer_gate(true, None)); |
| | | } |
| | | |
| | | #[test] |
| | | fn production_event_order_probe_then_track_publishes_ack_before_observer() { |
| | | let attributes = HashMap::from([ |
| | | ( |
| | | "inputSourceCategory".to_string(), |
| | | "controlled_fixture".to_string(), |
| | | ), |
| | | ( |
| | | "clientFixtureSequence".to_string(), |
| | | "fixture-01".to_string(), |
| | | ), |
| | | ]); |
| | | let effects = drive_pre_audio_order_test_seam(&[ |
| | | PreAudioOrderEvent::DataReceived { |
| | | sender: "user-1".to_string(), |
| | | sequence: "fixture-01".to_string(), |
| | | }, |
| | | PreAudioOrderEvent::TrackSubscribed { |
| | | participant: "user-1".to_string(), |
| | | attributes, |
| | | }, |
| | | ]); |
| | | assert_eq!(effects, ["ack_observed", "observer_started"]); |
| | | } |
| | | |
| | | #[test] |
| | | fn production_event_order_negative_probe_has_no_observer_or_session_effect() { |
| | | let mut invalid = HashMap::new(); |
| | | invalid.insert( |
| | | "inputSourceCategory".to_string(), |
| | | "ordinary_mic".to_string(), |
| | | ); |
| | | let effects = drive_pre_audio_order_test_seam(&[ |
| | | PreAudioOrderEvent::DataReceived { |
| | | sender: "user-1".to_string(), |
| | | sequence: "fixture-01".to_string(), |
| | | }, |
| | | PreAudioOrderEvent::TrackSubscribed { |
| | | participant: "user-1".to_string(), |
| | | attributes: invalid, |
| | | }, |
| | | ]); |
| | | assert!(effects.is_empty()); |
| | | } |
| | | |
| | | #[test] |
| | | fn production_audio_branch_orders_probe_before_spawn_callsite() { |
| | | let source = include_str!("main.rs"); |
| | | let branch = source |
| | | .find("RoomEvent::TrackSubscribed {\n track: RemoteTrack::Audio") |
| | | .expect("audio TrackSubscribed production branch"); |
| | | let branch_source = &source[branch..]; |
| | | let probe = branch_source |
| | | .find("let probe_result = process_controlled_fixture_probe") |
| | | .expect("probe must be processed in audio branch"); |
| | | let spawn = branch_source |
| | | .find("start_observer_after_controlled_fixture_probe") |
| | | .expect("spawn must use shared order entry"); |
| | | assert!( |
| | | probe < spawn, |
| | | "probe must precede shared observer spawn entry" |
| | | ); |
| | | } |
| | | } |