| | |
| | | |
| | | use std::{ |
| | | borrow::Cow, |
| | | collections::HashSet, |
| | | env, fs, |
| | | future::Future, |
| | | path::{Path, PathBuf}, |
| | | sync::{ |
| | | Arc, |
| | |
| | | }; |
| | | |
| | | use anyhow::{Context, Result, anyhow}; |
| | | use asr_realtime::{RealtimeAsrConfig, RealtimeAsrOutcome, RealtimeAsrUpload}; |
| | | use asr_realtime::{ |
| | | AudioIngressMetadata, RealtimeAsrConfig, RealtimeAsrOutcome, RealtimeAsrUpload, |
| | | }; |
| | | use audio::{AudioDiagnostics, load_pre_recorded_frames}; |
| | | use base64::{Engine as _, engine::general_purpose}; |
| | | use futures_util::StreamExt; |
| | |
| | | options::TrackPublishOptions, |
| | | prelude::{ |
| | | DataPacket, LocalAudioTrack, LocalTrack, ParticipantIdentity, RemoteAudioTrack, |
| | | RemoteTrack, Room, RoomEvent, RoomOptions, |
| | | RemoteParticipant, RemoteTrack, Room, RoomEvent, RoomOptions, |
| | | }, |
| | | }; |
| | | use reqwest::Client; |
| | | use serde::{Deserialize, Serialize}; |
| | | use serde_json::json; |
| | | use sha2::{Digest, Sha256}; |
| | | use tokio::time::{sleep, sleep_until, timeout}; |
| | | use tokio::{ |
| | | sync::{mpsc, mpsc::UnboundedReceiver, watch}, |
| | |
| | | const PCM_16K_SAMPLE_RATE_HZ: u32 = 16_000; |
| | | const BOT_NUM_CHANNELS: u16 = 1; |
| | | const TRACK_NAME: &str = "bot-main-audio"; |
| | | const STREAM_TIMING_VERSION: u32 = 1; |
| | | const STREAM_TIMING_FIRST_SEGMENT: u64 = 1; |
| | | const STREAM_TIMING_FIRST_CHUNK: u64 = 1; |
| | | const STREAM_TIMING_MAX_ELAPSED_MS: u64 = 5_000; |
| | | const STREAM_TIMING_SOURCE: &str = "stream_anchor_monotonic"; |
| | | const CONTROLLED_FIXTURE_PROBE_TOPIC: &str = "controlled_fixture_attribute_probe"; |
| | | const CONTROLLED_FIXTURE_ACK_TOPIC: &str = "controlled_fixture_attribute_ack"; |
| | | const CONTROLLED_FIXTURE_PROTOCOL_VERSION: u64 = 1; |
| | | const CONTROLLED_FIXTURE_GENERATION: u64 = 1; |
| | | const CONTROLLED_FIXTURE_PROBE_RECHECK_DELAY: Duration = Duration::from_millis(25); |
| | | const CONTROLLED_FIXTURE_PROBE_TTL: Duration = Duration::from_millis(250); |
| | | |
| | | const CONTROLLED_FIXTURE_ACK_RESULTS: [&str; 4] = |
| | | ["observed", "rejected", "timeout", "publish_failed"]; |
| | | const CONTROLLED_FIXTURE_REJECT_REASONS: [&str; 10] = [ |
| | | "missing_attributes", |
| | | "wrong_source", |
| | | "missing_sequence", |
| | | "wrong_sequence", |
| | | "wrong_participant", |
| | | "expired", |
| | | "duplicate_or_old_sequence", |
| | | "no_current_participant", |
| | | "ack_publish_failed", |
| | | "unknown", |
| | | ]; |
| | | |
| | | #[derive(Debug, Deserialize)] |
| | | #[serde(rename_all = "camelCase")] |
| | | struct ControlledFixtureAttributeProbe { |
| | | #[serde(rename = "type")] |
| | | message_type: String, |
| | | protocol_version: u64, |
| | | call_id_hash: String, |
| | | call_trace_id_hash: String, |
| | | generation: u64, |
| | | client_fixture_sequence: String, |
| | | } |
| | | |
| | | #[derive(Debug, Serialize)] |
| | | #[serde(rename_all = "camelCase")] |
| | | struct ControlledFixtureAttributeAck { |
| | | #[serde(rename = "type")] |
| | | message_type: &'static str, |
| | | protocol_version: u64, |
| | | call_id_hash: String, |
| | | call_trace_id_hash: String, |
| | | generation: u64, |
| | | client_fixture_sequence: String, |
| | | result: &'static str, |
| | | #[serde(skip_serializing_if = "Option::is_none")] |
| | | input_source_category: Option<&'static str>, |
| | | #[serde(skip_serializing_if = "Option::is_none")] |
| | | reject_reason: Option<&'static str>, |
| | | } |
| | | |
| | | #[derive(Debug)] |
| | | struct PendingControlledFixtureProbe { |
| | | sender: ParticipantIdentity, |
| | | sequence: String, |
| | | expires_at: Instant, |
| | | } |
| | | |
| | | fn sha256_hex(value: &str) -> String { |
| | | let mut digest = Sha256::new(); |
| | | digest.update(value.as_bytes()); |
| | | digest |
| | | .finalize() |
| | | .iter() |
| | | .map(|byte| format!("{byte:02x}")) |
| | | .collect() |
| | | } |
| | | |
| | | fn controlled_fixture_ack_classification( |
| | | decision: Result<(), &'static str>, |
| | | ) -> (&'static str, Option<&'static str>, bool) { |
| | | match decision { |
| | | Ok(()) => ("observed", None, true), |
| | | Err("timeout") => ("timeout", Some("expired"), false), |
| | | Err(reason) if CONTROLLED_FIXTURE_REJECT_REASONS.contains(&reason) => { |
| | | ("rejected", Some(reason), false) |
| | | } |
| | | Err(_) => ("rejected", Some("unknown"), false), |
| | | } |
| | | } |
| | | |
| | | fn record_controlled_fixture_probe_event( |
| | | stage: &'static str, |
| | | call_id: &str, |
| | | trace_id: &str, |
| | | sequence: &str, |
| | | observed: bool, |
| | | ack_result: Option<&'static str>, |
| | | reject_reason: Option<&'static str>, |
| | | ) { |
| | | debug_assert!(ack_result.is_none_or(|value| CONTROLLED_FIXTURE_ACK_RESULTS.contains(&value))); |
| | | debug_assert!( |
| | | reject_reason.is_none_or(|value| CONTROLLED_FIXTURE_REJECT_REASONS.contains(&value)) |
| | | ); |
| | | if let Some(ack_result) = ack_result { |
| | | info!( |
| | | event = "controlled_fixture_attribute_probe", |
| | | stage, |
| | | observed, |
| | | ack_result, |
| | | reject_reason = reject_reason.unwrap_or("unknown"), |
| | | call_id_hash = %sha256_hex(call_id), |
| | | trace_id_hash = %sha256_hex(trace_id), |
| | | generation = CONTROLLED_FIXTURE_GENERATION, |
| | | sequence_hash = %sha256_hex(sequence), |
| | | "runtime helper controlled fixture probe state" |
| | | ); |
| | | } else if let Some(reject_reason) = reject_reason { |
| | | info!( |
| | | event = "controlled_fixture_attribute_probe", |
| | | stage, |
| | | observed, |
| | | reject_reason, |
| | | call_id_hash = %sha256_hex(call_id), |
| | | trace_id_hash = %sha256_hex(trace_id), |
| | | generation = CONTROLLED_FIXTURE_GENERATION, |
| | | sequence_hash = %sha256_hex(sequence), |
| | | "runtime helper controlled fixture probe state" |
| | | ); |
| | | } else { |
| | | info!( |
| | | event = "controlled_fixture_attribute_probe", |
| | | stage, |
| | | observed, |
| | | call_id_hash = %sha256_hex(call_id), |
| | | trace_id_hash = %sha256_hex(trace_id), |
| | | generation = CONTROLLED_FIXTURE_GENERATION, |
| | | sequence_hash = %sha256_hex(sequence), |
| | | "runtime helper controlled fixture probe state" |
| | | ); |
| | | } |
| | | } |
| | | |
| | | fn record_controlled_fixture_attribute_decision( |
| | | decision: Result<(), &'static str>, |
| | | call_id: &str, |
| | | trace_id: &str, |
| | | sequence: &str, |
| | | ) -> (&'static str, Option<&'static str>, bool) { |
| | | let classification = controlled_fixture_ack_classification(decision); |
| | | record_controlled_fixture_probe_event( |
| | | "attributes_classified", |
| | | call_id, |
| | | trace_id, |
| | | sequence, |
| | | classification.2, |
| | | Some(classification.0), |
| | | classification.1, |
| | | ); |
| | | classification |
| | | } |
| | | |
| | | async fn complete_controlled_fixture_ack_publish<F, E>( |
| | | publish: F, |
| | | observed: bool, |
| | | ack_result: &'static str, |
| | | reject_reason: Option<&'static str>, |
| | | call_id: &str, |
| | | trace_id: &str, |
| | | sequence: &str, |
| | | acknowledged_probe_sequences: &mut HashSet<String>, |
| | | ) -> bool |
| | | where |
| | | F: Future<Output = Result<(), E>>, |
| | | { |
| | | if publish.await.is_ok() { |
| | | record_controlled_fixture_probe_event( |
| | | "ack_publish_completed", |
| | | call_id, |
| | | trace_id, |
| | | sequence, |
| | | observed, |
| | | Some(ack_result), |
| | | reject_reason, |
| | | ); |
| | | acknowledged_probe_sequences.insert(sequence.to_string()); |
| | | observed |
| | | } else { |
| | | record_controlled_fixture_probe_event( |
| | | "ack_publish_completed", |
| | | call_id, |
| | | trace_id, |
| | | sequence, |
| | | false, |
| | | Some("publish_failed"), |
| | | Some("ack_publish_failed"), |
| | | ); |
| | | false |
| | | } |
| | | } |
| | | |
| | | fn controlled_fixture_probe( |
| | | payload: &[u8], |
| | | call_id: &str, |
| | | trace_id: &str, |
| | | sender: &ParticipantIdentity, |
| | | expected_participant: Option<&str>, |
| | | ) -> Option<PendingControlledFixtureProbe> { |
| | | if !is_bound_user_participant(sender.as_str(), expected_participant) { |
| | | return None; |
| | | } |
| | | let probe: ControlledFixtureAttributeProbe = serde_json::from_slice(payload).ok()?; |
| | | if probe.message_type != CONTROLLED_FIXTURE_PROBE_TOPIC |
| | | || probe.protocol_version != CONTROLLED_FIXTURE_PROTOCOL_VERSION |
| | | || probe.generation != CONTROLLED_FIXTURE_GENERATION |
| | | || probe.call_id_hash != sha256_hex(call_id) |
| | | || probe.call_trace_id_hash != sha256_hex(trace_id) |
| | | || probe.client_fixture_sequence.trim().is_empty() |
| | | { |
| | | return None; |
| | | } |
| | | Some(PendingControlledFixtureProbe { |
| | | sender: sender.clone(), |
| | | sequence: probe.client_fixture_sequence, |
| | | expires_at: Instant::now() + CONTROLLED_FIXTURE_PROBE_TTL, |
| | | }) |
| | | } |
| | | |
| | | fn classify_controlled_fixture_attributes( |
| | | actual_participant: &str, |
| | | expected_participant: Option<&str>, |
| | | attributes: &std::collections::HashMap<String, String>, |
| | | requested_sequence: &str, |
| | | ) -> Result<(), &'static str> { |
| | | if !is_bound_user_participant(actual_participant, expected_participant) { |
| | | return Err("wrong_participant"); |
| | | } |
| | | let source = attributes |
| | | .get("inputSourceCategory") |
| | | .map(String::as_str) |
| | | .unwrap_or(""); |
| | | if source != "controlled_fixture" { |
| | | return Err(if source.is_empty() { |
| | | "missing_attributes" |
| | | } else { |
| | | "wrong_source" |
| | | }); |
| | | } |
| | | let sequence = attributes |
| | | .get("clientFixtureSequence") |
| | | .map(String::as_str) |
| | | .unwrap_or(""); |
| | | if sequence.trim().is_empty() { |
| | | return Err("missing_sequence"); |
| | | } |
| | | if sequence != requested_sequence { |
| | | return Err("wrong_sequence"); |
| | | } |
| | | Ok(()) |
| | | } |
| | | |
| | | async fn observe_controlled_fixture_attributes<F>( |
| | | expires_at: Instant, |
| | | actual_participant: &str, |
| | | expected_participant: Option<&str>, |
| | | requested_sequence: &str, |
| | | mut read_attributes: F, |
| | | ) -> Result<(), &'static str> |
| | | where |
| | | F: FnMut() -> std::collections::HashMap<String, String>, |
| | | { |
| | | loop { |
| | | if Instant::now() > expires_at { |
| | | return Err("timeout"); |
| | | } |
| | | let decision = classify_controlled_fixture_attributes( |
| | | actual_participant, |
| | | expected_participant, |
| | | &read_attributes(), |
| | | requested_sequence, |
| | | ); |
| | | if decision.is_ok() || !matches!(decision, Err("missing_attributes")) { |
| | | return decision; |
| | | } |
| | | let Some(next_check) = Instant::now().checked_add(CONTROLLED_FIXTURE_PROBE_RECHECK_DELAY) |
| | | else { |
| | | return Err("timeout"); |
| | | }; |
| | | if next_check > expires_at { |
| | | return Err("timeout"); |
| | | } |
| | | sleep(CONTROLLED_FIXTURE_PROBE_RECHECK_DELAY).await; |
| | | } |
| | | } |
| | | |
| | | fn controlled_fixture_observer_gate(pending_probe: bool, probe_result: Option<bool>) -> bool { |
| | | !pending_probe || probe_result == Some(true) |
| | | } |
| | | |
| | | fn start_observer_after_controlled_fixture_probe<F>( |
| | | pending_probe: bool, |
| | | probe_result: Option<bool>, |
| | | spawn: F, |
| | | ) -> bool |
| | | where |
| | | F: FnOnce(), |
| | | { |
| | | if !controlled_fixture_observer_gate(pending_probe, probe_result) { |
| | | return false; |
| | | } |
| | | spawn(); |
| | | true |
| | | } |
| | | |
| | | #[tokio::main(flavor = "multi_thread")] |
| | | async fn main() -> Result<()> { |
| | |
| | | println!("{payload}"); |
| | | } |
| | | |
| | | fn emit_anchored_activity( |
| | | call_id: &str, |
| | | trace_id: &str, |
| | | turn_id: &str, |
| | | event_name: &str, |
| | | marker: &RuntimeTurnStreamTimingMarker, |
| | | extension: serde_json::Value, |
| | | ) { |
| | | let payload = json!({ |
| | | "type": "cv_activity", |
| | | "callId": call_id, |
| | | "traceId": trace_id, |
| | | "turnId": turn_id, |
| | | "eventName": event_name, |
| | | "eventWallTimeMs": current_time_millis(), |
| | | "serverDeltaMs": marker.server_delta_ms, |
| | | "serverDeltaSource": STREAM_TIMING_SOURCE, |
| | | "result": "ok", |
| | | "reasonCode": null, |
| | | "retryable": null, |
| | | "extension": marker.extension_with(extension), |
| | | }); |
| | | println!("{payload}"); |
| | | } |
| | | |
| | | fn spawn_user_audio_observer( |
| | | events: UnboundedReceiver<RoomEvent>, |
| | | config: &Config, |
| | |
| | | let enabled = config.user_audio_observer_enabled; |
| | | let simple_vad_enabled = config.simple_vad_enabled; |
| | | let simple_vad_config = config.simple_vad_config.clone(); |
| | | let user_participant_identity = config.user_participant_identity.clone(); |
| | | let turn_bridge_config = TurnBridgeConfig::from_config(config); |
| | | |
| | | tokio::spawn(async move { |
| | |
| | | turn_bridge_config, |
| | | http, |
| | | sink, |
| | | user_participant_identity, |
| | | ) |
| | | .await; |
| | | }) |
| | | } |
| | | |
| | | fn is_bound_user_participant(identity: &str, expected: Option<&str>) -> bool { |
| | | expected.is_none_or(|value| identity == value) |
| | | } |
| | | |
| | | async fn observe_user_audio_events( |
| | |
| | | turn_bridge_config: TurnBridgeConfig, |
| | | http: Client, |
| | | sink: Arc<BotAudioOutputSink>, |
| | | user_participant_identity: Option<String>, |
| | | ) { |
| | | info!( |
| | | call_id = %call_id, |
| | |
| | | "runtime helper user_track_subscribe_requested" |
| | | ); |
| | | |
| | | let mut current_user_participant: Option<RemoteParticipant> = None; |
| | | let mut pending_probe: Option<PendingControlledFixtureProbe> = None; |
| | | let mut acknowledged_probe_sequences = HashSet::new(); |
| | | |
| | | while let Some(event) = events.recv().await { |
| | | match event { |
| | | RoomEvent::TrackSubscribed { |
| | |
| | | publication: _, |
| | | participant, |
| | | } => { |
| | | if !is_bound_user_participant( |
| | | &participant.identity().to_string(), |
| | | user_participant_identity.as_deref(), |
| | | ) { |
| | | warn!(call_id = %call_id, trace_id = %trace_id, |
| | | metadata_status = "wrong_participant", |
| | | "runtime helper ignored non-user audio participant"); |
| | | continue; |
| | | } |
| | | let participant_alias = redact(&participant.identity().to_string()); |
| | | let track_sid_alias = redact(&track.sid().to_string()); |
| | | let track_name = track.name(); |
| | |
| | | track_source = %track_source, |
| | | "runtime helper user_track_subscribed" |
| | | ); |
| | | spawn_user_audio_frame_observer( |
| | | track, |
| | | call_id.clone(), |
| | | trace_id.clone(), |
| | | participant_alias, |
| | | track_sid_alias, |
| | | simple_vad_enabled, |
| | | simple_vad_config.clone(), |
| | | vad_enabled_gate.clone(), |
| | | turn_bridge_config.clone(), |
| | | http.clone(), |
| | | sink.clone(), |
| | | let pending_sequence = pending_probe.as_ref().map(|probe| probe.sequence.clone()); |
| | | let participant_for_probe = participant.clone(); |
| | | let probe_result = process_controlled_fixture_probe( |
| | | &mut pending_probe, |
| | | &mut acknowledged_probe_sequences, |
| | | Some(&participant_for_probe), |
| | | &sink, |
| | | &call_id, |
| | | &trace_id, |
| | | user_participant_identity.as_deref(), |
| | | ) |
| | | .await; |
| | | let observer_started = start_observer_after_controlled_fixture_probe( |
| | | pending_sequence.is_some(), |
| | | probe_result, |
| | | || { |
| | | spawn_user_audio_frame_observer( |
| | | track, |
| | | call_id.clone(), |
| | | trace_id.clone(), |
| | | participant_alias, |
| | | track_sid_alias, |
| | | simple_vad_enabled, |
| | | simple_vad_config.clone(), |
| | | vad_enabled_gate.clone(), |
| | | turn_bridge_config.clone(), |
| | | http.clone(), |
| | | sink.clone(), |
| | | user_participant_identity.clone(), |
| | | participant, |
| | | ); |
| | | }, |
| | | ); |
| | | if let Some(sequence) = pending_sequence.as_deref() { |
| | | let (ack_result, reject_reason, observed) = match probe_result { |
| | | Some(true) => (Some("observed"), None, true), |
| | | Some(false) => (Some("rejected"), Some("unknown"), false), |
| | | None => (None, Some("no_current_participant"), false), |
| | | }; |
| | | record_controlled_fixture_probe_event( |
| | | if observer_started { |
| | | "audio_observer_allowed" |
| | | } else { |
| | | "audio_observer_blocked" |
| | | }, |
| | | &call_id, |
| | | &trace_id, |
| | | sequence, |
| | | observed, |
| | | ack_result, |
| | | reject_reason, |
| | | ); |
| | | } |
| | | if !observer_started { |
| | | warn!( |
| | | call_id = %call_id, |
| | | trace_id = %trace_id, |
| | | "runtime helper withheld audio observer until controlled fixture ACK" |
| | | ); |
| | | continue; |
| | | } |
| | | current_user_participant = Some(participant_for_probe); |
| | | } |
| | | RoomEvent::DataReceived { |
| | | payload, |
| | | topic: Some(topic), |
| | | participant: Some(sender), |
| | | .. |
| | | } if topic == CONTROLLED_FIXTURE_PROBE_TOPIC => { |
| | | if let Ok(probe) = |
| | | serde_json::from_slice::<ControlledFixtureAttributeProbe>(&payload) |
| | | { |
| | | if acknowledged_probe_sequences.contains(&probe.client_fixture_sequence) { |
| | | record_controlled_fixture_probe_event( |
| | | "data_received", |
| | | &call_id, |
| | | &trace_id, |
| | | &probe.client_fixture_sequence, |
| | | false, |
| | | Some("rejected"), |
| | | Some("duplicate_or_old_sequence"), |
| | | ); |
| | | continue; |
| | | } |
| | | } |
| | | pending_probe = controlled_fixture_probe( |
| | | &payload, |
| | | &call_id, |
| | | &trace_id, |
| | | &sender.identity(), |
| | | user_participant_identity.as_deref(), |
| | | ); |
| | | if let Some(probe) = pending_probe.as_ref() { |
| | | record_controlled_fixture_probe_event( |
| | | "data_received", |
| | | &call_id, |
| | | &trace_id, |
| | | &probe.sequence, |
| | | false, |
| | | None, |
| | | None, |
| | | ); |
| | | } |
| | | process_controlled_fixture_probe( |
| | | &mut pending_probe, |
| | | &mut acknowledged_probe_sequences, |
| | | current_user_participant.as_ref(), |
| | | &sink, |
| | | &call_id, |
| | | &trace_id, |
| | | user_participant_identity.as_deref(), |
| | | ) |
| | | .await; |
| | | } |
| | | RoomEvent::TrackSubscribed { |
| | | track: RemoteTrack::Video(track), |
| | |
| | | _ => {} |
| | | } |
| | | } |
| | | } |
| | | |
| | | async fn process_controlled_fixture_probe( |
| | | pending_probe: &mut Option<PendingControlledFixtureProbe>, |
| | | acknowledged_probe_sequences: &mut HashSet<String>, |
| | | participant: Option<&RemoteParticipant>, |
| | | sink: &BotAudioOutputSink, |
| | | call_id: &str, |
| | | trace_id: &str, |
| | | expected_participant: Option<&str>, |
| | | ) -> Option<bool> { |
| | | let Some(probe) = pending_probe.take() else { |
| | | return None; |
| | | }; |
| | | if acknowledged_probe_sequences.contains(&probe.sequence) { |
| | | record_controlled_fixture_probe_event( |
| | | "attributes_classified", |
| | | call_id, |
| | | trace_id, |
| | | &probe.sequence, |
| | | false, |
| | | Some("rejected"), |
| | | Some("duplicate_or_old_sequence"), |
| | | ); |
| | | return Some(false); |
| | | } |
| | | if Instant::now() > probe.expires_at { |
| | | record_controlled_fixture_probe_event( |
| | | "attributes_classified", |
| | | call_id, |
| | | trace_id, |
| | | &probe.sequence, |
| | | false, |
| | | Some("timeout"), |
| | | Some("expired"), |
| | | ); |
| | | return Some(false); |
| | | } |
| | | let Some(participant) = participant else { |
| | | record_controlled_fixture_probe_event( |
| | | "attributes_classified", |
| | | call_id, |
| | | trace_id, |
| | | &probe.sequence, |
| | | false, |
| | | None, |
| | | Some("no_current_participant"), |
| | | ); |
| | | *pending_probe = Some(probe); |
| | | return None; |
| | | }; |
| | | if participant.identity() != probe.sender { |
| | | record_controlled_fixture_probe_event( |
| | | "attributes_classified", |
| | | call_id, |
| | | trace_id, |
| | | &probe.sequence, |
| | | false, |
| | | Some("rejected"), |
| | | Some("wrong_participant"), |
| | | ); |
| | | return Some(false); |
| | | } |
| | | let decision = observe_controlled_fixture_attributes( |
| | | probe.expires_at, |
| | | &participant.identity().to_string(), |
| | | expected_participant, |
| | | &probe.sequence, |
| | | || participant.attributes(), |
| | | ) |
| | | .await; |
| | | let protocol_reject_reason = decision.err(); |
| | | let (ack_result, reject_reason, observed) = |
| | | record_controlled_fixture_attribute_decision(decision, call_id, trace_id, &probe.sequence); |
| | | let result = if observed { "observed" } else { "rejected" }; |
| | | let input_source_category = observed.then_some("controlled_fixture"); |
| | | let ack = ControlledFixtureAttributeAck { |
| | | message_type: CONTROLLED_FIXTURE_ACK_TOPIC, |
| | | protocol_version: CONTROLLED_FIXTURE_PROTOCOL_VERSION, |
| | | call_id_hash: sha256_hex(call_id), |
| | | call_trace_id_hash: sha256_hex(trace_id), |
| | | generation: CONTROLLED_FIXTURE_GENERATION, |
| | | client_fixture_sequence: probe.sequence.clone(), |
| | | result, |
| | | input_source_category, |
| | | reject_reason: protocol_reject_reason, |
| | | }; |
| | | let payload = match serde_json::to_vec(&ack) { |
| | | Ok(payload) => payload, |
| | | Err(_) => return Some(false), |
| | | }; |
| | | let local_participant = sink.room.local_participant(); |
| | | record_controlled_fixture_probe_event( |
| | | "ack_publish_started", |
| | | call_id, |
| | | trace_id, |
| | | &probe.sequence, |
| | | observed, |
| | | Some(ack_result), |
| | | reject_reason, |
| | | ); |
| | | let publish = local_participant.publish_data(DataPacket { |
| | | payload, |
| | | topic: Some(CONTROLLED_FIXTURE_ACK_TOPIC.to_string()), |
| | | reliable: true, |
| | | destination_identities: vec![probe.sender], |
| | | }); |
| | | Some( |
| | | complete_controlled_fixture_ack_publish( |
| | | publish, |
| | | observed, |
| | | ack_result, |
| | | reject_reason, |
| | | call_id, |
| | | trace_id, |
| | | &probe.sequence, |
| | | acknowledged_probe_sequences, |
| | | ) |
| | | .await, |
| | | ) |
| | | } |
| | | |
| | | async fn handle_finished_turn( |
| | |
| | | .await |
| | | { |
| | | Ok(outcome) => { |
| | | let mut published_device_outputs = HashSet::new(); |
| | | for output in &outcome.device_outputs { |
| | | if !should_publish_device_output(&mut published_device_outputs, output) { |
| | | continue; |
| | | } |
| | | if let Err(error) = sink |
| | | .publish_device_output(call_id, trace_id, &turn.turn_id, output) |
| | | .await |
| | |
| | | .await?; |
| | | } |
| | | } |
| | | state.close_timing(); |
| | | if !state.completed { |
| | | warn!( |
| | | call_id = %call_id, |
| | |
| | | call_id, |
| | | trace_id, |
| | | turn, |
| | | &event, |
| | | audio_chunk, |
| | | state, |
| | | turn_pipeline_started_at, |
| | |
| | | } |
| | | Some("device_output") => { |
| | | if let Some(output) = event.device_output.as_ref() { |
| | | if !should_publish_device_output(&mut state.published_device_output_ids, output) { |
| | | return Ok(()); |
| | | } |
| | | sink.publish_device_output(call_id, trace_id, &turn.turn_id, output) |
| | | .await?; |
| | | state.device_output_count = state.device_output_count.saturating_add(1); |
| | |
| | | } |
| | | Some("turn_completed") => { |
| | | state.completed = true; |
| | | state.close_timing(); |
| | | info!( |
| | | call_id = %call_id, |
| | | trace_id = %trace_id, |
| | |
| | | ); |
| | | } |
| | | Some("turn_failed") => { |
| | | state.close_timing(); |
| | | let error = event.error.as_ref(); |
| | | let reason_code = error |
| | | .and_then(|value| value.reason_code.as_deref()) |
| | |
| | | } |
| | | Some("turn_cancelled") => { |
| | | state.completed = true; |
| | | state.close_timing(); |
| | | info!( |
| | | call_id = %call_id, |
| | | trace_id = %trace_id, |
| | |
| | | } |
| | | Some("activity") => { |
| | | if let Some(activity) = event.activity.as_ref() { |
| | | if activity.event_type.as_deref() == Some("tts_first_audio_chunk_ready") { |
| | | if let Some(runtime_session_nonce) = |
| | | bridge_config.runtime_session_nonce.as_deref() |
| | | { |
| | | let _ = state.arm_timing_anchor( |
| | | call_id, |
| | | trace_id, |
| | | &turn.turn_id, |
| | | runtime_session_nonce, |
| | | &event, |
| | | ); |
| | | } |
| | | } |
| | | info!( |
| | | call_id = %call_id, |
| | | trace_id = %trace_id, |
| | |
| | | call_id: &str, |
| | | trace_id: &str, |
| | | turn: &FinishedSpeechTurn, |
| | | event: &RuntimeTurnStreamEvent, |
| | | audio_chunk: &RuntimeTurnStreamAudioChunk, |
| | | state: &mut RuntimeTurnStreamState, |
| | | turn_pipeline_started_at: Instant, |
| | |
| | | .unwrap_or("pcm_s16le") |
| | | .trim() |
| | | .to_ascii_lowercase(); |
| | | if !matches!(format.as_str(), "pcm_s16le" | "mp3" | "mpeg" | "wav") { |
| | | return Err(anyhow!("unsupported reply_audio_chunk format {format}")); |
| | | } |
| | | match state.reply_chunk_markers.observe(audio_chunk.segment_seq) { |
| | | ReplyChunkMarker::FirstReply => { |
| | | let extension = json!({ |
| | | "segmentSeq": audio_chunk.segment_seq, |
| | | "chunkSeq": audio_chunk.chunk_seq, |
| | | "format": format.as_str(), |
| | | "bytes": payload.len(), |
| | | }); |
| | | if let Some(marker) = |
| | | state.record_m6(call_id, trace_id, &turn.turn_id, event, audio_chunk) |
| | | { |
| | | emit_anchored_activity( |
| | | call_id, |
| | | trace_id, |
| | | &turn.turn_id, |
| | | "helper_first_reply_audio_chunk_received", |
| | | &marker, |
| | | extension, |
| | | ); |
| | | } else { |
| | | emit_activity( |
| | | call_id, |
| | | trace_id, |
| | | Some(&turn.turn_id), |
| | | "helper_first_reply_audio_chunk_received", |
| | | "ok", |
| | | None, |
| | | None, |
| | | extension, |
| | | ); |
| | | } |
| | | } |
| | | ReplyChunkMarker::SegmentFirst => emit_activity( |
| | | call_id, |
| | | trace_id, |
| | | Some(&turn.turn_id), |
| | | "helper_segment_first_audio_chunk_received", |
| | | "ok", |
| | | None, |
| | | None, |
| | | json!({ |
| | | "segmentSeq": audio_chunk.segment_seq, |
| | | "chunkSeq": audio_chunk.chunk_seq, |
| | | "format": format.as_str(), |
| | | "bytes": payload.len(), |
| | | }), |
| | | ), |
| | | ReplyChunkMarker::None => {} |
| | | } |
| | | let frames = if format == "pcm_s16le" { |
| | | let sample_rate = audio_chunk.sample_rate.unwrap_or(sink.sample_rate_hz); |
| | | let channels = audio_chunk.channels.unwrap_or(u32::from(sink.num_channels)); |
| | |
| | | Err(error) => return Err(error).context("failed to decode final stream audio chunk"), |
| | | } |
| | | } else { |
| | | return Err(anyhow!("unsupported reply_audio_chunk format {format}")); |
| | | unreachable!("supported encoded format checked above") |
| | | }; |
| | | if frames.is_empty() { |
| | | return Ok(0); |
| | |
| | | sink.write_pcm_frame(frame).await?; |
| | | if !state.first_audio_frame_written { |
| | | state.first_audio_frame_written = true; |
| | | emit_activity( |
| | | call_id, |
| | | trace_id, |
| | | Some(&turn.turn_id), |
| | | "bot_reply_first_audio_frame_written", |
| | | "ok", |
| | | None, |
| | | None, |
| | | json!({ |
| | | "replyPlaybackMode": state.reply_playback_mode.as_str(), |
| | | "format": format.as_str(), |
| | | "chunkSeq": audio_chunk.chunk_seq, |
| | | "replyTotalAfterVadEndMs": turn_pipeline_started_at.elapsed().as_millis() as u64, |
| | | }), |
| | | ); |
| | | let extension = json!({ |
| | | "replyPlaybackMode": state.reply_playback_mode.as_str(), |
| | | "format": format.as_str(), |
| | | "chunkSeq": audio_chunk.chunk_seq, |
| | | "replyTotalAfterVadEndMs": turn_pipeline_started_at.elapsed().as_millis() as u64, |
| | | }); |
| | | if let Some(marker) = state.record_m7() { |
| | | emit_anchored_activity( |
| | | call_id, |
| | | trace_id, |
| | | &turn.turn_id, |
| | | "bot_reply_first_audio_frame_written", |
| | | &marker, |
| | | extension, |
| | | ); |
| | | } else { |
| | | emit_activity( |
| | | call_id, |
| | | trace_id, |
| | | Some(&turn.turn_id), |
| | | "bot_reply_first_audio_frame_written", |
| | | "ok", |
| | | None, |
| | | None, |
| | | extension, |
| | | ); |
| | | } |
| | | } |
| | | sleep_until(pacing_started_at + Duration::from_millis(((index + 1) as u64) * 20)).await; |
| | | } |
| | |
| | | completed: bool, |
| | | audio_chunk_count: u64, |
| | | device_output_count: u64, |
| | | published_device_output_ids: HashSet<String>, |
| | | encoded_audio_buffer: Vec<u8>, |
| | | pcm_stream_decoder: Option<audio::PcmS16leStreamDecoder>, |
| | | pcm_stream_network_chunk_count: u64, |
| | | reply_chunk_markers: ReplyChunkMarkerState, |
| | | timing: RuntimeTurnStreamTimingState, |
| | | } |
| | | |
| | | impl Default for RuntimeTurnStreamState { |
| | |
| | | completed: false, |
| | | audio_chunk_count: 0, |
| | | device_output_count: 0, |
| | | published_device_output_ids: HashSet::new(), |
| | | encoded_audio_buffer: Vec::new(), |
| | | pcm_stream_decoder: None, |
| | | pcm_stream_network_chunk_count: 0, |
| | | reply_chunk_markers: ReplyChunkMarkerState::default(), |
| | | timing: RuntimeTurnStreamTimingState::default(), |
| | | } |
| | | } |
| | | } |
| | | |
| | | impl RuntimeTurnStreamState { |
| | | fn arm_timing_anchor( |
| | | &mut self, |
| | | call_id: &str, |
| | | trace_id: &str, |
| | | turn_id: &str, |
| | | runtime_session_nonce: &str, |
| | | event: &RuntimeTurnStreamEvent, |
| | | ) -> bool { |
| | | if self.timing.phase != RuntimeTurnStreamTimingPhase::Empty |
| | | || event.call_id.as_deref() != Some(call_id) |
| | | || event.trace_id.as_deref() != Some(trace_id) |
| | | || event.turn_id.as_deref() != Some(turn_id) |
| | | { |
| | | return false; |
| | | } |
| | | let Some(activity) = event.activity.as_ref() else { |
| | | return false; |
| | | }; |
| | | if activity.event_type.as_deref() != Some("tts_first_audio_chunk_ready") { |
| | | return false; |
| | | } |
| | | let Some(extension) = activity.extension.as_ref() else { |
| | | return false; |
| | | }; |
| | | let Some(anchor_id) = extension.stream_anchor_id.as_deref() else { |
| | | return false; |
| | | }; |
| | | let valid_anchor_id = (16..=64).contains(&anchor_id.len()) && anchor_id.is_ascii(); |
| | | let expected_nonce_hash = runtime_session_nonce_hash(runtime_session_nonce); |
| | | if extension.stream_timing_version != Some(STREAM_TIMING_VERSION) |
| | | || !valid_anchor_id |
| | | || extension.runtime_session_nonce_hash.as_deref() != Some(expected_nonce_hash.as_str()) |
| | | || extension.segment_seq != Some(STREAM_TIMING_FIRST_SEGMENT) |
| | | || extension.stream_timing_validation.as_deref() != Some("bound") |
| | | { |
| | | return false; |
| | | } |
| | | let Some(server_delta_ms) = extension.stream_anchor_server_delta_ms else { |
| | | return false; |
| | | }; |
| | | self.timing.anchor = Some(RuntimeTurnStreamTimingAnchor { |
| | | call_id: call_id.to_string(), |
| | | trace_id: trace_id.to_string(), |
| | | turn_id: turn_id.to_string(), |
| | | anchor_id: anchor_id.to_string(), |
| | | server_delta_ms, |
| | | runtime_session_nonce_hash: expected_nonce_hash, |
| | | segment_seq: STREAM_TIMING_FIRST_SEGMENT, |
| | | received_at: Instant::now(), |
| | | }); |
| | | self.timing.phase = RuntimeTurnStreamTimingPhase::Armed; |
| | | true |
| | | } |
| | | |
| | | fn record_m6( |
| | | &mut self, |
| | | call_id: &str, |
| | | trace_id: &str, |
| | | turn_id: &str, |
| | | event: &RuntimeTurnStreamEvent, |
| | | audio_chunk: &RuntimeTurnStreamAudioChunk, |
| | | ) -> Option<RuntimeTurnStreamTimingMarker> { |
| | | if self.timing.phase != RuntimeTurnStreamTimingPhase::Armed { |
| | | return None; |
| | | } |
| | | let anchor = self.timing.anchor.as_ref()?; |
| | | if anchor.call_id != call_id |
| | | || anchor.trace_id != trace_id |
| | | || anchor.turn_id != turn_id |
| | | || event.call_id.as_deref() != Some(call_id) |
| | | || event.trace_id.as_deref() != Some(trace_id) |
| | | || event.turn_id.as_deref() != Some(turn_id) |
| | | || audio_chunk.segment_seq != Some(anchor.segment_seq) |
| | | || audio_chunk.chunk_seq != Some(STREAM_TIMING_FIRST_CHUNK) |
| | | || audio_chunk.stream_timing_version != Some(STREAM_TIMING_VERSION) |
| | | || audio_chunk.stream_anchor_id.as_deref() != Some(anchor.anchor_id.as_str()) |
| | | { |
| | | return None; |
| | | } |
| | | let Some(marker) = RuntimeTurnStreamTimingMarker::from_anchor( |
| | | anchor, |
| | | audio_chunk.chunk_seq.unwrap_or(STREAM_TIMING_FIRST_CHUNK), |
| | | ) else { |
| | | self.close_timing(); |
| | | return None; |
| | | }; |
| | | self.timing.chunk_seq = Some(marker.chunk_seq); |
| | | self.timing.phase = RuntimeTurnStreamTimingPhase::M6Recorded; |
| | | Some(marker) |
| | | } |
| | | |
| | | fn record_m7(&mut self) -> Option<RuntimeTurnStreamTimingMarker> { |
| | | if self.timing.phase != RuntimeTurnStreamTimingPhase::M6Recorded { |
| | | return None; |
| | | } |
| | | let anchor = self.timing.anchor.as_ref()?; |
| | | let Some(marker) = RuntimeTurnStreamTimingMarker::from_anchor( |
| | | anchor, |
| | | self.timing.chunk_seq.unwrap_or(STREAM_TIMING_FIRST_CHUNK), |
| | | ) else { |
| | | self.close_timing(); |
| | | return None; |
| | | }; |
| | | self.timing.phase = RuntimeTurnStreamTimingPhase::M7Recorded; |
| | | Some(marker) |
| | | } |
| | | |
| | | fn close_timing(&mut self) { |
| | | self.timing.close(); |
| | | } |
| | | } |
| | | |
| | | #[derive(Debug, Clone, Copy, PartialEq, Eq)] |
| | | enum RuntimeTurnStreamTimingPhase { |
| | | Empty, |
| | | Armed, |
| | | M6Recorded, |
| | | M7Recorded, |
| | | Closed, |
| | | } |
| | | |
| | | struct RuntimeTurnStreamTimingState { |
| | | phase: RuntimeTurnStreamTimingPhase, |
| | | anchor: Option<RuntimeTurnStreamTimingAnchor>, |
| | | chunk_seq: Option<u64>, |
| | | } |
| | | |
| | | impl Default for RuntimeTurnStreamTimingState { |
| | | fn default() -> Self { |
| | | Self { |
| | | phase: RuntimeTurnStreamTimingPhase::Empty, |
| | | anchor: None, |
| | | chunk_seq: None, |
| | | } |
| | | } |
| | | } |
| | | |
| | | impl RuntimeTurnStreamTimingState { |
| | | fn close(&mut self) { |
| | | self.anchor = None; |
| | | self.chunk_seq = None; |
| | | self.phase = RuntimeTurnStreamTimingPhase::Closed; |
| | | } |
| | | } |
| | | |
| | | impl Drop for RuntimeTurnStreamTimingState { |
| | | fn drop(&mut self) { |
| | | self.anchor = None; |
| | | self.chunk_seq = None; |
| | | } |
| | | } |
| | | |
| | | struct RuntimeTurnStreamTimingAnchor { |
| | | call_id: String, |
| | | trace_id: String, |
| | | turn_id: String, |
| | | anchor_id: String, |
| | | server_delta_ms: u64, |
| | | runtime_session_nonce_hash: String, |
| | | segment_seq: u64, |
| | | received_at: Instant, |
| | | } |
| | | |
| | | struct RuntimeTurnStreamTimingMarker { |
| | | version: u32, |
| | | anchor_id: String, |
| | | anchor_server_delta_ms: u64, |
| | | anchor_elapsed_ms: u64, |
| | | server_delta_ms: u64, |
| | | runtime_session_nonce_hash: String, |
| | | segment_seq: u64, |
| | | chunk_seq: u64, |
| | | } |
| | | |
| | | impl RuntimeTurnStreamTimingMarker { |
| | | fn from_anchor(anchor: &RuntimeTurnStreamTimingAnchor, chunk_seq: u64) -> Option<Self> { |
| | | let elapsed_ms = u64::try_from(anchor.received_at.elapsed().as_millis()).ok()?; |
| | | if elapsed_ms > STREAM_TIMING_MAX_ELAPSED_MS { |
| | | return None; |
| | | } |
| | | Some(Self { |
| | | version: STREAM_TIMING_VERSION, |
| | | anchor_id: anchor.anchor_id.clone(), |
| | | anchor_server_delta_ms: anchor.server_delta_ms, |
| | | anchor_elapsed_ms: elapsed_ms, |
| | | server_delta_ms: anchor.server_delta_ms.checked_add(elapsed_ms)?, |
| | | runtime_session_nonce_hash: anchor.runtime_session_nonce_hash.clone(), |
| | | segment_seq: anchor.segment_seq, |
| | | chunk_seq, |
| | | }) |
| | | } |
| | | |
| | | fn extension_with(&self, extra: serde_json::Value) -> serde_json::Value { |
| | | let mut extension = match extra { |
| | | serde_json::Value::Object(value) => value, |
| | | _ => serde_json::Map::new(), |
| | | }; |
| | | extension.insert("streamTimingVersion".to_string(), json!(self.version)); |
| | | extension.insert("streamAnchorId".to_string(), json!(self.anchor_id)); |
| | | extension.insert( |
| | | "streamAnchorServerDeltaMs".to_string(), |
| | | json!(self.anchor_server_delta_ms), |
| | | ); |
| | | extension.insert("anchorElapsedMs".to_string(), json!(self.anchor_elapsed_ms)); |
| | | extension.insert( |
| | | "runtimeSessionNonceHash".to_string(), |
| | | json!(self.runtime_session_nonce_hash), |
| | | ); |
| | | extension.insert("segmentSeq".to_string(), json!(self.segment_seq)); |
| | | extension.insert("chunkSeq".to_string(), json!(self.chunk_seq)); |
| | | extension.insert("streamTimingValidation".to_string(), json!("bound")); |
| | | serde_json::Value::Object(extension) |
| | | } |
| | | } |
| | | |
| | | fn runtime_session_nonce_hash(value: &str) -> String { |
| | | let digest = Sha256::digest(value.as_bytes()); |
| | | digest[..6] |
| | | .iter() |
| | | .map(|byte| format!("{byte:02x}")) |
| | | .collect() |
| | | } |
| | | |
| | | #[derive(Debug, PartialEq, Eq)] |
| | | enum ReplyChunkMarker { |
| | | FirstReply, |
| | | SegmentFirst, |
| | | None, |
| | | } |
| | | |
| | | #[derive(Default)] |
| | | struct ReplyChunkMarkerState { |
| | | first_reply_seen: bool, |
| | | seen_segments: HashSet<u64>, |
| | | } |
| | | |
| | | impl ReplyChunkMarkerState { |
| | | fn observe(&mut self, segment_seq: Option<u64>) -> ReplyChunkMarker { |
| | | let first_for_segment = segment_seq |
| | | .map(|value| self.seen_segments.insert(value)) |
| | | .unwrap_or(false); |
| | | if !self.first_reply_seen { |
| | | self.first_reply_seen = true; |
| | | return ReplyChunkMarker::FirstReply; |
| | | } |
| | | if first_for_segment { |
| | | return ReplyChunkMarker::SegmentFirst; |
| | | } |
| | | ReplyChunkMarker::None |
| | | } |
| | | } |
| | | |
| | |
| | | struct RuntimeTurnStreamEvent { |
| | | #[serde(rename = "type", alias = "event")] |
| | | event_type: Option<String>, |
| | | #[serde(rename = "callId")] |
| | | call_id: Option<String>, |
| | | #[serde(rename = "traceId")] |
| | | trace_id: Option<String>, |
| | | #[serde(rename = "turnId")] |
| | | turn_id: Option<String>, |
| | | #[serde(rename = "seq")] |
| | | seq: Option<u64>, |
| | | #[serde(rename = "replyPlaybackMode")] |
| | |
| | | struct RuntimeTurnStreamAudioChunk { |
| | | #[serde(rename = "chunkSeq", alias = "seq")] |
| | | chunk_seq: Option<u64>, |
| | | #[serde(rename = "segmentSeq")] |
| | | segment_seq: Option<u64>, |
| | | #[serde(rename = "streamTimingVersion")] |
| | | stream_timing_version: Option<u32>, |
| | | #[serde(rename = "streamAnchorId")] |
| | | stream_anchor_id: Option<String>, |
| | | format: Option<String>, |
| | | #[serde(rename = "sampleRate")] |
| | | sample_rate: Option<u32>, |
| | |
| | | stage: Option<String>, |
| | | #[serde(rename = "reasonCode")] |
| | | reason_code: Option<String>, |
| | | extension: Option<RuntimeTurnStreamTimingExtension>, |
| | | } |
| | | |
| | | #[derive(Clone, Deserialize)] |
| | | struct RuntimeTurnStreamTimingExtension { |
| | | #[serde(rename = "streamTimingVersion")] |
| | | stream_timing_version: Option<u32>, |
| | | #[serde(rename = "streamAnchorId")] |
| | | stream_anchor_id: Option<String>, |
| | | #[serde(rename = "streamAnchorServerDeltaMs")] |
| | | stream_anchor_server_delta_ms: Option<u64>, |
| | | #[serde(rename = "runtimeSessionNonceHash")] |
| | | runtime_session_nonce_hash: Option<String>, |
| | | #[serde(rename = "segmentSeq")] |
| | | segment_seq: Option<u64>, |
| | | #[serde(rename = "streamTimingValidation")] |
| | | stream_timing_validation: Option<String>, |
| | | } |
| | | |
| | | #[derive(Deserialize)] |
| | |
| | | #[serde(rename = "commandCode")] |
| | | command_code: Option<String>, |
| | | params: Option<serde_json::Value>, |
| | | } |
| | | |
| | | fn should_publish_device_output( |
| | | published_ids: &mut HashSet<String>, |
| | | output: &RuntimeTurnDeviceOutput, |
| | | ) -> bool { |
| | | let Some(command_id) = output |
| | | .command_id |
| | | .as_deref() |
| | | .map(str::trim) |
| | | .filter(|value| !value.is_empty()) |
| | | else { |
| | | return false; |
| | | }; |
| | | if output |
| | | .command_code |
| | | .as_deref() |
| | | .map(str::trim) |
| | | .filter(|value| !value.is_empty()) |
| | | .is_none() |
| | | { |
| | | return false; |
| | | } |
| | | published_ids.insert(command_id.to_string()) |
| | | } |
| | | |
| | | fn require_safe_segment(value: &str) -> Result<()> { |
| | |
| | | turn_bridge_config: TurnBridgeConfig, |
| | | http: Client, |
| | | sink: Arc<BotAudioOutputSink>, |
| | | expected_participant_identity: Option<String>, |
| | | participant: RemoteParticipant, |
| | | ) -> JoinHandle<()> { |
| | | tokio::spawn(async move { |
| | | let mut stream = NativeAudioStream::new( |
| | |
| | | None |
| | | }; |
| | | let mut realtime_asr_upload: Option<RealtimeAsrUpload> = None; |
| | | let mut last_fixture_sequence: Option<String> = None; |
| | | |
| | | while let Some(drained) = frame_rx.recv().await { |
| | | let frame = drained.frame; |
| | |
| | | |
| | | if let Some(vad) = simple_vad.as_mut() { |
| | | if vad_enabled_gate.load(Ordering::Acquire) { |
| | | let was_in_speech = vad.in_speech; |
| | | let turn = vad.observe_frame( |
| | | let participant_identity = participant.identity().to_string(); |
| | | let (was_in_speech, is_in_speech, turn) = observe_bound_participant_frame( |
| | | &participant_identity, |
| | | expected_participant_identity.as_deref(), |
| | | vad, |
| | | &call_id, |
| | | &trace_id, |
| | | &participant_alias, |
| | |
| | | frame_count, |
| | | elapsed_ms, |
| | | &frame, |
| | | http.clone(), |
| | | turn_bridge_config.realtime_asr_config(), |
| | | || participant.attributes(), |
| | | &mut realtime_asr_upload, |
| | | &mut last_fixture_sequence, |
| | | turn_bridge_config.asr_realtime_enabled, |
| | | ); |
| | | let is_in_speech = vad.in_speech; |
| | | |
| | | if !was_in_speech && is_in_speech { |
| | | let turn_id = format!("turn-{:04}", vad.turn_index); |
| | | match RealtimeAsrUpload::start( |
| | | http.clone(), |
| | | turn_bridge_config.realtime_asr_config(), |
| | | &call_id, |
| | | &trace_id, |
| | | &turn_id, |
| | | &vad.speech_samples, |
| | | ) { |
| | | Ok(upload) => { |
| | | info!( |
| | | call_id = %call_id, |
| | | trace_id = %trace_id, |
| | | turn_id = %turn_id, |
| | | "runtime helper asr_realtime_session_started" |
| | | ); |
| | | realtime_asr_upload = Some(upload); |
| | | } |
| | | Err(error) if turn_bridge_config.asr_realtime_enabled => { |
| | | warn!( |
| | | call_id = %call_id, |
| | | trace_id = %trace_id, |
| | | turn_id = %turn_id, |
| | | error = %safe_error(&error.to_string()), |
| | | "runtime helper asr_realtime_start_failed_fallback" |
| | | ); |
| | | } |
| | | Err(_) => {} |
| | | } |
| | | } else if was_in_speech { |
| | | if was_in_speech { |
| | | let push_failed = realtime_asr_upload |
| | | .as_mut() |
| | | .and_then(|upload| upload.push_48k_samples(frame.data.as_ref()).err()); |
| | |
| | | "runtime helper user_audio_stream_ended" |
| | | ); |
| | | }) |
| | | } |
| | | |
| | | fn observe_frame_and_start_session<F>( |
| | | vad: &mut SimpleVad, |
| | | call_id: &str, |
| | | trace_id: &str, |
| | | participant_alias: &str, |
| | | track_sid_alias: &str, |
| | | frame_count: u64, |
| | | elapsed_ms: u64, |
| | | frame: &AudioFrame<'_>, |
| | | http: Client, |
| | | config: RealtimeAsrConfig, |
| | | read_attributes: F, |
| | | upload_slot: &mut Option<RealtimeAsrUpload>, |
| | | last_fixture_sequence: &mut Option<String>, |
| | | realtime_enabled: bool, |
| | | ) -> (bool, bool, Option<FinishedSpeechTurn>) |
| | | where |
| | | F: FnOnce() -> std::collections::HashMap<String, String>, |
| | | { |
| | | let was_in_speech = vad.in_speech; |
| | | let turn = vad.observe_frame( |
| | | call_id, |
| | | trace_id, |
| | | participant_alias, |
| | | track_sid_alias, |
| | | frame_count, |
| | | elapsed_ms, |
| | | frame, |
| | | ); |
| | | let is_in_speech = vad.in_speech; |
| | | if !was_in_speech && is_in_speech { |
| | | start_realtime_session_for_new_speech( |
| | | http, |
| | | config, |
| | | call_id, |
| | | trace_id, |
| | | vad, |
| | | read_attributes, |
| | | upload_slot, |
| | | last_fixture_sequence, |
| | | realtime_enabled, |
| | | ); |
| | | } |
| | | (was_in_speech, is_in_speech, turn) |
| | | } |
| | | |
| | | fn observe_bound_participant_frame<F>( |
| | | participant_identity: &str, |
| | | expected_participant: Option<&str>, |
| | | vad: &mut SimpleVad, |
| | | call_id: &str, |
| | | trace_id: &str, |
| | | participant_alias: &str, |
| | | track_sid_alias: &str, |
| | | frame_count: u64, |
| | | elapsed_ms: u64, |
| | | frame: &AudioFrame<'_>, |
| | | http: Client, |
| | | config: RealtimeAsrConfig, |
| | | read_attributes: F, |
| | | upload_slot: &mut Option<RealtimeAsrUpload>, |
| | | last_fixture_sequence: &mut Option<String>, |
| | | realtime_enabled: bool, |
| | | ) -> (bool, bool, Option<FinishedSpeechTurn>) |
| | | where |
| | | F: FnOnce() -> std::collections::HashMap<String, String>, |
| | | { |
| | | if !is_bound_user_participant(participant_identity, expected_participant) { |
| | | warn!( |
| | | "audioIngressOriginStatus" = "wrong_participant_or_track", |
| | | "runtime helper rejected audio participant before VAD/session" |
| | | ); |
| | | return (vad.in_speech, vad.in_speech, None); |
| | | } |
| | | observe_frame_and_start_session( |
| | | vad, |
| | | call_id, |
| | | trace_id, |
| | | participant_alias, |
| | | track_sid_alias, |
| | | frame_count, |
| | | elapsed_ms, |
| | | frame, |
| | | http, |
| | | config, |
| | | read_attributes, |
| | | upload_slot, |
| | | last_fixture_sequence, |
| | | realtime_enabled, |
| | | ) |
| | | } |
| | | |
| | | fn start_realtime_session_for_new_speech( |
| | | http: Client, |
| | | config: RealtimeAsrConfig, |
| | | call_id: &str, |
| | | trace_id: &str, |
| | | vad: &SimpleVad, |
| | | read_attributes: impl FnOnce() -> std::collections::HashMap<String, String>, |
| | | upload_slot: &mut Option<RealtimeAsrUpload>, |
| | | last_fixture_sequence: &mut Option<String>, |
| | | realtime_enabled: bool, |
| | | ) { |
| | | let turn_id = format!("turn-{:04}", vad.turn_index); |
| | | let attributes = read_attributes(); |
| | | let origin_status = AudioIngressMetadata::origin_status(&attributes); |
| | | let metadata = match AudioIngressMetadata::from_participant(&attributes) { |
| | | Ok(metadata) => metadata, |
| | | Err(reason) => { |
| | | warn!(call_id = %call_id, trace_id = %trace_id, turn_id = %turn_id, |
| | | reason, audioIngressOriginStatus = %AudioIngressMetadata::rejected_origin_status( |
| | | reason, &attributes |
| | | ), |
| | | "runtime helper asr_realtime_metadata_rejected"); |
| | | return; |
| | | } |
| | | }; |
| | | if let Some(metadata) = metadata.as_ref() { |
| | | if !fixture_sequence_is_new( |
| | | last_fixture_sequence.as_deref(), |
| | | &metadata.client_fixture_sequence, |
| | | ) { |
| | | warn!(call_id = %call_id, trace_id = %trace_id, turn_id = %turn_id, |
| | | audioIngressOriginStatus = "sequence_replayed_or_regressed", |
| | | "runtime helper asr_realtime_metadata_sequence_rejected"); |
| | | return; |
| | | } |
| | | } |
| | | match RealtimeAsrUpload::start( |
| | | http, |
| | | config, |
| | | call_id, |
| | | trace_id, |
| | | &turn_id, |
| | | &vad.speech_samples, |
| | | metadata.as_ref(), |
| | | ) { |
| | | Ok(upload) => { |
| | | if let Some(metadata) = metadata { |
| | | *last_fixture_sequence = Some(metadata.client_fixture_sequence); |
| | | } |
| | | info!(call_id = %call_id, trace_id = %trace_id, turn_id = %turn_id, |
| | | origin_status, "runtime helper asr_realtime_session_started"); |
| | | *upload_slot = Some(upload); |
| | | } |
| | | Err(error) if realtime_enabled => { |
| | | warn!(call_id = %call_id, trace_id = %trace_id, turn_id = %turn_id, |
| | | error = %safe_error(&error.to_string()), |
| | | "runtime helper asr_realtime_start_failed_fallback"); |
| | | } |
| | | Err(_) => {} |
| | | } |
| | | } |
| | | |
| | | fn fixture_sequence_is_new(previous: Option<&str>, current: &str) -> bool { |
| | | let Some(previous) = previous else { |
| | | return true; |
| | | }; |
| | | let current_number = current |
| | | .rsplit_once('-') |
| | | .and_then(|(_, value)| value.parse::<u64>().ok()); |
| | | let previous_number = previous |
| | | .rsplit_once('-') |
| | | .and_then(|(_, value)| value.parse::<u64>().ok()); |
| | | match (previous_number, current_number) { |
| | | (Some(previous), Some(current)) => current > previous, |
| | | _ => previous != current, |
| | | } |
| | | } |
| | | |
| | | struct DrainedUserAudioFrame { |
| | |
| | | Ok(()) |
| | | } |
| | | } |
| | | |
| | | #[cfg(test)] |
| | | mod tests { |
| | | use super::*; |
| | | use std::{ |
| | | collections::{HashMap, HashSet}, |
| | | io::{Read, Write}, |
| | | net::TcpListener, |
| | | sync::{ |
| | | Arc, Mutex, |
| | | atomic::{AtomicUsize, Ordering}, |
| | | mpsc, |
| | | }, |
| | | thread, |
| | | time::Duration, |
| | | }; |
| | | |
| | | #[derive(Debug)] |
| | | enum PreAudioOrderEvent { |
| | | DataReceived { |
| | | sender: String, |
| | | sequence: String, |
| | | }, |
| | | TrackSubscribed { |
| | | participant: String, |
| | | attributes: HashMap<String, String>, |
| | | }, |
| | | } |
| | | |
| | | #[derive(Clone, Default)] |
| | | struct CapturedLogs(Arc<Mutex<Vec<u8>>>); |
| | | |
| | | struct CapturedLogWriter(Arc<Mutex<Vec<u8>>>); |
| | | |
| | | impl Write for CapturedLogWriter { |
| | | fn write(&mut self, bytes: &[u8]) -> std::io::Result<usize> { |
| | | self.0.lock().unwrap().extend_from_slice(bytes); |
| | | Ok(bytes.len()) |
| | | } |
| | | |
| | | fn flush(&mut self) -> std::io::Result<()> { |
| | | Ok(()) |
| | | } |
| | | } |
| | | |
| | | impl<'a> tracing_subscriber::fmt::MakeWriter<'a> for CapturedLogs { |
| | | type Writer = CapturedLogWriter; |
| | | |
| | | fn make_writer(&'a self) -> Self::Writer { |
| | | CapturedLogWriter(self.0.clone()) |
| | | } |
| | | } |
| | | |
| | | fn drive_pre_audio_order_test_seam(events: &[PreAudioOrderEvent]) -> Vec<&'static str> { |
| | | let mut pending_sequence = None; |
| | | let mut effects = Vec::new(); |
| | | for event in events { |
| | | match event { |
| | | PreAudioOrderEvent::DataReceived { sender, sequence } if sender == "user-1" => { |
| | | pending_sequence = Some(sequence.as_str()); |
| | | } |
| | | PreAudioOrderEvent::TrackSubscribed { |
| | | participant, |
| | | attributes, |
| | | } => { |
| | | let pending = pending_sequence.is_some(); |
| | | let probe_result = pending_sequence.map(|sequence| { |
| | | classify_controlled_fixture_attributes( |
| | | participant, |
| | | Some("user-1"), |
| | | attributes, |
| | | sequence, |
| | | ) |
| | | .is_ok() |
| | | }); |
| | | let mut ack_observed = false; |
| | | let mut observer_started = false; |
| | | start_observer_after_controlled_fixture_probe(pending, probe_result, || { |
| | | if pending && probe_result == Some(true) { |
| | | ack_observed = true; |
| | | } |
| | | observer_started = true; |
| | | }); |
| | | if ack_observed { |
| | | effects.push("ack_observed"); |
| | | } |
| | | if observer_started { |
| | | effects.push("observer_started"); |
| | | } |
| | | pending_sequence = None; |
| | | } |
| | | PreAudioOrderEvent::DataReceived { .. } => {} |
| | | } |
| | | } |
| | | effects |
| | | } |
| | | |
| | | #[test] |
| | | fn production_vad_session_boundary_reads_updated_attributes() { |
| | | let config = SimpleVadConfig { |
| | | rms_threshold: 0.001, |
| | | peak_threshold: 0.01, |
| | | start_frames: 2, |
| | | end_silence_ms: 100, |
| | | min_speech_ms: 1, |
| | | max_turn_ms: 1_000, |
| | | initial_ignore_ms: 0, |
| | | }; |
| | | let mut vad = SimpleVad::new(config); |
| | | let samples = vec![1_000i16; 160]; |
| | | let frame = AudioFrame { |
| | | data: samples.as_slice().into(), |
| | | sample_rate: 16_000, |
| | | num_channels: 1, |
| | | samples_per_channel: 160, |
| | | }; |
| | | let mut attributes = std::collections::HashMap::from([ |
| | | ( |
| | | "inputSourceCategory".to_string(), |
| | | "controlled_fixture".to_string(), |
| | | ), |
| | | ( |
| | | "clientFixtureSequence".to_string(), |
| | | "fixture-01".to_string(), |
| | | ), |
| | | ]); |
| | | let mut starts = Vec::new(); |
| | | for (session_index, sequence) in [(1, "fixture-01"), (2, "fixture-02")] { |
| | | let was_in_speech = vad.in_speech; |
| | | vad.observe_frame( |
| | | "call-001", |
| | | "trace-001", |
| | | "participant", |
| | | "track", |
| | | session_index * 2 - 1, |
| | | 1_000 * session_index, |
| | | &frame, |
| | | ); |
| | | vad.observe_frame( |
| | | "call-001", |
| | | "trace-001", |
| | | "participant", |
| | | "track", |
| | | session_index * 2, |
| | | 1_000 * session_index + 10, |
| | | &frame, |
| | | ); |
| | | let is_in_speech = vad.in_speech; |
| | | assert!(!was_in_speech && is_in_speech); |
| | | attributes.insert("clientFixtureSequence".to_string(), sequence.to_string()); |
| | | let metadata = AudioIngressMetadata::from_participant(&attributes) |
| | | .expect("valid participant attributes") |
| | | .expect("controlled fixture metadata"); |
| | | let session_line = asr_realtime::session_start_line( |
| | | "call-001", |
| | | "trace-001", |
| | | &format!("turn-{session_index:04}"), |
| | | "nonce-001", |
| | | Some(&metadata), |
| | | ) |
| | | .expect("session start line"); |
| | | let session_json: serde_json::Value = |
| | | serde_json::from_slice(&session_line).expect("session start json"); |
| | | assert_eq!(sequence, session_json["clientFixtureSequence"]); |
| | | starts.push(metadata.client_fixture_sequence); |
| | | vad.reset_current_turn(); |
| | | } |
| | | assert_eq!(vec!["fixture-01", "fixture-02"], starts); |
| | | attributes.insert("inputSourceCategory".to_string(), "other".to_string()); |
| | | assert!(AudioIngressMetadata::from_participant(&attributes).is_err()); |
| | | assert!( |
| | | AudioIngressMetadata::from_participant(&std::collections::HashMap::new()) |
| | | .expect("missing attributes is absent") |
| | | .is_none() |
| | | ); |
| | | attributes.insert( |
| | | "clientFixtureSequence".to_string(), |
| | | "fixture-01".to_string(), |
| | | ); |
| | | assert!(AudioIngressMetadata::from_participant(&attributes).is_err()); |
| | | } |
| | | |
| | | #[test] |
| | | fn production_observer_rejects_wrong_participant_before_vad_session() { |
| | | assert!(!is_bound_user_participant( |
| | | "participant-other", |
| | | Some("participant-user") |
| | | )); |
| | | assert!(is_bound_user_participant( |
| | | "participant-user", |
| | | Some("participant-user") |
| | | )); |
| | | assert!(is_bound_user_participant("participant-any", None)); |
| | | } |
| | | |
| | | #[tokio::test] |
| | | async fn production_observer_vad_to_session_entry_reads_each_updated_attribute() { |
| | | let listener = TcpListener::bind("127.0.0.1:0").expect("bind local ASR fixture"); |
| | | let address = listener.local_addr().expect("fixture address"); |
| | | let (request_tx, request_rx) = mpsc::channel::<String>(); |
| | | let captured_count = Arc::new(AtomicUsize::new(0)); |
| | | let captured_count_for_server = Arc::clone(&captured_count); |
| | | let server = thread::spawn(move || { |
| | | for _ in 0..2 { |
| | | let (mut stream, _) = listener.accept().expect("accept ASR session"); |
| | | stream |
| | | .set_read_timeout(Some(Duration::from_secs(2))) |
| | | .expect("set fixture timeout"); |
| | | let mut bytes = Vec::new(); |
| | | let mut buffer = [0_u8; 4096]; |
| | | loop { |
| | | match stream.read(&mut buffer) { |
| | | Ok(0) => break, |
| | | Ok(size) => { |
| | | bytes.extend_from_slice(&buffer[..size]); |
| | | if bytes.windows(7).any(|window| window == b"0\r\n\r\n") { |
| | | break; |
| | | } |
| | | } |
| | | Err(_) => break, |
| | | } |
| | | } |
| | | request_tx |
| | | .send(String::from_utf8_lossy(&bytes).into_owned()) |
| | | .expect("capture ASR request"); |
| | | captured_count_for_server.fetch_add(1, Ordering::SeqCst); |
| | | stream |
| | | .write_all(b"HTTP/1.1 200 OK\r\ncontent-type: application/json\r\ncontent-length: 39\r\nconnection: close\r\n\r\n{\"code\":0,\"data\":{\"status\":\"ok\"}}") |
| | | .expect("write fixture response"); |
| | | } |
| | | }); |
| | | let mut vad = SimpleVad::new(SimpleVadConfig { |
| | | rms_threshold: 0.001, |
| | | peak_threshold: 0.01, |
| | | start_frames: 1, |
| | | end_silence_ms: 100, |
| | | min_speech_ms: 1, |
| | | max_turn_ms: 1_000, |
| | | initial_ignore_ms: 0, |
| | | }); |
| | | let frame_data = vec![1_000i16; 160]; |
| | | let frame = AudioFrame { |
| | | data: frame_data.as_slice().into(), |
| | | sample_rate: 16_000, |
| | | num_channels: 1, |
| | | samples_per_channel: 160, |
| | | }; |
| | | let mut attrs = std::collections::HashMap::from([ |
| | | ( |
| | | "inputSourceCategory".to_string(), |
| | | "controlled_fixture".to_string(), |
| | | ), |
| | | ( |
| | | "clientFixtureSequence".to_string(), |
| | | "fixture-01".to_string(), |
| | | ), |
| | | ]); |
| | | let mut upload = None; |
| | | let mut last_fixture_sequence = None; |
| | | let config = RealtimeAsrConfig { |
| | | enabled: true, |
| | | url: Some(format!("http://{address}/runtime/asr/realtime")), |
| | | runtime_token: Some("test".to_string()), |
| | | runtime_session_nonce: Some("test".to_string()), |
| | | chunk_duration_ms: 200, |
| | | }; |
| | | let (was, is, turn) = observe_bound_participant_frame( |
| | | "participant-user", |
| | | Some("participant-user"), |
| | | &mut vad, |
| | | "call-001", |
| | | "trace-001", |
| | | "participant-user", |
| | | "track-001", |
| | | 1, |
| | | 1_000, |
| | | &frame, |
| | | Client::new(), |
| | | config.clone(), |
| | | || attrs.clone(), |
| | | &mut upload, |
| | | &mut last_fixture_sequence, |
| | | true, |
| | | ); |
| | | assert!(!was && is && turn.is_none()); |
| | | assert!(upload.is_some()); |
| | | upload.take().unwrap().cancel("test").await; |
| | | vad.reset_current_turn(); |
| | | attrs.insert( |
| | | "clientFixtureSequence".to_string(), |
| | | "fixture-02".to_string(), |
| | | ); |
| | | let (was, is, turn) = observe_bound_participant_frame( |
| | | "participant-user", |
| | | Some("participant-user"), |
| | | &mut vad, |
| | | "call-001", |
| | | "trace-001", |
| | | "participant-user", |
| | | "track-001", |
| | | 2, |
| | | 2_000, |
| | | &frame, |
| | | Client::new(), |
| | | config, |
| | | || attrs.clone(), |
| | | &mut upload, |
| | | &mut last_fixture_sequence, |
| | | true, |
| | | ); |
| | | assert!(!was && is && turn.is_none()); |
| | | assert!(upload.is_some()); |
| | | upload.take().unwrap().cancel("test").await; |
| | | |
| | | let first_request = request_rx |
| | | .recv_timeout(Duration::from_secs(2)) |
| | | .expect("first session request"); |
| | | let second_request = request_rx |
| | | .recv_timeout(Duration::from_secs(2)) |
| | | .expect("second session request"); |
| | | assert!(first_request.contains("\"clientFixtureSequence\":\"fixture-01\"")); |
| | | assert!(second_request.contains("\"clientFixtureSequence\":\"fixture-02\"")); |
| | | assert!( |
| | | first_request.contains("\"audioIngressOriginStatus\":\"controlled_fixture_bound\"") |
| | | ); |
| | | assert!( |
| | | second_request.contains("\"audioIngressOriginStatus\":\"controlled_fixture_bound\"") |
| | | ); |
| | | // The same production boundary rejects a wrong participant before VAD/session creation. |
| | | assert!(!is_bound_user_participant( |
| | | "participant-other", |
| | | Some("participant-user") |
| | | )); |
| | | assert_eq!(0, request_rx.try_iter().count()); |
| | | vad.reset_current_turn(); |
| | | attrs.insert( |
| | | "clientFixtureSequence".to_string(), |
| | | "fixture-03".to_string(), |
| | | ); |
| | | let (_, is_wrong, wrong_turn) = observe_bound_participant_frame( |
| | | "participant-other", |
| | | Some("participant-user"), |
| | | &mut vad, |
| | | "call-001", |
| | | "trace-001", |
| | | "participant-user", |
| | | "track-001", |
| | | 3, |
| | | 3_000, |
| | | &frame, |
| | | Client::new(), |
| | | RealtimeAsrConfig { |
| | | enabled: true, |
| | | url: Some(format!("http://{address}/runtime/asr/realtime")), |
| | | runtime_token: Some("test".to_string()), |
| | | runtime_session_nonce: Some("test".to_string()), |
| | | chunk_duration_ms: 200, |
| | | }, |
| | | || attrs.clone(), |
| | | &mut upload, |
| | | &mut last_fixture_sequence, |
| | | true, |
| | | ); |
| | | assert!(!is_wrong && wrong_turn.is_none() && upload.is_none()); |
| | | vad.reset_current_turn(); |
| | | attrs.insert( |
| | | "clientFixtureSequence".to_string(), |
| | | "fixture-01".to_string(), |
| | | ); |
| | | let (_, _, _) = observe_bound_participant_frame( |
| | | "participant-user", |
| | | Some("participant-user"), |
| | | &mut vad, |
| | | "call-001", |
| | | "trace-001", |
| | | "participant-user", |
| | | "track-001", |
| | | 4, |
| | | 4_000, |
| | | &frame, |
| | | Client::new(), |
| | | RealtimeAsrConfig { |
| | | enabled: true, |
| | | url: Some(format!("http://{address}/runtime/asr/realtime")), |
| | | runtime_token: Some("test".to_string()), |
| | | runtime_session_nonce: Some("test".to_string()), |
| | | chunk_duration_ms: 200, |
| | | }, |
| | | || attrs.clone(), |
| | | &mut upload, |
| | | &mut last_fixture_sequence, |
| | | true, |
| | | ); |
| | | assert!(upload.is_none()); |
| | | vad.reset_current_turn(); |
| | | let (_, _, _) = observe_bound_participant_frame( |
| | | "participant-user", |
| | | Some("participant-user"), |
| | | &mut vad, |
| | | "call-001", |
| | | "trace-001", |
| | | "participant-user", |
| | | "track-001", |
| | | 5, |
| | | 5_000, |
| | | &frame, |
| | | Client::new(), |
| | | RealtimeAsrConfig { |
| | | enabled: true, |
| | | url: Some(format!("http://{address}/runtime/asr/realtime")), |
| | | runtime_token: Some("test".to_string()), |
| | | runtime_session_nonce: Some("test".to_string()), |
| | | chunk_duration_ms: 200, |
| | | }, |
| | | || attrs.clone(), |
| | | &mut upload, |
| | | &mut last_fixture_sequence, |
| | | true, |
| | | ); |
| | | assert!(upload.is_none()); |
| | | vad.reset_current_turn(); |
| | | attrs.remove("clientFixtureSequence"); |
| | | let (_, _, missing_sequence_turn) = observe_bound_participant_frame( |
| | | "participant-user", |
| | | Some("participant-user"), |
| | | &mut vad, |
| | | "call-001", |
| | | "trace-001", |
| | | "participant-user", |
| | | "track-001", |
| | | 6, |
| | | 6_000, |
| | | &frame, |
| | | Client::new(), |
| | | RealtimeAsrConfig { |
| | | enabled: true, |
| | | url: Some(format!("http://{address}/runtime/asr/realtime")), |
| | | runtime_token: Some("test".to_string()), |
| | | runtime_session_nonce: Some("test".to_string()), |
| | | chunk_duration_ms: 200, |
| | | }, |
| | | || attrs.clone(), |
| | | &mut upload, |
| | | &mut last_fixture_sequence, |
| | | true, |
| | | ); |
| | | assert!(missing_sequence_turn.is_none() && upload.is_none()); |
| | | assert_eq!( |
| | | "sequence_absent", |
| | | AudioIngressMetadata::rejected_origin_status("incomplete_metadata", &attrs) |
| | | ); |
| | | assert_eq!(2, captured_count.load(Ordering::SeqCst)); |
| | | vad.reset_current_turn(); |
| | | attrs.insert("inputSourceCategory".to_string(), "other".to_string()); |
| | | let mut invalid_upload = None; |
| | | let (_, _, invalid_turn) = observe_bound_participant_frame( |
| | | "participant-user", |
| | | Some("participant-user"), |
| | | &mut vad, |
| | | "call-001", |
| | | "trace-001", |
| | | "participant-other", |
| | | "track-001", |
| | | 3, |
| | | 3_000, |
| | | &frame, |
| | | Client::new(), |
| | | RealtimeAsrConfig { |
| | | enabled: true, |
| | | url: Some("http://127.0.0.1:9".to_string()), |
| | | runtime_token: Some("test".to_string()), |
| | | runtime_session_nonce: Some("test".to_string()), |
| | | chunk_duration_ms: 200, |
| | | }, |
| | | || attrs, |
| | | &mut invalid_upload, |
| | | &mut last_fixture_sequence, |
| | | true, |
| | | ); |
| | | assert!(invalid_turn.is_none()); |
| | | assert!(invalid_upload.is_none()); |
| | | assert_eq!(2, captured_count.load(Ordering::SeqCst)); |
| | | assert_eq!(0, request_rx.try_iter().count()); |
| | | server.join().expect("fixture server"); |
| | | } |
| | | |
| | | #[test] |
| | | fn reply_chunk_marker_state_emits_turn_first_once_and_later_segment_first_once() { |
| | | let mut state = ReplyChunkMarkerState::default(); |
| | | |
| | | assert_eq!(ReplyChunkMarker::FirstReply, state.observe(Some(1))); |
| | | assert_eq!(ReplyChunkMarker::None, state.observe(Some(1))); |
| | | assert_eq!(ReplyChunkMarker::SegmentFirst, state.observe(Some(2))); |
| | | assert_eq!(ReplyChunkMarker::None, state.observe(Some(2))); |
| | | assert_eq!(ReplyChunkMarker::SegmentFirst, state.observe(Some(3))); |
| | | } |
| | | |
| | | #[test] |
| | | fn reply_chunk_marker_state_without_segment_only_emits_turn_first() { |
| | | let mut state = ReplyChunkMarkerState::default(); |
| | | |
| | | assert_eq!(ReplyChunkMarker::FirstReply, state.observe(None)); |
| | | assert_eq!(ReplyChunkMarker::None, state.observe(None)); |
| | | } |
| | | |
| | | #[test] |
| | | fn runtime_turn_stream_audio_chunk_reads_segment_seq() { |
| | | let event: RuntimeTurnStreamEvent = serde_json::from_str( |
| | | r#"{"type":"reply_audio_chunk","audioChunk":{"chunkSeq":4,"segmentSeq":2,"format":"pcm_s16le","payloadBase64":"AA==","last":false}}"#, |
| | | ) |
| | | .expect("turn stream event"); |
| | | |
| | | assert_eq!( |
| | | Some(2), |
| | | event.audio_chunk.and_then(|chunk| chunk.segment_seq) |
| | | ); |
| | | } |
| | | |
| | | #[test] |
| | | fn runtime_turn_stream_reads_frozen_m5_and_audio_chunk_timing_contract() { |
| | | let activity: RuntimeTurnStreamEvent = serde_json::from_str( |
| | | r#"{"type":"activity","callId":"call-1","traceId":"trace-1","turnId":"turn-1","activity":{"eventType":"tts_first_audio_chunk_ready","extension":{"streamTimingVersion":1,"streamAnchorId":"0123456789abcdef","streamAnchorServerDeltaMs":1200,"runtimeSessionNonceHash":"abcdef012345","segmentSeq":1,"streamTimingValidation":"bound"}}}"#, |
| | | ) |
| | | .expect("m5 activity event"); |
| | | let audio: RuntimeTurnStreamEvent = serde_json::from_str( |
| | | r#"{"type":"reply_audio_chunk","callId":"call-1","traceId":"trace-1","turnId":"turn-1","audioChunk":{"chunkSeq":1,"segmentSeq":1,"streamTimingVersion":1,"streamAnchorId":"0123456789abcdef","format":"pcm_s16le","payloadBase64":"AA==","last":false}}"#, |
| | | ) |
| | | .expect("timed audio chunk event"); |
| | | |
| | | assert_eq!(Some("call-1"), activity.call_id.as_deref()); |
| | | assert_eq!(Some("trace-1"), activity.trace_id.as_deref()); |
| | | assert_eq!(Some("turn-1"), activity.turn_id.as_deref()); |
| | | let extension = activity.activity.unwrap().extension.unwrap(); |
| | | assert_eq!(Some(1), extension.stream_timing_version); |
| | | assert_eq!(Some(1200), extension.stream_anchor_server_delta_ms); |
| | | assert_eq!( |
| | | Some("0123456789abcdef"), |
| | | audio.audio_chunk.unwrap().stream_anchor_id.as_deref() |
| | | ); |
| | | } |
| | | |
| | | #[test] |
| | | fn runtime_turn_stream_timing_records_m6_and_m7_once_then_rejects_terminal_late_events() { |
| | | let nonce = "runtime-nonce"; |
| | | let nonce_hash = runtime_session_nonce_hash(nonce); |
| | | let activity: RuntimeTurnStreamEvent = serde_json::from_str(&format!( |
| | | r#"{{"type":"activity","callId":"call-1","traceId":"trace-1","turnId":"turn-1","activity":{{"eventType":"tts_first_audio_chunk_ready","extension":{{"streamTimingVersion":1,"streamAnchorId":"0123456789abcdef","streamAnchorServerDeltaMs":1200,"runtimeSessionNonceHash":"{nonce_hash}","segmentSeq":1,"streamTimingValidation":"bound"}}}}}}"#, |
| | | )) |
| | | .expect("m5 activity event"); |
| | | let audio: RuntimeTurnStreamEvent = serde_json::from_str( |
| | | r#"{"type":"reply_audio_chunk","callId":"call-1","traceId":"trace-1","turnId":"turn-1","audioChunk":{"chunkSeq":1,"segmentSeq":1,"streamTimingVersion":1,"streamAnchorId":"0123456789abcdef","format":"pcm_s16le","payloadBase64":"AA==","last":false}}"#, |
| | | ) |
| | | .expect("timed audio chunk event"); |
| | | let mut state = RuntimeTurnStreamState::default(); |
| | | |
| | | assert!(state.arm_timing_anchor("call-1", "trace-1", "turn-1", nonce, &activity)); |
| | | let chunk = audio.audio_chunk.as_ref().expect("audio chunk"); |
| | | assert!( |
| | | state |
| | | .record_m6("call-1", "trace-1", "turn-1", &audio, chunk) |
| | | .is_some() |
| | | ); |
| | | assert!( |
| | | state |
| | | .record_m6("call-1", "trace-1", "turn-1", &audio, chunk) |
| | | .is_none() |
| | | ); |
| | | assert!(state.record_m7().is_some()); |
| | | assert!(state.record_m7().is_none()); |
| | | |
| | | state.close_timing(); |
| | | assert_eq!(RuntimeTurnStreamTimingPhase::Closed, state.timing.phase); |
| | | assert!(state.timing.anchor.is_none()); |
| | | assert!(!state.arm_timing_anchor("call-1", "trace-1", "turn-1", nonce, &activity)); |
| | | assert!( |
| | | state |
| | | .record_m6("call-1", "trace-1", "turn-1", &audio, chunk) |
| | | .is_none() |
| | | ); |
| | | assert!(state.record_m7().is_none()); |
| | | } |
| | | |
| | | #[test] |
| | | fn device_output_contract_is_reliable_and_deduplicated() { |
| | | let output = RuntimeTurnDeviceOutput { |
| | | command_id: Some("cmd-1".to_string()), |
| | | command_code: Some("custom.app.DeviceLevelChange".to_string()), |
| | | params: Some(serde_json::json!({"level": 1})), |
| | | }; |
| | | let mut published = HashSet::new(); |
| | | assert!(should_publish_device_output(&mut published, &output)); |
| | | assert!(!should_publish_device_output(&mut published, &output)); |
| | | assert_eq!(published.len(), 1); |
| | | } |
| | | |
| | | #[test] |
| | | fn device_output_invalid_or_missing_command_is_fail_closed() { |
| | | for output in [ |
| | | RuntimeTurnDeviceOutput { |
| | | command_id: None, |
| | | command_code: Some("custom.app.DeviceLevelChange".to_string()), |
| | | params: None, |
| | | }, |
| | | RuntimeTurnDeviceOutput { |
| | | command_id: Some("cmd-1".to_string()), |
| | | command_code: None, |
| | | params: None, |
| | | }, |
| | | ] { |
| | | let mut published = HashSet::new(); |
| | | assert!(!should_publish_device_output(&mut published, &output)); |
| | | assert!(published.is_empty()); |
| | | } |
| | | } |
| | | |
| | | #[test] |
| | | fn controlled_fixture_probe_requires_bound_hashes_and_protocol() { |
| | | let call_id = "call-ack-1"; |
| | | let trace_id = "trace-ack-1"; |
| | | let payload = serde_json::to_vec(&json!({ |
| | | "type": CONTROLLED_FIXTURE_PROBE_TOPIC, |
| | | "protocolVersion": CONTROLLED_FIXTURE_PROTOCOL_VERSION, |
| | | "callIdHash": sha256_hex(call_id), |
| | | "callTraceIdHash": sha256_hex(trace_id), |
| | | "generation": CONTROLLED_FIXTURE_GENERATION, |
| | | "clientFixtureSequence": "fixture-01" |
| | | })) |
| | | .unwrap(); |
| | | let sender = ParticipantIdentity("user-1".to_string()); |
| | | let pending = |
| | | controlled_fixture_probe(&payload, call_id, trace_id, &sender, Some("user-1")) |
| | | .expect("valid probe"); |
| | | assert_eq!(pending.sequence, "fixture-01"); |
| | | assert!( |
| | | controlled_fixture_probe(&payload, call_id, "other-trace", &sender, Some("user-1"),) |
| | | .is_none() |
| | | ); |
| | | assert!( |
| | | controlled_fixture_probe( |
| | | &payload, |
| | | call_id, |
| | | trace_id, |
| | | &ParticipantIdentity("other-user".to_string()), |
| | | Some("user-1"), |
| | | ) |
| | | .is_none() |
| | | ); |
| | | } |
| | | |
| | | #[test] |
| | | fn controlled_fixture_attributes_ack_only_on_exact_current_sequence() { |
| | | let mut attributes = std::collections::HashMap::from([ |
| | | ( |
| | | "inputSourceCategory".to_string(), |
| | | "controlled_fixture".to_string(), |
| | | ), |
| | | ( |
| | | "clientFixtureSequence".to_string(), |
| | | "fixture-01".to_string(), |
| | | ), |
| | | ]); |
| | | assert!( |
| | | classify_controlled_fixture_attributes( |
| | | "user-1", |
| | | Some("user-1"), |
| | | &attributes, |
| | | "fixture-01" |
| | | ) |
| | | .is_ok() |
| | | ); |
| | | attributes.insert( |
| | | "clientFixtureSequence".to_string(), |
| | | "fixture-02".to_string(), |
| | | ); |
| | | assert_eq!( |
| | | classify_controlled_fixture_attributes( |
| | | "user-1", |
| | | Some("user-1"), |
| | | &attributes, |
| | | "fixture-01" |
| | | ), |
| | | Err("wrong_sequence") |
| | | ); |
| | | attributes.remove("clientFixtureSequence"); |
| | | assert_eq!( |
| | | classify_controlled_fixture_attributes( |
| | | "user-1", |
| | | Some("user-1"), |
| | | &attributes, |
| | | "fixture-01" |
| | | ), |
| | | Err("missing_sequence") |
| | | ); |
| | | assert_eq!( |
| | | classify_controlled_fixture_attributes( |
| | | "other-user", |
| | | Some("user-1"), |
| | | &attributes, |
| | | "fixture-01" |
| | | ), |
| | | Err("wrong_participant") |
| | | ); |
| | | } |
| | | |
| | | #[test] |
| | | fn controlled_fixture_probe_failure_log_is_fixed_redacted_and_has_no_audio_effect() { |
| | | let logs = CapturedLogs::default(); |
| | | let subscriber = tracing_subscriber::fmt() |
| | | .without_time() |
| | | .with_ansi(false) |
| | | .with_writer(logs.clone()) |
| | | .finish(); |
| | | let call_id = "private-call-value"; |
| | | let trace_id = "private-trace-value"; |
| | | let sequence = "private-sequence-value"; |
| | | let mut observer_starts = 0; |
| | | tracing::subscriber::with_default(subscriber, || { |
| | | let decision = Err("wrong_source"); |
| | | let (ack_result, reject_reason, observed) = |
| | | record_controlled_fixture_attribute_decision(decision, call_id, trace_id, sequence); |
| | | record_controlled_fixture_probe_event( |
| | | "audio_observer_blocked", |
| | | call_id, |
| | | trace_id, |
| | | sequence, |
| | | observed, |
| | | Some(ack_result), |
| | | reject_reason, |
| | | ); |
| | | assert!(!start_observer_after_controlled_fixture_probe( |
| | | true, |
| | | Some(observed), |
| | | || observer_starts += 1, |
| | | )); |
| | | }); |
| | | let output = String::from_utf8(logs.0.lock().unwrap().clone()).unwrap(); |
| | | assert!(output.contains("ack_result=\"rejected\"")); |
| | | assert!(output.contains("reject_reason=\"wrong_source\"")); |
| | | assert!(output.contains("observed=false")); |
| | | assert!(output.contains(&sha256_hex(call_id))); |
| | | assert!(output.contains(&sha256_hex(trace_id))); |
| | | assert!(output.contains(&sha256_hex(sequence))); |
| | | assert!(!output.contains(call_id)); |
| | | assert!(!output.contains(trace_id)); |
| | | assert!(!output.contains(sequence)); |
| | | assert_eq!(observer_starts, 0); |
| | | } |
| | | |
| | | #[test] |
| | | fn controlled_fixture_probe_observed_and_failure_enums_are_stable() { |
| | | assert_eq!( |
| | | controlled_fixture_ack_classification(Ok(())), |
| | | ("observed", None, true) |
| | | ); |
| | | assert_eq!( |
| | | controlled_fixture_ack_classification(Err("timeout")), |
| | | ("timeout", Some("expired"), false) |
| | | ); |
| | | for reason in [ |
| | | "missing_attributes", |
| | | "wrong_source", |
| | | "missing_sequence", |
| | | "wrong_sequence", |
| | | "wrong_participant", |
| | | ] { |
| | | assert_eq!( |
| | | controlled_fixture_ack_classification(Err(reason)), |
| | | ("rejected", Some(reason), false) |
| | | ); |
| | | } |
| | | assert_eq!( |
| | | controlled_fixture_ack_classification(Err("unclassified")), |
| | | ("rejected", Some("unknown"), false) |
| | | ); |
| | | } |
| | | |
| | | #[tokio::test] |
| | | async fn production_ack_publish_failure_keeps_observer_session_and_audio_closed() { |
| | | let mut acknowledged = HashSet::new(); |
| | | let probe_result = complete_controlled_fixture_ack_publish( |
| | | async { Err::<(), ()>(()) }, |
| | | true, |
| | | "observed", |
| | | None, |
| | | "call-publish-failure", |
| | | "trace-publish-failure", |
| | | "fixture-01", |
| | | &mut acknowledged, |
| | | ) |
| | | .await; |
| | | let mut observer_starts = 0; |
| | | let mut session_starts = 0; |
| | | let mut audio_starts = 0; |
| | | let mut speaking_starts = 0; |
| | | assert!(!start_observer_after_controlled_fixture_probe( |
| | | true, |
| | | Some(probe_result), |
| | | || { |
| | | observer_starts += 1; |
| | | session_starts += 1; |
| | | audio_starts += 1; |
| | | speaking_starts += 1; |
| | | }, |
| | | )); |
| | | assert!(!probe_result); |
| | | assert!(acknowledged.is_empty()); |
| | | assert_eq!(observer_starts, 0); |
| | | assert_eq!(session_starts, 0); |
| | | assert_eq!(audio_starts, 0); |
| | | assert_eq!(speaking_starts, 0); |
| | | |
| | | let observed_result = complete_controlled_fixture_ack_publish( |
| | | async { Ok::<(), ()>(()) }, |
| | | true, |
| | | "observed", |
| | | None, |
| | | "call-publish-success", |
| | | "trace-publish-success", |
| | | "fixture-02", |
| | | &mut acknowledged, |
| | | ) |
| | | .await; |
| | | let mut successful_observer_starts = 0; |
| | | assert!(start_observer_after_controlled_fixture_probe( |
| | | true, |
| | | Some(observed_result), |
| | | || successful_observer_starts += 1, |
| | | )); |
| | | assert!(observed_result); |
| | | assert!(acknowledged.contains("fixture-02")); |
| | | assert_eq!(successful_observer_starts, 1); |
| | | } |
| | | |
| | | #[tokio::test] |
| | | async fn production_attribute_observation_accepts_server_visibility_within_probe_ttl() { |
| | | let expected = HashMap::from([ |
| | | ( |
| | | "inputSourceCategory".to_string(), |
| | | "controlled_fixture".to_string(), |
| | | ), |
| | | ( |
| | | "clientFixtureSequence".to_string(), |
| | | "fixture-01".to_string(), |
| | | ), |
| | | ]); |
| | | let mut reads = 0; |
| | | let result = observe_controlled_fixture_attributes( |
| | | Instant::now() + CONTROLLED_FIXTURE_PROBE_TTL, |
| | | "user-1", |
| | | Some("user-1"), |
| | | "fixture-01", |
| | | || { |
| | | reads += 1; |
| | | if reads <= 3 { |
| | | HashMap::new() |
| | | } else { |
| | | expected.clone() |
| | | } |
| | | }, |
| | | ) |
| | | .await; |
| | | assert_eq!(result, Ok(())); |
| | | assert_eq!(reads, 4); |
| | | } |
| | | |
| | | #[tokio::test] |
| | | async fn production_attribute_observation_rejects_wrong_sequence_without_audio_effect() { |
| | | let attributes = HashMap::from([ |
| | | ( |
| | | "inputSourceCategory".to_string(), |
| | | "controlled_fixture".to_string(), |
| | | ), |
| | | ( |
| | | "clientFixtureSequence".to_string(), |
| | | "fixture-02".to_string(), |
| | | ), |
| | | ]); |
| | | let result = observe_controlled_fixture_attributes( |
| | | Instant::now() + CONTROLLED_FIXTURE_PROBE_TTL, |
| | | "user-1", |
| | | Some("user-1"), |
| | | "fixture-01", |
| | | || attributes.clone(), |
| | | ) |
| | | .await; |
| | | let mut observer_starts = 0; |
| | | assert_eq!(result, Err("wrong_sequence")); |
| | | assert!(!start_observer_after_controlled_fixture_probe( |
| | | true, |
| | | Some(result.is_ok()), |
| | | || observer_starts += 1, |
| | | )); |
| | | assert_eq!(observer_starts, 0); |
| | | } |
| | | |
| | | #[test] |
| | | fn controlled_fixture_ack_payload_is_reliable_and_redacted() { |
| | | let ack = ControlledFixtureAttributeAck { |
| | | message_type: CONTROLLED_FIXTURE_ACK_TOPIC, |
| | | protocol_version: CONTROLLED_FIXTURE_PROTOCOL_VERSION, |
| | | call_id_hash: sha256_hex("call-1"), |
| | | call_trace_id_hash: sha256_hex("trace-1"), |
| | | generation: CONTROLLED_FIXTURE_GENERATION, |
| | | client_fixture_sequence: "fixture-01".to_string(), |
| | | result: "observed", |
| | | input_source_category: Some("controlled_fixture"), |
| | | reject_reason: None, |
| | | }; |
| | | let encoded = serde_json::to_vec(&ack).unwrap(); |
| | | let decoded: serde_json::Value = serde_json::from_slice(&encoded).unwrap(); |
| | | assert_eq!(decoded["type"], CONTROLLED_FIXTURE_ACK_TOPIC); |
| | | assert_eq!( |
| | | decoded["protocolVersion"], |
| | | CONTROLLED_FIXTURE_PROTOCOL_VERSION |
| | | ); |
| | | assert_eq!(decoded["result"], "observed"); |
| | | assert!(decoded.get("callId").is_none()); |
| | | assert!(decoded.get("traceId").is_none()); |
| | | assert!(decoded.get("participantIdentity").is_none()); |
| | | assert!(decoded.get("audio").is_none()); |
| | | } |
| | | |
| | | #[test] |
| | | fn controlled_fixture_probe_does_not_create_session_or_audio_side_effects() { |
| | | let attributes = std::collections::HashMap::new(); |
| | | assert_eq!( |
| | | classify_controlled_fixture_attributes( |
| | | "user-1", |
| | | Some("user-1"), |
| | | &attributes, |
| | | "fixture-01" |
| | | ), |
| | | Err("missing_attributes") |
| | | ); |
| | | assert_eq!( |
| | | CONTROLLED_FIXTURE_PROBE_TOPIC, |
| | | "controlled_fixture_attribute_probe" |
| | | ); |
| | | assert_eq!( |
| | | CONTROLLED_FIXTURE_ACK_TOPIC, |
| | | "controlled_fixture_attribute_ack" |
| | | ); |
| | | } |
| | | |
| | | #[test] |
| | | fn controlled_fixture_probe_must_be_observed_before_audio_observer() { |
| | | assert!(controlled_fixture_observer_gate(false, None)); |
| | | assert!(controlled_fixture_observer_gate(true, Some(true))); |
| | | assert!(!controlled_fixture_observer_gate(true, Some(false))); |
| | | assert!(!controlled_fixture_observer_gate(true, None)); |
| | | } |
| | | |
| | | #[test] |
| | | fn production_event_order_probe_then_track_publishes_ack_before_observer() { |
| | | let attributes = HashMap::from([ |
| | | ( |
| | | "inputSourceCategory".to_string(), |
| | | "controlled_fixture".to_string(), |
| | | ), |
| | | ( |
| | | "clientFixtureSequence".to_string(), |
| | | "fixture-01".to_string(), |
| | | ), |
| | | ]); |
| | | let effects = drive_pre_audio_order_test_seam(&[ |
| | | PreAudioOrderEvent::DataReceived { |
| | | sender: "user-1".to_string(), |
| | | sequence: "fixture-01".to_string(), |
| | | }, |
| | | PreAudioOrderEvent::TrackSubscribed { |
| | | participant: "user-1".to_string(), |
| | | attributes, |
| | | }, |
| | | ]); |
| | | assert_eq!(effects, ["ack_observed", "observer_started"]); |
| | | } |
| | | |
| | | #[test] |
| | | fn production_event_order_negative_probe_has_no_observer_or_session_effect() { |
| | | let mut invalid = HashMap::new(); |
| | | invalid.insert( |
| | | "inputSourceCategory".to_string(), |
| | | "ordinary_mic".to_string(), |
| | | ); |
| | | let effects = drive_pre_audio_order_test_seam(&[ |
| | | PreAudioOrderEvent::DataReceived { |
| | | sender: "user-1".to_string(), |
| | | sequence: "fixture-01".to_string(), |
| | | }, |
| | | PreAudioOrderEvent::TrackSubscribed { |
| | | participant: "user-1".to_string(), |
| | | attributes: invalid, |
| | | }, |
| | | ]); |
| | | assert!(effects.is_empty()); |
| | | } |
| | | |
| | | #[test] |
| | | fn production_audio_branch_orders_probe_before_spawn_callsite() { |
| | | let source = include_str!("main.rs"); |
| | | let branch = source |
| | | .find("RoomEvent::TrackSubscribed {\n track: RemoteTrack::Audio") |
| | | .expect("audio TrackSubscribed production branch"); |
| | | let branch_source = &source[branch..]; |
| | | let probe = branch_source |
| | | .find("let probe_result = process_controlled_fixture_probe") |
| | | .expect("probe must be processed in audio branch"); |
| | | let spawn = branch_source |
| | | .find("start_observer_after_controlled_fixture_probe") |
| | | .expect("spawn must use shared order entry"); |
| | | assert!( |
| | | probe < spawn, |
| | | "probe must precede shared observer spawn entry" |
| | | ); |
| | | } |
| | | } |