| | |
| | | } |
| | | |
| | | fn record_controlled_fixture_probe_event( |
| | | runtime_call_id: &str, |
| | | runtime_trace_id: &str, |
| | | stage: &'static str, |
| | | call_id_hash: &str, |
| | | trace_id_hash: &str, |
| | |
| | | debug_assert!( |
| | | reject_reason.is_none_or(|value| CONTROLLED_FIXTURE_REJECT_REASONS.contains(&value)) |
| | | ); |
| | | if let Some(ack_result) = ack_result { |
| | | info!( |
| | | event = "controlled_fixture_attribute_probe", |
| | | println!( |
| | | "{}", |
| | | controlled_fixture_probe_event( |
| | | runtime_call_id, |
| | | runtime_trace_id, |
| | | stage, |
| | | call_id_hash, |
| | | trace_id_hash, |
| | | generation, |
| | | sequence, |
| | | observed, |
| | | ack_result, |
| | | reject_reason = reject_reason.unwrap_or("unknown"), |
| | | call_id_hash, |
| | | trace_id_hash, |
| | | generation, |
| | | sequence_hash = %sha256_hex(sequence), |
| | | "runtime helper controlled fixture probe state" |
| | | ); |
| | | } else if let Some(reject_reason) = reject_reason { |
| | | info!( |
| | | event = "controlled_fixture_attribute_probe", |
| | | stage, |
| | | observed, |
| | | reject_reason, |
| | | call_id_hash, |
| | | trace_id_hash, |
| | | generation, |
| | | sequence_hash = %sha256_hex(sequence), |
| | | "runtime helper controlled fixture probe state" |
| | | ); |
| | | } else { |
| | | info!( |
| | | event = "controlled_fixture_attribute_probe", |
| | | stage, |
| | | observed, |
| | | call_id_hash, |
| | | trace_id_hash, |
| | | generation, |
| | | sequence_hash = %sha256_hex(sequence), |
| | | "runtime helper controlled fixture probe state" |
| | | ); |
| | | } |
| | | ) |
| | | ); |
| | | } |
| | | |
| | | fn controlled_fixture_probe_event( |
| | | runtime_call_id: &str, |
| | | runtime_trace_id: &str, |
| | | stage: &'static str, |
| | | call_id_hash: &str, |
| | | trace_id_hash: &str, |
| | | generation: u64, |
| | | sequence: &str, |
| | | observed: bool, |
| | | ack_result: Option<&'static str>, |
| | | reject_reason: Option<&'static str>, |
| | | ) -> serde_json::Value { |
| | | json!({ |
| | | "type": "cv_activity", |
| | | "callId": runtime_call_id, |
| | | "traceId": runtime_trace_id, |
| | | "turnId": null, |
| | | "eventName": "controlled_fixture_attribute_probe", |
| | | "eventWallTimeMs": current_time_millis(), |
| | | "result": "ok", |
| | | "reasonCode": null, |
| | | "retryable": null, |
| | | "extension": { |
| | | "stage": stage, |
| | | "observed": observed, |
| | | "ack_result": ack_result, |
| | | "reject_reason": reject_reason, |
| | | "call_id_hash": call_id_hash, |
| | | "trace_id_hash": trace_id_hash, |
| | | "generation": generation, |
| | | "sequence_hash": sha256_hex(sequence), |
| | | }, |
| | | }) |
| | | } |
| | | |
| | | fn record_controlled_fixture_attribute_decision( |
| | | decision: Result<(), &'static str>, |
| | | runtime_call_id: &str, |
| | | runtime_trace_id: &str, |
| | | call_id_hash: &str, |
| | | trace_id_hash: &str, |
| | | generation: u64, |
| | |
| | | ) -> (&'static str, Option<&'static str>, bool) { |
| | | let classification = controlled_fixture_ack_classification(decision); |
| | | record_controlled_fixture_probe_event( |
| | | runtime_call_id, |
| | | runtime_trace_id, |
| | | "attributes_classified", |
| | | call_id_hash, |
| | | trace_id_hash, |
| | |
| | | |
| | | async fn complete_controlled_fixture_ack_publish<F, E>( |
| | | publish: F, |
| | | runtime_call_id: &str, |
| | | runtime_trace_id: &str, |
| | | observed: bool, |
| | | ack_result: &'static str, |
| | | reject_reason: Option<&'static str>, |
| | |
| | | { |
| | | if publish.await.is_ok() { |
| | | record_controlled_fixture_probe_event( |
| | | runtime_call_id, |
| | | runtime_trace_id, |
| | | "ack_publish_completed", |
| | | call_id_hash, |
| | | trace_id_hash, |
| | |
| | | observed |
| | | } else { |
| | | record_controlled_fixture_probe_event( |
| | | runtime_call_id, |
| | | runtime_trace_id, |
| | | "ack_publish_completed", |
| | | call_id_hash, |
| | | trace_id_hash, |
| | |
| | | Some(&participant_for_probe), |
| | | &sink, |
| | | user_participant_identity.as_deref(), |
| | | &call_id, |
| | | &trace_id, |
| | | ) |
| | | .await; |
| | | let observer_started = start_observer_after_controlled_fixture_probe( |
| | |
| | | None => (None, Some("no_current_participant"), false), |
| | | }; |
| | | record_controlled_fixture_probe_event( |
| | | &call_id, |
| | | &trace_id, |
| | | if observer_started { |
| | | "audio_observer_allowed" |
| | | } else { |
| | |
| | | { |
| | | if acknowledged_probe_sequences.contains(&probe.client_fixture_sequence) { |
| | | record_controlled_fixture_probe_event( |
| | | &call_id, |
| | | &trace_id, |
| | | "data_received", |
| | | &probe.call_id_hash, |
| | | &probe.call_trace_id_hash, |
| | |
| | | ); |
| | | if let Some(probe) = pending_probe.as_ref() { |
| | | record_controlled_fixture_probe_event( |
| | | &call_id, |
| | | &trace_id, |
| | | "data_received", |
| | | &probe.call_id_hash, |
| | | &probe.call_trace_id_hash, |
| | |
| | | current_user_participant.as_ref(), |
| | | &sink, |
| | | user_participant_identity.as_deref(), |
| | | &call_id, |
| | | &trace_id, |
| | | ) |
| | | .await; |
| | | } |
| | |
| | | participant: Option<&RemoteParticipant>, |
| | | sink: &BotAudioOutputSink, |
| | | expected_participant: Option<&str>, |
| | | runtime_call_id: &str, |
| | | runtime_trace_id: &str, |
| | | ) -> Option<bool> { |
| | | let Some(probe) = pending_probe.take() else { |
| | | return None; |
| | | }; |
| | | if acknowledged_probe_sequences.contains(&probe.sequence) { |
| | | record_controlled_fixture_probe_event( |
| | | runtime_call_id, |
| | | runtime_trace_id, |
| | | "attributes_classified", |
| | | &probe.call_id_hash, |
| | | &probe.call_trace_id_hash, |
| | |
| | | } |
| | | if Instant::now() > probe.expires_at { |
| | | record_controlled_fixture_probe_event( |
| | | runtime_call_id, |
| | | runtime_trace_id, |
| | | "attributes_classified", |
| | | &probe.call_id_hash, |
| | | &probe.call_trace_id_hash, |
| | |
| | | } |
| | | let Some(participant) = participant else { |
| | | record_controlled_fixture_probe_event( |
| | | runtime_call_id, |
| | | runtime_trace_id, |
| | | "attributes_classified", |
| | | &probe.call_id_hash, |
| | | &probe.call_trace_id_hash, |
| | |
| | | }; |
| | | if participant.identity() != probe.sender { |
| | | record_controlled_fixture_probe_event( |
| | | runtime_call_id, |
| | | runtime_trace_id, |
| | | "attributes_classified", |
| | | &probe.call_id_hash, |
| | | &probe.call_trace_id_hash, |
| | |
| | | .await; |
| | | let (ack_result, reject_reason, observed) = record_controlled_fixture_attribute_decision( |
| | | decision, |
| | | runtime_call_id, |
| | | runtime_trace_id, |
| | | &probe.call_id_hash, |
| | | &probe.call_trace_id_hash, |
| | | probe.generation, |
| | |
| | | }; |
| | | let local_participant = sink.room.local_participant(); |
| | | record_controlled_fixture_probe_event( |
| | | runtime_call_id, |
| | | runtime_trace_id, |
| | | "ack_publish_started", |
| | | &probe.call_id_hash, |
| | | &probe.call_trace_id_hash, |
| | |
| | | Some( |
| | | complete_controlled_fixture_ack_publish( |
| | | publish, |
| | | runtime_call_id, |
| | | runtime_trace_id, |
| | | observed, |
| | | ack_result, |
| | | reject_reason, |
| | |
| | | io::{Read, Write}, |
| | | net::TcpListener, |
| | | sync::{ |
| | | Arc, Mutex, |
| | | Arc, |
| | | atomic::{AtomicUsize, Ordering}, |
| | | mpsc, |
| | | }, |
| | |
| | | participant: String, |
| | | attributes: HashMap<String, String>, |
| | | }, |
| | | } |
| | | |
| | | #[derive(Clone, Default)] |
| | | struct CapturedLogs(Arc<Mutex<Vec<u8>>>); |
| | | |
| | | struct CapturedLogWriter(Arc<Mutex<Vec<u8>>>); |
| | | |
| | | impl Write for CapturedLogWriter { |
| | | fn write(&mut self, bytes: &[u8]) -> std::io::Result<usize> { |
| | | self.0.lock().unwrap().extend_from_slice(bytes); |
| | | Ok(bytes.len()) |
| | | } |
| | | |
| | | fn flush(&mut self) -> std::io::Result<()> { |
| | | Ok(()) |
| | | } |
| | | } |
| | | |
| | | impl<'a> tracing_subscriber::fmt::MakeWriter<'a> for CapturedLogs { |
| | | type Writer = CapturedLogWriter; |
| | | |
| | | fn make_writer(&'a self) -> Self::Writer { |
| | | CapturedLogWriter(self.0.clone()) |
| | | } |
| | | } |
| | | |
| | | fn drive_pre_audio_order_test_seam(events: &[PreAudioOrderEvent]) -> Vec<&'static str> { |
| | |
| | | } |
| | | |
| | | #[test] |
| | | fn controlled_fixture_probe_failure_log_is_fixed_redacted_and_has_no_audio_effect() { |
| | | let logs = CapturedLogs::default(); |
| | | let subscriber = tracing_subscriber::fmt() |
| | | .without_time() |
| | | .with_ansi(false) |
| | | .with_writer(logs.clone()) |
| | | .finish(); |
| | | fn controlled_fixture_probe_runtime_projection_binds_request_hashes_and_audio_gate() { |
| | | let call_id = "private-call-value"; |
| | | let trace_id = "private-trace-value"; |
| | | let sequence = "private-sequence-value"; |
| | | let call_id_hash = sha256_hex(call_id); |
| | | let trace_id_hash = sha256_hex(trace_id); |
| | | let mut observer_starts = 0; |
| | | tracing::subscriber::with_default(subscriber, || { |
| | | let decision = Err("wrong_source"); |
| | | let (ack_result, reject_reason, observed) = |
| | | record_controlled_fixture_attribute_decision( |
| | | decision, |
| | | &call_id_hash, |
| | | &trace_id_hash, |
| | | CONTROLLED_FIXTURE_GENERATION, |
| | | sequence, |
| | | ); |
| | | record_controlled_fixture_probe_event( |
| | | "audio_observer_blocked", |
| | | let (ack_result, reject_reason, observed) = |
| | | controlled_fixture_ack_classification(Err("wrong_source")); |
| | | let stages = [ |
| | | ("data_received", None, None), |
| | | ("attributes_classified", Some(ack_result), reject_reason), |
| | | ("ack_publish_started", Some(ack_result), reject_reason), |
| | | ("ack_publish_completed", Some(ack_result), reject_reason), |
| | | ("audio_observer_blocked", Some(ack_result), reject_reason), |
| | | ]; |
| | | for (stage, result, reason) in stages { |
| | | let event = controlled_fixture_probe_event( |
| | | call_id, |
| | | trace_id, |
| | | stage, |
| | | &call_id_hash, |
| | | &trace_id_hash, |
| | | CONTROLLED_FIXTURE_GENERATION, |
| | | sequence, |
| | | observed, |
| | | Some(ack_result), |
| | | reject_reason, |
| | | result, |
| | | reason, |
| | | ); |
| | | assert!(!start_observer_after_controlled_fixture_probe( |
| | | true, |
| | | Some(observed), |
| | | || observer_starts += 1, |
| | | )); |
| | | }); |
| | | let output = String::from_utf8(logs.0.lock().unwrap().clone()).unwrap(); |
| | | assert!(output.contains("ack_result=\"rejected\"")); |
| | | assert!(output.contains("reject_reason=\"wrong_source\"")); |
| | | assert!(output.contains("observed=false")); |
| | | assert!(output.contains(&sha256_hex(call_id))); |
| | | assert!(output.contains(&sha256_hex(trace_id))); |
| | | assert!(output.contains(&sha256_hex(sequence))); |
| | | assert!(!output.contains(call_id)); |
| | | assert!(!output.contains(trace_id)); |
| | | assert!(!output.contains(sequence)); |
| | | assert_eq!(event["type"], "cv_activity"); |
| | | assert_eq!(event["eventName"], "controlled_fixture_attribute_probe"); |
| | | assert_eq!(event["extension"]["call_id_hash"], call_id_hash); |
| | | assert_eq!(event["extension"]["trace_id_hash"], trace_id_hash); |
| | | assert_eq!(event["extension"]["stage"], stage); |
| | | let output = event.to_string(); |
| | | assert!(!output.contains(sequence)); |
| | | } |
| | | assert!(!start_observer_after_controlled_fixture_probe( |
| | | true, |
| | | Some(observed), |
| | | || observer_starts += 1, |
| | | )); |
| | | assert_eq!(observer_starts, 0); |
| | | |
| | | let (ack_result, reject_reason, observed) = controlled_fixture_ack_classification(Ok(())); |
| | | let allowed = controlled_fixture_probe_event( |
| | | call_id, |
| | | trace_id, |
| | | "audio_observer_allowed", |
| | | &call_id_hash, |
| | | &trace_id_hash, |
| | | CONTROLLED_FIXTURE_GENERATION, |
| | | sequence, |
| | | observed, |
| | | Some(ack_result), |
| | | reject_reason, |
| | | ); |
| | | assert_eq!(allowed["extension"]["trace_id_hash"], trace_id_hash); |
| | | assert!(start_observer_after_controlled_fixture_probe( |
| | | true, |
| | | Some(observed), |
| | | || observer_starts += 1, |
| | | )); |
| | | assert_eq!(observer_starts, 1); |
| | | } |
| | | |
| | | #[test] |
| | |
| | | let mut acknowledged = HashSet::new(); |
| | | let probe_result = complete_controlled_fixture_ack_publish( |
| | | async { Err::<(), ()>(()) }, |
| | | "runtime-call-publish-failure", |
| | | "runtime-trace-publish-failure", |
| | | true, |
| | | "observed", |
| | | None, |
| | |
| | | |
| | | let observed_result = complete_controlled_fixture_ack_publish( |
| | | async { Ok::<(), ()>(()) }, |
| | | "runtime-call-publish-success", |
| | | "runtime-trace-publish-success", |
| | | true, |
| | | "observed", |
| | | None, |