| | |
| | | const CONTROLLED_FIXTURE_GENERATION: u64 = 1; |
| | | const CONTROLLED_FIXTURE_PROBE_RECHECK_DELAY: Duration = Duration::from_millis(25); |
| | | const CONTROLLED_FIXTURE_PROBE_TTL: Duration = Duration::from_millis(250); |
| | | const CONTROLLED_FIXTURE_POST_EXPIRY_WINDOW: Duration = Duration::from_millis(2_000); |
| | | |
| | | const CONTROLLED_FIXTURE_ACK_RESULTS: [&str; 4] = |
| | | ["observed", "rejected", "timeout", "publish_failed"]; |
| | |
| | | call_trace_id_hash: String, |
| | | generation: u64, |
| | | sequence: String, |
| | | received_at: Instant, |
| | | expires_at: Instant, |
| | | } |
| | | |
| | | #[derive(Debug, PartialEq, Eq)] |
| | | struct ControlledFixtureVisibilityEvidence { |
| | | first_visible_bucket: &'static str, |
| | | visibility_source: &'static str, |
| | | visibility_result: &'static str, |
| | | binding_matched: bool, |
| | | } |
| | | |
| | | #[derive(Debug, PartialEq, Eq)] |
| | | struct ControlledFixtureAckPublishOutcome { |
| | | observed: bool, |
| | | published: bool, |
| | | } |
| | | |
| | | fn sha256_hex(value: &str) -> String { |
| | |
| | | } |
| | | |
| | | fn record_controlled_fixture_probe_event( |
| | | runtime_call_id: &str, |
| | | runtime_trace_id: &str, |
| | | stage: &'static str, |
| | | call_id_hash: &str, |
| | | trace_id_hash: &str, |
| | |
| | | println!( |
| | | "{}", |
| | | controlled_fixture_probe_event( |
| | | runtime_call_id, |
| | | runtime_trace_id, |
| | | stage, |
| | | call_id_hash, |
| | | trace_id_hash, |
| | |
| | | } |
| | | |
| | | fn controlled_fixture_probe_event( |
| | | runtime_call_id: &str, |
| | | runtime_trace_id: &str, |
| | | stage: &'static str, |
| | | call_id_hash: &str, |
| | | trace_id_hash: &str, |
| | |
| | | reject_reason: Option<&'static str>, |
| | | ) -> serde_json::Value { |
| | | json!({ |
| | | "event": "controlled_fixture_attribute_probe", |
| | | "stage": stage, |
| | | "observed": observed, |
| | | "ack_result": ack_result, |
| | | "reject_reason": reject_reason, |
| | | "call_id_hash": call_id_hash, |
| | | "trace_id_hash": trace_id_hash, |
| | | "generation": generation, |
| | | "sequence_hash": sha256_hex(sequence), |
| | | "type": "cv_activity", |
| | | "callId": runtime_call_id, |
| | | "traceId": runtime_trace_id, |
| | | "turnId": null, |
| | | "eventName": "controlled_fixture_attribute_probe", |
| | | "eventWallTimeMs": current_time_millis(), |
| | | "result": "ok", |
| | | "reasonCode": null, |
| | | "retryable": null, |
| | | "extension": { |
| | | "stage": stage, |
| | | "observed": observed, |
| | | "ack_result": ack_result, |
| | | "reject_reason": reject_reason, |
| | | "call_id_hash": call_id_hash, |
| | | "trace_id_hash": trace_id_hash, |
| | | "generation": generation, |
| | | "sequence_hash": sha256_hex(sequence), |
| | | }, |
| | | }) |
| | | } |
| | | |
| | | fn record_controlled_fixture_attribute_decision( |
| | | decision: Result<(), &'static str>, |
| | | runtime_call_id: &str, |
| | | runtime_trace_id: &str, |
| | | call_id_hash: &str, |
| | | trace_id_hash: &str, |
| | | generation: u64, |
| | |
| | | ) -> (&'static str, Option<&'static str>, bool) { |
| | | let classification = controlled_fixture_ack_classification(decision); |
| | | record_controlled_fixture_probe_event( |
| | | runtime_call_id, |
| | | runtime_trace_id, |
| | | "attributes_classified", |
| | | call_id_hash, |
| | | trace_id_hash, |
| | |
| | | |
| | | async fn complete_controlled_fixture_ack_publish<F, E>( |
| | | publish: F, |
| | | runtime_call_id: &str, |
| | | runtime_trace_id: &str, |
| | | observed: bool, |
| | | ack_result: &'static str, |
| | | reject_reason: Option<&'static str>, |
| | |
| | | generation: u64, |
| | | sequence: &str, |
| | | acknowledged_probe_sequences: &mut HashSet<String>, |
| | | ) -> bool |
| | | ) -> ControlledFixtureAckPublishOutcome |
| | | where |
| | | F: Future<Output = Result<(), E>>, |
| | | { |
| | | if publish.await.is_ok() { |
| | | record_controlled_fixture_probe_event( |
| | | runtime_call_id, |
| | | runtime_trace_id, |
| | | "ack_publish_completed", |
| | | call_id_hash, |
| | | trace_id_hash, |
| | |
| | | reject_reason, |
| | | ); |
| | | acknowledged_probe_sequences.insert(sequence.to_string()); |
| | | observed |
| | | ControlledFixtureAckPublishOutcome { |
| | | observed, |
| | | published: true, |
| | | } |
| | | } else { |
| | | record_controlled_fixture_probe_event( |
| | | runtime_call_id, |
| | | runtime_trace_id, |
| | | "ack_publish_completed", |
| | | call_id_hash, |
| | | trace_id_hash, |
| | |
| | | Some("publish_failed"), |
| | | Some("ack_publish_failed"), |
| | | ); |
| | | false |
| | | ControlledFixtureAckPublishOutcome { |
| | | observed: false, |
| | | published: false, |
| | | } |
| | | } |
| | | } |
| | | |
| | |
| | | { |
| | | return None; |
| | | } |
| | | let received_at = Instant::now(); |
| | | Some(PendingControlledFixtureProbe { |
| | | sender: sender.clone(), |
| | | call_id_hash: probe.call_id_hash, |
| | | call_trace_id_hash: probe.call_trace_id_hash, |
| | | generation: probe.generation, |
| | | sequence: probe.client_fixture_sequence, |
| | | expires_at: Instant::now() + CONTROLLED_FIXTURE_PROBE_TTL, |
| | | received_at, |
| | | expires_at: received_at + CONTROLLED_FIXTURE_PROBE_TTL, |
| | | }) |
| | | } |
| | | |
| | | fn controlled_fixture_probe_binding_decision( |
| | | probe_sender: &str, |
| | | current_audio_participant: Option<&str>, |
| | | expected_participant: Option<&str>, |
| | | ) -> Result<(), &'static str> { |
| | | if !is_bound_user_participant(probe_sender, expected_participant) { |
| | | return Err("wrong_participant"); |
| | | } |
| | | let Some(current_audio_participant) = current_audio_participant else { |
| | | return Err("no_current_participant"); |
| | | }; |
| | | if current_audio_participant != probe_sender |
| | | || !is_bound_user_participant(current_audio_participant, expected_participant) |
| | | { |
| | | return Err("wrong_participant"); |
| | | } |
| | | Ok(()) |
| | | } |
| | | |
| | | fn controlled_fixture_visibility_bucket(elapsed: Duration) -> &'static str { |
| | | if elapsed <= Duration::from_millis(250) { |
| | | "lte_250ms" |
| | | } else if elapsed <= Duration::from_millis(500) { |
| | | "250_500ms" |
| | | } else if elapsed <= Duration::from_millis(1_000) { |
| | | "500_1000ms" |
| | | } else { |
| | | "1000_2000ms" |
| | | } |
| | | } |
| | | |
| | | async fn observe_controlled_fixture_post_expiry_views<F, G>( |
| | | received_at: Instant, |
| | | observation_deadline: Instant, |
| | | held_participant: &str, |
| | | expected_participant: Option<&str>, |
| | | requested_sequence: &str, |
| | | lifecycle_active: Arc<AtomicBool>, |
| | | mut read_held_attributes: F, |
| | | mut read_current_participant: G, |
| | | ) -> Option<ControlledFixtureVisibilityEvidence> |
| | | where |
| | | F: FnMut() -> std::collections::HashMap<String, String>, |
| | | G: FnMut() -> Option<(String, std::collections::HashMap<String, String>)>, |
| | | { |
| | | loop { |
| | | if !lifecycle_active.load(Ordering::Acquire) { |
| | | return None; |
| | | } |
| | | let now = Instant::now(); |
| | | let held_decision = classify_controlled_fixture_attributes( |
| | | held_participant, |
| | | expected_participant, |
| | | &read_held_attributes(), |
| | | requested_sequence, |
| | | ); |
| | | match held_decision { |
| | | Ok(()) => { |
| | | return Some(ControlledFixtureVisibilityEvidence { |
| | | first_visible_bucket: controlled_fixture_visibility_bucket( |
| | | now.saturating_duration_since(received_at), |
| | | ), |
| | | visibility_source: "participant_attributes_poll", |
| | | visibility_result: "held_visible", |
| | | binding_matched: true, |
| | | }); |
| | | } |
| | | Err("missing_attributes") => {} |
| | | Err(_) => return None, |
| | | } |
| | | let Some((current_identity, current_attributes)) = read_current_participant() else { |
| | | return None; |
| | | }; |
| | | match classify_controlled_fixture_attributes( |
| | | ¤t_identity, |
| | | expected_participant, |
| | | ¤t_attributes, |
| | | requested_sequence, |
| | | ) { |
| | | Ok(()) => { |
| | | return Some(ControlledFixtureVisibilityEvidence { |
| | | first_visible_bucket: controlled_fixture_visibility_bucket( |
| | | now.saturating_duration_since(received_at), |
| | | ), |
| | | visibility_source: "current_room_lookup", |
| | | visibility_result: "held_stale_current_visible", |
| | | binding_matched: true, |
| | | }); |
| | | } |
| | | Err("missing_attributes") => {} |
| | | Err(_) => return None, |
| | | } |
| | | if now >= observation_deadline { |
| | | return Some(ControlledFixtureVisibilityEvidence { |
| | | first_visible_bucket: "never_visible_within_observation_window", |
| | | visibility_source: "held_and_current_room_lookup", |
| | | visibility_result: "unavailable_both", |
| | | binding_matched: true, |
| | | }); |
| | | } |
| | | sleep(CONTROLLED_FIXTURE_PROBE_RECHECK_DELAY).await; |
| | | } |
| | | } |
| | | |
| | | fn controlled_fixture_visibility_event( |
| | | runtime_call_id: &str, |
| | | runtime_trace_id: &str, |
| | | probe: &PendingControlledFixtureProbe, |
| | | evidence: &ControlledFixtureVisibilityEvidence, |
| | | ) -> serde_json::Value { |
| | | json!({ |
| | | "type": "cv_activity", |
| | | "callId": runtime_call_id, |
| | | "traceId": runtime_trace_id, |
| | | "turnId": null, |
| | | "eventName": "controlled_fixture_attribute_probe", |
| | | "eventWallTimeMs": current_time_millis(), |
| | | "result": "ok", |
| | | "reasonCode": null, |
| | | "retryable": null, |
| | | "extension": { |
| | | "stage": "post_expiry_visibility", |
| | | "first_visible_bucket": evidence.first_visible_bucket, |
| | | "visibility_source": evidence.visibility_source, |
| | | "visibility_result": evidence.visibility_result, |
| | | "binding_matched": evidence.binding_matched, |
| | | "call_id_hash": probe.call_id_hash, |
| | | "trace_id_hash": probe.call_trace_id_hash, |
| | | "generation": probe.generation, |
| | | "sequence_hash": sha256_hex(&probe.sequence), |
| | | "evidence_count": 1, |
| | | }, |
| | | }) |
| | | } |
| | | |
| | | fn spawn_controlled_fixture_post_expiry_observation( |
| | | probe: PendingControlledFixtureProbe, |
| | | participant: RemoteParticipant, |
| | | room: Arc<Room>, |
| | | expected_participant: Option<String>, |
| | | lifecycle_active: Arc<AtomicBool>, |
| | | runtime_call_id: String, |
| | | runtime_trace_id: String, |
| | | ) { |
| | | tokio::spawn(async move { |
| | | let participant_identity = participant.identity().to_string(); |
| | | let evidence = observe_controlled_fixture_post_expiry_views( |
| | | probe.received_at, |
| | | probe.received_at + CONTROLLED_FIXTURE_POST_EXPIRY_WINDOW, |
| | | &participant_identity, |
| | | expected_participant.as_deref(), |
| | | &probe.sequence, |
| | | lifecycle_active.clone(), |
| | | || participant.attributes(), |
| | | || { |
| | | room.remote_participants() |
| | | .get(&probe.sender) |
| | | .map(|current| (current.identity().to_string(), current.attributes())) |
| | | }, |
| | | ) |
| | | .await; |
| | | if lifecycle_active.load(Ordering::Acquire) { |
| | | if let Some(evidence) = evidence { |
| | | println!( |
| | | "{}", |
| | | controlled_fixture_visibility_event( |
| | | &runtime_call_id, |
| | | &runtime_trace_id, |
| | | &probe, |
| | | &evidence, |
| | | ) |
| | | ); |
| | | } |
| | | } |
| | | }); |
| | | } |
| | | |
| | | fn classify_controlled_fixture_attributes( |
| | |
| | | let mut current_user_participant: Option<RemoteParticipant> = None; |
| | | let mut pending_probe: Option<PendingControlledFixtureProbe> = None; |
| | | let mut acknowledged_probe_sequences = HashSet::new(); |
| | | let controlled_fixture_lifecycle_active = Arc::new(AtomicBool::new(true)); |
| | | |
| | | while let Some(event) = events.recv().await { |
| | | match event { |
| | |
| | | Some(&participant_for_probe), |
| | | &sink, |
| | | user_participant_identity.as_deref(), |
| | | &call_id, |
| | | &trace_id, |
| | | controlled_fixture_lifecycle_active.clone(), |
| | | ) |
| | | .await; |
| | | let observer_started = start_observer_after_controlled_fixture_probe( |
| | |
| | | None => (None, Some("no_current_participant"), false), |
| | | }; |
| | | record_controlled_fixture_probe_event( |
| | | &call_id, |
| | | &trace_id, |
| | | if observer_started { |
| | | "audio_observer_allowed" |
| | | } else { |
| | |
| | | } |
| | | current_user_participant = Some(participant_for_probe); |
| | | } |
| | | RoomEvent::TrackUnsubscribed { |
| | | track: RemoteTrack::Audio(_), |
| | | publication: _, |
| | | participant, |
| | | } => { |
| | | if current_user_participant |
| | | .as_ref() |
| | | .is_some_and(|current| current.identity() == participant.identity()) |
| | | { |
| | | current_user_participant = None; |
| | | } |
| | | } |
| | | RoomEvent::DataReceived { |
| | | payload, |
| | | topic: Some(topic), |
| | |
| | | { |
| | | if acknowledged_probe_sequences.contains(&probe.client_fixture_sequence) { |
| | | record_controlled_fixture_probe_event( |
| | | &call_id, |
| | | &trace_id, |
| | | "data_received", |
| | | &probe.call_id_hash, |
| | | &probe.call_trace_id_hash, |
| | |
| | | ); |
| | | if let Some(probe) = pending_probe.as_ref() { |
| | | record_controlled_fixture_probe_event( |
| | | &call_id, |
| | | &trace_id, |
| | | "data_received", |
| | | &probe.call_id_hash, |
| | | &probe.call_trace_id_hash, |
| | |
| | | current_user_participant.as_ref(), |
| | | &sink, |
| | | user_participant_identity.as_deref(), |
| | | &call_id, |
| | | &trace_id, |
| | | controlled_fixture_lifecycle_active.clone(), |
| | | ) |
| | | .await; |
| | | } |
| | |
| | | _ => {} |
| | | } |
| | | } |
| | | controlled_fixture_lifecycle_active.store(false, Ordering::Release); |
| | | } |
| | | |
| | | async fn process_controlled_fixture_probe( |
| | |
| | | participant: Option<&RemoteParticipant>, |
| | | sink: &BotAudioOutputSink, |
| | | expected_participant: Option<&str>, |
| | | runtime_call_id: &str, |
| | | runtime_trace_id: &str, |
| | | lifecycle_active: Arc<AtomicBool>, |
| | | ) -> Option<bool> { |
| | | let Some(probe) = pending_probe.take() else { |
| | | return None; |
| | | }; |
| | | if acknowledged_probe_sequences.contains(&probe.sequence) { |
| | | record_controlled_fixture_probe_event( |
| | | runtime_call_id, |
| | | runtime_trace_id, |
| | | "attributes_classified", |
| | | &probe.call_id_hash, |
| | | &probe.call_trace_id_hash, |
| | |
| | | } |
| | | if Instant::now() > probe.expires_at { |
| | | record_controlled_fixture_probe_event( |
| | | runtime_call_id, |
| | | runtime_trace_id, |
| | | "attributes_classified", |
| | | &probe.call_id_hash, |
| | | &probe.call_trace_id_hash, |
| | |
| | | } |
| | | let Some(participant) = participant else { |
| | | record_controlled_fixture_probe_event( |
| | | runtime_call_id, |
| | | runtime_trace_id, |
| | | "attributes_classified", |
| | | &probe.call_id_hash, |
| | | &probe.call_trace_id_hash, |
| | |
| | | }; |
| | | if participant.identity() != probe.sender { |
| | | record_controlled_fixture_probe_event( |
| | | runtime_call_id, |
| | | runtime_trace_id, |
| | | "attributes_classified", |
| | | &probe.call_id_hash, |
| | | &probe.call_trace_id_hash, |
| | |
| | | ); |
| | | return Some(false); |
| | | } |
| | | let decision = observe_controlled_fixture_attributes( |
| | | probe.expires_at, |
| | | &participant.identity().to_string(), |
| | | let decision = controlled_fixture_probe_binding_decision( |
| | | probe.sender.as_str(), |
| | | Some(participant.identity().as_str()), |
| | | expected_participant, |
| | | &probe.sequence, |
| | | || participant.attributes(), |
| | | ) |
| | | .await; |
| | | ); |
| | | if let Err(reason) = decision { |
| | | record_controlled_fixture_attribute_decision( |
| | | Err(reason), |
| | | runtime_call_id, |
| | | runtime_trace_id, |
| | | &probe.call_id_hash, |
| | | &probe.call_trace_id_hash, |
| | | probe.generation, |
| | | &probe.sequence, |
| | | ); |
| | | return Some(false); |
| | | } |
| | | let (ack_result, reject_reason, observed) = record_controlled_fixture_attribute_decision( |
| | | decision, |
| | | Ok(()), |
| | | runtime_call_id, |
| | | runtime_trace_id, |
| | | &probe.call_id_hash, |
| | | &probe.call_trace_id_hash, |
| | | probe.generation, |
| | |
| | | }; |
| | | let local_participant = sink.room.local_participant(); |
| | | record_controlled_fixture_probe_event( |
| | | runtime_call_id, |
| | | runtime_trace_id, |
| | | "ack_publish_started", |
| | | &probe.call_id_hash, |
| | | &probe.call_trace_id_hash, |
| | |
| | | payload, |
| | | topic: Some(CONTROLLED_FIXTURE_ACK_TOPIC.to_string()), |
| | | reliable: true, |
| | | destination_identities: vec![probe.sender], |
| | | destination_identities: vec![probe.sender.clone()], |
| | | }); |
| | | Some( |
| | | complete_controlled_fixture_ack_publish( |
| | | publish, |
| | | observed, |
| | | ack_result, |
| | | reject_reason, |
| | | &probe.call_id_hash, |
| | | &probe.call_trace_id_hash, |
| | | probe.generation, |
| | | &probe.sequence, |
| | | acknowledged_probe_sequences, |
| | | ) |
| | | .await, |
| | | let outcome = complete_controlled_fixture_ack_publish( |
| | | publish, |
| | | runtime_call_id, |
| | | runtime_trace_id, |
| | | observed, |
| | | ack_result, |
| | | reject_reason, |
| | | &probe.call_id_hash, |
| | | &probe.call_trace_id_hash, |
| | | probe.generation, |
| | | &probe.sequence, |
| | | acknowledged_probe_sequences, |
| | | ) |
| | | .await; |
| | | if outcome.published && ack_result == "timeout" && reject_reason == Some("expired") { |
| | | spawn_controlled_fixture_post_expiry_observation( |
| | | probe, |
| | | participant.clone(), |
| | | sink.room.clone(), |
| | | expected_participant.map(str::to_string), |
| | | lifecycle_active, |
| | | runtime_call_id.to_string(), |
| | | runtime_trace_id.to_string(), |
| | | ); |
| | | } |
| | | Some(outcome.observed) |
| | | } |
| | | |
| | | async fn handle_finished_turn( |
| | |
| | | |
| | | #[derive(Debug)] |
| | | enum PreAudioOrderEvent { |
| | | DataReceived { |
| | | sender: String, |
| | | sequence: String, |
| | | }, |
| | | TrackSubscribed { |
| | | participant: String, |
| | | attributes: HashMap<String, String>, |
| | | }, |
| | | DataReceived { sender: String, sequence: String }, |
| | | TrackSubscribed { participant: String }, |
| | | } |
| | | |
| | | fn drive_pre_audio_order_test_seam(events: &[PreAudioOrderEvent]) -> Vec<&'static str> { |
| | | let mut pending_sequence = None; |
| | | let call_id = "production-order-call"; |
| | | let trace_id = "production-order-trace"; |
| | | let mut pending_probe = None; |
| | | let mut effects = Vec::new(); |
| | | for event in events { |
| | | match event { |
| | | PreAudioOrderEvent::DataReceived { sender, sequence } if sender == "user-1" => { |
| | | pending_sequence = Some(sequence.as_str()); |
| | | PreAudioOrderEvent::DataReceived { sender, sequence } => { |
| | | let payload = serde_json::to_vec(&json!({ |
| | | "type": CONTROLLED_FIXTURE_PROBE_TOPIC, |
| | | "protocolVersion": CONTROLLED_FIXTURE_PROTOCOL_VERSION, |
| | | "callIdHash": sha256_hex(call_id), |
| | | "callTraceIdHash": sha256_hex(trace_id), |
| | | "generation": CONTROLLED_FIXTURE_GENERATION, |
| | | "clientFixtureSequence": sequence, |
| | | })) |
| | | .expect("production probe payload"); |
| | | pending_probe = controlled_fixture_probe( |
| | | &payload, |
| | | call_id, |
| | | trace_id, |
| | | &ParticipantIdentity(sender.clone()), |
| | | Some("user-1"), |
| | | ); |
| | | } |
| | | PreAudioOrderEvent::TrackSubscribed { |
| | | participant, |
| | | attributes, |
| | | } => { |
| | | let pending = pending_sequence.is_some(); |
| | | let probe_result = pending_sequence.map(|sequence| { |
| | | classify_controlled_fixture_attributes( |
| | | participant, |
| | | PreAudioOrderEvent::TrackSubscribed { participant } => { |
| | | let pending = pending_probe.is_some(); |
| | | let probe_result = pending_probe.as_ref().map(|probe| { |
| | | controlled_fixture_probe_binding_decision( |
| | | probe.sender.as_str(), |
| | | Some(participant), |
| | | Some("user-1"), |
| | | attributes, |
| | | sequence, |
| | | ) |
| | | .is_ok() |
| | | }); |
| | |
| | | if observer_started { |
| | | effects.push("observer_started"); |
| | | } |
| | | pending_sequence = None; |
| | | pending_probe = None; |
| | | } |
| | | PreAudioOrderEvent::DataReceived { .. } => {} |
| | | } |
| | | } |
| | | effects |
| | |
| | | ]; |
| | | for (stage, result, reason) in stages { |
| | | let event = controlled_fixture_probe_event( |
| | | call_id, |
| | | trace_id, |
| | | stage, |
| | | &call_id_hash, |
| | | &trace_id_hash, |
| | |
| | | result, |
| | | reason, |
| | | ); |
| | | assert_eq!(event["call_id_hash"], call_id_hash); |
| | | assert_eq!(event["trace_id_hash"], trace_id_hash); |
| | | assert_eq!(event["stage"], stage); |
| | | assert_eq!(event["type"], "cv_activity"); |
| | | assert_eq!(event["eventName"], "controlled_fixture_attribute_probe"); |
| | | assert_eq!(event["extension"]["call_id_hash"], call_id_hash); |
| | | assert_eq!(event["extension"]["trace_id_hash"], trace_id_hash); |
| | | assert_eq!(event["extension"]["stage"], stage); |
| | | let output = event.to_string(); |
| | | assert!(!output.contains(call_id)); |
| | | assert!(!output.contains(trace_id)); |
| | | assert!(!output.contains(sequence)); |
| | | } |
| | | assert!(!start_observer_after_controlled_fixture_probe( |
| | |
| | | |
| | | let (ack_result, reject_reason, observed) = controlled_fixture_ack_classification(Ok(())); |
| | | let allowed = controlled_fixture_probe_event( |
| | | call_id, |
| | | trace_id, |
| | | "audio_observer_allowed", |
| | | &call_id_hash, |
| | | &trace_id_hash, |
| | |
| | | Some(ack_result), |
| | | reject_reason, |
| | | ); |
| | | assert_eq!(allowed["trace_id_hash"], trace_id_hash); |
| | | assert_eq!(allowed["extension"]["trace_id_hash"], trace_id_hash); |
| | | assert!(start_observer_after_controlled_fixture_probe( |
| | | true, |
| | | Some(observed), |
| | |
| | | call_trace_id_hash: request_trace_hash.to_string(), |
| | | generation: CONTROLLED_FIXTURE_GENERATION, |
| | | sequence: "fixture-01".to_string(), |
| | | received_at: Instant::now(), |
| | | expires_at: Instant::now() + CONTROLLED_FIXTURE_PROBE_TTL, |
| | | }; |
| | | |
| | |
| | | let mut acknowledged = HashSet::new(); |
| | | let probe_result = complete_controlled_fixture_ack_publish( |
| | | async { Err::<(), ()>(()) }, |
| | | "runtime-call-publish-failure", |
| | | "runtime-trace-publish-failure", |
| | | true, |
| | | "observed", |
| | | None, |
| | |
| | | let mut speaking_starts = 0; |
| | | assert!(!start_observer_after_controlled_fixture_probe( |
| | | true, |
| | | Some(probe_result), |
| | | Some(probe_result.observed), |
| | | || { |
| | | observer_starts += 1; |
| | | session_starts += 1; |
| | |
| | | speaking_starts += 1; |
| | | }, |
| | | )); |
| | | assert!(!probe_result); |
| | | assert_eq!( |
| | | probe_result, |
| | | ControlledFixtureAckPublishOutcome { |
| | | observed: false, |
| | | published: false, |
| | | } |
| | | ); |
| | | assert!(acknowledged.is_empty()); |
| | | assert_eq!(observer_starts, 0); |
| | | assert_eq!(session_starts, 0); |
| | |
| | | |
| | | let observed_result = complete_controlled_fixture_ack_publish( |
| | | async { Ok::<(), ()>(()) }, |
| | | "runtime-call-publish-success", |
| | | "runtime-trace-publish-success", |
| | | true, |
| | | "observed", |
| | | None, |
| | |
| | | let mut successful_observer_starts = 0; |
| | | assert!(start_observer_after_controlled_fixture_probe( |
| | | true, |
| | | Some(observed_result), |
| | | Some(observed_result.observed), |
| | | || successful_observer_starts += 1, |
| | | )); |
| | | assert!(observed_result); |
| | | assert_eq!( |
| | | observed_result, |
| | | ControlledFixtureAckPublishOutcome { |
| | | observed: true, |
| | | published: true, |
| | | } |
| | | ); |
| | | assert!(acknowledged.contains("fixture-02")); |
| | | assert_eq!(successful_observer_starts, 1); |
| | | } |
| | |
| | | assert_eq!(observer_starts, 0); |
| | | } |
| | | |
| | | #[tokio::test] |
| | | async fn production_post_expiry_observation_records_bounded_visibility_without_second_ack() { |
| | | let started_at = Instant::now(); |
| | | let active = Arc::new(AtomicBool::new(true)); |
| | | let expected = HashMap::from([ |
| | | ( |
| | | "inputSourceCategory".to_string(), |
| | | "controlled_fixture".to_string(), |
| | | ), |
| | | ( |
| | | "clientFixtureSequence".to_string(), |
| | | "fixture-01".to_string(), |
| | | ), |
| | | ]); |
| | | let mut acknowledged = HashSet::new(); |
| | | let expired_ack = complete_controlled_fixture_ack_publish( |
| | | async { Ok::<(), ()>(()) }, |
| | | "runtime-call-post-expiry", |
| | | "runtime-trace-post-expiry", |
| | | false, |
| | | "timeout", |
| | | Some("expired"), |
| | | "call-post-expiry", |
| | | "trace-post-expiry", |
| | | CONTROLLED_FIXTURE_GENERATION, |
| | | "fixture-01", |
| | | &mut acknowledged, |
| | | ) |
| | | .await; |
| | | assert_eq!( |
| | | expired_ack, |
| | | ControlledFixtureAckPublishOutcome { |
| | | observed: false, |
| | | published: true, |
| | | } |
| | | ); |
| | | let evidence = observe_controlled_fixture_post_expiry_views( |
| | | started_at, |
| | | started_at + CONTROLLED_FIXTURE_POST_EXPIRY_WINDOW, |
| | | "user-1", |
| | | Some("user-1"), |
| | | "fixture-01", |
| | | active, |
| | | || { |
| | | if started_at.elapsed() >= Duration::from_millis(300) { |
| | | expected.clone() |
| | | } else { |
| | | HashMap::new() |
| | | } |
| | | }, |
| | | || Some(("user-1".to_string(), HashMap::new())), |
| | | ) |
| | | .await; |
| | | assert_eq!(acknowledged.len(), 1); |
| | | assert_eq!( |
| | | evidence, |
| | | Some(ControlledFixtureVisibilityEvidence { |
| | | first_visible_bucket: "250_500ms", |
| | | visibility_source: "participant_attributes_poll", |
| | | visibility_result: "held_visible", |
| | | binding_matched: true, |
| | | }) |
| | | ); |
| | | |
| | | let never_started_at = Instant::now(); |
| | | assert_eq!( |
| | | observe_controlled_fixture_post_expiry_views( |
| | | never_started_at, |
| | | never_started_at + Duration::from_millis(40), |
| | | "user-1", |
| | | Some("user-1"), |
| | | "fixture-01", |
| | | Arc::new(AtomicBool::new(true)), |
| | | HashMap::new, |
| | | || Some(("user-1".to_string(), HashMap::new())), |
| | | ) |
| | | .await, |
| | | Some(ControlledFixtureVisibilityEvidence { |
| | | first_visible_bucket: "never_visible_within_observation_window", |
| | | visibility_source: "held_and_current_room_lookup", |
| | | visibility_result: "unavailable_both", |
| | | binding_matched: true, |
| | | }) |
| | | ); |
| | | |
| | | assert_eq!( |
| | | observe_controlled_fixture_post_expiry_views( |
| | | Instant::now(), |
| | | Instant::now() + Duration::from_millis(50), |
| | | "cross-call-user", |
| | | Some("user-1"), |
| | | "fixture-01", |
| | | Arc::new(AtomicBool::new(true)), |
| | | || expected.clone(), |
| | | || Some(("user-1".to_string(), expected.clone())), |
| | | ) |
| | | .await, |
| | | None |
| | | ); |
| | | |
| | | let inactive = Arc::new(AtomicBool::new(false)); |
| | | assert_eq!( |
| | | observe_controlled_fixture_post_expiry_views( |
| | | Instant::now(), |
| | | Instant::now() + Duration::from_millis(50), |
| | | "user-1", |
| | | Some("user-1"), |
| | | "fixture-01", |
| | | inactive, |
| | | HashMap::new, |
| | | || Some(("user-1".to_string(), HashMap::new())), |
| | | ) |
| | | .await, |
| | | None |
| | | ); |
| | | assert_eq!(acknowledged.len(), 1); |
| | | |
| | | let probe = PendingControlledFixtureProbe { |
| | | sender: ParticipantIdentity("user-1".to_string()), |
| | | call_id_hash: "call-post-expiry".to_string(), |
| | | call_trace_id_hash: "trace-post-expiry".to_string(), |
| | | generation: CONTROLLED_FIXTURE_GENERATION, |
| | | sequence: "fixture-01".to_string(), |
| | | received_at: started_at, |
| | | expires_at: started_at + CONTROLLED_FIXTURE_PROBE_TTL, |
| | | }; |
| | | let event = controlled_fixture_visibility_event( |
| | | "runtime-call-post-expiry", |
| | | "runtime-trace-post-expiry", |
| | | &probe, |
| | | &evidence.unwrap(), |
| | | ); |
| | | let extension = event["extension"].as_object().unwrap(); |
| | | let mut keys = extension.keys().map(String::as_str).collect::<Vec<_>>(); |
| | | keys.sort_unstable(); |
| | | assert_eq!( |
| | | keys, |
| | | vec![ |
| | | "binding_matched", |
| | | "call_id_hash", |
| | | "evidence_count", |
| | | "first_visible_bucket", |
| | | "generation", |
| | | "sequence_hash", |
| | | "stage", |
| | | "trace_id_hash", |
| | | "visibility_result", |
| | | "visibility_source", |
| | | ] |
| | | ); |
| | | let encoded = event.to_string(); |
| | | for forbidden in [ |
| | | "\"participant\":", |
| | | "\"room\":", |
| | | "\"track\":", |
| | | "\"payload\":", |
| | | "\"audio\":", |
| | | ] { |
| | | assert!(!encoded.contains(forbidden)); |
| | | } |
| | | } |
| | | |
| | | #[tokio::test] |
| | | async fn production_post_expiry_observation_distinguishes_held_stale_from_current_room_view() { |
| | | let started_at = Instant::now(); |
| | | let current_attributes = HashMap::from([ |
| | | ( |
| | | "inputSourceCategory".to_string(), |
| | | "controlled_fixture".to_string(), |
| | | ), |
| | | ( |
| | | "clientFixtureSequence".to_string(), |
| | | "fixture-01".to_string(), |
| | | ), |
| | | ]); |
| | | let mut acknowledged = HashSet::new(); |
| | | let expired_ack = complete_controlled_fixture_ack_publish( |
| | | async { Ok::<(), ()>(()) }, |
| | | "runtime-call-current-view", |
| | | "runtime-trace-current-view", |
| | | false, |
| | | "timeout", |
| | | Some("expired"), |
| | | "call-current-view", |
| | | "trace-current-view", |
| | | CONTROLLED_FIXTURE_GENERATION, |
| | | "fixture-01", |
| | | &mut acknowledged, |
| | | ) |
| | | .await; |
| | | let evidence = observe_controlled_fixture_post_expiry_views( |
| | | started_at, |
| | | started_at + Duration::from_millis(100), |
| | | "user-1", |
| | | Some("user-1"), |
| | | "fixture-01", |
| | | Arc::new(AtomicBool::new(true)), |
| | | HashMap::new, |
| | | || Some(("user-1".to_string(), current_attributes.clone())), |
| | | ) |
| | | .await; |
| | | |
| | | assert_eq!( |
| | | expired_ack, |
| | | ControlledFixtureAckPublishOutcome { |
| | | observed: false, |
| | | published: true, |
| | | } |
| | | ); |
| | | assert_eq!(acknowledged.len(), 1); |
| | | assert_eq!( |
| | | evidence, |
| | | Some(ControlledFixtureVisibilityEvidence { |
| | | first_visible_bucket: "lte_250ms", |
| | | visibility_source: "current_room_lookup", |
| | | visibility_result: "held_stale_current_visible", |
| | | binding_matched: true, |
| | | }) |
| | | ); |
| | | let mut observer_starts = 0; |
| | | assert!(!start_observer_after_controlled_fixture_probe( |
| | | true, |
| | | Some(expired_ack.observed), |
| | | || observer_starts += 1, |
| | | )); |
| | | assert_eq!(observer_starts, 0); |
| | | |
| | | for current_view in [ |
| | | None, |
| | | Some(("cross-call-user".to_string(), current_attributes.clone())), |
| | | Some(( |
| | | "user-1".to_string(), |
| | | HashMap::from([ |
| | | ( |
| | | "inputSourceCategory".to_string(), |
| | | "controlled_fixture".to_string(), |
| | | ), |
| | | ( |
| | | "clientFixtureSequence".to_string(), |
| | | "fixture-old".to_string(), |
| | | ), |
| | | ]), |
| | | )), |
| | | ] { |
| | | assert_eq!( |
| | | observe_controlled_fixture_post_expiry_views( |
| | | Instant::now(), |
| | | Instant::now() + Duration::from_millis(20), |
| | | "user-1", |
| | | Some("user-1"), |
| | | "fixture-01", |
| | | Arc::new(AtomicBool::new(true)), |
| | | HashMap::new, |
| | | || current_view.clone(), |
| | | ) |
| | | .await, |
| | | None |
| | | ); |
| | | } |
| | | } |
| | | |
| | | #[test] |
| | | fn controlled_fixture_ack_payload_is_reliable_and_redacted() { |
| | | let ack = ControlledFixtureAttributeAck { |
| | |
| | | |
| | | #[test] |
| | | fn production_event_order_probe_then_track_publishes_ack_before_observer() { |
| | | let attributes = HashMap::from([ |
| | | ( |
| | | "inputSourceCategory".to_string(), |
| | | "controlled_fixture".to_string(), |
| | | ), |
| | | ( |
| | | "clientFixtureSequence".to_string(), |
| | | "fixture-01".to_string(), |
| | | ), |
| | | ]); |
| | | let effects = drive_pre_audio_order_test_seam(&[ |
| | | PreAudioOrderEvent::DataReceived { |
| | | sender: "user-1".to_string(), |
| | | sequence: "fixture-01".to_string(), |
| | | sequence: "1".to_string(), |
| | | }, |
| | | PreAudioOrderEvent::TrackSubscribed { |
| | | participant: "user-1".to_string(), |
| | | attributes, |
| | | }, |
| | | ]); |
| | | assert_eq!(effects, ["ack_observed", "observer_started"]); |
| | |
| | | |
| | | #[test] |
| | | fn production_event_order_negative_probe_has_no_observer_or_session_effect() { |
| | | let mut invalid = HashMap::new(); |
| | | invalid.insert( |
| | | "inputSourceCategory".to_string(), |
| | | "ordinary_mic".to_string(), |
| | | ); |
| | | let effects = drive_pre_audio_order_test_seam(&[ |
| | | PreAudioOrderEvent::DataReceived { |
| | | sender: "user-1".to_string(), |
| | | sequence: "fixture-01".to_string(), |
| | | sequence: "1".to_string(), |
| | | }, |
| | | PreAudioOrderEvent::TrackSubscribed { |
| | | participant: "user-1".to_string(), |
| | | attributes: invalid, |
| | | participant: "cross-call-user".to_string(), |
| | | }, |
| | | ]); |
| | | assert!(effects.is_empty()); |