From bf538f10cc411d978bc30863085cd98c5ca18b14 Mon Sep 17 00:00:00 2001
From: cai <cai@nbcai.cc>
Date: Wed, 12 Aug 2026 14:56:28 +0800
Subject: [PATCH] test(helper): observe post-expiry attribute visibility
---
src/main.rs | 378 ++++++++++++++++++++++++++++++++++++++++++++++++++---
src/service.rs | 31 ++++
2 files changed, 384 insertions(+), 25 deletions(-)
diff --git a/src/main.rs b/src/main.rs
index dc3ee70..8803bba 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -66,6 +66,7 @@
const CONTROLLED_FIXTURE_GENERATION: u64 = 1;
const CONTROLLED_FIXTURE_PROBE_RECHECK_DELAY: Duration = Duration::from_millis(25);
const CONTROLLED_FIXTURE_PROBE_TTL: Duration = Duration::from_millis(250);
+const CONTROLLED_FIXTURE_POST_EXPIRY_WINDOW: Duration = Duration::from_millis(2_000);
const CONTROLLED_FIXTURE_ACK_RESULTS: [&str; 4] =
["observed", "rejected", "timeout", "publish_failed"];
@@ -118,7 +119,21 @@
call_trace_id_hash: String,
generation: u64,
sequence: String,
+ received_at: Instant,
expires_at: Instant,
+}
+
+#[derive(Debug, PartialEq, Eq)]
+struct ControlledFixtureVisibilityEvidence {
+ first_visible_bucket: &'static str,
+ visibility_source: &'static str,
+ binding_matched: bool,
+}
+
+#[derive(Debug, PartialEq, Eq)]
+struct ControlledFixtureAckPublishOutcome {
+ observed: bool,
+ published: bool,
}
fn sha256_hex(value: &str) -> String {
@@ -249,7 +264,7 @@
generation: u64,
sequence: &str,
acknowledged_probe_sequences: &mut HashSet<String>,
-) -> bool
+) -> ControlledFixtureAckPublishOutcome
where
F: Future<Output = Result<(), E>>,
{
@@ -267,7 +282,10 @@
reject_reason,
);
acknowledged_probe_sequences.insert(sequence.to_string());
- observed
+ ControlledFixtureAckPublishOutcome {
+ observed,
+ published: true,
+ }
} else {
record_controlled_fixture_probe_event(
runtime_call_id,
@@ -281,7 +299,10 @@
Some("publish_failed"),
Some("ack_publish_failed"),
);
- false
+ ControlledFixtureAckPublishOutcome {
+ observed: false,
+ published: false,
+ }
}
}
@@ -323,14 +344,141 @@
{
return None;
}
+ let received_at = Instant::now();
Some(PendingControlledFixtureProbe {
sender: sender.clone(),
call_id_hash: probe.call_id_hash,
call_trace_id_hash: probe.call_trace_id_hash,
generation: probe.generation,
sequence: probe.client_fixture_sequence,
- expires_at: Instant::now() + CONTROLLED_FIXTURE_PROBE_TTL,
+ received_at,
+ expires_at: received_at + CONTROLLED_FIXTURE_PROBE_TTL,
})
+}
+
+fn controlled_fixture_visibility_bucket(elapsed: Duration) -> &'static str {
+ if elapsed <= Duration::from_millis(250) {
+ "lte_250ms"
+ } else if elapsed <= Duration::from_millis(500) {
+ "250_500ms"
+ } else if elapsed <= Duration::from_millis(1_000) {
+ "500_1000ms"
+ } else {
+ "1000_2000ms"
+ }
+}
+
+async fn observe_controlled_fixture_post_expiry<F>(
+ received_at: Instant,
+ observation_deadline: Instant,
+ actual_participant: &str,
+ expected_participant: Option<&str>,
+ requested_sequence: &str,
+ lifecycle_active: Arc<AtomicBool>,
+ mut read_attributes: F,
+) -> Option<ControlledFixtureVisibilityEvidence>
+where
+ F: FnMut() -> std::collections::HashMap<String, String>,
+{
+ loop {
+ if !lifecycle_active.load(Ordering::Acquire) {
+ return None;
+ }
+ let now = Instant::now();
+ let decision = classify_controlled_fixture_attributes(
+ actual_participant,
+ expected_participant,
+ &read_attributes(),
+ requested_sequence,
+ );
+ match decision {
+ Ok(()) => {
+ return Some(ControlledFixtureVisibilityEvidence {
+ first_visible_bucket: controlled_fixture_visibility_bucket(
+ now.saturating_duration_since(received_at),
+ ),
+ visibility_source: "participant_attributes_poll",
+ binding_matched: true,
+ });
+ }
+ Err("missing_attributes") if now < observation_deadline => {}
+ Err("missing_attributes") => {
+ return Some(ControlledFixtureVisibilityEvidence {
+ first_visible_bucket: "never_visible_within_observation_window",
+ visibility_source: "participant_attributes_poll",
+ binding_matched: true,
+ });
+ }
+ Err(_) => return None,
+ }
+ sleep(CONTROLLED_FIXTURE_PROBE_RECHECK_DELAY).await;
+ }
+}
+
+fn controlled_fixture_visibility_event(
+ runtime_call_id: &str,
+ runtime_trace_id: &str,
+ probe: &PendingControlledFixtureProbe,
+ evidence: &ControlledFixtureVisibilityEvidence,
+) -> serde_json::Value {
+ json!({
+ "type": "cv_activity",
+ "callId": runtime_call_id,
+ "traceId": runtime_trace_id,
+ "turnId": null,
+ "eventName": "controlled_fixture_attribute_probe",
+ "eventWallTimeMs": current_time_millis(),
+ "result": "ok",
+ "reasonCode": null,
+ "retryable": null,
+ "extension": {
+ "stage": "post_expiry_visibility",
+ "first_visible_bucket": evidence.first_visible_bucket,
+ "visibility_source": evidence.visibility_source,
+ "binding_matched": evidence.binding_matched,
+ "call_id_hash": probe.call_id_hash,
+ "trace_id_hash": probe.call_trace_id_hash,
+ "generation": probe.generation,
+ "sequence_hash": sha256_hex(&probe.sequence),
+ "evidence_count": 1,
+ },
+ })
+}
+
+fn spawn_controlled_fixture_post_expiry_observation(
+ probe: PendingControlledFixtureProbe,
+ participant: RemoteParticipant,
+ expected_participant: Option<String>,
+ lifecycle_active: Arc<AtomicBool>,
+ runtime_call_id: String,
+ runtime_trace_id: String,
+) {
+ tokio::spawn(async move {
+ let participant_identity = participant.identity().to_string();
+ let evidence = observe_controlled_fixture_post_expiry(
+ probe.received_at,
+ probe.received_at + CONTROLLED_FIXTURE_POST_EXPIRY_WINDOW,
+ &participant_identity,
+ expected_participant.as_deref(),
+ &probe.sequence,
+ lifecycle_active.clone(),
+ || participant.attributes(),
+ )
+ .await;
+ if lifecycle_active.load(Ordering::Acquire) {
+ if let Some(evidence) = evidence {
+ println!(
+ "{}",
+ controlled_fixture_visibility_event(
+ &runtime_call_id,
+ &runtime_trace_id,
+ &probe,
+ &evidence,
+ )
+ );
+ }
+ }
+ });
}
fn classify_controlled_fixture_attributes(
@@ -1130,6 +1278,7 @@
let mut current_user_participant: Option<RemoteParticipant> = None;
let mut pending_probe: Option<PendingControlledFixtureProbe> = None;
let mut acknowledged_probe_sequences = HashSet::new();
+ let controlled_fixture_lifecycle_active = Arc::new(AtomicBool::new(true));
while let Some(event) = events.recv().await {
match event {
@@ -1170,6 +1319,7 @@
user_participant_identity.as_deref(),
&call_id,
&trace_id,
+ controlled_fixture_lifecycle_active.clone(),
)
.await;
let observer_started = start_observer_after_controlled_fixture_probe(
@@ -1280,6 +1430,7 @@
user_participant_identity.as_deref(),
&call_id,
&trace_id,
+ controlled_fixture_lifecycle_active.clone(),
)
.await;
}
@@ -1322,6 +1473,7 @@
_ => {}
}
}
+ controlled_fixture_lifecycle_active.store(false, Ordering::Release);
}
async fn process_controlled_fixture_probe(
@@ -1332,6 +1484,7 @@
expected_participant: Option<&str>,
runtime_call_id: &str,
runtime_trace_id: &str,
+ lifecycle_active: Arc<AtomicBool>,
) -> Option<bool> {
let Some(probe) = pending_probe.take() else {
return None;
@@ -1436,24 +1589,33 @@
payload,
topic: Some(CONTROLLED_FIXTURE_ACK_TOPIC.to_string()),
reliable: true,
- destination_identities: vec![probe.sender],
+ destination_identities: vec![probe.sender.clone()],
});
- Some(
- complete_controlled_fixture_ack_publish(
- publish,
- runtime_call_id,
- runtime_trace_id,
- observed,
- ack_result,
- reject_reason,
- &probe.call_id_hash,
- &probe.call_trace_id_hash,
- probe.generation,
- &probe.sequence,
- acknowledged_probe_sequences,
- )
- .await,
+ let outcome = complete_controlled_fixture_ack_publish(
+ publish,
+ runtime_call_id,
+ runtime_trace_id,
+ observed,
+ ack_result,
+ reject_reason,
+ &probe.call_id_hash,
+ &probe.call_trace_id_hash,
+ probe.generation,
+ &probe.sequence,
+ acknowledged_probe_sequences,
)
+ .await;
+ if outcome.published && ack_result == "timeout" && reject_reason == Some("expired") {
+ spawn_controlled_fixture_post_expiry_observation(
+ probe,
+ participant.clone(),
+ expected_participant.map(str::to_string),
+ lifecycle_active,
+ runtime_call_id.to_string(),
+ runtime_trace_id.to_string(),
+ );
+ }
+ Some(outcome.observed)
}
async fn handle_finished_turn(
@@ -5484,6 +5646,7 @@
call_trace_id_hash: request_trace_hash.to_string(),
generation: CONTROLLED_FIXTURE_GENERATION,
sequence: "fixture-01".to_string(),
+ received_at: Instant::now(),
expires_at: Instant::now() + CONTROLLED_FIXTURE_PROBE_TTL,
};
@@ -5542,7 +5705,7 @@
let mut speaking_starts = 0;
assert!(!start_observer_after_controlled_fixture_probe(
true,
- Some(probe_result),
+ Some(probe_result.observed),
|| {
observer_starts += 1;
session_starts += 1;
@@ -5550,7 +5713,13 @@
speaking_starts += 1;
},
));
- assert!(!probe_result);
+ assert_eq!(
+ probe_result,
+ ControlledFixtureAckPublishOutcome {
+ observed: false,
+ published: false,
+ }
+ );
assert!(acknowledged.is_empty());
assert_eq!(observer_starts, 0);
assert_eq!(session_starts, 0);
@@ -5574,10 +5743,16 @@
let mut successful_observer_starts = 0;
assert!(start_observer_after_controlled_fixture_probe(
true,
- Some(observed_result),
+ Some(observed_result.observed),
|| successful_observer_starts += 1,
));
- assert!(observed_result);
+ assert_eq!(
+ observed_result,
+ ControlledFixtureAckPublishOutcome {
+ observed: true,
+ published: true,
+ }
+ );
assert!(acknowledged.contains("fixture-02"));
assert_eq!(successful_observer_starts, 1);
}
@@ -5644,6 +5819,161 @@
assert_eq!(observer_starts, 0);
}
+ #[tokio::test]
+ async fn production_post_expiry_observation_records_bounded_visibility_without_second_ack() {
+ let started_at = Instant::now();
+ let active = Arc::new(AtomicBool::new(true));
+ let expected = HashMap::from([
+ (
+ "inputSourceCategory".to_string(),
+ "controlled_fixture".to_string(),
+ ),
+ (
+ "clientFixtureSequence".to_string(),
+ "fixture-01".to_string(),
+ ),
+ ]);
+ let mut acknowledged = HashSet::new();
+ let expired_ack = complete_controlled_fixture_ack_publish(
+ async { Ok::<(), ()>(()) },
+ "runtime-call-post-expiry",
+ "runtime-trace-post-expiry",
+ false,
+ "timeout",
+ Some("expired"),
+ "call-post-expiry",
+ "trace-post-expiry",
+ CONTROLLED_FIXTURE_GENERATION,
+ "fixture-01",
+ &mut acknowledged,
+ )
+ .await;
+ assert_eq!(
+ expired_ack,
+ ControlledFixtureAckPublishOutcome {
+ observed: false,
+ published: true,
+ }
+ );
+ let evidence = observe_controlled_fixture_post_expiry(
+ started_at,
+ started_at + CONTROLLED_FIXTURE_POST_EXPIRY_WINDOW,
+ "user-1",
+ Some("user-1"),
+ "fixture-01",
+ active,
+ || {
+ if started_at.elapsed() >= Duration::from_millis(300) {
+ expected.clone()
+ } else {
+ HashMap::new()
+ }
+ },
+ )
+ .await;
+ assert_eq!(acknowledged.len(), 1);
+ assert_eq!(
+ evidence,
+ Some(ControlledFixtureVisibilityEvidence {
+ first_visible_bucket: "250_500ms",
+ visibility_source: "participant_attributes_poll",
+ binding_matched: true,
+ })
+ );
+
+ let never_started_at = Instant::now();
+ assert_eq!(
+ observe_controlled_fixture_post_expiry(
+ never_started_at,
+ never_started_at + Duration::from_millis(40),
+ "user-1",
+ Some("user-1"),
+ "fixture-01",
+ Arc::new(AtomicBool::new(true)),
+ HashMap::new,
+ )
+ .await,
+ Some(ControlledFixtureVisibilityEvidence {
+ first_visible_bucket: "never_visible_within_observation_window",
+ visibility_source: "participant_attributes_poll",
+ binding_matched: true,
+ })
+ );
+
+ assert_eq!(
+ observe_controlled_fixture_post_expiry(
+ Instant::now(),
+ Instant::now() + Duration::from_millis(50),
+ "cross-call-user",
+ Some("user-1"),
+ "fixture-01",
+ Arc::new(AtomicBool::new(true)),
+ || expected.clone(),
+ )
+ .await,
+ None
+ );
+
+ let inactive = Arc::new(AtomicBool::new(false));
+ assert_eq!(
+ observe_controlled_fixture_post_expiry(
+ Instant::now(),
+ Instant::now() + Duration::from_millis(50),
+ "user-1",
+ Some("user-1"),
+ "fixture-01",
+ inactive,
+ HashMap::new,
+ )
+ .await,
+ None
+ );
+ assert_eq!(acknowledged.len(), 1);
+
+ let probe = PendingControlledFixtureProbe {
+ sender: ParticipantIdentity("user-1".to_string()),
+ call_id_hash: "call-post-expiry".to_string(),
+ call_trace_id_hash: "trace-post-expiry".to_string(),
+ generation: CONTROLLED_FIXTURE_GENERATION,
+ sequence: "fixture-01".to_string(),
+ received_at: started_at,
+ expires_at: started_at + CONTROLLED_FIXTURE_PROBE_TTL,
+ };
+ let event = controlled_fixture_visibility_event(
+ "runtime-call-post-expiry",
+ "runtime-trace-post-expiry",
+ &probe,
+ &evidence.unwrap(),
+ );
+ let extension = event["extension"].as_object().unwrap();
+ let mut keys = extension.keys().map(String::as_str).collect::<Vec<_>>();
+ keys.sort_unstable();
+ assert_eq!(
+ keys,
+ vec![
+ "binding_matched",
+ "call_id_hash",
+ "evidence_count",
+ "first_visible_bucket",
+ "generation",
+ "sequence_hash",
+ "stage",
+ "trace_id_hash",
+ "visibility_source",
+ ]
+ );
+ let encoded = event.to_string();
+ for forbidden in [
+ "\"participant\":",
+ "\"room\":",
+ "\"track\":",
+ "\"payload\":",
+ "\"audio\":",
+ ] {
+ assert!(!encoded.contains(forbidden));
+ }
+ }
+
#[test]
fn controlled_fixture_ack_payload_is_reliable_and_redacted() {
let ack = ControlledFixtureAttributeAck {
diff --git a/src/service.rs b/src/service.rs
index 65a8622..76e2627 100644
--- a/src/service.rs
+++ b/src/service.rs
@@ -1255,7 +1255,11 @@
EventCallbackDispatch, deliver_event_callback_with, project_worker_stdout_event,
run_event_callback_worker,
};
- use crate::controlled_fixture_probe_event;
+ use crate::{
+ CONTROLLED_FIXTURE_GENERATION, CONTROLLED_FIXTURE_PROBE_TTL,
+ ControlledFixtureVisibilityEvidence, PendingControlledFixtureProbe,
+ controlled_fixture_probe_event, controlled_fixture_visibility_event,
+ };
use anyhow::Result;
use serde_json::{Value, json};
use std::{cell::RefCell, rc::Rc, sync::mpsc};
@@ -1301,6 +1305,31 @@
] {
assert!(project_worker_stdout_event(call_id, &invalid.to_string()).is_none());
}
+
+ let received_at = std::time::Instant::now();
+ let visibility = controlled_fixture_visibility_event(
+ call_id,
+ runtime_trace_id,
+ &PendingControlledFixtureProbe {
+ sender: livekit::prelude::ParticipantIdentity("user-1".to_string()),
+ call_id_hash: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
+ .to_string(),
+ call_trace_id_hash: trace_hash.to_string(),
+ generation: CONTROLLED_FIXTURE_GENERATION,
+ sequence: "fixture-01".to_string(),
+ received_at,
+ expires_at: received_at + CONTROLLED_FIXTURE_PROBE_TTL,
+ },
+ &ControlledFixtureVisibilityEvidence {
+ first_visible_bucket: "250_500ms",
+ visibility_source: "participant_attributes_poll",
+ binding_matched: true,
+ },
+ );
+ let projected = project_worker_stdout_event(call_id, &visibility.to_string())
+ .expect("post-expiry evidence must enter the production stdout projection");
+ assert_eq!(projected["extension"]["first_visible_bucket"], "250_500ms");
+ assert_eq!(projected["extension"]["binding_matched"], true);
}
#[test]
--
Gitblit v1.9.3