From da9563428e990244ff2145eb6a9cb453f9ef9d22 Mon Sep 17 00:00:00 2001
From: cai <cai@nbcai.cc>
Date: Wed, 12 Aug 2026 12:02:47 +0800
Subject: [PATCH] Fail closed when fixture ACK publish fails
---
src/main.rs | 412 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++-
1 files changed, 401 insertions(+), 11 deletions(-)
diff --git a/src/main.rs b/src/main.rs
index cd45c4d..d40ca9e 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -6,6 +6,7 @@
borrow::Cow,
collections::HashSet,
env, fs,
+ future::Future,
path::{Path, PathBuf},
sync::{
Arc,
@@ -66,6 +67,21 @@
const CONTROLLED_FIXTURE_PROBE_RECHECK_DELAY: Duration = Duration::from_millis(25);
const CONTROLLED_FIXTURE_PROBE_TTL: Duration = Duration::from_millis(250);
+const CONTROLLED_FIXTURE_ACK_RESULTS: [&str; 4] =
+ ["observed", "rejected", "timeout", "publish_failed"];
+const CONTROLLED_FIXTURE_REJECT_REASONS: [&str; 10] = [
+ "missing_attributes",
+ "wrong_source",
+ "missing_sequence",
+ "wrong_sequence",
+ "wrong_participant",
+ "expired",
+ "duplicate_or_old_sequence",
+ "no_current_participant",
+ "ack_publish_failed",
+ "unknown",
+];
+
#[derive(Debug, Deserialize)]
#[serde(rename_all = "camelCase")]
struct ControlledFixtureAttributeProbe {
@@ -110,6 +126,129 @@
.iter()
.map(|byte| format!("{byte:02x}"))
.collect()
+}
+
+fn controlled_fixture_ack_classification(
+ decision: Result<(), &'static str>,
+) -> (&'static str, Option<&'static str>, bool) {
+ match decision {
+ Ok(()) => ("observed", None, true),
+ Err("timeout") => ("timeout", Some("expired"), false),
+ Err(reason) if CONTROLLED_FIXTURE_REJECT_REASONS.contains(&reason) => {
+ ("rejected", Some(reason), false)
+ }
+ Err(_) => ("rejected", Some("unknown"), false),
+ }
+}
+
+fn record_controlled_fixture_probe_event(
+ stage: &'static str,
+ call_id: &str,
+ trace_id: &str,
+ sequence: &str,
+ observed: bool,
+ ack_result: Option<&'static str>,
+ reject_reason: Option<&'static str>,
+) {
+ debug_assert!(ack_result.is_none_or(|value| CONTROLLED_FIXTURE_ACK_RESULTS.contains(&value)));
+ debug_assert!(
+ reject_reason.is_none_or(|value| CONTROLLED_FIXTURE_REJECT_REASONS.contains(&value))
+ );
+ if let Some(ack_result) = ack_result {
+ info!(
+ event = "controlled_fixture_attribute_probe",
+ stage,
+ observed,
+ ack_result,
+ reject_reason = reject_reason.unwrap_or("unknown"),
+ call_id_hash = %sha256_hex(call_id),
+ trace_id_hash = %sha256_hex(trace_id),
+ generation = CONTROLLED_FIXTURE_GENERATION,
+ sequence_hash = %sha256_hex(sequence),
+ "runtime helper controlled fixture probe state"
+ );
+ } else if let Some(reject_reason) = reject_reason {
+ info!(
+ event = "controlled_fixture_attribute_probe",
+ stage,
+ observed,
+ reject_reason,
+ call_id_hash = %sha256_hex(call_id),
+ trace_id_hash = %sha256_hex(trace_id),
+ generation = CONTROLLED_FIXTURE_GENERATION,
+ sequence_hash = %sha256_hex(sequence),
+ "runtime helper controlled fixture probe state"
+ );
+ } else {
+ info!(
+ event = "controlled_fixture_attribute_probe",
+ stage,
+ observed,
+ call_id_hash = %sha256_hex(call_id),
+ trace_id_hash = %sha256_hex(trace_id),
+ generation = CONTROLLED_FIXTURE_GENERATION,
+ sequence_hash = %sha256_hex(sequence),
+ "runtime helper controlled fixture probe state"
+ );
+ }
+}
+
+fn record_controlled_fixture_attribute_decision(
+ decision: Result<(), &'static str>,
+ call_id: &str,
+ trace_id: &str,
+ sequence: &str,
+) -> (&'static str, Option<&'static str>, bool) {
+ let classification = controlled_fixture_ack_classification(decision);
+ record_controlled_fixture_probe_event(
+ "attributes_classified",
+ call_id,
+ trace_id,
+ sequence,
+ classification.2,
+ Some(classification.0),
+ classification.1,
+ );
+ classification
+}
+
+async fn complete_controlled_fixture_ack_publish<F, E>(
+ publish: F,
+ observed: bool,
+ ack_result: &'static str,
+ reject_reason: Option<&'static str>,
+ call_id: &str,
+ trace_id: &str,
+ sequence: &str,
+ acknowledged_probe_sequences: &mut HashSet<String>,
+) -> bool
+where
+ F: Future<Output = Result<(), E>>,
+{
+ if publish.await.is_ok() {
+ record_controlled_fixture_probe_event(
+ "ack_publish_completed",
+ call_id,
+ trace_id,
+ sequence,
+ observed,
+ Some(ack_result),
+ reject_reason,
+ );
+ acknowledged_probe_sequences.insert(sequence.to_string());
+ observed
+ } else {
+ record_controlled_fixture_probe_event(
+ "ack_publish_completed",
+ call_id,
+ trace_id,
+ sequence,
+ false,
+ Some("publish_failed"),
+ Some("ack_publish_failed"),
+ );
+ false
+ }
}
fn controlled_fixture_probe(
@@ -999,6 +1138,26 @@
);
},
);
+ if let Some(sequence) = pending_sequence.as_deref() {
+ let (ack_result, reject_reason, observed) = match probe_result {
+ Some(true) => (Some("observed"), None, true),
+ Some(false) => (Some("rejected"), Some("unknown"), false),
+ None => (None, Some("no_current_participant"), false),
+ };
+ record_controlled_fixture_probe_event(
+ if observer_started {
+ "audio_observer_allowed"
+ } else {
+ "audio_observer_blocked"
+ },
+ &call_id,
+ &trace_id,
+ sequence,
+ observed,
+ ack_result,
+ reject_reason,
+ );
+ }
if !observer_started {
warn!(
call_id = %call_id,
@@ -1019,6 +1178,15 @@
serde_json::from_slice::<ControlledFixtureAttributeProbe>(&payload)
{
if acknowledged_probe_sequences.contains(&probe.client_fixture_sequence) {
+ record_controlled_fixture_probe_event(
+ "data_received",
+ &call_id,
+ &trace_id,
+ &probe.client_fixture_sequence,
+ false,
+ Some("rejected"),
+ Some("duplicate_or_old_sequence"),
+ );
continue;
}
}
@@ -1029,6 +1197,17 @@
&sender.identity(),
user_participant_identity.as_deref(),
);
+ if let Some(probe) = pending_probe.as_ref() {
+ record_controlled_fixture_probe_event(
+ "data_received",
+ &call_id,
+ &trace_id,
+ &probe.sequence,
+ false,
+ None,
+ None,
+ );
+ }
process_controlled_fixture_probe(
&mut pending_probe,
&mut acknowledged_probe_sequences,
@@ -1093,14 +1272,53 @@
let Some(probe) = pending_probe.take() else {
return None;
};
- if acknowledged_probe_sequences.contains(&probe.sequence) || Instant::now() > probe.expires_at {
+ if acknowledged_probe_sequences.contains(&probe.sequence) {
+ record_controlled_fixture_probe_event(
+ "attributes_classified",
+ call_id,
+ trace_id,
+ &probe.sequence,
+ false,
+ Some("rejected"),
+ Some("duplicate_or_old_sequence"),
+ );
+ return Some(false);
+ }
+ if Instant::now() > probe.expires_at {
+ record_controlled_fixture_probe_event(
+ "attributes_classified",
+ call_id,
+ trace_id,
+ &probe.sequence,
+ false,
+ Some("timeout"),
+ Some("expired"),
+ );
return Some(false);
}
let Some(participant) = participant else {
+ record_controlled_fixture_probe_event(
+ "attributes_classified",
+ call_id,
+ trace_id,
+ &probe.sequence,
+ false,
+ None,
+ Some("no_current_participant"),
+ );
*pending_probe = Some(probe);
return None;
};
if participant.identity() != probe.sender {
+ record_controlled_fixture_probe_event(
+ "attributes_classified",
+ call_id,
+ trace_id,
+ &probe.sequence,
+ false,
+ Some("rejected"),
+ Some("wrong_participant"),
+ );
return Some(false);
}
let decision = observe_controlled_fixture_attributes(
@@ -1111,10 +1329,11 @@
|| participant.attributes(),
)
.await;
- let (result, input_source_category, reject_reason) = match decision {
- Ok(()) => ("observed", Some("controlled_fixture"), None),
- Err(reason) => ("rejected", None, Some(reason)),
- };
+ let protocol_reject_reason = decision.err();
+ let (ack_result, reject_reason, observed) =
+ record_controlled_fixture_attribute_decision(decision, call_id, trace_id, &probe.sequence);
+ let result = if observed { "observed" } else { "rejected" };
+ let input_source_category = observed.then_some("controlled_fixture");
let ack = ControlledFixtureAttributeAck {
message_type: CONTROLLED_FIXTURE_ACK_TOPIC,
protocol_version: CONTROLLED_FIXTURE_PROTOCOL_VERSION,
@@ -1124,23 +1343,41 @@
client_fixture_sequence: probe.sequence.clone(),
result,
input_source_category,
- reject_reason,
+ reject_reason: protocol_reject_reason,
};
let payload = match serde_json::to_vec(&ack) {
Ok(payload) => payload,
Err(_) => return Some(false),
};
let local_participant = sink.room.local_participant();
+ record_controlled_fixture_probe_event(
+ "ack_publish_started",
+ call_id,
+ trace_id,
+ &probe.sequence,
+ observed,
+ Some(ack_result),
+ reject_reason,
+ );
let publish = local_participant.publish_data(DataPacket {
payload,
topic: Some(CONTROLLED_FIXTURE_ACK_TOPIC.to_string()),
reliable: true,
destination_identities: vec![probe.sender],
});
- if publish.await.is_ok() {
- acknowledged_probe_sequences.insert(probe.sequence);
- }
- Some(result == "observed")
+ Some(
+ complete_controlled_fixture_ack_publish(
+ publish,
+ observed,
+ ack_result,
+ reject_reason,
+ call_id,
+ trace_id,
+ &probe.sequence,
+ acknowledged_probe_sequences,
+ )
+ .await,
+ )
}
async fn handle_finished_turn(
@@ -4392,7 +4629,7 @@
io::{Read, Write},
net::TcpListener,
sync::{
- Arc,
+ Arc, Mutex,
atomic::{AtomicUsize, Ordering},
mpsc,
},
@@ -4410,6 +4647,30 @@
participant: String,
attributes: HashMap<String, String>,
},
+ }
+
+ #[derive(Clone, Default)]
+ struct CapturedLogs(Arc<Mutex<Vec<u8>>>);
+
+ struct CapturedLogWriter(Arc<Mutex<Vec<u8>>>);
+
+ impl Write for CapturedLogWriter {
+ fn write(&mut self, bytes: &[u8]) -> std::io::Result<usize> {
+ self.0.lock().unwrap().extend_from_slice(bytes);
+ Ok(bytes.len())
+ }
+
+ fn flush(&mut self) -> std::io::Result<()> {
+ Ok(())
+ }
+ }
+
+ impl<'a> tracing_subscriber::fmt::MakeWriter<'a> for CapturedLogs {
+ type Writer = CapturedLogWriter;
+
+ fn make_writer(&'a self) -> Self::Writer {
+ CapturedLogWriter(self.0.clone())
+ }
}
fn drive_pre_audio_order_test_seam(events: &[PreAudioOrderEvent]) -> Vec<&'static str> {
@@ -5063,6 +5324,135 @@
);
}
+ #[test]
+ fn controlled_fixture_probe_failure_log_is_fixed_redacted_and_has_no_audio_effect() {
+ let logs = CapturedLogs::default();
+ let subscriber = tracing_subscriber::fmt()
+ .without_time()
+ .with_ansi(false)
+ .with_writer(logs.clone())
+ .finish();
+ let call_id = "private-call-value";
+ let trace_id = "private-trace-value";
+ let sequence = "private-sequence-value";
+ let mut observer_starts = 0;
+ tracing::subscriber::with_default(subscriber, || {
+ let decision = Err("wrong_source");
+ let (ack_result, reject_reason, observed) =
+ record_controlled_fixture_attribute_decision(decision, call_id, trace_id, sequence);
+ record_controlled_fixture_probe_event(
+ "audio_observer_blocked",
+ call_id,
+ trace_id,
+ sequence,
+ observed,
+ Some(ack_result),
+ reject_reason,
+ );
+ assert!(!start_observer_after_controlled_fixture_probe(
+ true,
+ Some(observed),
+ || observer_starts += 1,
+ ));
+ });
+ let output = String::from_utf8(logs.0.lock().unwrap().clone()).unwrap();
+ assert!(output.contains("ack_result=\"rejected\""));
+ assert!(output.contains("reject_reason=\"wrong_source\""));
+ assert!(output.contains("observed=false"));
+ assert!(output.contains(&sha256_hex(call_id)));
+ assert!(output.contains(&sha256_hex(trace_id)));
+ assert!(output.contains(&sha256_hex(sequence)));
+ assert!(!output.contains(call_id));
+ assert!(!output.contains(trace_id));
+ assert!(!output.contains(sequence));
+ assert_eq!(observer_starts, 0);
+ }
+
+ #[test]
+ fn controlled_fixture_probe_observed_and_failure_enums_are_stable() {
+ assert_eq!(
+ controlled_fixture_ack_classification(Ok(())),
+ ("observed", None, true)
+ );
+ assert_eq!(
+ controlled_fixture_ack_classification(Err("timeout")),
+ ("timeout", Some("expired"), false)
+ );
+ for reason in [
+ "missing_attributes",
+ "wrong_source",
+ "missing_sequence",
+ "wrong_sequence",
+ "wrong_participant",
+ ] {
+ assert_eq!(
+ controlled_fixture_ack_classification(Err(reason)),
+ ("rejected", Some(reason), false)
+ );
+ }
+ assert_eq!(
+ controlled_fixture_ack_classification(Err("unclassified")),
+ ("rejected", Some("unknown"), false)
+ );
+ }
+
+ #[tokio::test]
+ async fn production_ack_publish_failure_keeps_observer_session_and_audio_closed() {
+ let mut acknowledged = HashSet::new();
+ let probe_result = complete_controlled_fixture_ack_publish(
+ async { Err::<(), ()>(()) },
+ true,
+ "observed",
+ None,
+ "call-publish-failure",
+ "trace-publish-failure",
+ "fixture-01",
+ &mut acknowledged,
+ )
+ .await;
+ let mut observer_starts = 0;
+ let mut session_starts = 0;
+ let mut audio_starts = 0;
+ let mut speaking_starts = 0;
+ assert!(!start_observer_after_controlled_fixture_probe(
+ true,
+ Some(probe_result),
+ || {
+ observer_starts += 1;
+ session_starts += 1;
+ audio_starts += 1;
+ speaking_starts += 1;
+ },
+ ));
+ assert!(!probe_result);
+ assert!(acknowledged.is_empty());
+ assert_eq!(observer_starts, 0);
+ assert_eq!(session_starts, 0);
+ assert_eq!(audio_starts, 0);
+ assert_eq!(speaking_starts, 0);
+
+ let observed_result = complete_controlled_fixture_ack_publish(
+ async { Ok::<(), ()>(()) },
+ true,
+ "observed",
+ None,
+ "call-publish-success",
+ "trace-publish-success",
+ "fixture-02",
+ &mut acknowledged,
+ )
+ .await;
+ let mut successful_observer_starts = 0;
+ assert!(start_observer_after_controlled_fixture_probe(
+ true,
+ Some(observed_result),
+ || successful_observer_starts += 1,
+ ));
+ assert!(observed_result);
+ assert!(acknowledged.contains("fixture-02"));
+ assert_eq!(successful_observer_starts, 1);
+ }
+
#[tokio::test]
async fn production_attribute_observation_accepts_server_visibility_within_probe_ttl() {
let expected = HashMap::from([
--
Gitblit v1.9.3