backend/lm-livekit-helper.git

			@@ -1,7 +1,9 @@
			mod asr_realtime;
			mod audio;
			mod service;

			use std::{
			borrow::Cow,
			env, fs,
			path::{Path, PathBuf},
			sync::{
			@@ -12,6 +14,7 @@
			};

			use anyhow::{Context, Result, anyhow};
			use asr_realtime::{RealtimeAsrConfig, RealtimeAsrOutcome, RealtimeAsrUpload};
			use audio::{AudioDiagnostics, load_pre_recorded_frames};
			use base64::{Engine as _, engine::general_purpose};
			use futures_util::StreamExt;
			@@ -30,12 +33,18 @@
			use reqwest::Client;
			use serde::{Deserialize, Serialize};
			use serde_json::json;
			use tokio::time::{sleep, sleep_until};
			use tokio::{sync::mpsc::UnboundedReceiver, task::JoinHandle};
			use tokio::time::{sleep, sleep_until, timeout};
			use tokio::{
			sync::{mpsc, mpsc::UnboundedReceiver, watch},
			task::JoinHandle,
			};
			use tracing::{info, warn};

			const USER_AUDIO_SAMPLE_RATE_HZ: u32 = 48_000;
			const USER_AUDIO_NUM_CHANNELS: u16 = 1;
			const INBOUND_AUDIO_QUEUE_CAPACITY: usize = 100;
			const INBOUND_AUDIO_DROP_LOG_INTERVAL: u64 = 100;
			const AUDIO_DRAIN_STOP_GRACE: Duration = Duration::from_secs(2);
			const DEFAULT_BOT_AUDIO_PROFILE: &str = "pcm-16k";
			const LIVEKIT_48K_SAMPLE_RATE_HZ: u32 = 48_000;
			const PCM_16K_SAMPLE_RATE_HZ: u32 = 16_000;
			@@ -272,6 +281,9 @@
			runtime_turn_bridge_mode: String,
			runtime_asr_stream_enabled: bool,
			runtime_asr_stream_url: Option<String>,
			runtime_asr_realtime_enabled: bool,
			runtime_asr_realtime_url: Option<String>,
			runtime_asr_realtime_chunk_duration_ms: u64,
			runtime_turn_artifact_dir: Option<String>,
			runtime_session_nonce: Option<String>,
			user_audio_observer_enabled: bool,
			@@ -336,6 +348,9 @@
			bridge_mode: String,
			asr_stream_enabled: bool,
			asr_stream_url: Option<String>,
			asr_realtime_enabled: bool,
			asr_realtime_url: Option<String>,
			asr_realtime_chunk_duration_ms: u64,
			artifact_dir: Option<String>,
			runtime_session_nonce: Option<String>,
			audio_debug_dump_dir: Option<String>,
			@@ -349,6 +364,9 @@
			bridge_mode: config.runtime_turn_bridge_mode.clone(),
			asr_stream_enabled: config.runtime_asr_stream_enabled,
			asr_stream_url: config.runtime_asr_stream_url.clone(),
			asr_realtime_enabled: config.runtime_asr_realtime_enabled,
			asr_realtime_url: config.runtime_asr_realtime_url.clone(),
			asr_realtime_chunk_duration_ms: config.runtime_asr_realtime_chunk_duration_ms,
			artifact_dir: config.runtime_turn_artifact_dir.clone(),
			runtime_session_nonce: config.runtime_session_nonce.clone(),
			audio_debug_dump_dir: config.audio_debug_dump_dir.clone(),
			@@ -396,6 +414,16 @@
			.as_ref()
			.is_some_and(\|value\| !value.is_empty())
			}

			fn realtime_asr_config(&self) -> RealtimeAsrConfig {
			RealtimeAsrConfig {
			enabled: self.asr_realtime_enabled,
			url: self.asr_realtime_url.clone(),
			runtime_token: self.bridge_token.clone(),
			runtime_session_nonce: self.runtime_session_nonce.clone(),
			chunk_duration_ms: self.asr_realtime_chunk_duration_ms,
			}
			}
			}

			impl Config {
			@@ -419,6 +447,12 @@
			.unwrap_or_else(\|_\| "json".to_string()),
			runtime_asr_stream_enabled: bool_env("CV_RUNTIME_ASR_STREAM_ENABLED", false),
			runtime_asr_stream_url: optional_env("CV_RUNTIME_ASR_STREAM_URL"),
			runtime_asr_realtime_enabled: bool_env("CV_RUNTIME_ASR_REALTIME_ENABLED", false),
			runtime_asr_realtime_url: optional_env("CV_RUNTIME_ASR_REALTIME_URL"),
			runtime_asr_realtime_chunk_duration_ms: u64_env(
			"CV_RUNTIME_ASR_REALTIME_CHUNK_DURATION_MS",
			200,
			),
			runtime_turn_artifact_dir: optional_env("CV_RUNTIME_TURN_ARTIFACT_DIR"),
			runtime_session_nonce: optional_env("CV_RUNTIME_SESSION_NONCE"),
			user_audio_observer_enabled: bool_env("CV_ENABLE_USER_AUDIO_OBSERVER", true),
			@@ -775,6 +809,7 @@
			call_id: &str,
			trace_id: &str,
			turn: FinishedSpeechTurn,
			realtime_asr_result_ref: Option<String>,
			) {
			let turn_pipeline_started_at = Instant::now();
			if !bridge_config.is_ready() {
			@@ -834,8 +869,10 @@
			"endReason": turn.end_reason.as_str(),
			}),
			);
			let asr_result_ref =
			request_asr_result_ref(http, bridge_config, call_id, trace_id, &turn).await;
			let asr_result_ref = match realtime_asr_result_ref {
			Some(value) => Some(value),
			None => request_asr_result_ref(http, bridge_config, call_id, trace_id, &turn).await,
			};
			if bridge_config.is_stream_mode() {
			match request_turn_bridge_stream(
			http,
			@@ -2439,6 +2476,68 @@
			i32::from(USER_AUDIO_NUM_CHANNELS),
			);
			let started_at = Instant::now();
			let (frame_tx, mut frame_rx) =
			mpsc::channel::<DrainedUserAudioFrame>(INBOUND_AUDIO_QUEUE_CAPACITY);
			let (drain_shutdown_tx, mut drain_shutdown_rx) = watch::channel(false);
			let drain_call_id = call_id.clone();
			let drain_trace_id = trace_id.clone();
			let drain_task = tokio::spawn(async move {
			let mut received_frame_count: u64 = 0;
			let mut dropped_frame_count: u64 = 0;
			loop {
			tokio::select! {
			changed = drain_shutdown_rx.changed() => {
			match changed {
			Ok(()) if *drain_shutdown_rx.borrow() => break,
			Ok(()) => {}
			Err(_) => break,
			}
			}
			maybe_frame = stream.next() => {
			let Some(frame) = maybe_frame else {
			break;
			};
			received_frame_count = received_frame_count.saturating_add(1);
			let drained = DrainedUserAudioFrame {
			frame_index: received_frame_count,
			captured_elapsed_ms: started_at.elapsed().as_millis() as u64,
			frame: AudioFrame {
			data: Cow::Owned(frame.data.as_ref().to_vec()),
			sample_rate: frame.sample_rate,
			num_channels: frame.num_channels,
			samples_per_channel: frame.samples_per_channel,
			},
			};
			match frame_tx.try_send(drained) {
			Ok(()) => {}
			Err(mpsc::error::TrySendError::Full(_)) => {
			dropped_frame_count = dropped_frame_count.saturating_add(1);
			if dropped_frame_count % INBOUND_AUDIO_DROP_LOG_INTERVAL == 1 {
			warn!(
			call_id = %drain_call_id,
			trace_id = %drain_trace_id,
			dropped_frame_count,
			received_frame_count,
			queue_capacity = INBOUND_AUDIO_QUEUE_CAPACITY,
			"runtime helper inbound_audio_queue_full_dropping_newest"
			);
			}
			}
			Err(mpsc::error::TrySendError::Closed(_)) => break,
			}
			}
			}
			}
			stream.close();
			info!(
			call_id = %drain_call_id,
			trace_id = %drain_trace_id,
			received_frame_count,
			dropped_frame_count,
			queue_capacity = INBOUND_AUDIO_QUEUE_CAPACITY,
			"runtime helper user_audio_drain_ended"
			);
			});
			let mut frame_count: u64 = 0;
			let mut sample_count: u64 = 0;
			let mut simple_vad = if simple_vad_enabled {
			@@ -2446,11 +2545,13 @@
			} else {
			None
			};
			let mut realtime_asr_upload: Option<RealtimeAsrUpload> = None;

			while let Some(frame) = stream.next().await {
			frame_count += 1;
			while let Some(drained) = frame_rx.recv().await {
			let frame = drained.frame;
			frame_count = drained.frame_index;
			sample_count += u64::from(frame.samples_per_channel) * u64::from(frame.num_channels);
			let elapsed_ms = started_at.elapsed().as_millis() as u64;
			let elapsed_ms = drained.captured_elapsed_ms;
			if frame_count == 1 {
			info!(
			call_id = %call_id,
			@@ -2478,7 +2579,8 @@

			if let Some(vad) = simple_vad.as_mut() {
			if vad_enabled_gate.load(Ordering::Acquire) {
			if let Some(turn) = vad.observe_frame(
			let was_in_speech = vad.in_speech;
			let turn = vad.observe_frame(
			&call_id,
			&trace_id,
			&participant_alias,
			@@ -2486,7 +2588,65 @@
			frame_count,
			elapsed_ms,
			&frame,
			) {
			);
			let is_in_speech = vad.in_speech;

			if !was_in_speech && is_in_speech {
			let turn_id = format!("turn-{:04}", vad.turn_index);
			match RealtimeAsrUpload::start(
			http.clone(),
			turn_bridge_config.realtime_asr_config(),
			&call_id,
			&trace_id,
			&turn_id,
			&vad.speech_samples,
			) {
			Ok(upload) => {
			info!(
			call_id = %call_id,
			trace_id = %trace_id,
			turn_id = %turn_id,
			"runtime helper asr_realtime_session_started"
			);
			realtime_asr_upload = Some(upload);
			}
			Err(error) if turn_bridge_config.asr_realtime_enabled => {
			warn!(
			call_id = %call_id,
			trace_id = %trace_id,
			turn_id = %turn_id,
			error = %safe_error(&error.to_string()),
			"runtime helper asr_realtime_start_failed_fallback"
			);
			}
			Err(_) => {}
			}
			} else if was_in_speech {
			let push_failed = realtime_asr_upload
			.as_mut()
			.and_then(\|upload\| upload.push_48k_samples(frame.data.as_ref()).err());
			if let Some(error) = push_failed {
			warn!(
			call_id = %call_id,
			trace_id = %trace_id,
			error = %safe_error(&error.to_string()),
			"runtime helper asr_realtime_upload_failed_fallback"
			);
			if let Some(upload) = realtime_asr_upload.take() {
			tokio::spawn(async move {
			upload.cancel("upload_backpressure").await;
			});
			}
			}
			}

			if let Some(turn) = turn {
			let realtime_asr_result_ref = match realtime_asr_upload.take() {
			Some(upload) => {
			finish_realtime_asr_upload(upload, &call_id, &trace_id, &turn).await
			}
			None => None,
			};
			handle_finished_turn(
			&http,
			&turn_bridge_config,
			@@ -2494,10 +2654,22 @@
			&call_id,
			&trace_id,
			turn,
			realtime_asr_result_ref,
			)
			.await;
			} else if was_in_speech && !is_in_speech {
			if let Some(upload) = realtime_asr_upload.take() {
			tokio::spawn(async move {
			upload.cancel("speech_too_short").await;
			});
			}
			}
			} else {
			if let Some(upload) = realtime_asr_upload.take() {
			tokio::spawn(async move {
			upload.cancel("vad_disabled").await;
			});
			}
			vad.observe_disabled_frame(
			&call_id,
			&trace_id,
			@@ -2518,9 +2690,41 @@
			&track_sid_alias,
			started_at.elapsed().as_millis() as u64,
			) {
			handle_finished_turn(&http, &turn_bridge_config, &sink, &call_id, &trace_id, turn)
			.await;
			let realtime_asr_result_ref = match realtime_asr_upload.take() {
			Some(upload) => {
			finish_realtime_asr_upload(upload, &call_id, &trace_id, &turn).await
			}
			None => None,
			};
			handle_finished_turn(
			&http,
			&turn_bridge_config,
			&sink,
			&call_id,
			&trace_id,
			turn,
			realtime_asr_result_ref,
			)
			.await;
			}
			}
			if let Some(upload) = realtime_asr_upload.take() {
			upload.cancel("stream_end").await;
			}
			let _ = drain_shutdown_tx.send(true);
			let mut drain_task = drain_task;
			if timeout(AUDIO_DRAIN_STOP_GRACE, &mut drain_task)
			.await
			.is_err()
			{
			drain_task.abort();
			let _ = drain_task.await;
			warn!(
			call_id = %call_id,
			trace_id = %trace_id,
			stop_grace_ms = AUDIO_DRAIN_STOP_GRACE.as_millis() as u64,
			"runtime helper user_audio_drain_stop_timeout"
			);
			}

			info!(
			@@ -2536,6 +2740,64 @@
			})
			}

			struct DrainedUserAudioFrame {
			frame_index: u64,
			captured_elapsed_ms: u64,
			frame: AudioFrame<'static>,
			}

			async fn finish_realtime_asr_upload(
			upload: RealtimeAsrUpload,
			call_id: &str,
			trace_id: &str,
			turn: &FinishedSpeechTurn,
			) -> Option<String> {
			match upload.finish(turn.duration_ms, &turn.end_reason).await {
			Ok(RealtimeAsrOutcome {
			status,
			asr_result_ref,
			provider_alias,
			partial_count,
			fallback_reason,
			fallback_stage,
			chunk_count,
			audio_bytes,
			wall_ms,
			}) => {
			info!(
			call_id = %call_id,
			trace_id = %trace_id,
			turn_id = %turn.turn_id,
			status = %status,
			provider_alias = ?provider_alias,
			partial_count,
			chunk_count,
			audio_bytes,
			wall_ms,
			asr_result_ref_present = asr_result_ref.is_some(),
			fallback_reason = ?fallback_reason,
			fallback_stage = ?fallback_stage,
			"runtime helper asr_realtime_finished"
			);
			if status == "final" {
			asr_result_ref
			} else {
			None
			}
			}
			Err(error) => {
			warn!(
			call_id = %call_id,
			trace_id = %trace_id,
			turn_id = %turn.turn_id,
			error = %safe_error(&error.to_string()),
			"runtime helper asr_realtime_failed_fallback"
			);
			None
			}
			}
			}

			#[derive(Clone)]
			struct SimpleVadConfig {
			rms_threshold: f64,