From ea8cdf229544207066807a883cfd639b8961288d Mon Sep 17 00:00:00 2001
From: cai <cai@nbcai.cc>
Date: Sat, 08 Aug 2026 19:23:48 +0800
Subject: [PATCH] chore: bind observer identity candidate
---
src/audio.rs | 1386 ++++++++++++++++++++++++++++++++++++++++++++-------------
1 files changed, 1,069 insertions(+), 317 deletions(-)
diff --git a/src/audio.rs b/src/audio.rs
index 01d3a28..cec22f0 100644
--- a/src/audio.rs
+++ b/src/audio.rs
@@ -1,317 +1,1069 @@
-use std::{fs, io::Cursor, path::Path};
-
-use anyhow::{Context, Result, anyhow};
-use hound::{SampleFormat, WavReader};
-use minimp3::{Decoder as Mp3Decoder, Error as Mp3Error, Frame as Mp3Frame};
-use reqwest::Client;
-
-#[derive(Clone, Debug, PartialEq, Eq)]
-pub struct PcmFrame {
- pub data: Vec<i16>,
- pub sample_rate: u32,
- pub num_channels: u32,
- pub samples_per_channel: u32,
-}
-
-impl PcmFrame {
- pub fn new(
- data: Vec<i16>,
- sample_rate: u32,
- num_channels: u32,
- samples_per_channel: u32,
- ) -> Self {
- Self {
- data,
- sample_rate,
- num_channels,
- samples_per_channel,
- }
- }
-}
-
-pub async fn load_pre_recorded_frames(
- http: &Client,
- audio_file: Option<&str>,
- audio_url: Option<&str>,
- target_sample_rate_hz: u32,
- target_num_channels: u16,
-) -> Result<Option<Vec<PcmFrame>>> {
- if let Some(path) = audio_file.filter(|value| !value.trim().is_empty()) {
- let audio_bytes = fs::read(Path::new(path))
- .with_context(|| format!("failed to read greeting audio file {path}"))?;
- return decode_audio_frames(&audio_bytes, target_sample_rate_hz, target_num_channels)
- .map(Some);
- }
- if let Some(url) = audio_url.filter(|value| !value.trim().is_empty()) {
- let response = http
- .get(url)
- .send()
- .await
- .with_context(|| format!("failed to fetch greeting audio {url}"))?;
- if !response.status().is_success() {
- return Err(anyhow!(
- "greeting audio fetch failed for {url} with status {}",
- response.status()
- ));
- }
- let bytes = response
- .bytes()
- .await
- .with_context(|| format!("failed to read greeting audio body {url}"))?;
- return decode_audio_frames(&bytes, target_sample_rate_hz, target_num_channels).map(Some);
- }
- Ok(None)
-}
-
-fn decode_audio_frames(
- audio_bytes: &[u8],
- target_sample_rate_hz: u32,
- target_num_channels: u16,
-) -> Result<Vec<PcmFrame>> {
- if is_wav(audio_bytes) {
- return decode_wav_frames(audio_bytes, target_sample_rate_hz, target_num_channels);
- }
- if is_mp3(audio_bytes) {
- return decode_mp3_frames(audio_bytes, target_sample_rate_hz, target_num_channels);
- }
-
- let wav_result = decode_wav_frames(audio_bytes, target_sample_rate_hz, target_num_channels);
- if wav_result.is_ok() {
- return wav_result;
- }
- let wav_error = wav_result.err();
- decode_mp3_frames(audio_bytes, target_sample_rate_hz, target_num_channels).map_err(
- |mp3_error| {
- anyhow!(
- "failed to decode greeting audio as wav or mp3: wav={}, mp3={}",
- wav_error
- .map(|error| error.to_string())
- .unwrap_or_else(|| "unknown".to_string()),
- mp3_error
- )
- },
- )
-}
-
-fn decode_wav_frames(
- wav_bytes: &[u8],
- target_sample_rate_hz: u32,
- target_num_channels: u16,
-) -> Result<Vec<PcmFrame>> {
- let cursor = Cursor::new(wav_bytes.to_vec());
- let mut reader = WavReader::new(cursor).context("failed to open greeting wav bytes")?;
- let spec = reader.spec();
- let src_channels = spec.channels.max(1);
- let mut samples = match (spec.sample_format, spec.bits_per_sample) {
- (SampleFormat::Int, 16) => reader
- .samples::<i16>()
- .collect::<std::result::Result<Vec<_>, _>>()
- .context("failed to decode 16-bit wav samples")?,
- (SampleFormat::Float, 32) => reader
- .samples::<f32>()
- .map(|sample| sample.map(|value| (value.clamp(-1.0, 1.0) * i16::MAX as f32) as i16))
- .collect::<std::result::Result<Vec<_>, _>>()
- .context("failed to decode float wav samples")?,
- _ => {
- return Err(anyhow!(
- "unsupported greeting wav format: {:?} {}-bit",
- spec.sample_format,
- spec.bits_per_sample
- ));
- }
- };
-
- samples = remap_channels(samples, src_channels, target_num_channels);
- samples = resample_linear(
- samples,
- spec.sample_rate,
- target_sample_rate_hz,
- target_num_channels,
- );
-
- Ok(chunk_pcm_samples(
- samples,
- target_sample_rate_hz,
- target_num_channels,
- ))
-}
-
-fn decode_mp3_frames(
- mp3_bytes: &[u8],
- target_sample_rate_hz: u32,
- target_num_channels: u16,
-) -> Result<Vec<PcmFrame>> {
- let cursor = Cursor::new(mp3_bytes.to_vec());
- let mut decoder = Mp3Decoder::new(cursor);
- let mut samples = Vec::new();
-
- loop {
- match decoder.next_frame() {
- Ok(Mp3Frame {
- data,
- sample_rate,
- channels,
- ..
- }) => {
- let src_rate = u32::try_from(sample_rate)
- .map_err(|_| anyhow!("unsupported mp3 sample rate {sample_rate}"))?;
- let src_channels = u16::try_from(channels)
- .map_err(|_| anyhow!("unsupported mp3 channel count {channels}"))?;
- let mut frame_samples = remap_channels(data, src_channels, target_num_channels);
- frame_samples = resample_linear(
- frame_samples,
- src_rate,
- target_sample_rate_hz,
- target_num_channels,
- );
- samples.extend(frame_samples);
- }
- Err(Mp3Error::Eof) => break,
- Err(Mp3Error::SkippedData) | Err(Mp3Error::InsufficientData) => continue,
- Err(error) => return Err(anyhow!("failed to decode mp3 frame: {error:?}")),
- }
- }
-
- if samples.is_empty() {
- return Err(anyhow!("decoded mp3 contains no audio samples"));
- }
-
- Ok(chunk_pcm_samples(
- samples,
- target_sample_rate_hz,
- target_num_channels,
- ))
-}
-
-fn chunk_pcm_samples(
- samples: Vec<i16>,
- target_sample_rate_hz: u32,
- target_num_channels: u16,
-) -> Vec<PcmFrame> {
- const CHUNK_MS: u32 = 20;
- let samples_per_chunk = ((target_sample_rate_hz / 1000) * CHUNK_MS).max(1) as usize
- * usize::from(target_num_channels);
- let mut frames = Vec::new();
- for chunk in samples.chunks(samples_per_chunk) {
- let mut frame_data = chunk.to_vec();
- if frame_data.len() < samples_per_chunk {
- frame_data.resize(samples_per_chunk, 0);
- }
- let samples_per_channel =
- (frame_data.len() / usize::from(target_num_channels.max(1))) as u32;
- frames.push(PcmFrame::new(
- frame_data,
- target_sample_rate_hz,
- u32::from(target_num_channels),
- samples_per_channel,
- ));
- }
- frames
-}
-
-fn is_wav(bytes: &[u8]) -> bool {
- bytes.len() >= 12 && bytes.starts_with(b"RIFF") && &bytes[8..12] == b"WAVE"
-}
-
-fn is_mp3(bytes: &[u8]) -> bool {
- bytes.starts_with(b"ID3") || (bytes.len() >= 2 && bytes[0] == 0xff && bytes[1] & 0xe0 == 0xe0)
-}
-
-fn remap_channels(samples: Vec<i16>, src_channels: u16, dst_channels: u16) -> Vec<i16> {
- if src_channels == dst_channels {
- return samples;
- }
-
- let src_channels = usize::from(src_channels.max(1));
- let dst_channels = usize::from(dst_channels.max(1));
- let frames = samples.chunks(src_channels);
- let mut output = Vec::new();
- for frame in frames {
- match (src_channels, dst_channels) {
- (1, 2) => {
- let sample = frame.first().copied().unwrap_or_default();
- output.push(sample);
- output.push(sample);
- }
- (2, 1) => {
- let left = frame.first().copied().unwrap_or_default() as i32;
- let right = frame.get(1).copied().unwrap_or_default() as i32;
- output.push(((left + right) / 2) as i16);
- }
- _ => {
- for channel in 0..dst_channels {
- output.push(
- frame
- .get(channel % src_channels)
- .copied()
- .unwrap_or_default(),
- );
- }
- }
- }
- }
- output
-}
-
-fn resample_linear(samples: Vec<i16>, src_rate: u32, dst_rate: u32, channels: u16) -> Vec<i16> {
- if src_rate == dst_rate {
- return samples;
- }
-
- let channels = usize::from(channels.max(1));
- let src_frames = samples.len() / channels;
- if src_frames <= 1 {
- return samples;
- }
- let ratio = dst_rate as f64 / src_rate as f64;
- let dst_frames = ((src_frames as f64) * ratio).round().max(1.0) as usize;
- let mut output = Vec::with_capacity(dst_frames * channels);
-
- for dst_frame in 0..dst_frames {
- let src_pos = (dst_frame as f64) / ratio;
- let src_index = src_pos.floor() as usize;
- let next_index = (src_index + 1).min(src_frames - 1);
- let frac = (src_pos - src_index as f64) as f32;
- for channel in 0..channels {
- let a = samples[src_index * channels + channel] as f32;
- let b = samples[next_index * channels + channel] as f32;
- let mixed = a + (b - a) * frac;
- output.push(mixed.round().clamp(i16::MIN as f32, i16::MAX as f32) as i16);
- }
- }
-
- output
-}
-
-#[cfg(test)]
-mod tests {
- use super::load_pre_recorded_frames;
-
- #[tokio::test]
- async fn load_pre_recorded_frames_reads_local_wav() {
- let dir = std::env::temp_dir().join("cv-runtime-helper-tests");
- std::fs::create_dir_all(&dir).expect("create temp dir");
- let path = dir.join("fixture.wav");
- let spec = hound::WavSpec {
- channels: 1,
- sample_rate: 16_000,
- bits_per_sample: 16,
- sample_format: hound::SampleFormat::Int,
- };
- let mut writer = hound::WavWriter::create(&path, spec).expect("create wav");
- for _ in 0..16_000 {
- writer.write_sample(512_i16).expect("write sample");
- }
- writer.finalize().expect("finalize wav");
-
- let http = reqwest::Client::new();
- let frames = load_pre_recorded_frames(&http, path.to_str(), None, 48_000, 1)
- .await
- .expect("load frames")
- .expect("frames");
-
- assert!(!frames.is_empty());
- assert_eq!(frames[0].sample_rate, 48_000);
- assert_eq!(frames[0].num_channels, 1);
- }
-}
+use std::{
+ fs,
+ io::Cursor,
+ path::{Path, PathBuf},
+};
+
+use anyhow::{Context, Result, anyhow};
+use hound::{SampleFormat, WavReader, WavSpec, WavWriter};
+use minimp3::{Decoder as Mp3Decoder, Error as Mp3Error, Frame as Mp3Frame};
+use reqwest::Client;
+
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub struct PcmFrame {
+ pub data: Vec<i16>,
+ pub sample_rate: u32,
+ pub num_channels: u32,
+ pub samples_per_channel: u32,
+}
+
+impl PcmFrame {
+ pub fn new(
+ data: Vec<i16>,
+ sample_rate: u32,
+ num_channels: u32,
+ samples_per_channel: u32,
+ ) -> Self {
+ Self {
+ data,
+ sample_rate,
+ num_channels,
+ samples_per_channel,
+ }
+ }
+}
+
+#[derive(Clone, Debug)]
+pub struct LoadedAudio {
+ pub frames: Vec<PcmFrame>,
+ pub diagnostics: AudioDiagnostics,
+}
+
+#[derive(Clone, Debug)]
+pub struct AudioDiagnostics {
+ pub source_kind: &'static str,
+ pub source_format: &'static str,
+ pub source_bytes: usize,
+ pub source_sample_rate_hz: u32,
+ pub source_num_channels: u16,
+ pub decoded_sample_count: usize,
+ pub decoded_duration_ms: u64,
+ pub target_sample_rate_hz: u32,
+ pub target_num_channels: u16,
+ pub target_sample_count: usize,
+ pub target_duration_ms: u64,
+ pub frame_count: usize,
+ pub rms: f64,
+ pub peak: f64,
+ pub clipped_sample_count: usize,
+ pub silence_ratio: f64,
+ pub mp3_skipped_data_count: usize,
+ pub mp3_insufficient_data_count: usize,
+ pub debug_source_path: Option<String>,
+ pub debug_pcm_wav_path: Option<String>,
+}
+
+struct DecodeResult {
+ samples: Vec<i16>,
+ diagnostics: AudioDiagnostics,
+}
+
+pub async fn load_pre_recorded_frames(
+ http: &Client,
+ audio_file: Option<&str>,
+ audio_url: Option<&str>,
+ target_sample_rate_hz: u32,
+ target_num_channels: u16,
+ debug_dump_dir: Option<&str>,
+ call_id: &str,
+ debug_label: &str,
+) -> Result<Option<LoadedAudio>> {
+ if let Some(path) = audio_file.filter(|value| !value.trim().is_empty()) {
+ let audio_bytes = fs::read(Path::new(path))
+ .with_context(|| format!("failed to read greeting audio file {path}"))?;
+ return decode_audio_frames(
+ &audio_bytes,
+ "local_file",
+ target_sample_rate_hz,
+ target_num_channels,
+ debug_dump_dir,
+ call_id,
+ debug_label,
+ )
+ .map(Some);
+ }
+ if let Some(url) = audio_url.filter(|value| !value.trim().is_empty()) {
+ let response = http
+ .get(url)
+ .send()
+ .await
+ .with_context(|| format!("failed to fetch greeting audio {url}"))?;
+ if !response.status().is_success() {
+ return Err(anyhow!(
+ "greeting audio fetch failed for {url} with status {}",
+ response.status()
+ ));
+ }
+ let bytes = response
+ .bytes()
+ .await
+ .with_context(|| format!("failed to read greeting audio body {url}"))?;
+ return decode_audio_frames(
+ &bytes,
+ "downloaded_url",
+ target_sample_rate_hz,
+ target_num_channels,
+ debug_dump_dir,
+ call_id,
+ debug_label,
+ )
+ .map(Some);
+ }
+ Ok(None)
+}
+
+pub fn decode_audio_bytes_to_frames(
+ audio_bytes: &[u8],
+ source_kind: &'static str,
+ target_sample_rate_hz: u32,
+ target_num_channels: u16,
+ debug_dump_dir: Option<&str>,
+ call_id: &str,
+ debug_label: &str,
+) -> Result<LoadedAudio> {
+ decode_audio_frames(
+ audio_bytes,
+ source_kind,
+ target_sample_rate_hz,
+ target_num_channels,
+ debug_dump_dir,
+ call_id,
+ debug_label,
+ )
+}
+
+pub fn pcm_s16le_bytes_to_frames(
+ pcm_bytes: &[u8],
+ source_sample_rate_hz: u32,
+ source_num_channels: u32,
+ target_sample_rate_hz: u32,
+ target_num_channels: u16,
+) -> Result<Vec<PcmFrame>> {
+ if source_sample_rate_hz == 0 {
+ return Err(anyhow!("pcm_s16le source sample rate is zero"));
+ }
+ if source_num_channels == 0 {
+ return Err(anyhow!("pcm_s16le source channel count is zero"));
+ }
+ if pcm_bytes.len() % 2 != 0 {
+ return Err(anyhow!("pcm_s16le payload has odd byte length"));
+ }
+ let samples: Vec<i16> = pcm_bytes
+ .chunks_exact(2)
+ .map(|chunk| i16::from_le_bytes([chunk[0], chunk[1]]))
+ .collect();
+ if samples.is_empty() {
+ return Ok(Vec::new());
+ }
+ let source_channels = u16::try_from(source_num_channels)
+ .map_err(|_| anyhow!("unsupported pcm_s16le channel count {source_num_channels}"))?;
+ let mut target_samples = remap_channels(samples, source_channels, target_num_channels);
+ target_samples = resample_linear(
+ target_samples,
+ source_sample_rate_hz,
+ target_sample_rate_hz,
+ target_num_channels,
+ );
+ Ok(chunk_pcm_samples(
+ target_samples,
+ target_sample_rate_hz,
+ target_num_channels,
+ ))
+}
+
+#[derive(Debug, Clone, Default)]
+pub struct PcmS16leStreamChunkResult {
+ pub frames: Vec<PcmFrame>,
+ pub dropped_tail_bytes: usize,
+ pub buffered_source_bytes: usize,
+ pub buffered_source_samples: usize,
+}
+
+#[derive(Debug, Clone, Default)]
+pub struct PcmS16leStreamDebugDump {
+ pub debug_source_path: Option<String>,
+ pub debug_pcm_wav_path: Option<String>,
+ pub debug_pcm_wav_size_bytes: Option<u64>,
+}
+
+#[derive(Debug, Clone)]
+pub struct PcmS16leStreamDecoder {
+ source_sample_rate_hz: u32,
+ source_num_channels: u16,
+ target_sample_rate_hz: u32,
+ target_num_channels: u16,
+ source_byte_buffer: Vec<u8>,
+ source_sample_buffer: Vec<i16>,
+ debug_source_samples: Vec<i16>,
+ debug_target_samples: Vec<i16>,
+}
+
+impl PcmS16leStreamDecoder {
+ pub fn new(
+ source_sample_rate_hz: u32,
+ source_num_channels: u32,
+ target_sample_rate_hz: u32,
+ target_num_channels: u16,
+ ) -> Result<Self> {
+ if source_sample_rate_hz == 0 {
+ return Err(anyhow!("pcm_s16le source sample rate is zero"));
+ }
+ if source_num_channels == 0 {
+ return Err(anyhow!("pcm_s16le source channel count is zero"));
+ }
+ let source_num_channels = u16::try_from(source_num_channels)
+ .map_err(|_| anyhow!("unsupported pcm_s16le channel count {source_num_channels}"))?;
+ Ok(Self {
+ source_sample_rate_hz,
+ source_num_channels,
+ target_sample_rate_hz,
+ target_num_channels,
+ source_byte_buffer: Vec::new(),
+ source_sample_buffer: Vec::new(),
+ debug_source_samples: Vec::new(),
+ debug_target_samples: Vec::new(),
+ })
+ }
+
+ pub fn push_bytes(
+ &mut self,
+ pcm_bytes: &[u8],
+ source_sample_rate_hz: u32,
+ source_num_channels: u32,
+ last: bool,
+ ) -> Result<PcmS16leStreamChunkResult> {
+ let source_num_channels = u16::try_from(source_num_channels)
+ .map_err(|_| anyhow!("unsupported pcm_s16le channel count {source_num_channels}"))?;
+ if source_sample_rate_hz != self.source_sample_rate_hz
+ || source_num_channels != self.source_num_channels
+ {
+ return Err(anyhow!(
+ "pcm_s16le stream format changed from {}Hz/{}ch to {}Hz/{}ch",
+ self.source_sample_rate_hz,
+ self.source_num_channels,
+ source_sample_rate_hz,
+ source_num_channels
+ ));
+ }
+
+ self.source_byte_buffer.extend_from_slice(pcm_bytes);
+ let source_frame_alignment_bytes = usize::from(self.source_num_channels) * 2;
+ let aligned_len = self.source_byte_buffer.len()
+ - self.source_byte_buffer.len() % source_frame_alignment_bytes;
+ if aligned_len > 0 {
+ let aligned_bytes: Vec<u8> = self.source_byte_buffer.drain(..aligned_len).collect();
+ let source_samples: Vec<i16> = aligned_bytes
+ .chunks_exact(2)
+ .map(|chunk| i16::from_le_bytes([chunk[0], chunk[1]]))
+ .collect();
+ self.debug_source_samples.extend_from_slice(&source_samples);
+ let remapped = remap_channels(
+ source_samples,
+ self.source_num_channels,
+ self.target_num_channels,
+ );
+ self.source_sample_buffer.extend(remapped);
+ }
+
+ let dropped_tail_bytes = if last && !self.source_byte_buffer.is_empty() {
+ let dropped = self.source_byte_buffer.len();
+ self.source_byte_buffer.clear();
+ dropped
+ } else {
+ 0
+ };
+
+ let mut frames = self.drain_full_frames();
+ if last {
+ if let Some(frame) = self.drain_final_partial_frame() {
+ frames.push(frame);
+ }
+ }
+
+ Ok(PcmS16leStreamChunkResult {
+ frames,
+ dropped_tail_bytes,
+ buffered_source_bytes: self.source_byte_buffer.len(),
+ buffered_source_samples: self.source_sample_buffer.len(),
+ })
+ }
+
+ pub fn write_debug_dump(
+ &self,
+ dir: &str,
+ call_id: &str,
+ debug_label: &str,
+ ) -> Result<PcmS16leStreamDebugDump> {
+ let base_dir = PathBuf::from(dir);
+ fs::create_dir_all(&base_dir)
+ .with_context(|| format!("failed to create audio debug dump dir {dir}"))?;
+ let safe_call_id = sanitize_file_segment(call_id);
+ let safe_label = sanitize_file_segment(debug_label);
+
+ let source_path = base_dir.join(format!("{safe_call_id}-{safe_label}-source.wav"));
+ write_debug_wav(
+ &source_path,
+ &self.debug_source_samples,
+ self.source_sample_rate_hz,
+ self.source_num_channels,
+ )?;
+
+ let pcm_path = base_dir.join(format!("{safe_call_id}-{safe_label}-target.wav"));
+ write_debug_wav(
+ &pcm_path,
+ &self.debug_target_samples,
+ self.target_sample_rate_hz,
+ self.target_num_channels,
+ )?;
+ let pcm_size = fs::metadata(&pcm_path)
+ .with_context(|| format!("failed to stat audio debug wav {}", pcm_path.display()))?
+ .len();
+
+ Ok(PcmS16leStreamDebugDump {
+ debug_source_path: Some(source_path.to_string_lossy().to_string()),
+ debug_pcm_wav_path: Some(pcm_path.to_string_lossy().to_string()),
+ debug_pcm_wav_size_bytes: Some(pcm_size),
+ })
+ }
+
+ fn drain_full_frames(&mut self) -> Vec<PcmFrame> {
+ let source_samples_per_frame = self.source_samples_per_20ms_frame();
+ let mut frames = Vec::new();
+ while self.source_sample_buffer.len() >= source_samples_per_frame {
+ let source_samples: Vec<i16> = self
+ .source_sample_buffer
+ .drain(..source_samples_per_frame)
+ .collect();
+ frames.push(self.convert_source_samples_to_frame(source_samples, true));
+ }
+ frames
+ }
+
+ fn drain_final_partial_frame(&mut self) -> Option<PcmFrame> {
+ if self.source_sample_buffer.is_empty() {
+ return None;
+ }
+ let source_samples: Vec<i16> = self.source_sample_buffer.drain(..).collect();
+ Some(self.convert_source_samples_to_frame(source_samples, true))
+ }
+
+ fn convert_source_samples_to_frame(
+ &mut self,
+ source_samples: Vec<i16>,
+ pad_to_frame: bool,
+ ) -> PcmFrame {
+ let mut target_samples = resample_linear(
+ source_samples,
+ self.source_sample_rate_hz,
+ self.target_sample_rate_hz,
+ self.target_num_channels,
+ );
+ let target_frame_samples = self.target_samples_per_20ms_frame();
+ if target_samples.len() > target_frame_samples {
+ target_samples.truncate(target_frame_samples);
+ } else if pad_to_frame && target_samples.len() < target_frame_samples {
+ target_samples.resize(target_frame_samples, 0);
+ }
+ self.debug_target_samples.extend_from_slice(&target_samples);
+ PcmFrame::new(
+ target_samples,
+ self.target_sample_rate_hz,
+ u32::from(self.target_num_channels),
+ self.target_samples_per_channel_per_20ms_frame() as u32,
+ )
+ }
+
+ fn source_samples_per_20ms_frame(&self) -> usize {
+ let source_frames =
+ ((f64::from(self.source_sample_rate_hz) / 50.0).round() as usize).max(1);
+ source_frames * usize::from(self.target_num_channels)
+ }
+
+ fn target_samples_per_20ms_frame(&self) -> usize {
+ self.target_samples_per_channel_per_20ms_frame() * usize::from(self.target_num_channels)
+ }
+
+ fn target_samples_per_channel_per_20ms_frame(&self) -> usize {
+ ((self.target_sample_rate_hz / 1000) * 20).max(1) as usize
+ }
+}
+
+fn decode_audio_frames(
+ audio_bytes: &[u8],
+ source_kind: &'static str,
+ target_sample_rate_hz: u32,
+ target_num_channels: u16,
+ debug_dump_dir: Option<&str>,
+ call_id: &str,
+ debug_label: &str,
+) -> Result<LoadedAudio> {
+ if is_wav(audio_bytes) {
+ return finalize_decode_result(
+ audio_bytes,
+ decode_wav_samples(
+ audio_bytes,
+ source_kind,
+ target_sample_rate_hz,
+ target_num_channels,
+ )?,
+ target_sample_rate_hz,
+ target_num_channels,
+ debug_dump_dir,
+ call_id,
+ debug_label,
+ );
+ }
+ if is_mp3(audio_bytes) {
+ return finalize_decode_result(
+ audio_bytes,
+ decode_mp3_samples(
+ audio_bytes,
+ source_kind,
+ target_sample_rate_hz,
+ target_num_channels,
+ )?,
+ target_sample_rate_hz,
+ target_num_channels,
+ debug_dump_dir,
+ call_id,
+ debug_label,
+ );
+ }
+
+ let wav_result = decode_wav_samples(
+ audio_bytes,
+ source_kind,
+ target_sample_rate_hz,
+ target_num_channels,
+ );
+ if wav_result.is_ok() {
+ return finalize_decode_result(
+ audio_bytes,
+ wav_result?,
+ target_sample_rate_hz,
+ target_num_channels,
+ debug_dump_dir,
+ call_id,
+ debug_label,
+ );
+ }
+ let wav_error = wav_result.err();
+ let mp3_result = decode_mp3_samples(
+ audio_bytes,
+ source_kind,
+ target_sample_rate_hz,
+ target_num_channels,
+ );
+ match mp3_result {
+ Ok(result) => finalize_decode_result(
+ audio_bytes,
+ result,
+ target_sample_rate_hz,
+ target_num_channels,
+ debug_dump_dir,
+ call_id,
+ debug_label,
+ ),
+ Err(mp3_error) => Err(anyhow!(
+ "failed to decode greeting audio as wav or mp3: wav={}, mp3={}",
+ wav_error
+ .map(|error| error.to_string())
+ .unwrap_or_else(|| "unknown".to_string()),
+ mp3_error
+ )),
+ }
+}
+
+fn decode_wav_samples(
+ wav_bytes: &[u8],
+ source_kind: &'static str,
+ target_sample_rate_hz: u32,
+ target_num_channels: u16,
+) -> Result<DecodeResult> {
+ let cursor = Cursor::new(wav_bytes.to_vec());
+ let mut reader = WavReader::new(cursor).context("failed to open greeting wav bytes")?;
+ let spec = reader.spec();
+ let src_channels = spec.channels.max(1);
+ let decoded_samples = match (spec.sample_format, spec.bits_per_sample) {
+ (SampleFormat::Int, 16) => reader
+ .samples::<i16>()
+ .collect::<std::result::Result<Vec<_>, _>>()
+ .context("failed to decode 16-bit wav samples")?,
+ (SampleFormat::Float, 32) => reader
+ .samples::<f32>()
+ .map(|sample| sample.map(|value| (value.clamp(-1.0, 1.0) * i16::MAX as f32) as i16))
+ .collect::<std::result::Result<Vec<_>, _>>()
+ .context("failed to decode float wav samples")?,
+ _ => {
+ return Err(anyhow!(
+ "unsupported greeting wav format: {:?} {}-bit",
+ spec.sample_format,
+ spec.bits_per_sample
+ ));
+ }
+ };
+
+ let decoded_sample_count = decoded_samples.len();
+ let mut samples = remap_channels(decoded_samples, src_channels, target_num_channels);
+ samples = resample_linear(
+ samples,
+ spec.sample_rate,
+ target_sample_rate_hz,
+ target_num_channels,
+ );
+
+ Ok(DecodeResult {
+ diagnostics: build_diagnostics(
+ source_kind,
+ "wav",
+ wav_bytes.len(),
+ spec.sample_rate,
+ src_channels,
+ decoded_sample_count,
+ &samples,
+ target_sample_rate_hz,
+ target_num_channels,
+ 0,
+ 0,
+ ),
+ samples,
+ })
+}
+
+fn decode_mp3_samples(
+ mp3_bytes: &[u8],
+ source_kind: &'static str,
+ target_sample_rate_hz: u32,
+ target_num_channels: u16,
+) -> Result<DecodeResult> {
+ let cursor = Cursor::new(mp3_bytes.to_vec());
+ let mut decoder = Mp3Decoder::new(cursor);
+ let mut samples = Vec::new();
+ let mut decoded_sample_count = 0usize;
+ let mut source_sample_rate_hz = 0u32;
+ let mut source_num_channels = 0u16;
+ let mut skipped_data_count = 0usize;
+ let mut insufficient_data_count = 0usize;
+
+ loop {
+ match decoder.next_frame() {
+ Ok(Mp3Frame {
+ data,
+ sample_rate,
+ channels,
+ ..
+ }) => {
+ let src_rate = u32::try_from(sample_rate)
+ .map_err(|_| anyhow!("unsupported mp3 sample rate {sample_rate}"))?;
+ let src_channels = u16::try_from(channels)
+ .map_err(|_| anyhow!("unsupported mp3 channel count {channels}"))?;
+ if source_sample_rate_hz == 0 {
+ source_sample_rate_hz = src_rate;
+ }
+ if source_num_channels == 0 {
+ source_num_channels = src_channels;
+ }
+ decoded_sample_count += data.len();
+ let mut frame_samples = remap_channels(data, src_channels, target_num_channels);
+ frame_samples = resample_linear(
+ frame_samples,
+ src_rate,
+ target_sample_rate_hz,
+ target_num_channels,
+ );
+ samples.extend(frame_samples);
+ }
+ Err(Mp3Error::Eof) => break,
+ Err(Mp3Error::SkippedData) => {
+ skipped_data_count += 1;
+ continue;
+ }
+ Err(Mp3Error::InsufficientData) => {
+ insufficient_data_count += 1;
+ continue;
+ }
+ Err(error) => return Err(anyhow!("failed to decode mp3 frame: {error:?}")),
+ }
+ }
+
+ if samples.is_empty() {
+ return Err(anyhow!("decoded mp3 contains no audio samples"));
+ }
+
+ Ok(DecodeResult {
+ diagnostics: build_diagnostics(
+ source_kind,
+ "mp3",
+ mp3_bytes.len(),
+ source_sample_rate_hz,
+ source_num_channels.max(1),
+ decoded_sample_count,
+ &samples,
+ target_sample_rate_hz,
+ target_num_channels,
+ skipped_data_count,
+ insufficient_data_count,
+ ),
+ samples,
+ })
+}
+
+fn finalize_decode_result(
+ audio_bytes: &[u8],
+ mut decode_result: DecodeResult,
+ target_sample_rate_hz: u32,
+ target_num_channels: u16,
+ debug_dump_dir: Option<&str>,
+ call_id: &str,
+ debug_label: &str,
+) -> Result<LoadedAudio> {
+ if let Some(dir) = debug_dump_dir.filter(|value| !value.trim().is_empty()) {
+ let paths = dump_debug_audio(
+ dir,
+ call_id,
+ debug_label,
+ decode_result.diagnostics.source_format,
+ audio_bytes,
+ &decode_result.samples,
+ target_sample_rate_hz,
+ target_num_channels,
+ )?;
+ decode_result.diagnostics.debug_source_path = paths.0;
+ decode_result.diagnostics.debug_pcm_wav_path = paths.1;
+ }
+
+ let frames = chunk_pcm_samples(
+ decode_result.samples,
+ target_sample_rate_hz,
+ target_num_channels,
+ );
+ decode_result.diagnostics.frame_count = frames.len();
+
+ Ok(LoadedAudio {
+ frames,
+ diagnostics: decode_result.diagnostics,
+ })
+}
+
+fn build_diagnostics(
+ source_kind: &'static str,
+ source_format: &'static str,
+ source_bytes: usize,
+ source_sample_rate_hz: u32,
+ source_num_channels: u16,
+ decoded_sample_count: usize,
+ target_samples: &[i16],
+ target_sample_rate_hz: u32,
+ target_num_channels: u16,
+ mp3_skipped_data_count: usize,
+ mp3_insufficient_data_count: usize,
+) -> AudioDiagnostics {
+ let target_sample_count = target_samples.len();
+ let decoded_duration_ms = duration_ms(
+ decoded_sample_count,
+ source_sample_rate_hz,
+ source_num_channels,
+ );
+ let target_duration_ms = duration_ms(
+ target_sample_count,
+ target_sample_rate_hz,
+ target_num_channels,
+ );
+ let (rms, peak, clipped_sample_count, silence_ratio) = quality_stats(target_samples);
+ AudioDiagnostics {
+ source_kind,
+ source_format,
+ source_bytes,
+ source_sample_rate_hz,
+ source_num_channels,
+ decoded_sample_count,
+ decoded_duration_ms,
+ target_sample_rate_hz,
+ target_num_channels,
+ target_sample_count,
+ target_duration_ms,
+ frame_count: 0,
+ rms,
+ peak,
+ clipped_sample_count,
+ silence_ratio,
+ mp3_skipped_data_count,
+ mp3_insufficient_data_count,
+ debug_source_path: None,
+ debug_pcm_wav_path: None,
+ }
+}
+
+fn duration_ms(sample_count: usize, sample_rate_hz: u32, channels: u16) -> u64 {
+ if sample_count == 0 || sample_rate_hz == 0 || channels == 0 {
+ return 0;
+ }
+ let frames = sample_count as f64 / f64::from(channels.max(1));
+ ((frames / f64::from(sample_rate_hz)) * 1000.0).round() as u64
+}
+
+fn quality_stats(samples: &[i16]) -> (f64, f64, usize, f64) {
+ if samples.is_empty() {
+ return (0.0, 0.0, 0, 0.0);
+ }
+ const SILENCE_THRESHOLD: i32 = 256;
+ let mut sum_squares = 0f64;
+ let mut peak = 0i32;
+ let mut clipped = 0usize;
+ let mut silence = 0usize;
+ for &sample in samples {
+ let abs = i32::from(sample).abs();
+ peak = peak.max(abs);
+ if abs >= i32::from(i16::MAX) {
+ clipped += 1;
+ }
+ if abs <= SILENCE_THRESHOLD {
+ silence += 1;
+ }
+ let normalized = f64::from(sample) / f64::from(i16::MAX);
+ sum_squares += normalized * normalized;
+ }
+ let rms = (sum_squares / samples.len() as f64).sqrt();
+ let peak = f64::from(peak) / f64::from(i16::MAX);
+ let silence_ratio = silence as f64 / samples.len() as f64;
+ (round4(rms), round4(peak), clipped, round4(silence_ratio))
+}
+
+fn round4(value: f64) -> f64 {
+ (value * 10_000.0).round() / 10_000.0
+}
+
+fn dump_debug_audio(
+ dir: &str,
+ call_id: &str,
+ debug_label: &str,
+ source_format: &str,
+ source_bytes: &[u8],
+ target_samples: &[i16],
+ target_sample_rate_hz: u32,
+ target_num_channels: u16,
+) -> Result<(Option<String>, Option<String>)> {
+ let base_dir = PathBuf::from(dir);
+ fs::create_dir_all(&base_dir)
+ .with_context(|| format!("failed to create audio debug dump dir {dir}"))?;
+ let safe_call_id = sanitize_file_segment(call_id);
+ let safe_label = sanitize_file_segment(debug_label);
+ let source_path = base_dir.join(format!(
+ "{safe_call_id}-{safe_label}-source.{source_format}"
+ ));
+ fs::write(&source_path, source_bytes).with_context(|| {
+ format!(
+ "failed to write audio debug source {}",
+ source_path.display()
+ )
+ })?;
+
+ let pcm_path = base_dir.join(format!("{safe_call_id}-{safe_label}-target.wav"));
+ write_debug_wav(
+ &pcm_path,
+ target_samples,
+ target_sample_rate_hz,
+ target_num_channels,
+ )?;
+
+ Ok((
+ Some(source_path.to_string_lossy().to_string()),
+ Some(pcm_path.to_string_lossy().to_string()),
+ ))
+}
+
+fn write_debug_wav(path: &Path, samples: &[i16], sample_rate_hz: u32, channels: u16) -> Result<()> {
+ let spec = WavSpec {
+ channels,
+ sample_rate: sample_rate_hz,
+ bits_per_sample: 16,
+ sample_format: SampleFormat::Int,
+ };
+ let mut writer = WavWriter::create(path, spec)
+ .with_context(|| format!("failed to create audio debug wav {}", path.display()))?;
+ for &sample in samples {
+ writer
+ .write_sample(sample)
+ .with_context(|| format!("failed to write audio debug wav {}", path.display()))?;
+ }
+ writer
+ .finalize()
+ .with_context(|| format!("failed to finalize audio debug wav {}", path.display()))
+}
+
+pub fn write_pcm_wav(
+ path: &Path,
+ samples: &[i16],
+ sample_rate_hz: u32,
+ channels: u16,
+) -> Result<()> {
+ write_debug_wav(path, samples, sample_rate_hz, channels)
+}
+
+fn sanitize_file_segment(value: &str) -> String {
+ let sanitized: String = value
+ .chars()
+ .map(|ch| {
+ if ch.is_ascii_alphanumeric() || ch == '-' || ch == '_' || ch == '.' {
+ ch
+ } else {
+ '_'
+ }
+ })
+ .collect();
+ if sanitized.is_empty() {
+ "unknown".to_string()
+ } else {
+ sanitized
+ }
+}
+
+fn chunk_pcm_samples(
+ samples: Vec<i16>,
+ target_sample_rate_hz: u32,
+ target_num_channels: u16,
+) -> Vec<PcmFrame> {
+ const CHUNK_MS: u32 = 20;
+ let samples_per_chunk = ((target_sample_rate_hz / 1000) * CHUNK_MS).max(1) as usize
+ * usize::from(target_num_channels);
+ let mut frames = Vec::new();
+ for chunk in samples.chunks(samples_per_chunk) {
+ let mut frame_data = chunk.to_vec();
+ if frame_data.len() < samples_per_chunk {
+ frame_data.resize(samples_per_chunk, 0);
+ }
+ let samples_per_channel =
+ (frame_data.len() / usize::from(target_num_channels.max(1))) as u32;
+ frames.push(PcmFrame::new(
+ frame_data,
+ target_sample_rate_hz,
+ u32::from(target_num_channels),
+ samples_per_channel,
+ ));
+ }
+ frames
+}
+
+fn is_wav(bytes: &[u8]) -> bool {
+ bytes.len() >= 12 && bytes.starts_with(b"RIFF") && &bytes[8..12] == b"WAVE"
+}
+
+fn is_mp3(bytes: &[u8]) -> bool {
+ bytes.starts_with(b"ID3") || (bytes.len() >= 2 && bytes[0] == 0xff && bytes[1] & 0xe0 == 0xe0)
+}
+
+fn remap_channels(samples: Vec<i16>, src_channels: u16, dst_channels: u16) -> Vec<i16> {
+ if src_channels == dst_channels {
+ return samples;
+ }
+
+ let src_channels = usize::from(src_channels.max(1));
+ let dst_channels = usize::from(dst_channels.max(1));
+ let frames = samples.chunks(src_channels);
+ let mut output = Vec::new();
+ for frame in frames {
+ match (src_channels, dst_channels) {
+ (1, 2) => {
+ let sample = frame.first().copied().unwrap_or_default();
+ output.push(sample);
+ output.push(sample);
+ }
+ (2, 1) => {
+ let left = frame.first().copied().unwrap_or_default() as i32;
+ let right = frame.get(1).copied().unwrap_or_default() as i32;
+ output.push(((left + right) / 2) as i16);
+ }
+ _ => {
+ for channel in 0..dst_channels {
+ output.push(
+ frame
+ .get(channel % src_channels)
+ .copied()
+ .unwrap_or_default(),
+ );
+ }
+ }
+ }
+ }
+ output
+}
+
+fn resample_linear(samples: Vec<i16>, src_rate: u32, dst_rate: u32, channels: u16) -> Vec<i16> {
+ if src_rate == dst_rate {
+ return samples;
+ }
+
+ let channels = usize::from(channels.max(1));
+ let src_frames = samples.len() / channels;
+ if src_frames <= 1 {
+ return samples;
+ }
+ let ratio = dst_rate as f64 / src_rate as f64;
+ let dst_frames = ((src_frames as f64) * ratio).round().max(1.0) as usize;
+ let mut output = Vec::with_capacity(dst_frames * channels);
+
+ for dst_frame in 0..dst_frames {
+ let src_pos = (dst_frame as f64) / ratio;
+ let src_index = src_pos.floor() as usize;
+ let next_index = (src_index + 1).min(src_frames - 1);
+ let frac = (src_pos - src_index as f64) as f32;
+ for channel in 0..channels {
+ let a = samples[src_index * channels + channel] as f32;
+ let b = samples[next_index * channels + channel] as f32;
+ let mixed = a + (b - a) * frac;
+ output.push(mixed.round().clamp(i16::MIN as f32, i16::MAX as f32) as i16);
+ }
+ }
+
+ output
+}
+
+#[cfg(test)]
+mod tests {
+ use super::{PcmS16leStreamDecoder, load_pre_recorded_frames, pcm_s16le_bytes_to_frames};
+
+ #[tokio::test]
+ async fn load_pre_recorded_frames_reads_local_wav() {
+ let dir = std::env::temp_dir().join("cv-runtime-helper-tests");
+ std::fs::create_dir_all(&dir).expect("create temp dir");
+ let path = dir.join("fixture.wav");
+ let spec = hound::WavSpec {
+ channels: 1,
+ sample_rate: 16_000,
+ bits_per_sample: 16,
+ sample_format: hound::SampleFormat::Int,
+ };
+ let mut writer = hound::WavWriter::create(&path, spec).expect("create wav");
+ for _ in 0..16_000 {
+ writer.write_sample(512_i16).expect("write sample");
+ }
+ writer.finalize().expect("finalize wav");
+
+ let http = reqwest::Client::new();
+ let loaded = load_pre_recorded_frames(
+ &http,
+ path.to_str(),
+ None,
+ 48_000,
+ 1,
+ Some(dir.to_string_lossy().as_ref()),
+ "test-call",
+ "greeting",
+ )
+ .await
+ .expect("load frames")
+ .expect("frames");
+
+ assert!(!loaded.frames.is_empty());
+ assert_eq!(loaded.frames[0].sample_rate, 48_000);
+ assert_eq!(loaded.frames[0].num_channels, 1);
+ assert_eq!(loaded.diagnostics.source_format, "wav");
+ assert_eq!(loaded.diagnostics.source_sample_rate_hz, 16_000);
+ assert_eq!(loaded.diagnostics.target_sample_rate_hz, 48_000);
+ assert!(loaded.diagnostics.debug_source_path.is_some());
+ assert!(loaded.diagnostics.debug_pcm_wav_path.is_some());
+ }
+
+ #[test]
+ fn pcm_s16le_bytes_to_frames_resamples_elevenlabs_pcm_to_livekit_frames() {
+ let mut pcm = Vec::new();
+ for index in 0..4410 {
+ let sample = if index % 2 == 0 { 1024_i16 } else { -1024_i16 };
+ pcm.extend_from_slice(&sample.to_le_bytes());
+ }
+
+ let frames = pcm_s16le_bytes_to_frames(&pcm, 44_100, 1, 48_000, 1).expect("pcm frames");
+
+ assert!(!frames.is_empty());
+ assert_eq!(frames[0].sample_rate, 48_000);
+ assert_eq!(frames[0].num_channels, 1);
+ assert_eq!(frames[0].samples_per_channel, 960);
+ }
+
+ #[test]
+ fn pcm_s16le_stream_decoder_does_not_pad_each_network_chunk() {
+ let mut decoder = PcmS16leStreamDecoder::new(44_100, 1, 48_000, 1).expect("stream decoder");
+ let mut pcm = Vec::new();
+ for index in 0..4410 {
+ let sample = if index % 2 == 0 { 1024_i16 } else { -1024_i16 };
+ pcm.extend_from_slice(&sample.to_le_bytes());
+ }
+
+ let mut frames = Vec::new();
+ let chunk_size = 698 * 2;
+ for (index, chunk) in pcm.chunks(chunk_size).enumerate() {
+ let last = (index + 1) * chunk_size >= pcm.len();
+ let result = decoder
+ .push_bytes(chunk, 44_100, 1, last)
+ .expect("push pcm chunk");
+ frames.extend(result.frames);
+ }
+
+ assert_eq!(5, frames.len());
+ assert_eq!(
+ 4_800,
+ frames.iter().map(|frame| frame.data.len()).sum::<usize>()
+ );
+ assert!(frames.iter().all(|frame| frame.sample_rate == 48_000));
+ assert!(frames.iter().all(|frame| frame.num_channels == 1));
+ assert!(frames.iter().all(|frame| frame.samples_per_channel == 960));
+ }
+
+ #[test]
+ fn pcm_s16le_stream_decoder_waits_until_full_20ms_frame() {
+ let mut decoder = PcmS16leStreamDecoder::new(44_100, 1, 48_000, 1).expect("stream decoder");
+ let ten_ms = vec![0_u8; 441 * 2];
+
+ let first = decoder
+ .push_bytes(&ten_ms, 44_100, 1, false)
+ .expect("first chunk");
+ assert!(first.frames.is_empty());
+ assert_eq!(441, first.buffered_source_samples);
+
+ let second = decoder
+ .push_bytes(&ten_ms, 44_100, 1, false)
+ .expect("second chunk");
+ assert_eq!(1, second.frames.len());
+ assert_eq!(960, second.frames[0].samples_per_channel);
+ assert_eq!(0, second.buffered_source_samples);
+ }
+
+ #[test]
+ fn pcm_s16le_stream_decoder_can_keep_16k_native_audio_profile() {
+ let mut decoder = PcmS16leStreamDecoder::new(16_000, 1, 16_000, 1).expect("stream decoder");
+ let mut pcm = Vec::new();
+ for index in 0..1_600 {
+ let sample = if index % 2 == 0 { 768_i16 } else { -768_i16 };
+ pcm.extend_from_slice(&sample.to_le_bytes());
+ }
+
+ let mut frames = Vec::new();
+ let chunk_size = 250 * 2;
+ for (index, chunk) in pcm.chunks(chunk_size).enumerate() {
+ let last = (index + 1) * chunk_size >= pcm.len();
+ let result = decoder
+ .push_bytes(chunk, 16_000, 1, last)
+ .expect("push pcm chunk");
+ frames.extend(result.frames);
+ }
+
+ assert_eq!(5, frames.len());
+ assert!(frames.iter().all(|frame| frame.sample_rate == 16_000));
+ assert!(frames.iter().all(|frame| frame.num_channels == 1));
+ assert!(frames.iter().all(|frame| frame.samples_per_channel == 320));
+ assert_eq!(
+ 1_600,
+ frames.iter().map(|frame| frame.data.len()).sum::<usize>()
+ );
+ }
+}
--
Gitblit v1.9.3