add augment

.gitignore

@@ -10,7 +10,6 @@ wheels/
 .venv
 
 # data
-.checkpoints
 .data
 data
 runs

.vscode/launch.json

@@ -1,16 +0,0 @@
-{
-    // Use IntelliSense to learn about possible attributes.
-    // Hover to view descriptions of existing attributes.
-    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
-    "version": "0.2.0",
-    "configurations": [
-
-        {
-            "name": "Python Debugger: Current File",
-            "type": "debugpy",
-            "request": "launch",
-            "program": "src/train.py",
-            "console": "integratedTerminal"
-        }
-    ]
-}

pyproject.toml

@@ -8,22 +8,17 @@ dependencies = [
     "ipython>=9.10.1",
     "librosa>=0.11.0",
     "matplotlib>=3.10.8",
-    "mutagen>=1.47.0",
     "numpy>=2.4.4",
     "pandas>=3.0.2",
     "pillow>=12.2.0",
-    "pydub>=0.25.1",
     "pyrubberband>=0.4.0",
     "setuptools<82",
-    "silero-vad>=6.2.1",
     "tensorboard>=2.20.0",
     "tensorboardx>=2.6.5",
     "torch==2.8.0",
-    "torch-audiomentations>=0.12.0",
     "torchaudio==2.8.0",
     "torchcodec==0.7.0",
     "tqdm>=4.67.3",
-    "webrtcvad>=2.0.10",
 ]
 
 [[index]]

src/dataset.py

@@ -1,21 +1,22 @@
 import random
+import unicodedata
 
 import torch
 from torch import Tensor
 from torch.utils.data import Dataset, DataLoader
 import torchaudio
-from torch_audiomentations import ApplyImpulseResponse, Gain, PitchShift, LowPassFilter, HighPassFilter
-from torchaudio.transforms import Resample, TimeStretch
+from torchaudio.transforms import FrequencyMasking, MelSpectrogram, AmplitudeToDB, Resample, TimeMasking, TimeStretch
 from pathlib import Path
+import torchaudio.functional as F
 import pandas as pd
-from typing import List, TypedDict
+from typing import Dict, List, TypedDict
 from handle.text_normalizer import collapse_spaces, normalize_extended_uyghur_characters
 from tokenizer import ASRTokenizer
 
 
 # 单个样本的数据结构（Dataset.__getitem__ 返回）
 class BatchItem(TypedDict):
-    waveform: Tensor          # [time]
+    mel_spec: Tensor          # [n_mels, time] Mel频谱
     target_ids: Tensor        # [seq_len] 目标文本的token IDs
     target_text: str          # 原始文本
     audio_path: str           # 音频文件路径
@@ -23,9 +24,9 @@ class BatchItem(TypedDict):
 
 # 批量数据的数据结构（collate_fn 返回，DataLoader 输出）
 class Batch(TypedDict):
-    waveforms: Tensor         # [batch, time]
+    mel_specs: Tensor         # [batch, n_mels, time] padding后的Mel频谱
     targets: Tensor           # [batch, max_len] padding后的目标IDs
-    waveform_lengths: Tensor       # [batch] 每个样本的实际Waveform长度
+    mel_lengths: Tensor       # [batch] 每个样本的实际Mel长度
     target_lengths: Tensor    # [batch] 每个样本的实际目标长度
     target_texts: List[str]   # [batch] 原始文本列表
     audio_paths: List[str]    # [batch] 音频路径列表
@@ -40,68 +41,20 @@ class TsvFormat(TypedDict):
     gender: str
     locale: str
 
-class NoiseAugmentor:
-    def __init__(self, noise_root: Path, sample_rate: int=16000):
-        self.sample_rate = sample_rate
-        self.noise_files = list(Path(noise_root).rglob("*.wav"))
-
-    def apply_real_noise(self, waveform: Tensor):
-        # 1. 随机选一个噪音文件
-        noise_path = random.choice(self.noise_files)
-        noise_waveform, sr = torchaudio.load_with_torchcodec(noise_path)
-        
-        # Resample to target sample rate.
-        if sr != self.sample_rate:
-            noise_waveform = Resample(sr, self.sample_rate)(noise_waveform)
-        
-        # Convert to mono if it is setro.
-        if waveform.shape[0] > 1:
-            waveform = waveform.mean(dim=0, keepdim=True)
-        
-        # 3. 截取或填充，使其长度与语音一致
-        sig_len = waveform.shape[1]
-        noise_len = noise_waveform.shape[1]
-        
-        if noise_len >= sig_len:
-            # 随机截取一段
-            start = random.randint(0, noise_len - sig_len)
-            noise_waveform = noise_waveform[:, start:start + sig_len]
-        else:
-            full_noise = torch.zeros_like(waveform)
-            start = random.randint(0, sig_len - noise_len)
-            full_noise[:, start : start + noise_len] = noise_waveform
-            noise_waveform = full_noise
-            
-        # 4. 设定随机信噪比 SNR (5dB 到 20dB)
-        snr_db = random.uniform(5, 20)
-        
-        # 5. 混合
-        return self._mix_at_snr(waveform, noise_waveform, snr_db)
-
-    def _mix_at_snr(self, signal: Tensor, noise: Tensor, snr_db: float):
-        s_p = signal.pow(2).mean()
-        n_p = noise.pow(2).mean()
-        snr_linear = 10**(snr_db / 10)
-        scale = torch.sqrt(s_p / (n_p * snr_linear + 1e-8))
-        
-        noisy = signal + scale * noise
-        # 归一化，防止溢出
-        return noisy / (noisy.abs().max() + 1e-8)
-
 class CommonVoiceDataset(Dataset[BatchItem]):
     def __init__(
         self,
         tsv_path: Path,
         audio_dir: Path,
-        noise_dir: Path,
         tokenizer: ASRTokenizer,
         sample_rate: int = 16000,
+        n_mels: int = 80 * 4,
         max_audio_len: int = 480000,  # 30秒 @ 16kHz
         augment: bool = True,
-        augment_prob: float = 0.5,
+        augment_prob: float = 0.5,  # 数据增强的概率
     ) -> None:
         super().__init__()
-        self.noise_augmentor = NoiseAugmentor(noise_root=noise_dir, sample_rate=sample_rate)
+    
         self.audio_dir = Path(audio_dir)
         self.tokenizer = tokenizer
         self.sample_rate = sample_rate
@@ -119,13 +72,22 @@ class CommonVoiceDataset(Dataset[BatchItem]):
 
         self.data = self.data.loc[valid_indices].reset_index(drop=True)
 
-        self.gain_up = Gain(min_gain_in_db=4, max_gain_in_db=8, p=1.0, output_type='tensor')
-        self.gain_down = Gain(min_gain_in_db=-15, max_gain_in_db=-8, p=1.0, output_type='tensor')
-        self.pitch_up = PitchShift(min_transpose_semitones=1, max_transpose_semitones=4, p=1.0, sample_rate=self.sample_rate, output_type='tensor')
-        self.pitch_down = PitchShift(min_transpose_semitones=-4, max_transpose_semitones=-1, p=1.0, sample_rate=self.sample_rate, output_type='tensor')
-        self.lowpass = LowPassFilter(min_cutoff_freq=600, max_cutoff_freq=2000, p=1.0, output_type='tensor')
-        self.highpass = HighPassFilter(min_cutoff_freq=800, max_cutoff_freq=2000, p=1.0, output_type='tensor')
-        self.apply_ir = ApplyImpulseResponse(ir_paths=noise_dir,convolve_mode='same', p=1, output_type="tensor")
+        # Mel频谱转换
+        self.mel_transform = MelSpectrogram(
+            sample_rate=sample_rate,
+            n_fft=400,
+            win_length=400,
+            hop_length=160,
+            n_mels=n_mels,
+            f_min=0,
+            f_max=8000,
+            power=3.0
+        )
+        self.amplitude_to_db = AmplitudeToDB()
+
+        # SpecAugment 转换
+        self.time_masking = TimeMasking(time_mask_param=30)  # 遮蔽最多30帧
+        self.freq_masking = FrequencyMasking(freq_mask_param=15)  # 遮蔽最多15个频率
 
     def __len__(self):
         return len(self.data)
@@ -133,75 +95,42 @@ class CommonVoiceDataset(Dataset[BatchItem]):
     def _load_audio(self, audio_path: Path) -> Tensor:
         waveform, sample_rate = torchaudio.load_with_torchcodec(audio_path)
 
-        # Resample to target sample rate.
         if sample_rate != self.sample_rate:
             waveform = Resample(sample_rate, self.sample_rate)(waveform)
 
-        # Convert to mono if it is setro.
         if waveform.shape[0] > 1:
             waveform = waveform.mean(dim=0, keepdim=True)
 
-        # Normalization
-        waveform = waveform / waveform.abs().max()
+        max_val = waveform.abs().max()
+        if max_val > 0:
+            waveform = waveform / max_val
         
-        # Clip waveform exceeds from max length.
         if waveform.shape[1] > self.max_audio_len:
             waveform = waveform[:, :self.max_audio_len]
+
         return waveform
     
+    def _extract_features(self, waveform: Tensor) -> Tensor:
+        """ 提取 Mel 频谱特征 """
+        mel_spec: Tensor = self.mel_transform(waveform)
+        log_mel_spec: Tensor = self.amplitude_to_db(mel_spec)
+        return log_mel_spec.squeeze(0) # [n_mels, time]
+    
     def _augment_waveform(self, waveform: Tensor) -> Tensor:
         if not self.augment or random.random() > self.augment_prob:
             return waveform
         
         # 1. voice Stretch/Compress 
-        if random.random() < 0.6:
-            waveform = self._stretch_or_compress(waveform=waveform)
+        if random.random() < 0.5:
+            waveform = self._voice_stretch_or_compress(waveform=waveform)
         
         if random.random() < 0.4:
-            waveform = self.noise_augmentor.apply_real_noise(waveform)
-        
-        if random.random() < 0.3:
-            waveform = self._time_mask_waveform(waveform=waveform)
-
-        # torch_audiomentations: [1, time] -> [1, 1, time]
-        if waveform.dim() == 2:
-            waveform_3d = waveform.unsqueeze(0)
-        # 随机选择一种频谱增强
-        choice = random.random()
-        if choice < 0.15:
-            # 增益变化（上或下）
-            if random.random() < 0.5:
-                waveform_3d = self.gain_up(waveform_3d, sample_rate=self.sample_rate)
-            else:
-                waveform_3d = self.gain_down(waveform_3d, sample_rate=self.sample_rate)
-        elif choice < 0.25:
-            # 音高变化（上或下）
-            if random.random() < 0.5:
-                waveform_3d = self.pitch_up(waveform_3d, sample_rate=self.sample_rate)
-            else:
-                waveform_3d = self.pitch_down(waveform_3d, sample_rate=self.sample_rate)
-        elif choice < 0.30:
-            # 低通滤波（声音发闷）
-            waveform_3d = self.lowpass(waveform_3d, sample_rate=self.sample_rate)
-        # elif choice < 0.32:
-        #     # 低通滤波（声音发闷）
-        #     waveform_3d = self.apply_ir(waveform_3d, sample_rate=self.sample_rate)
-        elif choice < 0.35:
-            # 高通滤波（电话效果）
-            waveform_3d = self.highpass(waveform_3d, sample_rate=self.sample_rate)
-        
-        #  [1, 1, time] -> [1, time]
-        waveform = waveform_3d.squeeze(0)
-
-        # 防止多次 augment 后振幅溢出，最后归一化
-        max_amp = waveform.abs().max()
-        if max_amp > 1.0:
-            waveform = waveform / max_amp
+            waveform = self._add_noise(waveform)
         
         return waveform
     
-    def _stretch_or_compress(self, waveform: Tensor) -> Tensor:
-        speed_factor = random.uniform(0.85, 1.4) # (Speed Change: 0.85x - 1.4x)
+    def _voice_stretch_or_compress(self, waveform: Tensor) -> Tensor:
+        speed_factor = random.uniform(0.6, 1.4) # (Speed Change: 0.6x - 1.4x)
         spec = torch.stft(
             waveform.squeeze(0),
             n_fft=400,
@@ -211,37 +140,49 @@ class CommonVoiceDataset(Dataset[BatchItem]):
         )
         
         # 时间拉伸（不改变音高）
-        stretched_spec = TimeStretch(hop_length=160, n_freq=spec.shape[-2], fixed_rate=speed_factor)(spec)
+        stretch = TimeStretch(
+            hop_length=160,
+            n_freq=201,
+            fixed_rate=speed_factor
+        )
+        stretched_spec = stretch(spec)
         
         # 转回波形
-        waveform_stretched = torch.istft(stretched_spec, n_fft=400, hop_length=160, window=torch.hann_window(400).to(waveform.device)).unsqueeze(0)
+        waveform_stretched = torch.istft(
+            stretched_spec,
+            n_fft=400,
+            hop_length=160,
+            window=torch.hann_window(400).to(waveform.device)
+        ).unsqueeze(0)
     
         return waveform_stretched
     
-    def _time_mask_waveform(self, waveform: Tensor) -> Tensor:
-        audio_len = waveform.shape[1]
-        sr = self.sample_rate # 16000
+    def _add_noise(self, waveform: Tensor, snr_db: float = None) -> Tensor:
+        if snr_db is None:
+            snr_db = random.uniform(15, 25)
         
-        # 设置参数：单次遮盖最长 0.4 秒 (6400个点)
-        max_mask_time = 0.4
-        max_mask_samples = int(sr * max_mask_time)
+        signal_power = waveform.norm(p=2)
         
-        # 根据音频长度决定遮盖次数：
-        # 比如每 3 秒钟允许遮盖 1 次
-        num_masks = max(1, audio_len // (sr * 3)) 
+        snr_linear = 10 ** (snr_db / 10)
+        noise_power = signal_power / snr_linear
         
-        for _ in range(num_masks):
-            # 每次随机遮盖 0.1s 到 0.4s
-            current_mask_len = random.randint(int(sr * 0.1), max_mask_samples)
+        # Generate Gaussian noise
+        noise = torch.randn_like(waveform) * noise_power / waveform.shape[1] ** 0.5
         
-            if audio_len > current_mask_len:
-                start_pos = random.randint(0, audio_len - current_mask_len)
+        noisy_waveform: Tensor = waveform + noise
         
-                # 填充微小噪音（模拟环境底噪）
-                noise = torch.randn(1, current_mask_len).to(waveform.device) * 0.002
-                waveform[:, start_pos : start_pos + current_mask_len] = noise
+        max_val = noisy_waveform.abs().max()
+        if max_val > 0:
+            noisy_waveform = noisy_waveform / max_val
         
-        return waveform
+        return noisy_waveform
+    
+    def _augment_spec(self, mel_spec: Tensor) -> Tensor:
+        if not self.augment or random.random() > self.augment_prob:
+            return mel_spec
+        mel_spec = self.time_masking(mel_spec)
+        mel_spec = self.freq_masking(mel_spec)
+        return mel_spec
     
     def __getitem__(self, index) -> BatchItem:
         row: TsvFormat = self.data.iloc[index]
@@ -251,24 +192,26 @@ class CommonVoiceDataset(Dataset[BatchItem]):
 
         waveform = self._load_audio(audio_path=audio_path)
         waveform = self._augment_waveform(waveform)
-        waveform = waveform.squeeze(0)
+        mel_spec = self._extract_features(waveform=waveform)
+        mel_spec = self._augment_spec(mel_spec=mel_spec)
 
         return BatchItem(
-            waveform=waveform,
+            mel_spec=mel_spec,
             target_ids=torch.tensor(self.tokenizer.encode(text), dtype=torch.long),
             target_text=text,
             audio_path=str(audio_path)
         )
     
 def collate_fn(items: List[BatchItem]) -> Batch:
-    max_waveform_len = max(item['waveform'].shape[0] for item in items)
+    max_mel_len = max(item['mel_spec'].shape[1] for item in items)
     max_target_len = max(len(item['target_ids']) for item in items)
 
     batch_size = len(items)
+    n_mels = items[0]['mel_spec'].shape[0]
 
-    waveforms = torch.zeros(batch_size, max_waveform_len)
+    mel_specs = torch.zeros(batch_size, n_mels, max_mel_len)
     targets = torch.zeros(batch_size, max_target_len, dtype=torch.long)
-    waveform_lengths = torch.zeros(batch_size, dtype=torch.long)
+    mel_lengths = torch.zeros(batch_size, dtype=torch.long)
     target_lengths = torch.zeros(batch_size, dtype=torch.long)
 
     target_texts = []
@@ -276,29 +219,29 @@ def collate_fn(items: List[BatchItem]) -> Batch:
     
 
     for i, item in enumerate(items):
-        waveform_len = item['waveform'].shape[0]
+        mel_len = item['mel_spec'].shape[1]
         target_len = len(item['target_ids'])
 
-        waveforms[i, :waveform_len] = item['waveform']
+        mel_specs[i, :, :mel_len] = item['mel_spec']
         targets[i, :target_len] = item['target_ids']
-        waveform_lengths[i] = waveform_len
+        mel_lengths[i] = mel_len
         target_lengths[i] = target_len
 
         target_texts.append(item['target_text'])
         audio_paths.append(item['audio_path'])
 
     return Batch(
-        waveforms=waveforms,
+        mel_specs=mel_specs,
         targets=targets,
-        waveform_lengths=waveform_lengths,
+        mel_lengths=mel_lengths,
         target_lengths=target_lengths,
         target_texts=target_texts,
         audio_paths=audio_paths
     )
 
 
-def create_dataloader(tsv_path: Path, audio_dir: Path, noise_dir: Path, tokenizer: ASRTokenizer, batch_size: int = 8, shuffle: bool = True, augment: bool = True, augment_prob: int = 0.5) -> DataLoader:
-    dataset = CommonVoiceDataset(tsv_path=tsv_path, audio_dir=audio_dir, noise_dir=noise_dir, tokenizer=tokenizer, augment=augment, augment_prob=augment_prob)
+def create_dataloader(tsv_path: Path, audio_dir: Path, tokenizer: ASRTokenizer, batch_size: int = 8, shuffle: bool = True, augment: bool = True) -> DataLoader:
+    dataset = CommonVoiceDataset(tsv_path=tsv_path, audio_dir=audio_dir, tokenizer=tokenizer, augment=augment)
 
     return DataLoader(dataset=dataset, batch_size=batch_size, shuffle=shuffle, collate_fn=collate_fn, pin_memory=True, num_workers=8, prefetch_factor=8, persistent_workers=True)
 
@@ -332,100 +275,3 @@ if __name__ == "__main__":
         print(f"Target texts: {batch['target_texts']}")
         print(f"Audio paths: {batch['audio_paths']}")
         break
-
-
-
-
-
-import torch
-import torchaudio
-from silero_vad import load_silero_vad, read_audio, get_speech_timestamps
-
-# --- 1. 加载 VAD 模型 ---
-# 推荐使用 GPU (cuda) 如果可用，否则使用 CPU (cpu)
-vad_model = load_silero_vad(source="local", force_onnx=False)
-
-def process_audio_with_vad_and_asr(audio_path, inference_module):
-    """
-    使用 Silero VAD 分割音频，然后对每个语音片段进行 ASR 转录。
-
-    Args:
-        audio_path (str): 输入音频文件的路径。
-        inference_module: 您封装了 transcribe 方法的模块或对象。
-                          需要确保其 _load_audio 和 transcribe 方法可用。
-    """
-    print(f"正在加载音频: {audio_path}")
-    
-    # --- 2. 加载音频用于 VAD 检测 ---
-    # Silero VAD 推荐使用 16kHz 采样率
-    waveform_vad, sample_rate_vad = torchaudio.load(audio_path)
-    if sample_rate_vad != 16000:
-        # 如果采样率不是16kHz，需要重采样以供VAD使用
-        resampler = torchaudio.transforms.Resample(orig_freq=sample_rate_vad, new_freq=16000)
-        waveform_vad = resampler(waveform_vad)
-        sample_rate_vad = 16000
-
-    # --- 3. 获取语音时间段 ---
-    # get_speech_timestamps 返回一个列表，每个元素是 {'start': start_sample, 'end': end_sample} 的字典
-    # 采样率是16000，所以时间戳单位是 1/16000 秒
-    speech_timestamps = get_speech_timestamps(
-        waveform_vad, 
-        vad_model, 
-        sampling_rate=sample_rate_vad,
-        threshold=0.5,  # 可以根据需要调整阈值
-        min_speech_duration_ms=250, # 最小语音持续时间，防止短噪音被误判
-        max_speech_duration_s=float('inf'), # 最大语音持续时间，float('inf') 表示不限制
-        min_silence_duration_ms=100, # 最小静音间隔，用于分割语音块
-        window_size_samples=1536, # VAD窗口大小
-        speech_pad_ms=30 # 在语音块前后添加的填充时间
-    )
-
-    print(f"检测到 {len(speech_timestamps)} 个语音片段。")
-
-    full_text = ""
-    for i, ts in enumerate(speech_timestamps):
-        start_sample = int(ts['start'])
-        end_sample = int(ts['end'])
-
-        # 计算时间戳（秒）
-        start_time = start_sample / sample_rate_vad
-        end_time = end_sample / sample_rate_vad
-        
-        print(f"\n处理第 {i+1} 个片段: 时间范围 [{start_time:.2f}s - {end_time:.2f}s]")
-
-        # --- 4. 从原始音频中提取此片段 ---
-        # 注意：这里假设您的 transcribe 函数可以接受 waveform 张量。
-        # 我们需要从原始可能不同采样率的音频中提取片段，或者用VAD处理过的waveform。
-        # 为了匹配您原来的 _load_audio 方式，我们用 torchaudio 再次精确加载片段。
-        
-        # 计算原始音频中的样本索引（如果原始音频采样率与VAD不同）
-        original_waveform, original_sr = torchaudio.load(audio_path)
-        if sample_rate_vad != original_sr:
-             # 如果VAD和原始音频采样率不同，需要重新映射索引
-            start_idx_orig = int(start_sample * (original_sr / sample_rate_vad))
-            end_idx_orig = int(end_sample * (original_sr / sample_rate_vad))
-        else:
-            start_idx_orig = start_sample
-            end_idx_orig = end_sample
-            
-        segment_waveform = original_waveform[:, start_idx_orig:end_idx_orig]
-
-        # --- 5. 对该片段进行 ASR 转录 ---
-        # 这里调用您原有的 transcribe 方法
-        try:
-            # 假设 transcribe 方法接受一个 waveform tensor
-            segment_text = inference_module.transcribe(waveform=segment_waveform)
-            
-            print(f"  -> 转录结果: {segment_text}")
-            full_text += f"[{start_time:.2f}-{end_time:.2f}s] {segment_text}\n"
-        except Exception as e:
-            print(f"  -> 转录第 {i+1} 个片段时出错: {e}")
-
-    print("\n--- 完整转录结果 ---")
-    print(full_text)
-
-
-# --- 使用示例 ---
-# 假设您有一个名为 'inference' 的模块对象，它有 _load_audio 和 transcribe 方法
-# audio_path = "your_audio_file.wav"
-# process_audio_with_vad_and_asr(audio_path, inference)

src/handle/audio_analyze.py

@@ -1,70 +0,0 @@
-from pathlib import Path
-from tqdm import tqdm
-from mutagen.mp3 import MP3
-from mutagen import MutagenError
-import pandas as pd
-
-workspace_dir = Path(__file__).parent.parent.parent
-
-def format_duration(seconds):
-    """将秒数格式化为 时:分:秒 的格式"""
-    hours = int(seconds // 3600)
-    minutes = int((seconds % 3600) // 60)
-    secs = int(seconds % 60)
-    
-    if hours > 0:
-        return f"{hours}小时{minutes}分{secs}秒"
-    elif minutes > 0:
-        return f"{minutes}分{secs}秒"
-    else:
-        return f"{secs}秒"
-
-def get_mp3_duration(file_path: Path):
-    """获取单个MP3文件的时长（秒）"""
-    try:
-        audio = MP3(file_path)
-        return audio.info.length
-    except MutagenError as e:
-        print(f"错误：无法读取文件 {file_path} - {e}")
-        return None
-    except Exception as e:
-        print(f"错误：处理文件 {file_path} 时出错 - {e}")
-        return None
-
-def analyze_mp3_files(directory: Path, ):
-    """分析目录中的所有MP3文件"""
-    results = []
-    total_duration = 0
-    df = pd.read_csv(workspace_dir / '.data/ug/train_new.tsv', sep='\t')
-
-    for _, row in tqdm(df.iterrows(), total=len(df), desc="anlayze audio"):
-        mp3_file: Path = workspace_dir / '.data/ug/clips'  / row['path']
-        duration = get_mp3_duration(mp3_file)
-        
-        if duration is not None:
-            total_duration += duration
-            duration_str = format_duration(duration)
-            file_size = mp3_file.stat().st_size / (1024 * 1024)  # 转换为MB
-            
-            result_line = f"{mp3_file.name:<50} {duration_str:>20} ({file_size:.2f} MB)"
-            results.append(result_line)
-        else:
-            error_line = f"{mp3_file.name:<50} {'读取失败':>20}"
-            print(error_line)
-            results.append(error_line)
-
-    mp3_files: list[Path] = []
-    for ext in ['*.mp3', '*.MP3']:
-            mp3_files.extend(directory.rglob(ext))
-    
-    print(f"tsv 找到 {len(results)} 个MP3文件\n")
-    print(f"找到 {len(mp3_files)} 个MP3文件\n")
-    print(f"\n总时长: {format_duration(total_duration)}")
-    print(f"总时长(秒): {total_duration:.2f} 秒")
-    print(f"总时长(分钟): {total_duration/60:.2f} 分钟")
-    print(f"总时长(小时): {total_duration/3600:.2f} 小时")
-    print(f"文件总数: {len(mp3_files)}")
-
-
-audio_directory = Path(workspace_dir / '.data/ug/clips')
-analyze_mp3_files(directory=audio_directory)

src/handle/export_model_state_dict.py

@@ -4,10 +4,11 @@ from pathlib import Path
 import torch
 
 device = "cuda:0"
-workspace_dir = Path(__file__).parent.parent.parent
+workspace_dir = Path(__file__).parent.parent
 
-input_checkpoint = workspace_dir.joinpath('.checkpoints/best_wer_model.pt')
-output_checkpoint = workspace_dir.joinpath('.checkpoints/prodect_best_wer_model.pt')
+input_checkpoint = workspace_dir.joinpath('.checkpoints/checkpoint_step_9500.pt')
+output_checkpoint = workspace_dir.joinpath('.checkpoints/checkpoint_step.pt')
 
 checkpoint = torch.load(input_checkpoint, map_location=device)
 torch.save(checkpoint['model_state_dict'], output_checkpoint)
+

src/handle/export_new_tsv.py

@@ -1,37 +0,0 @@
-from pathlib import Path
-
-import pandas as pd
-workspace_dir = Path(__file__).parent.parent.parent
-
-# 读取你合并后的总表 (253,430条)
-df = pd.read_csv(workspace_dir / ".data/ug/validated.tsv", sep='\t')
-
-# 1. 获取所有唯一的说话人
-all_speakers = df['client_id'].unique()
-
-# 2. 随机打乱说话人顺序
-import random
-random.seed(42) # 固定随机种子，保证实验可重复
-random.shuffle(all_speakers)
-
-# 3. 挑选验证集说话人，直到录音总数达到 ~8000 条
-val_indices = []
-val_count = 0
-target_val_size = 8000
-
-for speaker in all_speakers:
-    speaker_data = df[df['client_id'] == speaker]
-    val_indices.extend(speaker_data.index.tolist())
-    val_count += len(speaker_data)
-    if val_count >= target_val_size:
-        break
-
-# 4. 划分文件
-df_val = df.loc[val_indices]
-df_train = df.drop(val_indices)
-
-print(f"训练集条数: {len(df_train)}")
-print(f"验证集条数: {len(df_val)}")
-
-df_train.to_csv(workspace_dir / ".data/ug/train_new.tsv", sep='\t', index=False)
-df_val.to_csv(workspace_dir / ".data/ug/val_new.tsv", sep='\t', index=False)

src/handle/test.ipynb

@@ -2,7 +2,7 @@
  "cells": [
   {
    "cell_type": "code",
-   "execution_count": 21,
+   "execution_count": 25,
    "id": "aa1d00f1",
    "metadata": {},
    "outputs": [
@@ -32,7 +32,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 22,
+   "execution_count": 26,
    "id": "a6d351e1",
    "metadata": {},
    "outputs": [
@@ -110,7 +110,7 @@
        "<DisplayHandle display_id=scaled>"
       ]
      },
-     "execution_count": 22,
+     "execution_count": 26,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -134,17 +134,9 @@
     "display(Audio(waveform, rate=sample_rate, element_id=\"scaled\", normalize=False), display_id=\"scaled\")"
    ]
   },
-  {
-   "cell_type": "markdown",
-   "id": "9cb24ba9",
-   "metadata": {},
-   "source": [
-    "## Mel spectrogram based approach"
-   ]
-  },
   {
    "cell_type": "code",
-   "execution_count": 23,
+   "execution_count": 27,
    "id": "59557614",
    "metadata": {},
    "outputs": [
@@ -162,14 +154,6 @@
       "  f_max: 8000\n",
       "  power: 3.0\n"
      ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "/home/blacksheep/projekts/study_asr/.venv/lib/python3.11/site-packages/torchaudio/functional/functional.py:585: UserWarning: At least one mel filterbank has all zero values. The value for `n_mels` (320) may be set too high. Or, the value for `n_freqs` (201) may be set too low.\n",
-      "  warnings.warn(\n"
-     ]
     }
    ],
    "source": [
@@ -195,7 +179,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 24,
+   "execution_count": 28,
    "id": "7b9f9e7f",
    "metadata": {},
    "outputs": [
@@ -255,7 +239,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 25,
+   "execution_count": 29,
    "id": "56db5576",
    "metadata": {},
    "outputs": [
@@ -273,7 +257,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 26,
+   "execution_count": null,
    "id": "cfa5546a",
    "metadata": {},
    "outputs": [
@@ -287,7 +271,7 @@
       "tensor([208])\n",
       "conformer out:  torch.Size([1, 208, 200])\n",
       "conformer length out:  tensor([208])\n",
-      "loss:  tensor(-3.9094, grad_fn=<MeanBackward0>)\n"
+      "loss:  tensor(-4.1674, grad_fn=<MeanBackward0>)\n"
      ]
     }
    ],
@@ -346,124 +330,6 @@
     "\n",
     "\n"
    ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "421d0749",
-   "metadata": {},
-   "source": [
-    "## Conv1D over waveform approach"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 27,
-   "id": "83fc7c1e",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "from torch.nn import Linear, Conv1d, ModuleList, GELU, GroupNorm\n",
-    "from torchaudio.models import Conformer\n",
-    "waveform = waveform.unsqueeze(0)[:, :, :16000]"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 28,
-   "id": "a76e8e18",
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Shape of waveform: torch.Size([1, 1, 16000])\n",
-      "Shape after 0th: layer: torch.Size([1, 512, 3199]), layer_instance: Conv1d(1, 512, kernel_size=(10,), stride=(5,))\n",
-      "Shape after 1th: layer: torch.Size([1, 512, 3199]), layer_instance: GroupNorm(32, 512, eps=1e-05, affine=True)\n",
-      "Shape after 2th: layer: torch.Size([1, 512, 3199]), layer_instance: GELU(approximate='none')\n",
-      "Shape after 3th: layer: torch.Size([1, 512, 1599]), layer_instance: Conv1d(512, 512, kernel_size=(3,), stride=(2,))\n",
-      "Shape after 4th: layer: torch.Size([1, 512, 1599]), layer_instance: GroupNorm(32, 512, eps=1e-05, affine=True)\n",
-      "Shape after 5th: layer: torch.Size([1, 512, 1599]), layer_instance: GELU(approximate='none')\n",
-      "Shape after 6th: layer: torch.Size([1, 512, 799]), layer_instance: Conv1d(512, 512, kernel_size=(3,), stride=(2,))\n",
-      "Shape after 7th: layer: torch.Size([1, 512, 799]), layer_instance: GroupNorm(32, 512, eps=1e-05, affine=True)\n",
-      "Shape after 8th: layer: torch.Size([1, 512, 799]), layer_instance: GELU(approximate='none')\n",
-      "Shape after 9th: layer: torch.Size([1, 512, 399]), layer_instance: Conv1d(512, 512, kernel_size=(3,), stride=(2,))\n",
-      "Shape after 10th: layer: torch.Size([1, 512, 399]), layer_instance: GroupNorm(32, 512, eps=1e-05, affine=True)\n",
-      "Shape after 11th: layer: torch.Size([1, 512, 399]), layer_instance: GELU(approximate='none')\n",
-      "Shape after 12th: layer: torch.Size([1, 512, 199]), layer_instance: Conv1d(512, 512, kernel_size=(3,), stride=(2,))\n",
-      "Shape after 13th: layer: torch.Size([1, 512, 199]), layer_instance: GroupNorm(32, 512, eps=1e-05, affine=True)\n",
-      "Shape after 14th: layer: torch.Size([1, 512, 199]), layer_instance: GELU(approximate='none')\n",
-      "Shape after 15th: layer: torch.Size([1, 512, 99]), layer_instance: Conv1d(512, 512, kernel_size=(2,), stride=(2,))\n",
-      "Shape after 16th: layer: torch.Size([1, 512, 99]), layer_instance: GroupNorm(32, 512, eps=1e-05, affine=True)\n",
-      "Shape after 17th: layer: torch.Size([1, 512, 99]), layer_instance: GELU(approximate='none')\n",
-      "Shape after 18th: layer: torch.Size([1, 512, 49]), layer_instance: Conv1d(512, 512, kernel_size=(2,), stride=(2,))\n",
-      "Shape after 19th: layer: torch.Size([1, 512, 49]), layer_instance: GroupNorm(32, 512, eps=1e-05, affine=True)\n",
-      "Shape after 20th: layer: torch.Size([1, 512, 49]), layer_instance: GELU(approximate='none')\n",
-      "torch.Size([1, 49, 256])\n"
-     ]
-    }
-   ],
-   "source": [
-    "\n",
-    "print(f\"Shape of waveform: {waveform.shape}\")\n",
-    "\n",
-    "filters = ModuleList([\n",
-    "    Conv1d(in_channels=1, out_channels=512, kernel_size=10, stride=5),\n",
-    "    GroupNorm(num_channels=512, num_groups=32),\n",
-    "    GELU(),\n",
-    "    Conv1d(in_channels=512, out_channels=512, kernel_size=3, stride=2),\n",
-    "    GroupNorm(num_channels=512, num_groups=32),\n",
-    "    GELU(),\n",
-    "    Conv1d(in_channels=512, out_channels=512, kernel_size=3, stride=2),\n",
-    "    GroupNorm(num_channels=512, num_groups=32),\n",
-    "    GELU(),\n",
-    "    Conv1d(in_channels=512, out_channels=512, kernel_size=3, stride=2),\n",
-    "    GroupNorm(num_channels=512, num_groups=32),\n",
-    "    GELU(),\n",
-    "    Conv1d(in_channels=512, out_channels=512, kernel_size=3, stride=2),\n",
-    "    GroupNorm(num_channels=512, num_groups=32),\n",
-    "    GELU(),\n",
-    "    Conv1d(in_channels=512, out_channels=512, kernel_size=2, stride=2),\n",
-    "    GroupNorm(num_channels=512, num_groups=32),\n",
-    "    GELU(),\n",
-    "    Conv1d(in_channels=512, out_channels=512, kernel_size=2, stride=2),\n",
-    "    GroupNorm(num_channels=512, num_groups=32),\n",
-    "    GELU(),\n",
-    "])\n",
-    "\n",
-    "projector = Linear(in_features=512, out_features=256)\n",
-    "\n",
-    "filter_out = waveform\n",
-    "for i, filter in enumerate(filters):\n",
-    "    filter_out: Tensor = filter(filter_out)\n",
-    "    print(f\"Shape after {i}th: layer: {filter_out.shape}, layer_instance: {filter}\")\n",
-    "filter_out = filter_out.permute(0, 2, 1)\n",
-    "filter_out = projector(filter_out)\n",
-    "print(filter_out.shape)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 36,
-   "id": "567df2f1",
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "49\n"
-     ]
-    }
-   ],
-   "source": [
-    "dummy_waveform = torch.zeros(1, 1, 16000)\n",
-    "for i, filter in enumerate(filters):\n",
-    "    # print(i)\n",
-    "    dummy_waveform = filter(dummy_waveform)\n",
-    "shink_factor = dummy_waveform.shape[2]\n",
-    "print(shink_factor)"
-   ]
   }
  ],
  "metadata": {

src/handle/test.py

@@ -196,7 +196,7 @@ text = "\" 15 يىل بۇرۇن مەن بىلەن سەي چۇڭشىن (ئال
 # text = "يۆ جيەنتاۋ يېقىنقى بىر مەزگىلدە، خىزمەتداشلىرى بىلەن نۇرغۇن قىيىنچىلىققا ئۇچرىغان شوپۇرغا ياردەم قىلغانلىقىنى، شۇنداقلا يېمەكلىك ۋە سۇ يەتكۈزۈپ بەرگەنلىكىنى، ئەمما ئاياغ سوۋغا قىلىشى تۇنجى قېتىم ئىكەنلىكىنى، بۈگۈنكىسى 20 يىللىق ساقچىلىق جەريانىدا تۇنجى قېتىم شوپۇرغا ئاياغ سوۋغا قىلىشى ئىكەنلىكىنى ئېيتتى."
 text = "يۆ جۇڭمىڭ مۇنداق دېدى: 2019- يىلى 12-ئايدا، مەملىكەتلىك خەلق قۇرۇلتىيى دائىمىي كومىتېتىنىڭ 44- قېتىملىق كومىتېت باشلىقلىرى يىغىنى مەملىكەتلىك خەلق قۇرۇلتىيى دائىمىي كومىتېتىنىڭ 2020-يىللىق قانۇن چىقىرىش خىزمىتى پىلانىنى پىرىنسىپ جەھەتتىن ماقۇللىدى، خىزمەت تەرتىپى بويىچە، 13-نۆۋەتلىك مەملىكەتلىك خەلق قۇرۇلتىيى 3-يىغىنىنىڭ روھى ۋە ۋەكىللەرنىڭ تەكلىپ-تەۋسىيەلىرىگە ئاساسەن پىلاننى تەڭشەش كېرەك. بۇ يىل 6-ئاينىڭ 1-كۈنى، 58-قېتىملىق كومىتېت باشلىقلىرى يىغىنى تەڭشەلگەندىن كېيىنكى يىللىق قانۇن چىقىرىش خىزمىتى پىلانىنى قاراپ چىقىپ ماقۇللىدى."
 text = "	ئاشقازان-ئۈچەينىڭ لۆمۈلدىشىنى ئىلگىرى سۈرىدىغان دورىنى تاماقتىن بۇرۇن ئىستېمال قىلىش كېرەك."
-text = "مەن مەكتەپكە باردىم"
+# text = "مەن مەكتەپكە باردىم"
 # text = "غەرىپئەللىرى"
 # text = "ئىكەنلىكىنى، بۈگۈنكىسى 20 يىللىق ساقچىلىق جەريانىدا تۇنجى"
 # text = "	يېزىدىكى كەڭ، ئازادە ئۆي، باغلىرىنى تاشلاپ، خەقنىڭ ھويلىسىدا قورۇنۇپ-ئەيمىنىپ يەر دەسسەپ يۈردى.	"
@@ -212,9 +212,9 @@ print(f"Original: {text}")
 print(f"Syllables: {result}")
 
 
-# tokenizer = ASRTokenizer(vocab_path=workspace_dir / 'config/asr_vocab.json')
+tokenizer = ASRTokenizer(vocab_path=workspace_dir / 'config/asr_vocab.json')
 
-# print(text)
-# ids = tokenizer.encode(text=text)
-# # print(ids)
-# print('|'.join(tokenizer.decode([id]) for id in ids))
+print(text)
+ids = tokenizer.encode(text=text)
+# print(ids)
+print('|'.join(tokenizer.decode([id]) for id in ids))

src/inference.py

@@ -1,37 +1,58 @@
+import random
+
+import librosa
 import torch
 import torchaudio
 from torch import Tensor, no_grad, device
-from torchaudio.transforms import Resample
+from torchaudio.transforms import MelSpectrogram, AmplitudeToDB, Resample, TimeStretch
 from pathlib import Path
-from typing import TypedDict
-from silero_vad import load_silero_vad, get_speech_timestamps
+import torchaudio.functional as F
+
 from tokenizer import ASRTokenizer
 from model import ASRModel
 
 CONFIG = {
     # 模型配置
-    'input_dim': 256,
+    'input_dim': 640,
     'num_heads': 8,
     'ffn_dim': 2048,
     'num_layers': 8,
     'dropout': 0.1,
 }
 
-class TimestampsType(TypedDict):
-    start: int
-    end: int
-
 class ASRInference:
-    def __init__(self, model_path: Path, vocab_path: Path, device: device, sample_rate: int = 16000) -> None:
+    def __init__(self, model_path: Path, vocab_path: Path, device: device, augment: bool = True, augment_prob: float = 0.5) -> None:
         self.device = device
-        self.sample_rate = sample_rate
+        self.augment: bool = augment
+        self.augment_prob: float = augment_prob
         self.tokenizer = ASRTokenizer(vocab_path=vocab_path)
-        self.model = ASRModel(vocab_size=self.tokenizer.vocab_size(), **CONFIG).to(device=device)
+        self.model = ASRModel(
+            vocab_size=self.tokenizer.vocab_size(),
+            input_dim=CONFIG['input_dim'],
+            num_heads=CONFIG['num_heads'],
+            ffn_dim=CONFIG['ffn_dim'],
+            num_layers=CONFIG['num_layers'],
+            dropout=CONFIG['dropout'],
+        ).to(device)
+
         self.model.load_state_dict(torch.load(model_path, map_location=device)['model_state_dict'])
         self.model.eval()
 
         print(f"params params: {self.model.get_num_params():,}",)
 
+        self.sample_rate = 16000
+        self.mel_transform = MelSpectrogram(
+            sample_rate=self.sample_rate,
+            n_fft=400,
+            win_length=400,
+            hop_length=160,
+            n_mels=80,
+            f_min=0,
+            f_max=8000,
+            power=2.0,
+        )
+        self.amplitude_to_db = AmplitudeToDB()
+
     def _load_audio(self, audio_path: Path)  -> Tensor:
         waveform, sample_rate = torchaudio.load_with_torchcodec(audio_path)
 
@@ -44,12 +65,87 @@ class ASRInference:
         waveform = waveform / (waveform.abs().max() + 1e-8)
         return waveform
     
-    def transcribe(self, waveform: Tensor) -> str:
-        waveform = waveform.to(device=self.device) # [1, time]
-        waveform_length = torch.tensor([waveform.shape[1]], dtype=torch.long, device=self.device)
+    def _extract_features(self, waveform: Tensor) -> Tensor:
+        mel_spec: Tensor = self.mel_transform(waveform)
+        log_mel_spec: Tensor = self.amplitude_to_db(mel_spec)
+        return log_mel_spec.squeeze(0) # [n_mels, time]
+    
+    def _augment_waveform(self, waveform: Tensor) -> Tensor:
+        if not self.augment or random.random() > self.augment_prob:
+            return waveform
+        
+        # 1. voice Stretch/Compress 
+        if random.random() < 0.5:
+            waveform = self._voice_stretch_or_compress(waveform=waveform)
+
+        if random.random() < 0.3:
+            waveform = self._drop_frames(waveform)
+
+        if random.random() < 0.4:
+            waveform = self._add_noise(waveform)
+        
+        return waveform
+    
+    def _voice_stretch_or_compress(self, waveform: Tensor) -> Tensor:
+        speed_factor = random.uniform(0.6, 1.4) # (Speed Change: 0.6x - 1.4x)
+        spec = torch.stft(
+            waveform.squeeze(0),
+            n_fft=400,
+            hop_length=160,
+            window=torch.hann_window(400).to(waveform.device),
+            return_complex=True
+        )
+        
+        # 时间拉伸（不改变音高）
+        stretch = TimeStretch(
+            hop_length=160,
+            n_freq=201,
+            fixed_rate=speed_factor
+        )
+        stretched_spec = stretch(spec)
+        
+        # 转回波形
+        waveform_stretched = torch.istft(
+            stretched_spec,
+            n_fft=400,
+            hop_length=160,
+            window=torch.hann_window(400).to(waveform.device)
+        ).unsqueeze(0)
+
+        return waveform_stretched
+
+    def _drop_frames(self, waveform: Tensor) -> Tensor:
+        audio_len = waveform.shape[1]
+        drop_ratio = random.uniform(0.05, 0.15)
+        drop_len = int(audio_len * drop_ratio)
+
+        if audio_len > drop_len:
+            start_pos = random.randint(0, audio_len - drop_len)
+            # clean
+            waveform = torch.cat([waveform[:, :start_pos], waveform[:, start_pos + drop_len:]], dim=1)
+
+        return waveform
+    
+    def _add_noise(self, waveform: Tensor) -> Tensor:
+        snr_db = random.uniform(10, 20)
+        signal_power = torch.mean(waveform ** 2)
+        
+        snr_linear = 10 ** (snr_db / 10)
+        noise_power = signal_power / snr_linear
+        
+        noise = torch.randn_like(waveform) * torch.sqrt(noise_power)
+        return waveform + noise
+    
+    def transcribe(self, audio_path: Path) -> str:
+        waveform = self._load_audio(audio_path=audio_path)
+        waveform = self._augment_waveform(waveform=waveform)
+        mel_spec = self._extract_features(waveform=waveform)
+
+        mel_spec = mel_spec.unsqueeze(0).to(self.device) # [1, n_mels, time]
+        mel_length = torch.tensor([mel_spec.shape[2]], dtype=torch.long, device=self.device)
 
         with no_grad():
-            log_probs, _ = self.model(waveforms=waveform, waveform_lengths=waveform_length) # [1, T, vocab]
+            log_probs, _ = self.model(mel_specs=mel_spec, mel_lengths=mel_length) # [1, T, vocab]
 
         text = self.tokenizer.ctc_greedy_decode(log_probs=log_probs[0])
         return text
@@ -63,61 +159,25 @@ class ASRInference:
 
         return results
 
-    def create(self, audio_path: Path, threshold_ms: int = 20000):
-        def merge_timestamps(timestamps: int, threshold_ms: int, sample_rate: int):
-            if not timestamps:
-                return []
-            threshold_samples = (threshold_ms / 1000) * sample_rate
-            merged = []
-            curr_start = timestamps[0]['start']
-            curr_end = timestamps[0]['end']
-            
-            for i in range(1, len(timestamps)):
-                # 如果当前积攒的长度不到 2 秒，就一直合并到当前的 end 上
-                if (curr_end - curr_start) < threshold_samples:
-                    curr_end = timestamps[i]['end']
-                else:
-                    merged.append({'start': curr_start, 'end': curr_end})
-                    curr_start = timestamps[i]['start']
-                    curr_end = timestamps[i]['end']
-            merged.append({'start': curr_start, 'end': curr_end})
-            return merged
-
-        vad_model = load_silero_vad(onnx=False)
-        waveform = self._load_audio(audio_path=audio_path)
-        speech_timestamps: list[TimestampsType] = get_speech_timestamps(
-            waveform, 
-            vad_model, 
-            sampling_rate=self.sample_rate,
-            threshold=0.5,  # 可以根据需要调整阈值
-            min_speech_duration_ms=100, # 最小语音持续时间，防止短噪音被误判
-            min_silence_duration_ms=200, # 最小静音间隔，用于分割语音块
-            speech_pad_ms=100, # 在语音块前后添加的填充时间
-        )
-        final_timestamps = merge_timestamps(timestamps=speech_timestamps, threshold_ms=threshold_ms, sample_rate=self.sample_rate)
-        for ts in final_timestamps:
-            ts: TimestampsType
-            start_sample = int(ts['start'])
-            end_sample = int(ts['end'])
-
-            segment_waveform = waveform[:, start_sample:end_sample]
-
-            segment_text = self.transcribe(waveform=segment_waveform)
-            print(segment_text, end=" ", flush=True)
-        print('\n')
 
 def main():
     workspace_dir = Path(__file__).parent.parent
-    device = torch.device('cuda:1')
+    device = torch.device('cuda:0')
 
-    checkpoint = workspace_dir / ".checkpoints/best_wer_model.pt"
-    inference = ASRInference(model_path=checkpoint, vocab_path=workspace_dir / 'config/asr_vocab.json' , device=device)
+    # checkpoint = sorted(workspace_dir.glob('.checkpoints/checkpoint_epoch_*.pt'), key=lambda p: int(p.stem.split('_')[1]))[-1]
+    checkpoint = workspace_dir / ".checkpoints/checkpoint_step_9500.pt"
+    print(f"Load Checkpoint: {checkpoint}")
+
+    inference = ASRInference(model_path=checkpoint, vocab_path=workspace_dir / 'config/uig_vocab.json' , device=device)
 
     audio_path = workspace_dir / 'data/test/F001_001.wav'
-    waveform = inference._load_audio(audio_path=audio_path)
-    text = inference.transcribe(waveform=waveform)
-    print("transcribe:", text)
-    inference.create(audio_path=audio_path)
+    print(f"\n转录音频: {audio_path}")
+    text = inference.transcribe(audio_path=audio_path)
+    print(f"\n识别结果: {text}")
 
 if __name__ == "__main__":
     main()
+# workspace_dir = Path(__file__).parent.parent
+# checkpoint = sorted(workspace_dir.glob('.checkpoints/checkpoint_step_*.pt'), key=lambda p: int(p.stem.split('_')[-1]))[-1]
+
+# print(checkpoint)

src/model.py

@@ -1,77 +1,36 @@
-from torch.nn import Module, Linear, functional, GELU, Conv1d, GroupNorm, ModuleList
+from torch.nn import Conv2d, Module, ReLU, Linear, functional
 from torch import Tensor
 from torchaudio.models import Conformer
 
-class WaveformFilter(Module):
-    def __init__(self):
-        super().__init__()
-        
-        self.filters = ModuleList([
-            Conv1d(in_channels=1, out_channels=512, kernel_size=10, stride=5),
-            GroupNorm(num_channels=512, num_groups=32),
-            GELU(),
-            Conv1d(in_channels=512, out_channels=512, kernel_size=3, stride=2),
-            GroupNorm(num_channels=512, num_groups=32),
-            GELU(),
-            Conv1d(in_channels=512, out_channels=512, kernel_size=3, stride=2),
-            GroupNorm(num_channels=512, num_groups=32),
-            GELU(),
-            Conv1d(in_channels=512, out_channels=512, kernel_size=3, stride=2),
-            GroupNorm(num_channels=512, num_groups=32),
-            GELU(),
-            Conv1d(in_channels=512, out_channels=512, kernel_size=3, stride=2),
-            GroupNorm(num_channels=512, num_groups=32),
-            GELU(),
-            Conv1d(in_channels=512, out_channels=512, kernel_size=2, stride=2),
-            GroupNorm(num_channels=512, num_groups=32),
-            GELU(),
-            Conv1d(in_channels=512, out_channels=512, kernel_size=2, stride=2),
-            GroupNorm(num_channels=512, num_groups=32),
-            GELU(),
-        ])
-    
-    def compute_lengths(self, waveform_lengths: Tensor) -> Tensor:
-        """Accurately compute output lengths after all convolutions."""
-        lengths = waveform_lengths
-        for module in self.filters:
-            if isinstance(module, Conv1d):
-                # Conv1d with padding=0, dilation=1:
-                # output = floor((input - kernel_size) / stride) + 1
-                lengths = (lengths - module.kernel_size[0]) // module.stride[0] + 1
-        return lengths
-    
-    def forward(self, x: Tensor) -> Tensor:
-        for filter in self.filters:
-            x = filter(x)
-        
-        return x
-
 
 class ASRModel(Module):
-    def __init__(self, vocab_size: int, input_dim: int = 256, num_heads: int = 8, ffn_dim: int = 2048, num_layers: int = 6, dropout: float = 0.1) -> None:
+    def __init__(self, vocab_size: int, input_dim: int = 640, num_heads: int = 8, ffn_dim: int = 2048, num_layers: int = 6, dropout: float = 0.1) -> None:
         super().__init__()
 
-        self.feature_extractor = WaveformFilter()
-        self.proj = Linear(in_features=512, out_features=input_dim, bias=False)
+        self.conv1 = Conv2d(in_channels=1, out_channels=16, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1))
+        self.conv2 = Conv2d(in_channels=16, out_channels=32, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1))
+        self.conv3 = Conv2d(in_channels=32, out_channels=32, kernel_size=(3, 3), stride=(2, 1), padding=(1, 1))
+        self.proj = Linear(in_features=1280, out_features=640)
+        self.relu = ReLU()
 
         self.encoder = Conformer(input_dim=input_dim, num_heads=num_heads, ffn_dim=ffn_dim, num_layers=num_layers, depthwise_conv_kernel_size=31, dropout=dropout)
 
         self.ctc_head = Linear(in_features=input_dim, out_features=vocab_size, bias=False)
 
-    def forward(self, waveforms: Tensor, waveform_lengths: Tensor) -> tuple[Tensor, Tensor]:
-        assert len(waveform_lengths.shape) == 1, "The waveform_lengths tensor must be shape of [B] tensor."
-        assert len(waveforms.shape) == 2, "The waveform tensor must be [B, time] tensor"
-        # waveforms: [B, time]
-        x: Tensor = waveforms.unsqueeze(1)  # [B, 1, time]
-        #x: [B, 1, time]
-        x: Tensor = self.feature_extractor(x)
-        #x: [B, 512, time]
-        x = x.permute(0, 2, 1)
-        #x: [B, time, 512]
+    def forward(self, mel_specs: Tensor, mel_lengths: Tensor) -> tuple[Tensor, Tensor]:
+        # mel_specs: [B, n_mels, time]
+        x: Tensor = mel_specs.unsqueeze(1)  # [B, 1, n_mels, time]
+
+        x = self.relu(self.conv1(x))  # [batch, 16, n_mels/2, time/2]
+        x = self.relu(self.conv2(x))  # [batch, 32, n_mels/4, time/4]
+        x = self.relu(self.conv3(x))  # [batch, 32, n_mels/8, time/4]
+
+        # [B, channels, freq, time] → [B, time, channels*freq]
+        batch, channels, freq, time = x.shape
+        x = x.permute(0, 3, 1, 2).reshape(batch, time, channels * freq)
         x = self.proj(x)
-        #x: [B, time, 256]
         # lengths = torch.tensor([time] * batch, dtype=torch.long, device=x.device)
-        lengths = self.feature_extractor.compute_lengths(waveform_lengths)
+        lengths = ((mel_lengths + 1) // 2 + 1) // 2  # 两层 stride=2
 
         x, lengths = self.encoder(x, lengths)
 

src/train.py

@@ -1,5 +1,3 @@
-import math
-
 import torch
 from torch import Tensor, device, no_grad, cuda
 from torch.optim import AdamW, Optimizer
@@ -19,16 +17,16 @@ from model import ASRModel
 # ============ 全局配置 ============
 CONFIG = {
     # 数据配置
-    'batch_size': 32,
+    'batch_size': 96,
     
     # 训练配置
     'num_epochs': 50,
-    'learning_rate': 2e-4,
+    'learning_rate': 1e-4,
     'weight_decay': 1e-4,
     'grad_clip_norm': 1.0,
     
     # 模型配置
-    'input_dim': 256,
+    'input_dim': 640,
     'num_heads': 8,
     'ffn_dim': 2048,
     'num_layers': 8,
@@ -93,15 +91,15 @@ def train_one_epoch(model: ASRModel, dataloader: DataLoader, criterion: CTCLoss,
     progress_bar = tqdm(dataloader, desc=f"Epoch {epoch}")
     for batch_index, batch in enumerate(progress_bar):
         batch: Batch
-        waveforms = batch['waveforms'].to(device)
+        mel_specs = batch['mel_specs'].to(device)
         targets = batch['targets'].to(device)
-        waveform_lengths = batch['waveform_lengths'].to(device)
+        mel_lengths = batch['mel_lengths'].to(device)
         target_lengths = batch['target_lengths'].to(device)
 
         optimizer.zero_grad()
 
         with torch.autocast(device_type='cuda', dtype=torch.bfloat16):
-            log_probs, lengths = model(waveforms=waveforms, waveform_lengths=waveform_lengths)
+            log_probs, lengths = model(mel_specs=mel_specs, mel_lengths=mel_lengths)
             log_probs: Tensor
             log_probs_ctc = log_probs.permute(1, 0, 2)
             loss: Tensor = criterion(log_probs=log_probs_ctc, targets=targets, input_lengths=lengths, target_lengths=target_lengths)
@@ -137,13 +135,13 @@ def validate(model: ASRModel, dataloader: DataLoader, criterion: CTCLoss, device
         progress_bar = tqdm(dataloader, desc="Validate", leave=False)
         for batch in progress_bar:
             batch: Batch
-            waveforms = batch['waveforms'].to(device)
+            mel_specs = batch['mel_specs'].to(device)
             targets = batch['targets'].to(device)
-            waveform_lengths = batch['waveform_lengths'].to(device)
+            mel_lengths = batch['mel_lengths'].to(device)
             target_lengths = batch['target_lengths'].to(device)
 
             with torch.autocast(device_type='cuda', dtype=torch.bfloat16):
-                log_probs, lengths = model(waveforms=waveforms, waveform_lengths=waveform_lengths)
+                log_probs, lengths = model(mel_specs=mel_specs, mel_lengths=mel_lengths)
                 log_probs: Tensor
                 log_probs_ctc = log_probs.permute(1, 0, 2)
                 loss: Tensor = criterion(log_probs=log_probs_ctc, targets=targets, input_lengths=lengths, target_lengths=target_lengths)
@@ -212,26 +210,20 @@ def main():
     workspace_dir = Path(__file__).parent.parent
     device = torch.device('cuda:0')
     tokenizer = ASRTokenizer(workspace_dir / 'config/asr_vocab.json')
-    final_prob = 0.5
-    warmup_epochs = 8
-    current_prob = 0.0
 
     # ============ 创建数据加载器 ============
     train_loader = create_dataloader(
-        tsv_path=workspace_dir / '.data/ug/train_new.tsv',
+        tsv_path=workspace_dir / '.data/ug/train.tsv',
         audio_dir=workspace_dir / '.data/ug/clips',
-        noise_dir='/mnt/dataset/dataset/audio/noise',
         tokenizer=tokenizer,
         batch_size=CONFIG['batch_size'],
         shuffle=True,
-        augment=True,
-        augment_prob=current_prob
+        augment=True
     )
     
     val_loader = create_dataloader(
-        tsv_path=workspace_dir / '.data/ug/val_new.tsv',
+        tsv_path=workspace_dir / '.data/ug/dev.tsv',
         audio_dir=workspace_dir / '.data/ug/clips',
-        noise_dir='/mnt/dataset/dataset/audio/noise',
         tokenizer=tokenizer,
         batch_size=CONFIG['batch_size'],
         shuffle=False,
@@ -259,9 +251,9 @@ def main():
         max_lr=CONFIG['learning_rate'],
         epochs=CONFIG['num_epochs'],
         steps_per_epoch=len(train_loader),
-        pct_start=0.15,
+        pct_start=0.1,  # 前 10% 步数用于 warmup
         anneal_strategy='cos',
-        div_factor=10.0,  # 初始 lr = max_lr / 10
+        div_factor=25.0,  # 初始 lr = max_lr / 25
         final_div_factor=1e4,  # 最终 lr = max_lr / 10000
     )
     scaler = GradScaler()
@@ -294,7 +286,6 @@ def main():
         print("🆕 从头开始训练\n")
 
     for epoch in range(start_epoch, CONFIG['num_epochs']):
-        current_prob = final_prob * (1 - math.cos(math.pi * epoch / warmup_epochs)) / 2
         train_loss, global_step = train_one_epoch(
             model=model, 
             dataloader=train_loader, 

uv.lock

@@ -6,10 +6,10 @@ resolution-markers = [
     "python_full_version >= '3.14' and sys_platform == 'emscripten'",
     "python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'",
     "python_full_version == '3.13.*' and sys_platform == 'win32'",
-    "python_full_version == '3.13.*' and sys_platform == 'emscripten'",
-    "python_full_version == '3.13.*' and sys_platform != 'emscripten' and sys_platform != 'win32'",
     "python_full_version == '3.12.*' and sys_platform == 'win32'",
+    "python_full_version == '3.13.*' and sys_platform == 'emscripten'",
     "python_full_version == '3.12.*' and sys_platform == 'emscripten'",
+    "python_full_version == '3.13.*' and sys_platform != 'emscripten' and sys_platform != 'win32'",
     "python_full_version == '3.12.*' and sys_platform != 'emscripten' and sys_platform != 'win32'",
     "python_full_version < '3.12' and sys_platform == 'win32'",
     "python_full_version < '3.12' and sys_platform == 'emscripten'",
@@ -449,11 +449,11 @@ wheels = [
 
 [[package]]
 name = "fsspec"
-version = "2026.4.0"
+version = "2026.3.0"
 source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/d5/8d/1c51c094345df128ca4a990d633fe1a0ff28726c9e6b3c41ba65087bba1d/fsspec-2026.4.0.tar.gz", hash = "sha256:301d8ac70ae90ef3ad05dcf94d6c3754a097f9b5fe4667d2787aa359ec7df7e4", size = 312760, upload-time = "2026-04-29T20:42:38.635Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/e1/cf/b50ddf667c15276a9ab15a70ef5f257564de271957933ffea49d2cdbcdfb/fsspec-2026.3.0.tar.gz", hash = "sha256:1ee6a0e28677557f8c2f994e3eea77db6392b4de9cd1f5d7a9e87a0ae9d01b41", size = 313547, upload-time = "2026-03-27T19:11:14.892Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/d5/0c/043d5e551459da400957a1395e0febbf771446ff34291afcbe3d8be2a279/fsspec-2026.4.0-py3-none-any.whl", hash = "sha256:11ef7bb35dab8a394fde6e608221d5cf3e8499401c249bebaeaad760a1a8dec2", size = 203402, upload-time = "2026-04-29T20:42:36.842Z" },
+    { url = "https://files.pythonhosted.org/packages/d5/1f/5f4a3cd9e4440e9d9bc78ad0a91a1c8d46b4d429d5239ebe6793c9fe5c41/fsspec-2026.3.0-py3-none-any.whl", hash = "sha256:d2ceafaad1b3457968ed14efa28798162f1638dbb5d2a6868a2db002a5ee39a4", size = 202595, upload-time = "2026-03-27T19:11:13.595Z" },
 ]
 
 [[package]]
@@ -553,14 +553,14 @@ wheels = [
 
 [[package]]
 name = "jedi"
-version = "0.20.0"
+version = "0.19.2"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
     { name = "parso" },
 ]
-sdist = { url = "https://files.pythonhosted.org/packages/46/b7/a3635f6a2d7cf5b5dd98064fc1d5fbbafcb25477bcea204a3a92145d158b/jedi-0.20.0.tar.gz", hash = "sha256:c3f4ccbd276696f4b19c54618d4fb18f9fc24b0aef02acf704b23f487daa1011", size = 3119416, upload-time = "2026-05-01T23:38:47.814Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/72/3a/79a912fbd4d8dd6fbb02bf69afd3bb72cf0c729bb3063c6f4498603db17a/jedi-0.19.2.tar.gz", hash = "sha256:4770dc3de41bde3966b02eb84fbcf557fb33cce26ad23da12c742fb50ecb11f0", size = 1231287, upload-time = "2024-11-11T01:41:42.873Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/9a/93/242e2eab5fe682ffcb8b0084bde703a41d51e17ee0f3a31ff0d9d813620a/jedi-0.20.0-py2.py3-none-any.whl", hash = "sha256:7bdd9c2634f56713299976f4cbd59cb3fa92165cc5e05ea811fb253480728b67", size = 4884812, upload-time = "2026-05-01T23:38:43.919Z" },
+    { url = "https://files.pythonhosted.org/packages/c0/5a/9cac0c82afec3d09ccd97c8b6502d48f165f9124db81b4bcb90b4af974ee/jedi-0.19.2-py2.py3-none-any.whl", hash = "sha256:a8ef22bde8490f57fe5c7681a3c83cb58874daf72b4784de3cce5b6ef6edb5b9", size = 1572278, upload-time = "2024-11-11T01:41:40.175Z" },
 ]
 
 [[package]]
@@ -584,15 +584,6 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/7b/91/984aca2ec129e2757d1e4e3c81c3fcda9d0f85b74670a094cc443d9ee949/joblib-1.5.3-py3-none-any.whl", hash = "sha256:5fc3c5039fc5ca8c0276333a188bbd59d6b7ab37fe6632daa76bc7f9ec18e713", size = 309071, upload-time = "2025-12-15T08:41:44.973Z" },
 ]
 
-[[package]]
-name = "julius"
-version = "0.2.7"
-source = { registry = "https://pypi.org/simple" }
-dependencies = [
-    { name = "torch" },
-]
-sdist = { url = "https://files.pythonhosted.org/packages/a1/19/c9e1596b5572c786b93428d0904280e964c930fae7e6c9368ed9e1b63922/julius-0.2.7.tar.gz", hash = "sha256:3c0f5f5306d7d6016fcc95196b274cae6f07e2c9596eed314e4e7641554fbb08", size = 59640, upload-time = "2022-09-19T16:13:34.2Z" }
-
 [[package]]
 name = "kiwisolver"
 version = "1.5.0"
@@ -986,15 +977,6 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/81/f2/08ace4142eb281c12701fc3b93a10795e4d4dc7f753911d836675050f886/msgpack-1.1.2-cp314-cp314t-win_arm64.whl", hash = "sha256:d99ef64f349d5ec3293688e91486c5fdb925ed03807f64d98d205d2713c60b46", size = 70868, upload-time = "2025-10-08T09:15:44.959Z" },
 ]
 
-[[package]]
-name = "mutagen"
-version = "1.47.0"
-source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/81/e6/64bc71b74eef4b68e61eb921dcf72dabd9e4ec4af1e11891bbd312ccbb77/mutagen-1.47.0.tar.gz", hash = "sha256:719fadef0a978c31b4cf3c956261b3c58b6948b32023078a2117b1de09f0fc99", size = 1274186, upload-time = "2023-09-03T16:33:33.411Z" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/b0/7a/620f945b96be1f6ee357d211d5bf74ab1b7fe72a9f1525aafbfe3aee6875/mutagen-1.47.0-py3-none-any.whl", hash = "sha256:edd96f50c5907a9539d8e5bba7245f62c9f520aef333d13392a79a4f70aca719", size = 194391, upload-time = "2023-09-03T16:33:29.955Z" },
-]
-
 [[package]]
 name = "networkx"
 version = "3.6.1"
@@ -1312,11 +1294,11 @@ wheels = [
 
 [[package]]
 name = "parso"
-version = "0.8.7"
+version = "0.8.6"
 source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/30/4b/90c937815137d43ce71ba043cd3566221e9df6b9c805f24b5d138c9d40a7/parso-0.8.7.tar.gz", hash = "sha256:eaaac4c9fdd5e9e8852dc778d2d7405897ec510f2a298071453e5e3a07914bb1", size = 401824, upload-time = "2026-05-01T23:13:02.138Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/81/76/a1e769043c0c0c9fe391b702539d594731a4362334cdf4dc25d0c09761e7/parso-0.8.6.tar.gz", hash = "sha256:2b9a0332696df97d454fa67b81618fd69c35a7b90327cbe6ba5c92d2c68a7bfd", size = 401621, upload-time = "2026-02-09T15:45:24.425Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/99/5d/8268b644392ee874ee82a635cd0df1773de230bde356c38de28e298392cc/parso-0.8.7-py2.py3-none-any.whl", hash = "sha256:a8926eb2a1b915486941fdbd31e86a4baf88fe8c210f25f2f35ecec5b574ca1c", size = 107025, upload-time = "2026-05-01T23:12:58.867Z" },
+    { url = "https://files.pythonhosted.org/packages/b6/61/fae042894f4296ec49e3f193aff5d7c18440da9e48102c3315e1bc4519a7/parso-0.8.6-py2.py3-none-any.whl", hash = "sha256:2c549f800b70a5c4952197248825584cb00f033b29c692671d3bf08bf380baff", size = 106894, upload-time = "2026-02-09T15:45:21.391Z" },
 ]
 
 [[package]]
@@ -1441,15 +1423,6 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/2a/2d/d4bf65e47cea8ff2c794a600c4fd1273a7902f268757c531e0ee9f18aa58/pooch-1.9.0-py3-none-any.whl", hash = "sha256:f265597baa9f760d25ceb29d0beb8186c243d6607b0f60b83ecf14078dbc703b", size = 67175, upload-time = "2026-01-30T19:15:08.36Z" },
 ]
 
-[[package]]
-name = "primepy"
-version = "1.3"
-source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/35/77/0cfa1b4697cfb5336f3a96e8bc73327f64610be3a64c97275f1801afb395/primePy-1.3.tar.gz", hash = "sha256:25fd7e25344b0789a5984c75d89f054fcf1f180bef20c998e4befbac92de4669", size = 3914, upload-time = "2018-05-29T17:18:18.683Z" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/74/c1/bb7e334135859c3a92ec399bc89293ea73f28e815e35b43929c8db6af030/primePy-1.3-py3-none-any.whl", hash = "sha256:5ed443718765be9bf7e2ff4c56cdff71b42140a15b39d054f9d99f0009e2317a", size = 4040, upload-time = "2018-05-29T17:18:17.53Z" },
-]
-
 [[package]]
 name = "prompt-toolkit"
 version = "3.0.52"
@@ -1532,15 +1505,6 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/0c/c3/44f3fbbfa403ea2a7c779186dc20772604442dde72947e7d01069cbe98e3/pycparser-3.0-py3-none-any.whl", hash = "sha256:b727414169a36b7d524c1c3e31839a521725078d7b2ff038656844266160a992", size = 48172, upload-time = "2026-01-21T14:26:50.693Z" },
 ]
 
-[[package]]
-name = "pydub"
-version = "0.25.1"
-source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/fe/9a/e6bca0eed82db26562c73b5076539a4a08d3cffd19c3cc5913a3e61145fd/pydub-0.25.1.tar.gz", hash = "sha256:980a33ce9949cab2a569606b65674d748ecbca4f0796887fd6f46173a7b0d30f", size = 38326, upload-time = "2021-03-10T02:09:54.659Z" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/a6/53/d78dc063216e62fc55f6b2eebb447f6a4b0a59f55c8406376f76bf959b08/pydub-0.25.1-py2.py3-none-any.whl", hash = "sha256:65617e33033874b59d87db603aa1ed450633288aefead953b30bded59cb599a6", size = 32327, upload-time = "2021-03-10T02:09:53.503Z" },
-]
-
 [[package]]
 name = "pygments"
 version = "2.20.0"
@@ -1729,20 +1693,6 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/e1/e3/c164c88b2e5ce7b24d667b9bd83589cf4f3520d97cad01534cd3c4f55fdb/setuptools-81.0.0-py3-none-any.whl", hash = "sha256:fdd925d5c5d9f62e4b74b30d6dd7828ce236fd6ed998a08d81de62ce5a6310d6", size = 1062021, upload-time = "2026-02-06T21:10:37.175Z" },
 ]
 
-[[package]]
-name = "silero-vad"
-version = "6.2.1"
-source = { registry = "https://pypi.org/simple" }
-dependencies = [
-    { name = "packaging" },
-    { name = "torch" },
-    { name = "torchaudio" },
-]
-sdist = { url = "https://files.pythonhosted.org/packages/32/d3/e31f526482782764aa4f70e20fd4545cf2e4a81a60b6fb0f089f6d107991/silero_vad-6.2.1.tar.gz", hash = "sha256:b23062b0e39fad17b1266fc23c1e7b4290219dbe82ce08510889e32f681f4b3b", size = 28913811, upload-time = "2026-02-24T08:41:59.329Z" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/0b/2b/48566f29a8b53d856ceb1994f209122749b3fda0a733a07e82047257de7a/silero_vad-6.2.1-py3-none-any.whl", hash = "sha256:09de93c4d874bb19c53e62a47dd38be5f163cedad2b5599583231f2a84ef79cb", size = 9146242, upload-time = "2026-02-24T08:41:56.955Z" },
-]
-
 [[package]]
 name = "six"
 version = "1.17.0"
@@ -1773,28 +1723,28 @@ wheels = [
 
 [[package]]
 name = "soxr"
-version = "1.1.0"
+version = "1.0.0"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
     { name = "numpy" },
 ]
-sdist = { url = "https://files.pythonhosted.org/packages/ed/11/27cebce4a108f77afea7c80545115536b45e3f11ebfb914f638fdd9ba847/soxr-1.1.0.tar.gz", hash = "sha256:9f228ae21c78fa9359ca98d8a5e8e91f30639e438e574133dace62c5b5309e44", size = 173067, upload-time = "2026-05-03T00:15:18.214Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/42/7e/f4b461944662ad75036df65277d6130f9411002bfb79e9df7dff40a31db9/soxr-1.0.0.tar.gz", hash = "sha256:e07ee6c1d659bc6957034f4800c60cb8b98de798823e34d2a2bba1caa85a4509", size = 171415, upload-time = "2025-09-07T13:22:21.317Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/8e/49/3e6bc84f87439f222f40b616e9a29a170f41fb564710ea510df19dc26907/soxr-1.1.0-cp311-cp311-macosx_10_14_x86_64.whl", hash = "sha256:34cc92208c3c412c046813e69da639c04a792c6a41fbfd7d909d359cd3e97a2d", size = 205699, upload-time = "2026-05-03T00:14:46.67Z" },
-    { url = "https://files.pythonhosted.org/packages/2f/94/216f46096a85b07d1e6ba7fd44491402e912a3d688cd4f36f0a600ca155f/soxr-1.1.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:bd30f7201eac896ebf5db7b09156e6f1a1b82601900d29d9c8449bdad8365b11", size = 167381, upload-time = "2026-05-03T00:14:48.012Z" },
-    { url = "https://files.pythonhosted.org/packages/94/cb/06caa463b8181ec1981bd6376d4a873748b7008193188b8cfb60391eb131/soxr-1.1.0-cp311-cp311-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1577865e993f98ffb261257c3060fa76ec3db44ed3f181b16464268000424464", size = 210938, upload-time = "2026-05-03T00:14:49.768Z" },
-    { url = "https://files.pythonhosted.org/packages/86/47/d5964551ca818b7f0c7ef7f3899056263b60ef098a801066350a9672ca8f/soxr-1.1.0-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3da87e3ffa3e41823d873b051c7ecb2acebd8d1b6b46b752f5facf10a0d84ab9", size = 245268, upload-time = "2026-05-03T00:14:51.422Z" },
-    { url = "https://files.pythonhosted.org/packages/8f/29/371467eb86c7ba6810df0bfe9409bcd9c52ec5615b111190fafe23e4d2e1/soxr-1.1.0-cp311-cp311-win_amd64.whl", hash = "sha256:ae30c48ac795378cf23ba3c7c640b8ff794af714ac388b9fd6b31a40b39e6e86", size = 176779, upload-time = "2026-05-03T00:14:53.09Z" },
-    { url = "https://files.pythonhosted.org/packages/06/8a/f3da7973b5f1b05d2d7e94d5376b881dcbc05297900cae6c3d33d95b209b/soxr-1.1.0-cp312-abi3-macosx_10_14_x86_64.whl", hash = "sha256:e0e09fa633ce2e67df08b298afced4d184f6e753fc330f241022250f1d0d61da", size = 204124, upload-time = "2026-05-03T00:14:54.505Z" },
-    { url = "https://files.pythonhosted.org/packages/03/dc/200013a74641f8774664bbcd2346c695c05c2e300ea792adcb40a293eed0/soxr-1.1.0-cp312-abi3-macosx_11_0_arm64.whl", hash = "sha256:d6a7ad82b8d5f3fcc04b1d2ca055562b96af571e1d4fa7c6c61d0fb509ac43b4", size = 165457, upload-time = "2026-05-03T00:14:56.007Z" },
-    { url = "https://files.pythonhosted.org/packages/88/2b/2e5eba817a762a2ec589ff165b8bc5955b25a0ad140045f7cd8e45410543/soxr-1.1.0-cp312-abi3-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bf98c0d7b7d5ef5bf072fee8d3020e8b664f2d195933ea7bc5089267c2e22a06", size = 206529, upload-time = "2026-05-03T00:14:57.646Z" },
-    { url = "https://files.pythonhosted.org/packages/5c/f1/0e55195893228609c9a08c3b13b7a83a46c3a992cd00d3304f0f320cfb07/soxr-1.1.0-cp312-abi3-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3b033078e86f3c4a658e5697fac8995764fad9e799563616b630136b613167f1", size = 240413, upload-time = "2026-05-03T00:14:59.363Z" },
-    { url = "https://files.pythonhosted.org/packages/b0/4d/621e4150e4815246ad552d215a8a294a90143fedd19ee442cf82d3b3abc8/soxr-1.1.0-cp312-abi3-win_amd64.whl", hash = "sha256:6ae2a174bffea94e8ead857dad85999d3f49f091774dbad5b046c0417d7092f4", size = 174357, upload-time = "2026-05-03T00:15:00.724Z" },
-    { url = "https://files.pythonhosted.org/packages/76/cd/77b74f1e95af0e11e52e9a034421aece7f7b45afd15a909afd41d5a5d102/soxr-1.1.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:a941f5aaa0b8abced24318105c1ea22576afcc1138c19f625716ce4e2f76ad64", size = 207990, upload-time = "2026-05-03T00:15:02.1Z" },
-    { url = "https://files.pythonhosted.org/packages/30/86/600cc31f982288167a59972746f117790162012546f995a32b5a55394b16/soxr-1.1.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:feebcba99ac99adb8009d46c8f4c1956b8c167576b0ae8a6fb47502e9a6f78e7", size = 169288, upload-time = "2026-05-03T00:15:03.75Z" },
-    { url = "https://files.pythonhosted.org/packages/39/e4/80cd9aae0645513db1076d4384e8b2d895faf5009218b4a04348012c54fc/soxr-1.1.0-cp314-cp314t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:52c9ca84e3dc656d83acc424574770e20ea8e0704dc3842d4e27b0fe9d3ba449", size = 211405, upload-time = "2026-05-03T00:15:05.395Z" },
-    { url = "https://files.pythonhosted.org/packages/a6/d6/cc3c80ac9b2289da4cf46c5d53b05e4327e6f5560a25868d06f9e2213af1/soxr-1.1.0-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f4977323ef9c3aa3c2a26ff5fe0191c84b8fd759daf7afb1f25a91a55ad8b730", size = 244617, upload-time = "2026-05-03T00:15:07.134Z" },
-    { url = "https://files.pythonhosted.org/packages/d3/9e/f7af5fae841ffe32ed8440234ea2ad6adecca3bd92b6101076268c429000/soxr-1.1.0-cp314-cp314t-win_amd64.whl", hash = "sha256:e17d4ef9b0185214b2c0935605ae63f827ea423bc74964be44763d68d2b6c21e", size = 187253, upload-time = "2026-05-03T00:15:08.813Z" },
+    { url = "https://files.pythonhosted.org/packages/65/ce/a3262bc8733d3a4ce5f660ed88c3d97f4b12658b0909e71334cba1721dcb/soxr-1.0.0-cp311-cp311-macosx_10_14_x86_64.whl", hash = "sha256:28e19d74a5ef45c0d7000f3c70ec1719e89077379df2a1215058914d9603d2d8", size = 206739, upload-time = "2025-09-07T13:21:54.572Z" },
+    { url = "https://files.pythonhosted.org/packages/64/dc/e8cbd100b652697cc9865dbed08832e7e135ff533f453eb6db9e6168d153/soxr-1.0.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f8dc69fc18884e53b72f6141fdf9d80997edbb4fec9dc2942edcb63abbe0d023", size = 165233, upload-time = "2025-09-07T13:21:55.887Z" },
+    { url = "https://files.pythonhosted.org/packages/75/12/4b49611c9ba5e9fe6f807d0a83352516808e8e573f8b4e712fc0c17f3363/soxr-1.0.0-cp311-cp311-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3f15450e6f65f22f02fcd4c5a9219c873b1e583a73e232805ff160c759a6b586", size = 208867, upload-time = "2025-09-07T13:21:57.076Z" },
+    { url = "https://files.pythonhosted.org/packages/cc/70/92146ab970a3ef8c43ac160035b1e52fde5417f89adb10572f7e788d9596/soxr-1.0.0-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1f73f57452f9df37b4de7a4052789fcbd474a5b28f38bba43278ae4b489d4384", size = 242633, upload-time = "2025-09-07T13:21:58.621Z" },
+    { url = "https://files.pythonhosted.org/packages/b5/a7/628479336206959463d08260bffed87905e7ba9e3bd83ca6b405a0736e94/soxr-1.0.0-cp311-cp311-win_amd64.whl", hash = "sha256:9f417c3d69236051cf5a1a7bad7c4bff04eb3d8fcaa24ac1cb06e26c8d48d8dc", size = 173814, upload-time = "2025-09-07T13:21:59.798Z" },
+    { url = "https://files.pythonhosted.org/packages/c5/c7/f92b81f1a151c13afb114f57799b86da9330bec844ea5a0d3fe6a8732678/soxr-1.0.0-cp312-abi3-macosx_10_14_x86_64.whl", hash = "sha256:abecf4e39017f3fadb5e051637c272ae5778d838e5c3926a35db36a53e3a607f", size = 205508, upload-time = "2025-09-07T13:22:01.252Z" },
+    { url = "https://files.pythonhosted.org/packages/ff/1d/c945fea9d83ea1f2be9d116b3674dbaef26ed090374a77c394b31e3b083b/soxr-1.0.0-cp312-abi3-macosx_11_0_arm64.whl", hash = "sha256:e973d487ee46aa8023ca00a139db6e09af053a37a032fe22f9ff0cc2e19c94b4", size = 163568, upload-time = "2025-09-07T13:22:03.558Z" },
+    { url = "https://files.pythonhosted.org/packages/b5/80/10640970998a1d2199bef6c4d92205f36968cddaf3e4d0e9fe35ddd405bd/soxr-1.0.0-cp312-abi3-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e8ce273cca101aff3d8c387db5a5a41001ba76ef1837883438d3c652507a9ccc", size = 204707, upload-time = "2025-09-07T13:22:05.125Z" },
+    { url = "https://files.pythonhosted.org/packages/b1/87/2726603c13c2126cb8ded9e57381b7377f4f0df6ba4408e1af5ddbfdc3dd/soxr-1.0.0-cp312-abi3-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e8f2a69686f2856d37823bbb7b78c3d44904f311fe70ba49b893af11d6b6047b", size = 238032, upload-time = "2025-09-07T13:22:06.428Z" },
+    { url = "https://files.pythonhosted.org/packages/ce/04/530252227f4d0721a5524a936336485dfb429bb206a66baf8e470384f4a2/soxr-1.0.0-cp312-abi3-win_amd64.whl", hash = "sha256:2a3b77b115ae7c478eecdbd060ed4f61beda542dfb70639177ac263aceda42a2", size = 172070, upload-time = "2025-09-07T13:22:07.62Z" },
+    { url = "https://files.pythonhosted.org/packages/99/77/d3b3c25b4f1b1aa4a73f669355edcaee7a52179d0c50407697200a0e55b9/soxr-1.0.0-cp314-cp314t-macosx_10_14_x86_64.whl", hash = "sha256:392a5c70c04eb939c9c176bd6f654dec9a0eaa9ba33d8f1024ed63cf68cdba0a", size = 209509, upload-time = "2025-09-07T13:22:08.773Z" },
+    { url = "https://files.pythonhosted.org/packages/8a/ee/3ca73e18781bb2aff92b809f1c17c356dfb9a1870652004bd432e79afbfa/soxr-1.0.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:fdc41a1027ba46777186f26a8fba7893be913383414135577522da2fcc684490", size = 167690, upload-time = "2025-09-07T13:22:10.259Z" },
+    { url = "https://files.pythonhosted.org/packages/bd/f0/eea8b5f587a2531657dc5081d2543a5a845f271a3bea1c0fdee5cebde021/soxr-1.0.0-cp314-cp314t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:449acd1dfaf10f0ce6dfd75c7e2ef984890df94008765a6742dafb42061c1a24", size = 209541, upload-time = "2025-09-07T13:22:11.739Z" },
+    { url = "https://files.pythonhosted.org/packages/64/59/2430a48c705565eb09e78346950b586f253a11bd5313426ced3ecd9b0feb/soxr-1.0.0-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:38b35c99e408b8f440c9376a5e1dd48014857cd977c117bdaa4304865ae0edd0", size = 243025, upload-time = "2025-09-07T13:22:12.877Z" },
+    { url = "https://files.pythonhosted.org/packages/3c/1b/f84a2570a74094e921bbad5450b2a22a85d58585916e131d9b98029c3e69/soxr-1.0.0-cp314-cp314t-win_amd64.whl", hash = "sha256:a39b519acca2364aa726b24a6fd55acf29e4c8909102e0b858c23013c38328e5", size = 184850, upload-time = "2025-09-07T13:22:14.068Z" },
 ]
 
 [[package]]
@@ -1853,22 +1803,17 @@ dependencies = [
     { name = "ipython" },
     { name = "librosa" },
     { name = "matplotlib" },
-    { name = "mutagen" },
     { name = "numpy" },
     { name = "pandas" },
     { name = "pillow" },
-    { name = "pydub" },
     { name = "pyrubberband" },
     { name = "setuptools" },
-    { name = "silero-vad" },
     { name = "tensorboard" },
     { name = "tensorboardx" },
     { name = "torch" },
-    { name = "torch-audiomentations" },
     { name = "torchaudio" },
     { name = "torchcodec" },
     { name = "tqdm" },
-    { name = "webrtcvad" },
 ]
 
 [package.metadata]
@@ -1876,22 +1821,17 @@ requires-dist = [
     { name = "ipython", specifier = ">=9.10.1" },
     { name = "librosa", specifier = ">=0.11.0" },
     { name = "matplotlib", specifier = ">=3.10.8" },
-    { name = "mutagen", specifier = ">=1.47.0" },
     { name = "numpy", specifier = ">=2.4.4" },
     { name = "pandas", specifier = ">=3.0.2" },
     { name = "pillow", specifier = ">=12.2.0" },
-    { name = "pydub", specifier = ">=0.25.1" },
     { name = "pyrubberband", specifier = ">=0.4.0" },
     { name = "setuptools", specifier = "<82" },
-    { name = "silero-vad", specifier = ">=6.2.1" },
     { name = "tensorboard", specifier = ">=2.20.0" },
     { name = "tensorboardx", specifier = ">=2.6.5" },
     { name = "torch", specifier = "==2.8.0" },
-    { name = "torch-audiomentations", specifier = ">=0.12.0" },
     { name = "torchaudio", specifier = "==2.8.0" },
     { name = "torchcodec", specifier = "==0.7.0" },
     { name = "tqdm", specifier = ">=4.67.3" },
-    { name = "webrtcvad", specifier = ">=2.0.10" },
 ]
 
 [[package]]
@@ -2006,36 +1946,6 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/04/6e/650bb7f28f771af0cb791b02348db8b7f5f64f40f6829ee82aa6ce99aabe/torch-2.8.0-cp313-none-macosx_11_0_arm64.whl", hash = "sha256:7b677e17f5a3e69fdef7eb3b9da72622f8d322692930297e4ccb52fefc6c8211", size = 73632395, upload-time = "2025-08-06T14:55:28.645Z" },
 ]
 
-[[package]]
-name = "torch-audiomentations"
-version = "0.12.0"
-source = { registry = "https://pypi.org/simple" }
-dependencies = [
-    { name = "julius" },
-    { name = "torch" },
-    { name = "torch-pitch-shift" },
-    { name = "torchaudio" },
-]
-sdist = { url = "https://files.pythonhosted.org/packages/31/8d/2f8fd7e34c75f5ee8de4310c3bd3f22270acd44d1f809e2fe7c12fbf35f8/torch_audiomentations-0.12.0.tar.gz", hash = "sha256:b02d4c5eb86376986a53eb405cca5e34f370ea9284411237508e720c529f7888", size = 52094, upload-time = "2025-01-15T09:07:01.071Z" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/21/9d/1ee04f49c15d2d632f6f7102061d7c07652858e6d91b58a091531034e84f/torch_audiomentations-0.12.0-py3-none-any.whl", hash = "sha256:1b80b91d2016ccf83979622cac8f702072a79b7dcc4c2bee40f00b26433a786b", size = 48506, upload-time = "2025-01-15T09:06:59.687Z" },
-]
-
-[[package]]
-name = "torch-pitch-shift"
-version = "1.2.5"
-source = { registry = "https://pypi.org/simple" }
-dependencies = [
-    { name = "packaging" },
-    { name = "primepy" },
-    { name = "torch" },
-    { name = "torchaudio" },
-]
-sdist = { url = "https://files.pythonhosted.org/packages/79/a6/722a832bca75d5079f6731e005b3d0c2eec7c6c6863d030620952d143d57/torch_pitch_shift-1.2.5.tar.gz", hash = "sha256:6e1c7531f08d0f407a4c55e5ff8385a41355c5c5d27ab7fa08632e51defbd0ed", size = 4725, upload-time = "2024-09-25T19:10:12.922Z" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/27/4c/96ac2a09efb56cc3c41fb3ce9b6f4d8c0604499f7481d4a13a7b03e21382/torch_pitch_shift-1.2.5-py3-none-any.whl", hash = "sha256:6f8500cbc13f1c98b11cde1805ce5084f82cdd195c285f34287541f168a7c6a7", size = 5005, upload-time = "2024-09-25T19:10:11.521Z" },
-]
-
 [[package]]
 name = "torchaudio"
 version = "2.8.0"
@@ -2142,19 +2052,13 @@ wheels = [
 
 [[package]]
 name = "wcwidth"
-version = "0.7.0"
+version = "0.6.0"
 source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/2c/ee/afaf0f85a9a18fe47a67f1e4422ed6cf1fe642f0ae0a2f81166231303c52/wcwidth-0.7.0.tar.gz", hash = "sha256:90e3a7ea092341c44b99562e75d09e4d5160fe7a3974c6fb842a101a95e7eed0", size = 182132, upload-time = "2026-05-02T16:04:12.653Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/35/a2/8e3becb46433538a38726c948d3399905a4c7cabd0df578ede5dc51f0ec2/wcwidth-0.6.0.tar.gz", hash = "sha256:cdc4e4262d6ef9a1a57e018384cbeb1208d8abbc64176027e2c2455c81313159", size = 159684, upload-time = "2026-02-06T19:19:40.919Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/41/52/e465037f5375f43533d1a80b6923955201596a99142ed524d77b571a1418/wcwidth-0.7.0-py3-none-any.whl", hash = "sha256:5d69154c429a82910e241c738cd0e2976fac8a2dd47a1a805f4afed1c0f136f2", size = 110825, upload-time = "2026-05-02T16:04:11.033Z" },
+    { url = "https://files.pythonhosted.org/packages/68/5a/199c59e0a824a3db2b89c5d2dade7ab5f9624dbf6448dc291b46d5ec94d3/wcwidth-0.6.0-py3-none-any.whl", hash = "sha256:1a3a1e510b553315f8e146c54764f4fb6264ffad731b3d78088cdb1478ffbdad", size = 94189, upload-time = "2026-02-06T19:19:39.646Z" },
 ]
 
-[[package]]
-name = "webrtcvad"
-version = "2.0.10"
-source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/89/34/e2de2d97f3288512b9ea56f92e7452f8207eb5a0096500badf9dfd48f5e6/webrtcvad-2.0.10.tar.gz", hash = "sha256:f1bed2fb25b63fb7b1a55d64090c993c9c9167b28485ae0bcdd81cf6ede96aea", size = 66156, upload-time = "2017-01-07T23:05:18.732Z" }
-
 [[package]]
 name = "werkzeug"
 version = "3.1.8"