audio_model/src/inference.py

import random

import librosa
import torch
import torchaudio
from torch import Tensor, no_grad, device
from torchaudio.transforms import MelSpectrogram, AmplitudeToDB, Resample, TimeStretch
from pathlib import Path
import torchaudio.functional as F

from tokenizer import ASRTokenizer
from model import ASRModel

CONFIG = {
    # 模型配置
    'input_dim': 640,
    'num_heads': 8,
    'ffn_dim': 2048,
    'num_layers': 8,
    'dropout': 0.1,
}

class ASRInference:
    def __init__(self, model_path: Path, vocab_path: Path, device: device, augment: bool = True, augment_prob: float = 0.5) -> None:
        self.device = device
        self.augment: bool = augment
        self.augment_prob: float = augment_prob
        self.tokenizer = ASRTokenizer(vocab_path=vocab_path)
        self.model = ASRModel(
            vocab_size=self.tokenizer.vocab_size(),
            input_dim=CONFIG['input_dim'],
            num_heads=CONFIG['num_heads'],
            ffn_dim=CONFIG['ffn_dim'],
            num_layers=CONFIG['num_layers'],
            dropout=CONFIG['dropout'],
        ).to(device)

        self.model.load_state_dict(torch.load(model_path, map_location=device)['model_state_dict'])
        self.model.eval()

        print(f"params params: {self.model.get_num_params():,}",)

        self.sample_rate = 16000
        self.mel_transform = MelSpectrogram(
            sample_rate=self.sample_rate,
            n_fft=400,
            win_length=400,
            hop_length=160,
            n_mels=80,
            f_min=0,
            f_max=8000,
            power=2.0,
        )
        self.amplitude_to_db = AmplitudeToDB()

    def _load_audio(self, audio_path: Path)  -> Tensor:
        waveform, sample_rate = torchaudio.load_with_torchcodec(audio_path)

        if sample_rate != self.sample_rate:
            waveform = Resample(sample_rate, self.sample_rate)(waveform)

        if waveform.shape[0] > 1:
            waveform = waveform.mean(dim=0, keepdim=True)
        
        waveform = waveform / (waveform.abs().max() + 1e-8)
        return waveform
    
    def _extract_features(self, waveform: Tensor) -> Tensor:
        mel_spec: Tensor = self.mel_transform(waveform)
        log_mel_spec: Tensor = self.amplitude_to_db(mel_spec)
        return log_mel_spec.squeeze(0) # [n_mels, time]
    
    def _augment_waveform(self, waveform: Tensor) -> Tensor:
        if not self.augment or random.random() > self.augment_prob:
            return waveform
        
        # 1. voice Stretch/Compress 
        if random.random() < 0.5:
            waveform = self._voice_stretch_or_compress(waveform=waveform)

        if random.random() < 0.3:
            waveform = self._drop_frames(waveform)

        if random.random() < 0.4:
            waveform = self._add_noise(waveform)
        
        return waveform
    
    def _voice_stretch_or_compress(self, waveform: Tensor) -> Tensor:
        speed_factor = random.uniform(0.6, 1.4) # (Speed Change: 0.6x - 1.4x)
        spec = torch.stft(
            waveform.squeeze(0),
            n_fft=400,
            hop_length=160,
            window=torch.hann_window(400).to(waveform.device),
            return_complex=True
        )
        
        # 时间拉伸（不改变音高）
        stretch = TimeStretch(
            hop_length=160,
            n_freq=201,
            fixed_rate=speed_factor
        )
        stretched_spec = stretch(spec)
        
        # 转回波形
        waveform_stretched = torch.istft(
            stretched_spec,
            n_fft=400,
            hop_length=160,
            window=torch.hann_window(400).to(waveform.device)
        ).unsqueeze(0)

        return waveform_stretched

    def _drop_frames(self, waveform: Tensor) -> Tensor:
        audio_len = waveform.shape[1]
        drop_ratio = random.uniform(0.05, 0.15)
        drop_len = int(audio_len * drop_ratio)

        if audio_len > drop_len:
            start_pos = random.randint(0, audio_len - drop_len)
            # clean
            waveform = torch.cat([waveform[:, :start_pos], waveform[:, start_pos + drop_len:]], dim=1)

        return waveform
    
    def _add_noise(self, waveform: Tensor) -> Tensor:
        snr_db = random.uniform(10, 20)
        signal_power = torch.mean(waveform ** 2)
        
        snr_linear = 10 ** (snr_db / 10)
        noise_power = signal_power / snr_linear
        
        noise = torch.randn_like(waveform) * torch.sqrt(noise_power)
        return waveform + noise
    
    def transcribe(self, audio_path: Path) -> str:
        waveform = self._load_audio(audio_path=audio_path)
        waveform = self._augment_waveform(waveform=waveform)
        mel_spec = self._extract_features(waveform=waveform)

        mel_spec = mel_spec.unsqueeze(0).to(self.device) # [1, n_mels, time]
        mel_length = torch.tensor([mel_spec.shape[2]], dtype=torch.long, device=self.device)

        with no_grad():
            log_probs, _ = self.model(mel_specs=mel_spec, mel_lengths=mel_length) # [1, T, vocab]

        text = self.tokenizer.ctc_greedy_decode(log_probs=log_probs[0])
        return text
        
    def transcribe_batch(self, audio_paths: list[Path]) -> list[str]:
        results = []

        for audio_path in audio_paths:
            text = self.transcribe(audio_path=audio_path)
            results.append(text)

        return results


def main():
    workspace_dir = Path(__file__).parent.parent
    device = torch.device('cuda:0')

    # checkpoint = sorted(workspace_dir.glob('.checkpoints/checkpoint_epoch_*.pt'), key=lambda p: int(p.stem.split('_')[1]))[-1]
    checkpoint = workspace_dir / ".checkpoints/checkpoint_step_9500.pt"
    print(f"Load Checkpoint: {checkpoint}")

    inference = ASRInference(model_path=checkpoint, vocab_path=workspace_dir / 'config/uig_vocab.json' , device=device)

    audio_path = workspace_dir / 'data/test/F001_001.wav'
    print(f"\n转录音频: {audio_path}")
    text = inference.transcribe(audio_path=audio_path)
    print(f"\n识别结果: {text}")

if __name__ == "__main__":
    main()
# workspace_dir = Path(__file__).parent.parent
# checkpoint = sorted(workspace_dir.glob('.checkpoints/checkpoint_step_*.pt'), key=lambda p: int(p.stem.split('_')[-1]))[-1]

# print(checkpoint)