audio diffusion pytorch

適用於 PyTorch 的功能齊全的音頻擴散庫。包含無條件音訊產生、文字條件音訊產生、擴散自動編碼、上方取樣和聲碼的模型。提供的模型是基於波形的，但是，U-Net（使用a-unet構建）、 DiffusionModel 、擴散方法和擴散採樣器對於任何維度都是通用的，並且可以高度自訂以處理其他格式。注意：（1）這裡沒有提供預先訓練的模型，（2）顯示的配置是指示性的且未經測試，有關本文中使用的配置，請參閱 Moûsai。

安裝

pip install audio-diffusion-pytorch

用法

無條件產生器

 from audio_diffusion_pytorch import DiffusionModel , UNetV0 , VDiffusion , VSampler

model = DiffusionModel (
    net_t = UNetV0 , # The model type used for diffusion (U-Net V0 in this case)
    in_channels = 2 , # U-Net: number of input/output (audio) channels
    channels = [ 8 , 32 , 64 , 128 , 256 , 512 , 512 , 1024 , 1024 ], # U-Net: channels at each layer
    factors = [ 1 , 4 , 4 , 4 , 2 , 2 , 2 , 2 , 2 ], # U-Net: downsampling and upsampling factors at each layer
    items = [ 1 , 2 , 2 , 2 , 2 , 2 , 2 , 4 , 4 ], # U-Net: number of repeating items at each layer
    attentions = [ 0 , 0 , 0 , 0 , 0 , 1 , 1 , 1 , 1 ], # U-Net: attention enabled/disabled at each layer
    attention_heads = 8 , # U-Net: number of attention heads per attention item
    attention_features = 64 , # U-Net: number of attention features per attention item
    diffusion_t = VDiffusion , # The diffusion method used
    sampler_t = VSampler , # The diffusion sampler used
)

# Train model with audio waveforms
audio = torch . randn ( 1 , 2 , 2 ** 18 ) # [batch_size, in_channels, length]
loss = model ( audio )
loss . backward ()

# Turn noise into new audio sample with diffusion
noise = torch . randn ( 1 , 2 , 2 ** 18 ) # [batch_size, in_channels, length]
sample = model . sample ( noise , num_steps = 10 ) # Suggested num_steps 10-100

文字條件產生器

使用t5-base文本嵌入來調節生成的文本到音頻擴散模型需要pip install transformers 。

 from audio_diffusion_pytorch import DiffusionModel , UNetV0 , VDiffusion , VSampler

model = DiffusionModel (
    # ... same as unconditional model
    use_text_conditioning = True , # U-Net: enables text conditioning (default T5-base)
    use_embedding_cfg = True , # U-Net: enables classifier free guidance
    embedding_max_length = 64 , # U-Net: text embedding maximum length (default for T5-base)
    embedding_features = 768 , # U-Net: text mbedding features (default for T5-base)
    cross_attentions = [ 0 , 0 , 0 , 1 , 1 , 1 , 1 , 1 , 1 ], # U-Net: cross-attention enabled/disabled at each layer
)

# Train model with audio waveforms
audio_wave = torch . randn ( 1 , 2 , 2 ** 18 ) # [batch, in_channels, length]
loss = model (
    audio_wave ,
    text = [ 'The audio description' ], # Text conditioning, one element per batch
    embedding_mask_proba = 0.1 # Probability of masking text with learned embedding (Classifier-Free Guidance Mask)
)
loss . backward ()

# Turn noise into new audio sample with diffusion
noise = torch . randn ( 1 , 2 , 2 ** 18 )
sample = model . sample (
    noise ,
    text = [ 'The audio description' ],
    embedding_scale = 5.0 , # Higher for more text importance, suggested range: 1-15 (Classifier-Free Guidance Scale)
    num_steps = 2 # Higher for better quality, suggested num_steps: 10-100
)

擴散上採樣器

使用擴散將音訊從較低取樣率上取樣到較高取樣率，例如 3kHz 到 48kHz。

 from audio_diffusion_pytorch import DiffusionUpsampler , UNetV0 , VDiffusion , VSampler

upsampler = DiffusionUpsampler (
    net_t = UNetV0 , # The model type used for diffusion
    upsample_factor = 16 , # The upsample factor (e.g. 16 can be used for 3kHz to 48kHz)
    in_channels = 2 , # U-Net: number of input/output (audio) channels
    channels = [ 8 , 32 , 64 , 128 , 256 , 512 , 512 , 1024 , 1024 ], # U-Net: channels at each layer
    factors = [ 1 , 4 , 4 , 4 , 2 , 2 , 2 , 2 , 2 ], # U-Net: downsampling and upsampling factors at each layer
    items = [ 1 , 2 , 2 , 2 , 2 , 2 , 2 , 4 , 4 ], # U-Net: number of repeating items at each layer
    diffusion_t = VDiffusion , # The diffusion method used
    sampler_t = VSampler , # The diffusion sampler used
)

# Train model with high sample rate audio waveforms
audio = torch . randn ( 1 , 2 , 2 ** 18 ) # [batch, in_channels, length]
loss = upsampler ( audio )
loss . backward ()

# Turn low sample rate audio into high sample rate
downsampled_audio = torch . randn ( 1 , 2 , 2 ** 14 ) # [batch, in_channels, length]
sample = upsampler . sample ( downsampled_audio , num_steps = 10 ) # Output has shape: [1, 2, 2**18]

擴散聲碼器

使用擴散將梅爾譜圖轉換為波譜。

 from audio_diffusion_pytorch import DiffusionVocoder , UNetV0 , VDiffusion , VSampler

vocoder = DiffusionVocoder (
    mel_n_fft = 1024 , # Mel-spectrogram n_fft
    mel_channels = 80 , # Mel-spectrogram channels
    mel_sample_rate = 48000 , # Mel-spectrogram sample rate
    mel_normalize_log = True , # Mel-spectrogram log normalization (alternative is mel_normalize=True for [-1,1] power normalization)
    net_t = UNetV0 , # The model type used for diffusion vocoding
    channels = [ 8 , 32 , 64 , 128 , 256 , 512 , 512 , 1024 , 1024 ], # U-Net: channels at each layer
    factors = [ 1 , 4 , 4 , 4 , 2 , 2 , 2 , 2 , 2 ], # U-Net: downsampling and upsampling factors at each layer
    items = [ 1 , 2 , 2 , 2 , 2 , 2 , 2 , 4 , 4 ], # U-Net: number of repeating items at each layer
    diffusion_t = VDiffusion , # The diffusion method used
    sampler_t = VSampler , # The diffusion sampler used
)

# Train model on waveforms (automatically converted to mel internally)
audio = torch . randn ( 1 , 2 , 2 ** 18 ) # [batch, in_channels, length]
loss = vocoder ( audio )
loss . backward ()

# Turn mel spectrogram into waveform
mel_spectrogram = torch . randn ( 1 , 2 , 80 , 1024 ) # [batch, in_channels, mel_channels, mel_length]
sample = vocoder . sample ( mel_spectrogram , num_steps = 10 ) # Output has shape: [1, 2, 2**18]

擴散自動編碼器

使用擴散將音訊自動編碼為壓縮潛在音訊。可以提供任何編碼器，只要它是EncoderBase類別的子類別或包含out_channels和downsample_factor欄位。

 from audio_diffusion_pytorch import DiffusionAE , UNetV0 , VDiffusion , VSampler
from audio_encoders_pytorch import MelE1d , TanhBottleneck

autoencoder = DiffusionAE (
    encoder = MelE1d ( # The encoder used, in this case a mel-spectrogram encoder
        in_channels = 2 ,
        channels = 512 ,
        multipliers = [ 1 , 1 ],
        factors = [ 2 ],
        num_blocks = [ 12 ],
        out_channels = 32 ,
        mel_channels = 80 ,
        mel_sample_rate = 48000 ,
        mel_normalize_log = True ,
        bottleneck = TanhBottleneck (),
    ),
    inject_depth = 6 ,
    net_t = UNetV0 , # The model type used for diffusion upsampling
    in_channels = 2 , # U-Net: number of input/output (audio) channels
    channels = [ 8 , 32 , 64 , 128 , 256 , 512 , 512 , 1024 , 1024 ], # U-Net: channels at each layer
    factors = [ 1 , 4 , 4 , 4 , 2 , 2 , 2 , 2 , 2 ], # U-Net: downsampling and upsampling factors at each layer
    items = [ 1 , 2 , 2 , 2 , 2 , 2 , 2 , 4 , 4 ], # U-Net: number of repeating items at each layer
    diffusion_t = VDiffusion , # The diffusion method used
    sampler_t = VSampler , # The diffusion sampler used
)

# Train autoencoder with audio samples
audio = torch . randn ( 1 , 2 , 2 ** 18 ) # [batch, in_channels, length]
loss = autoencoder ( audio )
loss . backward ()

# Encode/decode audio
audio = torch . randn ( 1 , 2 , 2 ** 18 ) # [batch, in_channels, length]
latent = autoencoder . encode ( audio ) # Encode
sample = autoencoder . decode ( latent , num_steps = 10 ) # Decode by sampling diffusion model conditioning on latent 

其他

修復

 from audio_diffusion_pytorch import UNetV0 , VInpainter

# The diffusion UNetV0 (this is an example, the net must be trained to work)
net = UNetV0 (
    dim = 1 ,
    in_channels = 2 , # U-Net: number of input/output (audio) channels
    channels = [ 8 , 32 , 64 , 128 , 256 , 512 , 512 , 1024 , 1024 ], # U-Net: channels at each layer
    factors = [ 1 , 4 , 4 , 4 , 2 , 2 , 2 , 2 , 2 ], # U-Net: downsampling and upsampling factors at each layer
    items = [ 1 , 2 , 2 , 2 , 2 , 2 , 2 , 4 , 4 ], # U-Net: number of repeating items at each layer
    attentions = [ 0 , 0 , 0 , 0 , 0 , 1 , 1 , 1 , 1 ], # U-Net: attention enabled/disabled at each layer
    attention_heads = 8 , # U-Net: number of attention heads per attention block
    attention_features = 64 , # U-Net: number of attention features per attention block,
)

# Instantiate inpainter with trained net
inpainter = VInpainter ( net = net )

# Inpaint source
y = inpainter (
    source = torch . randn ( 1 , 2 , 2 ** 18 ), # Start source
    mask = torch . randint ( 0 , 2 , ( 1 , 2 , 2 ** 18 ), dtype = torch . bool ),  # Set to `True` the parts you want to keep
    num_steps = 10 , # Number of inpainting steps
    num_resamples = 2 , # Number of resampling steps
    show_progress = True ,
) # [1, 2, 2 ** 18] 

欣賞

• StabilityAI 用於計算，Zach Evans 和 HarmonAI 的其他人進行了有趣的研究討論。
• 蘇黎世聯邦理工學院提供資源，Zhijing Jin、Bernhard Schoelkopf 和 Mrinmaya Sachan 指導本論文。
• Phil Wang 在擴散和 Imagen 方面做出了精彩的開源貢獻。
• 凱瑟琳·克勞森 (Katherine Crowson) 的 k 擴散實驗和瘋狂的採樣器集合。

引文

DDPM擴散

 @misc { 2006.11239 ,
Author = { Jonathan Ho and Ajay Jain and Pieter Abbeel } ,
Title = { Denoising Diffusion Probabilistic Models } ,
Year = { 2020 } ,
Eprint = { arXiv:2006.11239 } ,
}

DDIM（V 採樣器）

 @misc { 2010.02502 ,
Author = { Jiaming Song and Chenlin Meng and Stefano Ermon } ,
Title = { Denoising Diffusion Implicit Models } ,
Year = { 2020 } ,
Eprint = { arXiv:2010.02502 } ,
}

V-擴散

 @misc { 2202.00512 ,
Author = { Tim Salimans and Jonathan Ho } ,
Title = { Progressive Distillation for Fast Sampling of Diffusion Models } ,
Year = { 2022 } ,
Eprint = { arXiv:2202.00512 } ,
}

Imagen（T5 文本調節）

 @misc { 2205.11487 ,
Author = { Chitwan Saharia and William Chan and Saurabh Saxena and Lala Li and Jay Whang and Emily Denton and Seyed Kamyar Seyed Ghasemipour and Burcu Karagol Ayan and S. Sara Mahdavi and Rapha Gontijo Lopes and Tim Salimans and Jonathan Ho and David J Fleet and Mohammad Norouzi } ,
Title = { Photorealistic Text-to-Image Diffusion Models with Deep Language Understanding } ,
Year = { 2022 } ,
Eprint = { arXiv:2205.11487 } ,
}