audio diffusion pytorch
v0.1.3
PyTorch를 위한 모든 기능을 갖춘 오디오 확산 라이브러리입니다. 무조건 오디오 생성, 텍스트 조건부 오디오 생성, 확산 자동 인코딩, 업샘플링 및 보코딩을 위한 모델을 포함합니다. 제공된 모델은 파형 기반이지만 U-Net( a-unet 사용하여 구축됨), DiffusionModel , 확산 방법 및 확산 샘플러는 모두 모든 차원에 일반적이며 다른 형식에서 작동하도록 고도로 사용자 정의할 수 있습니다. 참고: (1) 여기에는 사전 훈련된 모델이 제공되지 않습니다. (2) 표시된 구성은 참고용이며 테스트되지 않았습니다. 논문에 사용된 구성은 Moûsai를 참조하세요.
pip install audio-diffusion-pytorch from audio_diffusion_pytorch import DiffusionModel , UNetV0 , VDiffusion , VSampler
model = DiffusionModel (
net_t = UNetV0 , # The model type used for diffusion (U-Net V0 in this case)
in_channels = 2 , # U-Net: number of input/output (audio) channels
channels = [ 8 , 32 , 64 , 128 , 256 , 512 , 512 , 1024 , 1024 ], # U-Net: channels at each layer
factors = [ 1 , 4 , 4 , 4 , 2 , 2 , 2 , 2 , 2 ], # U-Net: downsampling and upsampling factors at each layer
items = [ 1 , 2 , 2 , 2 , 2 , 2 , 2 , 4 , 4 ], # U-Net: number of repeating items at each layer
attentions = [ 0 , 0 , 0 , 0 , 0 , 1 , 1 , 1 , 1 ], # U-Net: attention enabled/disabled at each layer
attention_heads = 8 , # U-Net: number of attention heads per attention item
attention_features = 64 , # U-Net: number of attention features per attention item
diffusion_t = VDiffusion , # The diffusion method used
sampler_t = VSampler , # The diffusion sampler used
)
# Train model with audio waveforms
audio = torch . randn ( 1 , 2 , 2 ** 18 ) # [batch_size, in_channels, length]
loss = model ( audio )
loss . backward ()
# Turn noise into new audio sample with diffusion
noise = torch . randn ( 1 , 2 , 2 ** 18 ) # [batch_size, in_channels, length]
sample = model . sample ( noise , num_steps = 10 ) # Suggested num_steps 10-100 t5-base 텍스트 임베딩으로 생성을 조절하는 텍스트-오디오 확산 모델에는 pip install transformers 필요합니다.
from audio_diffusion_pytorch import DiffusionModel , UNetV0 , VDiffusion , VSampler
model = DiffusionModel (
# ... same as unconditional model
use_text_conditioning = True , # U-Net: enables text conditioning (default T5-base)
use_embedding_cfg = True , # U-Net: enables classifier free guidance
embedding_max_length = 64 , # U-Net: text embedding maximum length (default for T5-base)
embedding_features = 768 , # U-Net: text mbedding features (default for T5-base)
cross_attentions = [ 0 , 0 , 0 , 1 , 1 , 1 , 1 , 1 , 1 ], # U-Net: cross-attention enabled/disabled at each layer
)
# Train model with audio waveforms
audio_wave = torch . randn ( 1 , 2 , 2 ** 18 ) # [batch, in_channels, length]
loss = model (
audio_wave ,
text = [ 'The audio description' ], # Text conditioning, one element per batch
embedding_mask_proba = 0.1 # Probability of masking text with learned embedding (Classifier-Free Guidance Mask)
)
loss . backward ()
# Turn noise into new audio sample with diffusion
noise = torch . randn ( 1 , 2 , 2 ** 18 )
sample = model . sample (
noise ,
text = [ 'The audio description' ],
embedding_scale = 5.0 , # Higher for more text importance, suggested range: 1-15 (Classifier-Free Guidance Scale)
num_steps = 2 # Higher for better quality, suggested num_steps: 10-100
)확산을 사용하여 낮은 샘플 속도에서 높은 샘플 속도로 오디오를 업샘플링합니다(예: 3kHz~48kHz).
from audio_diffusion_pytorch import DiffusionUpsampler , UNetV0 , VDiffusion , VSampler
upsampler = DiffusionUpsampler (
net_t = UNetV0 , # The model type used for diffusion
upsample_factor = 16 , # The upsample factor (e.g. 16 can be used for 3kHz to 48kHz)
in_channels = 2 , # U-Net: number of input/output (audio) channels
channels = [ 8 , 32 , 64 , 128 , 256 , 512 , 512 , 1024 , 1024 ], # U-Net: channels at each layer
factors = [ 1 , 4 , 4 , 4 , 2 , 2 , 2 , 2 , 2 ], # U-Net: downsampling and upsampling factors at each layer
items = [ 1 , 2 , 2 , 2 , 2 , 2 , 2 , 4 , 4 ], # U-Net: number of repeating items at each layer
diffusion_t = VDiffusion , # The diffusion method used
sampler_t = VSampler , # The diffusion sampler used
)
# Train model with high sample rate audio waveforms
audio = torch . randn ( 1 , 2 , 2 ** 18 ) # [batch, in_channels, length]
loss = upsampler ( audio )
loss . backward ()
# Turn low sample rate audio into high sample rate
downsampled_audio = torch . randn ( 1 , 2 , 2 ** 14 ) # [batch, in_channels, length]
sample = upsampler . sample ( downsampled_audio , num_steps = 10 ) # Output has shape: [1, 2, 2**18]확산을 사용하여 멜 스펙트로그램을 파동으로 변환합니다.
from audio_diffusion_pytorch import DiffusionVocoder , UNetV0 , VDiffusion , VSampler
vocoder = DiffusionVocoder (
mel_n_fft = 1024 , # Mel-spectrogram n_fft
mel_channels = 80 , # Mel-spectrogram channels
mel_sample_rate = 48000 , # Mel-spectrogram sample rate
mel_normalize_log = True , # Mel-spectrogram log normalization (alternative is mel_normalize=True for [-1,1] power normalization)
net_t = UNetV0 , # The model type used for diffusion vocoding
channels = [ 8 , 32 , 64 , 128 , 256 , 512 , 512 , 1024 , 1024 ], # U-Net: channels at each layer
factors = [ 1 , 4 , 4 , 4 , 2 , 2 , 2 , 2 , 2 ], # U-Net: downsampling and upsampling factors at each layer
items = [ 1 , 2 , 2 , 2 , 2 , 2 , 2 , 4 , 4 ], # U-Net: number of repeating items at each layer
diffusion_t = VDiffusion , # The diffusion method used
sampler_t = VSampler , # The diffusion sampler used
)
# Train model on waveforms (automatically converted to mel internally)
audio = torch . randn ( 1 , 2 , 2 ** 18 ) # [batch, in_channels, length]
loss = vocoder ( audio )
loss . backward ()
# Turn mel spectrogram into waveform
mel_spectrogram = torch . randn ( 1 , 2 , 80 , 1024 ) # [batch, in_channels, mel_channels, mel_length]
sample = vocoder . sample ( mel_spectrogram , num_steps = 10 ) # Output has shape: [1, 2, 2**18] 확산을 사용하여 오디오를 압축된 잠재성으로 자동 인코딩합니다. EncoderBase 클래스를 하위 클래스로 분류하거나 out_channels 및 downsample_factor 필드를 포함하는 한 모든 인코더를 제공할 수 있습니다.
from audio_diffusion_pytorch import DiffusionAE , UNetV0 , VDiffusion , VSampler
from audio_encoders_pytorch import MelE1d , TanhBottleneck
autoencoder = DiffusionAE (
encoder = MelE1d ( # The encoder used, in this case a mel-spectrogram encoder
in_channels = 2 ,
channels = 512 ,
multipliers = [ 1 , 1 ],
factors = [ 2 ],
num_blocks = [ 12 ],
out_channels = 32 ,
mel_channels = 80 ,
mel_sample_rate = 48000 ,
mel_normalize_log = True ,
bottleneck = TanhBottleneck (),
),
inject_depth = 6 ,
net_t = UNetV0 , # The model type used for diffusion upsampling
in_channels = 2 , # U-Net: number of input/output (audio) channels
channels = [ 8 , 32 , 64 , 128 , 256 , 512 , 512 , 1024 , 1024 ], # U-Net: channels at each layer
factors = [ 1 , 4 , 4 , 4 , 2 , 2 , 2 , 2 , 2 ], # U-Net: downsampling and upsampling factors at each layer
items = [ 1 , 2 , 2 , 2 , 2 , 2 , 2 , 4 , 4 ], # U-Net: number of repeating items at each layer
diffusion_t = VDiffusion , # The diffusion method used
sampler_t = VSampler , # The diffusion sampler used
)
# Train autoencoder with audio samples
audio = torch . randn ( 1 , 2 , 2 ** 18 ) # [batch, in_channels, length]
loss = autoencoder ( audio )
loss . backward ()
# Encode/decode audio
audio = torch . randn ( 1 , 2 , 2 ** 18 ) # [batch, in_channels, length]
latent = autoencoder . encode ( audio ) # Encode
sample = autoencoder . decode ( latent , num_steps = 10 ) # Decode by sampling diffusion model conditioning on latent from audio_diffusion_pytorch import UNetV0 , VInpainter
# The diffusion UNetV0 (this is an example, the net must be trained to work)
net = UNetV0 (
dim = 1 ,
in_channels = 2 , # U-Net: number of input/output (audio) channels
channels = [ 8 , 32 , 64 , 128 , 256 , 512 , 512 , 1024 , 1024 ], # U-Net: channels at each layer
factors = [ 1 , 4 , 4 , 4 , 2 , 2 , 2 , 2 , 2 ], # U-Net: downsampling and upsampling factors at each layer
items = [ 1 , 2 , 2 , 2 , 2 , 2 , 2 , 4 , 4 ], # U-Net: number of repeating items at each layer
attentions = [ 0 , 0 , 0 , 0 , 0 , 1 , 1 , 1 , 1 ], # U-Net: attention enabled/disabled at each layer
attention_heads = 8 , # U-Net: number of attention heads per attention block
attention_features = 64 , # U-Net: number of attention features per attention block,
)
# Instantiate inpainter with trained net
inpainter = VInpainter ( net = net )
# Inpaint source
y = inpainter (
source = torch . randn ( 1 , 2 , 2 ** 18 ), # Start source
mask = torch . randint ( 0 , 2 , ( 1 , 2 , 2 ** 18 ), dtype = torch . bool ), # Set to `True` the parts you want to keep
num_steps = 10 , # Number of inpainting steps
num_resamples = 2 , # Number of resampling steps
show_progress = True ,
) # [1, 2, 2 ** 18] DDPM 확산
@misc { 2006.11239 ,
Author = { Jonathan Ho and Ajay Jain and Pieter Abbeel } ,
Title = { Denoising Diffusion Probabilistic Models } ,
Year = { 2020 } ,
Eprint = { arXiv:2006.11239 } ,
}DDIM(V-샘플러)
@misc { 2010.02502 ,
Author = { Jiaming Song and Chenlin Meng and Stefano Ermon } ,
Title = { Denoising Diffusion Implicit Models } ,
Year = { 2020 } ,
Eprint = { arXiv:2010.02502 } ,
}V-확산
@misc { 2202.00512 ,
Author = { Tim Salimans and Jonathan Ho } ,
Title = { Progressive Distillation for Fast Sampling of Diffusion Models } ,
Year = { 2022 } ,
Eprint = { arXiv:2202.00512 } ,
}Imagen(T5 텍스트 조절)
@misc { 2205.11487 ,
Author = { Chitwan Saharia and William Chan and Saurabh Saxena and Lala Li and Jay Whang and Emily Denton and Seyed Kamyar Seyed Ghasemipour and Burcu Karagol Ayan and S. Sara Mahdavi and Rapha Gontijo Lopes and Tim Salimans and Jonathan Ho and David J Fleet and Mohammad Norouzi } ,
Title = { Photorealistic Text-to-Image Diffusion Models with Deep Language Understanding } ,
Year = { 2022 } ,
Eprint = { arXiv:2205.11487 } ,
}
