phenaki pytorch 다운로드 - phenaki pytorch 소스 코드 다운로드

페나키 - Pytorch

Mask GIT를 사용하여 최대 2분 길이의 텍스트 안내 비디오를 생성하는 Phenaki Video를 Pytorch에서 구현합니다. 또한 잠재적으로 더 나은 세대를 위해 토큰 비평가와 관련된 또 다른 기술을 결합할 것입니다.

이 작품을 공개적으로 복제하는 데 관심이 있다면 가입하세요.

AI 커피브레이크 설명

감사

최첨단 인공지능 연구를 위한 Stability.ai의 아낌없는 후원
? 놀라운 트랜스포머와 가속 라이브러리를 위한 Huggingface
Guillem의 지속적인 기여
너? 훌륭한 기계 학습 엔지니어 및/또는 연구원이라면 오픈 소스 생성 AI의 개척에 자유롭게 기여해 주세요.

설치하다

$ pip install phenaki-pytorch

용법

C-ViViT

 import torch
from phenaki_pytorch import CViViT , CViViTTrainer

cvivit = CViViT (
    dim = 512 ,
    codebook_size = 65536 ,
    image_size = 256 ,
    patch_size = 32 ,
    temporal_patch_size = 2 ,
    spatial_depth = 4 ,
    temporal_depth = 4 ,
    dim_head = 64 ,
    heads = 8
). cuda ()

trainer = CViViTTrainer (
    cvivit ,
    folder = '/path/to/images/or/videos' ,
    batch_size = 4 ,
    grad_accum_every = 4 ,
    train_on_images = False ,  # you can train on images first, before fine tuning on video, for sample efficiency
    use_ema = False ,          # recommended to be turned on (keeps exponential moving averaged cvivit) unless if you don't have enough resources
    num_train_steps = 10000
)

trainer . train ()               # reconstructions and checkpoints will be saved periodically to ./results

페나키

 import torch
from phenaki_pytorch import CViViT , MaskGit , Phenaki

cvivit = CViViT (
    dim = 512 ,
    codebook_size = 65536 ,
    image_size = ( 256 , 128 ),  # video with rectangular screen allowed
    patch_size = 32 ,
    temporal_patch_size = 2 ,
    spatial_depth = 4 ,
    temporal_depth = 4 ,
    dim_head = 64 ,
    heads = 8
)

cvivit . load ( '/path/to/trained/cvivit.pt' )

maskgit = MaskGit (
    num_tokens = 5000 ,
    max_seq_len = 1024 ,
    dim = 512 ,
    dim_context = 768 ,
    depth = 6 ,
)

phenaki = Phenaki (
    cvivit = cvivit ,
    maskgit = maskgit
). cuda ()

videos = torch . randn ( 3 , 3 , 17 , 256 , 128 ). cuda () # (batch, channels, frames, height, width)
mask = torch . ones (( 3 , 17 )). bool (). cuda () # [optional] (batch, frames) - allows for co-training videos of different lengths as well as video and images in the same batch

texts = [
    'a whale breaching from afar' ,
    'young girl blowing out candles on her birthday cake' ,
    'fireworks with blue and green sparkles'
]

loss = phenaki ( videos , texts = texts , video_frame_mask = mask )
loss . backward ()

# do the above for many steps, then ...

video = phenaki . sample ( texts = 'a squirrel examines an acorn' , num_frames = 17 , cond_scale = 5. ) # (1, 3, 17, 256, 128)

# so in the paper, they do not really achieve 2 minutes of coherent video
# at each new scene with new text conditioning, they condition on the previous K frames
# you can easily achieve this with this framework as so

video_prime = video [:, :, - 3 :] # (1, 3, 3, 256, 128) # say K = 3

video_next = phenaki . sample ( texts = 'a cat watches the squirrel from afar' , prime_frames = video_prime , num_frames = 14 ) # (1, 3, 14, 256, 128)

# the total video

entire_video = torch . cat (( video , video_next ), dim = 2 ) # (1, 3, 17 + 14, 256, 128)

# and so on...

아니면 그냥 make_video 함수를 가져오세요.

 # ... above code

from phenaki_pytorch import make_video

entire_video , scenes = make_video ( phenaki , texts = [
    'a squirrel examines an acorn buried in the snow' ,
    'a cat watches the squirrel from a frosted window sill' ,
    'zoom out to show the entire living room, with the cat residing by the window sill'
], num_frames = ( 17 , 14 , 14 ), prime_lengths = ( 5 , 5 ))

entire_video . shape # (1, 3, 17 + 14 + 14 = 45, 256, 256)

# scenes - List[Tensor[3]] - video segment of each scene

그게 다야!

토큰 비평가

새로운 논문에서는 신뢰도 측정으로 각 토큰의 예측 확률에 의존하는 대신 샘플링 중에 반복적으로 마스크할 항목을 결정하도록 추가 비평가를 교육할 수 있다고 제안합니다. 아래와 같이 잠재적으로 더 나은 세대를 위해 이 비평가를 선택적으로 교육할 수 있습니다.

 import torch
from phenaki_pytorch import CViViT , MaskGit , TokenCritic , Phenaki

cvivit = CViViT (
    dim = 512 ,
    codebook_size = 65536 ,
    image_size = ( 256 , 128 ),
    patch_size = 32 ,
    temporal_patch_size = 2 ,
    spatial_depth = 4 ,
    temporal_depth = 4 ,
    dim_head = 64 ,
    heads = 8
)

maskgit = MaskGit (
    num_tokens = 65536 ,
    max_seq_len = 1024 ,
    dim = 512 ,
    dim_context = 768 ,
    depth = 6 ,
)

# (1) define the critic

critic = TokenCritic (
    num_tokens = 65536 ,
    max_seq_len = 1024 ,
    dim = 512 ,
    dim_context = 768 ,
    depth = 6 ,
    has_cross_attn = True
)

trainer = Phenaki (
    maskgit = maskgit ,
    cvivit = cvivit ,
    critic = critic    # and then (2) pass it into Phenaki
). cuda ()

texts = [
    'a whale breaching from afar' ,
    'young girl blowing out candles on her birthday cake' ,
    'fireworks with blue and green sparkles'
]

videos = torch . randn ( 3 , 3 , 3 , 256 , 128 ). cuda () # (batch, channels, frames, height, width)

loss = trainer ( videos = videos , texts = texts )
loss . backward ()

또는 더 간단하게 Phenaki 초기화에서 self_token_critic = True 설정하여 MaskGit 자체를 자기 비평가(Nijkamp et al)로 재사용할 수 있습니다.

 phenaki = Phenaki (
    ...,
    self_token_critic = True  # set this to True
)

이제 너희 세대는 크게 향상되어야 한다!

페나키 트레이너

이 저장소는 또한 연구원이 텍스트를 이미지로, 그런 다음 텍스트를 비디오로 교육할 수 있도록 노력할 것입니다. 마찬가지로, 무조건적인 훈련을 위해서는 연구자가 먼저 이미지를 훈련한 다음 비디오를 미세 조정할 수 있어야 합니다. 아래는 텍스트-비디오 예시입니다.

 import torch
from torch . utils . data import Dataset
from phenaki_pytorch import CViViT , MaskGit , Phenaki , PhenakiTrainer

cvivit = CViViT (
    dim = 512 ,
    codebook_size = 65536 ,
    image_size = 256 ,
    patch_size = 32 ,
    temporal_patch_size = 2 ,
    spatial_depth = 4 ,
    temporal_depth = 4 ,
    dim_head = 64 ,
    heads = 8
)

cvivit . load ( '/path/to/trained/cvivit.pt' )

maskgit = MaskGit (
    num_tokens = 5000 ,
    max_seq_len = 1024 ,
    dim = 512 ,
    dim_context = 768 ,
    depth = 6 ,
    unconditional = False
)

phenaki = Phenaki (
    cvivit = cvivit ,
    maskgit = maskgit
). cuda ()

# mock text video dataset
# you will have to extend your own, and return the (<video tensor>, <caption>) tuple

class MockTextVideoDataset ( Dataset ):
    def __init__ (
        self ,
        length = 100 ,
        image_size = 256 ,
        num_frames = 17
    ):
        super (). __init__ ()
        self . num_frames = num_frames
        self . image_size = image_size
        self . len = length

    def __len__ ( self ):
        return self . len

    def __getitem__ ( self , idx ):
        video = torch . randn ( 3 , self . num_frames , self . image_size , self . image_size )
        caption = 'video caption'
        return video , caption

dataset = MockTextVideoDataset ()

# pass in the dataset

trainer = PhenakiTrainer (
    phenaki = phenaki ,
    batch_size = 4 ,
    grad_accum_every = 4 ,
    train_on_images = False , # if your mock dataset above return (images, caption) pairs, set this to True
    dataset = dataset ,       # pass in your dataset here
    sample_texts_file_path = '/path/to/captions.txt' # each caption should be on a new line, during sampling, will be randomly drawn
)

trainer . train ()

무조건은 다음과 같습니다

전. 무조건적인 이미지 및 비디오 교육

 import torch
from phenaki_pytorch import CViViT , MaskGit , Phenaki , PhenakiTrainer

cvivit = CViViT (
    dim = 512 ,
    codebook_size = 65536 ,
    image_size = 256 ,
    patch_size = 32 ,
    temporal_patch_size = 2 ,
    spatial_depth = 4 ,
    temporal_depth = 4 ,
    dim_head = 64 ,
    heads = 8
)

cvivit . load ( '/path/to/trained/cvivit.pt' )

maskgit = MaskGit (
    num_tokens = 5000 ,
    max_seq_len = 1024 ,
    dim = 512 ,
    dim_context = 768 ,
    depth = 6 ,
    unconditional = False
)

phenaki = Phenaki (
    cvivit = cvivit ,
    maskgit = maskgit
). cuda ()

# pass in the folder to images or video

trainer = PhenakiTrainer (
    phenaki = phenaki ,
    batch_size = 4 ,
    grad_accum_every = 4 ,
    train_on_images = True ,                # for sake of example, bottom is folder of images
    dataset = '/path/to/images/or/video'
)

trainer . train ()

토도

인용

 @article { Villegas2022PhenakiVL ,
    title   = { Phenaki: Variable Length Video Generation From Open Domain Textual Description } ,
    author  = { Ruben Villegas and Mohammad Babaeizadeh and Pieter-Jan Kindermans and Hernan Moraldo and Han Zhang and Mohammad Taghi Saffar and Santiago Castro and Julius Kunze and D. Erhan } ,
    journal = { ArXiv } ,
    year    = { 2022 } ,
    volume  = { abs/2210.02399 }
}

 @article { Chang2022MaskGITMG ,
    title   = { MaskGIT: Masked Generative Image Transformer } ,
    author  = { Huiwen Chang and Han Zhang and Lu Jiang and Ce Liu and William T. Freeman } ,
    journal = { 2022 IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR) } ,
    year    = { 2022 } ,
    pages   = { 11305-11315 }
}

 @article { Lezama2022ImprovedMI ,
    title   = { Improved Masked Image Generation with Token-Critic } ,
    author  = { Jos{'e} Lezama and Huiwen Chang and Lu Jiang and Irfan Essa } ,
    journal = { ArXiv } ,
    year    = { 2022 } ,
    volume  = { abs/2209.04439 }
}

 @misc { ding2021cogview ,
    title   = { CogView: Mastering Text-to-Image Generation via Transformers } ,
    author  = { Ming Ding and Zhuoyi Yang and Wenyi Hong and Wendi Zheng and Chang Zhou and Da Yin and Junyang Lin and Xu Zou and Zhou Shao and Hongxia Yang and Jie Tang } ,
    year    = { 2021 } ,
    eprint  = { 2105.13290 } ,
    archivePrefix = { arXiv } ,
    primaryClass = { cs.CV }
}

 @misc { shazeer2020glu ,
    title   = { GLU Variants Improve Transformer } ,
    author  = { Noam Shazeer } ,
    year    = { 2020 } ,
    url     = { https://arxiv.org/abs/2002.05202 }
}

 @misc { press2021ALiBi ,
    title   = { Train Short, Test Long: Attention with Linear Biases Enable Input Length Extrapolation } ,
    author  = { Ofir Press and Noah A. Smith and Mike Lewis } ,
    year    = { 2021 } ,
    url     = { https://ofir.io/train_short_test_long.pdf }
}

 @article { Liu2022SwinTV ,
    title   = { Swin Transformer V2: Scaling Up Capacity and Resolution } ,
    author  = { Ze Liu and Han Hu and Yutong Lin and Zhuliang Yao and Zhenda Xie and Yixuan Wei and Jia Ning and Yue Cao and Zheng Zhang and Li Dong and Furu Wei and Baining Guo } ,
    journal = { 2022 IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR) } ,
    year    = { 2022 } ,
    pages   = { 11999-12009 }
}

 @inproceedings { Nijkamp2021SCRIPTSP ,
    title   = { SCRIPT: Self-Critic PreTraining of Transformers } ,
    author  = { Erik Nijkamp and Bo Pang and Ying Nian Wu and Caiming Xiong } ,
    booktitle = { North American Chapter of the Association for Computational Linguistics } ,
    year    = { 2021 }
}

 @misc { https://doi.org/10.48550/arxiv.2302.01327 ,
    doi     = { 10.48550/ARXIV.2302.01327 } ,
    url     = { https://arxiv.org/abs/2302.01327 } ,
    author  = { Kumar, Manoj and Dehghani, Mostafa and Houlsby, Neil } ,
    title   = { Dual PatchNorm } ,
    publisher = { arXiv } ,
    year    = { 2023 } ,
    copyright = { Creative Commons Attribution 4.0 International }
}

 @misc { gilmer2023intriguing
    title  = { Intriguing Properties of Transformer Training Instabilities } ,
    author = { Justin Gilmer, Andrea Schioppa, and Jeremy Cohen } ,
    year   = { 2023 } ,
    status = { to be published - one attention stabilization technique is circulating within Google Brain, being used by multiple teams }
}

 @misc { mentzer2023finite ,
    title   = { Finite Scalar Quantization: VQ-VAE Made Simple } ,
    author  = { Fabian Mentzer and David Minnen and Eirikur Agustsson and Michael Tschannen } ,
    year    = { 2023 } ,
    eprint  = { 2309.15505 } ,
    archivePrefix = { arXiv } ,
    primaryClass = { cs.CV }
}

 @misc { yu2023language ,
    title   = { Language Model Beats Diffusion -- Tokenizer is Key to Visual Generation } ,
    author  = { Lijun Yu and José Lezama and Nitesh B. Gundavarapu and Luca Versari and Kihyuk Sohn and David Minnen and Yong Cheng and Agrim Gupta and Xiuye Gu and Alexander G. Hauptmann and Boqing Gong and Ming-Hsuan Yang and Irfan Essa and David A. Ross and Lu Jiang } ,
    year    = { 2023 } ,
    eprint  = { 2310.05737 } ,
    archivePrefix = { arXiv } ,
    primaryClass = { cs.CV }
}