self rewarding lm pytorch下载 - self rewarding lm pytorch源代码下载

自我奖励语言模型

MetaAI 的自我奖励语言模型中提出的培训框架的实现

他们确实把 DPO 论文的标题牢记在心。

该库还包含 SPIN 的实现，Nous Research 的 Teknium 对此表示乐观。

欣赏

A16Z 开源人工智能资助计划和？感谢慷慨的赞助，以及我的其他赞助商，为我提供了开源当前人工智能研究的独立性

安装

$ pip install self-rewarding-lm-pytorch

用法

 import torch
from torch import Tensor

from self_rewarding_lm_pytorch import (
    SelfRewardingTrainer ,
    create_mock_dataset
)

from x_transformers import TransformerWrapper , Decoder

transformer = TransformerWrapper (
    num_tokens = 256 ,
    max_seq_len = 1024 ,
    attn_layers = Decoder (
        dim = 512 ,
        depth = 1 ,
        heads = 8
    )
)

sft_dataset = create_mock_dataset ( 100 , lambda : ( torch . randint ( 0 , 256 , ( 256 ,)), torch . tensor ( 1 )))
prompt_dataset = create_mock_dataset ( 100 , lambda : 'mock prompt' )

def decode_tokens ( tokens : Tensor ) -> str :
    decode_token = lambda token : str ( chr ( max ( 32 , token )))
    return '' . join ( list ( map ( decode_token , tokens )))

def encode_str ( seq_str : str ) -> Tensor :
    return Tensor ( list ( map ( ord , seq_str )))

trainer = SelfRewardingTrainer (
    transformer ,
    finetune_configs = dict (
        train_sft_dataset = sft_dataset ,
        self_reward_prompt_dataset = prompt_dataset ,
        dpo_num_train_steps = 1000
    ),
    tokenizer_decode = decode_tokens ,
    tokenizer_encode = encode_str ,
    accelerate_kwargs = dict (
        cpu = True
    )
)

trainer ( overwrite_checkpoints = True )

# checkpoints after each finetuning stage will be saved to ./checkpoints

SPIN 可以按如下方式进行训练 - 也可以将其添加到微调管道中，如自述文件中的最后一个示例所示。

 import torch

from self_rewarding_lm_pytorch import (
    SPINTrainer ,
    create_mock_dataset
)

from x_transformers import TransformerWrapper , Decoder

transformer = TransformerWrapper (
    num_tokens = 256 ,
    max_seq_len = 1024 ,
    attn_layers = Decoder (
        dim = 512 ,
        depth = 6 ,
        heads = 8
    )
)

sft_dataset = create_mock_dataset ( 100 , lambda : ( torch . randint ( 0 , 256 , ( 256 ,)), torch . tensor ( 1 )))

spin_trainer = SPINTrainer (
    transformer ,
    max_seq_len = 16 ,
    train_sft_dataset = sft_dataset ,
    checkpoint_every = 100 ,
    spin_kwargs = dict (
        λ = 0.1 ,
    ),
)

spin_trainer ()

假设您想尝试自己的奖励提示（法学硕士法官除外）。首先，您需要导入RewardConfig ，然后将其作为reward_prompt_config传递到训练器中

 # first import

from self_rewarding_lm_pytorch import RewardConfig

# then say you want to try asking the transformer nicely

# reward_regex_template is the string that will be looked for in the LLM response, for parsing out the reward where {{ reward }} is defined as a number

trainer = SelfRewardingTrainer (
    transformer ,
    ...,
    self_reward_prompt_config = RewardConfig (
        prompt_template = """
        Pretty please rate the following user prompt and response
        User: {{ prompt }}
        Response: {{ response }}

        Format your score as follows:
        Rating: <rating as integer from 0 - 10>
        """ ,
        reward_regex_template = """
        Rating: {{ reward }}
        """
    )
)

最后，如果您想尝试任意顺序的微调，您也可以通过将FinetuneConfig实例作为列表传递到finetune_configs来获得这种灵活性

前任。假设你想对交错 SPIN、外部奖励和自我奖励进行研究

这个想法源自 Teknium 的一个私人不和谐频道。

 # import the configs

from self_rewarding_lm_pytorch import (
    SFTConfig ,
    SelfRewardDPOConfig ,
    ExternalRewardDPOConfig ,
    SelfPlayConfig ,
)

trainer = SelfRewardingTrainer (
    model ,
    finetune_configs = [
        SFTConfig (...),
        SelfPlayConfig (...),
        ExternalRewardDPOConfig (...),
        SelfRewardDPOConfig (...),
        SelfPlayConfig (...),
        SelfRewardDPOConfig (...)
    ],
    ...
)

trainer ()

# checkpoints after each finetuning stage will be saved to ./checkpoints

托多

引文

 @misc { yuan2024selfrewarding ,
    title   = { Self-Rewarding Language Models } , 
    author  = { Weizhe Yuan and Richard Yuanzhe Pang and Kyunghyun Cho and Sainbayar Sukhbaatar and Jing Xu and Jason Weston } ,
    year    = { 2024 } ,
    eprint  = { 2401.10020 } ,
    archivePrefix = { arXiv } ,
    primaryClass = { cs.CL }
}

 @article { Chen2024SelfPlayFC ,
    title   = { Self-Play Fine-Tuning Converts Weak Language Models to Strong Language Models } ,
    author  = { Zixiang Chen and Yihe Deng and Huizhuo Yuan and Kaixuan Ji and Quanquan Gu } ,
    journal = { ArXiv } ,
    year    = { 2024 } ,
    volume  = { abs/2401.01335 } ,
    url     = { https://api.semanticscholar.org/CorpusID:266725672 }
}

 @article { Rafailov2023DirectPO ,
    title   = { Direct Preference Optimization: Your Language Model is Secretly a Reward Model } ,
    author  = { Rafael Rafailov and Archit Sharma and Eric Mitchell and Stefano Ermon and Christopher D. Manning and Chelsea Finn } ,
    journal = { ArXiv } ,
    year    = { 2023 } ,
    volume  = { abs/2305.18290 } ,
    url     = { https://api.semanticscholar.org/CorpusID:258959321 }
}

 @inproceedings { Guo2024DirectLM ,
    title   = { Direct Language Model Alignment from Online AI Feedback } ,
    author  = { Shangmin Guo and Biao Zhang and Tianlin Liu and Tianqi Liu and Misha Khalman and Felipe Llinares and Alexandre Rame and Thomas Mesnard and Yao Zhao and Bilal Piot and Johan Ferret and Mathieu Blondel } ,
    year    = { 2024 } ,
    url     = { https://api.semanticscholar.org/CorpusID:267522951 }
}