gpt2_ppo.yml

tokenizer:
  model_name: gpt2
  padding_side: left
  truncation_side: left
  pad_token_as_eos_token: True

reward_fn:
  id: "intent_accuracy"
  args:
    intent_coeff: 0.75
    auto_coeff: 0.25


datapool:
  id: "daily_dialog"
  args:
    context_size: 5

env:
  n_envs: 10
  args:
    max_prompt_length: 128
    max_episode_length: 20
    terminate_on_eos: True

alg:
  id: ppo
  args: 
    n_steps: 128
    batch_size: 64
    verbose: 1
    learning_rate: 0.000001
    n_epochs: 5
    
  kl_div:
    coeff: 0.2
    target_kl: 0.5
 
  policy:
    id: causal_lm_actor_critic_policy
    args:
      model_name: gpt2
      apply_model_parallel: True
      generation_kwargs:
        do_sample: True
        top_k: 20
        min_length: 2
        max_new_tokens: 20
      
train_evaluation:
  eval_batch_size: 32
  n_iters: 50
  eval_every: 5
  save_every: 10
  metrics:
    - id: intent_accuracy
    - id: causal_perplexity
      args:
        tokenizer_id: gpt2
        stride: 128
        model_type: causal
    - id: diversity
      args: {}
    - id: meteor
      args: {}
    - id: rouge
    - id: bleu
      args: {}
    - id: bert_score
      args:
        language: en
    - id: sacre_bleu
      args:
        tokenize: "intl"
  generation_kwargs:
    do_sample: True
    top_k: 20
    min_length: 2
    max_new_tokens: 20