File size: 5,676 Bytes

e38171c

ds_cfg:
  train_micro_batch_size_per_gpu: ${per_gpu_train_batch_size}
  gradient_accumulation_steps: ${gradient_accumulation_steps}
  scheduler:
    type: WarmupDecayLR
    params:
      total_num_steps: 989
      warmup_max_lr: ${learning_rate}
      warmup_num_steps: 59
      warmup_type: linear
  optimizer:
    type: AdamW
    params:
      lr: ${learning_rate}
      betas:
      - 0.9
      - 0.95
      eps: 1.0e-06
      weight_decay: ${weight_decay}
  bf16:
    enabled: true
  zero_optimization:
    stage: 1
    stage3_param_persistence_threshold: 100000.0
    stage3_max_live_parameters: 100000000.0
    stage3_prefetch_bucket_size: 100000000.0
    memory_efficient_linear: false
  steps_per_print: 25
  gradient_clipping: 1.0
  prescale_gradients: false
sft_model_dir: experiments/gemma.2b.it.meta_math_distil.H100.w4.v1.0/checkpoint-2000/
train_file: ${sft_model_dir}/meta_math_sub.25k.rap.train.0shot.n10.tem1.0.p0.7.v1.0_clean_fix.prm_cp3200_best_of_10.v1.0.(2,3).pos0.5.neg10.product.full_only_pos2pos.json
dev_file: null
test_file: null
torch_dtype:
  _target_: general_util.training_utils.return_torch_dtype
  dtype: bfloat16
tokenizer_init:
  _target_: general_util.tokenization_utils.init_tokenizer
  tokenizer_path: experiments/gemma.2b.it.meta_math_distil.H100.w4.v1.0/checkpoint-2000/
  padding_side: left
device_map:
  _target_: models.utils.return_single_device_map
model:
  _target_: models.gemma.GemmaForCausalLMDPO.from_pretrained_with_ref_model
  beta: 0.5
  gradient_checkpointing: false
  attn_implementation: flash_attention_2
  torch_dtype: ${torch_dtype}
  device_map: ${device_map}
  ref_model:
    _target_: models.gemma.GemmaForCausalLM.from_pretrained
    pretrained_model_name_or_path: ${model_name_or_path}
    torch_dtype: ${torch_dtype}
    attn_implementation: flash_attention_2
    device_map: ${device_map}
read_tensor_dpo:
  _target_: data.logic_combine.MultiMappingDataset
  file_path: experiments/gemma.2b.it.meta_math_distil.H100.w4.v1.0/checkpoint-2000//meta_math_sub.25k.rap.train.0shot.n10.tem1.0.p0.7.v1.0_clean_fix.json
  aligner:
    _target_: data.input_aligner.concat_aligner
    aligners:
    - _target_: data.input_aligner.dpo_pair_aligner_cleaned
      response_field: response
      id_field: id
      do_sample: false
  template:
    chosen: '{instruction}


      ### Question: {query}


      SubQuestion 1: {pos}<eos>'
    reject: '{instruction}


      ### Question: {query}


      SubQuestion 1: {neg}<eos>'
    prompt: '{instruction}


      ### Question: {query}


      SubQuestion 1:'
  instruction: 'Given a question, please decompose it into sub-questions. For each
    sub-question, please answer it in a complete sentence, ending with "The answer
    is". When the original question is answerable, please start the sub-question with
    "Now we can answer the question: ".'
  kv_mapping:
    chosen: chosen
    reject: reject
    id: index
    prompt: prompt
read_tensor_step_dpo:
  _target_: data.logic_combine.MultiMappingDataset
  aligner:
    _target_: data.logic_combine.field_extract_aligner
    input_index_field: id
    extract_index_field: id
    extract_fields:
    - query
    extra_file: experiments/gemma.2b.it.meta_math_distil.H100.w4.v1.0/checkpoint-2000//meta_math_sub.25k.rap.train.0shot.n10.tem1.0.p0.7.v1.0_clean_fix.json
  template:
    chosen: '{instruction}


      ### Question: {query}


      SubQuestion 1: {chosen}<eos>'
    reject: '{instruction}


      ### Question: {query}


      SubQuestion 1: {reject}<eos>'
    prompt: '{instruction}


      ### Question: {query}


      SubQuestion 1:'
  instruction: 'Given a question, please decompose it into sub-questions. For each
    sub-question, please answer it in a complete sentence, ending with "The answer
    is". When the original question is answerable, please start the sub-question with
    "Now we can answer the question: ".'
  kv_mapping:
    chosen: chosen
    reject: reject
    id: index
    prompt: prompt
read_tensor:
  _target_: data.general.ReplayDataset
  _recursive_: false
  new_dataset_cfg: ${read_tensor_step_dpo}
  old_dataset_cfg: ${read_tensor_dpo}
  replay_ratio: 0.2
dist_load_data_barrier: false
extended_vocab: null
collator:
  _target_: data.dpo.DPOCollator
  tokenizer: ${tokenizer_init}
  max_seq_length: 1024
num_workers: 8
prefetch_factor: 2
model_name_or_path: ${sft_model_dir}
pretrain: null
dp_size: 4
tp_size: 1
pp_size: 1
exp_name: gemma.2b.it.meta_math_rap.step.dpo.H100.w4.v1.0.s${seed}
exp_notes: null
output_dir: experiments/${exp_name}
do_train: true
evaluate_during_training: false
do_eval: false
eval_sub_path: checkpoint-100
per_gpu_train_batch_size: 2
per_gpu_eval_batch_size: 4
learning_rate: 1.0e-06
gradient_accumulation_steps: 8
weight_decay: 0.1
adam_epsilon: 1.0e-06
adam_betas: (0.9, 0.98)
total_dataset_len: 63348
max_grad_norm: 1.0
num_train_epochs: 1
max_steps: 0
warmup_proportion: 0.06
warmup_steps: 0
optimizer: null
use_nvlamb: null
bit_training: null
logging_steps: 5
save_ds_state: false
save_steps: 100
save_best: false
eval_steps: 400
ddp_eval: true
no_cuda: false
seed: 43
local_rank: 0
fp16: true
fp16_opt_level: O1
fp16_bfloat16: true
prediction_cfg:
  metric: loss
  measure: -1
  best_checkpoint: null
  best_result: null
eval_forward_fn:
  _target_: general_util.evaluator.DefaultForwardFn
post_process:
  _target_: post_processors.dpo.DPOEvalPostProcessor
summary_helper:
  _target_: general_util.tensorboard_helper.WandbWriter
  batch_index_or_keys: null
  outputs_index_or_keys:
    train/chosen_reward: chosen_reward
    train/rejected_reward: rejected_reward
n_gpu: 1
device: cuda:0
train_batch_size: 2
eval_batch_size: null
world_size: 4