| batch_size_training: '32' |
| checkpoint_type: StateDictType.FULL_STATE_DICT |
| dataset: foundational_dataset |
| dist_checkpoint_folder: fine-tuned |
| dist_checkpoint_root_folder: test_run_save |
| enable_fsdp: 'True' |
| freeze_layers: 'False' |
| fsdp_activation_checkpointing: 'True' |
| gamma: '0.9' |
| load_peft_model: 'False' |
| low_cpu_fsdp: 'False' |
| lr: '0.0001' |
| micro_batch_size: '32' |
| mixed_precision: 'True' |
| model_name: models/v3/llama7b-full-1e-4_low-chunk1024-009-017 |
| num_epochs: '1' |
| num_freeze_layers: '1' |
| num_workers_dataloader: '2' |
| one_gpu: 'False' |
| optimizer: AdamW |
| output_dir: PATH/to/save/PEFT/model |
| peft_method: lora |
| pure_bf16: 'True' |
| quantization: 'False' |
| run_validation: 'True' |
| save_model: 'True' |
| save_optimizer: 'False' |
| seed: '42' |
| sharding_strategy: ShardingStrategy.FULL_SHARD |
| type_of_model: foundational |
| use_fast_kernels: 'False' |
| use_fp16: 'False' |
| use_peft: 'False' |
| val_batch_size: '64' |
| weight_decay: '0.0' |
|
|
