| activation: softmax |
| adam_beta1: 0.9 |
| adam_beta2: 0.99 |
| adam_epsilon: 1.0e-06 |
| alpha: 0.1 |
| attn_implementation: null |
| beta: 0.125 |
| bf16: true |
| block_size: 512 |
| checkpoint_dir: mlruns/896390784617014591/892b97fa0aa6499288906c463545ae00/checkpoints |
| compile: false |
| config_path: configs/JZ/NRJ_base-wiki-original.yaml |
| dataloader_num_workers: 8 |
| dataset_path: /lustre/fswork/projects/rech/oou/uqh26ve/data/pre_training/en/en_wiki/wiki_20220301-cleaned-valid001/data-bin/wiki_20220301-cleaned-valid001-BPE30K/ |
| ddp_find_unused_parameters: false |
| disable_tqdm: true |
| do_eval: true |
| dropout: 0.1 |
| embedding_dim: 768 |
| eval_steps: 25000 |
| evaluation_strategy: steps |
| forward_memories: 3072 |
| fp16: false |
| gradient_accumulation_steps: 1 |
| ignore_lines: false |
| layer_norm: 1.0e-12 |
| learning_rate: 0.0007 |
| log_on_each_node: false |
| logging_steps: 1000 |
| logging_strategy: steps |
| lr_scheduler_kwargs: {} |
| lr_scheduler_type: cosine |
| max_steps: 500000 |
| model_name: NRJ-V_30000K_bpe-NL12-NH12-EMB768-FFN3072 |
| model_type: energyBERT |
| n_run: 51 |
| num_heads: 12 |
| num_layers: 12 |
| num_params: 50638896 |
| optimizer: adamw_torch |
| output_dir: null |
| per_device_eval_batch_size: 8 |
| per_device_train_batch_size: 64 |
| remove_unused_columns: false |
| report_to: mlflow |
| save_steps: 25000 |
| save_strategy: steps |
| seed: 42 |
| share_layers: false |
| test_file: /lustre/fswork/projects/rech/oou/uqh26ve/data/pre_training/en/en_wiki/wiki_20220301-cleaned-valid001/wikipedia.test.txt |
| tie_weights: false |
| tokenizer_path: /lustre/fswork/projects/rech/oou/uqh26ve/data/pre_training/en/en_wiki/wiki_20220301-cleaned-valid001/data-bin/wiki_20220301-cleaned-valid001-BPE30K/tokenizer |
| tokenizer_type: bpe |
| total_batch_size: 4096 |
| training_file: /lustre/fswork/projects/rech/oou/uqh26ve/data/pre_training/en/en_wiki/wiki_20220301-cleaned-valid001/wikipedia.train.txt |
| valid_file: /lustre/fswork/projects/rech/oou/uqh26ve/data/pre_training/en/en_wiki/wiki_20220301-cleaned-valid001/wikipedia.valid.txt |
| vocabulary_size: 30000 |
| warmup_ratio: 0.0 |
| warmup_steps: 24000 |
| weight_decay: 0.01 |
|
|
