Upload 135M base checkpoint trained on 10B FineWeb-Edu tokens

README.md

@@ -0,0 +1,79 @@
+---
+license: apache-2.0
+language:
+- en
+library_name: transformers
+pipeline_tag: text-generation
+tags:
+- causal-lm
+- pretraining
+- from-scratch
+- fineweb-edu
+- single-gpu
+- l20
+datasets:
+- HuggingFaceFW/fineweb-edu
+---
+
+# l20-edu-135m
+
+`l20-edu-135m` is a 134.5M-parameter causal language model pretrained from scratch on a single NVIDIA L20 GPU.
+
+This is a base model checkpoint, not an instruction-tuned chat model. It is intended for research, evaluation, and downstream fine-tuning.
+
+## Model Details
+
+- Architecture: Llama-style decoder-only Transformer
+- Parameters: 134,515,008
+- Layers: 30
+- Hidden size: 576
+- FFN size: 1536
+- Attention heads: 9 query heads, 3 key/value heads
+- Tokenizer: SmolLM2-135M tokenizer
+- Training data: FineWeb-Edu sample-10BT
+- Training tokens: 10,001,252,352 planned tokens
+- Final checkpoint: step 18,928
+- Hardware: single NVIDIA L20 GPU
+- Precision: mixed precision training
+
+## Evaluation
+
+Final validation:
+
+- Loss: 2.8731
+- Perplexity: 17.69
+
+lm-eval results for the final checkpoint:
+
+| Task | Metric | Score |
+| --- | --- | ---: |
+| ARC-Challenge | acc_norm | 0.2765 |
+| ARC-Easy | acc_norm | 0.5059 |
+| HellaSwag | acc_norm | 0.3272 |
+| LAMBADA OpenAI | acc | 0.2540 |
+| PIQA | acc_norm | 0.6224 |
+| Winogrande | acc | 0.5099 |
+
+Compared with public 100M-160M baselines on the same lm-eval task set, this model is competitive with several older baselines but is below modern heavily overtrained compact models such as SmolLM and SmolLM2, which use substantially larger pretraining budgets.
+
+## Intended Use
+
+This checkpoint is suitable for:
+
+- base model evaluation
+- continued pretraining experiments
+- supervised fine-tuning experiments
+- small-model training pipeline demonstrations
+
+It is not suitable as a production assistant without post-training, safety evaluation, and domain-specific validation.
+
+## Limitations
+
+- This is a small base model trained on 10B tokens.
+- It is not instruction-tuned and may not follow user requests reliably.
+- It can produce incorrect facts, repetitions, or incomplete generations.
+- Results should not be described as SOTA without controlled baselines and matched training-budget comparisons.
+
+## Citation
+
+If you use this checkpoint, please cite or link to this repository and include the training-token budget when comparing against other compact language models.

config.json

@@ -0,0 +1,30 @@
+{
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 0,
+  "dtype": "bfloat16",
+  "eos_token_id": 0,
+  "head_dim": 64,
+  "hidden_act": "silu",
+  "hidden_size": 576,
+  "initializer_range": 0.02,
+  "intermediate_size": 1536,
+  "max_position_embeddings": 2048,
+  "mlp_bias": false,
+  "model_type": "llama",
+  "num_attention_heads": 9,
+  "num_hidden_layers": 30,
+  "num_key_value_heads": 3,
+  "pad_token_id": 0,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": null,
+  "rope_theta": 10000.0,
+  "tie_word_embeddings": true,
+  "transformers_version": "4.57.3",
+  "use_cache": false,
+  "vocab_size": 49152
+}

eval/comparison.json

@@ -0,0 +1,213 @@
+{
+  "candidate": "l20-edu-135m-deepthin",
+  "tasks": {
+    "lambada_openai": {
+      "metric": "acc,none",
+      "value": 0.25402678051620414
+    },
+    "hellaswag": {
+      "metric": "acc_norm,none",
+      "value": 0.3272256522605059
+    },
+    "piqa": {
+      "metric": "acc_norm,none",
+      "value": 0.6224156692056583
+    },
+    "arc_easy": {
+      "metric": "acc_norm,none",
+      "value": 0.5058922558922558
+    },
+    "arc_challenge": {
+      "metric": "acc_norm,none",
+      "value": 0.2764505119453925
+    },
+    "winogrande": {
+      "metric": "acc,none",
+      "value": 0.5098658247829518
+    }
+  },
+  "baselines": {
+    "gpt2-small": {
+      "lambada_openai": {
+        "metric": "acc,none",
+        "value": 0.30758781292451
+      },
+      "hellaswag": {
+        "metric": "acc_norm,none",
+        "value": 0.31378211511651066
+      },
+      "piqa": {
+        "metric": "acc_norm,none",
+        "value": 0.6207834602829162
+      },
+      "arc_easy": {
+        "metric": "acc_norm,none",
+        "value": 0.39730639730639733
+      },
+      "arc_challenge": {
+        "metric": "acc_norm,none",
+        "value": 0.22610921501706485
+      },
+      "winogrande": {
+        "metric": "acc,none",
+        "value": 0.5067087608524072
+      }
+    },
+    "opt-125m": {
+      "lambada_openai": {
+        "metric": "acc,none",
+        "value": 0.3856006209974772
+      },
+      "hellaswag": {
+        "metric": "acc_norm,none",
+        "value": 0.31597291376219877
+      },
+      "piqa": {
+        "metric": "acc_norm,none",
+        "value": 0.6202393906420022
+      },
+      "arc_easy": {
+        "metric": "acc_norm,none",
+        "value": 0.398989898989899
+      },
+      "arc_challenge": {
+        "metric": "acc_norm,none",
+        "value": 0.22098976109215018
+      },
+      "winogrande": {
+        "metric": "acc,none",
+        "value": 0.5177584846093133
+      }
+    },
+    "gpt-neo-125m": {
+      "lambada_openai": {
+        "metric": "acc,none",
+        "value": 0.37647972055113527
+      },
+      "hellaswag": {
+        "metric": "acc_norm,none",
+        "value": 0.30551682931686913
+      },
+      "piqa": {
+        "metric": "acc_norm,none",
+        "value": 0.6213275299238302
+      },
+      "arc_easy": {
+        "metric": "acc_norm,none",
+        "value": 0.39646464646464646
+      },
+      "arc_challenge": {
+        "metric": "acc_norm,none",
+        "value": 0.23208191126279865
+      },
+      "winogrande": {
+        "metric": "acc,none",
+        "value": 0.5098658247829518
+      }
+    },
+    "cerebras-gpt-111m": {
+      "lambada_openai": {
+        "metric": "acc,none",
+        "value": 0.19115078594993207
+      },
+      "hellaswag": {
+        "metric": "acc_norm,none",
+        "value": 0.2719577773351922
+      },
+      "piqa": {
+        "metric": "acc_norm,none",
+        "value": 0.5810663764961915
+      },
+      "arc_easy": {
+        "metric": "acc_norm,none",
+        "value": 0.35058922558922556
+      },
+      "arc_challenge": {
+        "metric": "acc_norm,none",
+        "value": 0.2098976109215017
+      },
+      "winogrande": {
+        "metric": "acc,none",
+        "value": 0.49013417521704816
+      }
+    },
+    "pythia-160m": {
+      "lambada_openai": {
+        "metric": "acc,none",
+        "value": 0.12245294003493111
+      },
+      "hellaswag": {
+        "metric": "acc_norm,none",
+        "value": 0.30302728540131446
+      },
+      "piqa": {
+        "metric": "acc_norm,none",
+        "value": 0.5979325353645266
+      },
+      "arc_easy": {
+        "metric": "acc_norm,none",
+        "value": 0.3640572390572391
+      },
+      "arc_challenge": {
+        "metric": "acc_norm,none",
+        "value": 0.23122866894197952
+      },
+      "winogrande": {
+        "metric": "acc,none",
+        "value": 0.5074980268350434
+      }
+    },
+    "smollm-135m": {
+      "lambada_openai": {
+        "metric": "acc,none",
+        "value": 0.3757034737046381
+      },
+      "hellaswag": {
+        "metric": "acc_norm,none",
+        "value": 0.42650866361282613
+      },
+      "piqa": {
+        "metric": "acc_norm,none",
+        "value": 0.6822633297062024
+      },
+      "arc_easy": {
+        "metric": "acc_norm,none",
+        "value": 0.561026936026936
+      },
+      "arc_challenge": {
+        "metric": "acc_norm,none",
+        "value": 0.28754266211604096
+      },
+      "winogrande": {
+        "metric": "acc,none",
+        "value": 0.5272296764009471
+      }
+    },
+    "smollm2-135m": {
+      "lambada_openai": {
+        "metric": "acc,none",
+        "value": 0.4288763826896953
+      },
+      "hellaswag": {
+        "metric": "acc_norm,none",
+        "value": 0.43009360685122483
+      },
+      "piqa": {
+        "metric": "acc_norm,none",
+        "value": 0.6838955386289445
+      },
+      "arc_easy": {
+        "metric": "acc_norm,none",
+        "value": 0.5854377104377104
+      },
+      "arc_challenge": {
+        "metric": "acc_norm,none",
+        "value": 0.29692832764505117
+      },
+      "winogrande": {
+        "metric": "acc,none",
+        "value": 0.5248618784530387
+      }
+    }
+  }
+}

eval/comparison.md

@@ -0,0 +1,22 @@
+# lm-eval Comparison
+
+Candidate: `l20-edu-135m-deepthin`
+
+| Task | Metric | l20-edu-135m-deepthin | gpt2-small | opt-125m | gpt-neo-125m | cerebras-gpt-111m | pythia-160m | smollm-135m | smollm2-135m |
+| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |
+| arc_challenge | acc_norm,none | 0.2765 | 0.2261 | 0.2210 | 0.2321 | 0.2099 | 0.2312 | 0.2875 | 0.2969 |
+| arc_easy | acc_norm,none | 0.5059 | 0.3973 | 0.3990 | 0.3965 | 0.3506 | 0.3641 | 0.5610 | 0.5854 |
+| hellaswag | acc_norm,none | 0.3272 | 0.3138 | 0.3160 | 0.3055 | 0.2720 | 0.3030 | 0.4265 | 0.4301 |
+| lambada_openai | acc,none | 0.2540 | 0.3076 | 0.3856 | 0.3765 | 0.1912 | 0.1225 | 0.3757 | 0.4289 |
+| piqa | acc_norm,none | 0.6224 | 0.6208 | 0.6202 | 0.6213 | 0.5811 | 0.5979 | 0.6823 | 0.6839 |
+| winogrande | acc,none | 0.5099 | 0.5067 | 0.5178 | 0.5099 | 0.4901 | 0.5075 | 0.5272 | 0.5249 |
+
+## Win Rates
+
+- `gpt2-small`: 5/6 = 0.833
+- `opt-125m`: 4/6 = 0.667
+- `gpt-neo-125m`: 4/6 = 0.667
+- `cerebras-gpt-111m`: 6/6 = 1.000
+- `pythia-160m`: 6/6 = 1.000
+- `smollm-135m`: 0/6 = 0.000
+- `smollm2-135m`: 0/6 = 0.000

generation_config.json

@@ -0,0 +1,7 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 0,
+  "eos_token_id": 0,
+  "pad_token_id": 0,
+  "transformers_version": "4.57.3"
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7c53c309205a87b2cd1a113f26769dcb8e341064ed0ecda5afbd26cc30e47502
+size 269060552

pretrain_config.yaml

@@ -0,0 +1,51 @@
+run_name: l20-edu-135m-deepthin
+output_dir: runs/l20-edu-135m-deepthin
+seed: 1337
+tokenizer_name: HuggingFaceTB/SmolLM2-135M
+dataset:
+  name: HuggingFaceFW/fineweb-edu
+  config_name: sample-10BT
+  split: train
+  streaming: true
+  text_column: text
+  min_chars: 300
+  max_chars: 50000
+  min_score: 3.0
+  min_int_score: 3
+  append_eos: true
+  shuffle_buffer: 10000
+  max_docs: null
+  local_text_path: null
+model:
+  block_size: 2048
+  hidden_size: 576
+  intermediate_size: 1536
+  num_hidden_layers: 30
+  num_attention_heads: 9
+  num_key_value_heads: 3
+  rope_theta: 10000.0
+  rms_norm_eps: 1.0e-06
+  attention_dropout: 0.0
+  tie_word_embeddings: true
+  vocab_multiple: 64
+  attn_implementation: sdpa
+trainer:
+  micro_batch_size: 6
+  gradient_accumulation_steps: 43
+  max_steps: 18928
+  warmup_steps: 1000
+  learning_rate: 0.0004
+  min_lr_ratio: 0.1
+  weight_decay: 0.1
+  beta1: 0.9
+  beta2: 0.95
+  grad_clip: 1.0
+  dtype: bfloat16
+  compile: true
+  gradient_checkpointing: true
+  log_interval: 10
+  eval_interval: 500
+  eval_batches: 64
+  save_interval: 1000
+  keep_last_checkpoints: 2
+  num_workers: 0

special_tokens_map.json

@@ -0,0 +1,43 @@
+{
+  "additional_special_tokens": [
+    "<|endoftext|>",
+    "<|im_start|>",
+    "<|im_end|>",
+    "<repo_name>",
+    "<reponame>",
+    "<file_sep>",
+    "<filename>",
+    "<gh_stars>",
+    "<issue_start>",
+    "<issue_comment>",
+    "<issue_closed>",
+    "<jupyter_start>",
+    "<jupyter_text>",
+    "<jupyter_code>",
+    "<jupyter_output>",
+    "<jupyter_script>",
+    "<empty_output>"
+  ],
+  "bos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "<|endoftext|>",
+  "unk_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer_config.json

@@ -0,0 +1,169 @@
+{
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "<repo_name>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "4": {
+      "content": "<reponame>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "5": {
+      "content": "<file_sep>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "6": {
+      "content": "<filename>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "7": {
+      "content": "<gh_stars>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "8": {
+      "content": "<issue_start>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "9": {
+      "content": "<issue_comment>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "10": {
+      "content": "<issue_closed>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "11": {
+      "content": "<jupyter_start>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "12": {
+      "content": "<jupyter_text>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "13": {
+      "content": "<jupyter_code>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "14": {
+      "content": "<jupyter_output>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "15": {
+      "content": "<jupyter_script>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "16": {
+      "content": "<empty_output>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [
+    "<|endoftext|>",
+    "<|im_start|>",
+    "<|im_end|>",
+    "<repo_name>",
+    "<reponame>",
+    "<file_sep>",
+    "<filename>",
+    "<gh_stars>",
+    "<issue_start>",
+    "<issue_comment>",
+    "<issue_closed>",
+    "<jupyter_start>",
+    "<jupyter_text>",
+    "<jupyter_code>",
+    "<jupyter_output>",
+    "<jupyter_script>",
+    "<empty_output>"
+  ],
+  "bos_token": "<|endoftext|>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|endoftext|>",
+  "extra_special_tokens": {},
+  "model_max_length": 8192,
+  "pad_token": "<|endoftext|>",
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|endoftext|>",
+  "vocab_size": 49152
+}