from transformers import AutoConfig, AutoModelForCausalLM, AutoTokenizer
model_ckpt = "codeparrot"
org = "transformersbook"
def model_size(model):
    return sum(t.numel() for t in model.parameters())
tokenizer = AutoTokenizer.from_pretrained(org + "/" + model_ckpt)
config_small = AutoConfig.from_pretrained("gpt2", vocab_size=len(tokenizer))
model_small = AutoModelForCausalLM.from_config(config_small)

print(f'GPT-2 size: {model_size(model_small)/1000**2:.1f}M parameters')
     
model_small.save_pretrained("models/" + model_ckpt + "-small", push_to_hub=True)