Upload llama_and_8k_v3_seed_888/flexitok--llama-3.2-1b-tokenizer_overlap.json with huggingface_hub

llama_and_8k_v3_seed_888/flexitok--llama-3.2-1b-tokenizer_overlap.json

@@ -0,0 +1 @@
+{"1": {"ratio_to_total_tokens": 0.6969030688622755, "expected_training_ratio_in_superset": 0.03167741222101252, "num_tokens": 89382}, "2": {"ratio_to_total_tokens": 0.1986183882235529, "expected_training_ratio_in_superset": 0.018056217111232082, "num_tokens": 25474}, "3": {"ratio_to_total_tokens": 0.050523952095808386, "expected_training_ratio_in_superset": 0.006889629831246598, "num_tokens": 6480}, "4": {"ratio_to_total_tokens": 0.013091005489021956, "expected_training_ratio_in_superset": 0.0023801828161858106, "num_tokens": 1679}, "5": {"ratio_to_total_tokens": 0.007227732035928144, "expected_training_ratio_in_superset": 0.001642666371801851, "num_tokens": 927}, "6": {"ratio_to_total_tokens": 0.004506611776447106, "expected_training_ratio_in_superset": 0.0012290759390310288, "num_tokens": 578}, "7": {"ratio_to_total_tokens": 0.003274700598802395, "expected_training_ratio_in_superset": 0.0010419501905280348, "num_tokens": 420}, "8": {"ratio_to_total_tokens": 0.002307884231536926, "expected_training_ratio_in_superset": 0.0008392306296497913, "num_tokens": 296}, "9": {"ratio_to_total_tokens": 0.0018790543912175649, "expected_training_ratio_in_superset": 0.0007687040691344584, "num_tokens": 241}, "10": {"ratio_to_total_tokens": 0.0017075224550898204, "expected_training_ratio_in_superset": 0.0007761465704953729, "num_tokens": 219}, "11": {"ratio_to_total_tokens": 0.0012864895209580838, "expected_training_ratio_in_superset": 0.0006432447604790419, "num_tokens": 165}, "12": {"ratio_to_total_tokens": 0.001278692614770459, "expected_training_ratio_in_superset": 0.0006974686989657049, "num_tokens": 164}, "13": {"ratio_to_total_tokens": 0.0010525823353293414, "expected_training_ratio_in_superset": 0.000621980470876429, "num_tokens": 135}, "14": {"ratio_to_total_tokens": 0.0007485029940119761, "expected_training_ratio_in_superset": 0.00047632008709853023, "num_tokens": 96}, "15": {"ratio_to_total_tokens": 0.0010525823353293414, "expected_training_ratio_in_superset": 0.0007176697740881872, "num_tokens": 135}, "16": {"ratio_to_total_tokens": 0.0010525823353293414, "expected_training_ratio_in_superset": 0.0007655144256940665, "num_tokens": 135}, "17": {"ratio_to_total_tokens": 0.0004132360279441118, "expected_training_ratio_in_superset": 0.00031931874886590453, "num_tokens": 53}, "18": {"ratio_to_total_tokens": 0.0007017215568862275, "expected_training_ratio_in_superset": 0.0005741358192705499, "num_tokens": 90}, "19": {"ratio_to_total_tokens": 0.0003508607784431138, "expected_training_ratio_in_superset": 0.00030301612683723464, "num_tokens": 45}, "20": {"ratio_to_total_tokens": 0.00037425149700598805, "expected_training_ratio_in_superset": 0.00034022863364180725, "num_tokens": 48}, "21": {"ratio_to_total_tokens": 0.000280688622754491, "expected_training_ratio_in_superset": 0.00026793004899292326, "num_tokens": 36}, "22": {"ratio_to_total_tokens": 0.011367889221556886, "expected_training_ratio_in_superset": 0.011367889221556886, "num_tokens": 1458}, "total_training_compared_to_full_model": 0.08239593256668482}