| { |
| "schema_version": 1, |
| "overview": [ |
| { |
| "label": "Text decisions", |
| "n": 1991, |
| "ours": 95.68056253139126, |
| "jev": 97.23756906077348, |
| "source": "reports/selfjev_4b_treeserver/eval2/report.json", |
| "reference": "reports/external/eval2/typesafe_jev-latest/report.json" |
| }, |
| { |
| "label": "AI response review", |
| "n": 946, |
| "ours": 93.12896405919662, |
| "jev": 92.49471458773785, |
| "source": "reports/selfjev_4b_treeserver/eval_llm/report.json", |
| "reference": "data/all.jsonl.gz: eval_llm / jev", |
| "reference_model": "typesafe/jev-1.13-20260917" |
| }, |
| { |
| "label": "Public-source tasks", |
| "n": 3300, |
| "ours": 83.39393939393939, |
| "jev": 82.0909090909091, |
| "source": "reports/selfjev_4b_treeserver/test/report.json", |
| "reference": "reports/external/full/typesafe_jev-latest/report.json" |
| } |
| ], |
| "public_datasets": [ |
| { |
| "family": "heldout_intent_clinc", |
| "dataset": "CLINC150", |
| "task": "Intent routing", |
| "exposure": "held-out source", |
| "n": 300, |
| "ours": 95.0, |
| "jev": 94.33333333333333, |
| "dataset_url": "https://huggingface.co/datasets/clinc/clinc_oos", |
| "source": "reports/selfjev_4b_treeserver/test/report.json", |
| "reference": "reports/external/full/typesafe_jev-latest/report.json" |
| }, |
| { |
| "family": "heldout_topic_dbpedia", |
| "dataset": "DBpedia", |
| "task": "Topic classification", |
| "exposure": "held-out source", |
| "n": 300, |
| "ours": 98.0, |
| "jev": 98.0, |
| "dataset_url": "https://huggingface.co/datasets/fancyzhx/dbpedia_14", |
| "source": "reports/selfjev_4b_treeserver/test/report.json", |
| "reference": "reports/external/full/typesafe_jev-latest/report.json" |
| }, |
| { |
| "family": "heldout_question_type_trec", |
| "dataset": "TREC", |
| "task": "Question type", |
| "exposure": "held-out source", |
| "n": 300, |
| "ours": 91.33333333333333, |
| "jev": 94.0, |
| "dataset_url": "https://huggingface.co/datasets/SetFit/TREC-QC", |
| "source": "reports/selfjev_4b_treeserver/test/report.json", |
| "reference": "reports/external/full/typesafe_jev-latest/report.json" |
| }, |
| { |
| "family": "heldout_emotion_multiclass", |
| "dataset": "Emotion", |
| "task": "Emotion classification", |
| "exposure": "held-out source", |
| "n": 300, |
| "ours": 57.333333333333336, |
| "jev": 57.666666666666664, |
| "dataset_url": "https://huggingface.co/datasets/dair-ai/emotion", |
| "source": "reports/selfjev_4b_treeserver/test/report.json", |
| "reference": "reports/external/full/typesafe_jev-latest/report.json" |
| }, |
| { |
| "family": "heldout_boolq", |
| "dataset": "BoolQ", |
| "task": "Reading comprehension", |
| "exposure": "held-out source", |
| "n": 300, |
| "ours": 87.66666666666667, |
| "jev": 90.66666666666667, |
| "dataset_url": "https://huggingface.co/datasets/google/boolq", |
| "source": "reports/selfjev_4b_treeserver/test/report.json", |
| "reference": "reports/external/full/typesafe_jev-latest/report.json" |
| }, |
| { |
| "family": "heldout_sentiment_sst2", |
| "dataset": "SST-2", |
| "task": "Sentiment", |
| "exposure": "held-out source", |
| "n": 300, |
| "ours": 91.33333333333333, |
| "jev": 96.66666666666667, |
| "dataset_url": "https://huggingface.co/datasets/stanfordnlp/sst2", |
| "source": "reports/selfjev_4b_treeserver/test/report.json", |
| "reference": "reports/external/full/typesafe_jev-latest/report.json" |
| }, |
| { |
| "family": "hf_intent_banking77", |
| "dataset": "Banking77", |
| "task": "Banking intent", |
| "exposure": "seen source", |
| "n": 300, |
| "ours": 96.33333333333333, |
| "jev": 95.66666666666667, |
| "dataset_url": "https://huggingface.co/datasets/mteb/banking77", |
| "source": "reports/selfjev_4b_treeserver/test/report.json", |
| "reference": "reports/external/full/typesafe_jev-latest/report.json" |
| }, |
| { |
| "family": "hf_topic_agnews", |
| "dataset": "AG News", |
| "task": "News topic", |
| "exposure": "seen source", |
| "n": 300, |
| "ours": 90.33333333333333, |
| "jev": 87.33333333333333, |
| "dataset_url": "https://huggingface.co/datasets/fancyzhx/ag_news", |
| "source": "reports/selfjev_4b_treeserver/test/report.json", |
| "reference": "reports/external/full/typesafe_jev-latest/report.json" |
| }, |
| { |
| "family": "hf_sentiment_tweets", |
| "dataset": "TweetEval", |
| "task": "Tweet sentiment", |
| "exposure": "seen source", |
| "n": 300, |
| "ours": 67.33333333333333, |
| "jev": 64.33333333333333, |
| "dataset_url": "https://huggingface.co/datasets/cardiffnlp/tweet_eval", |
| "source": "reports/selfjev_4b_treeserver/test/report.json", |
| "reference": "reports/external/full/typesafe_jev-latest/report.json" |
| }, |
| { |
| "family": "hf_nli", |
| "dataset": "MNLI", |
| "task": "Entailment (binary)", |
| "exposure": "seen source", |
| "n": 300, |
| "ours": 95.0, |
| "jev": 92.66666666666667, |
| "dataset_url": "https://huggingface.co/datasets/nyu-mll/glue", |
| "source": "reports/selfjev_4b_treeserver/test/report.json", |
| "reference": "reports/external/full/typesafe_jev-latest/report.json" |
| }, |
| { |
| "family": "hf_emotions_multilabel", |
| "dataset": "GoEmotions", |
| "task": "Emotion tags (exact set)", |
| "exposure": "seen source", |
| "n": 300, |
| "ours": 47.666666666666664, |
| "jev": 31.666666666666668, |
| "dataset_url": "https://huggingface.co/datasets/google-research-datasets/go_emotions", |
| "source": "reports/selfjev_4b_treeserver/test/report.json", |
| "reference": "reports/external/full/typesafe_jev-latest/report.json" |
| } |
| ], |
| "accuracy_vs_size": [ |
| { |
| "model": "Qwen3 reranker 0.6B", |
| "nominal_backbone_billions": 0.6, |
| "regime": "stock", |
| "accuracy": 62.18181818181818, |
| "n": 3300, |
| "source": "reports/baseline/test/report.json" |
| }, |
| { |
| "model": "Qwen3 reranker 4B", |
| "nominal_backbone_billions": 4, |
| "regime": "stock", |
| "accuracy": 63.78787878787879, |
| "n": 3300, |
| "source": "reports/baseline_4b/test/report.json" |
| }, |
| { |
| "model": "Qwen3 reranker 8B", |
| "nominal_backbone_billions": 8, |
| "regime": "stock", |
| "accuracy": 67.03030303030303, |
| "n": 3300, |
| "source": "reports/baseline_8b/test/report.json" |
| }, |
| { |
| "model": "Qwen3 + LoRA 0.6B", |
| "nominal_backbone_billions": 0.6, |
| "regime": "early fine-tune", |
| "accuracy": 74.48484848484848, |
| "n": 3300, |
| "source": "reports/lora_pilot/test/report.json" |
| }, |
| { |
| "model": "Qwen3 + LoRA 4B", |
| "nominal_backbone_billions": 4, |
| "regime": "early fine-tune", |
| "accuracy": 80.81818181818181, |
| "n": 3300, |
| "source": "reports/lora_4b/test/report.json" |
| }, |
| { |
| "model": "Qwen3 + LoRA 8B", |
| "nominal_backbone_billions": 8, |
| "regime": "early fine-tune", |
| "accuracy": 80.96969696969697, |
| "n": 3300, |
| "source": "reports/lora_8b/test/report.json" |
| }, |
| { |
| "model": "SelfJev-4B", |
| "nominal_backbone_billions": 4, |
| "regime": "current release", |
| "accuracy": 83.39393939393939, |
| "n": 3300, |
| "source": "reports/selfjev_4b_treeserver/test/report.json" |
| } |
| ], |
| "research_path": [ |
| { |
| "model": "Initial tree", |
| "accuracy": 85.13309894525364, |
| "source": "reports/tree_4b/eval2/report.json", |
| "n": 1991 |
| }, |
| { |
| "model": "Verified hard cases", |
| "accuracy": 90.55750878955298, |
| "source": "reports/tree_4b_r2b/eval2/report.json", |
| "n": 1991 |
| }, |
| { |
| "model": "Instruct backbone", |
| "accuracy": 92.71722752385736, |
| "source": "reports/tree_4b_instruct_r2x64/eval2/report.json", |
| "n": 1991 |
| }, |
| { |
| "model": "Broader data + options", |
| "accuracy": 94.47513812154696, |
| "source": "reports/tree_4b_combo/eval2/report.json", |
| "n": 1991 |
| }, |
| { |
| "model": "Qwen3.5 + tree training", |
| "accuracy": 95.58011049723757, |
| "source": "reports/qwen35_4b_tree/eval2/report.json", |
| "n": 1991 |
| }, |
| { |
| "model": "SelfJev / current engine", |
| "accuracy": 95.68056253139126, |
| "source": "reports/selfjev_4b_treeserver/eval2/report.json", |
| "n": 1991 |
| } |
| ], |
| "latency": { |
| "gpu": [ |
| { |
| "hardware": "A10G", |
| "model": "tree_4b_combo (archived Qwen3-4B)", |
| "engine": "vLLM bf16", |
| "text_tokens": 8, |
| "questions": 1, |
| "candidates_per_question": 3, |
| "timed_calls": 10, |
| "metric": "server_ms", |
| "median_ms": 55.28760100060026, |
| "wall_median_ms": 124.98704150038975, |
| "samples_ms": [ |
| 55.370269999912125, |
| 55.36273899997468, |
| 38.954117999310256, |
| 55.06569299905095, |
| 58.78931900042517, |
| 55.11900499914191, |
| 55.21246300122584, |
| 58.02654000035545, |
| 55.09648099905462, |
| 58.76103499940655 |
| ], |
| "source": "reports/latency/requests_combo.jsonl", |
| "endpoint": "combo_vllm" |
| }, |
| { |
| "hardware": "A10G", |
| "model": "tree_4b_combo (archived Qwen3-4B)", |
| "engine": "vLLM bf16", |
| "text_tokens": 512, |
| "questions": 1, |
| "candidates_per_question": 3, |
| "timed_calls": 10, |
| "metric": "server_ms", |
| "median_ms": 136.1421874998996, |
| "wall_median_ms": 203.85035450090072, |
| "samples_ms": [ |
| 136.60056099979556, |
| 116.28793100135226, |
| 136.12962099978176, |
| 136.00464800038026, |
| 136.15475400001742, |
| 136.4384650005377, |
| 116.19678699935321, |
| 116.20972499986237, |
| 136.9518049996259, |
| 136.39452200004598 |
| ], |
| "source": "reports/latency/requests_combo.jsonl", |
| "endpoint": "combo_vllm" |
| }, |
| { |
| "hardware": "A10G", |
| "model": "tree_4b_combo (archived Qwen3-4B)", |
| "engine": "vLLM bf16", |
| "text_tokens": 2048, |
| "questions": 1, |
| "candidates_per_question": 3, |
| "timed_calls": 10, |
| "metric": "server_ms", |
| "median_ms": 360.9746049996829, |
| "wall_median_ms": 429.1081245000896, |
| "samples_ms": [ |
| 360.76974000025075, |
| 360.4158849993837, |
| 363.08945900054823, |
| 361.06656000083603, |
| 340.3361130003759, |
| 361.0516810003901, |
| 361.28494200056593, |
| 360.89752899897576, |
| 360.7139420000749, |
| 361.0689139986789 |
| ], |
| "source": "reports/latency/requests_combo.jsonl", |
| "endpoint": "combo_vllm" |
| }, |
| { |
| "hardware": "A10G", |
| "model": "tree_4b_combo (archived Qwen3-4B)", |
| "engine": "vLLM bf16", |
| "text_tokens": 4096, |
| "questions": 1, |
| "candidates_per_question": 3, |
| "timed_calls": 10, |
| "metric": "server_ms", |
| "median_ms": 698.2241815003363, |
| "wall_median_ms": 770.4885834991728, |
| "samples_ms": [ |
| 698.2829150001635, |
| 698.1607249999797, |
| 697.9792379988794, |
| 698.1654480005091, |
| 677.1453080000356, |
| 698.3787730005133, |
| 698.640732000058, |
| 698.3155820016691, |
| 677.0144709989836, |
| 702.3017700012133 |
| ], |
| "source": "reports/latency/requests_combo.jsonl", |
| "endpoint": "combo_vllm" |
| }, |
| { |
| "hardware": "A10G", |
| "model": "tree_4b_combo (archived Qwen3-4B)", |
| "engine": "vLLM bf16", |
| "text_tokens": 8, |
| "questions": 16, |
| "candidates_per_question": 3, |
| "timed_calls": 10, |
| "metric": "server_ms", |
| "median_ms": 400.7618610003192, |
| "wall_median_ms": 470.83658299925446, |
| "samples_ms": [ |
| 399.5544110002811, |
| 402.2729070002242, |
| 399.83477399982803, |
| 401.20150899929286, |
| 400.263260000429, |
| 405.31594999993104, |
| 403.85230599895294, |
| 400.13025400003244, |
| 404.5256089993927, |
| 400.3222130013455 |
| ], |
| "source": "reports/latency/requests_combo.jsonl", |
| "endpoint": "combo_vllm" |
| }, |
| { |
| "hardware": "A10G", |
| "model": "tree_4b_combo (archived Qwen3-4B)", |
| "engine": "vLLM bf16", |
| "text_tokens": 512, |
| "questions": 16, |
| "candidates_per_question": 3, |
| "timed_calls": 10, |
| "metric": "server_ms", |
| "median_ms": 496.0031340006026, |
| "wall_median_ms": 565.5848959995637, |
| "samples_ms": [ |
| 496.5896279991284, |
| 495.53610399925674, |
| 495.90942100076063, |
| 496.09684700044454, |
| 498.08515000040643, |
| 507.20587199975853, |
| 494.783912999992, |
| 508.56806299998425, |
| 495.86053200073366, |
| 495.0548369997705 |
| ], |
| "source": "reports/latency/requests_combo.jsonl", |
| "endpoint": "combo_vllm" |
| }, |
| { |
| "hardware": "A10G", |
| "model": "tree_4b_combo (archived Qwen3-4B)", |
| "engine": "vLLM bf16", |
| "text_tokens": 2048, |
| "questions": 16, |
| "candidates_per_question": 3, |
| "timed_calls": 10, |
| "metric": "server_ms", |
| "median_ms": 808.3618930004377, |
| "wall_median_ms": 881.7444380001689, |
| "samples_ms": [ |
| 809.0102930000285, |
| 808.5504170012427, |
| 807.333759001267, |
| 808.514283000477, |
| 807.2206779997941, |
| 808.2095030003984, |
| 808.2002600003761, |
| 809.2561360008403, |
| 807.4774739998247, |
| 808.9496079992387 |
| ], |
| "source": "reports/latency/requests_combo.jsonl", |
| "endpoint": "combo_vllm" |
| }, |
| { |
| "hardware": "A10G", |
| "model": "tree_4b_combo (archived Qwen3-4B)", |
| "engine": "vLLM bf16", |
| "text_tokens": 4096, |
| "questions": 16, |
| "candidates_per_question": 3, |
| "timed_calls": 10, |
| "metric": "server_ms", |
| "median_ms": 1263.495414999852, |
| "wall_median_ms": 1335.7231039999533, |
| "samples_ms": [ |
| 1261.9187909986067, |
| 1261.8359649986814, |
| 1266.046794000431, |
| 1266.8424380008219, |
| 1263.200176999817, |
| 1262.7665830004844, |
| 1261.9569859998592, |
| 1264.7399829984352, |
| 1265.7464590010932, |
| 1263.7906529998872 |
| ], |
| "source": "reports/latency/requests_combo.jsonl", |
| "endpoint": "combo_vllm" |
| }, |
| { |
| "hardware": "L40S", |
| "model": "tree_4b_combo (archived Qwen3-4B)", |
| "engine": "vLLM bf16", |
| "text_tokens": 8, |
| "questions": 1, |
| "candidates_per_question": 3, |
| "timed_calls": 10, |
| "metric": "server_ms", |
| "median_ms": 36.45959900006801, |
| "wall_median_ms": 158.34560450457502, |
| "samples_ms": [ |
| 36.38916199997766, |
| 36.1490689999755, |
| 36.53003600015836, |
| 37.10315000012088, |
| 35.73796799992124, |
| 21.666765999952986, |
| 36.83985399993617, |
| 38.05452899996453, |
| 35.37955100000545, |
| 36.64288100003432 |
| ], |
| "source": "reports/latency/requests_qwen35.jsonl", |
| "endpoint": "combo_vllm_l40s" |
| }, |
| { |
| "hardware": "L40S", |
| "model": "tree_4b_combo (archived Qwen3-4B)", |
| "engine": "vLLM bf16", |
| "text_tokens": 512, |
| "questions": 1, |
| "candidates_per_question": 3, |
| "timed_calls": 10, |
| "metric": "server_ms", |
| "median_ms": 54.91692000009607, |
| "wall_median_ms": 177.25756250001723, |
| "samples_ms": [ |
| 54.61964599999192, |
| 55.2488529999664, |
| 54.82836400005908, |
| 54.600072000084765, |
| 55.28374400000757, |
| 54.97968000008768, |
| 54.85416000010446, |
| 56.15273400007936, |
| 38.988259999996444, |
| 55.807231000017055 |
| ], |
| "source": "reports/latency/requests_qwen35.jsonl", |
| "endpoint": "combo_vllm_l40s" |
| }, |
| { |
| "hardware": "L40S", |
| "model": "tree_4b_combo (archived Qwen3-4B)", |
| "engine": "vLLM bf16", |
| "text_tokens": 2048, |
| "questions": 1, |
| "candidates_per_question": 3, |
| "timed_calls": 10, |
| "metric": "server_ms", |
| "median_ms": 121.32292549995327, |
| "wall_median_ms": 243.57995799800847, |
| "samples_ms": [ |
| 104.74272299984477, |
| 121.24930300001324, |
| 120.69148800014773, |
| 120.43462199994792, |
| 121.74739700003556, |
| 121.3965479998933, |
| 122.36573599989242, |
| 123.3871580000141, |
| 107.56961300012335, |
| 124.78330499993717 |
| ], |
| "source": "reports/latency/requests_qwen35.jsonl", |
| "endpoint": "combo_vllm_l40s" |
| }, |
| { |
| "hardware": "L40S", |
| "model": "tree_4b_combo (archived Qwen3-4B)", |
| "engine": "vLLM bf16", |
| "text_tokens": 4096, |
| "questions": 1, |
| "candidates_per_question": 3, |
| "timed_calls": 10, |
| "metric": "server_ms", |
| "median_ms": 227.5202175001141, |
| "wall_median_ms": 356.2396874986007, |
| "samples_ms": [ |
| 204.6169520001513, |
| 225.62721799999963, |
| 223.35811500011005, |
| 207.54149900017183, |
| 226.97410900013892, |
| 228.5486939999828, |
| 228.06632600008925, |
| 229.80536599993684, |
| 232.11173300001064, |
| 234.48715499989703 |
| ], |
| "source": "reports/latency/requests_qwen35.jsonl", |
| "endpoint": "combo_vllm_l40s" |
| }, |
| { |
| "hardware": "L40S", |
| "model": "tree_4b_combo (archived Qwen3-4B)", |
| "engine": "vLLM bf16", |
| "text_tokens": 8, |
| "questions": 16, |
| "candidates_per_question": 3, |
| "timed_calls": 10, |
| "metric": "server_ms", |
| "median_ms": 134.8911009998801, |
| "wall_median_ms": 199.10460399842123, |
| "samples_ms": [ |
| 132.70184499992865, |
| 134.9127859998589, |
| 133.87793299989426, |
| 134.86941599990132, |
| 133.53017600002204, |
| 135.4655110001204, |
| 133.62230500001715, |
| 135.95944700000473, |
| 137.00546900008703, |
| 138.80897199987885 |
| ], |
| "source": "reports/latency/requests_qwen35.jsonl", |
| "endpoint": "combo_vllm_l40s" |
| }, |
| { |
| "hardware": "L40S", |
| "model": "tree_4b_combo (archived Qwen3-4B)", |
| "engine": "vLLM bf16", |
| "text_tokens": 512, |
| "questions": 16, |
| "candidates_per_question": 3, |
| "timed_calls": 10, |
| "metric": "server_ms", |
| "median_ms": 162.7876434999962, |
| "wall_median_ms": 279.2183749988908, |
| "samples_ms": [ |
| 160.33683899991047, |
| 159.26829999989423, |
| 163.23814399993353, |
| 162.57371599999715, |
| 160.9234079999169, |
| 163.50454499979605, |
| 160.74670500006505, |
| 163.00157099999524, |
| 165.01971599996068, |
| 166.60063699987404 |
| ], |
| "source": "reports/latency/requests_qwen35.jsonl", |
| "endpoint": "combo_vllm_l40s" |
| }, |
| { |
| "hardware": "L40S", |
| "model": "tree_4b_combo (archived Qwen3-4B)", |
| "engine": "vLLM bf16", |
| "text_tokens": 2048, |
| "questions": 16, |
| "candidates_per_question": 3, |
| "timed_calls": 10, |
| "metric": "server_ms", |
| "median_ms": 267.89673350003795, |
| "wall_median_ms": 342.3541664960794, |
| "samples_ms": [ |
| 265.61998399984077, |
| 265.5573600000025, |
| 269.7477939998407, |
| 265.3392329998496, |
| 265.06465299985393, |
| 267.57590700003675, |
| 268.21756000003916, |
| 272.3697799999627, |
| 273.9279830000214, |
| 272.2678170000563 |
| ], |
| "source": "reports/latency/requests_qwen35.jsonl", |
| "endpoint": "combo_vllm_l40s" |
| }, |
| { |
| "hardware": "L40S", |
| "model": "tree_4b_combo (archived Qwen3-4B)", |
| "engine": "vLLM bf16", |
| "text_tokens": 4096, |
| "questions": 16, |
| "candidates_per_question": 3, |
| "timed_calls": 10, |
| "metric": "server_ms", |
| "median_ms": 423.97226049990877, |
| "wall_median_ms": 504.6778749965597, |
| "samples_ms": [ |
| 417.6726650000546, |
| 424.1281079998771, |
| 423.013415000014, |
| 423.81641299994044, |
| 422.8606829999535, |
| 420.63271100005295, |
| 429.60235900000043, |
| 433.76208300014696, |
| 433.9289730000928, |
| 432.4733450000622 |
| ], |
| "source": "reports/latency/requests_qwen35.jsonl", |
| "endpoint": "combo_vllm_l40s" |
| }, |
| { |
| "hardware": "H100", |
| "model": "tree_4b_combo (archived Qwen3-4B)", |
| "engine": "vLLM bf16", |
| "text_tokens": 8, |
| "questions": 1, |
| "candidates_per_question": 3, |
| "timed_calls": 10, |
| "metric": "server_ms", |
| "median_ms": 21.56023199995616, |
| "wall_median_ms": 139.68993749585934, |
| "samples_ms": [ |
| 21.245153999984723, |
| 20.78479499994046, |
| 20.641650000015943, |
| 22.149028000058024, |
| 21.16872700003114, |
| 21.875309999927595, |
| 22.11586399994303, |
| 22.3034570000209, |
| 21.161376000009113, |
| 22.325410999997075 |
| ], |
| "source": "reports/latency/requests_h100.jsonl", |
| "endpoint": "h100_bf16" |
| }, |
| { |
| "hardware": "H100", |
| "model": "tree_4b_combo (archived Qwen3-4B)", |
| "engine": "vLLM bf16", |
| "text_tokens": 512, |
| "questions": 1, |
| "candidates_per_question": 3, |
| "timed_calls": 10, |
| "metric": "server_ms", |
| "median_ms": 29.72500900000341, |
| "wall_median_ms": 149.15822900366038, |
| "samples_ms": [ |
| 31.458417999942867, |
| 29.484858999921926, |
| 29.528270000014345, |
| 29.456955999989987, |
| 29.97955400007868, |
| 29.82279100001506, |
| 28.950795000014296, |
| 29.82171999997263, |
| 29.628298000034192, |
| 29.9076060000516 |
| ], |
| "source": "reports/latency/requests_h100.jsonl", |
| "endpoint": "h100_bf16" |
| }, |
| { |
| "hardware": "H100", |
| "model": "tree_4b_combo (archived Qwen3-4B)", |
| "engine": "vLLM bf16", |
| "text_tokens": 2048, |
| "questions": 1, |
| "candidates_per_question": 3, |
| "timed_calls": 10, |
| "metric": "server_ms", |
| "median_ms": 47.23783000008552, |
| "wall_median_ms": 163.87020850379486, |
| "samples_ms": [ |
| 47.28402300008838, |
| 48.001042000009875, |
| 47.191637000082665, |
| 39.51898399998299, |
| 48.103888000014194, |
| 47.116929000026175, |
| 48.16269699995246, |
| 40.556042000048365, |
| 39.454456999919785, |
| 48.861916000078054 |
| ], |
| "source": "reports/latency/requests_h100.jsonl", |
| "endpoint": "h100_bf16" |
| }, |
| { |
| "hardware": "H100", |
| "model": "tree_4b_combo (archived Qwen3-4B)", |
| "engine": "vLLM bf16", |
| "text_tokens": 4096, |
| "questions": 1, |
| "candidates_per_question": 3, |
| "timed_calls": 10, |
| "metric": "server_ms", |
| "median_ms": 81.86599299995123, |
| "wall_median_ms": 200.37400000728667, |
| "samples_ms": [ |
| 80.500124000082, |
| 84.03132800003732, |
| 81.40461099992535, |
| 82.67275399998653, |
| 82.86948700003904, |
| 82.98848699996597, |
| 74.614677999989, |
| 74.00309600006949, |
| 73.97928599993975, |
| 82.32737499997711 |
| ], |
| "source": "reports/latency/requests_h100.jsonl", |
| "endpoint": "h100_bf16" |
| }, |
| { |
| "hardware": "H100", |
| "model": "tree_4b_combo (archived Qwen3-4B)", |
| "engine": "vLLM bf16", |
| "text_tokens": 8, |
| "questions": 16, |
| "candidates_per_question": 3, |
| "timed_calls": 10, |
| "metric": "server_ms", |
| "median_ms": 58.08685649998324, |
| "wall_median_ms": 178.56795899569988, |
| "samples_ms": [ |
| 58.254151999904025, |
| 57.70814099992094, |
| 57.460850999973445, |
| 57.497520999959306, |
| 57.621986999947694, |
| 58.632586000044284, |
| 57.91956100006246, |
| 58.77968599997985, |
| 58.65662400003657, |
| 58.74566300008155 |
| ], |
| "source": "reports/latency/requests_h100.jsonl", |
| "endpoint": "h100_bf16" |
| }, |
| { |
| "hardware": "H100", |
| "model": "tree_4b_combo (archived Qwen3-4B)", |
| "engine": "vLLM bf16", |
| "text_tokens": 512, |
| "questions": 16, |
| "candidates_per_question": 3, |
| "timed_calls": 10, |
| "metric": "server_ms", |
| "median_ms": 75.51745949996302, |
| "wall_median_ms": 165.61064551933669, |
| "samples_ms": [ |
| 79.35704300007274, |
| 74.19516200002363, |
| 75.14215899993815, |
| 76.22677799997746, |
| 77.03235300004962, |
| 76.20802600001753, |
| 74.63117800000418, |
| 74.63798900005258, |
| 74.36926400009725, |
| 75.8927599999879 |
| ], |
| "source": "reports/latency/requests_h100.jsonl", |
| "endpoint": "h100_bf16" |
| }, |
| { |
| "hardware": "H100", |
| "model": "tree_4b_combo (archived Qwen3-4B)", |
| "engine": "vLLM bf16", |
| "text_tokens": 2048, |
| "questions": 16, |
| "candidates_per_question": 3, |
| "timed_calls": 10, |
| "metric": "server_ms", |
| "median_ms": 119.95621649998611, |
| "wall_median_ms": 185.84285449469462, |
| "samples_ms": [ |
| 120.04537699999673, |
| 118.41631599997982, |
| 120.76750799997171, |
| 119.86705599997549, |
| 118.63536399994246, |
| 117.67707399997107, |
| 120.54047499998433, |
| 120.36383300005582, |
| 119.76625700003751, |
| 122.93751400000019 |
| ], |
| "source": "reports/latency/requests_h100.jsonl", |
| "endpoint": "h100_bf16" |
| }, |
| { |
| "hardware": "H100", |
| "model": "tree_4b_combo (archived Qwen3-4B)", |
| "engine": "vLLM bf16", |
| "text_tokens": 4096, |
| "questions": 16, |
| "candidates_per_question": 3, |
| "timed_calls": 10, |
| "metric": "server_ms", |
| "median_ms": 189.33901899998773, |
| "wall_median_ms": 250.28441649919841, |
| "samples_ms": [ |
| 189.939337999931, |
| 189.09927400000015, |
| 187.20732099995985, |
| 188.55110999993485, |
| 192.09218399998917, |
| 188.690078000036, |
| 190.02264199991714, |
| 189.5787639999753, |
| 187.71737500003383, |
| 189.88641399994322 |
| ], |
| "source": "reports/latency/requests_h100.jsonl", |
| "endpoint": "h100_bf16" |
| } |
| ], |
| "mac": [ |
| { |
| "hardware": "Apple M5 Pro / 48 GB", |
| "model": "SelfJev-4B (current adapter)", |
| "engine": "TreeServer / MPS bf16", |
| "text_tokens": 8, |
| "questions": 1, |
| "candidates_per_question": 3, |
| "timed_calls": 10, |
| "metric": "local_call_ms", |
| "median_ms": 688.325, |
| "wall_median_ms": null, |
| "samples_ms": [ |
| 582.376, |
| 721.25, |
| 825.346, |
| 661.189, |
| 693.301, |
| 703.62, |
| 669.614, |
| 662.967, |
| 683.349, |
| 694.881 |
| ], |
| "source": "reports/latency/mac_m5_pro_selfjev4b.json", |
| "endpoint": "TreeServer.score_requests" |
| }, |
| { |
| "hardware": "Apple M5 Pro / 48 GB", |
| "model": "SelfJev-4B (current adapter)", |
| "engine": "TreeServer / MPS bf16", |
| "text_tokens": 8, |
| "questions": 16, |
| "candidates_per_question": 3, |
| "timed_calls": 10, |
| "metric": "local_call_ms", |
| "median_ms": 6786.2605, |
| "wall_median_ms": null, |
| "samples_ms": [ |
| 6662.747, |
| 6815.696, |
| 6687.229, |
| 6762.426, |
| 6928.557, |
| 6867.698, |
| 6706.089, |
| 6784.906, |
| 6874.29, |
| 6787.615 |
| ], |
| "source": "reports/latency/mac_m5_pro_selfjev4b.json", |
| "endpoint": "TreeServer.score_requests" |
| }, |
| { |
| "hardware": "Apple M5 Pro / 48 GB", |
| "model": "SelfJev-4B (current adapter)", |
| "engine": "TreeServer / MPS bf16", |
| "text_tokens": 512, |
| "questions": 1, |
| "candidates_per_question": 3, |
| "timed_calls": 10, |
| "metric": "local_call_ms", |
| "median_ms": 2018.3355000000001, |
| "wall_median_ms": null, |
| "samples_ms": [ |
| 2041.356, |
| 1995.315, |
| 1872.847, |
| 2047.419, |
| 1971.03, |
| 2093.553, |
| 2070.734, |
| 2053.476, |
| 1980.676, |
| 1912.813 |
| ], |
| "source": "reports/latency/mac_m5_pro_selfjev4b.json", |
| "endpoint": "TreeServer.score_requests" |
| }, |
| { |
| "hardware": "Apple M5 Pro / 48 GB", |
| "model": "SelfJev-4B (current adapter)", |
| "engine": "TreeServer / MPS bf16", |
| "text_tokens": 512, |
| "questions": 16, |
| "candidates_per_question": 3, |
| "timed_calls": 10, |
| "metric": "local_call_ms", |
| "median_ms": 8598.7065, |
| "wall_median_ms": null, |
| "samples_ms": [ |
| 8624.271, |
| 8651.608, |
| 8528.286, |
| 8357.41, |
| 8620.205, |
| 8703.927, |
| 8858.519, |
| 8577.208, |
| 8215.803, |
| 8237.903 |
| ], |
| "source": "reports/latency/mac_m5_pro_selfjev4b.json", |
| "endpoint": "TreeServer.score_requests" |
| } |
| ], |
| "mac_meta": { |
| "date": "2026-09-28", |
| "chip": "Apple M5 Pro", |
| "gpu_cores": 20, |
| "unified_memory_gb": 48, |
| "platform": "macOS-26.5.2-arm64-arm-64bit", |
| "torch": "2.14.0", |
| "model": { |
| "rotary_buffer_precision": "native", |
| "adapter_sha256": "dfbf2834d883987893ec305a6093a345fd79c77f903f60f2f914cbe3b6058d1b", |
| "model": "Qwen/Qwen3.5-4B", |
| "revision": "851bf6e806efd8d0a36b00ddf55e13ccb7b8cd0a", |
| "adapter": "weights/selfjev_4b", |
| "device": "mps", |
| "dtype": "bfloat16", |
| "prompt": "challenger-state-first-v1", |
| "prompt_sha": "6c9d8459ac44", |
| "truncation": "none", |
| "max_length": 8192, |
| "adapter_merged": true, |
| "architecture": "shared-prefix tree, forward only: text once per request, each question once, then each candidate" |
| }, |
| "input_text": "synthetic `blue ` repeated to exactly the stated tokenizer-token length", |
| "method": "TreeServer.score_requests, one request at a time; MPS synchronized before and after each call; wall time includes tokenization and model work, excludes HTTP/network; 3-option multiclass questions; two warmups per cell", |
| "warmup": 2, |
| "reps": 10, |
| "local_cost_usd": 0 |
| }, |
| "scope": "Different models and engines. No current-model NVIDIA measurement. No new inference." |
| }, |
| "external_text_decisions": { |
| "model": "Eikos-4B", |
| "accuracy": 92.7674535409342, |
| "source": "reports/eikos_4b/eval2/report.json" |
| }, |
| "provenance_sha256": { |
| "data/all.jsonl.gz": "cc1db4b3c8c72832c128600c1213486cb01d6880df8cfdd8eda3710ca6712b20", |
| "reports/selfjev_4b_treeserver/test/report.json": "ec3da777a49358850820115c6e9d98a4a7a25d736bb7a23f74edb904462e751e", |
| "reports/external/full/typesafe_jev-latest/report.json": "02c56f7564b5d257194d996a0831d22f78f168b4247b14ac868c7106acf3add0", |
| "reports/selfjev_4b_treeserver/eval2/report.json": "6d3990d0f77e94985edb9d9c174e9a9a326671bc045d578761993da51dcdfa13", |
| "reports/external/eval2/typesafe_jev-latest/report.json": "a632fd3a6a5628b89aa7a371587bac0d362b8de846c4327c2174c796b4b6ae4a", |
| "reports/selfjev_4b_treeserver/eval_llm/report.json": "e98db0fbee4109e6fdb8585334662754abd747357d8474598bcf61e966cb8f6d", |
| "reports/baseline/test/report.json": "8659d24af1f35e9fe29b7b54b73f44080d445277a8fe74d01b581c0b8ed0dbc7", |
| "reports/baseline_4b/test/report.json": "db31f7e9b922590955d7056475732e68763f4aabca0b5d78fd1cf180a5db3b3a", |
| "reports/baseline_8b/test/report.json": "459c908e69636e6cae3d49f50801be5463e27c8d7777c18c7ed9ca4165e61ad9", |
| "reports/lora_pilot/test/report.json": "4cc1dfc68e954155fcb017a478ee19b30a9f91c204f79d906bac02cba00e4051", |
| "reports/lora_4b/test/report.json": "d385af3b3b180b003d9f11a613305c2293ac1d35a0866b304bd881ef42a1c487", |
| "reports/lora_8b/test/report.json": "2749882e4717d8310b2aced826bd8712279d0b8258ca5317e5e0d6ab25322ecb", |
| "reports/tree_4b/eval2/report.json": "a61afe4c7ebd85e37684b9b74a7f938375a7d252973be028d0f822b1d1d4cf8c", |
| "reports/tree_4b_r2b/eval2/report.json": "6438041ed5b3818b2a324de2356a20fd1394f6e1efdd1daaee7ed6499e3cf578", |
| "reports/tree_4b_instruct_r2x64/eval2/report.json": "2f2b94c58f1640bd6fc55831f35f7b937745763fbd5e825cebdcdf890d2f33af", |
| "reports/tree_4b_combo/eval2/report.json": "9bc9eed004d3a9477041925549b03e369efb4e73896fc1e0020ef609c6633dd7", |
| "reports/qwen35_4b_tree/eval2/report.json": "5ec21f63b8e4fe33986a5c110b5ed4558b8c50f8a9ca4eaa1dacdbee0991e55e", |
| "reports/eikos_4b/eval2/report.json": "ebd207e342f94c7a2d3bb4047fdcfd10dc802a2cf986eff820dd23217a9e19f9", |
| "reports/latency/requests_combo.jsonl": "424b8c71ce36a30bfec6675bf2aa88611fbf8063e5e8aff06a54489a1e8e9174", |
| "reports/latency/requests_qwen35.jsonl": "c604f1dd350691bfc46839947fa6963f0459140b14f6e174002060823ebea845", |
| "reports/latency/requests_h100.jsonl": "9822ca6be7e027c8f60c24c5b571d9e59614ffc2682c4d12152dab8c5fb42c23", |
| "reports/latency/mac_m5_pro_selfjev4b.json": "37e45d4d2bad001d6a8dcccada3f1d591ca5eb146c2366a1fb18ab27cd50ab2e" |
| }, |
| "scope": "Saved results only. No new inference. Public datasets adapted; no S1Bench claim." |
| } |
|
|
