{
  "schema_version": 1,
  "overview": [
    {
      "label": "Text decisions",
      "n": 1991,
      "ours": 95.68056253139126,
      "jev": 97.23756906077348,
      "source": "reports/selfjev_4b_treeserver/eval2/report.json",
      "reference": "reports/external/eval2/typesafe_jev-latest/report.json"
    },
    {
      "label": "AI response review",
      "n": 946,
      "ours": 93.12896405919662,
      "jev": 92.49471458773785,
      "source": "reports/selfjev_4b_treeserver/eval_llm/report.json",
      "reference": "data/all.jsonl.gz: eval_llm / jev",
      "reference_model": "typesafe/jev-1.13-20260917"
    },
    {
      "label": "Public-source tasks",
      "n": 3300,
      "ours": 83.39393939393939,
      "jev": 82.0909090909091,
      "source": "reports/selfjev_4b_treeserver/test/report.json",
      "reference": "reports/external/full/typesafe_jev-latest/report.json"
    }
  ],
  "public_datasets": [
    {
      "family": "heldout_intent_clinc",
      "dataset": "CLINC150",
      "task": "Intent routing",
      "exposure": "held-out source",
      "n": 300,
      "ours": 95.0,
      "jev": 94.33333333333333,
      "dataset_url": "https://huggingface.co/datasets/clinc/clinc_oos",
      "source": "reports/selfjev_4b_treeserver/test/report.json",
      "reference": "reports/external/full/typesafe_jev-latest/report.json"
    },
    {
      "family": "heldout_topic_dbpedia",
      "dataset": "DBpedia",
      "task": "Topic classification",
      "exposure": "held-out source",
      "n": 300,
      "ours": 98.0,
      "jev": 98.0,
      "dataset_url": "https://huggingface.co/datasets/fancyzhx/dbpedia_14",
      "source": "reports/selfjev_4b_treeserver/test/report.json",
      "reference": "reports/external/full/typesafe_jev-latest/report.json"
    },
    {
      "family": "heldout_question_type_trec",
      "dataset": "TREC",
      "task": "Question type",
      "exposure": "held-out source",
      "n": 300,
      "ours": 91.33333333333333,
      "jev": 94.0,
      "dataset_url": "https://huggingface.co/datasets/SetFit/TREC-QC",
      "source": "reports/selfjev_4b_treeserver/test/report.json",
      "reference": "reports/external/full/typesafe_jev-latest/report.json"
    },
    {
      "family": "heldout_emotion_multiclass",
      "dataset": "Emotion",
      "task": "Emotion classification",
      "exposure": "held-out source",
      "n": 300,
      "ours": 57.333333333333336,
      "jev": 57.666666666666664,
      "dataset_url": "https://huggingface.co/datasets/dair-ai/emotion",
      "source": "reports/selfjev_4b_treeserver/test/report.json",
      "reference": "reports/external/full/typesafe_jev-latest/report.json"
    },
    {
      "family": "heldout_boolq",
      "dataset": "BoolQ",
      "task": "Reading comprehension",
      "exposure": "held-out source",
      "n": 300,
      "ours": 87.66666666666667,
      "jev": 90.66666666666667,
      "dataset_url": "https://huggingface.co/datasets/google/boolq",
      "source": "reports/selfjev_4b_treeserver/test/report.json",
      "reference": "reports/external/full/typesafe_jev-latest/report.json"
    },
    {
      "family": "heldout_sentiment_sst2",
      "dataset": "SST-2",
      "task": "Sentiment",
      "exposure": "held-out source",
      "n": 300,
      "ours": 91.33333333333333,
      "jev": 96.66666666666667,
      "dataset_url": "https://huggingface.co/datasets/stanfordnlp/sst2",
      "source": "reports/selfjev_4b_treeserver/test/report.json",
      "reference": "reports/external/full/typesafe_jev-latest/report.json"
    },
    {
      "family": "hf_intent_banking77",
      "dataset": "Banking77",
      "task": "Banking intent",
      "exposure": "seen source",
      "n": 300,
      "ours": 96.33333333333333,
      "jev": 95.66666666666667,
      "dataset_url": "https://huggingface.co/datasets/mteb/banking77",
      "source": "reports/selfjev_4b_treeserver/test/report.json",
      "reference": "reports/external/full/typesafe_jev-latest/report.json"
    },
    {
      "family": "hf_topic_agnews",
      "dataset": "AG News",
      "task": "News topic",
      "exposure": "seen source",
      "n": 300,
      "ours": 90.33333333333333,
      "jev": 87.33333333333333,
      "dataset_url": "https://huggingface.co/datasets/fancyzhx/ag_news",
      "source": "reports/selfjev_4b_treeserver/test/report.json",
      "reference": "reports/external/full/typesafe_jev-latest/report.json"
    },
    {
      "family": "hf_sentiment_tweets",
      "dataset": "TweetEval",
      "task": "Tweet sentiment",
      "exposure": "seen source",
      "n": 300,
      "ours": 67.33333333333333,
      "jev": 64.33333333333333,
      "dataset_url": "https://huggingface.co/datasets/cardiffnlp/tweet_eval",
      "source": "reports/selfjev_4b_treeserver/test/report.json",
      "reference": "reports/external/full/typesafe_jev-latest/report.json"
    },
    {
      "family": "hf_nli",
      "dataset": "MNLI",
      "task": "Entailment (binary)",
      "exposure": "seen source",
      "n": 300,
      "ours": 95.0,
      "jev": 92.66666666666667,
      "dataset_url": "https://huggingface.co/datasets/nyu-mll/glue",
      "source": "reports/selfjev_4b_treeserver/test/report.json",
      "reference": "reports/external/full/typesafe_jev-latest/report.json"
    },
    {
      "family": "hf_emotions_multilabel",
      "dataset": "GoEmotions",
      "task": "Emotion tags (exact set)",
      "exposure": "seen source",
      "n": 300,
      "ours": 47.666666666666664,
      "jev": 31.666666666666668,
      "dataset_url": "https://huggingface.co/datasets/google-research-datasets/go_emotions",
      "source": "reports/selfjev_4b_treeserver/test/report.json",
      "reference": "reports/external/full/typesafe_jev-latest/report.json"
    }
  ],
  "accuracy_vs_size": [
    {
      "model": "Qwen3 reranker 0.6B",
      "nominal_backbone_billions": 0.6,
      "regime": "stock",
      "accuracy": 62.18181818181818,
      "n": 3300,
      "source": "reports/baseline/test/report.json"
    },
    {
      "model": "Qwen3 reranker 4B",
      "nominal_backbone_billions": 4,
      "regime": "stock",
      "accuracy": 63.78787878787879,
      "n": 3300,
      "source": "reports/baseline_4b/test/report.json"
    },
    {
      "model": "Qwen3 reranker 8B",
      "nominal_backbone_billions": 8,
      "regime": "stock",
      "accuracy": 67.03030303030303,
      "n": 3300,
      "source": "reports/baseline_8b/test/report.json"
    },
    {
      "model": "Qwen3 + LoRA 0.6B",
      "nominal_backbone_billions": 0.6,
      "regime": "early fine-tune",
      "accuracy": 74.48484848484848,
      "n": 3300,
      "source": "reports/lora_pilot/test/report.json"
    },
    {
      "model": "Qwen3 + LoRA 4B",
      "nominal_backbone_billions": 4,
      "regime": "early fine-tune",
      "accuracy": 80.81818181818181,
      "n": 3300,
      "source": "reports/lora_4b/test/report.json"
    },
    {
      "model": "Qwen3 + LoRA 8B",
      "nominal_backbone_billions": 8,
      "regime": "early fine-tune",
      "accuracy": 80.96969696969697,
      "n": 3300,
      "source": "reports/lora_8b/test/report.json"
    },
    {
      "model": "SelfJev-4B",
      "nominal_backbone_billions": 4,
      "regime": "current release",
      "accuracy": 83.39393939393939,
      "n": 3300,
      "source": "reports/selfjev_4b_treeserver/test/report.json"
    }
  ],
  "research_path": [
    {
      "model": "Initial tree",
      "accuracy": 85.13309894525364,
      "source": "reports/tree_4b/eval2/report.json",
      "n": 1991
    },
    {
      "model": "Verified hard cases",
      "accuracy": 90.55750878955298,
      "source": "reports/tree_4b_r2b/eval2/report.json",
      "n": 1991
    },
    {
      "model": "Instruct backbone",
      "accuracy": 92.71722752385736,
      "source": "reports/tree_4b_instruct_r2x64/eval2/report.json",
      "n": 1991
    },
    {
      "model": "Broader data + options",
      "accuracy": 94.47513812154696,
      "source": "reports/tree_4b_combo/eval2/report.json",
      "n": 1991
    },
    {
      "model": "Qwen3.5 + tree training",
      "accuracy": 95.58011049723757,
      "source": "reports/qwen35_4b_tree/eval2/report.json",
      "n": 1991
    },
    {
      "model": "SelfJev / current engine",
      "accuracy": 95.68056253139126,
      "source": "reports/selfjev_4b_treeserver/eval2/report.json",
      "n": 1991
    }
  ],
  "latency": {
    "gpu": [
      {
        "hardware": "A10G",
        "model": "tree_4b_combo (archived Qwen3-4B)",
        "engine": "vLLM bf16",
        "text_tokens": 8,
        "questions": 1,
        "candidates_per_question": 3,
        "timed_calls": 10,
        "metric": "server_ms",
        "median_ms": 55.28760100060026,
        "wall_median_ms": 124.98704150038975,
        "samples_ms": [
          55.370269999912125,
          55.36273899997468,
          38.954117999310256,
          55.06569299905095,
          58.78931900042517,
          55.11900499914191,
          55.21246300122584,
          58.02654000035545,
          55.09648099905462,
          58.76103499940655
        ],
        "source": "reports/latency/requests_combo.jsonl",
        "endpoint": "combo_vllm"
      },
      {
        "hardware": "A10G",
        "model": "tree_4b_combo (archived Qwen3-4B)",
        "engine": "vLLM bf16",
        "text_tokens": 512,
        "questions": 1,
        "candidates_per_question": 3,
        "timed_calls": 10,
        "metric": "server_ms",
        "median_ms": 136.1421874998996,
        "wall_median_ms": 203.85035450090072,
        "samples_ms": [
          136.60056099979556,
          116.28793100135226,
          136.12962099978176,
          136.00464800038026,
          136.15475400001742,
          136.4384650005377,
          116.19678699935321,
          116.20972499986237,
          136.9518049996259,
          136.39452200004598
        ],
        "source": "reports/latency/requests_combo.jsonl",
        "endpoint": "combo_vllm"
      },
      {
        "hardware": "A10G",
        "model": "tree_4b_combo (archived Qwen3-4B)",
        "engine": "vLLM bf16",
        "text_tokens": 2048,
        "questions": 1,
        "candidates_per_question": 3,
        "timed_calls": 10,
        "metric": "server_ms",
        "median_ms": 360.9746049996829,
        "wall_median_ms": 429.1081245000896,
        "samples_ms": [
          360.76974000025075,
          360.4158849993837,
          363.08945900054823,
          361.06656000083603,
          340.3361130003759,
          361.0516810003901,
          361.28494200056593,
          360.89752899897576,
          360.7139420000749,
          361.0689139986789
        ],
        "source": "reports/latency/requests_combo.jsonl",
        "endpoint": "combo_vllm"
      },
      {
        "hardware": "A10G",
        "model": "tree_4b_combo (archived Qwen3-4B)",
        "engine": "vLLM bf16",
        "text_tokens": 4096,
        "questions": 1,
        "candidates_per_question": 3,
        "timed_calls": 10,
        "metric": "server_ms",
        "median_ms": 698.2241815003363,
        "wall_median_ms": 770.4885834991728,
        "samples_ms": [
          698.2829150001635,
          698.1607249999797,
          697.9792379988794,
          698.1654480005091,
          677.1453080000356,
          698.3787730005133,
          698.640732000058,
          698.3155820016691,
          677.0144709989836,
          702.3017700012133
        ],
        "source": "reports/latency/requests_combo.jsonl",
        "endpoint": "combo_vllm"
      },
      {
        "hardware": "A10G",
        "model": "tree_4b_combo (archived Qwen3-4B)",
        "engine": "vLLM bf16",
        "text_tokens": 8,
        "questions": 16,
        "candidates_per_question": 3,
        "timed_calls": 10,
        "metric": "server_ms",
        "median_ms": 400.7618610003192,
        "wall_median_ms": 470.83658299925446,
        "samples_ms": [
          399.5544110002811,
          402.2729070002242,
          399.83477399982803,
          401.20150899929286,
          400.263260000429,
          405.31594999993104,
          403.85230599895294,
          400.13025400003244,
          404.5256089993927,
          400.3222130013455
        ],
        "source": "reports/latency/requests_combo.jsonl",
        "endpoint": "combo_vllm"
      },
      {
        "hardware": "A10G",
        "model": "tree_4b_combo (archived Qwen3-4B)",
        "engine": "vLLM bf16",
        "text_tokens": 512,
        "questions": 16,
        "candidates_per_question": 3,
        "timed_calls": 10,
        "metric": "server_ms",
        "median_ms": 496.0031340006026,
        "wall_median_ms": 565.5848959995637,
        "samples_ms": [
          496.5896279991284,
          495.53610399925674,
          495.90942100076063,
          496.09684700044454,
          498.08515000040643,
          507.20587199975853,
          494.783912999992,
          508.56806299998425,
          495.86053200073366,
          495.0548369997705
        ],
        "source": "reports/latency/requests_combo.jsonl",
        "endpoint": "combo_vllm"
      },
      {
        "hardware": "A10G",
        "model": "tree_4b_combo (archived Qwen3-4B)",
        "engine": "vLLM bf16",
        "text_tokens": 2048,
        "questions": 16,
        "candidates_per_question": 3,
        "timed_calls": 10,
        "metric": "server_ms",
        "median_ms": 808.3618930004377,
        "wall_median_ms": 881.7444380001689,
        "samples_ms": [
          809.0102930000285,
          808.5504170012427,
          807.333759001267,
          808.514283000477,
          807.2206779997941,
          808.2095030003984,
          808.2002600003761,
          809.2561360008403,
          807.4774739998247,
          808.9496079992387
        ],
        "source": "reports/latency/requests_combo.jsonl",
        "endpoint": "combo_vllm"
      },
      {
        "hardware": "A10G",
        "model": "tree_4b_combo (archived Qwen3-4B)",
        "engine": "vLLM bf16",
        "text_tokens": 4096,
        "questions": 16,
        "candidates_per_question": 3,
        "timed_calls": 10,
        "metric": "server_ms",
        "median_ms": 1263.495414999852,
        "wall_median_ms": 1335.7231039999533,
        "samples_ms": [
          1261.9187909986067,
          1261.8359649986814,
          1266.046794000431,
          1266.8424380008219,
          1263.200176999817,
          1262.7665830004844,
          1261.9569859998592,
          1264.7399829984352,
          1265.7464590010932,
          1263.7906529998872
        ],
        "source": "reports/latency/requests_combo.jsonl",
        "endpoint": "combo_vllm"
      },
      {
        "hardware": "L40S",
        "model": "tree_4b_combo (archived Qwen3-4B)",
        "engine": "vLLM bf16",
        "text_tokens": 8,
        "questions": 1,
        "candidates_per_question": 3,
        "timed_calls": 10,
        "metric": "server_ms",
        "median_ms": 36.45959900006801,
        "wall_median_ms": 158.34560450457502,
        "samples_ms": [
          36.38916199997766,
          36.1490689999755,
          36.53003600015836,
          37.10315000012088,
          35.73796799992124,
          21.666765999952986,
          36.83985399993617,
          38.05452899996453,
          35.37955100000545,
          36.64288100003432
        ],
        "source": "reports/latency/requests_qwen35.jsonl",
        "endpoint": "combo_vllm_l40s"
      },
      {
        "hardware": "L40S",
        "model": "tree_4b_combo (archived Qwen3-4B)",
        "engine": "vLLM bf16",
        "text_tokens": 512,
        "questions": 1,
        "candidates_per_question": 3,
        "timed_calls": 10,
        "metric": "server_ms",
        "median_ms": 54.91692000009607,
        "wall_median_ms": 177.25756250001723,
        "samples_ms": [
          54.61964599999192,
          55.2488529999664,
          54.82836400005908,
          54.600072000084765,
          55.28374400000757,
          54.97968000008768,
          54.85416000010446,
          56.15273400007936,
          38.988259999996444,
          55.807231000017055
        ],
        "source": "reports/latency/requests_qwen35.jsonl",
        "endpoint": "combo_vllm_l40s"
      },
      {
        "hardware": "L40S",
        "model": "tree_4b_combo (archived Qwen3-4B)",
        "engine": "vLLM bf16",
        "text_tokens": 2048,
        "questions": 1,
        "candidates_per_question": 3,
        "timed_calls": 10,
        "metric": "server_ms",
        "median_ms": 121.32292549995327,
        "wall_median_ms": 243.57995799800847,
        "samples_ms": [
          104.74272299984477,
          121.24930300001324,
          120.69148800014773,
          120.43462199994792,
          121.74739700003556,
          121.3965479998933,
          122.36573599989242,
          123.3871580000141,
          107.56961300012335,
          124.78330499993717
        ],
        "source": "reports/latency/requests_qwen35.jsonl",
        "endpoint": "combo_vllm_l40s"
      },
      {
        "hardware": "L40S",
        "model": "tree_4b_combo (archived Qwen3-4B)",
        "engine": "vLLM bf16",
        "text_tokens": 4096,
        "questions": 1,
        "candidates_per_question": 3,
        "timed_calls": 10,
        "metric": "server_ms",
        "median_ms": 227.5202175001141,
        "wall_median_ms": 356.2396874986007,
        "samples_ms": [
          204.6169520001513,
          225.62721799999963,
          223.35811500011005,
          207.54149900017183,
          226.97410900013892,
          228.5486939999828,
          228.06632600008925,
          229.80536599993684,
          232.11173300001064,
          234.48715499989703
        ],
        "source": "reports/latency/requests_qwen35.jsonl",
        "endpoint": "combo_vllm_l40s"
      },
      {
        "hardware": "L40S",
        "model": "tree_4b_combo (archived Qwen3-4B)",
        "engine": "vLLM bf16",
        "text_tokens": 8,
        "questions": 16,
        "candidates_per_question": 3,
        "timed_calls": 10,
        "metric": "server_ms",
        "median_ms": 134.8911009998801,
        "wall_median_ms": 199.10460399842123,
        "samples_ms": [
          132.70184499992865,
          134.9127859998589,
          133.87793299989426,
          134.86941599990132,
          133.53017600002204,
          135.4655110001204,
          133.62230500001715,
          135.95944700000473,
          137.00546900008703,
          138.80897199987885
        ],
        "source": "reports/latency/requests_qwen35.jsonl",
        "endpoint": "combo_vllm_l40s"
      },
      {
        "hardware": "L40S",
        "model": "tree_4b_combo (archived Qwen3-4B)",
        "engine": "vLLM bf16",
        "text_tokens": 512,
        "questions": 16,
        "candidates_per_question": 3,
        "timed_calls": 10,
        "metric": "server_ms",
        "median_ms": 162.7876434999962,
        "wall_median_ms": 279.2183749988908,
        "samples_ms": [
          160.33683899991047,
          159.26829999989423,
          163.23814399993353,
          162.57371599999715,
          160.9234079999169,
          163.50454499979605,
          160.74670500006505,
          163.00157099999524,
          165.01971599996068,
          166.60063699987404
        ],
        "source": "reports/latency/requests_qwen35.jsonl",
        "endpoint": "combo_vllm_l40s"
      },
      {
        "hardware": "L40S",
        "model": "tree_4b_combo (archived Qwen3-4B)",
        "engine": "vLLM bf16",
        "text_tokens": 2048,
        "questions": 16,
        "candidates_per_question": 3,
        "timed_calls": 10,
        "metric": "server_ms",
        "median_ms": 267.89673350003795,
        "wall_median_ms": 342.3541664960794,
        "samples_ms": [
          265.61998399984077,
          265.5573600000025,
          269.7477939998407,
          265.3392329998496,
          265.06465299985393,
          267.57590700003675,
          268.21756000003916,
          272.3697799999627,
          273.9279830000214,
          272.2678170000563
        ],
        "source": "reports/latency/requests_qwen35.jsonl",
        "endpoint": "combo_vllm_l40s"
      },
      {
        "hardware": "L40S",
        "model": "tree_4b_combo (archived Qwen3-4B)",
        "engine": "vLLM bf16",
        "text_tokens": 4096,
        "questions": 16,
        "candidates_per_question": 3,
        "timed_calls": 10,
        "metric": "server_ms",
        "median_ms": 423.97226049990877,
        "wall_median_ms": 504.6778749965597,
        "samples_ms": [
          417.6726650000546,
          424.1281079998771,
          423.013415000014,
          423.81641299994044,
          422.8606829999535,
          420.63271100005295,
          429.60235900000043,
          433.76208300014696,
          433.9289730000928,
          432.4733450000622
        ],
        "source": "reports/latency/requests_qwen35.jsonl",
        "endpoint": "combo_vllm_l40s"
      },
      {
        "hardware": "H100",
        "model": "tree_4b_combo (archived Qwen3-4B)",
        "engine": "vLLM bf16",
        "text_tokens": 8,
        "questions": 1,
        "candidates_per_question": 3,
        "timed_calls": 10,
        "metric": "server_ms",
        "median_ms": 21.56023199995616,
        "wall_median_ms": 139.68993749585934,
        "samples_ms": [
          21.245153999984723,
          20.78479499994046,
          20.641650000015943,
          22.149028000058024,
          21.16872700003114,
          21.875309999927595,
          22.11586399994303,
          22.3034570000209,
          21.161376000009113,
          22.325410999997075
        ],
        "source": "reports/latency/requests_h100.jsonl",
        "endpoint": "h100_bf16"
      },
      {
        "hardware": "H100",
        "model": "tree_4b_combo (archived Qwen3-4B)",
        "engine": "vLLM bf16",
        "text_tokens": 512,
        "questions": 1,
        "candidates_per_question": 3,
        "timed_calls": 10,
        "metric": "server_ms",
        "median_ms": 29.72500900000341,
        "wall_median_ms": 149.15822900366038,
        "samples_ms": [
          31.458417999942867,
          29.484858999921926,
          29.528270000014345,
          29.456955999989987,
          29.97955400007868,
          29.82279100001506,
          28.950795000014296,
          29.82171999997263,
          29.628298000034192,
          29.9076060000516
        ],
        "source": "reports/latency/requests_h100.jsonl",
        "endpoint": "h100_bf16"
      },
      {
        "hardware": "H100",
        "model": "tree_4b_combo (archived Qwen3-4B)",
        "engine": "vLLM bf16",
        "text_tokens": 2048,
        "questions": 1,
        "candidates_per_question": 3,
        "timed_calls": 10,
        "metric": "server_ms",
        "median_ms": 47.23783000008552,
        "wall_median_ms": 163.87020850379486,
        "samples_ms": [
          47.28402300008838,
          48.001042000009875,
          47.191637000082665,
          39.51898399998299,
          48.103888000014194,
          47.116929000026175,
          48.16269699995246,
          40.556042000048365,
          39.454456999919785,
          48.861916000078054
        ],
        "source": "reports/latency/requests_h100.jsonl",
        "endpoint": "h100_bf16"
      },
      {
        "hardware": "H100",
        "model": "tree_4b_combo (archived Qwen3-4B)",
        "engine": "vLLM bf16",
        "text_tokens": 4096,
        "questions": 1,
        "candidates_per_question": 3,
        "timed_calls": 10,
        "metric": "server_ms",
        "median_ms": 81.86599299995123,
        "wall_median_ms": 200.37400000728667,
        "samples_ms": [
          80.500124000082,
          84.03132800003732,
          81.40461099992535,
          82.67275399998653,
          82.86948700003904,
          82.98848699996597,
          74.614677999989,
          74.00309600006949,
          73.97928599993975,
          82.32737499997711
        ],
        "source": "reports/latency/requests_h100.jsonl",
        "endpoint": "h100_bf16"
      },
      {
        "hardware": "H100",
        "model": "tree_4b_combo (archived Qwen3-4B)",
        "engine": "vLLM bf16",
        "text_tokens": 8,
        "questions": 16,
        "candidates_per_question": 3,
        "timed_calls": 10,
        "metric": "server_ms",
        "median_ms": 58.08685649998324,
        "wall_median_ms": 178.56795899569988,
        "samples_ms": [
          58.254151999904025,
          57.70814099992094,
          57.460850999973445,
          57.497520999959306,
          57.621986999947694,
          58.632586000044284,
          57.91956100006246,
          58.77968599997985,
          58.65662400003657,
          58.74566300008155
        ],
        "source": "reports/latency/requests_h100.jsonl",
        "endpoint": "h100_bf16"
      },
      {
        "hardware": "H100",
        "model": "tree_4b_combo (archived Qwen3-4B)",
        "engine": "vLLM bf16",
        "text_tokens": 512,
        "questions": 16,
        "candidates_per_question": 3,
        "timed_calls": 10,
        "metric": "server_ms",
        "median_ms": 75.51745949996302,
        "wall_median_ms": 165.61064551933669,
        "samples_ms": [
          79.35704300007274,
          74.19516200002363,
          75.14215899993815,
          76.22677799997746,
          77.03235300004962,
          76.20802600001753,
          74.63117800000418,
          74.63798900005258,
          74.36926400009725,
          75.8927599999879
        ],
        "source": "reports/latency/requests_h100.jsonl",
        "endpoint": "h100_bf16"
      },
      {
        "hardware": "H100",
        "model": "tree_4b_combo (archived Qwen3-4B)",
        "engine": "vLLM bf16",
        "text_tokens": 2048,
        "questions": 16,
        "candidates_per_question": 3,
        "timed_calls": 10,
        "metric": "server_ms",
        "median_ms": 119.95621649998611,
        "wall_median_ms": 185.84285449469462,
        "samples_ms": [
          120.04537699999673,
          118.41631599997982,
          120.76750799997171,
          119.86705599997549,
          118.63536399994246,
          117.67707399997107,
          120.54047499998433,
          120.36383300005582,
          119.76625700003751,
          122.93751400000019
        ],
        "source": "reports/latency/requests_h100.jsonl",
        "endpoint": "h100_bf16"
      },
      {
        "hardware": "H100",
        "model": "tree_4b_combo (archived Qwen3-4B)",
        "engine": "vLLM bf16",
        "text_tokens": 4096,
        "questions": 16,
        "candidates_per_question": 3,
        "timed_calls": 10,
        "metric": "server_ms",
        "median_ms": 189.33901899998773,
        "wall_median_ms": 250.28441649919841,
        "samples_ms": [
          189.939337999931,
          189.09927400000015,
          187.20732099995985,
          188.55110999993485,
          192.09218399998917,
          188.690078000036,
          190.02264199991714,
          189.5787639999753,
          187.71737500003383,
          189.88641399994322
        ],
        "source": "reports/latency/requests_h100.jsonl",
        "endpoint": "h100_bf16"
      }
    ],
    "mac": [
      {
        "hardware": "Apple M5 Pro / 48 GB",
        "model": "SelfJev-4B (current adapter)",
        "engine": "TreeServer / MPS bf16",
        "text_tokens": 8,
        "questions": 1,
        "candidates_per_question": 3,
        "timed_calls": 10,
        "metric": "local_call_ms",
        "median_ms": 688.325,
        "wall_median_ms": null,
        "samples_ms": [
          582.376,
          721.25,
          825.346,
          661.189,
          693.301,
          703.62,
          669.614,
          662.967,
          683.349,
          694.881
        ],
        "source": "reports/latency/mac_m5_pro_selfjev4b.json",
        "endpoint": "TreeServer.score_requests"
      },
      {
        "hardware": "Apple M5 Pro / 48 GB",
        "model": "SelfJev-4B (current adapter)",
        "engine": "TreeServer / MPS bf16",
        "text_tokens": 8,
        "questions": 16,
        "candidates_per_question": 3,
        "timed_calls": 10,
        "metric": "local_call_ms",
        "median_ms": 6786.2605,
        "wall_median_ms": null,
        "samples_ms": [
          6662.747,
          6815.696,
          6687.229,
          6762.426,
          6928.557,
          6867.698,
          6706.089,
          6784.906,
          6874.29,
          6787.615
        ],
        "source": "reports/latency/mac_m5_pro_selfjev4b.json",
        "endpoint": "TreeServer.score_requests"
      },
      {
        "hardware": "Apple M5 Pro / 48 GB",
        "model": "SelfJev-4B (current adapter)",
        "engine": "TreeServer / MPS bf16",
        "text_tokens": 512,
        "questions": 1,
        "candidates_per_question": 3,
        "timed_calls": 10,
        "metric": "local_call_ms",
        "median_ms": 2018.3355000000001,
        "wall_median_ms": null,
        "samples_ms": [
          2041.356,
          1995.315,
          1872.847,
          2047.419,
          1971.03,
          2093.553,
          2070.734,
          2053.476,
          1980.676,
          1912.813
        ],
        "source": "reports/latency/mac_m5_pro_selfjev4b.json",
        "endpoint": "TreeServer.score_requests"
      },
      {
        "hardware": "Apple M5 Pro / 48 GB",
        "model": "SelfJev-4B (current adapter)",
        "engine": "TreeServer / MPS bf16",
        "text_tokens": 512,
        "questions": 16,
        "candidates_per_question": 3,
        "timed_calls": 10,
        "metric": "local_call_ms",
        "median_ms": 8598.7065,
        "wall_median_ms": null,
        "samples_ms": [
          8624.271,
          8651.608,
          8528.286,
          8357.41,
          8620.205,
          8703.927,
          8858.519,
          8577.208,
          8215.803,
          8237.903
        ],
        "source": "reports/latency/mac_m5_pro_selfjev4b.json",
        "endpoint": "TreeServer.score_requests"
      }
    ],
    "mac_meta": {
      "date": "2026-09-28",
      "chip": "Apple M5 Pro",
      "gpu_cores": 20,
      "unified_memory_gb": 48,
      "platform": "macOS-26.5.2-arm64-arm-64bit",
      "torch": "2.14.0",
      "model": {
        "rotary_buffer_precision": "native",
        "adapter_sha256": "dfbf2834d883987893ec305a6093a345fd79c77f903f60f2f914cbe3b6058d1b",
        "model": "Qwen/Qwen3.5-4B",
        "revision": "851bf6e806efd8d0a36b00ddf55e13ccb7b8cd0a",
        "adapter": "weights/selfjev_4b",
        "device": "mps",
        "dtype": "bfloat16",
        "prompt": "challenger-state-first-v1",
        "prompt_sha": "6c9d8459ac44",
        "truncation": "none",
        "max_length": 8192,
        "adapter_merged": true,
        "architecture": "shared-prefix tree, forward only: text once per request, each question once, then each candidate"
      },
      "input_text": "synthetic `blue ` repeated to exactly the stated tokenizer-token length",
      "method": "TreeServer.score_requests, one request at a time; MPS synchronized before and after each call; wall time includes tokenization and model work, excludes HTTP/network; 3-option multiclass questions; two warmups per cell",
      "warmup": 2,
      "reps": 10,
      "local_cost_usd": 0
    },
    "scope": "Different models and engines. No current-model NVIDIA measurement. No new inference."
  },
  "external_text_decisions": {
    "model": "Eikos-4B",
    "accuracy": 92.7674535409342,
    "source": "reports/eikos_4b/eval2/report.json"
  },
  "provenance_sha256": {
    "data/all.jsonl.gz": "cc1db4b3c8c72832c128600c1213486cb01d6880df8cfdd8eda3710ca6712b20",
    "reports/selfjev_4b_treeserver/test/report.json": "ec3da777a49358850820115c6e9d98a4a7a25d736bb7a23f74edb904462e751e",
    "reports/external/full/typesafe_jev-latest/report.json": "02c56f7564b5d257194d996a0831d22f78f168b4247b14ac868c7106acf3add0",
    "reports/selfjev_4b_treeserver/eval2/report.json": "6d3990d0f77e94985edb9d9c174e9a9a326671bc045d578761993da51dcdfa13",
    "reports/external/eval2/typesafe_jev-latest/report.json": "a632fd3a6a5628b89aa7a371587bac0d362b8de846c4327c2174c796b4b6ae4a",
    "reports/selfjev_4b_treeserver/eval_llm/report.json": "e98db0fbee4109e6fdb8585334662754abd747357d8474598bcf61e966cb8f6d",
    "reports/baseline/test/report.json": "8659d24af1f35e9fe29b7b54b73f44080d445277a8fe74d01b581c0b8ed0dbc7",
    "reports/baseline_4b/test/report.json": "db31f7e9b922590955d7056475732e68763f4aabca0b5d78fd1cf180a5db3b3a",
    "reports/baseline_8b/test/report.json": "459c908e69636e6cae3d49f50801be5463e27c8d7777c18c7ed9ca4165e61ad9",
    "reports/lora_pilot/test/report.json": "4cc1dfc68e954155fcb017a478ee19b30a9f91c204f79d906bac02cba00e4051",
    "reports/lora_4b/test/report.json": "d385af3b3b180b003d9f11a613305c2293ac1d35a0866b304bd881ef42a1c487",
    "reports/lora_8b/test/report.json": "2749882e4717d8310b2aced826bd8712279d0b8258ca5317e5e0d6ab25322ecb",
    "reports/tree_4b/eval2/report.json": "a61afe4c7ebd85e37684b9b74a7f938375a7d252973be028d0f822b1d1d4cf8c",
    "reports/tree_4b_r2b/eval2/report.json": "6438041ed5b3818b2a324de2356a20fd1394f6e1efdd1daaee7ed6499e3cf578",
    "reports/tree_4b_instruct_r2x64/eval2/report.json": "2f2b94c58f1640bd6fc55831f35f7b937745763fbd5e825cebdcdf890d2f33af",
    "reports/tree_4b_combo/eval2/report.json": "9bc9eed004d3a9477041925549b03e369efb4e73896fc1e0020ef609c6633dd7",
    "reports/qwen35_4b_tree/eval2/report.json": "5ec21f63b8e4fe33986a5c110b5ed4558b8c50f8a9ca4eaa1dacdbee0991e55e",
    "reports/eikos_4b/eval2/report.json": "ebd207e342f94c7a2d3bb4047fdcfd10dc802a2cf986eff820dd23217a9e19f9",
    "reports/latency/requests_combo.jsonl": "424b8c71ce36a30bfec6675bf2aa88611fbf8063e5e8aff06a54489a1e8e9174",
    "reports/latency/requests_qwen35.jsonl": "c604f1dd350691bfc46839947fa6963f0459140b14f6e174002060823ebea845",
    "reports/latency/requests_h100.jsonl": "9822ca6be7e027c8f60c24c5b571d9e59614ffc2682c4d12152dab8c5fb42c23",
    "reports/latency/mac_m5_pro_selfjev4b.json": "37e45d4d2bad001d6a8dcccada3f1d591ca5eb146c2366a1fb18ab27cd50ab2e"
  },
  "scope": "Saved results only. No new inference. Public datasets adapted; no S1Bench claim."
}