"""
LR Scheduler Benchmark — Telemetry edition
==========================================
Gradio Space, dark dashboard UI, sidebar scheduler picker, Plotly charts.
GPU training runs on Modal (user's credits). Results cached locally.
"""
from __future__ import annotations
import gradio as gr
import pandas as pd
import json
import os
from pathlib import Path
from charts import line_plot, horizontal_bar, color_for, THEMES, SCHEDULER_COLORS
# ─── Modal Integration ───────────────────────────────────────────────────────
MODAL_AVAILABLE = False
try:
import modal
if os.environ.get("MODAL_TOKEN_ID") and os.environ.get("MODAL_TOKEN_SECRET"):
MODAL_AVAILABLE = True
except ImportError:
pass
MODAL_APP_NAME = "lr-scheduler-benchmark"
MODAL_FULL_FN = "run_full_benchmark"
# ─── Scheduler registry ──────────────────────────────────────────────────────
ALL_SCHEDULERS = [
"cosine", "linear", "cosine_with_restarts", "polynomial",
"constant", "constant_with_warmup", "inverse_sqrt",
"wsd", "wsd_long",
"onecycle", "exponential",
"lucky_pick", "rlrs", "greedy_lr", "dlrs", "aecs",
"rex", "exp_increasing", "scaling_law_opt",
"step_decay", "cosine_annealing", "plateau_reduction",
"cyclic_lr", "mnist_example", "sqrt_scheduler",
]
SCHEDULER_DISPLAY = {
"cosine": "Cosine",
"linear": "Linear",
"cosine_with_restarts": "Cosine w/ Restarts",
"polynomial": "Polynomial (p=2)",
"constant": "Constant",
"constant_with_warmup": "Constant + Warmup",
"inverse_sqrt": "Inverse √",
"wsd": "WSD",
"wsd_long": "WSD-Long",
"onecycle": "OneCycleLR",
"exponential": "Exponential",
"lucky_pick": "Lucky Pick",
"rlrs": "RLRS",
"greedy_lr": "GreedyLR",
"dlrs": "DLRS",
"aecs": "AECS",
"rex": "REX",
"exp_increasing": "Exp Increasing",
"scaling_law_opt": "Scaling Law Optimal",
"step_decay": "StepLR",
"cosine_annealing": "Cosine Annealing",
"plateau_reduction": "Plateau Reduction",
"cyclic_lr": "CyclicLR (Triangular2)",
"mnist_example": "MNIST Example",
"sqrt_scheduler": "Square Root Decay",
}
SCHEDULER_FAMILY = {
"cosine": "decay", "linear": "decay", "polynomial": "decay", "inverse_sqrt": "decay",
"wsd": "decay", "wsd_long": "decay", "rex": "decay", "scaling_law_opt": "decay",
"exponential": "decay", "sqrt_scheduler": "decay", "cosine_annealing": "decay",
"cosine_with_restarts": "cyclical", "cyclic_lr": "cyclical", "onecycle": "cyclical",
"lucky_pick": "adaptive", "rlrs": "adaptive", "greedy_lr": "adaptive",
"dlrs": "adaptive", "aecs": "adaptive", "plateau_reduction": "adaptive",
"step_decay": "step", "mnist_example": "step",
"constant": "constant", "constant_with_warmup": "constant",
"exp_increasing": "exotic",
}
SOURCE_LABEL = {
"cosine": "HF", "linear": "HF", "cosine_with_restarts": "HF", "polynomial": "HF",
"constant": "HF", "constant_with_warmup": "HF", "inverse_sqrt": "HF",
"wsd": "HF", "wsd_long": "HF",
"onecycle": "PyTorch", "exponential": "PyTorch", "step_decay": "PyTorch",
"cosine_annealing": "PyTorch", "plateau_reduction": "PyTorch", "mnist_example": "PyTorch",
"lucky_pick": "GitHub", "aecs": "GitHub",
"rlrs": "This bench", "greedy_lr": "This bench", "dlrs": "This bench",
"rex": "Paper", "scaling_law_opt": "Paper", "cyclic_lr": "Paper",
"sqrt_scheduler": "Paper", "exp_increasing": "—",
}
# ─── Data Loading ────────────────────────────────────────────────────────────
DATA_DIR = Path("data")
DATA_DIR.mkdir(exist_ok=True)
def load_leaderboard():
p = DATA_DIR / "leaderboard.csv"
if p.exists():
return pd.read_csv(p)
return _empty_leaderboard()
def load_curves():
p = DATA_DIR / "curves.csv"
if p.exists():
return pd.read_csv(p)
return pd.DataFrame(columns=["scheduler", "seed", "step", "train_loss", "val_loss", "val_acc", "lr"])
def load_per_run():
p = DATA_DIR / "per_run.csv"
if p.exists():
return pd.read_csv(p)
return pd.DataFrame()
def load_config():
p = DATA_DIR / "config.json"
if p.exists():
with open(p) as f:
return json.load(f)
return _default_config()
def _default_config():
return {
"model": "distilbert/distilbert-base-uncased", "dataset": "nyu-mll/glue/sst2",
"batch_size": 32, "num_epochs": 3, "learning_rate": 2e-5,
"weight_decay": 0.01, "warmup_fraction": 0.06, "eval_every_steps": 50,
"target_loss": 0.35, "num_seeds": 3,
}
def _empty_leaderboard():
n = len(ALL_SCHEDULERS)
return pd.DataFrame({
"rank": list(range(1, n + 1)),
"scheduler": ALL_SCHEDULERS,
"final_val_loss_mean": [None]*n, "final_val_loss_std": [None]*n,
"final_val_acc_mean": [None]*n, "final_val_acc_std": [None]*n,
"auc_val_loss_mean": [None]*n, "steps_to_target_mean": [None]*n,
"steps_to_target_reached": [0]*n, "num_seeds": [0]*n,
})
# ─── Save / merge results ────────────────────────────────────────────────────
def save_results(result_data):
new_keys = {r["scheduler"] for r in result_data["leaderboard"]}
def _merge(new_rows, fname):
new_df = pd.DataFrame(new_rows)
path = DATA_DIR / fname
if path.exists():
old = pd.read_csv(path)
old = old[~old["scheduler"].isin(new_keys)]
merged = pd.concat([old, new_df], ignore_index=True)
else:
merged = new_df
return merged, path
lb, lb_path = _merge(result_data["leaderboard"], "leaderboard.csv")
lb = lb.sort_values("final_val_loss_mean", na_position="last").reset_index(drop=True)
lb["rank"] = range(1, len(lb) + 1)
lb.to_csv(lb_path, index=False)
pr, pr_path = _merge(result_data["per_run"], "per_run.csv")
pr.to_csv(pr_path, index=False)
cv, cv_path = _merge(result_data["curves"], "curves.csv")
cv.to_csv(cv_path, index=False)
with open(DATA_DIR / "config.json", "w") as f:
json.dump(result_data.get("config", _default_config()), f, indent=2)
# Persist to HF Space repo so data survives rebuilds
try:
from huggingface_hub import HfApi
token = os.environ.get("HF_TOKEN") or os.environ.get("HUGGING_FACE_HUB_TOKEN")
api = HfApi(token=token)
space_id = "juiceb0xc0de/lr-scheduler-benchmark"
for fname in ["leaderboard.csv", "curves.csv", "per_run.csv", "config.json"]:
fp = DATA_DIR / fname
if fp.exists():
api.upload_file(path_or_fileobj=str(fp), path_in_repo=f"data/{fname}",
repo_id=space_id, repo_type="space")
except Exception as e:
print(f"Warning: couldn't upload to Space: {e}")
# ─── Live state ──────────────────────────────────────────────────────────────
leaderboard_raw = load_leaderboard()
curves_raw = load_curves()
config = load_config()
def _has_data():
return leaderboard_raw["final_val_loss_mean"].notna().any()
# ─── HTML chrome builders ───────────────────────────────────────────────────
def header_html():
has = _has_data()
n_seeds = config.get("num_seeds", 3)
sub = f"v0.3 · {len(ALL_SCHEDULERS)} schedulers · SST-2/DistilBERT · {n_seeds} seeds"
return f"""
"""
def kpi_strip_html():
if not _has_data():
return """
Schedulers
""" + str(len(ALL_SCHEDULERS)) + """
awaiting results
Best val loss
—
run benchmark first
Best val accuracy
—
run benchmark first
Total runs
0
scheduler × seed
"""
df = leaderboard_raw[leaderboard_raw["final_val_loss_mean"].notna()].copy()
df = df.sort_values("final_val_loss_mean")
best = df.iloc[0]
best_name = SCHEDULER_DISPLAY.get(best["scheduler"], best["scheduler"])
best_acc = df["final_val_acc_mean"].max()
best_acc_row = df.loc[df["final_val_acc_mean"].idxmax()]
best_acc_name = SCHEDULER_DISPLAY.get(best_acc_row["scheduler"], best_acc_row["scheduler"])
total_runs = int(df["num_seeds"].sum()) if "num_seeds" in df.columns else 0
return f"""
Schedulers benchmarked
{len(df)}
of {len(ALL_SCHEDULERS)} total
Best val loss
{best['final_val_loss_mean']:.4f}
{best_name}
Best val accuracy
{best_acc*100:.2f}%
{best_acc_name}
Total GPU runs
{total_runs}
scheduler × seed
"""
# ─── Callbacks ──────────────────────────────────────────────────────────────
DISPLAY_TO_KEY = {v: k for k, v in SCHEDULER_DISPLAY.items()}
ALL_DISPLAY = [SCHEDULER_DISPLAY[k] for k in ALL_SCHEDULERS]
def display_to_keys(display_names):
return [DISPLAY_TO_KEY.get(n, n) for n in (display_names or [])]
def on_compare_change(selected_display):
keys = display_to_keys(selected_display)
val_fig = line_plot(curves_raw, keys, SCHEDULER_DISPLAY, "val_loss", "val loss", theme="dark", height=360)
lr_fig = line_plot(curves_raw, keys, SCHEDULER_DISPLAY, "lr", "lr", theme="dark", height=360)
train_fig = line_plot(curves_raw, keys, SCHEDULER_DISPLAY, "train_loss","train loss", theme="dark", height=300)
return val_fig, lr_fig, train_fig
def on_bar_metric_change(metric_label, selected_display):
metric_map = {
"Val loss": "final_val_loss_mean",
"Val acc": "final_val_acc_mean",
"AUC": "auc_val_loss_mean",
"Steps": "steps_to_target_mean",
}
metric = metric_map.get(metric_label, "final_val_loss_mean")
keys = set(display_to_keys(selected_display))
return horizontal_bar(leaderboard_raw, metric, SCHEDULER_DISPLAY, selected_keys=keys, theme="dark")
# ─── Leaderboard table ──────────────────────────────────────────────────────
SORT_OPTIONS = {
"Rank": ("rank", True),
"Val loss ↓": ("final_val_loss_mean", True),
"Val accuracy ↑": ("final_val_acc_mean", False),
"AUC ↓": ("auc_val_loss_mean", True),
"Steps to target ↓": ("steps_to_target_mean", True),
}
def format_leaderboard(df):
out = pd.DataFrame()
out["#"] = df["rank"].apply(lambda v: f"{int(v):02d}")
out["Scheduler"] = df["scheduler"].map(lambda s: SCHEDULER_DISPLAY.get(s, s))
out["Family"] = df["scheduler"].map(lambda s: SCHEDULER_FAMILY.get(s, "—").title())
out["Source"] = df["scheduler"].map(lambda s: SOURCE_LABEL.get(s, "—"))
if df["final_val_loss_mean"].notna().any():
out["Val loss ↓"] = df.apply(
lambda r: f"{r['final_val_loss_mean']:.4f} ±{r['final_val_loss_std']:.3f}"
if pd.notna(r["final_val_loss_mean"]) else "—", axis=1)
out["Val acc ↑"] = df.apply(
lambda r: f"{r['final_val_acc_mean']*100:.2f}% ±{r['final_val_acc_std']*100:.2f}"
if pd.notna(r["final_val_acc_mean"]) else "—", axis=1)
out["AUC ↓"] = df.apply(
lambda r: f"{r['auc_val_loss_mean']:.1f}" if pd.notna(r.get("auc_val_loss_mean")) else "—", axis=1)
out["Steps→target ↓"] = df.apply(
lambda r: f"{int(r['steps_to_target_mean'])}"
if pd.notna(r.get("steps_to_target_mean")) else "—", axis=1)
out["Seeds"] = df["num_seeds"].astype(int)
else:
for c in ["Val loss ↓", "Val acc ↑", "AUC ↓", "Steps→target ↓"]:
out[c] = "—"
out["Seeds"] = 0
return out
def sort_leaderboard(sort_label):
col, asc = SORT_OPTIONS.get(sort_label, ("rank", True))
df = leaderboard_raw.sort_values(col, ascending=asc, na_position="last").copy()
df["rank"] = range(1, len(df) + 1)
return format_leaderboard(df)
# ─── Modal callbacks ────────────────────────────────────────────────────────
def launch_benchmark(selected_display, num_seeds):
if not MODAL_AVAILABLE:
return None, ("> ⚠ **Modal not configured.** Add `MODAL_TOKEN_ID` and `MODAL_TOKEN_SECRET` "
"to this Space's secrets, then deploy with `modal deploy modal_benchmark.py`.")
if not selected_display:
return None, "> ⚠ Select at least one scheduler."
try:
keys = display_to_keys(selected_display)
fn = modal.Function.from_name(MODAL_APP_NAME, MODAL_FULL_FN)
call = fn.spawn(schedulers=keys, num_seeds=int(num_seeds))
return (call.object_id,
f"> 🚀 **Launched** — {len(keys)} schedulers × {int(num_seeds)} seeds = "
f"{len(keys)*int(num_seeds)} L40S containers.\n>\n"
f"> Job id: `{call.object_id}` · expected ~15–30 min · click **Check status** to poll.")
except Exception as e:
return None, f"> ❌ Failed to launch: `{e}`"
def check_benchmark_status(call_id):
if not call_id:
return "> No job running. Launch a benchmark first."
if not MODAL_AVAILABLE:
return "> ⚠ Modal not configured."
try:
fc = modal.FunctionCall.from_id(call_id)
result = fc.get(timeout=0)
if "error" in result:
return f"> ❌ Failed: `{result['error']}`"
save_results(result)
global leaderboard_raw, curves_raw, config
leaderboard_raw = load_leaderboard()
curves_raw = load_curves()
config = load_config()
n_runs = len(result["per_run"])
best = result["leaderboard"][0]
best_name = SCHEDULER_DISPLAY.get(best["scheduler"], best["scheduler"])
return (f"> ✅ **Complete** — {len(result['leaderboard'])} schedulers, {n_runs} runs.\n>\n"
f"> 🏆 Best: **{best_name}** — val loss {best['final_val_loss_mean']:.4f}, "
f"acc {best['final_val_acc_mean']*100:.2f}%.\n>\n"
f"> Reload the page to see updated charts.")
except TimeoutError:
return "> ⏳ Still running. Check again in a minute."
except Exception as e:
return f"> ⚠ Error: `{e}`"
# ─── UI ─────────────────────────────────────────────────────────────────────
CSS_PATH = Path(__file__).parent / "static" / "style.css"
CSS_TEXT = CSS_PATH.read_text() if CSS_PATH.exists() else ""
DEFAULT_SELECTED = ALL_DISPLAY[:5] # Top 5 by spec
with gr.Blocks(css=CSS_TEXT, theme=gr.themes.Base(), title="LR Scheduler Bench") as demo:
gr.HTML(header_html())
kpi_html = gr.HTML(kpi_strip_html())
with gr.Tabs():
# ════════════════════════════════════════════════════════════════════
# COMPARE
# ════════════════════════════════════════════════════════════════════
with gr.TabItem("Compare"):
with gr.Row(equal_height=False):
with gr.Column(scale=1, min_width=260, elem_classes=["tm-sidebar"]):
gr.HTML('')
with gr.Row():
all_btn = gr.Button("All", size="sm")
none_btn = gr.Button("None", size="sm")
top5_btn = gr.Button("Top 5", size="sm")
sched_select = gr.CheckboxGroup(
choices=ALL_DISPLAY,
value=DEFAULT_SELECTED,
label="",
show_label=False,
container=False,
)
with gr.Column(scale=4):
with gr.Row():
with gr.Column():
gr.Markdown('Validation loss
'
'Mean over seeds, per training step.
')
val_plot = gr.Plot(
value=line_plot(curves_raw, display_to_keys(DEFAULT_SELECTED),
SCHEDULER_DISPLAY, "val_loss", "val loss",
theme="dark", height=360),
show_label=False, container=False,
)
with gr.Column():
gr.Markdown('Learning rate schedule
'
'Per-step LR (single seed).
')
lr_plot = gr.Plot(
value=line_plot(curves_raw, display_to_keys(DEFAULT_SELECTED),
SCHEDULER_DISPLAY, "lr", "lr",
theme="dark", height=360),
show_label=False, container=False,
)
gr.Markdown('Train loss
'
'Mean over seeds.
')
train_plot = gr.Plot(
value=line_plot(curves_raw, display_to_keys(DEFAULT_SELECTED),
SCHEDULER_DISPLAY, "train_loss", "train loss",
theme="dark", height=300),
show_label=False, container=False,
)
with gr.Row():
gr.Markdown('Final ranking
'
'Horizontal bars — labels never overlap.
')
bar_metric = gr.Radio(
choices=["Val loss", "Val acc", "AUC", "Steps"],
value="Val loss",
label="", show_label=False, container=False,
)
bar_plot = gr.Plot(
value=horizontal_bar(leaderboard_raw, "final_val_loss_mean",
SCHEDULER_DISPLAY,
selected_keys=set(display_to_keys(DEFAULT_SELECTED)),
theme="dark"),
show_label=False, container=False,
)
# Wire callbacks
sched_select.change(on_compare_change, inputs=sched_select,
outputs=[val_plot, lr_plot, train_plot])
sched_select.change(on_bar_metric_change, inputs=[bar_metric, sched_select],
outputs=bar_plot)
bar_metric.change(on_bar_metric_change, inputs=[bar_metric, sched_select],
outputs=bar_plot)
all_btn.click( lambda: ALL_DISPLAY, outputs=sched_select)
none_btn.click(lambda: [], outputs=sched_select)
top5_btn.click(lambda: ALL_DISPLAY[:5], outputs=sched_select)
# ════════════════════════════════════════════════════════════════════
# LEADERBOARD
# ════════════════════════════════════════════════════════════════════
with gr.TabItem("Leaderboard"):
gr.Markdown(
'Full leaderboard
'
'Sortable. Best in column highlighted in mono numerals.
'
)
sort_dd = gr.Dropdown(
choices=list(SORT_OPTIONS.keys()), value="Rank",
label="Sort by", container=False, scale=0,
)
lb_table = gr.DataFrame(
value=format_leaderboard(leaderboard_raw),
interactive=False, max_height=620, wrap=False,
)
sort_dd.change(sort_leaderboard, sort_dd, lb_table)
# ════════════════════════════════════════════════════════════════════
# RUN BENCHMARK
# ════════════════════════════════════════════════════════════════════
with gr.TabItem("Run Benchmark"):
gr.Markdown(f"""
### Run on Modal GPU
Each (scheduler, seed) pair gets its own L40S container. Estimated cost ~$0.50–1.00 for
all schedulers × 3 seeds. Estimated time 15–30 min in parallel.
""")
if not MODAL_AVAILABLE:
gr.HTML('Modal not configured. '
'Add MODAL_TOKEN_ID and MODAL_TOKEN_SECRET '
'to Space secrets, then modal deploy modal_benchmark.py.
')
job_state = gr.State(value=None)
run_select = gr.CheckboxGroup(
choices=ALL_DISPLAY, value=ALL_DISPLAY,
label="Schedulers to benchmark",
)
num_seeds = gr.Slider(1, 5, value=3, step=1, label="Seeds per scheduler")
with gr.Row():
launch_btn = gr.Button("🚀 Launch benchmark", variant="primary",
interactive=MODAL_AVAILABLE)
check_btn = gr.Button("Check status")
status = gr.Markdown("> Ready to launch.")
launch_btn.click(launch_benchmark, [run_select, num_seeds], [job_state, status])
check_btn.click( check_benchmark_status, [job_state], [status])
# ════════════════════════════════════════════════════════════════════
# METHODOLOGY
# ════════════════════════════════════════════════════════════════════
with gr.TabItem("Methodology"):
gr.Markdown(f"""
### Benchmark design
Twenty-five schedulers under **identical conditions** — only the scheduler changes between runs.
| Parameter | Value |
|---|---|
| Model | `{config['model']}` (67M) |
| Task | SST-2 sentiment classification ([GLUE](https://huggingface.co/datasets/nyu-mll/glue)) |
| Optimizer | AdamW (β=0.9/0.999, ε=1e-8) |
| Peak LR | `{config['learning_rate']}` |
| Weight decay | `{config['weight_decay']}` |
| Batch size | {config['batch_size']} |
| Epochs | {config['num_epochs']} |
| Warmup | {int(config['warmup_fraction']*100)}% of steps |
| Grad clip | max-norm 1.0 |
| Seeds | {config['num_seeds']} |
### Metrics
| Metric | Description | Better |
|---|---|---|
| Final val loss | Cross-entropy on SST-2 val | ↓ |
| Final val accuracy | Classification accuracy | ↑ |
| AUC val loss | Area under val-loss curve (convergence speed) | ↓ |
| Steps→target | First step where val loss ≤ {config['target_loss']} | ↓ |
### Fairness notes
- All schedulers share the same peak LR (`{config['learning_rate']}`); this measures **out-of-the-box**
behavior at a fixed LR. Some schedulers benefit from per-scheduler tuning.
- OneCycleLR uses 10× peak as `max_lr` (standard super-convergence recipe).
- AECS uses a cosine backbone and modulates on top based on detected events (gradient spikes,
loss spikes, redundancy, instability, plateau).
- Results averaged across {config['num_seeds']} seeds with different shuffle orders.
### Want a scheduler added?
Open a thread in [Discussions](https://huggingface.co/spaces/juiceb0xc0de/lr-scheduler-benchmark/discussions)
with a link to the implementation (paper, repo, or PyTorch class) and a one-line description of
the schedule shape. Custom schedulers should expose a `get_lr(step)` method.
### References
- Cosine Annealing — Loshchilov & Hutter, 2016 ([arXiv](https://arxiv.org/abs/1608.03983))
- WSD — MiniCPM, 2024 ([arXiv](https://arxiv.org/abs/2404.06395))
- OneCycleLR — Smith & Topin, 2019 ([arXiv](https://arxiv.org/abs/1708.07120))
- Inverse √ — Vaswani et al., 2017 ([arXiv](https://arxiv.org/abs/1706.03762))
""")
with gr.Accordion("Raw per-run results", open=False):
pr = load_per_run()
if not pr.empty:
gr.DataFrame(value=pr, interactive=False, max_height=400)
else:
gr.Markdown("*Per-run data appears here after benchmark runs complete.*")
with gr.Accordion("Citation (BibTeX)", open=False):
gr.Code(
value="""@misc{lr-scheduler-benchmark-2026,
title={LR Scheduler Benchmark: Standardized Comparison of Learning Rate Schedules},
author={JuiceB0xC0de},
year={2026},
url={https://huggingface.co/spaces/juiceb0xc0de/lr-scheduler-benchmark}
}""",
language="markdown", interactive=False,
)
if __name__ == "__main__":
demo.launch(server_name="0.0.0.0")