""" LR Scheduler Benchmark — Telemetry edition ========================================== Gradio Space, dark dashboard UI, sidebar scheduler picker, Plotly charts. GPU training runs on Modal (user's credits). Results cached locally. """ from __future__ import annotations import gradio as gr import pandas as pd import json import os from pathlib import Path from charts import line_plot, horizontal_bar, color_for, THEMES, SCHEDULER_COLORS # ─── Modal Integration ─────────────────────────────────────────────────────── MODAL_AVAILABLE = False try: import modal if os.environ.get("MODAL_TOKEN_ID") and os.environ.get("MODAL_TOKEN_SECRET"): MODAL_AVAILABLE = True except ImportError: pass MODAL_APP_NAME = "lr-scheduler-benchmark" MODAL_FULL_FN = "run_full_benchmark" # ─── Scheduler registry ────────────────────────────────────────────────────── ALL_SCHEDULERS = [ "cosine", "linear", "cosine_with_restarts", "polynomial", "constant", "constant_with_warmup", "inverse_sqrt", "wsd", "wsd_long", "onecycle", "exponential", "lucky_pick", "rlrs", "greedy_lr", "dlrs", "aecs", "rex", "exp_increasing", "scaling_law_opt", "step_decay", "cosine_annealing", "plateau_reduction", "cyclic_lr", "mnist_example", "sqrt_scheduler", ] SCHEDULER_DISPLAY = { "cosine": "Cosine", "linear": "Linear", "cosine_with_restarts": "Cosine w/ Restarts", "polynomial": "Polynomial (p=2)", "constant": "Constant", "constant_with_warmup": "Constant + Warmup", "inverse_sqrt": "Inverse √", "wsd": "WSD", "wsd_long": "WSD-Long", "onecycle": "OneCycleLR", "exponential": "Exponential", "lucky_pick": "Lucky Pick", "rlrs": "RLRS", "greedy_lr": "GreedyLR", "dlrs": "DLRS", "aecs": "AECS", "rex": "REX", "exp_increasing": "Exp Increasing", "scaling_law_opt": "Scaling Law Optimal", "step_decay": "StepLR", "cosine_annealing": "Cosine Annealing", "plateau_reduction": "Plateau Reduction", "cyclic_lr": "CyclicLR (Triangular2)", "mnist_example": "MNIST Example", "sqrt_scheduler": "Square Root Decay", } SCHEDULER_FAMILY = { "cosine": "decay", "linear": "decay", "polynomial": "decay", "inverse_sqrt": "decay", "wsd": "decay", "wsd_long": "decay", "rex": "decay", "scaling_law_opt": "decay", "exponential": "decay", "sqrt_scheduler": "decay", "cosine_annealing": "decay", "cosine_with_restarts": "cyclical", "cyclic_lr": "cyclical", "onecycle": "cyclical", "lucky_pick": "adaptive", "rlrs": "adaptive", "greedy_lr": "adaptive", "dlrs": "adaptive", "aecs": "adaptive", "plateau_reduction": "adaptive", "step_decay": "step", "mnist_example": "step", "constant": "constant", "constant_with_warmup": "constant", "exp_increasing": "exotic", } SOURCE_LABEL = { "cosine": "HF", "linear": "HF", "cosine_with_restarts": "HF", "polynomial": "HF", "constant": "HF", "constant_with_warmup": "HF", "inverse_sqrt": "HF", "wsd": "HF", "wsd_long": "HF", "onecycle": "PyTorch", "exponential": "PyTorch", "step_decay": "PyTorch", "cosine_annealing": "PyTorch", "plateau_reduction": "PyTorch", "mnist_example": "PyTorch", "lucky_pick": "GitHub", "aecs": "GitHub", "rlrs": "This bench", "greedy_lr": "This bench", "dlrs": "This bench", "rex": "Paper", "scaling_law_opt": "Paper", "cyclic_lr": "Paper", "sqrt_scheduler": "Paper", "exp_increasing": "—", } # ─── Data Loading ──────────────────────────────────────────────────────────── DATA_DIR = Path("data") DATA_DIR.mkdir(exist_ok=True) def load_leaderboard(): p = DATA_DIR / "leaderboard.csv" if p.exists(): return pd.read_csv(p) return _empty_leaderboard() def load_curves(): p = DATA_DIR / "curves.csv" if p.exists(): return pd.read_csv(p) return pd.DataFrame(columns=["scheduler", "seed", "step", "train_loss", "val_loss", "val_acc", "lr"]) def load_per_run(): p = DATA_DIR / "per_run.csv" if p.exists(): return pd.read_csv(p) return pd.DataFrame() def load_config(): p = DATA_DIR / "config.json" if p.exists(): with open(p) as f: return json.load(f) return _default_config() def _default_config(): return { "model": "distilbert/distilbert-base-uncased", "dataset": "nyu-mll/glue/sst2", "batch_size": 32, "num_epochs": 3, "learning_rate": 2e-5, "weight_decay": 0.01, "warmup_fraction": 0.06, "eval_every_steps": 50, "target_loss": 0.35, "num_seeds": 3, } def _empty_leaderboard(): n = len(ALL_SCHEDULERS) return pd.DataFrame({ "rank": list(range(1, n + 1)), "scheduler": ALL_SCHEDULERS, "final_val_loss_mean": [None]*n, "final_val_loss_std": [None]*n, "final_val_acc_mean": [None]*n, "final_val_acc_std": [None]*n, "auc_val_loss_mean": [None]*n, "steps_to_target_mean": [None]*n, "steps_to_target_reached": [0]*n, "num_seeds": [0]*n, }) # ─── Save / merge results ──────────────────────────────────────────────────── def save_results(result_data): new_keys = {r["scheduler"] for r in result_data["leaderboard"]} def _merge(new_rows, fname): new_df = pd.DataFrame(new_rows) path = DATA_DIR / fname if path.exists(): old = pd.read_csv(path) old = old[~old["scheduler"].isin(new_keys)] merged = pd.concat([old, new_df], ignore_index=True) else: merged = new_df return merged, path lb, lb_path = _merge(result_data["leaderboard"], "leaderboard.csv") lb = lb.sort_values("final_val_loss_mean", na_position="last").reset_index(drop=True) lb["rank"] = range(1, len(lb) + 1) lb.to_csv(lb_path, index=False) pr, pr_path = _merge(result_data["per_run"], "per_run.csv") pr.to_csv(pr_path, index=False) cv, cv_path = _merge(result_data["curves"], "curves.csv") cv.to_csv(cv_path, index=False) with open(DATA_DIR / "config.json", "w") as f: json.dump(result_data.get("config", _default_config()), f, indent=2) # Persist to HF Space repo so data survives rebuilds try: from huggingface_hub import HfApi token = os.environ.get("HF_TOKEN") or os.environ.get("HUGGING_FACE_HUB_TOKEN") api = HfApi(token=token) space_id = "juiceb0xc0de/lr-scheduler-benchmark" for fname in ["leaderboard.csv", "curves.csv", "per_run.csv", "config.json"]: fp = DATA_DIR / fname if fp.exists(): api.upload_file(path_or_fileobj=str(fp), path_in_repo=f"data/{fname}", repo_id=space_id, repo_type="space") except Exception as e: print(f"Warning: couldn't upload to Space: {e}") # ─── Live state ────────────────────────────────────────────────────────────── leaderboard_raw = load_leaderboard() curves_raw = load_curves() config = load_config() def _has_data(): return leaderboard_raw["final_val_loss_mean"].notna().any() # ─── HTML chrome builders ─────────────────────────────────────────────────── def header_html(): has = _has_data() n_seeds = config.get("num_seeds", 3) sub = f"v0.3 · {len(ALL_SCHEDULERS)} schedulers · SST-2/DistilBERT · {n_seeds} seeds" return f"""
LR Scheduler Bench
{sub}
{'last run · live data' if has else 'awaiting first benchmark run'} · discussions ↗
""" def kpi_strip_html(): if not _has_data(): return """
Schedulers
""" + str(len(ALL_SCHEDULERS)) + """
awaiting results
Best val loss
—
run benchmark first
Best val accuracy
—
run benchmark first
Total runs
0
scheduler × seed
""" df = leaderboard_raw[leaderboard_raw["final_val_loss_mean"].notna()].copy() df = df.sort_values("final_val_loss_mean") best = df.iloc[0] best_name = SCHEDULER_DISPLAY.get(best["scheduler"], best["scheduler"]) best_acc = df["final_val_acc_mean"].max() best_acc_row = df.loc[df["final_val_acc_mean"].idxmax()] best_acc_name = SCHEDULER_DISPLAY.get(best_acc_row["scheduler"], best_acc_row["scheduler"]) total_runs = int(df["num_seeds"].sum()) if "num_seeds" in df.columns else 0 return f"""
Schedulers benchmarked
{len(df)}
of {len(ALL_SCHEDULERS)} total
Best val loss
{best['final_val_loss_mean']:.4f}
{best_name}
Best val accuracy
{best_acc*100:.2f}%
{best_acc_name}
Total GPU runs
{total_runs}
scheduler × seed
""" # ─── Callbacks ────────────────────────────────────────────────────────────── DISPLAY_TO_KEY = {v: k for k, v in SCHEDULER_DISPLAY.items()} ALL_DISPLAY = [SCHEDULER_DISPLAY[k] for k in ALL_SCHEDULERS] def display_to_keys(display_names): return [DISPLAY_TO_KEY.get(n, n) for n in (display_names or [])] def on_compare_change(selected_display): keys = display_to_keys(selected_display) val_fig = line_plot(curves_raw, keys, SCHEDULER_DISPLAY, "val_loss", "val loss", theme="dark", height=360) lr_fig = line_plot(curves_raw, keys, SCHEDULER_DISPLAY, "lr", "lr", theme="dark", height=360) train_fig = line_plot(curves_raw, keys, SCHEDULER_DISPLAY, "train_loss","train loss", theme="dark", height=300) return val_fig, lr_fig, train_fig def on_bar_metric_change(metric_label, selected_display): metric_map = { "Val loss": "final_val_loss_mean", "Val acc": "final_val_acc_mean", "AUC": "auc_val_loss_mean", "Steps": "steps_to_target_mean", } metric = metric_map.get(metric_label, "final_val_loss_mean") keys = set(display_to_keys(selected_display)) return horizontal_bar(leaderboard_raw, metric, SCHEDULER_DISPLAY, selected_keys=keys, theme="dark") # ─── Leaderboard table ────────────────────────────────────────────────────── SORT_OPTIONS = { "Rank": ("rank", True), "Val loss ↓": ("final_val_loss_mean", True), "Val accuracy ↑": ("final_val_acc_mean", False), "AUC ↓": ("auc_val_loss_mean", True), "Steps to target ↓": ("steps_to_target_mean", True), } def format_leaderboard(df): out = pd.DataFrame() out["#"] = df["rank"].apply(lambda v: f"{int(v):02d}") out["Scheduler"] = df["scheduler"].map(lambda s: SCHEDULER_DISPLAY.get(s, s)) out["Family"] = df["scheduler"].map(lambda s: SCHEDULER_FAMILY.get(s, "—").title()) out["Source"] = df["scheduler"].map(lambda s: SOURCE_LABEL.get(s, "—")) if df["final_val_loss_mean"].notna().any(): out["Val loss ↓"] = df.apply( lambda r: f"{r['final_val_loss_mean']:.4f} ±{r['final_val_loss_std']:.3f}" if pd.notna(r["final_val_loss_mean"]) else "—", axis=1) out["Val acc ↑"] = df.apply( lambda r: f"{r['final_val_acc_mean']*100:.2f}% ±{r['final_val_acc_std']*100:.2f}" if pd.notna(r["final_val_acc_mean"]) else "—", axis=1) out["AUC ↓"] = df.apply( lambda r: f"{r['auc_val_loss_mean']:.1f}" if pd.notna(r.get("auc_val_loss_mean")) else "—", axis=1) out["Steps→target ↓"] = df.apply( lambda r: f"{int(r['steps_to_target_mean'])}" if pd.notna(r.get("steps_to_target_mean")) else "—", axis=1) out["Seeds"] = df["num_seeds"].astype(int) else: for c in ["Val loss ↓", "Val acc ↑", "AUC ↓", "Steps→target ↓"]: out[c] = "—" out["Seeds"] = 0 return out def sort_leaderboard(sort_label): col, asc = SORT_OPTIONS.get(sort_label, ("rank", True)) df = leaderboard_raw.sort_values(col, ascending=asc, na_position="last").copy() df["rank"] = range(1, len(df) + 1) return format_leaderboard(df) # ─── Modal callbacks ──────────────────────────────────────────────────────── def launch_benchmark(selected_display, num_seeds): if not MODAL_AVAILABLE: return None, ("> ⚠ **Modal not configured.** Add `MODAL_TOKEN_ID` and `MODAL_TOKEN_SECRET` " "to this Space's secrets, then deploy with `modal deploy modal_benchmark.py`.") if not selected_display: return None, "> ⚠ Select at least one scheduler." try: keys = display_to_keys(selected_display) fn = modal.Function.from_name(MODAL_APP_NAME, MODAL_FULL_FN) call = fn.spawn(schedulers=keys, num_seeds=int(num_seeds)) return (call.object_id, f"> 🚀 **Launched** — {len(keys)} schedulers × {int(num_seeds)} seeds = " f"{len(keys)*int(num_seeds)} L40S containers.\n>\n" f"> Job id: `{call.object_id}` · expected ~15–30 min · click **Check status** to poll.") except Exception as e: return None, f"> ❌ Failed to launch: `{e}`" def check_benchmark_status(call_id): if not call_id: return "> No job running. Launch a benchmark first." if not MODAL_AVAILABLE: return "> ⚠ Modal not configured." try: fc = modal.FunctionCall.from_id(call_id) result = fc.get(timeout=0) if "error" in result: return f"> ❌ Failed: `{result['error']}`" save_results(result) global leaderboard_raw, curves_raw, config leaderboard_raw = load_leaderboard() curves_raw = load_curves() config = load_config() n_runs = len(result["per_run"]) best = result["leaderboard"][0] best_name = SCHEDULER_DISPLAY.get(best["scheduler"], best["scheduler"]) return (f"> ✅ **Complete** — {len(result['leaderboard'])} schedulers, {n_runs} runs.\n>\n" f"> 🏆 Best: **{best_name}** — val loss {best['final_val_loss_mean']:.4f}, " f"acc {best['final_val_acc_mean']*100:.2f}%.\n>\n" f"> Reload the page to see updated charts.") except TimeoutError: return "> ⏳ Still running. Check again in a minute." except Exception as e: return f"> ⚠ Error: `{e}`" # ─── UI ───────────────────────────────────────────────────────────────────── CSS_PATH = Path(__file__).parent / "static" / "style.css" CSS_TEXT = CSS_PATH.read_text() if CSS_PATH.exists() else "" DEFAULT_SELECTED = ALL_DISPLAY[:5] # Top 5 by spec with gr.Blocks(css=CSS_TEXT, theme=gr.themes.Base(), title="LR Scheduler Bench") as demo: gr.HTML(header_html()) kpi_html = gr.HTML(kpi_strip_html()) with gr.Tabs(): # ════════════════════════════════════════════════════════════════════ # COMPARE # ════════════════════════════════════════════════════════════════════ with gr.TabItem("Compare"): with gr.Row(equal_height=False): with gr.Column(scale=1, min_width=260, elem_classes=["tm-sidebar"]): gr.HTML('
Schedulers
') with gr.Row(): all_btn = gr.Button("All", size="sm") none_btn = gr.Button("None", size="sm") top5_btn = gr.Button("Top 5", size="sm") sched_select = gr.CheckboxGroup( choices=ALL_DISPLAY, value=DEFAULT_SELECTED, label="", show_label=False, container=False, ) with gr.Column(scale=4): with gr.Row(): with gr.Column(): gr.Markdown('
Validation loss
' '
Mean over seeds, per training step.
') val_plot = gr.Plot( value=line_plot(curves_raw, display_to_keys(DEFAULT_SELECTED), SCHEDULER_DISPLAY, "val_loss", "val loss", theme="dark", height=360), show_label=False, container=False, ) with gr.Column(): gr.Markdown('
Learning rate schedule
' '
Per-step LR (single seed).
') lr_plot = gr.Plot( value=line_plot(curves_raw, display_to_keys(DEFAULT_SELECTED), SCHEDULER_DISPLAY, "lr", "lr", theme="dark", height=360), show_label=False, container=False, ) gr.Markdown('
Train loss
' '
Mean over seeds.
') train_plot = gr.Plot( value=line_plot(curves_raw, display_to_keys(DEFAULT_SELECTED), SCHEDULER_DISPLAY, "train_loss", "train loss", theme="dark", height=300), show_label=False, container=False, ) with gr.Row(): gr.Markdown('
Final ranking
' '
Horizontal bars — labels never overlap.
') bar_metric = gr.Radio( choices=["Val loss", "Val acc", "AUC", "Steps"], value="Val loss", label="", show_label=False, container=False, ) bar_plot = gr.Plot( value=horizontal_bar(leaderboard_raw, "final_val_loss_mean", SCHEDULER_DISPLAY, selected_keys=set(display_to_keys(DEFAULT_SELECTED)), theme="dark"), show_label=False, container=False, ) # Wire callbacks sched_select.change(on_compare_change, inputs=sched_select, outputs=[val_plot, lr_plot, train_plot]) sched_select.change(on_bar_metric_change, inputs=[bar_metric, sched_select], outputs=bar_plot) bar_metric.change(on_bar_metric_change, inputs=[bar_metric, sched_select], outputs=bar_plot) all_btn.click( lambda: ALL_DISPLAY, outputs=sched_select) none_btn.click(lambda: [], outputs=sched_select) top5_btn.click(lambda: ALL_DISPLAY[:5], outputs=sched_select) # ════════════════════════════════════════════════════════════════════ # LEADERBOARD # ════════════════════════════════════════════════════════════════════ with gr.TabItem("Leaderboard"): gr.Markdown( '
Full leaderboard
' '
Sortable. Best in column highlighted in mono numerals.
' ) sort_dd = gr.Dropdown( choices=list(SORT_OPTIONS.keys()), value="Rank", label="Sort by", container=False, scale=0, ) lb_table = gr.DataFrame( value=format_leaderboard(leaderboard_raw), interactive=False, max_height=620, wrap=False, ) sort_dd.change(sort_leaderboard, sort_dd, lb_table) # ════════════════════════════════════════════════════════════════════ # RUN BENCHMARK # ════════════════════════════════════════════════════════════════════ with gr.TabItem("Run Benchmark"): gr.Markdown(f""" ### Run on Modal GPU Each (scheduler, seed) pair gets its own L40S container. Estimated cost ~$0.50–1.00 for all schedulers × 3 seeds. Estimated time 15–30 min in parallel. """) if not MODAL_AVAILABLE: gr.HTML('
Modal not configured. ' 'Add MODAL_TOKEN_ID and MODAL_TOKEN_SECRET ' 'to Space secrets, then modal deploy modal_benchmark.py.
') job_state = gr.State(value=None) run_select = gr.CheckboxGroup( choices=ALL_DISPLAY, value=ALL_DISPLAY, label="Schedulers to benchmark", ) num_seeds = gr.Slider(1, 5, value=3, step=1, label="Seeds per scheduler") with gr.Row(): launch_btn = gr.Button("🚀 Launch benchmark", variant="primary", interactive=MODAL_AVAILABLE) check_btn = gr.Button("Check status") status = gr.Markdown("> Ready to launch.") launch_btn.click(launch_benchmark, [run_select, num_seeds], [job_state, status]) check_btn.click( check_benchmark_status, [job_state], [status]) # ════════════════════════════════════════════════════════════════════ # METHODOLOGY # ════════════════════════════════════════════════════════════════════ with gr.TabItem("Methodology"): gr.Markdown(f""" ### Benchmark design Twenty-five schedulers under **identical conditions** — only the scheduler changes between runs. | Parameter | Value | |---|---| | Model | `{config['model']}` (67M) | | Task | SST-2 sentiment classification ([GLUE](https://huggingface.co/datasets/nyu-mll/glue)) | | Optimizer | AdamW (β=0.9/0.999, ε=1e-8) | | Peak LR | `{config['learning_rate']}` | | Weight decay | `{config['weight_decay']}` | | Batch size | {config['batch_size']} | | Epochs | {config['num_epochs']} | | Warmup | {int(config['warmup_fraction']*100)}% of steps | | Grad clip | max-norm 1.0 | | Seeds | {config['num_seeds']} | ### Metrics | Metric | Description | Better | |---|---|---| | Final val loss | Cross-entropy on SST-2 val | ↓ | | Final val accuracy | Classification accuracy | ↑ | | AUC val loss | Area under val-loss curve (convergence speed) | ↓ | | Steps→target | First step where val loss ≤ {config['target_loss']} | ↓ | ### Fairness notes - All schedulers share the same peak LR (`{config['learning_rate']}`); this measures **out-of-the-box** behavior at a fixed LR. Some schedulers benefit from per-scheduler tuning. - OneCycleLR uses 10× peak as `max_lr` (standard super-convergence recipe). - AECS uses a cosine backbone and modulates on top based on detected events (gradient spikes, loss spikes, redundancy, instability, plateau). - Results averaged across {config['num_seeds']} seeds with different shuffle orders. ### Want a scheduler added? Open a thread in [Discussions](https://huggingface.co/spaces/juiceb0xc0de/lr-scheduler-benchmark/discussions) with a link to the implementation (paper, repo, or PyTorch class) and a one-line description of the schedule shape. Custom schedulers should expose a `get_lr(step)` method. ### References - Cosine Annealing — Loshchilov & Hutter, 2016 ([arXiv](https://arxiv.org/abs/1608.03983)) - WSD — MiniCPM, 2024 ([arXiv](https://arxiv.org/abs/2404.06395)) - OneCycleLR — Smith & Topin, 2019 ([arXiv](https://arxiv.org/abs/1708.07120)) - Inverse √ — Vaswani et al., 2017 ([arXiv](https://arxiv.org/abs/1706.03762)) """) with gr.Accordion("Raw per-run results", open=False): pr = load_per_run() if not pr.empty: gr.DataFrame(value=pr, interactive=False, max_height=400) else: gr.Markdown("*Per-run data appears here after benchmark runs complete.*") with gr.Accordion("Citation (BibTeX)", open=False): gr.Code( value="""@misc{lr-scheduler-benchmark-2026, title={LR Scheduler Benchmark: Standardized Comparison of Learning Rate Schedules}, author={JuiceB0xC0de}, year={2026}, url={https://huggingface.co/spaces/juiceb0xc0de/lr-scheduler-benchmark} }""", language="markdown", interactive=False, ) if __name__ == "__main__": demo.launch(server_name="0.0.0.0")