Spaces:
Sleeping
Sleeping
Download app.py from juiceb0xc0de/lr-scheduler-benchmark: direct link, hf CLI and curl.
- Browser
- Download file 28.5 kB
-
https://huggingface.co/spaces/juiceb0xc0de/lr-scheduler-benchmark/resolve/main/app.py
- Command line
-
hf download hf://spaces/juiceb0xc0de/lr-scheduler-benchmark/app.py
-
curl -L -o app.py https://huggingface.co/spaces/juiceb0xc0de/lr-scheduler-benchmark/resolve/main/app.py
28.5 kB
| """ | |
| LR Scheduler Benchmark — Telemetry edition | |
| ========================================== | |
| Gradio Space, dark dashboard UI, sidebar scheduler picker, Plotly charts. | |
| GPU training runs on Modal (user's credits). Results cached locally. | |
| """ | |
| from __future__ import annotations | |
| import gradio as gr | |
| import pandas as pd | |
| import json | |
| import os | |
| from pathlib import Path | |
| from charts import line_plot, horizontal_bar, color_for, THEMES, SCHEDULER_COLORS | |
| # ─── Modal Integration ─────────────────────────────────────────────────────── | |
| MODAL_AVAILABLE = False | |
| try: | |
| import modal | |
| if os.environ.get("MODAL_TOKEN_ID") and os.environ.get("MODAL_TOKEN_SECRET"): | |
| MODAL_AVAILABLE = True | |
| except ImportError: | |
| pass | |
| MODAL_APP_NAME = "lr-scheduler-benchmark" | |
| MODAL_FULL_FN = "run_full_benchmark" | |
| # ─── Scheduler registry ────────────────────────────────────────────────────── | |
| ALL_SCHEDULERS = [ | |
| "cosine", "linear", "cosine_with_restarts", "polynomial", | |
| "constant", "constant_with_warmup", "inverse_sqrt", | |
| "wsd", "wsd_long", | |
| "onecycle", "exponential", | |
| "lucky_pick", "rlrs", "greedy_lr", "dlrs", "aecs", | |
| "rex", "exp_increasing", "scaling_law_opt", | |
| "step_decay", "cosine_annealing", "plateau_reduction", | |
| "cyclic_lr", "mnist_example", "sqrt_scheduler", | |
| ] | |
| SCHEDULER_DISPLAY = { | |
| "cosine": "Cosine", | |
| "linear": "Linear", | |
| "cosine_with_restarts": "Cosine w/ Restarts", | |
| "polynomial": "Polynomial (p=2)", | |
| "constant": "Constant", | |
| "constant_with_warmup": "Constant + Warmup", | |
| "inverse_sqrt": "Inverse √", | |
| "wsd": "WSD", | |
| "wsd_long": "WSD-Long", | |
| "onecycle": "OneCycleLR", | |
| "exponential": "Exponential", | |
| "lucky_pick": "Lucky Pick", | |
| "rlrs": "RLRS", | |
| "greedy_lr": "GreedyLR", | |
| "dlrs": "DLRS", | |
| "aecs": "AECS", | |
| "rex": "REX", | |
| "exp_increasing": "Exp Increasing", | |
| "scaling_law_opt": "Scaling Law Optimal", | |
| "step_decay": "StepLR", | |
| "cosine_annealing": "Cosine Annealing", | |
| "plateau_reduction": "Plateau Reduction", | |
| "cyclic_lr": "CyclicLR (Triangular2)", | |
| "mnist_example": "MNIST Example", | |
| "sqrt_scheduler": "Square Root Decay", | |
| } | |
| SCHEDULER_FAMILY = { | |
| "cosine": "decay", "linear": "decay", "polynomial": "decay", "inverse_sqrt": "decay", | |
| "wsd": "decay", "wsd_long": "decay", "rex": "decay", "scaling_law_opt": "decay", | |
| "exponential": "decay", "sqrt_scheduler": "decay", "cosine_annealing": "decay", | |
| "cosine_with_restarts": "cyclical", "cyclic_lr": "cyclical", "onecycle": "cyclical", | |
| "lucky_pick": "adaptive", "rlrs": "adaptive", "greedy_lr": "adaptive", | |
| "dlrs": "adaptive", "aecs": "adaptive", "plateau_reduction": "adaptive", | |
| "step_decay": "step", "mnist_example": "step", | |
| "constant": "constant", "constant_with_warmup": "constant", | |
| "exp_increasing": "exotic", | |
| } | |
| SOURCE_LABEL = { | |
| "cosine": "HF", "linear": "HF", "cosine_with_restarts": "HF", "polynomial": "HF", | |
| "constant": "HF", "constant_with_warmup": "HF", "inverse_sqrt": "HF", | |
| "wsd": "HF", "wsd_long": "HF", | |
| "onecycle": "PyTorch", "exponential": "PyTorch", "step_decay": "PyTorch", | |
| "cosine_annealing": "PyTorch", "plateau_reduction": "PyTorch", "mnist_example": "PyTorch", | |
| "lucky_pick": "GitHub", "aecs": "GitHub", | |
| "rlrs": "This bench", "greedy_lr": "This bench", "dlrs": "This bench", | |
| "rex": "Paper", "scaling_law_opt": "Paper", "cyclic_lr": "Paper", | |
| "sqrt_scheduler": "Paper", "exp_increasing": "—", | |
| } | |
| # ─── Data Loading ──────────────────────────────────────────────────────────── | |
| DATA_DIR = Path("data") | |
| DATA_DIR.mkdir(exist_ok=True) | |
| def load_leaderboard(): | |
| p = DATA_DIR / "leaderboard.csv" | |
| if p.exists(): | |
| return pd.read_csv(p) | |
| return _empty_leaderboard() | |
| def load_curves(): | |
| p = DATA_DIR / "curves.csv" | |
| if p.exists(): | |
| return pd.read_csv(p) | |
| return pd.DataFrame(columns=["scheduler", "seed", "step", "train_loss", "val_loss", "val_acc", "lr"]) | |
| def load_per_run(): | |
| p = DATA_DIR / "per_run.csv" | |
| if p.exists(): | |
| return pd.read_csv(p) | |
| return pd.DataFrame() | |
| def load_config(): | |
| p = DATA_DIR / "config.json" | |
| if p.exists(): | |
| with open(p) as f: | |
| return json.load(f) | |
| return _default_config() | |
| def _default_config(): | |
| return { | |
| "model": "distilbert/distilbert-base-uncased", "dataset": "nyu-mll/glue/sst2", | |
| "batch_size": 32, "num_epochs": 3, "learning_rate": 2e-5, | |
| "weight_decay": 0.01, "warmup_fraction": 0.06, "eval_every_steps": 50, | |
| "target_loss": 0.35, "num_seeds": 3, | |
| } | |
| def _empty_leaderboard(): | |
| n = len(ALL_SCHEDULERS) | |
| return pd.DataFrame({ | |
| "rank": list(range(1, n + 1)), | |
| "scheduler": ALL_SCHEDULERS, | |
| "final_val_loss_mean": [None]*n, "final_val_loss_std": [None]*n, | |
| "final_val_acc_mean": [None]*n, "final_val_acc_std": [None]*n, | |
| "auc_val_loss_mean": [None]*n, "steps_to_target_mean": [None]*n, | |
| "steps_to_target_reached": [0]*n, "num_seeds": [0]*n, | |
| }) | |
| # ─── Save / merge results ──────────────────────────────────────────────────── | |
| def save_results(result_data): | |
| new_keys = {r["scheduler"] for r in result_data["leaderboard"]} | |
| def _merge(new_rows, fname): | |
| new_df = pd.DataFrame(new_rows) | |
| path = DATA_DIR / fname | |
| if path.exists(): | |
| old = pd.read_csv(path) | |
| old = old[~old["scheduler"].isin(new_keys)] | |
| merged = pd.concat([old, new_df], ignore_index=True) | |
| else: | |
| merged = new_df | |
| return merged, path | |
| lb, lb_path = _merge(result_data["leaderboard"], "leaderboard.csv") | |
| lb = lb.sort_values("final_val_loss_mean", na_position="last").reset_index(drop=True) | |
| lb["rank"] = range(1, len(lb) + 1) | |
| lb.to_csv(lb_path, index=False) | |
| pr, pr_path = _merge(result_data["per_run"], "per_run.csv") | |
| pr.to_csv(pr_path, index=False) | |
| cv, cv_path = _merge(result_data["curves"], "curves.csv") | |
| cv.to_csv(cv_path, index=False) | |
| with open(DATA_DIR / "config.json", "w") as f: | |
| json.dump(result_data.get("config", _default_config()), f, indent=2) | |
| # Persist to HF Space repo so data survives rebuilds | |
| try: | |
| from huggingface_hub import HfApi | |
| token = os.environ.get("HF_TOKEN") or os.environ.get("HUGGING_FACE_HUB_TOKEN") | |
| api = HfApi(token=token) | |
| space_id = "juiceb0xc0de/lr-scheduler-benchmark" | |
| for fname in ["leaderboard.csv", "curves.csv", "per_run.csv", "config.json"]: | |
| fp = DATA_DIR / fname | |
| if fp.exists(): | |
| api.upload_file(path_or_fileobj=str(fp), path_in_repo=f"data/{fname}", | |
| repo_id=space_id, repo_type="space") | |
| except Exception as e: | |
| print(f"Warning: couldn't upload to Space: {e}") | |
| # ─── Live state ────────────────────────────────────────────────────────────── | |
| leaderboard_raw = load_leaderboard() | |
| curves_raw = load_curves() | |
| config = load_config() | |
| def _has_data(): | |
| return leaderboard_raw["final_val_loss_mean"].notna().any() | |
| # ─── HTML chrome builders ─────────────────────────────────────────────────── | |
| def header_html(): | |
| has = _has_data() | |
| n_seeds = config.get("num_seeds", 3) | |
| sub = f"v0.3 · {len(ALL_SCHEDULERS)} schedulers · SST-2/DistilBERT · {n_seeds} seeds" | |
| return f""" | |
| <div class="tm-header"> | |
| <div class="tm-header__brand"> | |
| <svg width="28" height="28" viewBox="0 0 28 28" fill="none"> | |
| <rect x="1" y="1" width="26" height="26" rx="6" stroke="currentColor" stroke-width="1.5"/> | |
| <path d="M5 20 Q 9 18, 11 14 T 17 8 T 23 6" stroke="currentColor" stroke-width="1.6" fill="none"/> | |
| <circle cx="23" cy="6" r="2" fill="currentColor"/> | |
| </svg> | |
| <div> | |
| <div class="tm-header__title">LR Scheduler Bench</div> | |
| <div class="tm-header__sub">{sub}</div> | |
| </div> | |
| </div> | |
| <div class="tm-header__meta"> | |
| {'last run · live data' if has else 'awaiting first benchmark run'} · | |
| <a href="https://huggingface.co/spaces/juiceb0xc0de/lr-scheduler-benchmark/discussions" target="_blank">discussions ↗</a> | |
| </div> | |
| </div> | |
| """ | |
| def kpi_strip_html(): | |
| if not _has_data(): | |
| return """ | |
| <div class="tm-kpis"> | |
| <div class="tm-kpi"><div class="tm-kpi__label">Schedulers</div> | |
| <div class="tm-kpi__value">""" + str(len(ALL_SCHEDULERS)) + """</div> | |
| <div class="tm-kpi__sub">awaiting results</div></div> | |
| <div class="tm-kpi"><div class="tm-kpi__label">Best val loss</div> | |
| <div class="tm-kpi__value">—</div><div class="tm-kpi__sub">run benchmark first</div></div> | |
| <div class="tm-kpi"><div class="tm-kpi__label">Best val accuracy</div> | |
| <div class="tm-kpi__value">—</div><div class="tm-kpi__sub">run benchmark first</div></div> | |
| <div class="tm-kpi"><div class="tm-kpi__label">Total runs</div> | |
| <div class="tm-kpi__value">0</div><div class="tm-kpi__sub">scheduler × seed</div></div> | |
| </div> | |
| """ | |
| df = leaderboard_raw[leaderboard_raw["final_val_loss_mean"].notna()].copy() | |
| df = df.sort_values("final_val_loss_mean") | |
| best = df.iloc[0] | |
| best_name = SCHEDULER_DISPLAY.get(best["scheduler"], best["scheduler"]) | |
| best_acc = df["final_val_acc_mean"].max() | |
| best_acc_row = df.loc[df["final_val_acc_mean"].idxmax()] | |
| best_acc_name = SCHEDULER_DISPLAY.get(best_acc_row["scheduler"], best_acc_row["scheduler"]) | |
| total_runs = int(df["num_seeds"].sum()) if "num_seeds" in df.columns else 0 | |
| return f""" | |
| <div class="tm-kpis"> | |
| <div class="tm-kpi"> | |
| <div class="tm-kpi__label">Schedulers benchmarked</div> | |
| <div class="tm-kpi__value">{len(df)}</div> | |
| <div class="tm-kpi__sub">of {len(ALL_SCHEDULERS)} total</div> | |
| </div> | |
| <div class="tm-kpi"> | |
| <div class="tm-kpi__label">Best val loss</div> | |
| <div class="tm-kpi__value tm-kpi__value--success">{best['final_val_loss_mean']:.4f}</div> | |
| <div class="tm-kpi__sub">{best_name}</div> | |
| </div> | |
| <div class="tm-kpi"> | |
| <div class="tm-kpi__label">Best val accuracy</div> | |
| <div class="tm-kpi__value">{best_acc*100:.2f}%</div> | |
| <div class="tm-kpi__sub">{best_acc_name}</div> | |
| </div> | |
| <div class="tm-kpi"> | |
| <div class="tm-kpi__label">Total GPU runs</div> | |
| <div class="tm-kpi__value tm-kpi__value--accent">{total_runs}</div> | |
| <div class="tm-kpi__sub">scheduler × seed</div> | |
| </div> | |
| </div> | |
| """ | |
| # ─── Callbacks ────────────────────────────────────────────────────────────── | |
| DISPLAY_TO_KEY = {v: k for k, v in SCHEDULER_DISPLAY.items()} | |
| ALL_DISPLAY = [SCHEDULER_DISPLAY[k] for k in ALL_SCHEDULERS] | |
| def display_to_keys(display_names): | |
| return [DISPLAY_TO_KEY.get(n, n) for n in (display_names or [])] | |
| def on_compare_change(selected_display): | |
| keys = display_to_keys(selected_display) | |
| val_fig = line_plot(curves_raw, keys, SCHEDULER_DISPLAY, "val_loss", "val loss", theme="dark", height=360) | |
| lr_fig = line_plot(curves_raw, keys, SCHEDULER_DISPLAY, "lr", "lr", theme="dark", height=360) | |
| train_fig = line_plot(curves_raw, keys, SCHEDULER_DISPLAY, "train_loss","train loss", theme="dark", height=300) | |
| return val_fig, lr_fig, train_fig | |
| def on_bar_metric_change(metric_label, selected_display): | |
| metric_map = { | |
| "Val loss": "final_val_loss_mean", | |
| "Val acc": "final_val_acc_mean", | |
| "AUC": "auc_val_loss_mean", | |
| "Steps": "steps_to_target_mean", | |
| } | |
| metric = metric_map.get(metric_label, "final_val_loss_mean") | |
| keys = set(display_to_keys(selected_display)) | |
| return horizontal_bar(leaderboard_raw, metric, SCHEDULER_DISPLAY, selected_keys=keys, theme="dark") | |
| # ─── Leaderboard table ────────────────────────────────────────────────────── | |
| SORT_OPTIONS = { | |
| "Rank": ("rank", True), | |
| "Val loss ↓": ("final_val_loss_mean", True), | |
| "Val accuracy ↑": ("final_val_acc_mean", False), | |
| "AUC ↓": ("auc_val_loss_mean", True), | |
| "Steps to target ↓": ("steps_to_target_mean", True), | |
| } | |
| def format_leaderboard(df): | |
| out = pd.DataFrame() | |
| out["#"] = df["rank"].apply(lambda v: f"{int(v):02d}") | |
| out["Scheduler"] = df["scheduler"].map(lambda s: SCHEDULER_DISPLAY.get(s, s)) | |
| out["Family"] = df["scheduler"].map(lambda s: SCHEDULER_FAMILY.get(s, "—").title()) | |
| out["Source"] = df["scheduler"].map(lambda s: SOURCE_LABEL.get(s, "—")) | |
| if df["final_val_loss_mean"].notna().any(): | |
| out["Val loss ↓"] = df.apply( | |
| lambda r: f"{r['final_val_loss_mean']:.4f} ±{r['final_val_loss_std']:.3f}" | |
| if pd.notna(r["final_val_loss_mean"]) else "—", axis=1) | |
| out["Val acc ↑"] = df.apply( | |
| lambda r: f"{r['final_val_acc_mean']*100:.2f}% ±{r['final_val_acc_std']*100:.2f}" | |
| if pd.notna(r["final_val_acc_mean"]) else "—", axis=1) | |
| out["AUC ↓"] = df.apply( | |
| lambda r: f"{r['auc_val_loss_mean']:.1f}" if pd.notna(r.get("auc_val_loss_mean")) else "—", axis=1) | |
| out["Steps→target ↓"] = df.apply( | |
| lambda r: f"{int(r['steps_to_target_mean'])}" | |
| if pd.notna(r.get("steps_to_target_mean")) else "—", axis=1) | |
| out["Seeds"] = df["num_seeds"].astype(int) | |
| else: | |
| for c in ["Val loss ↓", "Val acc ↑", "AUC ↓", "Steps→target ↓"]: | |
| out[c] = "—" | |
| out["Seeds"] = 0 | |
| return out | |
| def sort_leaderboard(sort_label): | |
| col, asc = SORT_OPTIONS.get(sort_label, ("rank", True)) | |
| df = leaderboard_raw.sort_values(col, ascending=asc, na_position="last").copy() | |
| df["rank"] = range(1, len(df) + 1) | |
| return format_leaderboard(df) | |
| # ─── Modal callbacks ──────────────────────────────────────────────────────── | |
| def launch_benchmark(selected_display, num_seeds): | |
| if not MODAL_AVAILABLE: | |
| return None, ("> ⚠ **Modal not configured.** Add `MODAL_TOKEN_ID` and `MODAL_TOKEN_SECRET` " | |
| "to this Space's secrets, then deploy with `modal deploy modal_benchmark.py`.") | |
| if not selected_display: | |
| return None, "> ⚠ Select at least one scheduler." | |
| try: | |
| keys = display_to_keys(selected_display) | |
| fn = modal.Function.from_name(MODAL_APP_NAME, MODAL_FULL_FN) | |
| call = fn.spawn(schedulers=keys, num_seeds=int(num_seeds)) | |
| return (call.object_id, | |
| f"> 🚀 **Launched** — {len(keys)} schedulers × {int(num_seeds)} seeds = " | |
| f"{len(keys)*int(num_seeds)} L40S containers.\n>\n" | |
| f"> Job id: `{call.object_id}` · expected ~15–30 min · click **Check status** to poll.") | |
| except Exception as e: | |
| return None, f"> ❌ Failed to launch: `{e}`" | |
| def check_benchmark_status(call_id): | |
| if not call_id: | |
| return "> No job running. Launch a benchmark first." | |
| if not MODAL_AVAILABLE: | |
| return "> ⚠ Modal not configured." | |
| try: | |
| fc = modal.FunctionCall.from_id(call_id) | |
| result = fc.get(timeout=0) | |
| if "error" in result: | |
| return f"> ❌ Failed: `{result['error']}`" | |
| save_results(result) | |
| global leaderboard_raw, curves_raw, config | |
| leaderboard_raw = load_leaderboard() | |
| curves_raw = load_curves() | |
| config = load_config() | |
| n_runs = len(result["per_run"]) | |
| best = result["leaderboard"][0] | |
| best_name = SCHEDULER_DISPLAY.get(best["scheduler"], best["scheduler"]) | |
| return (f"> ✅ **Complete** — {len(result['leaderboard'])} schedulers, {n_runs} runs.\n>\n" | |
| f"> 🏆 Best: **{best_name}** — val loss {best['final_val_loss_mean']:.4f}, " | |
| f"acc {best['final_val_acc_mean']*100:.2f}%.\n>\n" | |
| f"> Reload the page to see updated charts.") | |
| except TimeoutError: | |
| return "> ⏳ Still running. Check again in a minute." | |
| except Exception as e: | |
| return f"> ⚠ Error: `{e}`" | |
| # ─── UI ───────────────────────────────────────────────────────────────────── | |
| CSS_PATH = Path(__file__).parent / "static" / "style.css" | |
| CSS_TEXT = CSS_PATH.read_text() if CSS_PATH.exists() else "" | |
| DEFAULT_SELECTED = ALL_DISPLAY[:5] # Top 5 by spec | |
| with gr.Blocks(css=CSS_TEXT, theme=gr.themes.Base(), title="LR Scheduler Bench") as demo: | |
| gr.HTML(header_html()) | |
| kpi_html = gr.HTML(kpi_strip_html()) | |
| with gr.Tabs(): | |
| # ════════════════════════════════════════════════════════════════════ | |
| # COMPARE | |
| # ════════════════════════════════════════════════════════════════════ | |
| with gr.TabItem("Compare"): | |
| with gr.Row(equal_height=False): | |
| with gr.Column(scale=1, min_width=260, elem_classes=["tm-sidebar"]): | |
| gr.HTML('<div class="tm-sidebar__title">Schedulers</div>') | |
| with gr.Row(): | |
| all_btn = gr.Button("All", size="sm") | |
| none_btn = gr.Button("None", size="sm") | |
| top5_btn = gr.Button("Top 5", size="sm") | |
| sched_select = gr.CheckboxGroup( | |
| choices=ALL_DISPLAY, | |
| value=DEFAULT_SELECTED, | |
| label="", | |
| show_label=False, | |
| container=False, | |
| ) | |
| with gr.Column(scale=4): | |
| with gr.Row(): | |
| with gr.Column(): | |
| gr.Markdown('<div class="tm-card__title">Validation loss</div>' | |
| '<div class="tm-card__sub">Mean over seeds, per training step.</div>') | |
| val_plot = gr.Plot( | |
| value=line_plot(curves_raw, display_to_keys(DEFAULT_SELECTED), | |
| SCHEDULER_DISPLAY, "val_loss", "val loss", | |
| theme="dark", height=360), | |
| show_label=False, container=False, | |
| ) | |
| with gr.Column(): | |
| gr.Markdown('<div class="tm-card__title">Learning rate schedule</div>' | |
| '<div class="tm-card__sub">Per-step LR (single seed).</div>') | |
| lr_plot = gr.Plot( | |
| value=line_plot(curves_raw, display_to_keys(DEFAULT_SELECTED), | |
| SCHEDULER_DISPLAY, "lr", "lr", | |
| theme="dark", height=360), | |
| show_label=False, container=False, | |
| ) | |
| gr.Markdown('<div class="tm-card__title">Train loss</div>' | |
| '<div class="tm-card__sub">Mean over seeds.</div>') | |
| train_plot = gr.Plot( | |
| value=line_plot(curves_raw, display_to_keys(DEFAULT_SELECTED), | |
| SCHEDULER_DISPLAY, "train_loss", "train loss", | |
| theme="dark", height=300), | |
| show_label=False, container=False, | |
| ) | |
| with gr.Row(): | |
| gr.Markdown('<div class="tm-card__title">Final ranking</div>' | |
| '<div class="tm-card__sub">Horizontal bars — labels never overlap.</div>') | |
| bar_metric = gr.Radio( | |
| choices=["Val loss", "Val acc", "AUC", "Steps"], | |
| value="Val loss", | |
| label="", show_label=False, container=False, | |
| ) | |
| bar_plot = gr.Plot( | |
| value=horizontal_bar(leaderboard_raw, "final_val_loss_mean", | |
| SCHEDULER_DISPLAY, | |
| selected_keys=set(display_to_keys(DEFAULT_SELECTED)), | |
| theme="dark"), | |
| show_label=False, container=False, | |
| ) | |
| # Wire callbacks | |
| sched_select.change(on_compare_change, inputs=sched_select, | |
| outputs=[val_plot, lr_plot, train_plot]) | |
| sched_select.change(on_bar_metric_change, inputs=[bar_metric, sched_select], | |
| outputs=bar_plot) | |
| bar_metric.change(on_bar_metric_change, inputs=[bar_metric, sched_select], | |
| outputs=bar_plot) | |
| all_btn.click( lambda: ALL_DISPLAY, outputs=sched_select) | |
| none_btn.click(lambda: [], outputs=sched_select) | |
| top5_btn.click(lambda: ALL_DISPLAY[:5], outputs=sched_select) | |
| # ════════════════════════════════════════════════════════════════════ | |
| # LEADERBOARD | |
| # ════════════════════════════════════════════════════════════════════ | |
| with gr.TabItem("Leaderboard"): | |
| gr.Markdown( | |
| '<div class="tm-card__title">Full leaderboard</div>' | |
| '<div class="tm-card__sub">Sortable. Best in column highlighted in mono numerals.</div>' | |
| ) | |
| sort_dd = gr.Dropdown( | |
| choices=list(SORT_OPTIONS.keys()), value="Rank", | |
| label="Sort by", container=False, scale=0, | |
| ) | |
| lb_table = gr.DataFrame( | |
| value=format_leaderboard(leaderboard_raw), | |
| interactive=False, max_height=620, wrap=False, | |
| ) | |
| sort_dd.change(sort_leaderboard, sort_dd, lb_table) | |
| # ════════════════════════════════════════════════════════════════════ | |
| # RUN BENCHMARK | |
| # ════════════════════════════════════════════════════════════════════ | |
| with gr.TabItem("Run Benchmark"): | |
| gr.Markdown(f""" | |
| ### Run on Modal GPU | |
| Each (scheduler, seed) pair gets its own L40S container. Estimated cost ~$0.50–1.00 for | |
| all schedulers × 3 seeds. Estimated time 15–30 min in parallel. | |
| """) | |
| if not MODAL_AVAILABLE: | |
| gr.HTML('<div class="tm-note"><strong>Modal not configured.</strong> ' | |
| 'Add <code>MODAL_TOKEN_ID</code> and <code>MODAL_TOKEN_SECRET</code> ' | |
| 'to Space secrets, then <code>modal deploy modal_benchmark.py</code>.</div>') | |
| job_state = gr.State(value=None) | |
| run_select = gr.CheckboxGroup( | |
| choices=ALL_DISPLAY, value=ALL_DISPLAY, | |
| label="Schedulers to benchmark", | |
| ) | |
| num_seeds = gr.Slider(1, 5, value=3, step=1, label="Seeds per scheduler") | |
| with gr.Row(): | |
| launch_btn = gr.Button("🚀 Launch benchmark", variant="primary", | |
| interactive=MODAL_AVAILABLE) | |
| check_btn = gr.Button("Check status") | |
| status = gr.Markdown("> Ready to launch.") | |
| launch_btn.click(launch_benchmark, [run_select, num_seeds], [job_state, status]) | |
| check_btn.click( check_benchmark_status, [job_state], [status]) | |
| # ════════════════════════════════════════════════════════════════════ | |
| # METHODOLOGY | |
| # ════════════════════════════════════════════════════════════════════ | |
| with gr.TabItem("Methodology"): | |
| gr.Markdown(f""" | |
| ### Benchmark design | |
| Twenty-five schedulers under **identical conditions** — only the scheduler changes between runs. | |
| | Parameter | Value | | |
| |---|---| | |
| | Model | `{config['model']}` (67M) | | |
| | Task | SST-2 sentiment classification ([GLUE](https://huggingface.co/datasets/nyu-mll/glue)) | | |
| | Optimizer | AdamW (β=0.9/0.999, ε=1e-8) | | |
| | Peak LR | `{config['learning_rate']}` | | |
| | Weight decay | `{config['weight_decay']}` | | |
| | Batch size | {config['batch_size']} | | |
| | Epochs | {config['num_epochs']} | | |
| | Warmup | {int(config['warmup_fraction']*100)}% of steps | | |
| | Grad clip | max-norm 1.0 | | |
| | Seeds | {config['num_seeds']} | | |
| ### Metrics | |
| | Metric | Description | Better | | |
| |---|---|---| | |
| | Final val loss | Cross-entropy on SST-2 val | ↓ | | |
| | Final val accuracy | Classification accuracy | ↑ | | |
| | AUC val loss | Area under val-loss curve (convergence speed) | ↓ | | |
| | Steps→target | First step where val loss ≤ {config['target_loss']} | ↓ | | |
| ### Fairness notes | |
| - All schedulers share the same peak LR (`{config['learning_rate']}`); this measures **out-of-the-box** | |
| behavior at a fixed LR. Some schedulers benefit from per-scheduler tuning. | |
| - OneCycleLR uses 10× peak as `max_lr` (standard super-convergence recipe). | |
| - AECS uses a cosine backbone and modulates on top based on detected events (gradient spikes, | |
| loss spikes, redundancy, instability, plateau). | |
| - Results averaged across {config['num_seeds']} seeds with different shuffle orders. | |
| ### Want a scheduler added? | |
| Open a thread in [Discussions](https://huggingface.co/spaces/juiceb0xc0de/lr-scheduler-benchmark/discussions) | |
| with a link to the implementation (paper, repo, or PyTorch class) and a one-line description of | |
| the schedule shape. Custom schedulers should expose a `get_lr(step)` method. | |
| ### References | |
| - Cosine Annealing — Loshchilov & Hutter, 2016 ([arXiv](https://arxiv.org/abs/1608.03983)) | |
| - WSD — MiniCPM, 2024 ([arXiv](https://arxiv.org/abs/2404.06395)) | |
| - OneCycleLR — Smith & Topin, 2019 ([arXiv](https://arxiv.org/abs/1708.07120)) | |
| - Inverse √ — Vaswani et al., 2017 ([arXiv](https://arxiv.org/abs/1706.03762)) | |
| """) | |
| with gr.Accordion("Raw per-run results", open=False): | |
| pr = load_per_run() | |
| if not pr.empty: | |
| gr.DataFrame(value=pr, interactive=False, max_height=400) | |
| else: | |
| gr.Markdown("*Per-run data appears here after benchmark runs complete.*") | |
| with gr.Accordion("Citation (BibTeX)", open=False): | |
| gr.Code( | |
| value="""@misc{lr-scheduler-benchmark-2026, | |
| title={LR Scheduler Benchmark: Standardized Comparison of Learning Rate Schedules}, | |
| author={JuiceB0xC0de}, | |
| year={2026}, | |
| url={https://huggingface.co/spaces/juiceb0xc0de/lr-scheduler-benchmark} | |
| }""", | |
| language="markdown", interactive=False, | |
| ) | |
| if __name__ == "__main__": | |
| demo.launch(server_name="0.0.0.0") | |