Add static model performance leaderboard for world-in-world

app.py

@@ -1,204 +1,94 @@
 import gradio as gr
 from gradio_leaderboard import Leaderboard, ColumnFilter, SelectColumns
 import pandas as pd
-from apscheduler.schedulers.background import BackgroundScheduler
-from huggingface_hub import snapshot_download
 
-from src.about import (
-    CITATION_BUTTON_LABEL,
-    CITATION_BUTTON_TEXT,
-    EVALUATION_QUEUE_TEXT,
-    INTRODUCTION_TEXT,
-    LLM_BENCHMARKS_TEXT,
-    TITLE,
-)
-from src.display.css_html_js import custom_css
-from src.display.utils import (
-    BENCHMARK_COLS,
-    COLS,
-    EVAL_COLS,
-    EVAL_TYPES,
-    AutoEvalColumn,
-    ModelType,
-    fields,
-    WeightType,
-    Precision
-)
-from src.envs import API, EVAL_REQUESTS_PATH, EVAL_RESULTS_PATH, QUEUE_REPO, REPO_ID, RESULTS_REPO, TOKEN
-from src.populate import get_evaluation_queue_df, get_leaderboard_df
-from src.submission.submit import add_new_eval
-
-
-def restart_space():
-    API.restart_space(repo_id=REPO_ID)
-
-### Space initialisation
-try:
-    print(EVAL_REQUESTS_PATH)
-    snapshot_download(
-        repo_id=QUEUE_REPO, local_dir=EVAL_REQUESTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30, token=TOKEN
-    )
-except Exception:
-    restart_space()
-try:
-    print(EVAL_RESULTS_PATH)
-    snapshot_download(
-        repo_id=RESULTS_REPO, local_dir=EVAL_RESULTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30, token=TOKEN
-    )
-except Exception:
-    restart_space()
-
-
-LEADERBOARD_DF = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, COLS, BENCHMARK_COLS)
-
-(
-    finished_eval_queue_df,
-    running_eval_queue_df,
-    pending_eval_queue_df,
-) = get_evaluation_queue_df(EVAL_REQUESTS_PATH, EVAL_COLS)
+# Static data
+STATIC_DATA = [
+    ["VLM", "w/o WM", "–", "RGB", "72B", 50.27, 6.24],
+    ["Image Gen.", "PathDreamer [36]", "Viewpoint", "RGB-D; Sem; Pano", "0.69B", 56.99, 5.28],
+    ["Image Gen.", "SE3DS [11]", "Viewpoint", "RGB-D; Pano", "1.1B", 57.53, 5.29],
+    ["Video Gen.", "NWM [25]", "Trajectory", "RGB", "1B", 57.35, 5.68],
+    ["Video Gen.", "SVD [6]", "Image", "RGB", "1.5B", 57.71, 5.29],
+    ["Video Gen.", "LTX-Video [5]", "Text", "RGB", "2B", 56.08, 5.37],
+    ["Video Gen.", "Hunyuan [4]", "Text", "RGB", "13B", 57.71, 5.21],
+    ["Video Gen.", "Wan2.1 [23]", "Text", "RGB", "14B", 58.26, 5.24],
+    ["Video Gen.", "Cosmos [1]", "Text", "RGB", "2B", 52.27, 5.898],
+    ["Video Gen.", "Runway", "Text", "–", "–", "–", "–"],
+    ["Video Gen. Post-Train", "SVD† [6]", "Action", "RGB; Pano", "1.5B", 60.98, 5.02],
+    ["Video Gen. Post-Train", "LTX† [5]", "Action", "RGB; Pano", "2B", 57.53, 5.49],
+    ["Video Gen. Post-Train", "WAN2.1† [23]", "Action", "RGB; Pano", "14B", "XXX", "XXX"],
+    ["Video Gen. Post-Train", "Cosmos† [1]", "Action", "RGB; Pano", "2B", 60.25, 5.08],
+]
+
+COLUMNS = ["Model Type", "Method", "Control Type", "Input Type", "#Param.", "Acc. ↑", "Mean Traj. ↓"]
+LEADERBOARD_DF = pd.DataFrame(STATIC_DATA, columns=COLUMNS)
+
+# Custom CSS (simplified)
+custom_css = """
+/* Add any custom styling here */
+.gradio-container {
+    max-width: 1200px !important;
+}
+"""
 
 def init_leaderboard(dataframe):
     if dataframe is None or dataframe.empty:
         raise ValueError("Leaderboard DataFrame is empty or None.")
     return Leaderboard(
         value=dataframe,
-        datatype=[c.type for c in fields(AutoEvalColumn)],
+        datatype=["str", "str", "str", "str", "str", "number", "number"],
         select_columns=SelectColumns(
-            default_selection=[c.name for c in fields(AutoEvalColumn) if c.displayed_by_default],
-            cant_deselect=[c.name for c in fields(AutoEvalColumn) if c.never_hidden],
+            default_selection=COLUMNS,
+            cant_deselect=["Model Type", "Method", "Acc. ↑"],
             label="Select Columns to Display:",
         ),
-        search_columns=[AutoEvalColumn.model.name, AutoEvalColumn.license.name],
-        hide_columns=[c.name for c in fields(AutoEvalColumn) if c.hidden],
+        search_columns=["Model Type", "Method"],
+        hide_columns=[],
         filter_columns=[
-            ColumnFilter(AutoEvalColumn.model_type.name, type="checkboxgroup", label="Model types"),
-            ColumnFilter(AutoEvalColumn.precision.name, type="checkboxgroup", label="Precision"),
-            ColumnFilter(
-                AutoEvalColumn.params.name,
-                type="slider",
-                min=0.01,
-                max=150,
-                label="Select the number of parameters (B)",
-            ),
-            ColumnFilter(
-                AutoEvalColumn.still_on_hub.name, type="boolean", label="Deleted/incomplete", default=True
-            ),
+            ColumnFilter("Model Type", type="checkboxgroup", label="Model types"),
+            ColumnFilter("Control Type", type="checkboxgroup", label="Control types"),
+            ColumnFilter("Input Type", type="checkboxgroup", label="Input types"),
         ],
         bool_checkboxgroup_label="Hide models",
         interactive=False,
     )
 
-
-demo = gr.Blocks(css=custom_css)
+demo = gr.Blocks(css=custom_css, title="Model Performance Leaderboard")
 with demo:
-    gr.HTML(TITLE)
-    gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")
+    gr.HTML("<h1 style='text-align: center'>🏆 Model Performance Leaderboard</h1>")
+    gr.Markdown("""
+    **Performance comparison across vision-language models, image generation, and video generation models.**
+    
+    📊 **Metrics:** Acc. ↑ (Accuracy - higher is better) | Mean Traj. ↓ (Mean Trajectory error - lower is better)
+    """, elem_classes="markdown-text")
 
     with gr.Tabs(elem_classes="tab-buttons") as tabs:
-        with gr.TabItem("🏅 LLM Benchmark", elem_id="llm-benchmark-tab-table", id=0):
+        with gr.TabItem("🏅 Leaderboard", elem_id="leaderboard-tab", id=0):
             leaderboard = init_leaderboard(LEADERBOARD_DF)
 
-        with gr.TabItem("📝 About", elem_id="llm-benchmark-tab-table", id=2):
-            gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text")
-
-        with gr.TabItem("🚀 Submit here! ", elem_id="llm-benchmark-tab-table", id=3):
-            with gr.Column():
-                with gr.Row():
-                    gr.Markdown(EVALUATION_QUEUE_TEXT, elem_classes="markdown-text")
-
-                with gr.Column():
-                    with gr.Accordion(
-                        f"✅ Finished Evaluations ({len(finished_eval_queue_df)})",
-                        open=False,
-                    ):
-                        with gr.Row():
-                            finished_eval_table = gr.components.Dataframe(
-                                value=finished_eval_queue_df,
-                                headers=EVAL_COLS,
-                                datatype=EVAL_TYPES,
-                                row_count=5,
-                            )
-                    with gr.Accordion(
-                        f"🔄 Running Evaluation Queue ({len(running_eval_queue_df)})",
-                        open=False,
-                    ):
-                        with gr.Row():
-                            running_eval_table = gr.components.Dataframe(
-                                value=running_eval_queue_df,
-                                headers=EVAL_COLS,
-                                datatype=EVAL_TYPES,
-                                row_count=5,
-                            )
-
-                    with gr.Accordion(
-                        f"⏳ Pending Evaluation Queue ({len(pending_eval_queue_df)})",
-                        open=False,
-                    ):
-                        with gr.Row():
-                            pending_eval_table = gr.components.Dataframe(
-                                value=pending_eval_queue_df,
-                                headers=EVAL_COLS,
-                                datatype=EVAL_TYPES,
-                                row_count=5,
-                            )
-            with gr.Row():
-                gr.Markdown("# ✉️✨ Submit your model here!", elem_classes="markdown-text")
-
-            with gr.Row():
-                with gr.Column():
-                    model_name_textbox = gr.Textbox(label="Model name")
-                    revision_name_textbox = gr.Textbox(label="Revision commit", placeholder="main")
-                    model_type = gr.Dropdown(
-                        choices=[t.to_str(" : ") for t in ModelType if t != ModelType.Unknown],
-                        label="Model type",
-                        multiselect=False,
-                        value=None,
-                        interactive=True,
-                    )
-
-                with gr.Column():
-                    precision = gr.Dropdown(
-                        choices=[i.value.name for i in Precision if i != Precision.Unknown],
-                        label="Precision",
-                        multiselect=False,
-                        value="float16",
-                        interactive=True,
-                    )
-                    weight_type = gr.Dropdown(
-                        choices=[i.value.name for i in WeightType],
-                        label="Weights type",
-                        multiselect=False,
-                        value="Original",
-                        interactive=True,
-                    )
-                    base_model_name_textbox = gr.Textbox(label="Base model (for delta or adapter weights)")
-
-            submit_button = gr.Button("Submit Eval")
-            submission_result = gr.Markdown()
-            submit_button.click(
-                add_new_eval,
-                [
-                    model_name_textbox,
-                    base_model_name_textbox,
-                    revision_name_textbox,
-                    precision,
-                    weight_type,
-                    model_type,
-                ],
-                submission_result,
-            )
-
-    with gr.Row():
-        with gr.Accordion("📙 Citation", open=False):
-            citation_button = gr.Textbox(
-                value=CITATION_BUTTON_TEXT,
-                label=CITATION_BUTTON_LABEL,
-                lines=20,
-                elem_id="citation-button",
-                show_copy_button=True,
-            )
-
-scheduler = BackgroundScheduler()
-scheduler.add_job(restart_space, "interval", seconds=1800)
-scheduler.start()
-demo.queue(default_concurrency_limit=40).launch()
+        with gr.TabItem("📝 About", elem_id="about-tab", id=1):
+            gr.Markdown("""
+            # About This Leaderboard
+            
+            This leaderboard showcases performance metrics across different types of AI models:
+            
+            ## Model Categories
+            - **VLM**: Vision-Language Models
+            - **Image Gen.**: Image Generation Models  
+            - **Video Gen.**: Video Generation Models
+            - **Video Gen. Post-Train**: Post-training specialized Video Generation Models
+            
+            ## Metrics Explained
+            - **Acc. ↑**: Accuracy score (higher values indicate better performance)
+            - **Mean Traj. ↓**: Mean trajectory error (lower values indicate better performance)
+            
+            ## Notes
+            - † indicates post-training specialized models
+            - XXX indicates results pending/unavailable
+            - – indicates not applicable or not available
+            
+            *Results may vary across different evaluation settings and benchmarks.*
+            """, elem_classes="markdown-text")
+
+if __name__ == "__main__":
+    demo.launch()

src/leaderboard/read_evals.py

@@ -1,3 +1,5 @@
+# src/leaderboard/read_evals.py
+
 import glob
 import json
 import math