MMTU leaderboard init

README.md

@@ -1,5 +1,5 @@
 ---
-title: Example Leaderboard Template
+title: MMTU Leaderboard
 emoji: 🥇
 colorFrom: green
 colorTo: indigo

app.py

@@ -10,6 +10,7 @@ from src.about import (
     EVALUATION_QUEUE_TEXT,
     INTRODUCTION_TEXT,
     LLM_BENCHMARKS_TEXT,
+    SUBMIT_INTRODUCTION,
     TITLE,
 )
 from src.display.css_html_js import custom_css
@@ -24,38 +25,43 @@ from src.display.utils import (
     WeightType,
     Precision
 )
-from src.envs import API, EVAL_REQUESTS_PATH, EVAL_RESULTS_PATH, QUEUE_REPO, REPO_ID, RESULTS_REPO, TOKEN
+from src.envs import API, REPO_ID, TOKEN
 from src.populate import get_evaluation_queue_df, get_leaderboard_df
-from src.submission.submit import add_new_eval
+# from src.submission.submit import add_new_eval
 
 
 def restart_space():
     API.restart_space(repo_id=REPO_ID)
 
-### Space initialisation
-try:
-    print(EVAL_REQUESTS_PATH)
-    snapshot_download(
-        repo_id=QUEUE_REPO, local_dir=EVAL_REQUESTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30, token=TOKEN
-    )
-except Exception:
-    restart_space()
-try:
-    print(EVAL_RESULTS_PATH)
-    snapshot_download(
-        repo_id=RESULTS_REPO, local_dir=EVAL_RESULTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30, token=TOKEN
-    )
-except Exception:
-    restart_space()
-
-
-LEADERBOARD_DF = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, COLS, BENCHMARK_COLS)
-
-(
-    finished_eval_queue_df,
-    running_eval_queue_df,
-    pending_eval_queue_df,
-) = get_evaluation_queue_df(EVAL_REQUESTS_PATH, EVAL_COLS)
+# ### Space initialisation
+# try:
+#     print(EVAL_REQUESTS_PATH)
+#     snapshot_download(
+#         repo_id=QUEUE_REPO, local_dir=EVAL_REQUESTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30, token=TOKEN
+#     )
+# except Exception:
+#     restart_space()
+# try:
+#     print(EVAL_RESULTS_PATH)
+#     snapshot_download(
+#         repo_id=RESULTS_REPO, local_dir=EVAL_RESULTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30, token=TOKEN
+#     )
+# except Exception:
+#     restart_space()
+
+def get_df():
+    df = pd.read_json("results.json")
+    df.sort_values(by="Overall", ascending=False, inplace=True)
+    return df
+
+
+LEADERBOARD_DF = get_df()
+
+# (
+#     finished_eval_queue_df,
+#     running_eval_queue_df,
+#     pending_eval_queue_df,
+# ) = get_evaluation_queue_df(EVAL_REQUESTS_PATH, EVAL_COLS)
 
 def init_leaderboard(dataframe):
     if dataframe is None or dataframe.empty:
@@ -68,22 +74,22 @@ def init_leaderboard(dataframe):
             cant_deselect=[c.name for c in fields(AutoEvalColumn) if c.never_hidden],
             label="Select Columns to Display:",
         ),
-        search_columns=[AutoEvalColumn.model.name, AutoEvalColumn.license.name],
-        hide_columns=[c.name for c in fields(AutoEvalColumn) if c.hidden],
-        filter_columns=[
-            ColumnFilter(AutoEvalColumn.model_type.name, type="checkboxgroup", label="Model types"),
-            ColumnFilter(AutoEvalColumn.precision.name, type="checkboxgroup", label="Precision"),
-            ColumnFilter(
-                AutoEvalColumn.params.name,
-                type="slider",
-                min=0.01,
-                max=150,
-                label="Select the number of parameters (B)",
-            ),
-            ColumnFilter(
-                AutoEvalColumn.still_on_hub.name, type="boolean", label="Deleted/incomplete", default=True
-            ),
-        ],
+        search_columns=[AutoEvalColumn.model.name],
+        # hide_columns=[c.name for c in fields(AutoEvalColumn) if c.hidden],
+        # filter_columns=[
+        #     ColumnFilter(AutoEvalColumn.model_type.name, type="checkboxgroup", label="Model types"),
+        #     ColumnFilter(AutoEvalColumn.precision.name, type="checkboxgroup", label="Precision"),
+        #     ColumnFilter(
+        #         AutoEvalColumn.params.name,
+        #         type="slider",
+        #         min=0.01,
+        #         max=150,
+        #         label="Select the number of parameters (B)",
+        #     ),
+        #     ColumnFilter(
+        #         AutoEvalColumn.still_on_hub.name, type="boolean", label="Deleted/incomplete", default=True
+        #     ),
+        # ],
         bool_checkboxgroup_label="Hide models",
         interactive=False,
     )
@@ -95,101 +101,17 @@ with demo:
     gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")
 
     with gr.Tabs(elem_classes="tab-buttons") as tabs:
-        with gr.TabItem("🏅 LLM Benchmark", elem_id="llm-benchmark-tab-table", id=0):
+        with gr.TabItem("🏅 MMTU", elem_id="llm-benchmark-tab-table", id=0):
             leaderboard = init_leaderboard(LEADERBOARD_DF)
 
         with gr.TabItem("📝 About", elem_id="llm-benchmark-tab-table", id=2):
             gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text")
 
-        with gr.TabItem("🚀 Submit here! ", elem_id="llm-benchmark-tab-table", id=3):
-            with gr.Column():
-                with gr.Row():
-                    gr.Markdown(EVALUATION_QUEUE_TEXT, elem_classes="markdown-text")
-
-                with gr.Column():
-                    with gr.Accordion(
-                        f"✅ Finished Evaluations ({len(finished_eval_queue_df)})",
-                        open=False,
-                    ):
-                        with gr.Row():
-                            finished_eval_table = gr.components.Dataframe(
-                                value=finished_eval_queue_df,
-                                headers=EVAL_COLS,
-                                datatype=EVAL_TYPES,
-                                row_count=5,
-                            )
-                    with gr.Accordion(
-                        f"🔄 Running Evaluation Queue ({len(running_eval_queue_df)})",
-                        open=False,
-                    ):
-                        with gr.Row():
-                            running_eval_table = gr.components.Dataframe(
-                                value=running_eval_queue_df,
-                                headers=EVAL_COLS,
-                                datatype=EVAL_TYPES,
-                                row_count=5,
-                            )
-
-                    with gr.Accordion(
-                        f"⏳ Pending Evaluation Queue ({len(pending_eval_queue_df)})",
-                        open=False,
-                    ):
-                        with gr.Row():
-                            pending_eval_table = gr.components.Dataframe(
-                                value=pending_eval_queue_df,
-                                headers=EVAL_COLS,
-                                datatype=EVAL_TYPES,
-                                row_count=5,
-                            )
-            with gr.Row():
-                gr.Markdown("# ✉️✨ Submit your model here!", elem_classes="markdown-text")
-
-            with gr.Row():
-                with gr.Column():
-                    model_name_textbox = gr.Textbox(label="Model name")
-                    revision_name_textbox = gr.Textbox(label="Revision commit", placeholder="main")
-                    model_type = gr.Dropdown(
-                        choices=[t.to_str(" : ") for t in ModelType if t != ModelType.Unknown],
-                        label="Model type",
-                        multiselect=False,
-                        value=None,
-                        interactive=True,
-                    )
-
-                with gr.Column():
-                    precision = gr.Dropdown(
-                        choices=[i.value.name for i in Precision if i != Precision.Unknown],
-                        label="Precision",
-                        multiselect=False,
-                        value="float16",
-                        interactive=True,
-                    )
-                    weight_type = gr.Dropdown(
-                        choices=[i.value.name for i in WeightType],
-                        label="Weights type",
-                        multiselect=False,
-                        value="Original",
-                        interactive=True,
-                    )
-                    base_model_name_textbox = gr.Textbox(label="Base model (for delta or adapter weights)")
-
-            submit_button = gr.Button("Submit Eval")
-            submission_result = gr.Markdown()
-            submit_button.click(
-                add_new_eval,
-                [
-                    model_name_textbox,
-                    base_model_name_textbox,
-                    revision_name_textbox,
-                    precision,
-                    weight_type,
-                    model_type,
-                ],
-                submission_result,
-            )
+        with gr.TabItem("🚀 Submit here!", elem_id="llm-benchmark-tab-table", id=3):
+            gr.Markdown(SUBMIT_INTRODUCTION, elem_classes="markdown-text")
 
     with gr.Row():
-        with gr.Accordion("📙 Citation", open=False):
+        with gr.Accordion("📙 Citation", open=True):
             citation_button = gr.Textbox(
                 value=CITATION_BUTTON_TEXT,
                 label=CITATION_BUTTON_LABEL,

results.json

@@ -0,0 +1,82 @@
+[
+    {
+        "Model": "o4-mini (2024-11-20)",
+        "Model type": "Reasoning",
+        "Model size": "unknown",
+        "Overall": 0.639,
+        "Data Source": "MMTU",
+        "Date": "2025-06-10"
+    },
+    {
+        "Model": "Deepseek-R1",
+        "Model type": "Reasoning",
+        "Model size": "unknown",
+        "Overall": 0.596 ,
+        "Data Source": "MMTU",
+        "Date": "2025-06-10"
+    },
+    {
+        "Model": "Deepseek-V3",
+        "Model type": "Chat",
+        "Model size": "unknown",
+        "Overall": 0.517,
+        "Data Source": "MMTU",
+        "Date": "2025-06-10"
+    },
+    {
+        "Model": "GPT-4o (2024-11-20)",
+        "Model type": "Chat",
+        "Model size": "unknown",
+        "Overall": 0.491 ,
+        "Data Source": "MMTU",
+        "Date": "2025-06-10"
+    },
+    {
+        "Model": "Llama-3.3-70B",
+        "Model type": "Chat",
+        "Model size": "70B",
+        "Overall": 0.438,
+        "Data Source": "MMTU",
+        "Date": "2025-06-10"
+    },
+    {
+        "Model": "Mistral-Large-2411",
+        "Model type": "Chat",
+        "Model size": "123B",
+        "Overall": 0.430,
+        "Data Source": "MMTU",
+        "Date": "2025-06-10"
+    },
+    {
+        "Model": "Llama-3.1-8B",
+        "Model type": "Chat",
+        "Model size": "8B",
+        "Overall": 0.259,
+        "Data Source": "MMTU",
+        "Date": "2025-06-10"
+    },
+    {
+        "Model": "Mistral-Small-2503",
+        "Model type": "Chat",
+        "Model size": "70B",
+        "Overall": 0.426,
+        "Data Source": "MMTU",
+        "Date": "2025-06-10"
+    },
+    {
+        "Model": "Llama-3.3-8B-Instruct",
+        "Model type": "Chat",
+        "Model size": "24B",
+        "Overall": 0.402,
+        "Data Source": "MMTU",
+        "Date": "2025-06-10"
+    },
+    {
+        "Model": "GPT-4o-mini (2024-07-18)",
+        "Model type": "Chat",
+        "Model size": "unknown",
+        "Overall": 0.386,
+        "Data Source": "MMTU",
+        "Date": "2025-06-10"
+    }
+]

src/about.py

@@ -21,20 +21,28 @@ NUM_FEWSHOT = 0 # Change with your few shot
 
 
 # Your leaderboard name
-TITLE = """<h1 align="center" id="space-title">Demo leaderboard</h1>"""
+TITLE = """<h1 align="center" id="space-title">MMTU leaderboard</h1>"""
 
 # What does your leaderboard evaluate?
 INTRODUCTION_TEXT = """
-Intro text
+|[**🤗 Dataset**](https://huggingface.co/datasets/MMTU-benchmark/MMTU) |[**🛠️GitHub**](https://github.com/MMTU-Benchmark/MMTU/tree/main) |[**🏆Leaderboard**](https://huggingface.co/spaces/MMTU-benchmark/mmtu-leaderboard)|[**📖 Paper**](https://arxiv.org/abs/2506.05587) |
+
+Tables and table-based use cases play a crucial role in many real-world applications, such as spreadsheets, databases, and computational notebooks, which traditionally require expert-level users like data engineers, analysts, and database administrators to operate. Although LLMs have shown remarkable progress in working with tables, comprehensive benchmarking of such capabilities remains limited, often narrowly focusing on tasks like NL-to-SQL and Table-QA, while overlooking the broader spectrum of real-world tasks that professional users face today. 
+
+We introduce **MMTU**, a large-scale benchmark with over **30K questions** across **25 real-world table tasks**, designed to comprehensively evaluate models ability to understand, reason, and manipulate real tables at the expert-level. These tasks are drawn from decades' worth of computer science research on tabular data, with a focus on complex table tasks faced by professional users. We show that MMTU require a combination of skills -- including table understanding, reasoning, and coding -- that remain challenging for today's frontier models, where even frontier reasoning models like OpenAI o4-mini and DeepSeek R1 score only around 60%, suggesting significant room for improvement. Our evaluation code is available at [GitHub](https://github.com/MMTU-Benchmark/MMTU/tree/main).
+
 """
 
 # Which evaluations are you running? how can people reproduce what you have?
 LLM_BENCHMARKS_TEXT = f"""
-## How it works
+Please visit [**🤗 Dataset**](https://huggingface.co/datasets/MMTU-benchmark/MMTU) and [**📖 Paper**](https://arxiv.org/abs/2506.05587) to see the full list of tasks and their descriptions.
+
+"""
 
-## Reproducibility
-To reproduce our results, here is the commands you can run:
+SUBMIT_INTRODUCTION = """# Submit on MMTU Leaderboard Introduction
+## ⚠ Please note that you need to submit the JSONL file with your model output.
 
+You can generate an output file using the evaluation script provided in our GitHub repository. For your convenience, the script and detailed instructions are available at GitHub: https://github.com/MMTU-Benchmark/MMTU. After generating the file, please send us an email at [email protected], attaching the output file.
 """
 
 EVALUATION_QUEUE_TEXT = """
@@ -67,6 +75,16 @@ Make sure you have followed the above steps first.
 If everything is done, check you can launch the EleutherAIHarness on your model locally, using the above command without modifications (you can add `--limit` to limit the number of examples per task).
 """
 
-CITATION_BUTTON_LABEL = "Copy the following snippet to cite these results"
-CITATION_BUTTON_TEXT = r"""
+CITATION_BUTTON_LABEL = "Copy the following snippet to cite MMTU"
+CITATION_BUTTON_TEXT = \
+r"""
+@misc{xing2025mmtumassivemultitasktable,
+      title={MMTU: A Massive Multi-Task Table Understanding and Reasoning Benchmark}, 
+      author={Junjie Xing and Yeye He and Mengyu Zhou and Haoyu Dong and Shi Han and Lingjiao Chen and Dongmei Zhang and Surajit Chaudhuri and H. V. Jagadish},
+      year={2025},
+      eprint={2506.05587},
+      archivePrefix={arXiv},
+      primaryClass={cs.AI},
+      url={https://arxiv.org/abs/2506.05587}, 
+}
 """

src/display/utils.py

@@ -23,22 +23,23 @@ class ColumnContent:
 ## Leaderboard columns
 auto_eval_column_dict = []
 # Init
-auto_eval_column_dict.append(["model_type_symbol", ColumnContent, ColumnContent("T", "str", True, never_hidden=True)])
+# auto_eval_column_dict.append(["model_type_symbol", ColumnContent, ColumnContent("T", "str", True, never_hidden=True)])
 auto_eval_column_dict.append(["model", ColumnContent, ColumnContent("Model", "markdown", True, never_hidden=True)])
 #Scores
-auto_eval_column_dict.append(["average", ColumnContent, ColumnContent("Average ⬆️", "number", True)])
-for task in Tasks:
-    auto_eval_column_dict.append([task.name, ColumnContent, ColumnContent(task.value.col_name, "number", True)])
+auto_eval_column_dict.append(["overall", ColumnContent, ColumnContent("Overall", "number", True)])
+# for task in Tasks:
+#     auto_eval_column_dict.append([task.name, ColumnContent, ColumnContent(task.value.col_name, "number", True)])
 # Model information
-auto_eval_column_dict.append(["model_type", ColumnContent, ColumnContent("Type", "str", False)])
-auto_eval_column_dict.append(["architecture", ColumnContent, ColumnContent("Architecture", "str", False)])
-auto_eval_column_dict.append(["weight_type", ColumnContent, ColumnContent("Weight type", "str", False, True)])
-auto_eval_column_dict.append(["precision", ColumnContent, ColumnContent("Precision", "str", False)])
-auto_eval_column_dict.append(["license", ColumnContent, ColumnContent("Hub License", "str", False)])
-auto_eval_column_dict.append(["params", ColumnContent, ColumnContent("#Params (B)", "number", False)])
-auto_eval_column_dict.append(["likes", ColumnContent, ColumnContent("Hub ❤️", "number", False)])
-auto_eval_column_dict.append(["still_on_hub", ColumnContent, ColumnContent("Available on the hub", "bool", False)])
-auto_eval_column_dict.append(["revision", ColumnContent, ColumnContent("Model sha", "str", False, False)])
+auto_eval_column_dict.append(["data_source", ColumnContent, ColumnContent("Data Source", "str", True)])
+auto_eval_column_dict.append(["date", ColumnContent, ColumnContent("Date", "str", True)])
+auto_eval_column_dict.append(["model_type", ColumnContent, ColumnContent("Model type", "str", False)])
+auto_eval_column_dict.append(["model_size", ColumnContent, ColumnContent("Model size", "str", False)])
+# auto_eval_column_dict.append(["precision", ColumnContent, ColumnContent("Precision", "str", False)])
+# auto_eval_column_dict.append(["license", ColumnContent, ColumnContent("Hub License", "str", False)])
+# auto_eval_column_dict.append(["params", ColumnContent, ColumnContent("#Params (B)", "number", False)])
+# auto_eval_column_dict.append(["likes", ColumnContent, ColumnContent("Hub ❤️", "number", False)])
+# auto_eval_column_dict.append(["still_on_hub", ColumnContent, ColumnContent("Available on the hub", "bool", False)])
+# auto_eval_column_dict.append(["revision", ColumnContent, ColumnContent("Model sha", "str", False, False)])
 
 # We use make dataclass to dynamically fill the scores from Tasks
 AutoEvalColumn = make_dataclass("AutoEvalColumn", auto_eval_column_dict, frozen=True)

src/envs.py

@@ -6,20 +6,20 @@ from huggingface_hub import HfApi
 # ----------------------------------
 TOKEN = os.environ.get("HF_TOKEN") # A read/write token for your org
 
-OWNER = "demo-leaderboard-backend" # Change to your org - don't forget to create a results and request dataset, with the correct format!
+OWNER = "MMTU-benchmark" # Change to your org - don't forget to create a results and request dataset, with the correct format!
 # ----------------------------------
 
 REPO_ID = f"{OWNER}/leaderboard"
-QUEUE_REPO = f"{OWNER}/requests"
-RESULTS_REPO = f"{OWNER}/results"
+# QUEUE_REPO = f"{OWNER}/requests"
+# RESULTS_REPO = f"{OWNER}/results"
 
 # If you setup a cache later, just change HF_HOME
 CACHE_PATH=os.getenv("HF_HOME", ".")
 
 # Local caches
-EVAL_REQUESTS_PATH = os.path.join(CACHE_PATH, "eval-queue")
-EVAL_RESULTS_PATH = os.path.join(CACHE_PATH, "eval-results")
-EVAL_REQUESTS_PATH_BACKEND = os.path.join(CACHE_PATH, "eval-queue-bk")
-EVAL_RESULTS_PATH_BACKEND = os.path.join(CACHE_PATH, "eval-results-bk")
+# EVAL_REQUESTS_PATH = os.path.join(CACHE_PATH, "eval-queue")
+# EVAL_RESULTS_PATH = os.path.join(CACHE_PATH, "eval-results")
+# EVAL_REQUESTS_PATH_BACKEND = os.path.join(CACHE_PATH, "eval-queue-bk")
+# EVAL_RESULTS_PATH_BACKEND = os.path.join(CACHE_PATH, "eval-results-bk")
 
 API = HfApi(token=TOKEN)