use docker and add exectuion

Dockerfile

@@ -0,0 +1,5 @@
+FROM bigcodebench/bigcodebench-gradio:latest
+COPY . /app
+EXPOSE 7860
+ENV GRADIO_SERVER_NAME="0.0.0.0"
+CMD ["python", "app.py"]

README.md

@@ -3,8 +3,7 @@ title: BigCodeBench Leaderboard
 emoji: 🥇
 colorFrom: green
 colorTo: indigo
-sdk: gradio
-sdk_version: 4.36.1
+sdk: docker
 app_file: app.py
 disable_embedding: true
 pinned: false

app.py

@@ -1,7 +1,6 @@
 import os
 import logging
 import time
-import schedule
 import datetime
 import gradio as gr
 from threading import Thread
@@ -49,6 +48,7 @@ from src.envs import (
     HF_HOME,
 )
 from src.populate import get_evaluation_queue_df, get_leaderboard_df
+from src.execute import generate_command, is_running, default_command, stream_logs, find_result_file
 from src.tools.plots import plot_elo_mle, plot_solve_rate
 # from src.voting.vote_system import VoteManager, run_scheduler
 
@@ -482,6 +482,7 @@ with main_block as demo:
                                                 ], instruct_map)
         with gr.TabItem("📝 About", id=3):
             gr.Markdown(ABOUT_TEXT, elem_classes="markdown-text")
+
         with gr.TabItem("🔎 Data Viewer", id="viewer"):
             search_input = gr.Textbox(label="Search by keyword")
             count_output = gr.Number(label="Number of filtered items")
@@ -508,7 +509,67 @@ with main_block as demo:
                 inputs=[search_input, index_slider, show_test],
                 outputs=[task_id_output, code_completion, nl_instruction, test_cases, count_output, index_slider]
             )
-
+            
+        with gr.TabItem("🛠️ Code Execution (Beta)", id=5):
+            gr.Markdown("## Upload your sanitized JSONL file to evaluate")
+            
+            with gr.Row():
+                jsonl_file = gr.File(label="Upload JSONL file", file_types=[".jsonl"])
+                split = gr.Dropdown(choices=["complete", "instruct"], label="Split", value="complete")
+                subset = gr.Dropdown(choices=["hard (gt pass rate: 100%)", "full (gt pass rate: 99.6%)"], label="Subset", value="hard")
+            
+            with gr.Row():
+                parallel = gr.Number(label="Parallel (optional)", precision=0)
+                min_time_limit = gr.Number(label="Min Time Limit", value=1, precision=1)
+                max_as_limit = gr.Number(label="Max AS Limit", value=25*1024, precision=0)
+            
+            with gr.Row():
+                max_data_limit = gr.Number(label="Max Data Limit", value=25*1024, precision=0)
+                max_stack_limit = gr.Number(label="Max Stack Limit", value=10, precision=0)
+                check_gt_only = gr.Checkbox(label="Check GT Only", value=False, visible=False)
+                no_gt = gr.Checkbox(label="No GT", value=False, visible=False)
+            
+            command_output = gr.Textbox(label="Command", value=default_command, interactive=False)
+            with gr.Row():
+                submit_btn = gr.Button("Run Evaluation")
+                download_btn = gr.DownloadButton(label="Download Result")
+            log_output = gr.Textbox(label="Execution Logs", lines=20)
+            
+            input_components = [
+                jsonl_file, split, subset, parallel,
+                min_time_limit, max_as_limit, max_data_limit, max_stack_limit,
+                check_gt_only, no_gt
+            ]
+            
+            for component in input_components:
+                component.change(generate_command, inputs=input_components, outputs=command_output)
+                
+            
+            def start_evaluation(command, jsonl_file, subset, split):
+                extra = subset + "_" if subset != "full" else ""
+                if jsonl_file is not None:
+                    result_path = os.path.basename(jsonl_file.name).replace(".jsonl", f"_{extra}eval_results.json")
+                else:
+                    result_path = None
+
+                for log in stream_logs(command, jsonl_file):
+                    if jsonl_file is not None:
+                        yield log, gr.update(value=result_path, label=result_path), gr.update()
+                    else:
+                        yield log, gr.update(), gr.update()
+                result_file = find_result_file()
+                if result_file:
+                    return gr.update(label="Evaluation completed. Result file found."), gr.update(value=result_file)
+                            # gr.Button(visible=False)#,
+                            # gr.DownloadButton(label="Download Result", value=result_file, visible=True))
+                else:
+                    return gr.update(label="Evaluation completed. No result file found."), gr.update(value=result_path)
+                            # gr.Button("Run Evaluation", visible=True),
+                            # gr.DownloadButton(visible=False))
+            submit_btn.click(start_evaluation,
+                        inputs=[command_output, jsonl_file, subset, split],
+                        outputs=[log_output, download_btn])
+        
         with gr.TabItem("🚀 Request", id=4):
             gr.Markdown(SUBMISSION_TEXT_3)
     
@@ -583,5 +644,5 @@ def update_leaderboard(payload: WebhookPayload) -> None:
 webhooks_server.launch()
 
 scheduler = BackgroundScheduler()
-scheduler.add_job(restart_space, "interval", hours=5) # restarted every 3h as backup in case automatic updates are not working
-scheduler.start()
+scheduler.add_job(restart_space, "interval", hours=3) # restarted every 3h as backup in case automatic updates are not working
+scheduler.start()

src/execute.py

@@ -0,0 +1,194 @@
+import gradio as gr
+import subprocess
+import sys
+import os
+import threading
+import time
+import uuid
+import glob
+import shutil
+from pathlib import Path
+
+default_command = "bigcodebench.evaluate"
+is_running = False
+
+def generate_command(
+    jsonl_file, split, subset, parallel,
+    min_time_limit, max_as_limit, max_data_limit, max_stack_limit,
+    check_gt_only, no_gt
+):
+    command = [default_command]
+    
+    if jsonl_file is not None:
+        # Copy the uploaded file to the current directory
+        local_filename = os.path.basename(jsonl_file.name)
+        shutil.copy(jsonl_file.name, local_filename)
+        command.extend(["--samples", local_filename])
+    
+    command.extend(["--split", split, "--subset", subset])
+    
+    if parallel is not None and parallel != 0:
+        command.extend(["--parallel", str(int(parallel))])
+    
+    command.extend([
+        "--min-time-limit", str(min_time_limit),
+        "--max-as-limit", str(int(max_as_limit)),
+        "--max-data-limit", str(int(max_data_limit)),
+        "--max-stack-limit", str(int(max_stack_limit))
+    ])
+    
+    if check_gt_only:
+        command.append("--check-gt-only")
+    
+    if no_gt:
+        command.append("--no-gt")
+    
+    return " ".join(command)
+
+
+def cleanup_previous_files(jsonl_file):
+    if jsonl_file is not None:
+        file_list = ['Dockerfile', 'app.py', 'README.md', os.path.basename(jsonl_file.name), "__pycache__"]
+    else:
+        file_list = ['Dockerfile', 'app.py', 'README.md', "__pycache__"]
+    for file in glob.glob("*"):
+        try:
+            if file not in file_list:
+                os.remove(file)
+        except Exception as e:
+            print(f"Error during cleanup of {file}: {e}")
+
+def find_result_file():
+    json_files = glob.glob("*.json")
+    if json_files:
+        return max(json_files, key=os.path.getmtime)
+    return None
+
+def run_bigcodebench(command):
+    global is_running
+    if is_running:
+        yield "A command is already running. Please wait for it to finish.\n"
+        return
+    is_running = True
+
+    try:
+        yield f"Executing command: {command}\n"
+        
+        process = subprocess.Popen(command.split(), stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True)
+        
+        def kill_process():
+            if process.poll() is None:  # If the process is still running
+                process.terminate()
+                is_running = False
+                yield "Process terminated after 12 minutes timeout.\n"
+
+        # Start a timer to kill the process after 12 minutes
+        timer = threading.Timer(720, kill_process)
+        timer.start()
+        
+        for line in process.stdout:
+            yield line
+        
+        # process.wait()
+        
+        timer.cancel()
+        
+        if process.returncode != 0:
+            yield f"Error: Command exited with status {process.returncode}\n"
+        
+        yield "Evaluation completed.\n"
+        
+        result_file = find_result_file()
+        if result_file:
+            yield f"Result file found: {result_file}\n"
+        else:
+            yield "No result file found.\n"
+    finally:
+        is_running = False
+
+def stream_logs(command, jsonl_file=None):
+    global is_running
+        
+    if is_running:
+        yield "A command is already running. Please wait for it to finish.\n"
+        return
+    
+    cleanup_previous_files(jsonl_file)
+    yield "Cleaned up previous files.\n"
+
+    log_content = []
+    for log_line in run_bigcodebench(command):
+        log_content.append(log_line)
+        yield "".join(log_content)
+        
+# with gr.Blocks() as demo:
+#     gr.Markdown("# BigCodeBench Evaluator")
+    
+#     with gr.Row():
+#         jsonl_file = gr.File(label="Upload JSONL file", file_types=[".jsonl"])
+#         split = gr.Dropdown(choices=["complete", "instruct"], label="Split", value="complete")
+#         subset = gr.Dropdown(choices=["hard", "full"], label="Subset", value="hard")
+    
+#     with gr.Row():
+#         parallel = gr.Number(label="Parallel (optional)", precision=0)
+#         min_time_limit = gr.Number(label="Min Time Limit", value=1, precision=1)
+#         max_as_limit = gr.Number(label="Max AS Limit", value=25*1024, precision=0)
+    
+#     with gr.Row():
+#         max_data_limit = gr.Number(label="Max Data Limit", value=25*1024, precision=0)
+#         max_stack_limit = gr.Number(label="Max Stack Limit", value=10, precision=0)
+#         check_gt_only = gr.Checkbox(label="Check GT Only")
+#         no_gt = gr.Checkbox(label="No GT")
+    
+#     command_output = gr.Textbox(label="Command", value=default_command, interactive=False)
+#     with gr.Row():
+#         submit_btn = gr.Button("Run Evaluation")
+#         download_btn = gr.DownloadButton(label="Download Result")
+#     log_output = gr.Textbox(label="Execution Logs", lines=20)
+    
+#     input_components = [
+#         jsonl_file, split, subset, parallel,
+#         min_time_limit, max_as_limit, max_data_limit, max_stack_limit,
+#         check_gt_only, no_gt
+#     ]
+    
+#     for component in input_components:
+#         component.change(generate_command, inputs=input_components, outputs=command_output)
+        
+    
+#     def start_evaluation(command, jsonl_file, subset, split):
+#         extra = subset + "_" if subset != "full" else ""
+#         if jsonl_file is not None:
+#             result_path = os.path.basename(jsonl_file.name).replace(".jsonl", f"_{extra}eval_results.json")
+#         else:
+#             result_path = None
+
+#         for log in stream_logs(command, jsonl_file):
+#             if jsonl_file is not None:
+#                 yield log, gr.update(value=result_path, label=result_path), gr.update()
+#             else:
+#                 yield log, gr.update(), gr.update()
+#         result_file = find_result_file()
+#         if result_file:
+#             return gr.update(label="Evaluation completed. Result file found."), gr.update(value=result_file)
+#                     # gr.Button(visible=False)#,
+#                     # gr.DownloadButton(label="Download Result", value=result_file, visible=True))
+#         else:
+#             return gr.update(label="Evaluation completed. No result file found."), gr.update(value=result_path)
+#                     # gr.Button("Run Evaluation", visible=True),
+#                     # gr.DownloadButton(visible=False))
+#     submit_btn.click(start_evaluation,
+#                  inputs=[command_output, jsonl_file, subset, split],
+#                  outputs=[log_output, download_btn])
+
+# REPO_ID = "bigcode/bigcodebench-evaluator"
+# HF_TOKEN = os.environ.get("HF_TOKEN", None)
+# API = HfApi(token=HF_TOKEN)
+
+# def restart_space():
+#     API.restart_space(repo_id=REPO_ID, token=HF_TOKEN)
+
+# demo.queue(max_size=300).launch(share=True, server_name="0.0.0.0", server_port=7860)
+# scheduler = BackgroundScheduler()
+# scheduler.add_job(restart_space, "interval", hours=3) # restarted every 3h as backup in case automatic updates are not working
+# scheduler.start()