Changed submission ID and added more informations from submission

app.py

@@ -19,29 +19,37 @@ from server import LeaderboardServer
 leaderboard_server = LeaderboardServer()
 
 
+SUBMISSION_INPUTS = dict.fromkeys((
+    "team_name",
+    "model_name",
+    "model_type",
+    "parameters",
+    "precision",
+    "description",
+    "link_to_model",
+    "submission_file",
+)).keys()
+
 def on_submit_pressed():
     return gr.update(value='Processing submission...', interactive=False)
 
 
-def validate_submission_inputs(team_name, submission_id, link_to_model, submission_file):
-    if not team_name or not submission_id or not link_to_model or not submission_file:
+def validate_submission_inputs(**kwargs):
+    if SUBMISSION_INPUTS - set(kwargs) != set():
         raise ValueError('Please fill in all fields')
-    if not os.path.exists(submission_file):
+    if not os.path.exists(kwargs["submission_file"]):
         raise ValueError('File does not exist')
 
-
-def process_submission(team_name, submission_id, description, link_to_model, submission_file):
+def process_submission(*inputs):
     try:
-        validate_submission_inputs(team_name, submission_id, link_to_model, submission_file)
-        metadata = {
-            "team_name": team_name,
-            "submission_id": submission_id,
-            "description": description,
-            "link_to_model": link_to_model,
-        }
+        inputs = dict(zip(SUBMISSION_INPUTS, inputs))
+        validate_submission_inputs(**inputs)
+        metadata = SUBMISSION_INPUTS - {"submission_file"}
+        metadata = {key: inputs[key] for key in metadata}
+        
         gr.Info('Submission valid, running local tournament...')
 
-        leaderboard_server.prepare_model_for_submission(submission_file, metadata)
+        leaderboard_server.prepare_model_for_submission(inputs["submission_file"], metadata)
     except ValueError as err:
         gr.Warning(str(err))
         return (
@@ -226,15 +234,22 @@ with gr.Blocks(theme=gr.themes.Soft(text_size=text_md), css=custom_css) as main:
         with gr.Tab('Submission'):
             with gr.Column():
                 gr.Markdown(SUBMISSION_TAB_TITLE_MARKDOWN)
+                
+                submission_inputs = dict.fromkeys(SUBMISSION_INPUTS)
+                
                 with gr.Row():
-                    submission_team_name_tb = gr.Textbox(label='Team Name')
-                    submission_id_tb = gr.Textbox(label='Submission ID')
+                    submission_inputs["team_name"] = gr.Textbox(label='Team name')
+                    submission_inputs["model_name"] = gr.Textbox(label='Model name')
+                    submission_inputs["model_type"] = gr.Textbox(label='Model type')
+                    submission_inputs["parameters"] = gr.Textbox(label='Parameters')
 
                 with gr.Row():
-                    description_tb = gr.Textbox(label='Description', type='text')
-                    link_to_model_tb = gr.Textbox(label='Link to model', type='text')
+                    submission_inputs["precision"] = gr.Textbox(label='Precision')
+                    submission_inputs["description"] = gr.Textbox(label='Description', type='text')
+                    submission_inputs["link_to_model"] = gr.Textbox(label='Link to model', type='text')
 
-                submission_file_path = gr.File(label='Upload your results', type='filepath')
+                submission_inputs["submission_file"] = gr.File(label='Upload your results', type='filepath')
+                
                 pre_submission_btn = gr.Button(value='Pre-submit model', interactive=True)
 
                 submit_prompt = gr.Markdown(
@@ -263,13 +278,7 @@ with gr.Blocks(theme=gr.themes.Soft(text_size=text_md), css=custom_css) as main:
                     outputs=[pre_submission_btn],
                 ).then(
                     fn=process_submission,
-                    inputs=[
-                        submission_team_name_tb,
-                        submission_id_tb,
-                        description_tb,
-                        link_to_model_tb,
-                        submission_file_path,
-                    ],
+                    inputs=list(submission_inputs.values()),
                     outputs=[
                         pre_submission_btn,
                         submit_prompt,

server.py

@@ -2,6 +2,7 @@ import copy
 import glob
 import json
 import os
+import hashlib
 
 import gradio as gr
 import pandas as pd
@@ -18,7 +19,6 @@ REPO = f"{ORG}/LLM_benchmark_data"
 HF_TOKEN = os.environ.get("HF_TOKEN")
 TASKS_METADATA_PATH = "./tasks_metadata.json"
 
-
 class LeaderboardServer:
     def __init__(self):
         self.server_address = REPO
@@ -29,7 +29,7 @@ class LeaderboardServer:
             token=HF_TOKEN,
             local_dir="./",
         )
-        self.submisssion_id_to_file = {}  # Map submission ids to file paths
+        self.submission_id_to_file = {}  # Map submission ids to file paths
         self.tasks_metadata = json.load(open(TASKS_METADATA_PATH))
         self.tasks_categories = {self.tasks_metadata[task]["category"] for task in self.tasks_metadata}
         self.submission_ids = set()
@@ -57,15 +57,15 @@ class LeaderboardServer:
 
     def fetch_existing_models(self):
         # Models data
-        for submission in glob.glob(os.path.join(self.local_leaderboard, "data") + "/*.json"):
-            data = json.load(open(submission))
+        for submission_file in glob.glob(os.path.join(self.local_leaderboard, "data") + "/*.json"):
+            data = json.load(open(submission_file))
             metadata = data.get('metadata')
             if metadata is None:
                 continue
-            submission_id = metadata["team_name"] + "_" + metadata["submission_id"]
+            submission_id = metadata["submission_id"]
             self.submission_ids.add(submission_id)
 
-            self.submisssion_id_to_file[submission_id] = submission
+            self.submission_id_to_file[submission_id] = submission_file
 
     def get_leaderboard(self, tournament_results=None):
         results = tournament_results if tournament_results else self.tournament_results
@@ -74,24 +74,26 @@ class LeaderboardServer:
             return pd.DataFrame(columns=['No submissions yet'])
         else:
             processed_results = []
-            for submission in results.keys():
-                path = self.submisssion_id_to_file.get(submission)
+            for submission_id in results.keys():
+                path = self.submission_id_to_file.get(submission_id)
                 if path is None:
-                    if self.pre_submit and submission == self.pre_submit[1]:
+                    if self.pre_submit and submission_id == self.pre_submit[1]:
                         data = json.load(open(self.pre_submit[2]))
                     else:
-                        raise gr.Error(f"Internal error: Submission [{submission}] not found")
+                        raise gr.Error(f"Internal error: Submission [{submission_id}] not found")
                 elif path:
                     data = json.load(open(path))
                 else:
-                    raise gr.Error(f"Submission [{submission}] not found")
-                submission_id = data["metadata"]["team_name"] + "_" + data["metadata"]["submission_id"]
+                    raise gr.Error(f"Submission [{submission_id}] not found")
+                
+                if submission_id != data["metadata"]["submission_id"]:
+                    raise gr.Error(f"Proper submission [{submission_id}] not found")
 
                 local_results = {}
                 for task in self.tasks_metadata.keys():
                     local_results[task] = 0
-                    for model in results[submission].keys():
-                        if results[submission][model][task]:
+                    for model in results[submission_id].keys():
+                        if results[submission_id][model][task]:
                             local_results[task] += 1
                     for metric in VISIBLE_METRICS:
                         metric_value = data['results'][task].get(metric)
@@ -100,7 +102,7 @@ class LeaderboardServer:
 
                 local_results["submission_id"] = submission_id
 
-                if self.pre_submit and submission == self.pre_submit[1]:
+                if self.pre_submit and submission_id == self.pre_submit[1]:
                     processed_results.insert(0, local_results)
                 else:
                     processed_results.append(local_results)
@@ -120,42 +122,71 @@ class LeaderboardServer:
             )
             return dataframe
 
-    def start_tournament(self, new_model_id, new_model_file):
+    def start_tournament(self, new_submission_id, new_model_file):
         new_tournament = copy.deepcopy(self.tournament_results)
-        new_tournament[new_model_id] = {}
-        new_tournament[new_model_id][new_model_id] = {
+        new_tournament[new_submission_id] = {}
+        new_tournament[new_submission_id][new_submission_id] = {
             task: False for task in self.tasks_metadata.keys()
         }
 
-        for model in self.submission_ids:
-            res = check_significance(new_model_file, self.submisssion_id_to_file[model])
-            res_inverse = check_significance(self.submisssion_id_to_file[model], new_model_file)
-            new_tournament[new_model_id][model] = {
+        for submission_id in self.submission_ids:
+            res = check_significance(new_model_file, self.submission_id_to_file[submission_id])
+            res_inverse = check_significance(self.submission_id_to_file[submission_id], new_model_file)
+            new_tournament[new_submission_id][submission_id] = {
                 task: data["significant"] for task, data in res.items()
             }
-            new_tournament[model][new_model_id] = {
+            new_tournament[submission_id][new_submission_id] = {
                 task: data["significant"] for task, data in res_inverse.items()
             }
         return new_tournament
 
+    @staticmethod
+    def create_submission_id(metadata):
+        # Délka ID můsí být omezena, protože se používá v názvu souboru
+        submission_id = "_".join([metadata[key][:7] for key in (
+            "team_name",
+            "model_name",
+            "model_predictions_sha256",
+            "model_results_sha256",
+        )])
+        return submission_id
+
+    @staticmethod
+    def get_sha256_hexdigest(obj):
+        data = json.dumps(
+            obj,
+            separators=(',', ':'),
+            sort_keys=True,
+            ensure_ascii=True,
+        ).encode()
+        result = hashlib.sha256(data).hexdigest()
+        return result
+
     def prepare_model_for_submission(self, file, metadata) -> None:
         with open(file, "r") as f:
             data = json.load(f)
+        
         data["metadata"] = metadata
+        
+        metadata["model_predictions_sha256"] = self.get_sha256_hexdigest(data["predictions"])
+        metadata["model_results_sha256"] = self.get_sha256_hexdigest(data["results"])
+        
+        # Délka ID můsí být omezena, protože se používá v názvu souboru
+        submission_id = self.create_submission_id(metadata)
+        metadata["submission_id"] = submission_id
+        
         with open(file, "w") as f:
             json.dump(data, f, separators=(',', ':'))  # compact JSON
-
-        model_id = metadata["team_name"] + "_" + metadata["submission_id"]
-        tournament_results = self.start_tournament(model_id, file)
-        self.pre_submit = tournament_results, model_id, file
+        
+        tournament_results = self.start_tournament(submission_id, file)
+        self.pre_submit = tournament_results, submission_id, file
 
     def save_pre_submit(self):
         if self.pre_submit:
-            tournament_results, model_id, file = self.pre_submit
-            filename = os.path.basename(file)
+            tournament_results, submission_id, file = self.pre_submit
             api.upload_file(
                 path_or_fileobj=file,
-                path_in_repo=f"data/{model_id}_{filename}",
+                path_in_repo=f"data/{submission_id}.json",
                 repo_id=self.server_address,
                 repo_type=self.repo_type,
                 token=HF_TOKEN,
@@ -175,7 +206,7 @@ class LeaderboardServer:
             )
 
     def get_model_detail(self, submission_id):
-        path = self.submisssion_id_to_file.get(submission_id)
+        path = self.submission_id_to_file.get(submission_id)
         if path is None:
             raise gr.Error(f"Submission [{submission_id}] not found")
         data = json.load(open(path))