Adjusted for hebrew

app.py

@@ -30,14 +30,11 @@ from src.envs import API, DEVICE, EVAL_REQUESTS_PATH, EVAL_RESULTS_PATH, QUEUE_R
 from src.populate import get_evaluation_queue_df, get_leaderboard_df
 from src.submission.submit import add_new_eval
 
-
-subprocess.run(["python", "scripts/fix_harness_import.py"])
-
 def restart_space():
     API.restart_space(repo_id=REPO_ID)
 
-def launch_backend():
-    _ = subprocess.run(["python", "main_backend.py"])
+# def launch_backend():
+#    _ = subprocess.run(["python", "main_backend.py"])
 
 try:
     print(EVAL_REQUESTS_PATH)
@@ -348,6 +345,6 @@ with demo:
 
 scheduler = BackgroundScheduler()
 scheduler.add_job(restart_space, "interval", seconds=1800)
-scheduler.add_job(launch_backend, "interval", seconds=100) # will only allow one job to be run at the same time
+# scheduler.add_job(launch_backend, "interval", seconds=100) # will only allow one job to be run at the same time
 scheduler.start()
 demo.queue(default_concurrency_limit=40).launch()

scripts/fix_harness_import.py

@@ -1,11 +0,0 @@
-"""This file should be used after pip install -r requirements.
-It creates a folder not ported during harness package creation (as they don't use a Manifest file atm and it ignore `.json` files).
-It will need to be updated if we want to use the harness' version of big bench to actually copy the json files.
-"""
-import os
-
-import lm_eval
-
-if __name__ == "__main__":
-    lm_eval_path = lm_eval.__path__[0]
-    os.makedirs(os.path.join(lm_eval_path, "datasets", "bigbench_resources"), exist_ok=True)

src/about.py

@@ -14,7 +14,7 @@ class Tasks(Enum):
     # task_key in the json file, metric_key in the json file, name to display in the leaderboard 
     task0 = Task("anli_r1", "acc", "ANLI")
     task1 = Task("logiqa", "acc_norm", "LogiQA")
-
+    
 NUM_FEWSHOT = 0 # Change with your few shot
 # ---------------------------------------------------
 

src/envs.py

@@ -6,7 +6,7 @@ from huggingface_hub import HfApi
 # ----------------------------------
 TOKEN = os.environ.get("TOKEN") # A read/write token for your org
 
-OWNER = "demo-leaderboard-backend" # Change to your org - don't forget to create a results and request file
+OWNER = "dicta-hebrew-llm-leaderboard" # Change to your org - don't forget to create a results and request file
 DEVICE = "cpu" # "cuda:0" if you add compute
 LIMIT = 20 # !!!! Should be None for actual evaluations!!!
 # ----------------------------------