Merge branch 'main' of https://huggingface.co/spaces/BramVanroy/open_dutch_llm_leaderboard

README.md

@@ -8,3 +8,18 @@ sdk_version: 4.11.0
 app_file: app.py
 pinned: false
 ---
+
+If you use the results of this leaderboard or refer to the leaderboard as a whole, please cite:
+
+Vanroy, B. (2023). *Language Resources for Dutch Large Language Modelling*. [https://arxiv.org/abs/2312.12852](https://arxiv.org/abs/2312.12852)
+
+```bibtext
+@misc{vanroy2023language,
+      title={Language Resources for {Dutch} Large Language Modelling}, 
+      author={Bram Vanroy},
+      year={2023},
+      eprint={2312.12852},
+      archivePrefix={arXiv},
+      primaryClass={cs.CL}
+}
+```

content.py

@@ -39,21 +39,32 @@ This leaderboard has borrowed heavily from the following sources:
 """
 
 
-CITATION = f"""## Citation
+CITATION = """## Citation
 
 
 If you use or cite the Dutch benchmark results or this specific leaderboard page, please cite the following paper:
 
-TBA
+Vanroy, B. (2023). *Language Resources for Dutch Large Language Modelling*. [https://arxiv.org/abs/2312.12852](https://arxiv.org/abs/2312.12852)
+
+```bibtext
+@misc{vanroy2023language,
+      title={Language Resources for {Dutch} Large Language Modelling}, 
+      author={Bram Vanroy},
+      year={2023},
+      eprint={2312.12852},
+      archivePrefix={arXiv},
+      primaryClass={cs.CL}
+}
+```
 
 
 If you use the multilingual benchmarks, please cite the following paper:
 
 ```bibtex
-@misc{{lai2023openllmbenchmark,
-    author = {{Viet Lai and Nghia Trung Ngo and Amir Pouran Ben Veyseh and Franck Dernoncourt and Thien Huu Nguyen}},
-    title={{Open Multilingual LLM Evaluation Leaderboard}},
-    year={{2023}}
-}}
+@misc{lai2023openllmbenchmark,
+    title={Open Multilingual {LLM} Evaluation Leaderboard},
+    author={Viet Lai and Nghia Trung Ngo and Amir Pouran Ben Veyseh and Franck Dernoncourt and Thien Huu Nguyen},
+    year={2023}
+}
 ```
 """

evals/mmlu/mmlu_nl_GEITje-7B-chat.json

@@ -0,0 +1,23 @@
+{
+  "results": {
+    "mmlu_nl": {
+      "acc": 0.38362297943386203,
+      "acc_stderr": 0.004236270988211309,
+      "acc_norm": 0.3342187144266525,
+      "acc_norm_stderr": 0.004109503747402399
+    }
+  },
+  "versions": {
+    "mmlu_nl": 0
+  },
+  "config": {
+    "model": "hf-auto",
+    "model_args": "pretrained=Rijgersberg/GEITje-7B-chat,use_accelerate=True,device_map_option=auto,dtype=auto,load_in_8bit=True",
+    "batch_size": 2,
+    "device": "cuda",
+    "no_cache": false,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

evals/mmlu/mmlu_nl_GEITje-7B.json

@@ -0,0 +1,23 @@
+{
+  "results": {
+    "mmlu_nl": {
+      "acc": 0.38202929346588754,
+      "acc_stderr": 0.004232924126485795,
+      "acc_norm": 0.32488426804280185,
+      "acc_norm_stderr": 0.004080014081775866
+    }
+  },
+  "versions": {
+    "mmlu_nl": 0
+  },
+  "config": {
+    "model": "hf-auto",
+    "model_args": "pretrained=Rijgersberg/GEITje-7B,use_accelerate=True,device_map_option=auto,dtype=auto,load_in_8bit=True",
+    "batch_size": 2,
+    "device": "cuda",
+    "no_cache": false,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}