Spaces:
Running
Running
File size: 3,917 Bytes
6dc2db5 d75a844 6dc2db5 d75a844 6dc2db5 d75a844 6dc2db5 d75a844 6dc2db5 d75a844 6dc2db5 d75a844 6dc2db5 d75a844 6dc2db5 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 |
import fire
import json
import pandas as pd
import pickle
def main(
model_info_file: str,
elo_rating_pkl: str,
output_csv: str
):
model_info = json.load(open(model_info_file))
with open(elo_rating_pkl, "rb") as fin:
elo_rating_results = pickle.load(fin)
# Model, Dim Elo rating (anony), Arena Elo rating (anony), Link, Orgnization
model_ratings = model_info
fields = ["key", "Model"]
for dim, dim_results in elo_rating_results.items():
anony_elo_rating_results = dim_results["anony"]
full_elo_rating_results = dim_results["full"]
anony_leaderboard_data = anony_elo_rating_results["leaderboard_table_df"]
full_leaderboard_data = full_elo_rating_results["leaderboard_table_df"]
fields += [f"{dim} Elo rating"]
all_models = anony_leaderboard_data.index.tolist()
for model in all_models:
if not model in model_ratings:
# set Organization and license to empty
model_ratings[model] = {}
model_ratings[model]["Organization"] = "N/A"
model_ratings[model]["Link"] = "N/A"
model_ratings[model]["Model"] = model
model_ratings[model]["key"] = model
if model in anony_leaderboard_data.index:
model_ratings[model][f"{dim} Elo rating"] = anony_leaderboard_data.loc[model, "rating"]
else:
model_ratings[model][f"{dim} Elo rating"] = 0
if "Arena Elo rating" not in model_ratings[model].keys():
model_ratings[model]["Arena Elo rating"] = 0
model_ratings[model]["Arena Elo rating"] += model_ratings[model][f"{dim} Elo rating"]
## Anony
# if model in anony_leaderboard_data.index:
# model_ratings[model][f"{dim} Elo rating (anony)"] = anony_leaderboard_data.loc[model, "rating"]
# else:
# model_ratings[model][f"{dim} Elo rating (anony)"] = 0
# if "Arena Elo rating (anony)" not in model_ratings[model].keys():
# model_ratings[model]["Arena Elo rating (anony)"] = 0
# model_ratings[model]["Arena Elo rating (anony)"] += model_ratings[model][f"{dim} Elo rating (anony)"]
## Anony + Named
# if model in full_elo_rating_results["leaderboard_table_df"].index:
# model_ratings[model][f"{dim} Elo rating (full)"] = full_leaderboard_data.loc[model, "rating"]
# else:
# model_ratings[model][f"{dim} Elo rating (full)"] = 0
# if "Arena Elo rating (full)" not in model_ratings[model].keys():
# model_ratings[model]["Arena Elo rating (full)"] = 0
# model_ratings[model]["Arena Elo rating (full)"] += model_ratings[model][f"{dim} Elo rating (full)"]
fields += ["Arena Elo rating", "Link", "Organization"]
# fields += ["Arena Elo rating (anony)", "Arena Elo rating (full)", "Link", "Organization"]
final_model_info = {}
print(model_ratings)
for model in model_ratings:
if "Model" in model_ratings[model]:
# model_ratings[model]["Arena Elo rating (anony)"] /= 5
# model_ratings[model]["Arena Elo rating (full)"] /= 5
model_ratings[model]["Arena Elo rating"] /= 5
final_model_info[model] = model_ratings[model]
model_info = final_model_info
exclude_keys = ['starting_from']
for key in exclude_keys:
for model in model_info:
if key in model_info[model]:
del model_info[model][key]
df = pd.DataFrame(model_info).T
df = df[fields]
# sort by anony rating
df = df.sort_values(by=["Arena Elo rating"], ascending=False)
df.to_csv(output_csv, index=False)
print("Leaderboard data saved to", output_csv)
print(df)
if __name__ == "__main__":
fire.Fire(main) |