import requests
import pandas as pd
from tqdm.auto import tqdm
from utils import *
import gradio as gr
from huggingface_hub import HfApi, hf_hub_download
from huggingface_hub.repocard import metadata_load
class DeepRL_Leaderboard:
def __init__(self) -> None:
self.leaderboard= {}
def add_leaderboard(self,id=None, title=None):
if id is not None and title is not None:
id = id.strip()
title = title.strip()
self.leaderboard.update({id:{'title':title,'data':get_data_per_env(id)}})
def get_data(self):
return self.leaderboard
def get_ids(self):
return list(self.leaderboard.keys())
# CSS file for the
with open('app.css','r') as f:
BLOCK_CSS = f.read()
LOADED_MODEL_IDS = {}
def get_data(rl_env):
global LOADED_MODEL_IDS
data = []
model_ids = get_model_ids(rl_env)
LOADED_MODEL_IDS[rl_env]=model_ids
for model_id in tqdm(model_ids):
meta = get_metadata(model_id)
if meta is None:
continue
row={}
row["metadata"] = meta
data.append(row)
return pd.DataFrame.from_records(data)
def get_data_per_env(rl_env):
dataframe = get_data(rl_env)
return dataframe,dataframe.empty
rl_leaderboard = DeepRL_Leaderboard()
rl_leaderboard.add_leaderboard('CarRacing-v0'," The Car Racing 🏎️ Leaderboard 🚀")
rl_leaderboard.add_leaderboard('MountainCar-v0',"The Mountain Car ⛰️ 🚗 Leaderboard 🚀")
rl_leaderboard.add_leaderboard('LunarLander-v2',"The Lunar Lander 🌕 Leaderboard 🚀")
rl_leaderboard.add_leaderboard('BipedalWalker-v3',"The BipedalWalker Leaderboard 🚀")
rl_leaderboard.add_leaderboard('Taxi-v3','The Taxi-v3🚖 Leaderboard 🚀')
rl_leaderboard.add_leaderboard('FrozenLake-v1-4x4-no_slippery','The FrozenLake-v1-4x4-no_slippery Leaderboard 🚀')
rl_leaderboard.add_leaderboard('FrozenLake-v1-8x8-no_slippery','The FrozenLake-v1-8x8-no_slippery Leaderboard 🚀')
rl_leaderboard.add_leaderboard('FrozenLake-v1-4x4','The FrozenLake-v1-4x4 Leaderboard 🚀')
rl_leaderboard.add_leaderboard('FrozenLake-v1-8x8','The FrozenLake-v1-8x8 Leaderboard 🚀')
rl_leaderboard.add_leaderboard('SpaceInvadersNoFrameskip-v4','The SpaceInvadersNoFrameskip-v4 Leaderboard 🚀')
RL_ENVS = rl_leaderboard.get_ids()
RL_DETAILS = rl_leaderboard.get_data()
def update_data(rl_env):
global LOADED_MODEL_IDS
data = []
model_ids = [x for x in get_model_ids(rl_env) if x not in LOADED_MODEL_IDS[rl_env]]
LOADED_MODEL_IDS[rl_env]+=model_ids
for model_id in tqdm(model_ids):
meta = get_metadata(model_id)
if meta is None:
continue
row = {}
row["metadata"] = meta
data.append(row)
return pd.DataFrame.from_records(data)
def update_data_per_env(rl_env):
global RL_DETAILS
old_dataframe,_ = RL_DETAILS[rl_env]['data']
new_dataframe = update_data(rl_env)
new_dataframe = new_dataframe.fillna("")
dataframe = pd.concat([old_dataframe,new_dataframe])
return dataframe,dataframe.empty
def get_info_display(dataframe,env_name,name_leaderboard,is_empty):
if not is_empty:
markdown = """
{name_leaderboard}
This is a leaderboard of {len_dataframe} agents, from {num_unique_users} unique users, playing {env_name} 👩🚀.
We use lower bound result to sort the models: mean_reward - std_reward.
You can click on the model's name to be redirected to its model card which includes documentation.
You want to try your model? Read this Unit 1 of Deep Reinforcement Learning Class.
""".format(len_dataframe = len(dataframe),env_name = env_name,name_leaderboard = name_leaderboard,num_unique_users = len(set(dataframe['User'].values)))
else:
markdown = """
{name_leaderboard}
""".format(name_leaderboard = name_leaderboard)
return markdown
def reload_all_data():
global RL_DETAILS,RL_ENVS
for rl_env in RL_ENVS:
RL_DETAILS[rl_env]['data'] = update_data_per_env(rl_env)
html = """
✅ Leaderboard updated! Click `Show Statistics` to see the current statistics.
"""
return html
def reload_leaderboard(rl_env):
global RL_DETAILS
data_dataframe,is_empty = RL_DETAILS[rl_env]['data']
markdown = get_info_display(data_dataframe,rl_env,RL_DETAILS[rl_env]['title'],is_empty)
return markdown
def get_units_stat():
# gets the number of models per unit
units={'Unit 1':[],'Unit 2':[],'Unit 3':[]}
for rl_env in RL_ENVS:
rl_env_metadata,is_empty = RL_DETAILS[rl_env]['data']
if is_empty is False:
# All good! Carry on
metadata_list = rl_env_metadata['metadata'].values
units['Unit 1'].extend([m for m in metadata_list if 'stable-baselines3' in m['tags']])
units['Unit 2'].extend([m for m in metadata_list if 'custom-implementation' in m['tags']])
units['Unit 3'].extend([m for m in metadata_list if 'stable-baselines3' in m['tags'] and 'SpaceInvadersNoFrameskip-v4'.lower() in [tag.lower for tag in m['tags']]])
# get count
for k in units.keys():
units[k] = len(units[k])
return plot_bar(value = list(units.values),name = list(units.keys()),x_name = "Units",y_name = "Number of model submissions",title="Number of model submissions per unit")
def get_models_stat():
# gets the number of models per unit
units={}
for rl_env in RL_ENVS:
rl_env_metadata,is_empty = RL_DETAILS[rl_env]['data']
if is_empty is False:
# All good! Carry on
metadata_list = rl_env_metadata['metadata'].values
units[rl_env] = [m for m in metadata_list]
# get count
for k in units.keys():
units[k] = len(units[k])
return plot_bar(value = list(units.values),name = list(units.keys()),x_name = "RL Environment",y_name = "Number of model submissions",title="Number of model submissions per RL environment")
def get_user_stat():
# gets the number of models per unit
users={}
for rl_env in RL_ENVS:
rl_env_metadata,is_empty = RL_DETAILS[rl_env]['data']
if is_empty is False:
# All good! Carry on
metadata_list = rl_env_metadata['metadata'].values
users[rl_env] = [m['model_id'].split('/')[0] for m in metadata_list]
# get count
for k in users.keys():
users[k] = len(set(users[k]))
return plot_bar(value = list(users.values),name = list(users.keys()),x_name = "RL Environment",y_name = "Number of user submissions",title="Number of user submissions per RL environment")
def get_stat():
# gets the number of models per unit
units={'Unit 1':[],'Unit 2':[],'Unit 3':[]}
users={}
models={}
for rl_env in RL_ENVS:
rl_env_metadata,is_empty = RL_DETAILS[rl_env]['data']
if is_empty is False:
# All good! Carry on
metadata_list = rl_env_metadata['metadata'].values
units['Unit 1'].extend([m for m in metadata_list if 'stable-baselines3' in m['tags']])
units['Unit 2'].extend([m for m in metadata_list if 'custom-implementation' in m['tags']])
units['Unit 3'].extend([m for m in metadata_list if 'stable-baselines3' in m['tags'] and 'spaceinvadersNoFrameskip-v4'.lower() in [tag.lower() for tag in m['tags']]])
users[rl_env] = [m['model_id'].split('/')[0] for m in metadata_list]
models[rl_env] = [m for m in metadata_list]
# get count
for k in units.keys():
units[k] = len(units[k])
for k in users.keys():
users[k] = len(set(users[k]))
for k in models.keys():
models[k] = len(models[k])
units_plot = plot_bar(value = list(units.values()),name = list(units.keys()),x_name = "Units",y_name = "Number of model submissions",title="Number of model submissions per unit")
user_plot = plot_barh(value = list(users.values()),name = list(users.keys()),x_name = "RL Environment",y_name = "Number of unique user submissions",title="Number of unique user submissions per RL environment")
model_plot = plot_barh(value = list(models.values()),name = list(models.keys()),x_name = "RL Environment",y_name = "Number of model submissions",title="Number of model submissions per RL environment")
return units_plot,user_plot,model_plot
block = gr.Blocks(css=BLOCK_CSS)
with block:
notification = gr.HTML("""
⌛ Updating leaderboard...
""")
block.load(reload_all_data,[],[notification])
with gr.Tabs():
with gr.TabItem("Dashboard") as rl_tab:
# Stats of user submission per units
# 2. # model submissions per environment
# 3. # unique users per environment
# get_units_stat()
#data_html,data_dataframe,is_empty = RL_DETAILS[rl_env]['data']
#markdown = get_info_display(data_dataframe,rl_env,RL_DETAILS[rl_env]['title'],is_empty)
#env_state =gr.Variable(default_value=rl_env)
#output_markdown = gr.HTML(markdown)
reload = gr.Button('Show Statistics')
units_plot = gr.Plot(type="matplotlib")
model_plot = gr.Plot(type="matplotlib")
user_plot = gr.Plot(type="matplotlib")
#plot_gender = gr.Plot(type="matplotlib")
#output_html = gr.HTML(data_html)
reload.click(get_stat,[],[units_plot,user_plot,model_plot])
#rl_tab.select(reload_leaderboard,inputs=[env_state],outputs=[output_markdown,output_html])
block.launch()