meg-huggingface commited on
Commit
72d2b05
1 Parent(s): c9946e0

Switching to normalized task name.

Browse files
Files changed (1) hide show
  1. app.py +28 -32
app.py CHANGED
@@ -9,7 +9,7 @@ from huggingface_hub import HfApi, snapshot_download, ModelInfo, list_models
9
  from enum import Enum
10
 
11
 
12
- OWNER = "EnergyStarAI"
13
  COMPUTE_SPACE = f"{OWNER}/launch-computation-example"
14
 
15
 
@@ -22,7 +22,6 @@ task_mappings = {'automatic speech recognition':'automatic-speech-recognition',
22
  'Image to Text':'image-to-text', 'Question Answering':'question-answering', 'Text Generation': 'text-generation',
23
  'Image Classification':'image-classification', 'Sentence Similarity': 'sentence-similarity',
24
  'Image Generation':'image-generation', 'Summarization':'summarization'}
25
-
26
  @dataclass
27
  class ModelDetails:
28
  name: str
@@ -47,29 +46,25 @@ def add_docker_eval(zip_file):
47
  new_fid = new_fid_list[-1]
48
  if new_fid.endswith('.zip'):
49
  API.upload_file(
50
- path_or_fileobj=zip_file ,
51
- repo_id="EnergyStarAI/tested_proprietary_models",
52
  path_in_repo='submitted_models/'+new_fid,
53
  repo_type="dataset",
54
  commit_message="Adding logs via submission Space.",
55
- token= TOKEN
56
  )
57
  gr.Info('Uploaded logs to dataset! We will validate their validity and add them to the next version of the leaderboard.')
58
  else:
59
  gr.Info('You can only upload .zip files here!')
60
 
61
 
62
- def add_new_eval(
63
- repo_id: str,
64
- task: str,
65
- ):
66
  model_owner = repo_id.split("/")[0]
67
- model_name = repo_id.split("/")[1]
68
- model_list=[]
69
  current_time = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
70
- requests= load_dataset("EnergyStarAI/requests_debug", split="test", token=TOKEN)
71
  requests_dset = requests.to_pandas()
72
- model_list= requests_dset[requests_dset['status'] == 'COMPLETED']['model'].tolist()
73
  task_models = list(API.list_models(filter=task_mappings[task]))
74
  task_model_names = [m.id for m in task_models]
75
  if repo_id in model_list:
@@ -80,20 +75,21 @@ def add_new_eval(
80
  # Is the model info correctly filled?
81
  try:
82
  model_info = API.model_info(repo_id=repo_id)
 
 
83
  except Exception:
84
- gr.Info("Could not find information for model %s" % (model))
85
-
86
- model_size = get_model_size(model_info=model_info)
87
-
88
  gr.Info("Adding request")
89
-
90
-
91
  request_dict = {
92
  "model": repo_id,
93
  "status": "PENDING",
94
  "submitted_time": pd.to_datetime(current_time),
95
- "task": task,
96
- "likes": model_info.likes,
97
  "params": model_size,
98
  "leaderboard_version": "v0",}
99
  #"license": license,
@@ -104,17 +100,17 @@ def add_new_eval(
104
  df_request_dict = pd.DataFrame([request_dict])
105
  print(df_request_dict)
106
  df_final = pd.concat([requests_dset, df_request_dict], ignore_index=True)
107
- updated_dset =Dataset.from_pandas(df_final)
108
- updated_dset.push_to_hub("EnergyStarAI/requests_debug", split="test", token=TOKEN)
109
 
110
  gr.Info("Starting compute space at %s " % COMPUTE_SPACE)
111
  return start_compute_space()
112
 
113
 
114
  def print_existing_models():
115
- requests= load_dataset("EnergyStarAI/requests_debug", split="test", token=TOKEN)
116
  requests_dset = requests.to_pandas()
117
- model_df= requests_dset[['model','status']]
118
  model_df = model_df[model_df['status'] == 'COMPLETED']
119
  return model_df
120
 
@@ -127,7 +123,7 @@ def highlight_cols(x):
127
 
128
  # Applying the style function
129
  existing_models = print_existing_models()
130
- formatted_df = existing_models.style.apply(highlight_cols, axis = None)
131
 
132
  def get_leaderboard_models():
133
  path = r'leaderboard_v0_data/energy'
@@ -148,9 +144,9 @@ with gr.Blocks() as demo:
148
  with gr.Row():
149
  with gr.Column():
150
  task = gr.Dropdown(
151
- choices=task_mappings.keys(),
152
  label="Choose a benchmark task",
153
- value = 'Text Generation',
154
  multiselect=False,
155
  interactive=True,
156
  )
@@ -171,15 +167,15 @@ with gr.Blocks() as demo:
171
  )
172
  with gr.Row():
173
  with gr.Column():
174
- with gr.Accordion("Submit log files from a Docker run:", open = False):
175
  gr.Markdown("If you've already benchmarked your model using the [Docker file](https://github.com/huggingface/EnergyStarAI/) provided, please upload the **entire run log directory** (in .zip format) below:")
176
  file_output = gr.File(visible=False)
177
  u = gr.UploadButton("Upload a zip file with logs", file_count="single")
178
- u.upload(add_docker_eval,u, file_output)
179
  with gr.Row():
180
  with gr.Column():
181
- with gr.Accordion("Models that are in the latest leaderboard version:", open = False):
182
  gr.Dataframe(get_leaderboard_models())
183
- with gr.Accordion("Models that have been benchmarked recently:", open = False):
184
  gr.Dataframe(formatted_df)
185
  demo.launch()
 
9
  from enum import Enum
10
 
11
 
12
+ OWNER = "AIEnergyScore"
13
  COMPUTE_SPACE = f"{OWNER}/launch-computation-example"
14
 
15
 
 
22
  'Image to Text':'image-to-text', 'Question Answering':'question-answering', 'Text Generation': 'text-generation',
23
  'Image Classification':'image-classification', 'Sentence Similarity': 'sentence-similarity',
24
  'Image Generation':'image-generation', 'Summarization':'summarization'}
 
25
  @dataclass
26
  class ModelDetails:
27
  name: str
 
46
  new_fid = new_fid_list[-1]
47
  if new_fid.endswith('.zip'):
48
  API.upload_file(
49
+ path_or_fileobj=zip_file,
50
+ repo_id="AIEnergyScore/tested_proprietary_models",
51
  path_in_repo='submitted_models/'+new_fid,
52
  repo_type="dataset",
53
  commit_message="Adding logs via submission Space.",
54
+ token=TOKEN
55
  )
56
  gr.Info('Uploaded logs to dataset! We will validate their validity and add them to the next version of the leaderboard.')
57
  else:
58
  gr.Info('You can only upload .zip files here!')
59
 
60
 
61
+ def add_new_eval(repo_id: str, task: str):
 
 
 
62
  model_owner = repo_id.split("/")[0]
63
+ model_name = repo_id.split("/")[1]
 
64
  current_time = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
65
+ requests = load_dataset("AIEnergyScore/requests_debug", split="test", token=TOKEN)
66
  requests_dset = requests.to_pandas()
67
+ model_list = requests_dset[requests_dset['status'] == 'COMPLETED']['model'].tolist()
68
  task_models = list(API.list_models(filter=task_mappings[task]))
69
  task_model_names = [m.id for m in task_models]
70
  if repo_id in model_list:
 
75
  # Is the model info correctly filled?
76
  try:
77
  model_info = API.model_info(repo_id=repo_id)
78
+ model_size = get_model_size(model_info=model_info)
79
+ likes = model_info.likes
80
  except Exception:
81
+ gr.Info("Could not find information for model %s" % (model_name))
82
+ model_size = None
83
+ likes = None
84
+
85
  gr.Info("Adding request")
86
+
 
87
  request_dict = {
88
  "model": repo_id,
89
  "status": "PENDING",
90
  "submitted_time": pd.to_datetime(current_time),
91
+ "task": task_mappings[task],
92
+ "likes": likes,
93
  "params": model_size,
94
  "leaderboard_version": "v0",}
95
  #"license": license,
 
100
  df_request_dict = pd.DataFrame([request_dict])
101
  print(df_request_dict)
102
  df_final = pd.concat([requests_dset, df_request_dict], ignore_index=True)
103
+ updated_dset = Dataset.from_pandas(df_final)
104
+ updated_dset.push_to_hub("AIEnergyScore/requests_debug", split="test", token=TOKEN)
105
 
106
  gr.Info("Starting compute space at %s " % COMPUTE_SPACE)
107
  return start_compute_space()
108
 
109
 
110
  def print_existing_models():
111
+ requests= load_dataset("AIEnergyScore/requests_debug", split="test", token=TOKEN)
112
  requests_dset = requests.to_pandas()
113
+ model_df= requests_dset[['model', 'status']]
114
  model_df = model_df[model_df['status'] == 'COMPLETED']
115
  return model_df
116
 
 
123
 
124
  # Applying the style function
125
  existing_models = print_existing_models()
126
+ formatted_df = existing_models.style.apply(highlight_cols, axis=None)
127
 
128
  def get_leaderboard_models():
129
  path = r'leaderboard_v0_data/energy'
 
144
  with gr.Row():
145
  with gr.Column():
146
  task = gr.Dropdown(
147
+ choices=list(task_mappings.keys()),
148
  label="Choose a benchmark task",
149
+ value='Text Generation',
150
  multiselect=False,
151
  interactive=True,
152
  )
 
167
  )
168
  with gr.Row():
169
  with gr.Column():
170
+ with gr.Accordion("Submit log files from a Docker run:", open=False):
171
  gr.Markdown("If you've already benchmarked your model using the [Docker file](https://github.com/huggingface/EnergyStarAI/) provided, please upload the **entire run log directory** (in .zip format) below:")
172
  file_output = gr.File(visible=False)
173
  u = gr.UploadButton("Upload a zip file with logs", file_count="single")
174
+ u.upload(add_docker_eval, u, file_output)
175
  with gr.Row():
176
  with gr.Column():
177
+ with gr.Accordion("Models that are in the latest leaderboard version:", open=False):
178
  gr.Dataframe(get_leaderboard_models())
179
+ with gr.Accordion("Models that have been benchmarked recently:", open=False):
180
  gr.Dataframe(formatted_df)
181
  demo.launch()