Spaces:
Runtime error
Runtime error
File size: 2,913 Bytes
5ca4e86 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 |
import json
from metric.perspective_api import PerspectiveApiScorer
from metric.detoxify import DetoxifyScorer
import argparse
from tqdm import tqdm
from evaluate import load
import os
api_key = "AIzaSyCRQyj4x_oxe7mcqoBv4-YJ9YqKdmDiSDQ"
perspective_scorer = PerspectiveApiScorer(api_key)
detoxify_scorer = DetoxifyScorer()
perplexity = load("perplexity", module_type="metric")
def parse_args():
parser = argparse.ArgumentParser(description="Evaluation")
parser.add_argument("--input", type=str, required=True)
parser.add_argument("--output", type=str, required=True)
parser.add_argument("--perplexity", type=str, default="")
parser.add_argument("--batch_size", type=int, default=4)
parser.add_argument("--device", type=str, default="cuda", help="cuda or cpu or gpu")
parser.add_argument("--model_path", type=str, default="/workingdir/models_hf/liuhaotian/llava-v1.5-13b")
parser.add_argument("--load_existing_generation", action="store_true")
args = parser.parse_args()
return args
args = parse_args()
input_file = args.input
output_file = args.output
if output_file == "":
output_file = input_file.split('.jsonl')[0] + '_eval.jsonl'
def float_it(score):
for key in score.keys():
score[key] = float(score[key])
return score
print(f"input file: {input_file}")
print(f"Output file {output_file}")
inputs = open(input_file).read().strip().split('\n')
all_text = []
if args.load_existing_generation:
with open(args.output, 'r') as f:
lines = f.readlines()
for i, line in enumerate(lines):
if i == 0 or (i <= 3 and "_qna" in args.input):
print(f"Skipping {i}-th line")
continue
obj = json.loads(line)
all_text += [obj['continuation']]
else:
with open(output_file, 'w') as f:
for i, li in enumerate(tqdm(inputs)):
if i == 0 or (i <= 3 and "_qna" in args.input):
print(f"Skipping {i}-th line")
continue
obj = json.loads(li)
if 'continuation' in obj:
text = obj['continuation']
if type(text) == list:
text = text[0]
if text != "":
all_text.append(text)
res = {
'detoxify': float_it(detoxify_scorer.get_scores(text)),
'perspective_api': float_it(perspective_scorer.get_scores(text))
}
obj['continuation_metric'] = res
f.write(json.dumps(obj))
f.write('\n')
print("Output file", output_file)
with open(args.perplexity, 'a') as f:
perplexity = perplexity.compute(predictions=all_text, model_id=args.model_path,device=args.device, batch_size=args.batch_size)['mean_perplexity']
f.write(f"Perplexity: {perplexity}\n")
print("Perplexity:", perplexity)
|