Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Adithyare/peft metric calculation #7304

Merged
merged 4 commits into from
Aug 25, 2023
Merged

Adithyare/peft metric calculation #7304

Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
bda5bea
all purpose metric calculation script, computes f1/accuracy/rouge
arendu Aug 23, 2023
35d2607
removed squad metric calc script
arendu Aug 23, 2023
6ef7345
[pre-commit.ci] auto fixes from pre-commit.com hooks
pre-commit-ci[bot] Aug 23, 2023
d2d377e
Merge branch 'main' into adithyare/peft_metric_calculation
arendu Aug 24, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 12 additions & 6 deletions ...s/metric_calculation/squad_metric_calc.py → ...ts/metric_calculation/peft_metric_calc.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
import re
import string
from collections import Counter
from rouge_score import rouge_scorer


"""
Expand Down Expand Up @@ -117,31 +118,36 @@ def main():

ground_truth_file = args.ground_truth
pred_file = args.preds

scorer = rouge_scorer.RougeScorer(['rougeL'], use_stemmer=True)
preds = open(pred_file, encoding="utf-8").readlines()
ground_truth = open(ground_truth_file).readlines()
f1 = exact_match = total = 0
f1 = exact_match = total = r_score = 0

for i in range(len(preds)):
truth = json.loads(ground_truth[i])
pred_answer = preds[i]
pred_answer = json.loads(preds[i])

# Need to separate out preditions from prompt, spliting on the provided "split string"
if args.split_string is not None:
pred_answer = pred_answer.split(args.split_string)[-1].strip()
pred_answer = pred_answer["sentence"].split(args.split_string)[-1].strip()

true_answers = truth[args.answer_field]
if not isinstance(true_answers, list):
true_answers = [true_answers]

r_scores = []
for ta in true_answers:
r_scores.append(scorer.score(ta, pred_answer)['rougeL'].fmeasure)
r_score += max(r_scores)
exact_match += metric_max_over_ground_truths(exact_match_score, pred_answer, true_answers)
f1 += metric_max_over_ground_truths(f1_score, pred_answer, true_answers)
total += 1

exact_match = 100.0 * exact_match / total
f1 = 100.0 * f1 / total

print({'exact_match': exact_match, 'f1': f1, 'total': total})
r_score = 100 * (r_score / total)
res = {'exact_match': exact_match, 'f1': f1, "rougeL": r_score, 'total': total}
print('\t'.join([f"{k} {v:.3f}" for k, v in res.items()]))


if __name__ == "__main__":
Expand Down