-
Notifications
You must be signed in to change notification settings - Fork 8
/
Copy pathevaluation.py
79 lines (55 loc) · 3.45 KB
/
evaluation.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
import os
import sys
sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
import argparse
import pandas as pd
from sklearn.metrics import cohen_kappa_score
from utils.file_utils import load_json
def calculate_average_score(data, start, end):
return sum(k for i, k in data.items() if start <= i <= end) / (end - start + 1)
def calculate_average_authenticity(file_path, method):
df = pd.json_normalize(load_json(file_path), record_path=['questionnaire'], meta=['iteration'], record_prefix='questionnaire_')
authenticity_avg_per_iteration = {}
for iteration, group in df.groupby('iteration'):
human_ratings = group['questionnaire_answer.human_rating'].tolist()
method_ratings = group[f'questionnaire_answer.{method}.rating'].tolist()
authenticity_avg_per_iteration[iteration] = cohen_kappa_score(human_ratings, method_ratings)
average_kappa = calculate_average_score(authenticity_avg_per_iteration, 1, 10)
kappa_iteration_5 = authenticity_avg_per_iteration.get(5, "No data for 5th iteration")
kappa_iteration_10 = authenticity_avg_per_iteration.get(10, "No data for 10th iteration")
print(f'======= {method} Authenticity =======')
# print(f'0th iteration authenticity: {rationality_avg_per_iteration[0]}')
print(f'Average authenticity: {average_kappa}')
print(f'5th iteration authenticity: {kappa_iteration_5}')
print(f'10th iteration authenticity: {kappa_iteration_10}')
return authenticity_avg_per_iteration
def calculate_average_rationality(file_path, method):
df = pd.json_normalize(load_json(file_path), record_path=['questionnaire'], meta=['iteration'], record_prefix='questionnaire_')
rationality_avg_per_iteration = {}
for iteration, group in df.groupby('iteration'):
method_rationality = group[f'questionnaire_answer.{method}.rationality'].tolist()
rationality_avg_per_iteration[int(iteration)] = sum(method_rationality) / len(method_rationality)
average_rationality = calculate_average_score(rationality_avg_per_iteration, 1, 10)
rationality_iteration_5 = rationality_avg_per_iteration.get(5, "No data for 5th iteration")
rationality_iteration_10 = rationality_avg_per_iteration.get(10, "No data for 10th iteration")
print(f'======= {method} Rationality =======')
# print(f'0th iteration rationality: {rationality_avg_per_iteration[0]}')
print(f'Average rationality: {average_rationality}')
print(f'5th iteration rationality: {rationality_iteration_5}')
print(f'10th iteration rationality: {rationality_iteration_10}')
return rationality_avg_per_iteration
def add_args():
parser = argparse.ArgumentParser()
parser.add_argument("--file_path", type=str, default=os.path.join('data', 'datasets', 'huggingface', 'english', 'eval_a.json'), help="Path of the file.")
parser.add_argument("--method", type=str, default="CoT", help="Method to calculate metrics.")
parser.add_argument("--authenticity", action='store_true', default=True, help="Whether to calculate the authenticity metric.")
parser.add_argument("--rationality", action='store_true', default=True, help="Whether to calculate the rationality metric.")
return parser.parse_args()
def main():
args = add_args()
if args.authenticity:
calculate_average_authenticity(args.file_path, args.method)
if args.rationality:
calculate_average_rationality(args.file_path, args.method)
if __name__ == '__main__':
sys.exit(main())