from scoring.gpt_scorer import GPTScorer from scoring.rogue_scorer import get_rouge_score import pandas as pd import time from tqdm import tqdm class AssessmentEngine: def __init__(self, save_result_dir, api_key): self.save_result_dir = save_result_dir self.gpt_scorer = GPTScorer(api_key) def eval_subject(self, subject_name, csv_file_name): qa_result_df = pd.read_csv(self.save_result_dir + '/' + csv_file_name) start_time = time.time() row_count = 0 rouge_score_sum = 0 for row_index, row in tqdm(qa_result_df.iterrows(), total=len(qa_result_df)): row_count += 1 test_question = row['question'] model_response = row['model_output'] reference_answer = row['answer'] rouge_score = get_rouge_score(model_response, reference_answer) rouge_1_f_score = rouge_score['rouge-1']['f'] rouge_score_sum += rouge_1_f_score qa_result_df.loc[row_index, 'rouge_score'] = rouge_1_f_score self.gpt_scorer.mode("accuracy") gpt_response_acc, gpt_score_acc = self.gpt_scorer.score_with_chatgpt(test_question, model_response, reference_answer) qa_result_df.loc[row_index, 'gpt_score_acc'] = gpt_score_acc qa_result_df.loc[row_index, 'gpt_response_acc'] = gpt_response_acc end_time = time.time() elapsed_time = end_time - start_time print("共评估结果" + str(row_count) + "条，总共用时：", elapsed_time, "秒") synthesis_score = rouge_score_sum / row_count qa_result_df.to_csv(self.save_result_dir + '/' + subject_name + '_qa_test_score_' + str(synthesis_score) + '.csv', index=False) def eval_result_diff(self, csv_file_name, file_type='csv'): if file_type == 'json': result_diff_df = pd.read_json(self.save_result_dir + '/' + csv_file_name) elif file_type == 'csv': result_diff_df = pd.read_csv(self.save_result_dir + '/' + csv_file_name) else: print("Unknown file type:" + file_type) return result_diff_df['rouge_score_finetune'] = '0' result_diff_df['rouge_score_origin'] = '0' result_diff_df['acc_finetune'] = '0' result_diff_df['acc_origin'] = '0' result_diff_df['fluency_finetune'] = '0' result_diff_df['fluency_origin'] = '0' result_diff_df['diff_score'] = '0' result_diff_df['acc_response_finetune'] = '0' result_diff_df['acc_response_origin'] = '0' result_diff_df['fluency_response_finetune'] = '0' result_diff_df['fluency_response_origin'] = '0' result_diff_df['diff_score_response'] = '0' start_time = time.time() finetune_rouge_score_sum = 0 origin_rouge_score_sum = 0 finetune_acc_score_sum = 0 origin_acc_score_sum = 0 finetune_fluency_score_sum = 0 origin_fluency_score_sum = 0 model_better_score_sum = 0 row_count = 0 for row_index, row in tqdm(result_diff_df.iterrows(), total=len(result_diff_df)): if row['question'] == '': continue row_count += 1 test_question = row['question'] finetune_model_response = row['predict_finetune'] original_model_response = row['predict_origin'] reference_answer = row['answer'] # 计算ROUGE分数 finetune_rouge_score = get_rouge_score(finetune_model_response, reference_answer) finetune_rouge_1_f_score = finetune_rouge_score['rouge-1']['f'] finetune_rouge_score_sum += finetune_rouge_1_f_score result_diff_df.loc[row_index, 'rouge_score_finetune'] = finetune_rouge_1_f_score origin_rouge_score = get_rouge_score(original_model_response, reference_answer) origin_rouge_1_f_score = origin_rouge_score['rouge-1']['f'] origin_rouge_score_sum += origin_rouge_1_f_score result_diff_df.loc[row_index, 'rouge_score_origin'] = origin_rouge_1_f_score self.gpt_scorer.mode("accuracy") gpt_response_acc, gpt_score_acc = (self.gpt_scorer.score_with_chatgpt(test_question, finetune_model_response, reference_answer)) result_diff_df.loc[row_index, 'acc_finetune'] = gpt_score_acc result_diff_df.loc[row_index, 'acc_response_finetune'] = gpt_response_acc if (gpt_score_acc is not None) and gpt_score_acc.isdigit(): finetune_acc_score_sum += float(gpt_score_acc) gpt_response_acc, gpt_score_acc = (self.gpt_scorer.score_with_chatgpt(test_question, original_model_response, reference_answer)) result_diff_df.loc[row_index, 'acc_origin'] = gpt_score_acc result_diff_df.loc[row_index, 'acc_response_origin'] = gpt_response_acc if (gpt_score_acc is not None) and gpt_score_acc.isdigit(): origin_acc_score_sum += float(gpt_score_acc) self.gpt_scorer.mode("fluency") gpt_response_fluency, gpt_score_fluency = (self.gpt_scorer.score_with_chatgpt(test_question, finetune_model_response, reference_answer)) result_diff_df.loc[row_index, 'fluency_finetune'] = gpt_score_fluency result_diff_df.loc[row_index, 'fluency_response_finetune'] = gpt_response_fluency if (gpt_score_fluency is not None) and gpt_score_fluency.isdigit(): finetune_fluency_score_sum += float(gpt_score_fluency) gpt_response_fluency, gpt_score_fluency = (self.gpt_scorer.score_with_chatgpt(test_question, original_model_response, reference_answer)) result_diff_df.loc[row_index, 'fluency_origin'] = gpt_score_fluency result_diff_df.loc[row_index, 'fluency_response_origin'] = gpt_response_fluency if (gpt_score_fluency is not None) and gpt_score_fluency.isdigit(): origin_fluency_score_sum += float(gpt_score_fluency) self.gpt_scorer.mode("diff") gpt_response_diff, gpt_score_diff = (self.gpt_scorer.score_with_chatgpt(test_question, finetune_model_response, reference_answer, original_model_response)) result_diff_df.loc[row_index, 'diff_score'] = gpt_score_diff result_diff_df.loc[row_index, 'diff_score_response'] = gpt_response_diff if (gpt_score_diff is not None) and gpt_score_diff.isdigit(): model_better_score_sum += float(gpt_score_diff) result_diff_df.to_csv(self.save_result_dir + '/result_diff_test_score_tmp.csv', index=False) end_time = time.time() elapsed_time = end_time - start_time print("共评估结果" + str(row_count) + "条，总共用时：", elapsed_time, "秒") synthesis_rouge_score = finetune_rouge_score_sum / row_count original_rouge_score = origin_rouge_score_sum / row_count synthesis_acc_score = finetune_acc_score_sum / row_count original_acc_score = origin_acc_score_sum / row_count synthesis_fluency_score = finetune_fluency_score_sum / row_count original_fluency_score = origin_fluency_score_sum / row_count synthesis_diff_score = model_better_score_sum / row_count print("微调模型ROUGE分数：", synthesis_rouge_score) print("原模型ROUGE分数：", original_rouge_score) print("微调模型准确性分数：", synthesis_acc_score) print("原模型准确性分数：", original_acc_score) print("微调模型流畅度分数：", synthesis_fluency_score) print("原模型流畅度分数：", original_fluency_score) print("微调模型优于原模型分数：", synthesis_diff_score) synthesis_score = (synthesis_rouge_score * 100 + synthesis_acc_score * 100 / 4 + synthesis_fluency_score * 100 / 3 + synthesis_diff_score * 100 / 3) / 4 print("综合评分：", synthesis_score) original_synthesis_score = (original_rouge_score * 100 + original_acc_score * 100 / 4 + original_fluency_score * 100 / 3 + 66) / 4 print("原模型综合评分：", original_synthesis_score) # 获取当前时间的字符串 current_time = time.strftime("%Y%m%d%H%M%S", time.localtime(time.time())) result_diff_df.to_csv(self.save_result_dir + '/' + current_time + '_result_diff_test_score_' + str(synthesis_score) + '.csv', index=False) def eval_result(self, file_name, file_type='csv'): start_time = time.time() if file_type == 'json': result_df = pd.read_json(self.save_result_dir + '/' + file_name) elif file_type == 'csv': result_df = pd.read_csv(self.save_result_dir + '/' + file_name) else: print("Unsupported file type:" + file_type) return result_df['rouge_score_finetune'] = '0' result_df['acc_finetune'] = '0' result_df['fluency_finetune'] = '0' result_df['acc_response_finetune'] = '0' result_df['fluency_response_finetune'] = '0' rouge_score_sum = 0 acc_score_sum = 0 fluency_score_sum = 0 row_count = 0 for row_index, row in tqdm(result_df.iterrows(), total=len(result_df)): row_count += 1 test_question = row['question'] model_response = row['Predict'] reference_answer = row['answer'] rouge_score = get_rouge_score(model_response, reference_answer) rouge_1_f_score = rouge_score['rouge-1']['f'] rouge_score_sum += rouge_1_f_score result_df.loc[row_index, 'rouge_score_finetune'] = rouge_1_f_score self.gpt_scorer.mode("accuracy") gpt_response_acc, gpt_score_acc = self.gpt_scorer.score_with_chatgpt(test_question, model_response, reference_answer) result_df.loc[row_index, 'acc_finetune'] = gpt_score_acc result_df.loc[row_index, 'acc_response_finetune'] = gpt_response_acc if (gpt_score_acc is not None) and gpt_score_acc.isdigit(): acc_score_sum += float(gpt_score_acc) self.gpt_scorer.mode("fluency") gpt_response_fluency, gpt_score_fluency = self.gpt_scorer.score_with_chatgpt(test_question, model_response, reference_answer) result_df.loc[row_index, 'fluency_finetune'] = gpt_score_fluency result_df.loc[row_index, 'fluency_response_finetune'] = gpt_response_fluency if (gpt_score_fluency is not None) and gpt_score_fluency.isdigit(): fluency_score_sum += float(gpt_score_fluency) result_df.to_csv(self.save_result_dir + '/result_test_score_tmp.csv', index=False) end_time = time.time() elapsed_time = end_time - start_time print("共评估结果" + str(row_count) + "条，总共用时：", elapsed_time, "秒") rouge_score = rouge_score_sum / row_count acc_score = acc_score_sum / row_count fluency_score = fluency_score_sum / row_count print("ROUGE分数：", rouge_score) print("准确性分数：", acc_score) print("流畅度分数：", fluency_score) synthesis_score = (rouge_score * 100 + acc_score * 100 / 4 + fluency_score * 100 / 3 + 66) / 4 print("综合评分：", synthesis_score) result_df.to_csv(self.save_result_dir + f'/result_test_score_{synthesis_score}.csv', index=False)