You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
39 lines
1.4 KiB
Python
39 lines
1.4 KiB
Python
import openai
|
|
from scoring.llm_scorer import extract_score, LanguageModelScorer
|
|
import time
|
|
|
|
|
|
def request_gpt(prompt, retries=3):
|
|
def ordinal(n):
|
|
return str(n) + {1: "st", 2: "nd", 3: "rd"}.get(10 <= n % 100 <= 20 and n or n % 10, "th")
|
|
|
|
for i in range(retries):
|
|
try:
|
|
response = openai.ChatCompletion.create(
|
|
model="gpt-3.5-turbo",
|
|
messages=prompt,
|
|
)
|
|
return response.choices[0]['message']['content']
|
|
except Exception as e:
|
|
print(f"\nAn error occurred while scoring with ChatGPT: {e}, it's the {ordinal(i + 1)} time.")
|
|
time.sleep(1)
|
|
print("Failed to get response from ChatGPT. Use default score.")
|
|
return None
|
|
|
|
|
|
class GPTScorer(LanguageModelScorer):
|
|
def __init__(self, api_key):
|
|
super().__init__()
|
|
self.api_key = api_key
|
|
openai.api_key = api_key
|
|
|
|
def score_with_llm(self, question, model_result, reference, origin_model_result=None):
|
|
prompt = self.generate_scoring_prompt(question, model_result, reference, origin_model_result)
|
|
try:
|
|
chatgpt_response = request_gpt(prompt, retries=5)
|
|
chatgpt_score = extract_score(chatgpt_response)
|
|
return chatgpt_response, chatgpt_score
|
|
except Exception as e:
|
|
print("\nAn error occurred while extract score:", e)
|
|
return None, '2'
|