Added retries for low rate limits models

tests/analysis.py

@@ -80,9 +80,23 @@ def complete_and_grade(interview_params, exp_name, grader_models, candidate_mode
     interview_type, attempt_num, llm_config = interview_params
 
     feedback_list = []
+    attempt_successful = False
+    for attempt in range(3):  # Retry up to 3 times
+        try:
+            file_path, _ = complete_interview(interview_type, exp_name, llm_config, model=candidate_model, pause=attempt * 5)
+            print(
+                f"Attempt {attempt_num + 1}, retry {attempt + 1} interview simulation of {interview_type} by {llm_config.name} completed successfully"
+            )
+            attempt_successful = True
+            break
+        except Exception as e:
+            print(f"Retry {attempt + 1} for attempt {attempt_num + 1} of {interview_type} by {llm_config.name} failed with error: {e}")
+
+    if not attempt_successful:
+        print(f"All retries failed for attempt {attempt_num + 1} of {interview_type} by {llm_config.name}")
+        return feedback_list
+
     try:
-        file_path, _ = complete_interview(interview_type, exp_name, llm_config, model=candidate_model)
-        print(f"Attempt {attempt_num + 1} interview simulation of {interview_type} by {llm_config.name} completed successfully")
         for i, grader_model in enumerate(grader_models):
             feedback = grade_attempt(file_path, grader_model, i)
             if feedback:
@@ -91,7 +105,7 @@ def complete_and_grade(interview_params, exp_name, grader_models, candidate_mode
                 print(f"Overall score: {feedback['overall_score']}")
 
     except Exception as e:
-        print(f"Attempt {attempt_num + 1} of {interview_type} by {llm_config.name} failed with error: {e}")
+        print(f"Grading for attempt {attempt_num + 1} of {interview_type} by {llm_config.name} failed with error: {e}")
 
     if len(feedback_list) == 0:
         print(f"Attempt {attempt_num + 1} of {interview_type} by {llm_config.name} returned an empty list")

tests/candidate.py

@@ -15,7 +15,7 @@ from resources.prompts import prompts
 from tests.testing_prompts import candidate_prompt
 
 
-def complete_interview(interview_type, exp_name, llm_config=None, requirements="", difficulty="", topic="", model="gpt-3.5-turbo"):
+def complete_interview(interview_type, exp_name, llm_config=None, requirements="", difficulty="", topic="", model="gpt-3.5-turbo", pause=0):
     client = OpenAI(base_url="https://api.openai.com/v1")
     config = Config()
     if llm_config:
@@ -98,6 +98,8 @@ def complete_interview(interview_type, exp_name, llm_config=None, requirements="
         if len(message_split) > 1:
             interview_data["transcript"].append(f"INTERVIEWER HIDDEN NOTE: {message_split[1]}")
 
+        time.sleep(pause)  # to prevent exceeding rate limits
+
     interview_data["feedback"] = llm.end_interview_full(problem_statement_text, messages_interviewer, interview_type)
     interview_data["average_response_time_seconds"] = round(sum(response_times) / len(response_times), 2) if response_times else 0