Added streaming to feedback

api/llm.py

@@ -91,8 +91,9 @@ class LLMManager:
         return chat_history, chat_display, "", code
 
     def end_interview(self, problem_description, chat_history):
+
         if not chat_history or len(chat_history) <= 2:
-            return "No interview content available to review."
+            yield "No interview content available to review."
 
         transcript = [f"{message['role'].capitalize()}: {message['content']}" for message in chat_history[1:]]
 
@@ -100,21 +101,34 @@ class LLMManager:
         if self.is_demo:
             system_prompt += f" Keep your response very short and simple, no more than {self.demo_word_limit} words."
 
-        try:
-            response = self.client.chat.completions.create(
-                model=self.config.llm.name,
-                messages=[
-                    {"role": "system", "content": system_prompt},
-                    {"role": "user", "content": f"The original problem to solve: {problem_description}"},
-                    {"role": "user", "content": "\n\n".join(transcript)},
-                    {"role": "user", "content": "Grade the interview based on the transcript provided and give feedback."},
-                ],
-                temperature=0.5,
-            )
-            if not response.choices:
-                raise APIError("LLM End Interview Error", details="No choices in response")
-            feedback = response.choices[0].message.content.strip()
-        except Exception as e:
-            raise APIError(f"LLM End Interview Error: Unexpected error: {e}")
+        messages = [
+            {"role": "system", "content": system_prompt},
+            {"role": "user", "content": f"The original problem to solve: {problem_description}"},
+            {"role": "user", "content": "\n\n".join(transcript)},
+            {"role": "user", "content": "Grade the interview based on the transcript provided and give feedback."},
+        ]
 
-        return feedback
+        if os.getenv("STREAMING", False):
+            try:
+                response = self.client.chat.completions.create(
+                    model=self.config.llm.name,
+                    messages=messages,
+                    temperature=0.5,
+                    stream=True,
+                )
+            except Exception as e:
+                raise APIError(f"LLM End Interview Error: Unexpected error: {e}")
+
+            feedback = ""
+            for chunk in response:
+                if chunk.choices[0].delta.content:
+                    feedback += chunk.choices[0].delta.content
+                yield feedback
+        #     else:
+        #         response = self.client.chat.completions.create(
+        #             model=self.config.llm.name,
+        #             messages=messages,
+        #             temperature=0.5,
+        #         )
+        #         feedback = response.choices[0].message.content.strip()
+        #         return feedback

app.py

@@ -139,16 +139,16 @@ with gr.Blocks(title="AI Interviewer") as demo:
     coding_tab.select(fn=add_interviewer_message(fixed_messages["intro"]), inputs=[chat, started_coding], outputs=[chat])
 
     start_btn.click(fn=add_interviewer_message(fixed_messages["start"]), inputs=[chat], outputs=[chat]).then(
-        fn=lambda: True, inputs=None, outputs=[started_coding]
+        fn=lambda: True, outputs=[started_coding]
     ).then(
         fn=llm.get_problem,
         inputs=[requirements, difficulty_select, topic_select],
         outputs=[description, chat_history],
         scroll_to_output=True,
     ).then(
-        fn=hide_settings, inputs=None, outputs=[init_acc, start_btn]
+        fn=hide_settings, outputs=[init_acc, start_btn]
     ).then(
-        fn=show_solution, inputs=None, outputs=[solution_acc, end_btn, audio_input]
+        fn=show_solution, outputs=[solution_acc, end_btn, audio_input]
     )
 
     end_btn.click(
@@ -156,11 +156,11 @@ with gr.Blocks(title="AI Interviewer") as demo:
         inputs=[chat],
         outputs=[chat],
     ).then(
-        fn=llm.end_interview, inputs=[description, chat_history], outputs=feedback
-    ).then(fn=hide_solution, inputs=None, outputs=[solution_acc, end_btn, problem_acc, audio_input])
+        fn=hide_solution, outputs=[solution_acc, end_btn, problem_acc, audio_input]
+    ).then(fn=llm.end_interview, inputs=[description, chat_history], outputs=[feedback])
 
     audio_input.stop_recording(fn=stt.speech_to_text, inputs=[audio_input], outputs=[message]).then(
-        fn=lambda: None, inputs=None, outputs=[audio_input]
+        fn=lambda: None, outputs=[audio_input]
     ).then(fn=add_candidate_message, inputs=[message, chat], outputs=[chat]).then(
         fn=llm.send_request,
         inputs=[code, previous_code, message, chat_history, chat],