Add UI changes

app.py

@@ -45,9 +45,10 @@ model = models.load_fromage('./', args_path, ckpt_path)
 def upload_image(state, image_input):
     conversation = state[0]
     chat_history = state[1]
-    conversation += [(f"![](/file={image_input.name})", "")]
     input_image = Image.open(image_input.name).resize(
         (224, 224)).convert('RGB')
+    input_image.save(image_input.name)  # Overwrite with smaller image.
+    conversation += [(f"![](/file={image_input.name})", "")]
     return [conversation, chat_history, input_image], conversation
 
 
@@ -69,7 +70,11 @@ def save_image_to_local(image: Image.Image):
     return filename
 
 
-def generate_for_prompt(input_text, state, ret_scale_factor, max_nm_rets, num_words, temperature):
+def generate_for_prompt(input_text, state, ret_scale_factor, max_num_rets, num_words, temperature):
+    # Ignore empty inputs.
+    if len(input_text) == 0:
+        return state, state[0], gr.update(visible=True)
+
     input_prompt = 'Q: ' + input_text + '\nA:'
     conversation = state[0]
     chat_history = state[1]
@@ -93,7 +98,7 @@ def generate_for_prompt(input_text, state, ret_scale_factor, max_nm_rets, num_wo
           model_inputs, flush=True)
     model_outputs = model.generate_for_images_and_texts(model_inputs,
                                                         num_words=max(num_words, 1), ret_scale_factor=ret_scale_factor, top_p=top_p,
-                                                        temperature=temperature, max_num_rets=max_nm_rets)
+                                                        temperature=temperature, max_num_rets=max_num_rets)
     print('model_outputs', model_outputs, flush=True)
 
     im_names = []
@@ -104,12 +109,16 @@ def generate_for_prompt(input_text, state, ret_scale_factor, max_nm_rets, num_wo
             text_outputs.append(output)
             response += output
         elif type(output) == list:
+            response += '<br/>'  # Add line break between images.
             for image in output:
                 filename = save_image_to_local(image)
-                response += f'<img src="/file={filename}">'
+                response += f'<img src="/file={filename}" style="display: inline-block;">'
+            response += '<br/>'
         elif type(output) == Image.Image:
             filename = save_image_to_local(output)
-            response += f'<img src="/file={filename}">'
+            response += '<br/>'
+            response += f'<img src="/file={filename}" style="display: inline-block;">'
+            response += '<br/>'
 
     # TODO(jykoh): Persist image inputs.
     chat_history = model_inputs + \
@@ -165,10 +174,14 @@ with gr.Blocks(css=css) as demo:
                     clear_btn = gr.Button("Clear All")
 
     text_input.submit(generate_for_prompt, [text_input, gr_state, ret_scale_factor,
-                      max_ret_images, gr_max_len, gr_temperature], [gr_state, chatbot, share_group])
+                      max_ret_images, gr_max_len, gr_temperature], [gr_state, chatbot, share_group],
+                      )
+                      # _js = "() => document.getElementById('#chatbot').scrollTop = document.getElementById('#chatbot').scrollHeight")
     text_input.submit(lambda: "", None, text_input)  # Reset chatbox.
     submit_btn.click(generate_for_prompt, [text_input, gr_state, ret_scale_factor,
-                     max_ret_images, gr_max_len, gr_temperature], [gr_state, chatbot, share_group])
+                     max_ret_images, gr_max_len, gr_temperature], [gr_state, chatbot, share_group],
+                     )
+                     # _js = "() => document.getElementById('#chatbot').scrollTop = document.getElementById('#chatbot').scrollHeight")
     submit_btn.click(lambda: "", None, text_input)  # Reset chatbox.
 
     image_btn.upload(upload_image, [gr_state, image_btn], [gr_state, chatbot])
@@ -177,5 +190,5 @@ with gr.Blocks(css=css) as demo:
     share_button.click(None, [], [], _js=share_js)
 
 
-demo.queue(concurrency_count=1, api_open=False, max_size=16)
-demo.launch(debug=True, server_name="0.0.0.0")
+# demo.queue(concurrency_count=1, api_open=False, max_size=16)
+demo.launch(debug=True, server_name="127.0.0.1")

fromage/models.py

@@ -635,7 +635,7 @@ def load_fromage(embeddings_dir: str, model_args_path: str, model_ckpt_path: str
   assert len(ret_token_idx) == 1, ret_token_idx
   model_kwargs['retrieval_token_idx'] = ret_token_idx[0]
 
-  debug = False
+  debug = True
   if debug:
     model_kwargs['opt_version'] = 'facebook/opt-125m'
     model_kwargs['visual_encoder'] = 'openai/clip-vit-base-patch32'