update

app.py

@@ -31,6 +31,19 @@ from model import (
     get_model,
 )
 
+# seed for reproducibility
+import random
+import numpy as np
+import torch
+seed = 42
+random.seed(seed)
+np.random.seed(seed)
+torch.manual_seed(seed)
+if torch.cuda.is_available():
+    torch.cuda.manual_seed_all(seed)
+    torch.backends.cudnn.deterministic = True
+    torch.backends.cudnn.benchmark = False
+
 logger = build_logger("dimple", "dimple.log")
 
 no_change_btn = gr.Button()
@@ -233,8 +246,6 @@ def http_bot(
         )
     
     thread = threading.Thread(target=run_generate)
-    thread.start()
-
     logger.info(f"==== wait for first token ====\n")
     state.append_message(Conversation.ASSISTANT, state.streaming_placeholder)
     yield (
@@ -243,6 +254,10 @@ def http_bot(
         gr.MultimodalTextbox(interactive=False),
     ) + (disable_btn,) * 5
 
+    num_steps = 0
+    start_tstamp = time.time()
+    thread.start()
+
     try:
         # Stream output
         for ans in streamer:
@@ -252,11 +267,22 @@ def http_bot(
                 ans = ans[0]
                 
             state.update_message(Conversation.ASSISTANT, ans, None)
+            num_steps += 1
             yield (
                 state,
                 state.to_gradio_chatbot(),
                 gr.MultimodalTextbox(interactive=False),
             ) + (disable_btn,) * 5
+        end_tstamp = time.time()
+        total_time = end_tstamp - start_tstamp
+        tps = int(max_new_tokens) / total_time
+        stat_info = f"\n\n[#Tokens: {int(max_new_tokens)}, #Steps: {int(num_steps)}, TPS: {tps:.2f} tokens/s, Latency: {total_time:.2f}s]"
+        state.update_message(Conversation.ASSISTANT, ans+stat_info, None)
+        yield (
+                state,
+                state.to_gradio_chatbot(),
+                gr.MultimodalTextbox(interactive=False),
+            ) + (disable_btn,) * 5
     except Exception as e:
         state.update_message(Conversation.ASSISTANT, server_error_msg, None)
         yield (
@@ -422,7 +448,7 @@ def build_demo():
                         [
                             {
                                 "files": [
-                                    "gallery/1.jpg",
+                                    "gallery/1_resize.jpg",
                                 ],
                                 "text": "Please help me describe the image.",
                             }
@@ -430,7 +456,7 @@ def build_demo():
                         [
                             {
                                 "files": [
-                                    "gallery/2.jpg",
+                                    "gallery/2_resize.jpg",
                                 ],
                                 "text": "First please perform reasoning, and think step by step to provide best answer to the following question: Is this airplane taking off or landing?",
                             }
@@ -438,7 +464,7 @@ def build_demo():
                         [
                             {
                                 "files": [
-                                    "gallery/3.jpg",
+                                    "gallery/3_resize.jpg",
                                 ],
                                 "text": "First please perform reasoning, and think step by step to provide best answer to the following question: What is the lamp on, a side table or a nightstand?",
                             }

model.py

@@ -5,8 +5,14 @@ from transformers import AutoProcessor, AutoModel, TextIteratorStreamer
 class FullSequenceStreamer(TextIteratorStreamer):
     def __init__(self, tokenizer, **kwargs):
         super().__init__(tokenizer, **kwargs)
+        self.mask_token = tokenizer.mask_token_id
+        self.placeholder_token = tokenizer.convert_tokens_to_ids("_")
+        self.placeholder_token = tokenizer.encode("␣")[0]
 
     def put(self, value, stream_end=False):
+        # change mask tokens to space token
+        value = value.clone()
+        value[value == self.mask_token] = self.placeholder_token
         # Assume full token_ids are passed in every time
         decoded = self.tokenizer.batch_decode(value, **self.decode_kwargs)
         self.text_queue.put(decoded)
@@ -33,3 +39,17 @@ def get_model(device):
 
     return model, processor
 
+from transformers import Qwen2VLForConditionalGeneration, AutoTokenizer, AutoProcessor
+
+def get_qwen(device):
+    
+    model_name = "Qwen/Qwen2-VL-7B-Instruct"
+    processor = AutoProcessor.from_pretrained("Qwen/Qwen2-VL-7B-Instruct")
+    model = Qwen2VLForConditionalGeneration.from_pretrained(
+        model_name,
+        torch_dtype=torch.bfloat16,
+    )
+    model = model.eval()
+    model = model.to(device)
+
+    return model, processor