add files

app.py

@@ -47,10 +47,21 @@ def get_args() -> argparse.Namespace:
     """
     parser = ArgumentParser(description="ERNIE models web chat demo.")
 
-    parser.add_argument("--server-port", type=int, default=7860, help="Demo server port.")
-    parser.add_argument("--server-name", type=str, default="0.0.0.0", help="Demo server name.")
-    parser.add_argument("--max_char", type=int, default=8000, help="Maximum character limit for messages.")
-    parser.add_argument("--max_retry_num", type=int, default=3, help="Maximum retry number for request.")
+    parser.add_argument(
+        "--server-port", type=int, default=7860, help="Demo server port."
+    )
+    parser.add_argument(
+        "--server-name", type=str, default="0.0.0.0", help="Demo server name."
+    )
+    parser.add_argument(
+        "--max_char",
+        type=int,
+        default=8000,
+        help="Maximum character limit for messages.",
+    )
+    parser.add_argument(
+        "--max_retry_num", type=int, default=3, help="Maximum retry number for request."
+    )
     parser.add_argument(
         "--model_name_map",
         type=str,
@@ -97,7 +108,15 @@ def get_args() -> argparse.Namespace:
             * ERNIE-4.5-VL[-*]: Multimodal models (image+text)
             """,
     )
-    parser.add_argument("--api_key", type=str, default="bce-v3/xxx", help="Model service API key.")
+    parser.add_argument(
+        "--api_key", type=str, default="bce-v3/xxx", help="Model service API key."
+    )
+    parser.add_argument(
+        "--concurrency_limit", type=int, default=10, help="Default concurrency limit."
+    )
+    parser.add_argument(
+        "--max_queue_size", type=int, default=50, help="Maximum queue size for request."
+    )
 
     args = parser.parse_args()
     try:
@@ -202,7 +221,12 @@ class GradioEvents:
             if idx in image_history:
                 content = []
                 content.append(
-                    {"type": "image_url", "image_url": {"url": GradioEvents.get_image_url(image_history[idx])}}
+                    {
+                        "type": "image_url",
+                        "image_url": {
+                            "url": GradioEvents.get_image_url(image_history[idx])
+                        },
+                    }
                 )
                 content.append({"type": "text", "text": query_h})
                 conversation.append({"role": "user", "content": content})
@@ -211,9 +235,16 @@ class GradioEvents:
             conversation.append({"role": "assistant", "content": response_h})
 
         content = []
-        if file_url and (len(image_history) == 0 or file_url != list(image_history.values())[-1]):
+        if file_url and (
+            len(image_history) == 0 or file_url != list(image_history.values())[-1]
+        ):
             image_history[len(task_history)] = file_url
-            content.append({"type": "image_url", "image_url": {"url": GradioEvents.get_image_url(file_url)}})
+            content.append(
+                {
+                    "type": "image_url",
+                    "image_url": {"url": GradioEvents.get_image_url(file_url)},
+                }
+            )
             content.append({"type": "text", "text": query})
             conversation.append({"role": "user", "content": content})
         else:
@@ -222,7 +253,9 @@ class GradioEvents:
         try:
             req_data = {"messages": conversation}
             model_name = model_name_map.get(model_name, model_name)
-            for chunk in bot_client.process_stream(model_name, req_data, max_tokens, temperature, top_p):
+            for chunk in bot_client.process_stream(
+                model_name, req_data, max_tokens, temperature, top_p
+            ):
                 if "error" in chunk:
                     raise Exception(chunk["error"])
 
@@ -395,7 +428,9 @@ class GradioEvents:
         """
         GradioEvents.gc()
 
-        reset_result = namedtuple("reset_result", ["chatbot", "task_history", "image_history", "file_btn"])
+        reset_result = namedtuple(
+            "reset_result", ["chatbot", "task_history", "image_history", "file_btn"]
+        )
         return reset_result(
             [],  # clear chatbot
             [],  # clear task_history
@@ -421,7 +456,9 @@ class GradioEvents:
         Returns:
             gr.update: An update object representing the visibility of the file button.
         """
-        return gr.update(visible=model_name.upper().startswith(MULTI_MODEL_PREFIX))  # file_btn
+        return gr.update(
+            visible=model_name.upper().startswith(MULTI_MODEL_PREFIX)
+        )  # file_btn
 
 
 def launch_demo(args: argparse.Namespace, bot_client: BotClient):
@@ -477,11 +514,16 @@ def launch_demo(args: argparse.Namespace, bot_client: BotClient):
 (本演示基于文心大模型实现。)</center>"""
         )
 
-        chatbot = gr.Chatbot(label="ERNIE", elem_classes="control-height", type="messages")
+        chatbot = gr.Chatbot(
+            label="ERNIE", elem_classes="control-height", type="messages"
+        )
         model_names = list(args.model_name_map.keys())
         with gr.Row():
             model_name = gr.Dropdown(
-                label="Select Model", choices=model_names, value=model_names[0], allow_custom_value=True
+                label="Select Model",
+                choices=model_names,
+                value=model_names[0],
+                allow_custom_value=True,
             )
             file_btn = gr.File(
                 label="Image upload (Active only for multimodal models. Accepted formats: PNG, JPEG, JPG)",
@@ -497,55 +539,89 @@ def launch_demo(args: argparse.Namespace, bot_client: BotClient):
             submit_btn = gr.Button("🚀 Submit(发送)", elem_id="submit-button")
             regen_btn = gr.Button("🤔️ Regenerate(重试)")
 
-        with gr.Accordion("⚙️ Advanced Config", open=False):  # open=False means collapsed by default
+        with gr.Accordion(
+            "⚙️ Advanced Config", open=False
+        ):  # open=False means collapsed by default
             system_message = gr.Textbox(value="", label="System message", visible=True)
             additional_inputs = [
                 system_message,
-                gr.Slider(minimum=1, maximum=4096, value=2048, step=1, label="Max new tokens"),
-                gr.Slider(minimum=0.1, maximum=1.0, value=1.0, step=0.05, label="Temperature"),
-                gr.Slider(minimum=0.1, maximum=1.0, value=0.7, step=0.05, label="Top-p (nucleus sampling)"),
+                gr.Slider(
+                    minimum=1, maximum=4096, value=2048, step=1, label="Max new tokens"
+                ),
+                gr.Slider(
+                    minimum=0.1, maximum=1.0, value=1.0, step=0.05, label="Temperature"
+                ),
+                gr.Slider(
+                    minimum=0.1,
+                    maximum=1.0,
+                    value=0.7,
+                    step=0.05,
+                    label="Top-p (nucleus sampling)",
+                ),
             ]
 
         task_history = gr.State([])
         image_history = gr.State({})
 
-        model_name.change(GradioEvents.toggle_components_visibility, inputs=model_name, outputs=file_btn)
         model_name.change(
-            GradioEvents.reset_state, outputs=[chatbot, task_history, image_history, file_btn], show_progress=True
+            GradioEvents.toggle_components_visibility,
+            inputs=model_name,
+            outputs=file_btn,
+        )
+        model_name.change(
+            GradioEvents.reset_state,
+            outputs=[chatbot, task_history, image_history, file_btn],
+            show_progress=True,
         )
         predict_with_clients = partial(
-            GradioEvents.predict_stream, model_name_map=args.model_name_map, bot_client=bot_client
+            GradioEvents.predict_stream,
+            model_name_map=args.model_name_map,
+            bot_client=bot_client,
         )
         regenerate_with_clients = partial(
-            GradioEvents.regenerate, model_name_map=args.model_name_map, bot_client=bot_client
+            GradioEvents.regenerate,
+            model_name_map=args.model_name_map,
+            bot_client=bot_client,
         )
         query.submit(
             predict_with_clients,
-            inputs=[query, chatbot, task_history, image_history, model_name, file_btn] + additional_inputs,
+            inputs=[query, chatbot, task_history, image_history, model_name, file_btn]
+            + additional_inputs,
             outputs=[chatbot],
             show_progress=True,
         )
         query.submit(GradioEvents.reset_user_input, [], [query])
         submit_btn.click(
             predict_with_clients,
-            inputs=[query, chatbot, task_history, image_history, model_name, file_btn] + additional_inputs,
+            inputs=[query, chatbot, task_history, image_history, model_name, file_btn]
+            + additional_inputs,
             outputs=[chatbot],
             show_progress=True,
         )
         submit_btn.click(GradioEvents.reset_user_input, [], [query])
         empty_btn.click(
-            GradioEvents.reset_state, outputs=[chatbot, task_history, image_history, file_btn], show_progress=True
+            GradioEvents.reset_state,
+            outputs=[chatbot, task_history, image_history, file_btn],
+            show_progress=True,
         )
         regen_btn.click(
             regenerate_with_clients,
-            inputs=[chatbot, task_history, image_history, model_name, file_btn] + additional_inputs,
+            inputs=[chatbot, task_history, image_history, model_name, file_btn]
+            + additional_inputs,
             outputs=[chatbot],
             show_progress=True,
         )
 
-        demo.load(GradioEvents.toggle_components_visibility, inputs=gr.State(model_names[0]), outputs=file_btn)
+        demo.load(
+            GradioEvents.toggle_components_visibility,
+            inputs=gr.State(model_names[0]),
+            outputs=file_btn,
+        )
 
-    demo.queue().launch(server_port=args.server_port, server_name=args.server_name)
+    demo.queue(
+        default_concurrency_limit=args.concurrency_limit, max_size=args.max_queue_size
+    )
+    demo.launch(server_port=args.server_port, server_name=args.server_name)
 
 
 def main():

bot_requests.py

@@ -40,17 +40,21 @@ class BotClient:
         """
         self.logger = logging.getLogger(__name__)
 
-        self.max_retry_num = getattr(args, 'max_retry_num', 3)
-        self.max_char = getattr(args, 'max_char', 8000)
+        self.max_retry_num = getattr(args, "max_retry_num", 3)
+        self.max_char = getattr(args, "max_char", 8000)
 
-        self.model_map = getattr(args, 'model_map', {})
+        self.model_map = getattr(args, "model_map", {})
         self.api_key = os.environ.get("API_KEY")
 
-        self.embedding_service_url = getattr(args, 'embedding_service_url', 'embedding_service_url')
-        self.embedding_model = getattr(args, 'embedding_model', 'embedding_model')
+        self.embedding_service_url = getattr(
+            args, "embedding_service_url", "embedding_service_url"
+        )
+        self.embedding_model = getattr(args, "embedding_model", "embedding_model")
 
-        self.web_search_service_url = getattr(args, 'web_search_service_url', 'web_search_service_url')
-        self.max_search_results_num = getattr(args, 'max_search_results_num', 15)
+        self.web_search_service_url = getattr(
+            args, "web_search_service_url", "web_search_service_url"
+        )
+        self.max_search_results_num = getattr(args, "max_search_results_num", 15)
 
         self.qianfan_api_key = os.environ.get("API_KEY")
 
@@ -109,7 +113,12 @@ class BotClient:
             raise
 
     def process(
-        self, model_name: str, req_data: dict, max_tokens: int = 2048, temperature: float = 1.0, top_p: float = 0.7
+        self,
+        model_name: str,
+        req_data: dict,
+        max_tokens: int = 2048,
+        temperature: float = 1.0,
+        top_p: float = 0.7,
     ) -> dict:
         """
         Handles chat completion requests by mapping the model name to its endpoint, preparing request parameters
@@ -152,7 +161,12 @@ class BotClient:
         return res
 
     def process_stream(
-        self, model_name: str, req_data: dict, max_tokens: int = 2048, temperature: float = 1.0, top_p: float = 0.7
+        self,
+        model_name: str,
+        req_data: dict,
+        max_tokens: int = 2048,
+        temperature: float = 1.0,
+        top_p: float = 0.7,
     ) -> dict:
         """
         Processes streaming requests by mapping the model name to its endpoint, configuring request parameters,
@@ -188,7 +202,9 @@ class BotClient:
 
             except Exception as e:
                 last_error = e
-                self.logger.error(f"Stream request failed (attempt {_ + 1}/{self.max_retry_num}): {e}")
+                self.logger.error(
+                    f"Stream request failed (attempt {_ + 1}/{self.max_retry_num}): {e}"
+                )
 
         self.logger.error("All retry attempts failed for stream request")
         yield {"error": str(last_error)}
@@ -209,7 +225,9 @@ class BotClient:
         en_ch_words = []
 
         for word in words:
-            if word.isalpha() and not any("\u4e00" <= char <= "\u9fff" for char in word):
+            if word.isalpha() and not any(
+                "\u4e00" <= char <= "\u9fff" for char in word
+            ):
                 en_ch_words.append(word)
             else:
                 en_ch_words.extend(list(word))
@@ -341,7 +359,9 @@ class BotClient:
         Returns:
             list: A list of floats representing the embedding.
         """
-        client = OpenAI(base_url=self.embedding_service_url, api_key=self.qianfan_api_key)
+        client = OpenAI(
+            base_url=self.embedding_service_url, api_key=self.qianfan_api_key
+        )
         response = client.embeddings.create(input=[text], model=self.embedding_model)
         return response.data[0].embedding
 
@@ -355,7 +375,10 @@ class BotClient:
         Returns:
             list: List of responses from the AI Search service.
         """
-        headers = {"Authorization": "Bearer " + self.qianfan_api_key, "Content-Type": "application/json"}
+        headers = {
+            "Authorization": "Bearer " + self.qianfan_api_key,
+            "Content-Type": "application/json",
+        }
 
         results = []
         top_k = self.max_search_results_num // len(query_list)
@@ -364,7 +387,9 @@ class BotClient:
                 "messages": [{"role": "user", "content": query}],
                 "resource_type_filter": [{"type": "web", "top_k": top_k}],
             }
-            response = requests.post(self.web_search_service_url, headers=headers, json=payload)
+            response = requests.post(
+                self.web_search_service_url, headers=headers, json=payload
+            )
 
             if response.status_code == 200:
                 response = response.json()