Update README.md

README.md

@@ -68,32 +68,41 @@ This optimization reduces the number of bits per parameter from 16 to 8, reducin
 
 ### Use with vLLM
 
-This model can be deployed efficiently using the [vLLM](https://docs.vllm.ai/en/latest/) backend, as shown in the example below.
+1. Initialize vLLM server:
+```
+vllm serve RedHatAI/Mistral-Small-24B-Instruct-2501-FP8-dynamic --tensor_parallel_size 1 --tokenizer_mode mistral
+```
+
+2. Send requests to the server:
 
 ```python
-from transformers import AutoTokenizer
-from vllm import LLM, SamplingParams
+from openai import OpenAI
 
-max_model_len, tp_size = 4096, 1
-model_name = "neuralmagic/Mistral-Small-24B-Instruct-2501-FP8-Dynamic"
-tokenizer = AutoTokenizer.from_pretrained(model_name)
-llm = LLM(model=model_name, tensor_parallel_size=tp_size, max_model_len=max_model_len, trust_remote_code=True)
-sampling_params = SamplingParams(temperature=0.3, max_tokens=256, stop_token_ids=[tokenizer.eos_token_id])
+# Modify OpenAI's API key and API base to use vLLM's API server.
+openai_api_key = "EMPTY"
+openai_api_base = "http://<your-server-host>:8000/v1"
 
-messages_list = [
-    [{"role": "user", "content": "Who are you? Please respond in pirate speak!"}],
-]
+client = OpenAI(
+    api_key=openai_api_key,
+    base_url=openai_api_base,
+)
 
-prompt_token_ids = [tokenizer.apply_chat_template(messages, add_generation_prompt=True) for messages in messages_list]
+model = "RedHatAI/Mistral-Small-24B-Instruct-2501-FP8-dynamic"
 
-outputs = llm.generate(prompt_token_ids=prompt_token_ids, sampling_params=sampling_params)
 
-generated_text = [output.outputs[0].text for output in outputs]
+messages = [
+    {"role": "user", "content": "Explain quantum mechanics clearly and concisely."},
+]
+
+outputs = client.chat.completions.create(
+    model=model,
+    messages=messages,
+)
+
+generated_text = outputs.choices[0].message.content
 print(generated_text)
 ```
 
-vLLM also supports OpenAI-compatible serving. See the [documentation](https://docs.vllm.ai/en/latest/) for more details.
-
 <details>
   <summary>Deploy on <strong>Red Hat AI Inference Server</strong></summary>