add all files for qwen

no new max tokens

demo

remove max

test

remove dialog

.gitignore

@@ -0,0 +1,103 @@
+# Python build
+.eggs/
+gradio.egg-info
+dist/
+dist-lite/
+*.pyc
+__pycache__/
+*.py[cod]
+*$py.class
+build/
+!js/build/
+!js/build/dist/
+__tmp/*
+*.pyi
+!gradio/stubs/**/*.pyi
+.ipynb_checkpoints/
+.python-version
+=23.2
+
+# JS build
+gradio/templates/*
+gradio/node/*
+gradio/_frontend_code/*
+js/gradio-preview/test/*
+
+# Secrets
+.env
+
+# Gradio run artifacts
+*.db
+*.sqlite3
+gradio/launches.json
+gradio/hash_seed.txt
+.gradio/
+
+tmp.zip
+
+# Tests
+.coverage
+coverage.xml
+test.txt
+**/snapshots/**/*.png
+playwright-report/
+.hypothesis
+.lite-perf.json
+
+# Demos
+demo/tmp.zip
+demo/files/*.avi
+demo/files/*.mp4
+demo/all_demos/demos/*
+demo/all_demos/requirements.txt
+demo/*/config.json
+demo/annotatedimage_component/*.png
+demo/fake_diffusion_with_gif/*.gif
+demo/cancel_events/cancel_events_output_log.txt
+demo/unload_event_test/output_log.txt
+demo/stream_video_out/output_*.ts
+demo/stream_video_out/output_*.mp4
+demo/stream_audio_out/*.mp3
+#demo/image_editor_story/*.png
+
+# Etc
+.idea/*
+.DS_Store
+*.bak
+workspace.code-workspace
+*.h5
+
+# dev containers
+.pnpm-store/
+
+# log files
+.pnpm-debug.log
+
+# Local virtualenv for devs
+.venv*
+
+# FRP
+gradio/frpc_*
+.vercel
+
+# js
+node_modules
+public/build/
+test-results
+client/js/dist/*
+client/js/test.js
+.config/test.py
+.svelte-kit
+
+
+# storybook
+storybook-static
+build-storybook.log
+js/storybook/theme.css
+#js/storybook/public/output-image.png
+
+# playwright
+.config/playwright/.cache
+
+# VSCode
+.lh

app.py

@@ -0,0 +1,129 @@
+import os
+import logging
+import gradio as gr
+from typing import Iterator
+from gateway import request_generation
+
+# Setup logging
+logging.basicConfig(level=logging.INFO)
+
+# Validate environment variables
+CLOUD_GATEWAY_API = os.getenv("API_ENDPOINT")
+if not CLOUD_GATEWAY_API:
+    raise EnvironmentError("API_ENDPOINT is not set.")
+
+MODEL_NAME: str = os.getenv("MODEL_NAME")
+if not MODEL_NAME:
+    raise EnvironmentError("MODEL_NAME is not set.")
+
+# Get API Key
+API_KEY = os.getenv("API_KEY")
+if not API_KEY:  # simple check to validate API Key
+    raise Exception("API Key not valid.")
+
+# Create a header, avoid declaring multiple times
+HEADER = {"x-api-key": f"{API_KEY}"}
+
+def generate(
+    message: str,
+    chat_history: list,
+    system_prompt: str,
+    temperature: float = 0.6,
+    frequency_penalty: float = 0.0,
+    presence_penalty: float = 0.0,
+) -> Iterator[str]:
+    """Send a request to backend, fetch the streaming responses and emit to the UI.
+
+    Args:
+        message (str): input message from the user
+        chat_history (list[tuple[str, str]]): entire chat history of the session
+        system_prompt (str): system prompt
+        temperature (float, optional): the value used to module the next token probabilities. Defaults to 0.6.
+        top_p (float, optional): if set to float<1, only the smallest set of most probable tokens with probabilities
+                                    that add up to top_p or higher are kept for generation. Defaults to 0.9.
+        top_k (int, optional): the number of highest probability vocabulary tokens to keep for top-k-filtering.
+                                Defaults to 50.
+        repetition_penalty (float, optional): the parameter for repetition penalty. 1.0 means no penalty.
+                                Defaults to 1.2.
+
+    Yields:
+        Iterator[str]: Streaming responses to the UI
+    """
+    # sample method to yield responses from the llm model
+    outputs = []
+    for text in request_generation(
+        header=HEADER,
+        message=message,
+        system_prompt=system_prompt,
+        temperature=temperature,
+        presence_penalty=presence_penalty,
+        frequency_penalty=frequency_penalty,
+        cloud_gateway_api=CLOUD_GATEWAY_API,
+        model_name=MODEL_NAME,
+    ):
+        outputs.append(text)
+        yield "".join(outputs)
+
+
+description = """
+This Space is an Alpha release that demonstrates the [Qwen3-30B-A3B](https://huggingface.co/Qwen/Qwen3-30B-A3B) model running on AMD MI300 infrastructure. The space is built with Qwen 3 [License](https://huggingface.co/Qwen/Qwen3-30B-A3B/blob/main/LICENSE). Feel free to play with it!
+"""
+
+demo = gr.ChatInterface(
+    fn=generate,
+    type="messages",
+    chatbot=gr.Chatbot(
+        type="messages",
+        scale=2,
+        allow_tags=True,
+    ),
+    stop_btn=None,
+    additional_inputs=[
+        gr.Textbox(
+            label="System prompt",
+            value="You are a highly capable AI assistant. Provide accurate, concise, and fact-based responses that are directly relevant to the user's query. Avoid speculation, ensure logical consistency, and maintain clarity in longer outputs. Keep answers well-structured and under 1200 tokens unless explicitly requested otherwise.",
+            lines=3,
+        ),
+        gr.Slider(
+            label="Temperature",
+            minimum=0.1,
+            maximum=4.0,
+            step=0.1,
+            value=0.3,
+        ),
+        gr.Slider(
+            label="Frequency penalty",
+            minimum=-2.0,
+            maximum=2.0,
+            step=0.1,
+            value=0.0,
+        ),
+        gr.Slider(
+            label="Presence penalty",
+            minimum=-2.0,
+            maximum=2.0,
+            step=0.1,
+            value=0.0,
+        ),
+    ],
+    examples=[
+        ["Plan a three-day trip to Washington DC for Cherry Blossom Festival."],
+        [
+            "Compose a short, joyful musical piece for kids celebrating spring sunshine and blossom."
+        ],
+        ["Can you explain briefly to me what is the Python programming language?"],
+        ["Explain the plot of Cinderella in a sentence."],
+        ["How many hours does it take a man to eat a Helicopter?"],
+        ["Write a 100-word article on 'Benefits of Open-Source in AI research'."],
+    ],
+    cache_examples=False,
+    title="Qwen3-30B-A3B",
+    description=description,
+)
+
+
+if __name__ == "__main__":
+    demo.queue(
+        max_size=int(os.getenv("QUEUE")),
+        default_concurrency_limit=int(os.getenv("CONCURRENCY_LIMIT")),
+    ).launch()

gateway.py

@@ -0,0 +1,128 @@
+import json
+import logging
+import requests
+import urllib3
+
+urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
+
+# Setup logging
+logging.basicConfig(level=logging.INFO)
+
+
+def check_server_health(cloud_gateway_api: str, header: dict) -> bool:
+    """
+    Use the appropriate API endpoint to check the server health.
+    Args:
+        cloud_gateway_api: API endpoint to probe.
+        header: Header for Authorization.
+
+    Returns:
+        True if server is active, false otherwise.
+    """
+    try:
+        response = requests.get(
+            cloud_gateway_api + "model/info",
+            headers=header,
+            verify=False,
+        )
+        response.raise_for_status()
+        return True
+    except requests.RequestException as e:
+        logging.error(f"Failed to check server health: {e}")
+        return False
+
+
+def request_generation(
+    header: dict,
+    message: str,
+    system_prompt: str,
+    cloud_gateway_api: str,
+    model_name: str,
+    temperature: float = 0.3,
+    frequency_penalty: float = 0.0,
+    presence_penalty: float = 0.0,
+):
+    """
+    Request streaming generation from the cloud gateway API. Uses the simple requests module with stream=True to utilize
+    token-by-token generation from LLM.
+
+    Args:
+        header: authorization header for the API.
+        message: prompt from the user.
+        system_prompt: system prompt to append.
+        cloud_gateway_api (str): API endpoint to send the request.
+        temperature: the value used to module the next token probabilities.
+        top_p: if set to float<1, only the smallest set of most probable tokens with probabilities that add up to top_p
+                or higher are kept for generation.
+        repetition_penalty: the parameter for repetition penalty. 1.0 means no penalty.
+
+    Returns:
+
+    """
+
+    payload = {
+        "model": model_name,
+        "messages": [
+            {"role": "system", "content": system_prompt},
+            {"role": "user", "content": message},
+        ],
+        "temperature": temperature,
+        "frequency_penalty": frequency_penalty,
+        "presence_penalty": presence_penalty,
+        "stream": True,  # Enable streaming
+        "serving_runtime": "vllm",
+    }
+
+    try:
+        response = requests.post(
+            cloud_gateway_api + "chat/conversation",
+            headers=header,
+            json=payload,
+            verify=False,
+        )
+        response.raise_for_status()
+
+        # Append the conversation ID with the key X-Conversation-ID to the header
+        header["X-Conversation-ID"] = response.json()["conversationId"]
+
+        with requests.get(
+            cloud_gateway_api + f"conversation/stream",
+            headers=header,
+            verify=False,
+            stream=True,
+        ) as response:
+            for chunk in response.iter_lines():
+                if chunk:
+                    # Convert the chunk from bytes to a string and then parse it as json
+                    chunk_str = chunk.decode("utf-8")
+
+                    # Remove the `data: ` prefix from the chunk if it exists
+                    for _ in range(2):
+                        if chunk_str.startswith("data: "):
+                            chunk_str = chunk_str[len("data: ") :]
+
+                    # Skip empty chunks
+                    if chunk_str.strip() == "[DONE]":
+                        break
+
+                    # Parse the chunk into a JSON object
+                    try:
+                        chunk_json = json.loads(chunk_str)
+
+                        # Extract the "content" field from the choices
+                        if "choices" in chunk_json and chunk_json["choices"]:
+                            content = chunk_json["choices"][0]["delta"].get(
+                                "content", ""
+                            )
+                        else:
+                            content = ""
+
+                        # Print the generated content as it's streamed
+                        if content:
+                            yield content
+                    except json.JSONDecodeError:
+                        # Handle any potential errors in decoding
+                        continue
+    except requests.RequestException as e:
+        logging.error(f"Failed to generate response: {e}")
+        yield "Server not responding. Please try again later."

requirements.txt

@@ -0,0 +1,4 @@
+numpy
+pillow
+fastapi
+websockets

style.css

@@ -0,0 +1,10 @@
+h1 {
+    text-align: center;
+    display: block;
+}
+
+.contain {
+    max-width: 900px;
+    margin: auto;
+    padding-top: 1.5rem;
+}