Local server

geniex pull ai-hub-models/Qwen3-4B-Instruct-2507

geniex serve

{
  "model": "ai-hub-models/Qwen3-4B-Instruct-2507",
  "messages": [
    {"role": "user", "content": "Hello! Briefly introduce yourself."}
  ],
  "max_tokens": 256,
  "temperature": 0.7,
  "stream": false
}

{
  "model": "unsloth/Qwen2.5-Coder-3B-GGUF:Q4_0",
  "prompt": "<|fim_prefix|>def fibonacci(n):\n    <|fim_suffix|>\n    return a<|fim_middle|>",
  "max_tokens": 64,
  "temperature": 0.2,
  "stop": ["<|endoftext|>"],
  "stream": false
}

from openai import OpenAI

client = OpenAI(
    base_url="http://127.0.0.1:18181/v1",
    api_key="geniex",  # any non-empty string; the server does not check it
)

stream = client.chat.completions.create(
    model="unsloth/Qwen3-4B-GGUF:Q4_0",
    messages=[
        {"role": "user", "content": "Hello! Briefly introduce yourself."},
    ],
    max_tokens=256,
    temperature=0.7,
    stream=True,
)

for chunk in stream:
    delta = chunk.choices[0].delta.content
    if delta:
        print(delta, end="", flush=True)
print()

resp = client.chat.completions.create(
    model="unsloth/Qwen3-4B-GGUF:Q4_0",
    messages=[
        {"role": "user", "content": "Hello! Briefly introduce yourself."},
    ],
    max_tokens=128,
    temperature=0.7,
    extra_body={"enable_think": False},
)

print(resp.choices[0].message.content)
print("finish_reason:", resp.choices[0].finish_reason)
print("usage:", resp.usage)

resp = client.chat.completions.create(
    model="unsloth/Qwen3-4B-GGUF:Q4_0",
    messages=[
        {"role": "user", "content": "What is 2+2? Answer briefly."},
    ],
    max_tokens=128,
)

msg = resp.choices[0].message
print("reasoning:", msg.model_extra.get("reasoning_content"))
print("content:", msg.content)

import json

from ddgs import DDGS

tools = [
    {
        "type": "function",
        "function": {
            "name": "web_search",
            "description": "Search the web for travel information about a location.",
            "parameters": {
                "type": "object",
                "properties": {
                    "query": {"type": "string", "description": "Search query, e.g. 'things to do in Kyoto'"},
                },
                "required": ["query"],
            },
        },
    }
]

def web_search(query: str) -> list[dict]:
    return [
        {"title": r["title"], "snippet": r["body"], "url": r["href"]}
        for r in DDGS().text(query, max_results=3)
    ]

IMAGE_URL = "https://images.pexels.com/photos/402028/pexels-photo-402028.jpeg?w=1024"

system_prompt = (
    "You are a travel assistant. Call the web_search tool to look up any location "
    "the user asks about before answering. Once you receive the tool results, do not "
    "call the tool again - use them to write a short, friendly reply for the user. "
    "Emit tool calls in the exact format: "
    '<tool_call>{"name": "web_search", "arguments": {"query": "<your query>"}}</tool_call>'
)

messages = [
    {"role": "system", "content": system_prompt},
    {
        "role": "user",
        "content": [
            {"type": "image_url", "image_url": {"url": IMAGE_URL}},
            {"type": "text", "text": "Identify the landmark, then call web_search for travel tips about it."},
        ],
    },
]

# Step 1 - VLM identifies the landmark and requests a web_search call.
first = client.chat.completions.create(
    model="qualcomm/Qwen3-VL-4B-Instruct",
    messages=messages,
    tools=tools,
    tool_choice="auto",
    max_tokens=512,
    extra_body={"enable_think": False},
)
call = first.choices[0].message.tool_calls[0]
print("finish_reason:", first.choices[0].finish_reason)  # -> "tool_calls"
print("call:", call.function.name, call.function.arguments)

# Step 2 - run the tool, feed the result back as a fresh text-only conversation.
result = web_search(**json.loads(call.function.arguments))

followup = [
    {"role": "system", "content": system_prompt},
    {"role": "user", "content": "Summarize travel tips using the search results below."},
    first.choices[0].message,
    {"role": "tool", "tool_call_id": call.id, "content": json.dumps(result)},
]

final = client.chat.completions.create(
    model="qualcomm/Qwen3-VL-4B-Instruct",
    messages=followup,
    max_tokens=256,
    extra_body={"enable_think": False},
)
print(final.choices[0].message.content)