Gemma 4 Usage Guide¶

uvvenv
source.venv/bin/activate
uvpipinstall-Uvllm--pre\
--extra-index-urlhttps://wheels.vllm.ai/nightly/cu129\
--extra-index-urlhttps://download.pytorch.org/whl/cu129\
--index-strategyunsafe-best-match

uvvenv--python3.12
source.venv/bin/activate
uvpipinstallvllm--pre\
--extra-index-urlhttps://wheels.vllm.ai/rocm/nightly/rocm721--upgrade

dockerpullvllm/vllm-openai:latest# For CUDA 12.9
dockerpullvllm/vllm-openai:latest-cu130# For CUDA 13.0
dockerpullvllm/vllm-openai-rocm:latest# For AMD GPUs
dockerpullvllm/vllm-tpu:gemma4# For Cloud TPUs
dockerpullvllm/vllm-openai-cpu:latest-x86_64# For Intel Xeon 6

vllmservegoogle/gemma-4-E4B-it\
--max-model-len<n_of_tokens># up to 131072

dockerrun-itd--namegemma4\
--ipc=host\
--networkhost\
--shm-size16G\
--gpusall\
-v~/.cache/huggingface:/root/.cache/huggingface\
vllm/vllm-openai:latest\
--modelgoogle/gemma-4-31B-it\
--tensor-parallel-size2\
--max-model-len32768\
--gpu-memory-utilization0.90\
--host0.0.0.0\
--port8000

dockerrun-itd--namegemma4-tpu\
--privileged\
--networkhost\
--shm-size16G\
-v/dev/shm:/dev/shm\
-eHF_TOKEN=$HF_TOKEN\
vllm/vllm-tpu:gemma4\
--modelgoogle/gemma-4-31B-it\
--tensor-parallel-size8\
--max-model-len16384\
--disable_chunked_mm_input\
--host0.0.0.0\
--port8000

dockerrun-itd--namegemma4-rocm\
--ipc=host\
--network=host\
--privileged\
--cap-add=CAP_SYS_ADMIN\
--device=/dev/kfd\
--device=/dev/dri\
--group-add=video\
--cap-add=SYS_PTRACE\
--security-opt=seccomp=unconfined\
--shm-size16G\
-v~/.cache/huggingface:/root/.cache/huggingface\
vllm/vllm-openai-rocm:latest\
--model<MODEL>\
--host0.0.0.0\
--port8000

dockerrun-itd--namegemma4-cpu\
--network=host\
--shm-size16G\
-v~/.cache/huggingface:/root/.cache/huggingface\
vllm/vllm-openai-cpu:latest-x86_64\
--model<MODEL>\
--host0.0.0.0\
--port8000

dockerrun-itd--namediffusiongemma\
--ipc=host\
--networkhost\
--gpusall\
-v~/.cache/huggingface:/root/.cache/huggingface\
vllm/vllm-openai:gemma\
--modelgoogle/diffusiongemma-26B-A4B-it\
--max-model-len262144\
--max-num-seqs4\
--gpu-memory-utilization0.85\
--generation-configvllm\
--enable-chunked-prefill\
--host0.0.0.0\
--port8000

fromopenaiimport OpenAI
client = OpenAI(
 base_url="http://localhost:8000/v1",
 api_key="EMPTY"
)
response = client.chat.completions.create(
 model="google/gemma-4-31B-it",
 messages=[
 {"role": "user", "content": "Write a poem about the ocean."}
 ],
 max_tokens=512,
 temperature=0.7
)
print(response.choices[0].message.content)

curlhttp://localhost:8000/v1/chat/completions\
-H"Content-Type: application/json"\
-d'{
 "model": "google/gemma-4-31B-it",
 "messages": [
 {"role": "user", "content": "Explain quantum entanglement in simple terms."}
 ],
 "max_tokens": 512,
 "temperature": 0.7
 }'

fromvllmimport LLM, SamplingParams
fromtransformersimport AutoTokenizer
model_path = "google/gemma-4-31B-it"
tokenizer = AutoTokenizer.from_pretrained(model_path)
llm = LLM(
 model=model_path,
 tensor_parallel_size=2,
 max_model_len=8192,
 gpu_memory_utilization=0.90,
 trust_remote_code=True
)
messages = [
 {"role": "user", "content": "What are the three laws of thermodynamics?"}
]
prompt = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
outputs = llm.generate(prompt, SamplingParams(temperature=0.0, max_tokens=1024))
print(outputs[0].outputs[0].text)

fromopenaiimport OpenAI
client = OpenAI(
 base_url="http://localhost:8000/v1",
 api_key="EMPTY"
)
response = client.chat.completions.create(
 model="google/gemma-4-31B-it",
 messages=[
 {
 "role": "user",
 "content": [
 {
 "type": "image_url",
 "image_url": {"url": "https://upload.wikimedia.org/wikipedia/commons/thumb/3/3a/Cat03.jpg/1200px-Cat03.jpg"}
 },
 {
 "type": "text",
 "text": "Describe this image in detail."
 }
 ]
 }
 ],
 max_tokens=1024
)
print(response.choices[0].message.content)

response = client.chat.completions.create(
 model="google/gemma-4-31B-it",
 messages=[
 {
 "role": "user",
 "content": [
 {
 "type": "image_url",
 "image_url": {"url": "https://upload.wikimedia.org/wikipedia/commons/thumb/3/3a/Cat03.jpg/1200px-Cat03.jpg"}
 },
 {
 "type": "image_url",
 "image_url": {"url": "https://upload.wikimedia.org/wikipedia/commons/thumb/4/4d/Cat_November_2010-1a.jpg/1200px-Cat_November_2010-1a.jpg"}
 },
 {
 "type": "text",
 "text": "What are the key similarities and differences between these two images?"
 }
 ]
 }
 ],
 max_tokens=1024
)

vllmservegoogle/gemma-4-31B-it\
--mm-processor-kwargs'{"max_soft_tokens": 560}'

fromvllmimport LLM, SamplingParams
fromPILimport Image
fromtransformersimport AutoProcessor
model_path = "google/gemma-4-31B-it"
processor = AutoProcessor.from_pretrained(model_path)
llm = LLM(
 model=model_path,
 max_model_len=8192,
 trust_remote_code=True,
 limit_mm_per_prompt={"image": 4},
 # Set the maximum capacity the vision tower will allocate for
 hf_overrides={
 "vision_config": {"default_output_length": 1120},
 "vision_soft_tokens_per_image": 1120
 },
 # Default token budget when no per-request override is given
 mm_processor_kwargs={"max_soft_tokens": 280}
)
image = Image.open("photo.jpg").convert("RGB")
messages = [{"role": "user", "content": [
 {"type": "image"},
 {"type": "text", "text": "Describe this image in detail."}
]}]
prompt = processor.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
# Override to 1120 tokens for this specific request
outputs = llm.generate(
 {
 "prompt": prompt,
 "multi_modal_data": {"image": image},
 "mm_processor_kwargs": {"max_soft_tokens": 1120}
 },
 sampling_params=SamplingParams(temperature=0.0, max_tokens=512),
)
print(outputs[0].outputs[0].text)

vllmservegoogle/gemma-4-31B-it\
--max-model-len8192\
--limit-mm-per-prompt'{"image": 4, "audio": 1}'

fromopenaiimport OpenAI
client = OpenAI(
 base_url="http://localhost:8000/v1",
 api_key="EMPTY"
)
response = client.chat.completions.create(
 model="google/gemma-4-E2B-it",
 messages=[
 {
 "role": "user",
 "content": [
 {
 "type": "audio_url",
 "audio_url": {"url": "https://upload.wikimedia.org/wikipedia/commons/2/22/Beatbox_by_Wikipedia_user_Wikipedia_Brown.ogg"}
 },
 {
 "type": "text",
 "text": "Provide a verbatim, word-for-word transcription of the audio."
 }
 ]
 }
 ],
 max_tokens=512
)
print(response.choices[0].message.content)

curlhttp://localhost:8000/v1/chat/completions\
-H"Content-Type: application/json"\
-d'{
 "model": "google/gemma-4-E4B-it",
 "messages": [
 {
 "role": "user",
 "content": [
 {"type": "audio_url", "audio_url": {"url": "https://example.com/audio.wav"}},
 {"type": "text", "text": "Transcribe this audio."}
 ]
 }
 ],
 "max_tokens": 512
 }'

vllmservegoogle/gemma-4-E2B-it\
--max-model-len8192\
--limit-mm-per-prompt'{"image": 4, "video": 1}'

fromopenaiimport OpenAI
client = OpenAI(
 base_url="http://localhost:8000/v1",
 api_key="EMPTY"
)
response = client.chat.completions.create(
 model="google/gemma-4-E2B-it",
 messages=[
 {
 "role": "user",
 "content": [
 {
 "type": "video_url",
 "video_url": {"url": "https://example.com/sample_video.mp4"}
 },
 {
 "type": "text",
 "text": "Summarize what happens in this video."
 }
 ]
 }
 ],
 max_tokens=1024
)
print(response.choices[0].message.content)

fromvllmimport LLM, SamplingParams
fromvllm.multimodal.utilsimport fetch_video
fromtransformersimport AutoProcessor
model_path = "google/gemma-4-E2B-it"
processor = AutoProcessor.from_pretrained(model_path)
llm = LLM(
 model=model_path,
 max_model_len=8192,
 trust_remote_code=True,
 limit_mm_per_prompt={"image": 4, "video": 1}
)
video_url = "https://example.com/sample_video.mp4"
video_data = fetch_video(video_url)
messages = [{"role": "user", "content": [
 {"type": "video"},
 {"type": "text", "text": "Summarize what happens in this video."}
]}]
prompt = processor.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
outputs = llm.generate(
 {
 "prompt": prompt,
 "multi_modal_data": {"video": [video_data]}
 },
 sampling_params=SamplingParams(temperature=0.0, max_tokens=1024),
)
print(outputs[0].outputs[0].text)

vllmservegoogle/gemma-4-31B-it\
--max-model-len16384\
--enable-auto-tool-choice\
--reasoning-parsergemma4\
--tool-call-parsergemma4\
--chat-templateexamples/tool_chat_template_gemma4.jinja

fromopenaiimport OpenAI
client = OpenAI(
 base_url="http://localhost:8000/v1",
 api_key="EMPTY"
)
response = client.chat.completions.create(
 model="google/gemma-4-31B-it",
 messages=[
 {"role": "user", "content": "A snail is at the bottom of a 20-foot well. Each day it climbs 3 feet, but at night it slides back 2 feet. How many days will it take to reach the top?"}
 ],
 max_tokens=4096,
 extra_body={
 "chat_template_kwargs": {"enable_thinking": True}
 }
)
message = response.choices[0].message
# The thinking process is in reasoning
if hasattr(message, "reasoning") and message.reasoning:
 print("=== Thinking ===")
 print(message.reasoning)
print("\n=== Answer ===")
print(message.content)

curlhttp://localhost:8000/v1/chat/completions\
-H"Content-Type: application/json"\
-d'{
 "model": "google/gemma-4-31B-it",
 "messages": [
 {"role": "user", "content": "What is the derivative of x^3 * ln(x)?"}
 ],
 "max_tokens": 4096,
 "chat_template_kwargs": {"enable_thinking": true}
 }'

vllmservegoogle/gemma-4-31B-it\
--max-model-len8192\
--enable-auto-tool-choice\
--tool-call-parsergemma4\
--reasoning-parsergemma4\
--chat-templateexamples/tool_chat_template_gemma4.jinja

fromopenaiimport OpenAI
importjson
client = OpenAI(
 base_url="http://localhost:8000/v1",
 api_key="EMPTY"
)
# Define tools
tools = [
 {
 "type": "function",
 "function": {
 "name": "get_weather",
 "description": "Get the current weather for a location",
 "parameters": {
 "type": "object",
 "properties": {
 "location": {
 "type": "string",
 "description": "City name, e.g. 'San Francisco, CA'"
 },
 "unit": {
 "type": "string",
 "enum": ["celsius", "fahrenheit"],
 "description": "Temperature unit"
 }
 },
 "required": ["location"]
 }
 }
 }
]
# Step 1: Send user message with tools
response = client.chat.completions.create(
 model="google/gemma-4-31B-it",
 messages=[
 {"role": "user", "content": "What is the weather in Tokyo today?"}
 ],
 tools=tools,
 max_tokens=1024
)
message = response.choices[0].message
# Step 2: Process tool calls
if message.tool_calls:
 tool_call = message.tool_calls[0]
 print(f"Tool: {tool_call.function.name}")
 print(f"Args: {tool_call.function.arguments}")
 # Step 3: Feed back tool result and get final answer
 response = client.chat.completions.create(
 model="google/gemma-4-31B-it",
 messages=[
 {"role": "user", "content": "What is the weather in Tokyo today?"},
 message, # assistant's tool call message
 {
 "role": "tool",
 "tool_call_id": tool_call.id,
 "content": json.dumps({"temperature": 22, "condition": "Partly cloudy", "unit": "celsius"})
 }
 ],
 tools=tools,
 max_tokens=1024
 )
 print(f"\nFinal answer: {response.choices[0].message.content}")

response = client.chat.completions.create(
 model="google/gemma-4-31B-it",
 messages=[
 {"role": "user", "content": "I need to know the weather in Tokyo and then calculate the wind chill factor."}
 ],
 tools=tools,
 max_tokens=4096,
 extra_body={
 "chat_template_kwargs": {"enable_thinking": True}
 }
)

response = client.chat.completions.create(
 model="google/gemma-4-31B-it",
 messages=[
 {
 "role": "user",
 "content": [
 {
 "type": "image_url",
 "image_url": {"url": "https://upload.wikimedia.org/wikipedia/commons/thumb/c/c9/Christ_the_Redeemer_-_Rio_de_Janeiro%2C_Brazil.jpg/800px-Christ_the_Redeemer_-_Rio_de_Janeiro%2C_Brazil.jpg"}
 },
 {
 "type": "text",
 "text": "What city is shown in this image? What is the current weather there?"
 }
 ]
 }
 ],
 tools=tools,
 max_tokens=1024
)

fromopenaiimport OpenAI
client = OpenAI(
 base_url="http://localhost:8000/v1",
 api_key="EMPTY"
)
json_schema = {
 "type": "object",
 "properties": {
 "city": {"type": "string"},
 "country": {"type": "string"},
 "population": {"type": "integer"},
 "landmarks": {
 "type": "array",
 "items": {"type": "string"}
 }
 },
 "required": ["city", "country", "population", "landmarks"]
}
response = client.chat.completions.create(
 model="google/gemma-4-31B-it",
 messages=[
 {
 "role": "system",
 "content": "Extract city information as structured JSON."
 },
 {
 "role": "user",
 "content": "Tell me about Paris, France."
 }
 ],
 response_format={
 "type": "json_schema",
 "json_schema": {
 "name": "city-info",
 "schema": json_schema
 }
 },
 max_tokens=512
)
importjson
data = json.loads(response.choices[0].message.content)
print(data)
# {"city": "Paris", "country": "France", "population": 2161000, "landmarks": ["Eiffel Tower", "Louvre Museum", ...]}

fromtypingimport Optional
frompydanticimport BaseModel, Field
fromopenaiimport OpenAI
client = OpenAI(
 base_url="http://localhost:8000/v1",
 api_key="EMPTY"
)
classWeatherReport(BaseModel):
 air_temperature: Optional[float] = Field(None, description="Temperature in Fahrenheit")
 wind_speed: Optional[float] = Field(None, description="Wind speed in mph")
 comments_or_answer: str = Field(description="Comments or answer to the user's question")
response = client.chat.completions.create(
 model="google/gemma-4-31B-it",
 messages=[
 {
 "role": "system",
 "content": (
 "Extract the weather information. Output JSON with these fields:\n"
 "- air_temperature: float, converted to Fahrenheit\n"
 "- wind_speed: float, converted to mph\n"
 "- comments_or_answer: string, answer the user's question"
 )
 },
 {
 "role": "user",
 "content": "The current weather in Seattle is 22.0°C with a wind speed of 6.0 km/h."
 }
 ],
 response_format={
 "type": "json_schema",
 "json_schema": {
 "name": "weather-report",
 "schema": WeatherReport.model_json_schema()
 }
 },
 max_tokens=256
)

response = client.chat.completions.create(
 model="google/gemma-4-31B-it",
 messages=[
 {
 "role": "system",
 "content": (
 "Analyze the text and extract entities. Output JSON with:\n"
 "- people: list of person names mentioned\n"
 "- organizations: list of organization names\n"
 "- locations: list of location names\n"
 "- summary: one-sentence summary of the text"
 )
 },
 {
 "role": "user",
 "content": "Dr. Elena Torres, lead researcher at the Riverside Institute, presented her findings on marine biodiversity at the annual symposium in Cape Marina. The Oceanic Wildlife Fund and the Global Conservation Alliance both pledged support."
 }
 ],
 response_format={
 "type": "json_schema",
 "json_schema": {
 "name": "entity-extraction",
 "schema": {
 "type": "object",
 "properties": {
 "people": {"type": "array", "items": {"type": "string"}},
 "organizations": {"type": "array", "items": {"type": "string"}},
 "locations": {"type": "array", "items": {"type": "string"}},
 "summary": {"type": "string"}
 },
 "required": ["people", "organizations", "locations", "summary"]
 }
 }
 },
 max_tokens=4096,
 extra_body={
 "chat_template_kwargs": {"enable_thinking": True}
 }
)
message = response.choices[0].message
if hasattr(message, "reasoning") and message.reasoning:
 print("=== Thinking ===")
 print(message.reasoning)
print("\n=== Structured Output ===")
print(message.content)

fromvllmimport LLM, SamplingParams
fromPILimport Image
fromtransformersimport AutoProcessor
model_path = "google/gemma-4-31B-it"
processor = AutoProcessor.from_pretrained(model_path)
llm = LLM(
 model=model_path,
 tensor_parallel_size=2,
 max_model_len=8192,
 gpu_memory_utilization=0.90,
 trust_remote_code=True,
 limit_mm_per_prompt={"image": 4, "audio": 1}
)
# Text + Image example
image = Image.open("photo.jpg").convert("RGB")
messages = [
 {
 "role": "user",
 "content": [
 {"type": "image"},
 {"type": "text", "text": "Describe this image in detail."}
 ]
 }
]
prompt = processor.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
sampling_params = SamplingParams(temperature=0.0, max_tokens=1024)
outputs = llm.generate(
 {"prompt": prompt, "multi_modal_data": {"image": image}},
 sampling_params=sampling_params,
)
print(outputs[0].outputs[0].text)

vllmservegoogle/gemma-4-31B-it\
--tensor-parallel-size2\
--max-model-len8192\
--speculative-config'{"model": "google/gemma-4-31B-it-assistant", "num_speculative_tokens": 4}'

fromvllmimport LLM, SamplingParams
fromtransformersimport AutoTokenizer
model_path = "google/gemma-4-E4B-it"
tokenizer = AutoTokenizer.from_pretrained(model_path)
llm = LLM(
 model=model_path,
 speculative_config={
 "model": "google/gemma-4-E4B-it-assistant",
 "num_speculative_tokens": 4,
 },
 max_model_len=8192,
 trust_remote_code=True,
)
messages = [{"role": "user", "content": "What are the three laws of thermodynamics?"}]
prompt = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
outputs = llm.generate(prompt, SamplingParams(temperature=0.0, max_tokens=1024))
print(outputs[0].outputs[0].text)

vllmservegoogle/gemma-4-31B-it-qat-w4a16-ct\
--max-model-len32768\
--gpu-memory-utilization0.90

vllmservegoogle/gemma-4-26B-A4B-it\
--quantizationint8_per_channel_weight_only\
--max-model-len32768\
--gpu-memory-utilization0.90

vllmservegoogle/gemma-4-E4B-it-qat-mobile-ct

vllmservegoogle/gemma-4-31B-it\
--tensor-parallel-size2\
--max-model-len32768\
--no-enable-prefix-caching\
--limit-mm-per-prompt'{"image": 0, "audio": 0}'\
--async-scheduling

# Prompt-heavy benchmark (8k input / 1k output)
vllmbenchserve\
--modelgoogle/gemma-4-31B-it\
--dataset-namerandom\
--random-input-len8000\
--random-output-len1000\
--request-rate10000\
--num-prompts16\
--ignore-eos

============ Serving Benchmark Result ============
Successful requests: N
Benchmark duration (s): xxx.xx
Total input tokens: xxxxx
Total generated tokens: xxxxx
Request throughput (req/s): xxx.xx
Output token throughput (tok/s): xxx.xx
Total Token throughput (tok/s): xxx.xx
---------------Time to First Token----------------
Mean TTFT (ms): xxx.xx
Median TTFT (ms): xxx.xx
P99 TTFT (ms): xxx.xx
-----Time per Output Token (excl. 1st token)------
Mean TPOT (ms): xxx.xx
Median TPOT (ms): xxx.xx
P99 TPOT (ms): xxx.xx
==================================================

vllmservegoogle/gemma-4-31B-it\
--tensor-parallel-size2\
--max-model-len16384\
--gpu-memory-utilization0.90\
--enable-auto-tool-choice\
--reasoning-parsergemma4\
--tool-call-parsergemma4\
--chat-templateexamples/tool_chat_template_gemma4.jinja\
--limit-mm-per-prompt'{"image": 4, "audio": 1}'\
--async-scheduling\
--host0.0.0.0\
--port8000

pipinstallmodal
modalsetup# one-time: authenticate with Modal
modaldeploygemma4-modal.py

modalrungemma4-modal.py