Gemma 4: From Edge E2B to Cloud 31B—Choosing the Right Model and Implementation Patterns

from transformers import AutoProcessor, AutoModelForCausalLM
import torch
 
model = AutoModelForCausalLM.from_pretrained(
    "google/gemma-4-31b-it",
    torch_dtype=torch.bfloat16,
    device_map="auto"
)
processor = AutoProcessor.from_pretrained("google/gemma-4-31b-it")
 
text = "Explain quantum computing in simple terms."
inputs = processor(text=text, return_tensors="pt")
outputs = model.generate(**inputs, max_new_tokens=256)
print(processor.decode(outputs[0], skip_special_tokens=True))

from PIL import Image
 
image = Image.open("path/to/image.jpg")
# Or: image = Image.open(urlopen("https://example.com/image.jpg"))
 
text = "What is shown in this image? Describe in detail."
inputs = processor(text=text, images=image, return_tensors="pt")
outputs = model.generate(**inputs, max_new_tokens=512)
print(processor.decode(outputs[0], skip_special_tokens=True))

images = [Image.open(f"image_{i}.jpg") for i in range(3)]
 
text = "Compare these three images. What are their primary differences?"
inputs = processor(text=text, images=images, return_tensors="pt")
outputs = model.generate(**inputs, max_new_tokens=1024)

from PIL import Image
 
def smart_resize(image, target_size=512):
    """Maintain aspect ratio during resize"""
    width, height = image.size
    aspect = width / height
    
    if aspect > 1:  # Landscape
        new_width = target_size
        new_height = int(target_size / aspect)
    else:  # Portrait
        new_height = target_size
        new_width = int(target_size * aspect)
    
    return image.resize((new_width, new_height), Image.Resampling.LANCZOS)

pip install transformers torch pillow datasets
pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu121
pip install bitsandbytes accelerate

from transformers import AutoModelForCausalLM, AutoProcessor, BitsAndBytesConfig
import torch
 
bnb_config = BitsAndBytesConfig(
    load_in_8bit=True,
    llm_int8_threshold=200.0,
    llm_int8_has_fp16_weight=False
)
 
model = AutoModelForCausalLM.from_pretrained(
    "google/gemma-4-31b-it",
    quantization_config=bnb_config,
    device_map="auto"
)
processor = AutoProcessor.from_pretrained("google/gemma-4-31b-it")

model = AutoModelForCausalLM.from_pretrained(
    "google/gemma-4-31b-it",
    torch_dtype=torch.bfloat16,
    device_map="auto"
)

def generate_response(prompt, max_tokens=256, temperature=0.7, top_p=0.95):
    """Core text generation function"""
    inputs = processor(text=prompt, return_tensors="pt").to(model.device)
    
    with torch.no_grad():
        outputs = model.generate(
            **inputs,
            max_new_tokens=max_tokens,
            temperature=temperature,
            top_p=top_p,
            do_sample=True
        )
    
    return processor.decode(outputs[0], skip_special_tokens=True)
 
response = generate_response("Best practices for reducing technical debt?")
print(response)

from transformers import TextIteratorStreamer
from threading import Thread
 
def stream_response(prompt, max_tokens=512):
    """Streaming output implementation"""
    inputs = processor(text=prompt, return_tensors="pt").to(model.device)
    
    streamer = TextIteratorStreamer(
        processor,
        skip_special_tokens=True,
        skip_prompt=True
    )
    
    generation_kwargs = dict(
        inputs,
        max_new_tokens=max_tokens,
        streamer=streamer,
        temperature=0.7,
        top_p=0.95
    )
    
    thread = Thread(target=model.generate, kwargs=generation_kwargs)
    thread.start()
    
    for text in streamer:
        print(text, end="", flush=True)
    
    thread.join()

curl -fsSL https://ollama.ai/install.sh | sh

ollama run gemma-4-e2b      # Edge-optimized
ollama run gemma-4-e4b      # Balanced
ollama run gemma-4-26b      # Enterprise (GPU recommended)
ollama run gemma-4-31b      # Maximum performance (high-end GPU)

import requests
import json
 
def ollama_inference(prompt, model="gemma-4-31b"):
    response = requests.post(
        "http://localhost:11434/api/generate",
        json={"model": model, "prompt": prompt, "stream": False}
    )
    return json.loads(response.text)["response"]
 
response = ollama_inference("Explain microservices architecture advantages")
print(response)

tools = [
    {
        "name": "get_weather",
        "description": "Retrieve weather for specified city",
        "parameters": {
            "type": "object",
            "properties": {
                "city": {"type": "string", "description": "City name"},
                "units": {
                    "type": "string",
                    "enum": ["celsius", "fahrenheit"]
                }
            },
            "required": ["city"]
        }
    },
    {
        "name": "web_search",
        "description": "Search the web",
        "parameters": {
            "type": "object",
            "properties": {"query": {"type": "string"}},
            "required": ["query"]
        }
    }
]

def agent_loop(initial_prompt, max_iterations=10):
    messages = [{"role": "user", "content": initial_prompt}]
    
    for iteration in range(max_iterations):
        response = model.generate(
            messages=messages,
            tools=tools,
            temperature=0.7
        )
        
        if response.get("type") == "function_call":
            func_name = response["function_name"]
            result = execute_function(func_name, response["arguments"])
            
            messages.append({
                "role": "assistant",
                "content": f"Executed {func_name}"
            })
            messages.append({
                "role": "user",
                "content": f"Result: {result}"
            })
        else:
            return response["content"]
    
    return "Max iterations reached"

from peft import get_peft_model, LoraConfig, TaskType
from transformers import Trainer, TrainingArguments
 
lora_config = LoraConfig(
    r=16,
    lora_alpha=32,
    target_modules=["q_proj", "v_proj", "k_proj", "o_proj"],
    lora_dropout=0.05,
    bias="none",
    task_type=TaskType.CAUSAL_LM
)
 
model = get_peft_model(model, lora_config)
 
training_args = TrainingArguments(
    output_dir="./results",
    num_train_epochs=3,
    per_device_train_batch_size=4,
    gradient_accumulation_steps=4,
    learning_rate=2e-4,
    warmup_steps=100,
    weight_decay=0.01,
    fp16=True
)
 
trainer = Trainer(
    model=model,
    args=training_args,
    train_dataset=train_dataset,
    eval_dataset=eval_dataset
)
 
trainer.train()