Building AI-Powered iOS and Android Apps with the Gemini API — Image Recognition, Voice Analysis, Chat, and Monetization

// ImageAnalyzer.swift
import Foundation
import UIKit
 
class ImageAnalyzer {
    private let backendURL = "https://YOUR_REGION-YOUR_PROJECT.cloudfunctions.net/analyzeImage"
 
    // Analyze an image and return text results
    func analyze(image: UIImage, prompt: String) async throws -> String {
        // Resize to 800px max dimension to reduce token count and cost
        let resized = resizeImage(image, maxDimension: 800)
        guard let imageData = resized.jpegData(compressionQuality: 0.7) else {
            throw AnalyzerError.imageConversionFailed
        }
        let base64 = imageData.base64EncodedString()
 
        // Get Firebase Auth token
        let token = try await fetchAuthToken()
 
        // Send to backend proxy
        var request = URLRequest(url: URL(string: backendURL)!)
        request.httpMethod = "POST"
        request.setValue("Bearer \(token)", forHTTPHeaderField: "Authorization")
        request.setValue("application/json", forHTTPHeaderField: "Content-Type")
 
        let body: [String: Any] = ["imageBase64": base64, "prompt": prompt]
        request.httpBody = try JSONSerialization.data(withJSONObject: body)
 
        let (data, _) = try await URLSession.shared.data(for: request)
        let response = try JSONDecoder().decode(AnalysisResponse.self, from: data)
        return response.text
    }
 
    private func resizeImage(_ image: UIImage, maxDimension: CGFloat) -> UIImage {
        let size = image.size
        let ratio = min(maxDimension / size.width, maxDimension / size.height)
        if ratio >= 1.0 { return image }
        let newSize = CGSize(width: size.width * ratio, height: size.height * ratio)
        UIGraphicsBeginImageContextWithOptions(newSize, false, 1.0)
        image.draw(in: CGRect(origin: .zero, size: newSize))
        let resized = UIGraphicsGetImageFromCurrentImageContext()!
        UIGraphicsEndImageContext()
        return resized
    }
}
 
struct AnalysisResponse: Codable {
    let text: String
}
 
enum AnalyzerError: Error {
    case imageConversionFailed
}

// ImageAnalysisViewModel.kt
import androidx.lifecycle.ViewModel
import androidx.lifecycle.viewModelScope
import kotlinx.coroutines.flow.MutableStateFlow
import kotlinx.coroutines.flow.StateFlow
import kotlinx.coroutines.launch
import android.graphics.Bitmap
 
class ImageAnalysisViewModel(
    private val geminiRepository: GeminiRepository
) : ViewModel() {
 
    private val _analysisResult = MutableStateFlow<AnalysisState>(AnalysisState.Idle)
    val analysisResult: StateFlow<AnalysisState> = _analysisResult
 
    fun analyzeImage(bitmap: Bitmap, prompt: String) {
        viewModelScope.launch {
            _analysisResult.value = AnalysisState.Loading
 
            // Resize to 800px for cost and speed optimization
            val resized = resizeBitmap(bitmap, 800)
 
            geminiRepository.analyzeImage(resized, prompt)
                .onSuccess { result ->
                    _analysisResult.value = AnalysisState.Success(result)
                }
                .onFailure { error ->
                    _analysisResult.value = AnalysisState.Error(error.message ?: "Unknown error")
                }
        }
    }
 
    private fun resizeBitmap(bitmap: Bitmap, maxDimension: Int): Bitmap {
        val ratio = minOf(
            maxDimension.toFloat() / bitmap.width,
            maxDimension.toFloat() / bitmap.height
        )
        if (ratio >= 1.0f) return bitmap
        val newWidth = (bitmap.width * ratio).toInt()
        val newHeight = (bitmap.height * ratio).toInt()
        return Bitmap.createScaledBitmap(bitmap, newWidth, newHeight, true)
    }
}
 
sealed class AnalysisState {
    object Idle : AnalysisState()
    object Loading : AnalysisState()
    data class Success(val result: String) : AnalysisState()
    data class Error(val message: String) : AnalysisState()
}

// VoiceAnalyzer.swift (iOS)
import Foundation
import AVFoundation
 
class VoiceAnalyzer: NSObject, AVAudioRecorderDelegate {
    private var audioRecorder: AVAudioRecorder?
    private var recordingURL: URL?
 
    // Start recording
    func startRecording() throws {
        let session = AVAudioSession.sharedInstance()
        try session.setCategory(.record, mode: .default)
        try session.setActive(true)
 
        let documentsPath = FileManager.default.urls(for: .documentDirectory, in: .userDomainMask)[0]
        recordingURL = documentsPath.appendingPathComponent("temp_recording.m4a")
 
        let settings: [String: Any] = [
            AVFormatIDKey: Int(kAudioFormatMPEG4AAC),
            AVSampleRateKey: 44100,
            AVNumberOfChannelsKey: 1,
            AVEncoderAudioQualityKey: AVAudioQuality.high.rawValue
        ]
 
        audioRecorder = try AVAudioRecorder(url: recordingURL!, settings: settings)
        audioRecorder?.record()
    }
 
    // Stop recording and analyze
    func stopAndAnalyze(task: String) async throws -> String {
        audioRecorder?.stop()
        guard let url = recordingURL else { throw VoiceError.noRecording }
 
        let audioData = try Data(contentsOf: url)
        let base64Audio = audioData.base64EncodedString()
 
        // Route through backend proxy to Gemini API
        return try await sendAudioToBackend(
            audioBase64: base64Audio,
            mimeType: "audio/mp4",
            task: task
        )
    }
}
 
enum VoiceError: Error {
    case noRecording
}

// VoiceAnalysisRepository.kt (Android)
import android.content.Context
import android.media.MediaRecorder
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.withContext
import java.io.File
 
class VoiceAnalysisRepository(
    private val context: Context,
    private val backendClient: BackendClient
) {
    private var mediaRecorder: MediaRecorder? = null
    private var outputFile: File? = null
 
    fun startRecording() {
        outputFile = File(context.cacheDir, "temp_audio.m4a")
        mediaRecorder = MediaRecorder(context).apply {
            setAudioSource(MediaRecorder.AudioSource.MIC)
            setOutputFormat(MediaRecorder.OutputFormat.MPEG_4)
            setAudioEncoder(MediaRecorder.AudioEncoder.AAC)
            setAudioSamplingRate(44100)
            setAudioEncodingBitRate(128000)
            setOutputFile(outputFile!!.absolutePath)
            prepare()
            start()
        }
    }
 
    suspend fun stopAndAnalyze(task: String): Result<String> = withContext(Dispatchers.IO) {
        mediaRecorder?.stop()
        mediaRecorder?.release()
        mediaRecorder = null
 
        val audioData = outputFile?.readBytes()
            ?: return@withContext Result.failure(Exception("No recording found"))
        val base64Audio = android.util.Base64.encodeToString(audioData, android.util.Base64.NO_WRAP)
 
        backendClient.analyzeAudio(base64Audio, "audio/mp4", task)
    }
}

// ChatSession.swift (iOS)
import Foundation
 
class ChatSession: ObservableObject {
    @Published var messages: [ChatMessage] = []
    private let backendClient: BackendClient
    private let systemInstruction: String
 
    init(
        backendClient: BackendClient,
        systemInstruction: String = "You are a helpful assistant."
    ) {
        self.backendClient = backendClient
        self.systemInstruction = systemInstruction
    }
 
    // Send a message with streaming support
    func sendMessage(_ text: String) async {
        let userMessage = ChatMessage(role: "user", content: text)
        messages.append(userMessage)
 
        // Placeholder for the assistant's streaming response
        let assistantMessage = ChatMessage(role: "assistant", content: "")
        messages.append(assistantMessage)
        let assistantIndex = messages.count - 1
 
        do {
            let stream = try await backendClient.streamChat(
                history: messages.dropLast(),
                systemInstruction: systemInstruction
            )
 
            for try await chunk in stream {
                await MainActor.run {
                    messages[assistantIndex].content += chunk
                }
            }
        } catch {
            await MainActor.run {
                messages[assistantIndex].content = "An error occurred: \(error.localizedDescription)"
            }
        }
    }
}
 
struct ChatMessage: Identifiable {
    let id = UUID()
    let role: String
    var content: String
}

// Firebase Functions: stream-chat.ts
import * as functions from "firebase-functions";
import * as admin from "firebase-admin";
import { GoogleGenerativeAI, Content } from "@google/generative-ai";
 
const genAI = new GoogleGenerativeAI(process.env.GEMINI_API_KEY!);
 
export const streamChat = functions.https.onRequest(async (req, res) => {
  const token = req.headers.authorization?.split("Bearer ")[1];
  if (!token) { res.status(401).send("Unauthorized"); return; }
  await admin.auth().verifyIdToken(token);
 
  const { history, systemInstruction } = req.body;
 
  // Server-Sent Events headers
  res.setHeader("Content-Type", "text/event-stream");
  res.setHeader("Cache-Control", "no-cache");
  res.setHeader("Connection", "keep-alive");
 
  const model = genAI.getGenerativeModel({
    model: "gemini-2.5-flash",
    systemInstruction: systemInstruction,
  });
 
  // Convert history to Gemini API format
  const geminiHistory: Content[] = history.map((msg: any) => ({
    role: msg.role === "assistant" ? "model" : "user",
    parts: [{ text: msg.content }],
  }));
 
  const chat = model.startChat({ history: geminiHistory });
 
  const lastMessage = history[history.length - 1].content;
  const streamResult = await chat.sendMessageStream(lastMessage);
 
  for await (const chunk of streamResult.stream) {
    const text = chunk.text();
    res.write(`data: ${JSON.stringify({ text })}\n\n`);
  }
 
  res.write("data: [DONE]\n\n");
  res.end();
});

// personalized-system-prompt.ts
function buildSystemPrompt(userProfile: UserProfile): string {
  return `You are a personal AI assistant for ${userProfile.name}.
 
User context:
- Language preference: ${userProfile.language}
- Expertise level: ${userProfile.level} (beginner/intermediate/expert)
- Interests: ${userProfile.interests.join(", ")}
- Previous interaction count: ${userProfile.interactionCount}
 
Guidelines:
- Respond in ${userProfile.language}
- Adjust complexity to their ${userProfile.level} level
- Reference their interests when genuinely relevant
- Be concise: maximum 3 paragraphs unless asked for detail
- Never request personal information`;
}

// model-selector.ts
type ModelTier = "flash-lite" | "flash" | "pro";
 
function selectModel(task: TaskType, contextLength: number): string {
  const modelMap: Record<ModelTier, string> = {
    "flash-lite": "gemini-2.5-flash-lite",   // Cheapest, fastest
    "flash": "gemini-2.5-flash",              // Best balance
    "pro": "gemini-2.5-pro",                 // Highest accuracy
  };
 
  // Simple classification tasks — Flash Lite is sufficient
  if (task === "sentiment" || task === "classification") {
    return modelMap["flash-lite"];
  }
 
  // General chat and image analysis — Flash
  if (contextLength < 10000) {
    return modelMap["flash"];
  }
 
  // Long context or complex reasoning — Pro
  return modelMap["pro"];
}

// response-cache.ts (Cloudflare Workers KV example)
class ResponseCache {
  private readonly TTL = 3600; // Cache for 1 hour
 
  async get(cacheKey: string): Promise<string | null> {
    return await CACHE_KV.get(cacheKey);
  }
 
  async set(cacheKey: string, response: string): Promise<void> {
    await CACHE_KV.put(cacheKey, response, { expirationTtl: this.TTL });
  }
 
  buildKey(prompt: string, imageHash?: string): string {
    const base = imageHash ? `${imageHash}_${prompt}` : prompt;
    return btoa(base).slice(0, 32);
  }
}

// image-optimizer.ts
function getTargetResolution(purpose: string): number {
  const resolutionMap: Record<string, number> = {
    "ocr": 1200,         // Text recognition needs higher resolution
    "object-detect": 800, // Object detection works at medium resolution
    "thumbnail": 400,    // Thumbnail analysis is fine at low resolution
    "default": 600,
  };
  return resolutionMap[purpose] ?? resolutionMap["default"];
}
// Apply resizing before base64 encoding to reduce token count significantly

# batch_processor.py
import google.generativeai as genai
import asyncio
from typing import List, Dict
 
genai.configure(api_key="YOUR_GEMINI_API_KEY")
 
async def process_batch(items: List[Dict]) -> List[str]:
    """Process multiple items in parallel with rate limiting"""
    model = genai.GenerativeModel("gemini-2.5-flash")
 
    async def process_single(item: Dict) -> str:
        response = await model.generate_content_async(
            item["prompt"],
            generation_config=genai.types.GenerationConfig(
                max_output_tokens=512,  # Limit output tokens
                temperature=0.7,
            )
        )
        return response.text
 
    # Max 5 concurrent requests to respect rate limits
    semaphore = asyncio.Semaphore(5)
 
    async def bounded_process(item: Dict) -> str:
        async with semaphore:
            return await process_single(item)
 
    results = await asyncio.gather(*[bounded_process(item) for item in items])
    return list(results)
 
# Example usage:
# items = [{"prompt": "Summarize: ..."}, {"prompt": "Translate: ..."}]
# Expected output: ["Summary text", "Translated text"]

// secure-proxy.ts
import { GoogleGenerativeAI } from "@google/generative-ai";
 
// ✅ Correct: API key stored only on the server
const genAI = new GoogleGenerativeAI(process.env.GEMINI_API_KEY!);
 
// ❌ Never do this: API keys in client code are exposed during reverse engineering
// const API_KEY = "YOUR_API_KEY";  // GitHub will flag this with Secret Scanning

// PrivacyManager.swift (iOS)
class PrivacyManager {
    // Mask PII before sending to the API
    static func sanitizeForAPI(_ text: String) -> String {
        var sanitized = text
 
        // Mask email addresses
        let emailPattern = "[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\\.[A-Za-z]{2,}"
        sanitized = sanitized.replacingOccurrences(
            of: emailPattern,
            with: "[EMAIL]",
            options: .regularExpression
        )
 
        // Mask phone numbers
        let phonePattern = "\\d{2,4}-\\d{2,4}-\\d{4}"
        sanitized = sanitized.replacingOccurrences(
            of: phonePattern,
            with: "[PHONE]",
            options: .regularExpression
        )
 
        return sanitized
    }
}

// rate-limiter.ts
import * as admin from "firebase-admin";
 
class RateLimiter {
  private readonly HOURLY_LIMIT = 20;
  private readonly db = admin.firestore();
 
  async checkAndIncrement(userId: string): Promise<boolean> {
    const now = Date.now();
    const windowStart = now - 3600000; // 1 hour
    const ref = this.db.collection("rate_limits").doc(userId);
 
    return this.db.runTransaction(async (transaction) => {
      const doc = await transaction.get(ref);
      const data = doc.data() ?? { requests: [] };
 
      const recentRequests = (data.requests as number[]).filter(
        (t) => t > windowStart
      );
 
      if (recentRequests.length >= this.HOURLY_LIMIT) {
        return false; // Rate limit exceeded
      }
 
      transaction.set(ref, { requests: [...recentRequests, now] });
      return true;
    });
  }
}

// PurchaseManager.swift (StoreKit 2)
import StoreKit
 
class PurchaseManager: ObservableObject {
    @Published var subscriptionStatus: SubscriptionStatus = .free
 
    func purchaseSubscription(_ product: Product) async throws {
        let result = try await product.purchase()
 
        switch result {
        case .success(let verification):
            let transaction = try checkVerified(verification)
            await updateSubscriptionStatus()
            await transaction.finish()
        case .userCancelled:
            break
        case .pending:
            break
        @unknown default:
            break
        }
    }
 
    func canUseFeature(_ feature: AIFeature) -> Bool {
        switch (subscriptionStatus, feature) {
        case (.free, .basicChat):
            return remainingFreeRequests > 0
        case (.standard, _), (.premium, _):
            return true
        default:
            return false
        }
    }
}

// exponential-backoff.ts
async function withRetry<T>(
  fn: () => Promise<T>,
  maxRetries = 3
): Promise<T> {
  for (let attempt = 0; attempt <= maxRetries; attempt++) {
    try {
      return await fn();
    } catch (error: any) {
      if (error?.status === 429 && attempt < maxRetries) {
        const delay = Math.pow(2, attempt) * 1000 + Math.random() * 1000;
        console.log(`Rate limited. Retrying in ${delay}ms...`);
        await new Promise(resolve => setTimeout(resolve, delay));
      } else {
        throw error;
      }
    }
  }
  throw new Error("Max retries exceeded");
}

// model-router.ts — per-feature model routing (proxy server side)
type Feature = "chat" | "image_describe" | "tag_classify" | "voice_summary";
 
const MODEL_BY_FEATURE: Record<Feature, string> = {
  chat: "gemini-3.5-flash",            // conversation quality first
  image_describe: "gemini-3.5-flash",  // keep multimodal on Flash
  tag_classify: "gemma-4",             // classification is fine on a light model
  voice_summary: "gemini-3.5-flash-lite", // summaries optimize for cost
};
 
export function resolveModel(feature: Feature): string {
  return MODEL_BY_FEATURE[feature] ?? "gemini-3.5-flash-lite";
}