Apple Vision Framework × Gemini API のハイブリッド画像認識 ─ 壁紙アプリでクラウド推論コストを 7 割削減した設計メモ

import Vision
import CoreImage
 
struct ImagePrefilterResult {
    let shouldSendToCloud: Bool
    let coarseCategory: CoarseCategory
    let sharpnessScore: Double
    let saliencyArea: Double
    let reasonIfRejected: String?
}
 
enum CoarseCategory: String, Codable {
    case photoLike, illustrationLike, abstract, unknown
}
 
final class ImagePrefilter {
    private let saliencyRequest = VNGenerateAttentionBasedSaliencyImageRequest()
    private let classifyRequest = VNClassifyImageRequest()
 
    func analyze(ciImage: CIImage) async throws -> ImagePrefilterResult {
        let handler = VNImageRequestHandler(ciImage: ciImage, options: [:])
 
        try handler.perform([saliencyRequest, classifyRequest])
 
        let sharpness = sharpnessScore(of: ciImage)
        let saliencyArea = saliencyAreaRatio(from: saliencyRequest.results?.first)
        let coarse = coarseCategory(from: classifyRequest.results ?? [])
 
        if sharpness < 0.18 {
            return .init(shouldSendToCloud: false,
                         coarseCategory: coarse,
                         sharpnessScore: sharpness,
                         saliencyArea: saliencyArea,
                         reasonIfRejected: "too_blurry")
        }
        if saliencyArea < 0.06 {
            return .init(shouldSendToCloud: false,
                         coarseCategory: coarse,
                         sharpnessScore: sharpness,
                         saliencyArea: saliencyArea,
                         reasonIfRejected: "low_saliency")
        }
        if isOwnAppGeneratedWatermark(ciImage: ciImage) {
            return .init(shouldSendToCloud: false,
                         coarseCategory: coarse,
                         sharpnessScore: sharpness,
                         saliencyArea: saliencyArea,
                         reasonIfRejected: "own_watermark")
        }
        return .init(shouldSendToCloud: true,
                     coarseCategory: coarse,
                     sharpnessScore: sharpness,
                     saliencyArea: saliencyArea,
                     reasonIfRejected: nil)
    }
 
    private func sharpnessScore(of image: CIImage) -> Double {
        // Laplacian variance approximation via CIConvolution3X3
        let kernel: [CGFloat] = [0, -1, 0, -1, 4, -1, 0, -1, 0]
        let filter = CIFilter.convolution3X3()
        filter.inputImage = image
        filter.weights = CIVector(values: kernel.map { Float($0) }, count: 9)
        filter.bias = 0.5
        guard let output = filter.outputImage else { return 0 }
        let stats = variance(of: output)
        // Normalize to 0...1 (壁紙アプリの実データで decile を取って正規化)
        return min(stats / 0.42, 1.0)
    }
}

import GoogleGenerativeAI
 
struct WallpaperClassification: Codable {
    let primaryCategory: String
    let secondaryCategory: String?
    let mood: String
    let recommendedTags: [String]
    let dominantColors: [String]
    let suitableScreenTypes: [String]
}
 
final class WallpaperClassifier {
    private let model: GenerativeModel
 
    init(apiKey: String) {
        self.model = GenerativeModel(
            name: "gemini-2.5-flash",
            apiKey: apiKey,
            generationConfig: GenerationConfig(
                temperature: 0.2,
                topP: 0.95,
                maxOutputTokens: 512,
                responseMIMEType: "application/json",
                responseSchema: Self.responseSchema
            )
        )
    }
 
    func classify(image: UIImage,
                  coarseHint: CoarseCategory) async throws -> WallpaperClassification {
        let resized = image.resizedKeepingAspect(maxSide: 512)
        let jpegData = resized.jpegData(compressionQuality: 0.78)!
 
        let prompt = """
        あなたは壁紙アプリの分類エンジンです。次の画像を 1 枚見て、JSON で分類結果を返してください。
        オンデバイスの粗い事前分類では「\(coarseHint.rawValue)」と判定されています。これは参考情報です。
 
        primaryCategory は次の 18 カテゴリから 1 つ:
        nature, abstract, person, animal, vehicle, architecture, space, urban,
        anime, japanese_art, cyberpunk, lofi, minimalist, geometric, food, sports,
        seasonal, other
 
        mood は次から 1 つ:
        calm, energetic, mysterious, romantic, melancholic, playful, sacred, neutral
 
        recommendedTags は 3〜6 件、英小文字スネークケース。
        dominantColors は #rrggbb の 3 件まで。
        suitableScreenTypes は "iphone_lock", "iphone_home", "ipad", "mac_desktop" から該当するもの。
        """
 
        let response = try await model.generateContent(
            prompt,
            ModelContent.Part.data(mimetype: "image/jpeg", jpegData)
        )
        guard let text = response.text,
              let data = text.data(using: .utf8) else {
            throw ClassifierError.emptyResponse
        }
        return try JSONDecoder().decode(WallpaperClassification.self, from: data)
    }
}