Gemini 機能の「使われ方」を測る — 受け入れ率・再生成率・編集距離で見る製品計測の設計

// Before: 生成するだけで、何も記録に残らない
import FirebaseAI
 
let model = FirebaseAI.firebaseAI(backend: .googleAI())
    .generativeModel(modelName: "gemini-3.5-flash")
 
func generateDailyMessage(context: DailyContext) async throws -> String {
    let response = try await model.generateContent(buildPrompt(context))
    return response.text ?? ""
}

// After: 生成・再生成・失敗・受け入れがすべて1箇所で記録される
import FirebaseAI
import FirebaseAnalytics
 
struct GenerationResult {
    let text: String
    let latencyMs: Int
    let attempt: Int
}
 
final class MeasuredGenerator {
    static let feature = "daily_message"
    static let promptVersion = "daily_message_v2"
    static let modelId = "gemini-3.5-flash"
 
    private let model = FirebaseAI.firebaseAI(backend: .googleAI())
        .generativeModel(modelName: modelId)
 
    // attempt: 1 = 初回生成、2以降 = ユーザーの「作り直す」タップ
    func generate(context: DailyContext, attempt: Int) async throws -> GenerationResult {
        let started = Date()
        do {
            let response = try await model.generateContent(buildPrompt(context))
            let text = response.text ?? ""
            let latencyMs = Int(Date().timeIntervalSince(started) * 1000)
            Analytics.logEvent(attempt == 1 ? "ai_generated" : "ai_regenerated", parameters: [
                "feature": Self.feature,
                "prompt_version": Self.promptVersion,
                "model_id": Self.modelId,
                "latency_ms": latencyMs,
                "output_chars": text.count,
                "attempt": attempt,
            ])
            return GenerationResult(text: text, latencyMs: latencyMs, attempt: attempt)
        } catch {
            Analytics.logEvent("ai_generation_failed", parameters: [
                "feature": Self.feature,
                "prompt_version": Self.promptVersion,
                "model_id": Self.modelId,
                "error_domain": (error as NSError).domain,
                "error_code": (error as NSError).code,
            ])
            throw error
        }
    }
 
    // 保存・シェアなど「受け入れ」操作のタイミングで呼ぶ
    func logAccepted(generated: String, saved: String) {
        let distance = normalizedEditDistance(generated, saved)
        Analytics.logEvent("ai_accepted", parameters: [
            "feature": Self.feature,
            "prompt_version": Self.promptVersion,
            // GA4 のパラメータは数値型が安全なので、0〜1 の距離を 100 倍した整数で送る
            "edit_distance_x100": Int(distance * 100),
        ])
    }
}

func normalizedEditDistance(_ a: String, _ b: String) -> Double {
    let s = Array(a), t = Array(b)
    if s.isEmpty || t.isEmpty {
        return s.count == t.count ? 0 : 1
    }
    var prev = Array(0...t.count)
    var curr = [Int](repeating: 0, count: t.count + 1)
    for i in 1...s.count {
        curr[0] = i
        for j in 1...t.count {
            let cost = s[i - 1] == t[j - 1] ? 0 : 1
            curr[j] = min(prev[j] + 1, curr[j - 1] + 1, prev[j - 1] + cost)
        }
        swap(&prev, &curr)
    }
    return Double(prev[t.count]) / Double(max(s.count, t.count))
}

WITH events AS (
  SELECT
    event_name,
    (SELECT value.string_value FROM UNNEST(event_params) WHERE key = 'prompt_version') AS prompt_version,
    (SELECT value.int_value    FROM UNNEST(event_params) WHERE key = 'latency_ms')         AS latency_ms,
    (SELECT value.int_value    FROM UNNEST(event_params) WHERE key = 'edit_distance_x100') AS edit_x100
  FROM `myapp.analytics_123456789.events_*`
  WHERE _TABLE_SUFFIX BETWEEN '20260601' AND '20260614'
    AND (SELECT value.string_value FROM UNNEST(event_params) WHERE key = 'feature') = 'daily_message'
)
SELECT
  prompt_version,
  COUNTIF(event_name = 'ai_generated')   AS generated,
  COUNTIF(event_name = 'ai_regenerated') AS regenerated,
  COUNTIF(event_name = 'ai_accepted')    AS accepted,
  SAFE_DIVIDE(
    COUNTIF(event_name = 'ai_accepted'),
    COUNTIF(event_name = 'ai_generated')
  ) AS acceptance_rate,
  SAFE_DIVIDE(
    COUNTIF(event_name = 'ai_regenerated'),
    COUNTIF(event_name = 'ai_generated') + COUNTIF(event_name = 'ai_regenerated')
  ) AS regenerate_share,
  APPROX_QUANTILES(latency_ms, 100)[OFFSET(95)] AS latency_p95_ms,
  APPROX_QUANTILES(edit_x100, 100)[OFFSET(50)]  AS edit_distance_median_x100
FROM events
GROUP BY prompt_version
ORDER BY prompt_version