Gemini Computer Use で構築する業務自動化ワークフロー — ブラウザ操作AIエージェントの設計と実装

import { chromium, Browser, Page } from "playwright";
 
let browser: Browser;
let page: Page;
 
async function initializeBrowser(): Promise<void> {
  browser = await chromium.launch({
    headless: false  // デバッグ時は表示
  });
 
  page = await browser.newPage();
  await page.setViewportSize({ width: 1280, height: 720 });
}
 
async function closeBrowser(): Promise<void> {
  await browser.close();
}
 
export { initializeBrowser, closeBrowser, page };

interface BoundingBox {
  x: number;
  y: number;
  width: number;
  height: number;
}
 
async function getElementCoordinates(selector: string): Promise<[number, number]> {
  const box = await page.locator(selector).boundingBox();
  if (!box) {
    throw new Error(`Element not found: ${selector}`);
  }
 
  // ボックスの中心座標
  const x = Math.round(box.x + box.width / 2);
  const y = Math.round(box.y + box.height / 2);
 
  return [x, y];
}
 
async function fillFormField(selector: string, value: string): Promise<void> {
  await page.locator(selector).fill(value);
  // 入力後のバリデーションイベント待機
  await page.waitForTimeout(500);
}
 
async function clickButton(selector: string): Promise<void> {
  await page.locator(selector).click();
  // ナビゲーション完了待機
  await page.waitForNavigation({ waitUntil: "networkidle" });
}

interface ExpenseReport {
  employeeId: string;
  date: string;
  items: {
    description: string;
    amount: number;
    category: string;
  }[];
  totalAmount: number;
}
 
async function automateExpenseSubmission(report: ExpenseReport): Promise<void> {
  console.log(`Processing expense report for employee ${report.employeeId}`);
 
  const instruction = `
You are an enterprise automation agent. Complete the following task:
 
1. Navigate to the expense reporting system at https://expenses.company.internal/
2. Log in with the provided credentials (use environment variables EXPENSE_USER and EXPENSE_PASS)
3. Click "New Expense Report"
4. Fill in the following details:
   - Employee ID: ${report.employeeId}
   - Report Date: ${report.date}
5. For each of the following expense items, add a row:
${report.items
  .map(
    (item) =>
      `   - Description: ${item.description}, Amount: ${item.amount}, Category: ${item.category}`
  )
  .join("\n")}
6. Verify the total matches ${report.totalAmount}
7. Click "Submit for Approval"
8. Confirm submission and take a final screenshot
 
After completing each step, explain what you've done and any observations.
  `;
 
  const result = await executeComputerUseAgent(instruction);
  console.log("Automation result:", result);
}
 
// 使用例
const testReport: ExpenseReport = {
  employeeId: "EMP12345",
  date: "2026-03-30",
  items: [
    { description: "Client meeting lunch", amount: 45.50, category: "meals" },
    { description: "Flight to NYC", amount: 350.00, category: "travel" },
    { description: "Hotel 2 nights", amount: 280.00, category: "lodging" }
  ],
  totalAmount: 675.50
};
 
await automateExpenseSubmission(testReport);

interface DataCollectionTask {
  sourceUrl: string;
  dataPoints: {
    name: string;
    selector: string;
  }[];
}
 
async function collectDashboardData(tasks: DataCollectionTask[]): Promise<Record<string, any>> {
  const collectedData: Record<string, any> = {};
 
  for (const task of tasks) {
    console.log(`Collecting data from ${task.sourceUrl}`);
 
    const instruction = `
You are a data collection agent. Complete this task:
 
1. Navigate to: ${task.sourceUrl}
2. Take a screenshot to see the current page
3. Locate and extract the following data points:
${task.dataPoints.map((point) => `   - ${point.name}: Use selector "${point.selector}" if available`).join("\n")}
4. If any data point requires scrolling or clicking to reveal, do so
5. Return the extracted data in JSON format with keys matching the data point names
 
Ensure all values are accurate and handle any authentication popups that may appear.
    `;
 
    const result = await executeComputerUseAgent(instruction);
 
    // 結果をパース（モデルが JSON 形式で返すことを期待）
    const jsonMatch = result.match(/\{[\s\S]*\}/);
    if (jsonMatch) {
      collectedData[task.sourceUrl] = JSON.parse(jsonMatch[0]);
    }
  }
 
  return collectedData;
}
 
// 使用例
const dashboardTasks: DataCollectionTask[] = [
  {
    sourceUrl: "https://analytics.example.com/dashboard",
    dataPoints: [
      { name: "totalRevenue", selector: ".revenue-metric" },
      { name: "activeUsers", selector: ".users-metric" },
      { name: "conversionRate", selector: ".conversion-metric" }
    ]
  }
];
 
const data = await collectDashboardData(dashboardTasks);
console.log("Collected data:", data);

interface UITestCase {
  name: string;
  steps: {
    action: "click" | "type" | "navigate" | "assert";
    target?: string;
    value?: string;
    expectedText?: string;
  }[];
}
 
async function runUITest(testCase: UITestCase): Promise<boolean> {
  console.log(`Running test: ${testCase.name}`);
 
  const stepsDescription = testCase.steps
    .map((step, index) => {
      switch (step.action) {
        case "navigate":
          return `${index + 1}. Navigate to: ${step.target}`;
        case "click":
          return `${index + 1}. Click on element: ${step.target}`;
        case "type":
          return `${index + 1}. Type "${step.value}" into: ${step.target}`;
        case "assert":
          return `${index + 1}. Verify text "${step.expectedText}" appears on the page`;
        default:
          return `${index + 1}. Unknown action`;
      }
    })
    .join("\n");
 
  const instruction = `
You are a UI testing agent. Execute the following test case and verify each step:
 
Test: ${testCase.name}
 
Steps to execute:
${stepsDescription}
 
After each step, take a screenshot and verify the expected outcome.
If any step fails, explain what went wrong.
At the end, provide a PASS or FAIL result.
  `;
 
  const result = await executeComputerUseAgent(instruction);
 
  // PASS/FAIL を判定
  const passed = result.toLowerCase().includes("pass");
  return passed;
}
 
// 使用例
const testCases: UITestCase[] = [
  {
    name: "User Login Flow",
    steps: [
      { action: "navigate", target: "https://app.example.com/login" },
      { action: "type", target: "#email-input", value: "test@example.com" },
      { action: "type", target: "#password-input", value: "password123" },
      { action: "click", target: "#login-button" },
      { action: "assert", expectedText: "Welcome to Dashboard" }
    ]
  },
  {
    name: "Create New Project",
    steps: [
      { action: "click", target: "#new-project-btn" },
      { action: "type", target: "#project-name", value: "Test Project" },
      { action: "type", target: "#project-description", value: "A test project" },
      { action: "click", target: "#create-btn" },
      { action: "assert", expectedText: "Project created successfully" }
    ]
  }
];
 
for (const test of testCases) {
  const passed = await runUITest(test);
  console.log(`${test.name}: ${passed ? "✓ PASS" : "✗ FAIL"}`);
}

interface SafetyPolicy {
  allowedDomains: string[];
  forbiddenActions: string[];
  requireUserConfirmation: boolean;
  maxIterations: number;
}
 
class SafetyService {
  private policy: SafetyPolicy;
 
  constructor(policy: SafetyPolicy) {
    this.policy = policy;
  }
 
  async validateAction(
    action: string,
    context: { currentUrl: string; elementSelector?: string }
  ): Promise<boolean> {
    // ドメイン確認
    const url = new URL(context.currentUrl);
    if (!this.policy.allowedDomains.includes(url.hostname)) {
      console.warn(`Domain not allowed: ${url.hostname}`);
      return false;
    }
 
    // 禁止アクション確認
    if (this.policy.forbiddenActions.some((forbidden) => action.includes(forbidden))) {
      console.warn(`Action not allowed: ${action}`);
      return false;
    }
 
    return true;
  }
 
  async requestUserConfirmation(message: string): Promise<boolean> {
    if (!this.policy.requireUserConfirmation) {
      return true;
    }
 
    // インタラクティブプロンプト（実装は環境依存）
    console.log(`SAFETY: ${message}`);
    console.log("Waiting for user confirmation...");
    return true;  // 本番では標準入力またはキューイングシステム
  }
}
 
// 使用例
const safetyPolicy: SafetyPolicy = {
  allowedDomains: ["app.company.internal", "expenses.company.internal"],
  forbiddenActions: ["navigate to external", "download executable"],
  requireUserConfirmation: true,
  maxIterations: 50
};
 
const safetyService = new SafetyService(safetyPolicy);
 
// エージェント実行前に確認
const canProceed = await safetyService.validateAction("click", {
  currentUrl: "https://app.company.internal/dashboard"
});
 
if (!canProceed) {
  console.error("Safety check failed. Aborting automation.");
}

interface ConfirmationRequest {
  action: string;
  detail: string;
  requiresApproval: boolean;
}
 
async function executeWithConfirmation(
  request: ConfirmationRequest
): Promise<boolean> {
  if (!request.requiresApproval) {
    return true;
  }
 
  console.log(`\n⚠️  APPROVAL REQUIRED`);
  console.log(`Action: ${request.action}`);
  console.log(`Details: ${request.detail}`);
  console.log("\nProceed? (yes/no)");
 
  // 実装: イベントキュー、ダッシュボード、メール通知など
  return await waitForUserApproval(request);
}
 
// エージェント内で使用
const confirmRequest: ConfirmationRequest = {
  action: "Submit expense report",
  detail: "Total amount: $675.50, Employee: EMP12345",
  requiresApproval: true
};
 
const approved = await executeWithConfirmation(confirmRequest);
if (!approved) {
  console.log("User rejected the action. Aborting.");
  process.exit(1);
}

interface RetryPolicy {
  maxAttempts: number;
  backoffMultiplier: number;
  initialDelayMs: number;
}
 
async function executeWithRetry<T>(
  fn: () => Promise<T>,
  policy: RetryPolicy
): Promise<T> {
  let lastError: Error | null = null;
  let delay = policy.initialDelayMs;
 
  for (let attempt = 1; attempt <= policy.maxAttempts; attempt++) {
    try {
      console.log(`Attempt ${attempt}/${policy.maxAttempts}`);
      return await fn();
    } catch (error) {
      lastError = error as Error;
 
      if (attempt < policy.maxAttempts) {
        console.log(`Attempt ${attempt} failed. Retrying in ${delay}ms...`);
        await new Promise((resolve) => setTimeout(resolve, delay));
        delay *= policy.backoffMultiplier;
      }
    }
  }
 
  throw lastError || new Error("All retry attempts failed");
}
 
// 使用例
const retryPolicy: RetryPolicy = {
  maxAttempts: 3,
  backoffMultiplier: 2,
  initialDelayMs: 1000
};
 
const result = await executeWithRetry(
  () => automateExpenseSubmission(testReport),
  retryPolicy
);

enum ErrorType {
  NETWORK_ERROR = "NETWORK_ERROR",
  AUTHENTICATION_FAILED = "AUTHENTICATION_FAILED",
  UI_NOT_FOUND = "UI_NOT_FOUND",
  VALIDATION_ERROR = "VALIDATION_ERROR",
  UNKNOWN = "UNKNOWN"
}
 
function classifyError(error: Error): ErrorType {
  const message = error.message.toLowerCase();
 
  if (message.includes("timeout") || message.includes("network")) {
    return ErrorType.NETWORK_ERROR;
  }
  if (message.includes("unauthorized") || message.includes("forbidden")) {
    return ErrorType.AUTHENTICATION_FAILED;
  }
  if (message.includes("not found") || message.includes("selector")) {
    return ErrorType.UI_NOT_FOUND;
  }
  if (message.includes("validation") || message.includes("invalid")) {
    return ErrorType.VALIDATION_ERROR;
  }
 
  return ErrorType.UNKNOWN;
}
 
async function handleError(error: Error, context: any): Promise<void> {
  const errorType = classifyError(error);
 
  switch (errorType) {
    case ErrorType.NETWORK_ERROR:
      console.log("Network error detected. Waiting 30 seconds before retry...");
      await new Promise((resolve) => setTimeout(resolve, 30000));
      break;
 
    case ErrorType.AUTHENTICATION_FAILED:
      console.error(
        "Authentication failed. Check credentials and try again."
      );
      throw error;
 
    case ErrorType.UI_NOT_FOUND:
      console.log("UI element not found. Taking screenshot for manual review.");
      const screenshot = await takeScreenshot();
      await saveScreenshot(`error_${Date.now()}.png`, screenshot);
      throw error;
 
    case ErrorType.VALIDATION_ERROR:
      console.log("Validation error. Correcting input data...");
      // 入力データ修正ロジック
      break;
 
    default:
      console.error("Unknown error:", error);
      throw error;
  }
}

async function processMultipleReports(
  reports: ExpenseReport[],
  concurrency: number = 3
): Promise<void> {
  const queue = [...reports];
  const running = new Set<Promise<void>>();
 
  while (queue.length > 0 || running.size > 0) {
    // 並列数制限内でタスク開始
    while (running.size < concurrency && queue.length > 0) {
      const report = queue.shift()!;
 
      const task = automateExpenseSubmission(report)
        .then(() => {
          console.log(`✓ Processed ${report.employeeId}`);
          running.delete(task);
        })
        .catch((error) => {
          console.error(`✗ Failed ${report.employeeId}:`, error);
          running.delete(task);
        });
 
      running.add(task);
    }
 
    // いずれかのタスク完了まで待機
    if (running.size > 0) {
      await Promise.race(running);
    }
  }
}

async function takeOptimizedScreenshot(): Promise<string> {
  // 低品質で取得（ファイルサイズ削減）
  const screenshot = await page.screenshot({
    encoding: "base64",
    quality: 60,  // JPEG 品質
    type: "jpeg"
  });
 
  return screenshot;
}

interface AutomationJob {
  id: string;
  type: "expense" | "data_collection" | "ui_test";
  status: "pending" | "running" | "completed" | "failed";
  data: any;
  createdAt: Date;
  completedAt?: Date;
  result?: any;
  error?: string;
}
 
class AutomationOrchestrator {
  private jobQueue: AutomationJob[] = [];
  private safetyService: SafetyService;
  private retryPolicy: RetryPolicy;
 
  constructor(safetyPolicy: SafetyPolicy, retryPolicy: RetryPolicy) {
    this.safetyService = new SafetyService(safetyPolicy);
    this.retryPolicy = retryPolicy;
  }
 
  async enqueueJob(job: AutomationJob): Promise<void> {
    this.jobQueue.push(job);
    console.log(`Job ${job.id} queued. Queue length: ${this.jobQueue.length}`);
  }
 
  async processQueue(): Promise<void> {
    while (this.jobQueue.length > 0) {
      const job = this.jobQueue.shift()!;
      await this.processJob(job);
    }
  }
 
  private async processJob(job: AutomationJob): Promise<void> {
    console.log(`\nProcessing job ${job.id} (type: ${job.type})`);
    job.status = "running";
 
    try {
      switch (job.type) {
        case "expense":
          job.result = await executeWithRetry(
            () => automateExpenseSubmission(job.data),
            this.retryPolicy
          );
          break;
 
        case "data_collection":
          job.result = await executeWithRetry(
            () => collectDashboardData(job.data),
            this.retryPolicy
          );
          break;
 
        case "ui_test":
          job.result = await executeWithRetry(
            () => runUITest(job.data),
            this.retryPolicy
          );
          break;
      }
 
      job.status = "completed";
      job.completedAt = new Date();
 
      console.log(`✓ Job ${job.id} completed successfully`);
    } catch (error) {
      job.status = "failed";
      job.error = (error as Error).message;
      job.completedAt = new Date();
 
      console.error(`✗ Job ${job.id} failed:`, error);
    }
 
    // ジョブ結果をログに記録
    await this.logJobResult(job);
  }
 
  private async logJobResult(job: AutomationJob): Promise<void> {
    // 実装: データベース、ログサービス、メール通知など
    console.log(`Logging result for job ${job.id}`);
  }
}
 
// 使用例
const orchestrator = new AutomationOrchestrator(safetyPolicy, retryPolicy);
 
// ジョブを投入
const expenseJob: AutomationJob = {
  id: "job-001",
  type: "expense",
  status: "pending",
  data: testReport,
  createdAt: new Date()
};
 
await orchestrator.enqueueJob(expenseJob);
await orchestrator.processQueue();

import * as logging from "@google-cloud/logging";
 
const log = logging.log("automation-system");
 
async function logAutomationEvent(
  event: string,
  details: any
): Promise<void> {
  const metadata = {
    severity: "INFO",
    timestamp: new Date().toISOString()
  };
 
  log.write(log.entry(metadata, { event, details }));
}
 
// 使用例
await logAutomationEvent("job_started", {
  jobId: "job-001",
  type: "expense",
  employeeId: "EMP12345"
});

import * as schedule from "node-schedule";
 
// 毎日午前 2 時に経費処理を実行
schedule.scheduleJob("0 2 * * *", async () => {
  console.log("Running scheduled expense processing...");
 
  const reports = await fetchPendingExpenseReports();
  await orchestrator.processMultipleReports(reports);
});

interface AutomationMetrics {
  totalJobsProcessed: number;
  successCount: number;
  failureCount: number;
  averageExecutionTimeMs: number;
  lastExecutionAt: Date;
}
 
function getMetrics(): AutomationMetrics {
  // 実装: メトリクス集計
  return {
    totalJobsProcessed: 1250,
    successCount: 1200,
    failureCount: 50,
    averageExecutionTimeMs: 45000,
    lastExecutionAt: new Date()
  };
}

// ❌ モデルが返した座標をそのまま渡すと、2x DPR 環境でクリックがズレる
const { x, y } = action.coordinate;
await page.mouse.click(x, y);

// ✅ スクリーンショットのピクセル空間 → CSS ピクセルへ正規化してから渡す
const dpr = await page.evaluate(() => window.devicePixelRatio || 1);
const cssX = action.coordinate.x / dpr;
const cssY = action.coordinate.y / dpr;
await page.mouse.click(cssX, cssY);