Building Enterprise Automation Workflows with Gemini Computer Use — Designing and Implementing Browser-Based AI Agents

import { Browser, Page } from "playwright";
 
class BrowserAutomation {
  private browser: Browser;
  private page: Page;
 
  async initialize(): Promise<void> {
    this.browser = await chromium.launch({
      headless: process.env.HEADLESS !== "false"
    });
 
    this.page = await this.browser.newPage();
    await this.page.setViewportSize({ width: 1280, height: 720 });
 
    // Event logging for debugging
    this.page.on("console", (msg) =>
      console.log(`Browser console: ${msg.text()}`)
    );
    this.page.on("pageerror", (error) =>
      console.error(`Browser error: ${error}`)
    );
  }
 
  async navigateTo(url: string): Promise<void> {
    await this.page.goto(url, { waitUntil: "networkidle" });
    await this.page.waitForTimeout(1000);  // Additional stabilization
  }
 
  async takeScreenshot(): Promise<string> {
    return await this.page.screenshot({
      encoding: "base64",
      quality: 80
    });
  }
 
  async getElementCoordinates(selector: string): Promise<[number, number]> {
    const box = await this.page.locator(selector).boundingBox();
    if (!box) throw new Error(`Selector not found: ${selector}`);
 
    return [
      Math.round(box.x + box.width / 2),
      Math.round(box.y + box.height / 2)
    ];
  }
 
  async fillField(selector: string, value: string): Promise<void> {
    await this.page.locator(selector).clear();
    await this.page.locator(selector).fill(value);
    await this.page.waitForTimeout(300);  // Validation debounce
  }
 
  async clickButton(selector: string, waitNav: boolean = true): Promise<void> {
    if (waitNav) {
      await Promise.all([
        this.page.waitForNavigation({ waitUntil: "networkidle" }),
        this.page.locator(selector).click()
      ]);
    } else {
      await this.page.locator(selector).click();
    }
  }
 
  async getPageText(): Promise<string> {
    return await this.page.evaluate(() => document.body.innerText);
  }
 
  async close(): Promise<void> {
    await this.browser.close();
  }
}
 
export { BrowserAutomation };

interface ExpenseItem {
  description: string;
  amount: number;
  category: "meals" | "travel" | "lodging" | "other";
  receipt?: string;
}
 
interface ExpenseReport {
  employeeId: string;
  reportDate: string;
  items: ExpenseItem[];
  totalAmount: number;
  approverEmail?: string;
}
 
async function automateExpenseReportSubmission(
  report: ExpenseReport
): Promise<void> {
  const instruction = `
You are an enterprise automation agent. Process an expense report with these details:
 
Employee ID: ${report.employeeId}
Report Date: ${report.reportDate}
Total Amount: $${report.totalAmount}
 
Expense items:
${report.items
  .map(
    (item) =>
      `- ${item.description}: $${item.amount} (${item.category}${item.receipt ? ` - Receipt: ${item.receipt}` : ""})`
  )
  .join("\n")}
 
Steps to complete:
1. Navigate to the company expense system
2. Login if prompted (use available browser session)
3. Start a new expense report
4. Fill all required fields for the report header
5. Add each expense item, selecting the correct category
6. Attach receipts if provided
7. Verify the total matches
8. Submit for approval
9. Confirm the submission message
 
After each major step, verify the action succeeded.
  `;
 
  const result = await executeAutomationTask(instruction);
  console.log("Expense submission result:", result.message);
 
  // Store audit trail
  await logAutomationEvent("expense_submitted", {
    employeeId: report.employeeId,
    amount: report.totalAmount,
    itemCount: report.items.length,
    timestamp: new Date().toISOString()
  });
}
 
// Usage
const testExpense: ExpenseReport = {
  employeeId: "EMP-2026-001",
  reportDate: "2026-03-30",
  items: [
    {
      description: "Client meeting at restaurant",
      amount: 67.50,
      category: "meals"
    },
    {
      description: "Round-trip flight NYC to SF",
      amount: 425.00,
      category: "travel"
    },
    {
      description: "Hotel accommodations 2 nights",
      amount: 320.00,
      category: "lodging"
    }
  ],
  totalAmount: 812.50
};
 
await automateExpenseReportSubmission(testExpense);

interface DataPoint {
  label: string;
  description: string;
}
 
interface DataCollectionJob {
  name: string;
  sourceUrl: string;
  loginRequired: boolean;
  dataPoints: DataPoint[];
}
 
async function collectDashboardData(job: DataCollectionJob): Promise<Record<string, any>> {
  const instruction = `
You are a data extraction agent. Complete this task:
 
Task Name: ${job.name}
Source URL: ${job.sourceUrl}
${job.loginRequired ? "Note: You may need to login" : ""}
 
Extract the following data points:
${job.dataPoints
  .map((point) => `- ${point.label}: ${point.description}`)
  .join("\n")}
 
Steps:
1. Navigate to ${job.sourceUrl}
2. Perform any necessary login
3. Take a screenshot to understand the layout
4. Locate each data point
5. Handle any pagination or expansion needed
6. Return extracted values in JSON format: {"label": value, ...}
 
Be thorough and ensure accuracy. If a data point is not visible after reasonable effort, indicate "not found".
  `;
 
  const result = await executeAutomationTask(instruction);
 
  // Parse JSON from response
  const jsonMatch = result.message.match(/\{[\s\S]*\}/);
  if (!jsonMatch) {
    throw new Error("Failed to extract JSON from automation result");
  }
 
  return JSON.parse(jsonMatch[0]);
}
 
// Usage
const analyticsJob: DataCollectionJob = {
  name: "Quarterly Analytics Pull",
  sourceUrl: "https://analytics.company.internal/dashboard",
  loginRequired: false,
  dataPoints: [
    { label: "Total Revenue Q1", description: "Sum of all Q1 revenue" },
    { label: "Active Users", description: "Current monthly active users" },
    { label: "Conversion Rate", description: "Overall site conversion %" },
    { label: "Top 3 Products", description: "Best-selling products by revenue" }
  ]
};
 
const data = await collectDashboardData(analyticsJob);
console.log("Collected analytics:", data);

interface TestStep {
  description: string;
  action: "navigate" | "click" | "fill" | "assert" | "screenshot";
  selector?: string;
  text?: string;
  expectedText?: string;
}
 
interface TestCase {
  name: string;
  description: string;
  steps: TestStep[];
}
 
async function runUITest(test: TestCase): Promise<{ passed: boolean; evidence: string }> {
  const stepsDescription = test.steps
    .map((step, i) => {
      switch (step.action) {
        case "navigate":
          return `${i + 1}. Navigate to ${step.text}`;
        case "click":
          return `${i + 1}. Click element containing "${step.text}"`;
        case "fill":
          return `${i + 1}. Fill form field with "${step.text}"`;
        case "assert":
          return `${i + 1}. Verify text "${step.expectedText}" appears`;
        case "screenshot":
          return `${i + 1}. Take screenshot for verification`;
        default:
          return `${i + 1}. Unknown action`;
      }
    })
    .join("\n");
 
  const instruction = `
You are a UI testing agent. Execute this test case and report results:
 
Test: ${test.name}
Description: ${test.description}
 
Execute these steps in order:
${stepsDescription}
 
For each step:
1. Execute the action
2. Wait for any loading to complete
3. Take a screenshot
4. Verify the expected outcome
 
After all steps, provide:
- Overall PASS/FAIL status
- Any failures with details
- Screenshots for evidence
 
If any step fails, stop and report why.
  `;
 
  const result = await executeAutomationTask(instruction);
 
  const passed = result.message.toLowerCase().includes("pass");
  return {
    passed,
    evidence: result.message
  };
}
 
// Usage
const loginTest: TestCase = {
  name: "User Login Flow",
  description: "Verify login functionality works correctly",
  steps: [
    {
      description: "Navigate to login page",
      action: "navigate",
      text: "https://app.example.com/login"
    },
    {
      description: "Enter email",
      action: "fill",
      text: "testuser@example.com"
    },
    {
      description: "Enter password",
      action: "fill",
      text: "TestPassword123!"
    },
    {
      description: "Click login button",
      action: "click",
      text: "Login"
    },
    {
      description: "Verify dashboard appears",
      action: "assert",
      expectedText: "Welcome to Dashboard"
    },
    {
      description: "Capture final state",
      action: "screenshot"
    }
  ]
};
 
const testResult = await runUITest(loginTest);
console.log(`Test ${loginTest.name}: ${testResult.passed ? "✓ PASS" : "✗ FAIL"}`);
console.log(testResult.evidence);

interface SafetyPolicy {
  allowedDomains: string[];
  allowedActions: string[];
  requireApprovalFor: string[];
  maxSequentialActions: number;
  timeoutMs: number;
}
 
class SafetyGate {
  private policy: SafetyPolicy;
 
  constructor(policy: SafetyPolicy) {
    this.policy = policy;
  }
 
  validateDomain(url: string): boolean {
    try {
      const hostname = new URL(url).hostname;
      return this.policy.allowedDomains.some(
        (domain) => hostname === domain || hostname.endsWith(`.${domain}`)
      );
    } catch {
      return false;
    }
  }
 
  requiresApproval(action: string): boolean {
    return this.policy.requireApprovalFor.some((approval) =>
      action.toLowerCase().includes(approval.toLowerCase())
    );
  }
 
  async validateAndExecute(
    action: string,
    fn: () => Promise<void>
  ): Promise<void> {
    if (!this.policy.allowedActions.includes(action)) {
      throw new Error(`Action not allowed: ${action}`);
    }
 
    if (this.requiresApproval(action)) {
      const approved = await this.requestUserApproval(
        `Approval required for: ${action}`
      );
      if (!approved) throw new Error("User rejected action");
    }
 
    await Promise.race([
      fn(),
      new Promise((_, reject) =>
        setTimeout(
          () => reject(new Error("Execution timeout")),
          this.policy.timeoutMs
        )
      )
    ]);
  }
 
  private async requestUserApproval(message: string): Promise<boolean> {
    // Implementation: webhook, queue, dashboard notification
    console.log(`APPROVAL REQUIRED: ${message}`);
    return true;  // Placeholder
  }
}
 
// Configuration
const safetyPolicy: SafetyPolicy = {
  allowedDomains: [
    "app.company.internal",
    "expenses.company.internal",
    "analytics.company.internal"
  ],
  allowedActions: ["click", "type", "screenshot", "scroll"],
  requireApprovalFor: ["submit", "delete", "transfer"],
  maxSequentialActions: 100,
  timeoutMs: 300000
};
 
const safetyGate = new SafetyGate(safetyPolicy);

interface AuditEntry {
  id: string;
  timestamp: Date;
  action: string;
  status: "pending_approval" | "approved" | "rejected" | "executed";
  details: any;
  executedBy?: string;
  approvedBy?: string;
}
 
class AuditLogger {
  private entries: AuditEntry[] = [];
 
  log(action: string, details: any): void {
    const entry: AuditEntry = {
      id: `audit-${Date.now()}-${Math.random()}`,
      timestamp: new Date(),
      action,
      status: "pending_approval",
      details
    };
 
    this.entries.push(entry);
    console.log(`[AUDIT] ${action}:`, details);
  }
 
  approve(entryId: string, approverEmail: string): void {
    const entry = this.entries.find((e) => e.id === entryId);
    if (entry) {
      entry.status = "approved";
      entry.approvedBy = approverEmail;
    }
  }
 
  async exportLog(format: "json" | "csv"): Promise<string> {
    if (format === "json") {
      return JSON.stringify(this.entries, null, 2);
    }
 
    // CSV format
    const headers = ["Timestamp", "Action", "Status", "Details"];
    const rows = this.entries.map((e) => [
      e.timestamp.toISOString(),
      e.action,
      e.status,
      JSON.stringify(e.details)
    ]);
 
    return [headers, ...rows].map((row) => row.join(",")).join("\n");
  }
}
 
const auditLogger = new AuditLogger();

interface RetryConfig {
  maxAttempts: number;
  baseDelayMs: number;
  backoffFactor: number;
  timeoutMs: number;
}
 
async function executeWithRetry<T>(
  task: () => Promise<T>,
  config: RetryConfig
): Promise<T> {
  let lastError: Error;
  let delay = config.baseDelayMs;
 
  for (let attempt = 1; attempt <= config.maxAttempts; attempt++) {
    try {
      console.log(
        `Attempt ${attempt}/${config.maxAttempts}: Starting task...`
      );
 
      const result = await Promise.race([
        task(),
        new Promise<T>((_, reject) =>
          setTimeout(
            () => reject(new Error("Task timeout")),
            config.timeoutMs
          )
        )
      ]);
 
      console.log(`✓ Task succeeded on attempt ${attempt}`);
      return result;
    } catch (error) {
      lastError = error as Error;
      console.error(`✗ Attempt ${attempt} failed: ${lastError.message}`);
 
      if (attempt < config.maxAttempts) {
        console.log(`Waiting ${delay}ms before retry...`);
        await new Promise((resolve) => setTimeout(resolve, delay));
        delay *= config.backoffFactor;
      }
    }
  }
 
  throw new Error(`Failed after ${config.maxAttempts} attempts: ${lastError.message}`);
}
 
// Usage
const retryConfig: RetryConfig = {
  maxAttempts: 3,
  baseDelayMs: 2000,
  backoffFactor: 2,
  timeoutMs: 120000
};
 
try {
  await executeWithRetry(
    () => automateExpenseReportSubmission(testExpense),
    retryConfig
  );
} catch (error) {
  console.error("Automation failed:", error);
  await notifyAdmins(`Automation failed: ${error}`);
}

enum ErrorCategory {
  TRANSIENT = "transient",     // Retry likely succeeds
  AUTH = "authentication",      // Requires manual intervention
  UI_NOT_FOUND = "ui_not_found", // Selector/element changed
  VALIDATION = "validation",    // Input data invalid
  NETWORK = "network",          // Connectivity issue
  UNKNOWN = "unknown"
}
 
function classifyError(error: Error): ErrorCategory {
  const msg = error.message.toLowerCase();
 
  const patterns = {
    [ErrorCategory.TRANSIENT]: [
      "econnreset",
      "etimedout",
      "temporarily unavailable"
    ],
    [ErrorCategory.AUTH]: ["401", "403", "unauthorized", "forbidden"],
    [ErrorCategory.UI_NOT_FOUND]: ["selector", "not found", "element"],
    [ErrorCategory.VALIDATION]: ["invalid", "required field", "format"],
    [ErrorCategory.NETWORK]: ["enotfound", "getaddrinfo", "connection refused"]
  };
 
  for (const [category, keywords] of Object.entries(patterns)) {
    if (keywords.some((kw) => msg.includes(kw))) {
      return category as ErrorCategory;
    }
  }
 
  return ErrorCategory.UNKNOWN;
}
 
function shouldRetry(category: ErrorCategory): boolean {
  return [
    ErrorCategory.TRANSIENT,
    ErrorCategory.NETWORK,
    ErrorCategory.UI_NOT_FOUND
  ].includes(category);
}

async function executeParallelTasks(
  tasks: ExpenseReport[],
  concurrency: number = 5
): Promise<Map<string, any>> {
  const results = new Map<string, any>();
  const queue = [...tasks];
  const running = new Set<Promise<void>>();
 
  while (queue.length > 0 || running.size > 0) {
    // Fill concurrency slots
    while (running.size < concurrency && queue.length > 0) {
      const report = queue.shift()!;
 
      const task = automateExpenseReportSubmission(report)
        .then(() => {
          results.set(report.employeeId, { status: "success" });
          running.delete(task);
        })
        .catch((error) => {
          results.set(report.employeeId, {
            status: "failed",
            error: error.message
          });
          running.delete(task);
        });
 
      running.add(task);
    }
 
    // Wait for slot to open
    if (running.size >= concurrency) {
      await Promise.race(running);
    }
  }
 
  return results;
}
 
// Usage
const reports = await fetchPendingExpenses();
const results = await executeParallelTasks(reports, 5);
 
let successCount = 0;
let failureCount = 0;
 
for (const [empId, result] of results) {
  if (result.status === "success") successCount++;
  else failureCount++;
}
 
console.log(
  `Batch complete: ${successCount} succeeded, ${failureCount} failed`
);

async function captureOptimizedScreenshot(page: Page): Promise<string> {
  return await page.screenshot({
    encoding: "base64",
    type: "jpeg",
    quality: 65  // Balance quality vs. size
  });
}

interface ScheduledJob {
  id: string;
  type: string;
  schedule: string;  // Cron expression
  enabled: boolean;
  lastRun?: Date;
  nextRun?: Date;
}
 
class OrchestrationEngine {
  private jobs: ScheduledJob[] = [];
  private executing: Set<string> = new Set();
 
  registerJob(job: ScheduledJob): void {
    this.jobs.push(job);
    this.scheduleJob(job);
  }
 
  private scheduleJob(job: ScheduledJob): void {
    // Implementation: node-schedule, cron, or cloud scheduler
    console.log(`Scheduled job: ${job.id} (${job.schedule})`);
  }
 
  async executeJob(jobId: string): Promise<void> {
    if (this.executing.has(jobId)) {
      console.log(`Job already running: ${jobId}`);
      return;
    }
 
    this.executing.add(jobId);
 
    try {
      const job = this.jobs.find((j) => j.id === jobId);
      if (!job || !job.enabled) return;
 
      console.log(`[${new Date().toISOString()}] Starting job: ${jobId}`);
 
      // Execute job logic based on type
      // This would route to appropriate handlers
 
      job.lastRun = new Date();
      await this.logJobExecution(job);
    } finally {
      this.executing.delete(jobId);
    }
  }
 
  private async logJobExecution(job: ScheduledJob): Promise<void> {
    // Write to monitoring system, database, logs
    console.log(`Job ${job.id} completed at ${job.lastRun}`);
  }
}

// ❌ Passing the model's raw coordinates drifts on 2x-DPR displays
const { x, y } = action.coordinate;
await page.mouse.click(x, y);

// ✅ Normalize from screenshot pixel space to CSS pixels first
const dpr = await page.evaluate(() => window.devicePixelRatio || 1);
const cssX = action.coordinate.x / dpr;
const cssY = action.coordinate.y / dpr;
await page.mouse.click(cssX, cssY);