Computer Use API: Letting AI Click Through GUIs

import anthropic
from PIL import Image
import base64, io, pyautogui

client = anthropic.Anthropic()

def screenshot_b64():
    img = pyautogui.screenshot()
    buf = io.BytesIO()
    img.save(buf, format="PNG")
    return base64.b64encode(buf.getvalue()).decode()

tools = [{
    "type": "computer_20250124",
    "name": "computer",
    "display_width_px": 1920,
    "display_height_px": 1080,
}]

messages = [{"role": "user", "content": "Open Calculator and compute 137*42."}]

while True:
    resp = client.beta.messages.create(
        model="claude-sonnet-4-6",
        max_tokens=2048,
        tools=tools,
        messages=messages,
        betas=["computer-use-2025-01-24"],
    )
    messages.append({"role": "assistant", "content": resp.content})
    if resp.stop_reason == "end_turn":
        break
    for block in resp.content:
        if block.type == "tool_use" and block.name == "computer":
            act = block.input
            if act["action"] == "screenshot":
                result = {"type": "image", "source": {"type": "base64",
                    "media_type": "image/png", "data": screenshot_b64()}}
            elif act["action"] == "left_click":
                pyautogui.click(act["coordinate"][0], act["coordinate"][1])
                result = "clicked"
            elif act["action"] == "type":
                pyautogui.typewrite(act["text"])
                result = "typed"
            # ... key, scroll, mouse_move, etc.
            messages.append({"role": "user", "content": [{
                "type": "tool_result",
                "tool_use_id": block.id,
                "content": [result] if isinstance(result, dict) else result
            }]})