diff --git a/README.md b/README.md index 01992cb..4c82196 100644 --- a/README.md +++ b/README.md @@ -51,6 +51,7 @@ Then follow the guide's own README to install dependencies and run it. | [Amp Code Coding Agent](typescript/amp/amp-sdk) | Coding agent powered by the Amp Code CLI. | | [Claude Two-Agent System](typescript/anthropic/multi-agent-claude-sdk) | Two-agent Claude system coordinating work across sandboxes. | | [Claude Coding Agent](typescript/anthropic/single-claude-agent-sdk) | Single Claude Code agent you drive from the CLI. | +| [Claude Draws](typescript/computer-use/claude-draws) | Claude drawing on a browser sketchpad in a sandbox desktop, via the Anthropic computer toolset. | | [Brainbase Universal Harness API](typescript/brainbase/universal-harness-api) | Background coding agent on any harness via Brainbase's Universal Harness API, running in Daytona sandboxes. | | [Devin CLI Coding Agent](typescript/cognition/devin-cli) | Coding agent powered by Cognition's Devin CLI. | | [Convex AI App Builder](typescript/convex/ai-app-builder) | AI app builder on the Daytona Convex component — LLM-generated apps run in sandboxes with live build state and iframe previews. | diff --git a/python/computer-use/claude-draws/README.md b/python/computer-use/claude-draws/README.md index cf66f5a..b87297a 100644 --- a/python/computer-use/claude-draws/README.md +++ b/python/computer-use/claude-draws/README.md @@ -2,6 +2,8 @@ This example opens a canvas sketchpad in a Chromium window on a Daytona sandbox desktop and lets Claude draw in it with the mouse, through the Anthropic SDK's computer toolset. +A [TypeScript version](../../../typescript/computer-use/claude-draws) of this example is also available. + ## What you will build - A sandbox desktop at 1280x800, created and deleted by `DaytonaComputer` from the [`daytona-claude-toolsets`](https://pypi.org/project/daytona-claude-toolsets/) package. diff --git a/python/computer-use/claude-draws/pyproject.toml b/python/computer-use/claude-draws/pyproject.toml index 3ccd0ea..693dd73 100644 --- a/python/computer-use/claude-draws/pyproject.toml +++ b/python/computer-use/claude-draws/pyproject.toml @@ -8,8 +8,8 @@ dependencies = [ # `draw.py` imports `daytona` itself, so it is declared here rather than relied on as a # transitive dependency. Same range `daytona-claude-toolsets` pins, so the two cannot disagree. "daytona>=0.223.0,<0.224.0", - # Floor: the release that ships `anthropic.tools.computer` (computer_toolset_20260801). - "anthropic>=1.9.0", + # Floor: the first release that ships `anthropic.tools.computer` (computer_toolset_20260801). + "anthropic>=1.12.0", ] [tool.setuptools] diff --git a/typescript/computer-use/claude-draws/.env.example b/typescript/computer-use/claude-draws/.env.example new file mode 100644 index 0000000..46bce14 --- /dev/null +++ b/typescript/computer-use/claude-draws/.env.example @@ -0,0 +1,14 @@ +# Required: Daytona API key, from https://app.daytona.io/dashboard/keys +DAYTONA_API_KEY= + +# Required: Anthropic API key, from https://console.anthropic.com/settings/keys +ANTHROPIC_API_KEY= + +# Optional: override the model the drawing loop uses +# MODEL=claude-sonnet-5-5 + +# Optional: cap the number of model/tool iterations (default: 120) +# MAX_TURNS=120 + +# Optional: save a final desktop screenshot after the model finishes +# OUTPUT_PATH=./claude-draws.png diff --git a/typescript/computer-use/claude-draws/.gitignore b/typescript/computer-use/claude-draws/.gitignore new file mode 100644 index 0000000..2c5c837 --- /dev/null +++ b/typescript/computer-use/claude-draws/.gitignore @@ -0,0 +1,15 @@ +# Dependencies +node_modules/ + +# Environment variables +.env + +# Build output +dist/ + +# Logs +*.log + +# OS generated files +.DS_Store +Thumbs.db diff --git a/typescript/computer-use/claude-draws/README.md b/typescript/computer-use/claude-draws/README.md new file mode 100644 index 0000000..4bc596e --- /dev/null +++ b/typescript/computer-use/claude-draws/README.md @@ -0,0 +1,87 @@ +# Claude Draws on Daytona + +This example opens a canvas sketchpad in a Chromium window on a Daytona sandbox desktop and lets Claude draw in it with the mouse, through the Anthropic SDK's computer toolset. + +The [Python version](../../../python/computer-use/claude-draws) of this example is also available. + +## What you will build + +- A sandbox desktop at 1280x800, created and deleted by `DaytonaComputer` from the [`@daytona/claude-toolsets`](https://www.npmjs.com/package/@daytona/claude-toolsets) package. +- A self-contained HTML sketchpad, `paint.html`, uploaded into the sandbox and opened in a visible Chromium window with no tab strip or omnibox. +- A drawing loop where `client.beta.messages.toolRunner` hands the computer toolset to the model, which takes screenshots, scrolls the color palette, picks swatches by name, and drags the mouse to draw. +- An unattended `confirm` callback that logs every action the model takes and approves it, because the desktop is a throwaway sandbox. + +## Requirements + +- Node.js 20 or higher on your machine. +- A Daytona account and API key. +- An Anthropic API key. + +## Setup + +```bash +git clone https://github.com/daytona/guides.git +cd guides/typescript/computer-use/claude-draws +npm install +cp .env.example .env # macOS/Linux +# Windows PowerShell: Copy-Item .env.example .env +# edit .env with your Daytona and Anthropic API keys +``` + +On macOS and Linux, load and export `.env` in the shell as shown below. PowerShell users can set the variables with `$env:` instead. + +Environment variables: + +- `DAYTONA_API_KEY`: required. Get it from the [Daytona Dashboard](https://app.daytona.io/dashboard/keys). +- `ANTHROPIC_API_KEY`: required, for the model loop. +- `MODEL`: optional, the model the drawing loop uses. Defaults to `claude-sonnet-5-5`. +- `MAX_TURNS`: optional, maximum model/tool iterations. Defaults to `120`. +- `OUTPUT_PATH`: optional path for a final PNG screenshot after the model finishes. + +## Workflow + +1. Put your keys in `.env` and run the script: + + ```bash + set -a; source .env; set +a # macOS/Linux + npm run start + ``` + + On Windows PowerShell, set them in the session instead: + + ```powershell + $env:DAYTONA_API_KEY = "" + $env:ANTHROPIC_API_KEY = "" + npm run start + ``` + +2. Watch the run in your terminal. Each line is one step of the loop: + + ```text + sandbox 7f3c…, screen 1280x800 + sketchpad open at file:///tmp/claude-draws/paint.html; handing the desktop to the model + [confirm] screenshot {} + [screenshot] {} + [text] I can see the sketchpad. The palette along the bottom… + [confirm] scroll {"coordinate":[640,720],"scroll_direction":"right","scroll_amount":3} + ``` + +3. The last line totals what the drawing loop cost, summed over every assistant turn: + + ```text + tokens: 412905 input, 6124 output + ``` + + Screenshots dominate the input count: the model takes one after most actions, and each is billed as an image. Cache and server-tool counters are reported separately by the API and are not folded into these two numbers. + +`src/draw.ts` creates the sandbox, starts its desktop, uploads `paint.html`, and launches Chromium through an async Daytona process session — a foreground GUI process never returns, so a synchronous command would block until the browser exits. It waits for Chromium's loopback debugging port to answer before handing the desktop to the model, then prints the model's text and tool calls until the task finishes. The sandbox is deleted when the run ends. + +The task prompt is fixed, in `TASK` at the top of `src/draw.ts`. It targets colors by their visible label (`orange`, `teal`, `rose`), never by position, because the palette scrolls horizontally and swatch coordinates are not stable. It also keeps the model inside the drawing band, roughly 120 to 620 pixels from the top, where the floating docks do not intercept clicks. Edit `TASK` to draw something else, and keep both of those constraints. + +Model-driven GUI runs are not deterministic: the same prompt will not produce the same picture twice, and the model can misread the screen or miss a swatch. Treat the result as a demo of the loop, not as a reproducible render. + +## Files + +- `src/draw.ts`: creates the sandbox, uploads and opens the sketchpad, and runs the model's drawing loop. +- `paint.html`: the self-contained sketchpad — canvas, brush sizes, a 38-color palette, and a caption box, with no external assets. It also exposes a small read-only `window.paint` inspection surface (pixel probes, the selected color, a log of the strokes drawn) so a run can be checked from the devtools console or a browser test instead of by eye. +- `.env.example`: environment variable template. diff --git a/typescript/computer-use/claude-draws/package.json b/typescript/computer-use/claude-draws/package.json new file mode 100644 index 0000000..27787bb --- /dev/null +++ b/typescript/computer-use/claude-draws/package.json @@ -0,0 +1,24 @@ +{ + "name": "claude-draws", + "version": "0.1.0", + "private": true, + "description": "Let Claude draw on a canvas sketchpad in a Daytona sandbox desktop", + "type": "module", + "engines": { + "node": ">=20" + }, + "scripts": { + "start": "tsx src/draw.ts", + "typecheck": "tsc --noEmit" + }, + "dependencies": { + "@anthropic-ai/sdk": ">=0.132.0", + "@daytona/claude-toolsets": "^1.0.0", + "@daytona/sdk": "^0.223.0" + }, + "devDependencies": { + "@types/node": "^24.0.0", + "tsx": "^4.20.0", + "typescript": "^5.9.0" + } +} diff --git a/typescript/computer-use/claude-draws/paint.html b/typescript/computer-use/claude-draws/paint.html new file mode 100644 index 0000000..f8d0b04 --- /dev/null +++ b/typescript/computer-use/claude-draws/paint.html @@ -0,0 +1,891 @@ + + + + + +Paint — canvas sketchpad + + + + + + +
+
+ + Paint +
+ +
+ +
+ Brush +
+
+ +
+ + + +
+ +
+ + black · 8px +
+ + +
+ +
+
+ Palette + scroll sideways +
+
+
+
+
+ + + + diff --git a/typescript/computer-use/claude-draws/src/draw.ts b/typescript/computer-use/claude-draws/src/draw.ts new file mode 100644 index 0000000..b7f75c0 --- /dev/null +++ b/typescript/computer-use/claude-draws/src/draw.ts @@ -0,0 +1,207 @@ +/** Let Claude draw on a sketchpad running in a Daytona sandbox desktop. */ + +import Anthropic from '@anthropic-ai/sdk' +import type { BetaComputerConfirmContext } from '@anthropic-ai/sdk/helpers/beta/toolsets' +import { DaytonaComputer } from '@daytona/claude-toolsets' +import type { Sandbox } from '@daytona/sdk' +import { randomInt } from 'node:crypto' +import { readFile, stat, writeFile } from 'node:fs/promises' +import { fileURLToPath } from 'node:url' +import { setTimeout as sleep } from 'node:timers/promises' + +const PAINT_HTML = fileURLToPath(new URL('../paint.html', import.meta.url)) + +// Paths inside the sandbox. The profile doubles as HOME for the scrubbed Chromium environment. +const REMOTE_DIR = '/tmp/claude-draws' +const REMOTE_HTML = `${REMOTE_DIR}/paint.html` +const PROFILE = '/tmp/claude-draws-profile' +const CHROMIUM = 'chromium' +const SESSION_ID = 'claude-draws-chromium' + +// The desktop size the sandbox boots with, and the window Chromium opens at. +const WIDTH = 1280 +const HEIGHT = 800 + +// The task names swatches rather than relying on positions because the palette scrolls horizontally. +const TASK = `The screen shows a sketchpad web app called Paint, already open and maximized. Draw in it with the mouse. + +How the app works: + +- A floating toolbar docks along the top of the window: brush sizes labeled fine, small, medium and bold, a caption box, a live readout, and a Clear button. +- A floating color palette docks along the bottom. It scrolls horizontally and only shows part of itself at a time. Every swatch shows its color name as text under the color chip. +- The canvas fills the whole window behind those two docks. + +How to work: + +1. Take a screenshot first and find the toolbar and the palette. +2. Choose colors by NAME, never by position. Scroll the palette sideways until the swatch whose label you want is visible, then click that swatch. The readout in the top toolbar then shows the selected color name, so you can confirm the click landed before you draw. +3. Draw only in the middle band of the screen, roughly 120 to 620 pixels from the top. The floating toolbar and palette sit outside that band and will swallow clicks meant for the canvas. +4. Drag the mouse to draw freehand. Hold shift and click to draw a straight line from the last point you drew to the point you click, so shift-clicks chain into a polyline. + +What to draw, a simple sunset over water: + +- A large round sun in the upper middle of the drawing band, with the swatch labeled "orange". +- A straight horizon line across the drawing band and a few wave strokes under it, with the swatch labeled "teal". +- Two or three small clouds above the horizon, with the swatch labeled "rose". +- Finally click the caption box in the top toolbar and type: sunset by claude + +Take a screenshot when you are finished and describe what you drew.` + +function confirm(context: BetaComputerConfirmContext): boolean { + console.log(`[confirm] ${context.member} ${JSON.stringify(context.input)}`) + return true +} + +function shellQuote(value: string): string { + return `'${value.replaceAll("'", `'"'"'`)}'` +} + +function positiveInteger(name: string, fallback: number): number { + const raw = process.env[name] + if (raw === undefined) return fallback + const value = Number(raw) + if (!Number.isSafeInteger(value) || value <= 0) { + throw new Error(`${name} must be a positive integer`) + } + return value +} + +async function uploadSketchpad(sandbox: Sandbox): Promise { + await sandbox.process.executeCommand( + `mkdir -p -m 700 ${shellQuote(REMOTE_DIR)} ${shellQuote(PROFILE)}`, + ) + await sandbox.fs.uploadFile(await readFile(PAINT_HTML), REMOTE_HTML) +} + +async function readChromiumLog(sandbox: Sandbox): Promise { + try { + const result = await sandbox.process.executeCommand( + `tail -n 40 ${shellQuote(`${PROFILE}/chromium.log`)}`, + undefined, + undefined, + 10, + ) + if (result.exitCode !== 0) { + return `` + } + return result.result.trim() || '' + } catch (error) { + return `` + } +} + +async function launchChromium(sandbox: Sandbox): Promise { + const port = randomInt(20_000, 40_000) + const isRoot = (await sandbox.process.executeCommand('id -u')).result.trim() === '0' + const flags = [ + '--remote-debugging-address=127.0.0.1', + `--remote-debugging-port=${port}`, + `--user-data-dir=${PROFILE}`, + `--window-size=${WIDTH},${HEIGHT}`, + '--start-maximized', + '--no-first-run', + '--no-default-browser-check', + '--disable-background-networking', + '--disable-component-update', + '--disable-default-apps', + '--disable-extensions', + '--disable-sync', + '--password-store=basic', + '--disable-features=BackForwardCache', + ...(isRoot ? ['--no-sandbox'] : []), + `--app=file://${REMOTE_HTML}`, + ] + const environment = [ + `HOME=${PROFILE}`, + 'PATH=/usr/local/bin:/usr/bin:/bin', + 'LANG=C.UTF-8', + 'DISPLAY=:0', + ] + const command = [ + 'env -i', + ...environment.map(shellQuote), + shellQuote(CHROMIUM), + ...flags.map(shellQuote), + `>${shellQuote(`${PROFILE}/chromium.log`)} 2>&1`, + ].join(' ') + + await sandbox.process.createSession(SESSION_ID) + await sandbox.process.executeSessionCommand(SESSION_ID, { command, runAsync: true }) + + const deadline = Date.now() + 60_000 + const probe = `curl -sf --max-time 2 -o /dev/null http://127.0.0.1:${port}/json/version` + while (Date.now() < deadline) { + const remainingSeconds = Math.max(1, Math.ceil((deadline - Date.now()) / 1000)) + const result = await sandbox.process.executeCommand( + probe, + undefined, + undefined, + remainingSeconds, + ) + if (result.exitCode === 0) return + await sleep(500) + } + + throw new Error( + `Chromium did not start in the sandbox. Last lines of ${PROFILE}/chromium.log:\n${await readChromiumLog(sandbox)}`, + ) +} + +async function saveScreenshot(sandbox: Sandbox, outputPath: string): Promise { + const screenshot = await sandbox.computerUse.screenshot.takeFullScreen(false) + if (!screenshot.screenshot) throw new Error('The sandbox returned an empty final screenshot') + await writeFile(outputPath, Buffer.from(screenshot.screenshot, 'base64')) + console.log(`screenshot: ${outputPath}`) +} + +async function main(): Promise { + try { + if (!(await stat(PAINT_HTML)).isFile()) throw new Error('not a file') + } catch { + throw new Error( + `${PAINT_HTML} is missing. Run this example from its checkout: clone the repo, run npm install, then npm run start from this directory.`, + ) + } + + const client = new Anthropic() + const maxIterations = positiveInteger('MAX_TURNS', 120) + const computer = await DaytonaComputer.create({ confirm, resolution: [WIDTH, HEIGHT] }) + try { + console.log(`sandbox ${computer.sandbox.id}, screen ${computer.width}x${computer.height}`) + await uploadSketchpad(computer.sandbox) + await launchChromium(computer.sandbox) + console.log(`sketchpad open at file://${REMOTE_HTML}; handing the desktop to the model`) + + let inputTokens = 0 + let outputTokens = 0 + let turns = 0 + const runner = client.beta.messages.toolRunner({ + model: process.env.MODEL ?? 'claude-sonnet-5-5', + max_tokens: 4096, + max_iterations: maxIterations, + tools: [computer], + messages: [{ role: 'user', content: TASK }], + }) + for await (const message of runner) { + turns += 1 + inputTokens += message.usage.input_tokens + outputTokens += message.usage.output_tokens + for (const block of message.content) { + if (block.type === 'text') console.log(`[text] ${block.text}`) + else if (block.type === 'tool_use') console.log(`[${block.name}] ${JSON.stringify(block.input)}`) + } + } + + const outputPath = process.env.OUTPUT_PATH + if (outputPath) await saveScreenshot(computer.sandbox, outputPath) + console.log(`turns: ${turns}`) + console.log(`tokens: ${inputTokens} input, ${outputTokens} output`) + } finally { + await computer.close() + } +} + +main().catch((error: unknown) => { + console.error(error) + process.exitCode = 1 +}) diff --git a/typescript/computer-use/claude-draws/tsconfig.json b/typescript/computer-use/claude-draws/tsconfig.json new file mode 100644 index 0000000..89585ac --- /dev/null +++ b/typescript/computer-use/claude-draws/tsconfig.json @@ -0,0 +1,15 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "NodeNext", + "moduleResolution": "NodeNext", + "strict": true, + "noUncheckedIndexedAccess": true, + "exactOptionalPropertyTypes": true, + "esModuleInterop": true, + "skipLibCheck": true, + "forceConsistentCasingInFileNames": true, + "noEmit": true + }, + "include": ["src/**/*.ts"] +}