From a3b12fe5642df96d7e7425211b1f5b2a4be6f2b7 Mon Sep 17 00:00:00 2001 From: Carl Atupem Date: Sat, 28 Jun 2025 00:01:05 -0400 Subject: [PATCH] Update MCP tool descriptions, and compress screenshots to 1MB --- README.md | 16 + docs/core-concepts/desktop-environment.mdx | 21 + packages/bytebotd/package-lock.json | 504 +++++++++++++++++- packages/bytebotd/package.json | 5 +- packages/bytebotd/src/mcp/compressor.ts | 233 ++++++++ .../bytebotd/src/mcp/computer-use.tools.ts | 237 +++++--- 6 files changed, 942 insertions(+), 74 deletions(-) create mode 100644 packages/bytebotd/src/mcp/compressor.ts diff --git a/README.md b/README.md index 4fcbcf7..e5ff34d 100644 --- a/README.md +++ b/README.md @@ -265,6 +265,22 @@ Use the computer control API for precise automation: The core container also exposes an [MCP](https://github.com/rekog-labs/MCP-Nest) endpoint. Connect your MCP client to `http://localhost:9990/sse` to invoke these tools over SSE. +```json +{ + "mcpServers": { + "bytebot": { + "command": "npx", + "args": [ + "mcp-remote", + "http://127.0.0.1:9990/sse", + "--transport", + "http-first" + ] + } + } +} +``` + ```javascript // Take screenshot POST http://localhost:9990/computer-use diff --git a/docs/core-concepts/desktop-environment.mdx b/docs/core-concepts/desktop-environment.mdx index b9cb463..930e98b 100644 --- a/docs/core-concepts/desktop-environment.mdx +++ b/docs/core-concepts/desktop-environment.mdx @@ -132,6 +132,27 @@ Navigate to `http://localhost:9990/vnc` for instant access: - Supports touch devices - Clipboard sharing +### MCP Control + +The core container also exposes an [MCP](https://github.com/rekog-labs/MCP-Nest) endpoint. +Connect your MCP client to `http://localhost:9990/sse` to invoke these tools over SSE. + +```json +{ + "mcpServers": { + "bytebot": { + "command": "npx", + "args": [ + "mcp-remote", + "http://127.0.0.1:9990/sse", + "--transport", + "http-first" + ] + } + } +} +``` + ### Direct API Control Most efficient for automation: ```bash diff --git a/packages/bytebotd/package-lock.json b/packages/bytebotd/package-lock.json index f4002ed..cc89fd8 100644 --- a/packages/bytebotd/package-lock.json +++ b/packages/bytebotd/package-lock.json @@ -23,6 +23,7 @@ "http-proxy-middleware": "^3.0.5", "reflect-metadata": "^0.2.2", "rxjs": "^7.8.1", + "sharp": "^0.34.2", "socket.io": "^4.8.1", "uiohook-napi": "^1.5.4" }, @@ -775,6 +776,16 @@ "@jridgewell/sourcemap-codec": "^1.4.10" } }, + "node_modules/@emnapi/runtime": { + "version": "1.4.3", + "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.4.3.tgz", + "integrity": "sha512-pBPWdu6MLKROBX05wSNKcNb++m5Er+KQ9QkB+WVM+pW2Kx9hoSrVTnu3BdkI5eBLZoKu/J6mW/B6i6bJB2ytXQ==", + "license": "MIT", + "optional": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, "node_modules/@eslint-community/eslint-utils": { "version": "4.5.1", "resolved": "https://registry.npmjs.org/@eslint-community/eslint-utils/-/eslint-utils-4.5.1.tgz", @@ -992,6 +1003,402 @@ "url": "https://github.com/sponsors/nzakas" } }, + "node_modules/@img/sharp-darwin-arm64": { + "version": "0.34.2", + "resolved": "https://registry.npmjs.org/@img/sharp-darwin-arm64/-/sharp-darwin-arm64-0.34.2.tgz", + "integrity": "sha512-OfXHZPppddivUJnqyKoi5YVeHRkkNE2zUFT2gbpKxp/JZCFYEYubnMg+gOp6lWfasPrTS+KPosKqdI+ELYVDtg==", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-darwin-arm64": "1.1.0" + } + }, + "node_modules/@img/sharp-darwin-x64": { + "version": "0.34.2", + "resolved": "https://registry.npmjs.org/@img/sharp-darwin-x64/-/sharp-darwin-x64-0.34.2.tgz", + "integrity": "sha512-dYvWqmjU9VxqXmjEtjmvHnGqF8GrVjM2Epj9rJ6BUIXvk8slvNDJbhGFvIoXzkDhrJC2jUxNLz/GUjjvSzfw+g==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-darwin-x64": "1.1.0" + } + }, + "node_modules/@img/sharp-libvips-darwin-arm64": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-arm64/-/sharp-libvips-darwin-arm64-1.1.0.tgz", + "integrity": "sha512-HZ/JUmPwrJSoM4DIQPv/BfNh9yrOA8tlBbqbLz4JZ5uew2+o22Ik+tHQJcih7QJuSa0zo5coHTfD5J8inqj9DA==", + "cpu": [ + "arm64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "darwin" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-darwin-x64": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-x64/-/sharp-libvips-darwin-x64-1.1.0.tgz", + "integrity": "sha512-Xzc2ToEmHN+hfvsl9wja0RlnXEgpKNmftriQp6XzY/RaSfwD9th+MSh0WQKzUreLKKINb3afirxW7A0fz2YWuQ==", + "cpu": [ + "x64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "darwin" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-arm": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm/-/sharp-libvips-linux-arm-1.1.0.tgz", + "integrity": "sha512-s8BAd0lwUIvYCJyRdFqvsj+BJIpDBSxs6ivrOPm/R7piTs5UIwY5OjXrP2bqXC9/moGsyRa37eYWYCOGVXxVrA==", + "cpu": [ + "arm" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-arm64": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm64/-/sharp-libvips-linux-arm64-1.1.0.tgz", + "integrity": "sha512-IVfGJa7gjChDET1dK9SekxFFdflarnUB8PwW8aGwEoF3oAsSDuNUTYS+SKDOyOJxQyDC1aPFMuRYLoDInyV9Ew==", + "cpu": [ + "arm64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-ppc64": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-ppc64/-/sharp-libvips-linux-ppc64-1.1.0.tgz", + "integrity": "sha512-tiXxFZFbhnkWE2LA8oQj7KYR+bWBkiV2nilRldT7bqoEZ4HiDOcePr9wVDAZPi/Id5fT1oY9iGnDq20cwUz8lQ==", + "cpu": [ + "ppc64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-s390x": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-s390x/-/sharp-libvips-linux-s390x-1.1.0.tgz", + "integrity": "sha512-xukSwvhguw7COyzvmjydRb3x/09+21HykyapcZchiCUkTThEQEOMtBj9UhkaBRLuBrgLFzQ2wbxdeCCJW/jgJA==", + "cpu": [ + "s390x" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-x64": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-x64/-/sharp-libvips-linux-x64-1.1.0.tgz", + "integrity": "sha512-yRj2+reB8iMg9W5sULM3S74jVS7zqSzHG3Ol/twnAAkAhnGQnpjj6e4ayUz7V+FpKypwgs82xbRdYtchTTUB+Q==", + "cpu": [ + "x64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linuxmusl-arm64": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-arm64/-/sharp-libvips-linuxmusl-arm64-1.1.0.tgz", + "integrity": "sha512-jYZdG+whg0MDK+q2COKbYidaqW/WTz0cc1E+tMAusiDygrM4ypmSCjOJPmFTvHHJ8j/6cAGyeDWZOsK06tP33w==", + "cpu": [ + "arm64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linuxmusl-x64": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-x64/-/sharp-libvips-linuxmusl-x64-1.1.0.tgz", + "integrity": "sha512-wK7SBdwrAiycjXdkPnGCPLjYb9lD4l6Ze2gSdAGVZrEL05AOUJESWU2lhlC+Ffn5/G+VKuSm6zzbQSzFX/P65A==", + "cpu": [ + "x64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-linux-arm": { + "version": "0.34.2", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm/-/sharp-linux-arm-0.34.2.tgz", + "integrity": "sha512-0DZzkvuEOqQUP9mo2kjjKNok5AmnOr1jB2XYjkaoNRwpAYMDzRmAqUIa1nRi58S2WswqSfPOWLNOr0FDT3H5RQ==", + "cpu": [ + "arm" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-arm": "1.1.0" + } + }, + "node_modules/@img/sharp-linux-arm64": { + "version": "0.34.2", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm64/-/sharp-linux-arm64-0.34.2.tgz", + "integrity": "sha512-D8n8wgWmPDakc83LORcfJepdOSN6MvWNzzz2ux0MnIbOqdieRZwVYY32zxVx+IFUT8er5KPcyU3XXsn+GzG/0Q==", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-arm64": "1.1.0" + } + }, + "node_modules/@img/sharp-linux-s390x": { + "version": "0.34.2", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-s390x/-/sharp-linux-s390x-0.34.2.tgz", + "integrity": "sha512-EGZ1xwhBI7dNISwxjChqBGELCWMGDvmxZXKjQRuqMrakhO8QoMgqCrdjnAqJq/CScxfRn+Bb7suXBElKQpPDiw==", + "cpu": [ + "s390x" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-s390x": "1.1.0" + } + }, + "node_modules/@img/sharp-linux-x64": { + "version": "0.34.2", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-x64/-/sharp-linux-x64-0.34.2.tgz", + "integrity": "sha512-sD7J+h5nFLMMmOXYH4DD9UtSNBD05tWSSdWAcEyzqW8Cn5UxXvsHAxmxSesYUsTOBmUnjtxghKDl15EvfqLFbQ==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-x64": "1.1.0" + } + }, + "node_modules/@img/sharp-linuxmusl-arm64": { + "version": "0.34.2", + "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-arm64/-/sharp-linuxmusl-arm64-0.34.2.tgz", + "integrity": "sha512-NEE2vQ6wcxYav1/A22OOxoSOGiKnNmDzCYFOZ949xFmrWZOVII1Bp3NqVVpvj+3UeHMFyN5eP/V5hzViQ5CZNA==", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linuxmusl-arm64": "1.1.0" + } + }, + "node_modules/@img/sharp-linuxmusl-x64": { + "version": "0.34.2", + "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-x64/-/sharp-linuxmusl-x64-0.34.2.tgz", + "integrity": "sha512-DOYMrDm5E6/8bm/yQLCWyuDJwUnlevR8xtF8bs+gjZ7cyUNYXiSf/E8Kp0Ss5xasIaXSHzb888V1BE4i1hFhAA==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linuxmusl-x64": "1.1.0" + } + }, + "node_modules/@img/sharp-wasm32": { + "version": "0.34.2", + "resolved": "https://registry.npmjs.org/@img/sharp-wasm32/-/sharp-wasm32-0.34.2.tgz", + "integrity": "sha512-/VI4mdlJ9zkaq53MbIG6rZY+QRN3MLbR6usYlgITEzi4Rpx5S6LFKsycOQjkOGmqTNmkIdLjEvooFKwww6OpdQ==", + "cpu": [ + "wasm32" + ], + "license": "Apache-2.0 AND LGPL-3.0-or-later AND MIT", + "optional": true, + "dependencies": { + "@emnapi/runtime": "^1.4.3" + }, + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-win32-arm64": { + "version": "0.34.2", + "resolved": "https://registry.npmjs.org/@img/sharp-win32-arm64/-/sharp-win32-arm64-0.34.2.tgz", + "integrity": "sha512-cfP/r9FdS63VA5k0xiqaNaEoGxBg9k7uE+RQGzuK9fHt7jib4zAVVseR9LsE4gJcNWgT6APKMNnCcnyOtmSEUQ==", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0 AND LGPL-3.0-or-later", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-win32-ia32": { + "version": "0.34.2", + "resolved": "https://registry.npmjs.org/@img/sharp-win32-ia32/-/sharp-win32-ia32-0.34.2.tgz", + "integrity": "sha512-QLjGGvAbj0X/FXl8n1WbtQ6iVBpWU7JO94u/P2M4a8CFYsvQi4GW2mRy/JqkRx0qpBzaOdKJKw8uc930EX2AHw==", + "cpu": [ + "ia32" + ], + "license": "Apache-2.0 AND LGPL-3.0-or-later", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-win32-x64": { + "version": "0.34.2", + "resolved": "https://registry.npmjs.org/@img/sharp-win32-x64/-/sharp-win32-x64-0.34.2.tgz", + "integrity": "sha512-aUdT6zEYtDKCaxkofmmJDJYGCf0+pJg3eU9/oBuqvEeoB9dKI6ZLc/1iLJCTuJQDO4ptntAlkUmHgGjyuobZbw==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0 AND LGPL-3.0-or-later", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, "node_modules/@inquirer/checkbox": { "version": "4.1.4", "resolved": "https://registry.npmjs.org/@inquirer/checkbox/-/checkbox-4.1.4.tgz", @@ -5494,11 +5901,23 @@ "dev": true, "license": "MIT" }, + "node_modules/color": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/color/-/color-4.2.3.tgz", + "integrity": "sha512-1rXeuUUiGGrykh+CeBdu5Ie7OJwinCgQY0bc7GCRxy5xVHy+moaqkpL/jqQq0MtQOeYcrqEz4abc5f0KtU7W4A==", + "license": "MIT", + "dependencies": { + "color-convert": "^2.0.1", + "color-string": "^1.9.0" + }, + "engines": { + "node": ">=12.5.0" + } + }, "node_modules/color-convert": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", - "dev": true, "license": "MIT", "dependencies": { "color-name": "~1.1.4" @@ -5511,9 +5930,18 @@ "version": "1.1.4", "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", - "dev": true, "license": "MIT" }, + "node_modules/color-string": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/color-string/-/color-string-1.9.1.tgz", + "integrity": "sha512-shrVawQFojnZv6xM40anx4CkoDP+fZsw/ZerEMsW/pyzsRbElpsL/DBVW7q3ExxwusdNXI3lXpuhEZkzs8p5Eg==", + "license": "MIT", + "dependencies": { + "color-name": "^1.0.0", + "simple-swizzle": "^0.2.2" + } + }, "node_modules/combined-stream": { "version": "1.0.8", "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", @@ -5859,6 +6287,15 @@ "node": ">= 0.8" } }, + "node_modules/detect-libc": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.0.4.tgz", + "integrity": "sha512-3UDv+G9CsCKO1WKMGw9fwq/SWJYbI0c5Y7LU1AXYoDdbhE2AHQ6N6Nb34sG8Fj7T5APy8qXDCKuuIHd1BR0tVA==", + "license": "Apache-2.0", + "engines": { + "node": ">=8" + } + }, "node_modules/detect-newline": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/detect-newline/-/detect-newline-3.1.0.tgz", @@ -10721,10 +11158,9 @@ } }, "node_modules/semver": { - "version": "7.7.1", - "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.1.tgz", - "integrity": "sha512-hlq8tAfn0m/61p4BVRcPzIGr6LKiMwo4VM6dGi6pt4qcRkmNzTcWq6eCEjEh+qXjkMDvPlOFFSGwQjoEa6gyMA==", - "dev": true, + "version": "7.7.2", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.2.tgz", + "integrity": "sha512-RF0Fw+rO5AMf9MAyaRXI4AV0Ulj5lMHqVxxdSgiVbixSCXoEmmX/jk0CuJw4+3SqroYO9VoUh+HcuJivvtJemA==", "license": "ISC", "bin": { "semver": "bin/semver.js" @@ -10815,6 +11251,47 @@ "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==", "license": "ISC" }, + "node_modules/sharp": { + "version": "0.34.2", + "resolved": "https://registry.npmjs.org/sharp/-/sharp-0.34.2.tgz", + "integrity": "sha512-lszvBmB9QURERtyKT2bNmsgxXK0ShJrL/fvqlonCo7e6xBF8nT8xU6pW+PMIbLsz0RxQk3rgH9kd8UmvOzlMJg==", + "hasInstallScript": true, + "license": "Apache-2.0", + "dependencies": { + "color": "^4.2.3", + "detect-libc": "^2.0.4", + "semver": "^7.7.2" + }, + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-darwin-arm64": "0.34.2", + "@img/sharp-darwin-x64": "0.34.2", + "@img/sharp-libvips-darwin-arm64": "1.1.0", + "@img/sharp-libvips-darwin-x64": "1.1.0", + "@img/sharp-libvips-linux-arm": "1.1.0", + "@img/sharp-libvips-linux-arm64": "1.1.0", + "@img/sharp-libvips-linux-ppc64": "1.1.0", + "@img/sharp-libvips-linux-s390x": "1.1.0", + "@img/sharp-libvips-linux-x64": "1.1.0", + "@img/sharp-libvips-linuxmusl-arm64": "1.1.0", + "@img/sharp-libvips-linuxmusl-x64": "1.1.0", + "@img/sharp-linux-arm": "0.34.2", + "@img/sharp-linux-arm64": "0.34.2", + "@img/sharp-linux-s390x": "0.34.2", + "@img/sharp-linux-x64": "0.34.2", + "@img/sharp-linuxmusl-arm64": "0.34.2", + "@img/sharp-linuxmusl-x64": "0.34.2", + "@img/sharp-wasm32": "0.34.2", + "@img/sharp-win32-arm64": "0.34.2", + "@img/sharp-win32-ia32": "0.34.2", + "@img/sharp-win32-x64": "0.34.2" + } + }, "node_modules/shebang-command": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", @@ -10921,6 +11398,21 @@ "url": "https://github.com/sponsors/isaacs" } }, + "node_modules/simple-swizzle": { + "version": "0.2.2", + "resolved": "https://registry.npmjs.org/simple-swizzle/-/simple-swizzle-0.2.2.tgz", + "integrity": "sha512-JA//kQgZtbuY83m+xT+tXJkmJncGMTFT+C+g2h2R9uxkYIrE2yy9sgmcLhCnw57/WSD+Eh3J97FPEDFnbXnDUg==", + "license": "MIT", + "dependencies": { + "is-arrayish": "^0.3.1" + } + }, + "node_modules/simple-swizzle/node_modules/is-arrayish": { + "version": "0.3.2", + "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.3.2.tgz", + "integrity": "sha512-eVRqCvVlZbuw3GrM63ovNSNAeA1K16kaR/LRY/92w0zxQ5/1YzwblUX652i4Xs9RwAGjW9d9y6X88t8OaAJfWQ==", + "license": "MIT" + }, "node_modules/sisteransi": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/sisteransi/-/sisteransi-1.0.5.tgz", diff --git a/packages/bytebotd/package.json b/packages/bytebotd/package.json index fcd677f..f903498 100644 --- a/packages/bytebotd/package.json +++ b/packages/bytebotd/package.json @@ -34,14 +34,15 @@ "@nestjs/serve-static": "^5.0.3", "@nestjs/websockets": "^11.1.2", "@nut-tree-fork/nut-js": "^4.2.6", + "@rekog/mcp-nest": "^1.6.2", "class-transformer": "^0.5.1", "class-validator": "^0.14.2", "http-proxy-middleware": "^3.0.5", "reflect-metadata": "^0.2.2", "rxjs": "^7.8.1", + "sharp": "^0.34.2", "socket.io": "^4.8.1", - "uiohook-napi": "^1.5.4", - "@rekog/mcp-nest": "^1.6.2" + "uiohook-napi": "^1.5.4" }, "devDependencies": { "@eslint/eslintrc": "^3.2.0", diff --git a/packages/bytebotd/src/mcp/compressor.ts b/packages/bytebotd/src/mcp/compressor.ts new file mode 100644 index 0000000..d669720 --- /dev/null +++ b/packages/bytebotd/src/mcp/compressor.ts @@ -0,0 +1,233 @@ +import * as sharp from 'sharp'; + +interface CompressionOptions { + targetSizeKB?: number; + initialQuality?: number; + minQuality?: number; + format?: 'png' | 'jpeg' | 'webp'; + maxIterations?: number; +} + +interface CompressionResult { + base64: string; + sizeBytes: number; + sizeKB: number; + sizeMB: number; + quality: number; + format: string; + iterations: number; +} + +class Base64ImageCompressor { + /** + * Compress a base64 PNG string to under specified size (default 1MB) + */ + static async compressToSize( + base64String: string, + options: CompressionOptions = {}, + ): Promise { + const { + targetSizeKB = 1024, // 1MB default + initialQuality = 95, + minQuality = 10, + format = 'png', + maxIterations = 10, + } = options; + + // Extract base64 data (remove data URL prefix if present) + const base64Data = base64String.replace(/^data:image\/\w+;base64,/, ''); + const inputBuffer = Buffer.from(base64Data, 'base64'); + + let quality = initialQuality; + let outputBuffer: Buffer; + let iterations = 0; + + // Binary search for optimal quality + let low = minQuality; + let high = initialQuality; + let bestResult: { buffer: Buffer; quality: number } | null = null; + + while (low <= high && iterations < maxIterations) { + quality = Math.floor((low + high) / 2); + + outputBuffer = await this.compressBuffer(inputBuffer, quality, format); + const sizeKB = outputBuffer.length / 1024; + + if (sizeKB <= targetSizeKB) { + // Size is acceptable, try higher quality + bestResult = { buffer: outputBuffer, quality }; + low = quality + 1; + } else { + // Size too large, reduce quality + high = quality - 1; + } + + iterations++; + } + + // If no result found under target size, use lowest quality + if (!bestResult) { + outputBuffer = await this.compressBuffer(inputBuffer, minQuality, format); + quality = minQuality; + } else { + outputBuffer = bestResult.buffer; + quality = bestResult.quality; + } + + // Convert back to base64 + const outputBase64 = outputBuffer.toString('base64'); + const sizeBytes = outputBuffer.length; + + return { + base64: outputBase64, + sizeBytes, + sizeKB: sizeBytes / 1024, + sizeMB: sizeBytes / (1024 * 1024), + quality, + format, + iterations, + }; + } + + /** + * Compress buffer with specified quality + */ + private static async compressBuffer( + inputBuffer: Buffer, + quality: number, + format: 'png' | 'jpeg' | 'webp', + ): Promise { + let sharpInstance = sharp(inputBuffer); + + switch (format) { + case 'png': + return sharpInstance + .png({ + quality, + compressionLevel: 9, + adaptiveFiltering: true, + palette: true, + }) + .toBuffer(); + + case 'jpeg': + return sharpInstance + .jpeg({ + quality, + progressive: true, + mozjpeg: true, + optimizeScans: true, + }) + .toBuffer(); + + case 'webp': + return sharpInstance + .webp({ + quality, + alphaQuality: quality, + lossless: false, + nearLossless: false, + smartSubsample: true, + }) + .toBuffer(); + + default: + throw new Error(`Unsupported format: ${format}`); + } + } + + /** + * Compress with dimension reduction if quality alone isn't enough + */ + static async compressWithResize( + base64String: string, + options: CompressionOptions & { + maxWidth?: number; + maxHeight?: number; + } = {}, + ): Promise { + const { + targetSizeKB = 1024, + maxWidth = 2048, + maxHeight = 2048, + ...compressionOptions + } = options; + + // First try compression without resizing + let result = await this.compressToSize(base64String, compressionOptions); + + // If still too large, apply progressive resizing + if (result.sizeKB > targetSizeKB) { + const base64Data = base64String.replace(/^data:image\/\w+;base64,/, ''); + const inputBuffer = Buffer.from(base64Data, 'base64'); + + const metadata = await sharp(inputBuffer).metadata(); + const originalWidth = metadata.width || maxWidth; + const originalHeight = metadata.height || maxHeight; + + let scale = 0.9; // Start with 90% of original size + + while (result.sizeKB > targetSizeKB && scale > 0.3) { + const newWidth = Math.floor(originalWidth * scale); + const newHeight = Math.floor(originalHeight * scale); + + const resizedBuffer = await sharp(inputBuffer) + .resize(newWidth, newHeight, { + fit: 'inside', + withoutEnlargement: true, + }) + .toBuffer(); + + const resizedBase64 = resizedBuffer.toString('base64'); + + result = await this.compressToSize(resizedBase64, compressionOptions); + scale -= 0.1; + } + } + + return result; + } + + /** + * Get size information for a base64 string + */ + static getBase64SizeInfo(base64String: string): { + bytes: number; + kb: number; + mb: number; + formatted: string; + } { + const base64Data = base64String.replace(/^data:image\/\w+;base64,/, ''); + const bytes = Buffer.from(base64Data, 'base64').length; + const kb = bytes / 1024; + const mb = bytes / (1024 * 1024); + + let formatted: string; + if (mb >= 1) { + formatted = `${mb.toFixed(2)} MB`; + } else if (kb >= 1) { + formatted = `${kb.toFixed(2)} KB`; + } else { + formatted = `${bytes} bytes`; + } + + return { bytes, kb, mb, formatted }; + } +} + +// Utility function for quick compression +export async function compressPngBase64Under1MB( + base64String: string, +): Promise { + const result = await Base64ImageCompressor.compressToSize(base64String, { + targetSizeKB: 1024, + format: 'png', + initialQuality: 95, + minQuality: 10, + }); + + return result.base64; +} + +// Export the class for more control +export { Base64ImageCompressor, CompressionOptions, CompressionResult }; diff --git a/packages/bytebotd/src/mcp/computer-use.tools.ts b/packages/bytebotd/src/mcp/computer-use.tools.ts index 2922aa7..72744ff 100644 --- a/packages/bytebotd/src/mcp/computer-use.tools.ts +++ b/packages/bytebotd/src/mcp/computer-use.tools.ts @@ -2,7 +2,7 @@ import { Injectable } from '@nestjs/common'; import { Tool } from '@rekog/mcp-nest'; import { z } from 'zod'; import { ComputerUseService } from '../computer-use/computer-use.service'; -import { MessageContentType } from '@bytebot/shared'; +import { compressPngBase64Under1MB } from './compressor'; @Injectable() export class ComputerUseTools { @@ -10,20 +10,23 @@ export class ComputerUseTools { @Tool({ name: 'computer_move_mouse', - description: 'Move the mouse to specific coordinates.', + description: 'Moves the mouse cursor to the specified coordinates.', parameters: z.object({ - coordinates: z.object({ x: z.number(), y: z.number() }), + coordinates: z.object({ + x: z.number().describe('The x-coordinate to move the mouse to.'), + y: z.number().describe('The y-coordinate to move the mouse to.'), + }), }), }) async moveMouse({ coordinates }: { coordinates: { x: number; y: number } }) { try { await this.computerUse.action({ action: 'move_mouse', coordinates }); - return { content: [{ type: MessageContentType.Text, text: 'mouse moved' }] }; + return { content: [{ type: 'text', text: 'mouse moved' }] }; } catch (err) { return { content: [ { - type: MessageContentType.Text, + type: 'text', text: `Error moving mouse: ${(err as Error).message}`, }, ], @@ -33,10 +36,21 @@ export class ComputerUseTools { @Tool({ name: 'computer_trace_mouse', - description: 'Trace a path with the mouse.', + description: + 'Moves the mouse cursor along a specified path of coordinates.', parameters: z.object({ - path: z.array(z.object({ x: z.number(), y: z.number() })), - holdKeys: z.array(z.string()).optional(), + path: z + .array( + z.object({ + x: z.number().describe('The x-coordinate to move the mouse to.'), + y: z.number().describe('The y-coordinate to move the mouse to.'), + }), + ) + .describe('An array of coordinate objects representing the path.'), + holdKeys: z + .array(z.string()) + .optional() + .describe('Optional array of keys to hold during the trace.'), }), }) async traceMouse({ @@ -49,13 +63,13 @@ export class ComputerUseTools { try { await this.computerUse.action({ action: 'trace_mouse', path, holdKeys }); return { - content: [{ type: MessageContentType.Text, text: 'mouse traced' }], + content: [{ type: 'text', text: 'mouse traced' }], }; } catch (err) { return { content: [ { - type: MessageContentType.Text, + type: 'text', text: `Error tracing mouse: ${(err as Error).message}`, }, ], @@ -65,12 +79,28 @@ export class ComputerUseTools { @Tool({ name: 'computer_click_mouse', - description: 'Perform mouse clicks.', + description: + 'Performs a mouse click at the specified coordinates or current position.', parameters: z.object({ - coordinates: z.object({ x: z.number(), y: z.number() }).optional(), - button: z.enum(['left', 'right', 'middle']), - holdKeys: z.array(z.string()).optional(), - numClicks: z.number(), + coordinates: z + .object({ + x: z.number().describe('The x-coordinate to move the mouse to.'), + y: z.number().describe('The y-coordinate to move the mouse to.'), + }) + .optional() + .describe( + 'Optional coordinates for the click. If not provided, clicks at the current mouse position.', + ), + button: z + .enum(['left', 'right', 'middle']) + .describe('The mouse button to click.'), + holdKeys: z + .array(z.string()) + .optional() + .describe('Optional array of keys to hold during the click.'), + numClicks: z + .number() + .describe('Number of clicks to perform (e.g., 2 for double-click).'), }), }) async clickMouse({ @@ -93,13 +123,13 @@ export class ComputerUseTools { numClicks, }); return { - content: [{ type: MessageContentType.Text, text: 'mouse clicked' }], + content: [{ type: 'text', text: 'mouse clicked' }], }; } catch (err) { return { content: [ { - type: MessageContentType.Text, + type: 'text', text: `Error clicking mouse: ${(err as Error).message}`, }, ], @@ -109,11 +139,24 @@ export class ComputerUseTools { @Tool({ name: 'computer_press_mouse', - description: 'Press or release a mouse button.', + description: + 'Presses or releases a specified mouse button at the given coordinates or current position.', parameters: z.object({ - coordinates: z.object({ x: z.number(), y: z.number() }).optional(), - button: z.enum(['left', 'right', 'middle']), - press: z.enum(['down', 'up']), + coordinates: z + .object({ + x: z.number().describe('The x-coordinate for the mouse action.'), + y: z.number().describe('The y-coordinate for the mouse action.'), + }) + .optional() + .describe( + 'Optional coordinates for the mouse press/release. If not provided, uses the current mouse position.', + ), + button: z + .enum(['left', 'right', 'middle']) + .describe('The mouse button to press or release.'), + press: z + .enum(['down', 'up']) + .describe('The action to perform (press or release).'), }), }) async pressMouse({ @@ -133,13 +176,13 @@ export class ComputerUseTools { press, }); return { - content: [{ type: MessageContentType.Text, text: 'mouse pressed' }], + content: [{ type: 'text', text: 'mouse pressed' }], }; } catch (err) { return { content: [ { - type: MessageContentType.Text, + type: 'text', text: `Error pressing mouse: ${(err as Error).message}`, }, ], @@ -149,11 +192,30 @@ export class ComputerUseTools { @Tool({ name: 'computer_drag_mouse', - description: 'Drag the mouse along a path.', + description: + 'Drags the mouse from a starting point along a path while holding a specified button.', parameters: z.object({ - path: z.array(z.object({ x: z.number(), y: z.number() })), - button: z.enum(['left', 'right', 'middle']), - holdKeys: z.array(z.string()).optional(), + path: z + .array( + z.object({ + x: z + .number() + .describe('The x-coordinate of a point in the drag path.'), + y: z + .number() + .describe('The y-coordinate of a point in the drag path.'), + }), + ) + .describe( + 'An array of coordinate objects representing the drag path. The first coordinate is the start point.', + ), + button: z + .enum(['left', 'right', 'middle']) + .describe('The mouse button to hold while dragging.'), + holdKeys: z + .array(z.string()) + .optional() + .describe('Optional array of keys to hold during the drag.'), }), }) async dragMouse({ @@ -173,13 +235,13 @@ export class ComputerUseTools { holdKeys, }); return { - content: [{ type: MessageContentType.Text, text: 'mouse dragged' }], + content: [{ type: 'text', text: 'mouse dragged' }], }; } catch (err) { return { content: [ { - type: MessageContentType.Text, + type: 'text', text: `Error dragging mouse: ${(err as Error).message}`, }, ], @@ -189,12 +251,35 @@ export class ComputerUseTools { @Tool({ name: 'computer_scroll', - description: 'Scroll the mouse wheel.', + description: 'Scrolls the mouse wheel up, down, left, or right.', parameters: z.object({ - coordinates: z.object({ x: z.number(), y: z.number() }).optional(), - direction: z.enum(['up', 'down', 'left', 'right']), - numScrolls: z.number(), - holdKeys: z.array(z.string()).optional(), + coordinates: z + .object({ + x: z + .number() + .describe( + 'The x-coordinate for the scroll action (if applicable).', + ), + y: z + .number() + .describe( + 'The y-coordinate for the scroll action (if applicable).', + ), + }) + .optional() + .describe( + 'Coordinates for where the scroll should occur. Behavior might depend on the OS/application.', + ), + direction: z + .enum(['up', 'down', 'left', 'right']) + .describe('The direction to scroll the mouse wheel.'), + numScrolls: z + .number() + .describe('The number of times to scroll the mouse wheel.'), + holdKeys: z + .array(z.string()) + .optional() + .describe('Optional array of keys to hold during the scroll.'), }), }) async scroll({ @@ -216,12 +301,12 @@ export class ComputerUseTools { numScrolls, holdKeys, }); - return { content: [{ type: MessageContentType.Text, text: 'scrolled' }] }; + return { content: [{ type: 'text', text: 'scrolled' }] }; } catch (err) { return { content: [ { - type: MessageContentType.Text, + type: 'text', text: `Error scrolling: ${(err as Error).message}`, }, ], @@ -231,21 +316,29 @@ export class ComputerUseTools { @Tool({ name: 'computer_type_keys', - description: 'Press a sequence of keys.', + description: + 'Simulates typing a sequence of keys, often used for shortcuts involving modifier keys (e.g., Ctrl+C). Presses and releases each key in order.', parameters: z.object({ - keys: z.array(z.string()), - delay: z.number().optional(), + keys: z + .array(z.string()) + .describe( + 'An array of key names to type in sequence (e.g., ["control", "c"]).', + ), + delay: z + .number() + .optional() + .describe('Optional delay in milliseconds between key presses.'), }), }) async typeKeys({ keys, delay }: { keys: string[]; delay?: number }) { try { await this.computerUse.action({ action: 'type_keys', keys, delay }); - return { content: [{ type: MessageContentType.Text, text: 'keys typed' }] }; + return { content: [{ type: 'text', text: 'keys typed' }] }; } catch (err) { return { content: [ { - type: MessageContentType.Text, + type: 'text', text: `Error typing keys: ${(err as Error).message}`, }, ], @@ -255,21 +348,28 @@ export class ComputerUseTools { @Tool({ name: 'computer_press_keys', - description: 'Hold or release keys.', + description: + 'Simulates pressing down or releasing specific keys. Useful for holding modifier keys.', parameters: z.object({ - keys: z.array(z.string()), - press: z.enum(['down', 'up']), + keys: z + .array(z.string()) + .describe( + 'An array of key names to press or release (e.g., ["shift"]).', + ), + press: z + .enum(['down', 'up']) + .describe('Whether to press the keys down or release them up.'), }), }) async pressKeys({ keys, press }: { keys: string[]; press: 'down' | 'up' }) { try { await this.computerUse.action({ action: 'press_keys', keys, press }); - return { content: [{ type: MessageContentType.Text, text: 'keys pressed' }] }; + return { content: [{ type: 'text', text: 'keys pressed' }] }; } catch (err) { return { content: [ { - type: MessageContentType.Text, + type: 'text', text: `Error pressing keys: ${(err as Error).message}`, }, ], @@ -279,21 +379,24 @@ export class ComputerUseTools { @Tool({ name: 'computer_type_text', - description: 'Type a text string.', + description: 'Simulates typing a string of text character by character.', parameters: z.object({ - text: z.string(), - delay: z.number().optional(), + text: z.string().describe('The text string to type.'), + delay: z + .number() + .optional() + .describe('Optional delay in milliseconds between key presses.'), }), }) async typeText({ text, delay }: { text: string; delay?: number }) { try { await this.computerUse.action({ action: 'type_text', text, delay }); - return { content: [{ type: MessageContentType.Text, text: 'text typed' }] }; + return { content: [{ type: 'text', text: 'text typed' }] }; } catch (err) { return { content: [ { - type: MessageContentType.Text, + type: 'text', text: `Error typing text: ${(err as Error).message}`, }, ], @@ -303,18 +406,23 @@ export class ComputerUseTools { @Tool({ name: 'computer_wait', - description: 'Wait for a period of time.', - parameters: z.object({ duration: z.number() }), + description: 'Pauses execution for a specified duration.', + parameters: z.object({ + duration: z + .number() + .default(500) + .describe('The duration to wait in milliseconds.'), + }), }) async wait({ duration }: { duration: number }) { try { await this.computerUse.action({ action: 'wait', duration }); - return { content: [{ type: MessageContentType.Text, text: 'waiting done' }] }; + return { content: [{ type: 'text', text: 'waiting done' }] }; } catch (err) { return { content: [ { - type: MessageContentType.Text, + type: 'text', text: `Error waiting: ${(err as Error).message}`, }, ], @@ -324,7 +432,7 @@ export class ComputerUseTools { @Tool({ name: 'computer_screenshot', - description: 'Capture a screenshot of the desktop.', + description: 'Captures a screenshot of the current screen.', }) async screenshot() { try { @@ -334,12 +442,9 @@ export class ComputerUseTools { return { content: [ { - type: MessageContentType.Image, - source: { - type: 'base64', - media_type: 'image/png', - data: shot.image, - }, + type: 'image', + data: await compressPngBase64Under1MB(shot.image), + mimeType: 'image/png', }, ], }; @@ -347,7 +452,7 @@ export class ComputerUseTools { return { content: [ { - type: MessageContentType.Text, + type: 'text', text: `Error taking screenshot: ${(err as Error).message}`, }, ], @@ -357,7 +462,7 @@ export class ComputerUseTools { @Tool({ name: 'computer_cursor_position', - description: 'Get the current cursor position.', + description: 'Gets the current (x, y) coordinates of the mouse cursor.', }) async cursorPosition() { try { @@ -367,7 +472,7 @@ export class ComputerUseTools { return { content: [ { - type: MessageContentType.Text, + type: 'text', text: JSON.stringify(pos), }, ], @@ -376,7 +481,7 @@ export class ComputerUseTools { return { content: [ { - type: MessageContentType.Text, + type: 'text', text: `Error getting cursor position: ${(err as Error).message}`, }, ],