opensource/bytebot
3
0
Fork 0
mirror of https://github.com/bytebot-ai/bytebot.git synced 2026-09-01 15:30:04 +08:00
Code Issues Packages Projects Releases Wiki Activity

Update MCP tool descriptions, and compress screenshots to 1MB

Browse Source
This commit is contained in:
Carl Atupem
2025-06-28 00:01:05 -04:00
parent 53126f1499
commit a3b12fe564
6 changed files with 942 additions and 74 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.md
+16
View File
@@ -265,6 +265,22 @@ Use the computer control API for precise automation:
The core container also exposes an [MCP](https://github.com/rekog-labs/MCP-Nest) endpoint.
Connect your MCP client to `http://localhost:9990/sse` to invoke these tools over SSE.
```json
{
"mcpServers": {
"bytebot": {
"command": "npx",
"args": [
"mcp-remote",
"http://127.0.0.1:9990/sse",
"--transport",
"http-first"
]
}
}
}
```
```javascript
// Take screenshot
POST http://localhost:9990/computer-use
docs/core-concepts/desktop-environment.mdx
+21
View File
@@ -132,6 +132,27 @@ Navigate to `http://localhost:9990/vnc` for instant access:
- Supports touch devices
- Clipboard sharing
### MCP Control
The core container also exposes an [MCP](https://github.com/rekog-labs/MCP-Nest) endpoint.
Connect your MCP client to `http://localhost:9990/sse` to invoke these tools over SSE.
```json
{
"mcpServers": {
"bytebot": {
"command": "npx",
"args": [
"mcp-remote",
"http://127.0.0.1:9990/sse",
"--transport",
"http-first"
]
}
}
}
```
### Direct API Control
Most efficient for automation:
```bash
packages/bytebotd/package-lock.json
Generated
+498 -6
View File
@@ -23,6 +23,7 @@
"http-proxy-middleware": "^3.0.5",
"reflect-metadata": "^0.2.2",
"rxjs": "^7.8.1",
"sharp": "^0.34.2",
"socket.io": "^4.8.1",
"uiohook-napi": "^1.5.4"
},
@@ -775,6 +776,16 @@
"@jridgewell/sourcemap-codec": "^1.4.10"
}
},
"node_modules/@emnapi/runtime": {
"version": "1.4.3",
"resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.4.3.tgz",
"integrity": "sha512-pBPWdu6MLKROBX05wSNKcNb++m5Er+KQ9QkB+WVM+pW2Kx9hoSrVTnu3BdkI5eBLZoKu/J6mW/B6i6bJB2ytXQ==",
"license": "MIT",
"optional": true,
"dependencies": {
"tslib": "^2.4.0"
}
},
"node_modules/@eslint-community/eslint-utils": {
"version": "4.5.1",
"resolved": "https://registry.npmjs.org/@eslint-community/eslint-utils/-/eslint-utils-4.5.1.tgz",
@@ -992,6 +1003,402 @@
"url": "https://github.com/sponsors/nzakas"
}
},
"node_modules/@img/sharp-darwin-arm64": {
"version": "0.34.2",
"resolved": "https://registry.npmjs.org/@img/sharp-darwin-arm64/-/sharp-darwin-arm64-0.34.2.tgz",
"integrity": "sha512-OfXHZPppddivUJnqyKoi5YVeHRkkNE2zUFT2gbpKxp/JZCFYEYubnMg+gOp6lWfasPrTS+KPosKqdI+ELYVDtg==",
"cpu": [
"arm64"
],
"license": "Apache-2.0",
"optional": true,
"os": [
"darwin"
],
"engines": {
"node": "^18.17.0 || ^20.3.0 || >=21.0.0"
},
"funding": {
"url": "https://opencollective.com/libvips"
},
"optionalDependencies": {
"@img/sharp-libvips-darwin-arm64": "1.1.0"
}
},
"node_modules/@img/sharp-darwin-x64": {
"version": "0.34.2",
"resolved": "https://registry.npmjs.org/@img/sharp-darwin-x64/-/sharp-darwin-x64-0.34.2.tgz",
"integrity": "sha512-dYvWqmjU9VxqXmjEtjmvHnGqF8GrVjM2Epj9rJ6BUIXvk8slvNDJbhGFvIoXzkDhrJC2jUxNLz/GUjjvSzfw+g==",
"cpu": [
"x64"
],
"license": "Apache-2.0",
"optional": true,
"os": [
"darwin"
],
"engines": {
"node": "^18.17.0 || ^20.3.0 || >=21.0.0"
},
"funding": {
"url": "https://opencollective.com/libvips"
},
"optionalDependencies": {
"@img/sharp-libvips-darwin-x64": "1.1.0"
}
},
"node_modules/@img/sharp-libvips-darwin-arm64": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-arm64/-/sharp-libvips-darwin-arm64-1.1.0.tgz",
"integrity": "sha512-HZ/JUmPwrJSoM4DIQPv/BfNh9yrOA8tlBbqbLz4JZ5uew2+o22Ik+tHQJcih7QJuSa0zo5coHTfD5J8inqj9DA==",
"cpu": [
"arm64"
],
"license": "LGPL-3.0-or-later",
"optional": true,
"os": [
"darwin"
],
"funding": {
"url": "https://opencollective.com/libvips"
}
},
"node_modules/@img/sharp-libvips-darwin-x64": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-x64/-/sharp-libvips-darwin-x64-1.1.0.tgz",
"integrity": "sha512-Xzc2ToEmHN+hfvsl9wja0RlnXEgpKNmftriQp6XzY/RaSfwD9th+MSh0WQKzUreLKKINb3afirxW7A0fz2YWuQ==",
"cpu": [
"x64"
],
"license": "LGPL-3.0-or-later",
"optional": true,
"os": [
"darwin"
],
"funding": {
"url": "https://opencollective.com/libvips"
}
},
"node_modules/@img/sharp-libvips-linux-arm": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm/-/sharp-libvips-linux-arm-1.1.0.tgz",
"integrity": "sha512-s8BAd0lwUIvYCJyRdFqvsj+BJIpDBSxs6ivrOPm/R7piTs5UIwY5OjXrP2bqXC9/moGsyRa37eYWYCOGVXxVrA==",
"cpu": [
"arm"
],
"license": "LGPL-3.0-or-later",
"optional": true,
"os": [
"linux"
],
"funding": {
"url": "https://opencollective.com/libvips"
}
},
"node_modules/@img/sharp-libvips-linux-arm64": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm64/-/sharp-libvips-linux-arm64-1.1.0.tgz",
"integrity": "sha512-IVfGJa7gjChDET1dK9SekxFFdflarnUB8PwW8aGwEoF3oAsSDuNUTYS+SKDOyOJxQyDC1aPFMuRYLoDInyV9Ew==",
"cpu": [
"arm64"
],
"license": "LGPL-3.0-or-later",
"optional": true,
"os": [
"linux"
],
"funding": {
"url": "https://opencollective.com/libvips"
}
},
"node_modules/@img/sharp-libvips-linux-ppc64": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-ppc64/-/sharp-libvips-linux-ppc64-1.1.0.tgz",
"integrity": "sha512-tiXxFZFbhnkWE2LA8oQj7KYR+bWBkiV2nilRldT7bqoEZ4HiDOcePr9wVDAZPi/Id5fT1oY9iGnDq20cwUz8lQ==",
"cpu": [
"ppc64"
],
"license": "LGPL-3.0-or-later",
"optional": true,
"os": [
"linux"
],
"funding": {
"url": "https://opencollective.com/libvips"
}
},
"node_modules/@img/sharp-libvips-linux-s390x": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-s390x/-/sharp-libvips-linux-s390x-1.1.0.tgz",
"integrity": "sha512-xukSwvhguw7COyzvmjydRb3x/09+21HykyapcZchiCUkTThEQEOMtBj9UhkaBRLuBrgLFzQ2wbxdeCCJW/jgJA==",
"cpu": [
"s390x"
],
"license": "LGPL-3.0-or-later",
"optional": true,
"os": [
"linux"
],
"funding": {
"url": "https://opencollective.com/libvips"
}
},
"node_modules/@img/sharp-libvips-linux-x64": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-x64/-/sharp-libvips-linux-x64-1.1.0.tgz",
"integrity": "sha512-yRj2+reB8iMg9W5sULM3S74jVS7zqSzHG3Ol/twnAAkAhnGQnpjj6e4ayUz7V+FpKypwgs82xbRdYtchTTUB+Q==",
"cpu": [
"x64"
],
"license": "LGPL-3.0-or-later",
"optional": true,
"os": [
"linux"
],
"funding": {
"url": "https://opencollective.com/libvips"
}
},
"node_modules/@img/sharp-libvips-linuxmusl-arm64": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-arm64/-/sharp-libvips-linuxmusl-arm64-1.1.0.tgz",
"integrity": "sha512-jYZdG+whg0MDK+q2COKbYidaqW/WTz0cc1E+tMAusiDygrM4ypmSCjOJPmFTvHHJ8j/6cAGyeDWZOsK06tP33w==",
"cpu": [
"arm64"
],
"license": "LGPL-3.0-or-later",
"optional": true,
"os": [
"linux"
],
"funding": {
"url": "https://opencollective.com/libvips"
}
},
"node_modules/@img/sharp-libvips-linuxmusl-x64": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-x64/-/sharp-libvips-linuxmusl-x64-1.1.0.tgz",
"integrity": "sha512-wK7SBdwrAiycjXdkPnGCPLjYb9lD4l6Ze2gSdAGVZrEL05AOUJESWU2lhlC+Ffn5/G+VKuSm6zzbQSzFX/P65A==",
"cpu": [
"x64"
],
"license": "LGPL-3.0-or-later",
"optional": true,
"os": [
"linux"
],
"funding": {
"url": "https://opencollective.com/libvips"
}
},
"node_modules/@img/sharp-linux-arm": {
"version": "0.34.2",
"resolved": "https://registry.npmjs.org/@img/sharp-linux-arm/-/sharp-linux-arm-0.34.2.tgz",
"integrity": "sha512-0DZzkvuEOqQUP9mo2kjjKNok5AmnOr1jB2XYjkaoNRwpAYMDzRmAqUIa1nRi58S2WswqSfPOWLNOr0FDT3H5RQ==",
"cpu": [
"arm"
],
"license": "Apache-2.0",
"optional": true,
"os": [
"linux"
],
"engines": {
"node": "^18.17.0 || ^20.3.0 || >=21.0.0"
},
"funding": {
"url": "https://opencollective.com/libvips"
},
"optionalDependencies": {
"@img/sharp-libvips-linux-arm": "1.1.0"
}
},
"node_modules/@img/sharp-linux-arm64": {
"version": "0.34.2",
"resolved": "https://registry.npmjs.org/@img/sharp-linux-arm64/-/sharp-linux-arm64-0.34.2.tgz",
"integrity": "sha512-D8n8wgWmPDakc83LORcfJepdOSN6MvWNzzz2ux0MnIbOqdieRZwVYY32zxVx+IFUT8er5KPcyU3XXsn+GzG/0Q==",
"cpu": [
"arm64"
],
"license": "Apache-2.0",
"optional": true,
"os": [
"linux"
],
"engines": {
"node": "^18.17.0 || ^20.3.0 || >=21.0.0"
},
"funding": {
"url": "https://opencollective.com/libvips"
},
"optionalDependencies": {
"@img/sharp-libvips-linux-arm64": "1.1.0"
}
},
"node_modules/@img/sharp-linux-s390x": {
"version": "0.34.2",
"resolved": "https://registry.npmjs.org/@img/sharp-linux-s390x/-/sharp-linux-s390x-0.34.2.tgz",
"integrity": "sha512-EGZ1xwhBI7dNISwxjChqBGELCWMGDvmxZXKjQRuqMrakhO8QoMgqCrdjnAqJq/CScxfRn+Bb7suXBElKQpPDiw==",
"cpu": [
"s390x"
],
"license": "Apache-2.0",
"optional": true,
"os": [
"linux"
],
"engines": {
"node": "^18.17.0 || ^20.3.0 || >=21.0.0"
},
"funding": {
"url": "https://opencollective.com/libvips"
},
"optionalDependencies": {
"@img/sharp-libvips-linux-s390x": "1.1.0"
}
},
"node_modules/@img/sharp-linux-x64": {
"version": "0.34.2",
"resolved": "https://registry.npmjs.org/@img/sharp-linux-x64/-/sharp-linux-x64-0.34.2.tgz",
"integrity": "sha512-sD7J+h5nFLMMmOXYH4DD9UtSNBD05tWSSdWAcEyzqW8Cn5UxXvsHAxmxSesYUsTOBmUnjtxghKDl15EvfqLFbQ==",
"cpu": [
"x64"
],
"license": "Apache-2.0",
"optional": true,
"os": [
"linux"
],
"engines": {
"node": "^18.17.0 || ^20.3.0 || >=21.0.0"
},
"funding": {
"url": "https://opencollective.com/libvips"
},
"optionalDependencies": {
"@img/sharp-libvips-linux-x64": "1.1.0"
}
},
"node_modules/@img/sharp-linuxmusl-arm64": {
"version": "0.34.2",
"resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-arm64/-/sharp-linuxmusl-arm64-0.34.2.tgz",
"integrity": "sha512-NEE2vQ6wcxYav1/A22OOxoSOGiKnNmDzCYFOZ949xFmrWZOVII1Bp3NqVVpvj+3UeHMFyN5eP/V5hzViQ5CZNA==",
"cpu": [
"arm64"
],
"license": "Apache-2.0",
"optional": true,
"os": [
"linux"
],
"engines": {
"node": "^18.17.0 || ^20.3.0 || >=21.0.0"
},
"funding": {
"url": "https://opencollective.com/libvips"
},
"optionalDependencies": {
"@img/sharp-libvips-linuxmusl-arm64": "1.1.0"
}
},
"node_modules/@img/sharp-linuxmusl-x64": {
"version": "0.34.2",
"resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-x64/-/sharp-linuxmusl-x64-0.34.2.tgz",
"integrity": "sha512-DOYMrDm5E6/8bm/yQLCWyuDJwUnlevR8xtF8bs+gjZ7cyUNYXiSf/E8Kp0Ss5xasIaXSHzb888V1BE4i1hFhAA==",
"cpu": [
"x64"
],
"license": "Apache-2.0",
"optional": true,
"os": [
"linux"
],
"engines": {
"node": "^18.17.0 || ^20.3.0 || >=21.0.0"
},
"funding": {
"url": "https://opencollective.com/libvips"
},
"optionalDependencies": {
"@img/sharp-libvips-linuxmusl-x64": "1.1.0"
}
},
"node_modules/@img/sharp-wasm32": {
"version": "0.34.2",
"resolved": "https://registry.npmjs.org/@img/sharp-wasm32/-/sharp-wasm32-0.34.2.tgz",
"integrity": "sha512-/VI4mdlJ9zkaq53MbIG6rZY+QRN3MLbR6usYlgITEzi4Rpx5S6LFKsycOQjkOGmqTNmkIdLjEvooFKwww6OpdQ==",
"cpu": [
"wasm32"
],
"license": "Apache-2.0 AND LGPL-3.0-or-later AND MIT",
"optional": true,
"dependencies": {
"@emnapi/runtime": "^1.4.3"
},
"engines": {
"node": "^18.17.0 || ^20.3.0 || >=21.0.0"
},
"funding": {
"url": "https://opencollective.com/libvips"
}
},
"node_modules/@img/sharp-win32-arm64": {
"version": "0.34.2",
"resolved": "https://registry.npmjs.org/@img/sharp-win32-arm64/-/sharp-win32-arm64-0.34.2.tgz",
"integrity": "sha512-cfP/r9FdS63VA5k0xiqaNaEoGxBg9k7uE+RQGzuK9fHt7jib4zAVVseR9LsE4gJcNWgT6APKMNnCcnyOtmSEUQ==",
"cpu": [
"arm64"
],
"license": "Apache-2.0 AND LGPL-3.0-or-later",
"optional": true,
"os": [
"win32"
],
"engines": {
"node": "^18.17.0 || ^20.3.0 || >=21.0.0"
},
"funding": {
"url": "https://opencollective.com/libvips"
}
},
"node_modules/@img/sharp-win32-ia32": {
"version": "0.34.2",
"resolved": "https://registry.npmjs.org/@img/sharp-win32-ia32/-/sharp-win32-ia32-0.34.2.tgz",
"integrity": "sha512-QLjGGvAbj0X/FXl8n1WbtQ6iVBpWU7JO94u/P2M4a8CFYsvQi4GW2mRy/JqkRx0qpBzaOdKJKw8uc930EX2AHw==",
"cpu": [
"ia32"
],
"license": "Apache-2.0 AND LGPL-3.0-or-later",
"optional": true,
"os": [
"win32"
],
"engines": {
"node": "^18.17.0 || ^20.3.0 || >=21.0.0"
},
"funding": {
"url": "https://opencollective.com/libvips"
}
},
"node_modules/@img/sharp-win32-x64": {
"version": "0.34.2",
"resolved": "https://registry.npmjs.org/@img/sharp-win32-x64/-/sharp-win32-x64-0.34.2.tgz",
"integrity": "sha512-aUdT6zEYtDKCaxkofmmJDJYGCf0+pJg3eU9/oBuqvEeoB9dKI6ZLc/1iLJCTuJQDO4ptntAlkUmHgGjyuobZbw==",
"cpu": [
"x64"
],
"license": "Apache-2.0 AND LGPL-3.0-or-later",
"optional": true,
"os": [
"win32"
],
"engines": {
"node": "^18.17.0 || ^20.3.0 || >=21.0.0"
},
"funding": {
"url": "https://opencollective.com/libvips"
}
},
"node_modules/@inquirer/checkbox": {
"version": "4.1.4",
"resolved": "https://registry.npmjs.org/@inquirer/checkbox/-/checkbox-4.1.4.tgz",
@@ -5494,11 +5901,23 @@
"dev": true,
"license": "MIT"
},
"node_modules/color": {
"version": "4.2.3",
"resolved": "https://registry.npmjs.org/color/-/color-4.2.3.tgz",
"integrity": "sha512-1rXeuUUiGGrykh+CeBdu5Ie7OJwinCgQY0bc7GCRxy5xVHy+moaqkpL/jqQq0MtQOeYcrqEz4abc5f0KtU7W4A==",
"license": "MIT",
"dependencies": {
"color-convert": "^2.0.1",
"color-string": "^1.9.0"
},
"engines": {
"node": ">=12.5.0"
}
},
"node_modules/color-convert": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
"integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==",
"dev": true,
"license": "MIT",
"dependencies": {
"color-name": "~1.1.4"
@@ -5511,9 +5930,18 @@
"version": "1.1.4",
"resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
"integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==",
"dev": true,
"license": "MIT"
},
"node_modules/color-string": {
"version": "1.9.1",
"resolved": "https://registry.npmjs.org/color-string/-/color-string-1.9.1.tgz",
"integrity": "sha512-shrVawQFojnZv6xM40anx4CkoDP+fZsw/ZerEMsW/pyzsRbElpsL/DBVW7q3ExxwusdNXI3lXpuhEZkzs8p5Eg==",
"license": "MIT",
"dependencies": {
"color-name": "^1.0.0",
"simple-swizzle": "^0.2.2"
}
},
"node_modules/combined-stream": {
"version": "1.0.8",
"resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz",
@@ -5859,6 +6287,15 @@
"node": ">= 0.8"
}
},
"node_modules/detect-libc": {
"version": "2.0.4",
"resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.0.4.tgz",
"integrity": "sha512-3UDv+G9CsCKO1WKMGw9fwq/SWJYbI0c5Y7LU1AXYoDdbhE2AHQ6N6Nb34sG8Fj7T5APy8qXDCKuuIHd1BR0tVA==",
"license": "Apache-2.0",
"engines": {
"node": ">=8"
}
},
"node_modules/detect-newline": {
"version": "3.1.0",
"resolved": "https://registry.npmjs.org/detect-newline/-/detect-newline-3.1.0.tgz",
@@ -10721,10 +11158,9 @@
}
},
"node_modules/semver": {
"version": "7.7.1",
"resolved": "https://registry.npmjs.org/semver/-/semver-7.7.1.tgz",
"integrity": "sha512-hlq8tAfn0m/61p4BVRcPzIGr6LKiMwo4VM6dGi6pt4qcRkmNzTcWq6eCEjEh+qXjkMDvPlOFFSGwQjoEa6gyMA==",
"dev": true,
"version": "7.7.2",
"resolved": "https://registry.npmjs.org/semver/-/semver-7.7.2.tgz",
"integrity": "sha512-RF0Fw+rO5AMf9MAyaRXI4AV0Ulj5lMHqVxxdSgiVbixSCXoEmmX/jk0CuJw4+3SqroYO9VoUh+HcuJivvtJemA==",
"license": "ISC",
"bin": {
"semver": "bin/semver.js"
@@ -10815,6 +11251,47 @@
"integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==",
"license": "ISC"
},
"node_modules/sharp": {
"version": "0.34.2",
"resolved": "https://registry.npmjs.org/sharp/-/sharp-0.34.2.tgz",
"integrity": "sha512-lszvBmB9QURERtyKT2bNmsgxXK0ShJrL/fvqlonCo7e6xBF8nT8xU6pW+PMIbLsz0RxQk3rgH9kd8UmvOzlMJg==",
"hasInstallScript": true,
"license": "Apache-2.0",
"dependencies": {
"color": "^4.2.3",
"detect-libc": "^2.0.4",
"semver": "^7.7.2"
},
"engines": {
"node": "^18.17.0 || ^20.3.0 || >=21.0.0"
},
"funding": {
"url": "https://opencollective.com/libvips"
},
"optionalDependencies": {
"@img/sharp-darwin-arm64": "0.34.2",
"@img/sharp-darwin-x64": "0.34.2",
"@img/sharp-libvips-darwin-arm64": "1.1.0",
"@img/sharp-libvips-darwin-x64": "1.1.0",
"@img/sharp-libvips-linux-arm": "1.1.0",
"@img/sharp-libvips-linux-arm64": "1.1.0",
"@img/sharp-libvips-linux-ppc64": "1.1.0",
"@img/sharp-libvips-linux-s390x": "1.1.0",
"@img/sharp-libvips-linux-x64": "1.1.0",
"@img/sharp-libvips-linuxmusl-arm64": "1.1.0",
"@img/sharp-libvips-linuxmusl-x64": "1.1.0",
"@img/sharp-linux-arm": "0.34.2",
"@img/sharp-linux-arm64": "0.34.2",
"@img/sharp-linux-s390x": "0.34.2",
"@img/sharp-linux-x64": "0.34.2",
"@img/sharp-linuxmusl-arm64": "0.34.2",
"@img/sharp-linuxmusl-x64": "0.34.2",
"@img/sharp-wasm32": "0.34.2",
"@img/sharp-win32-arm64": "0.34.2",
"@img/sharp-win32-ia32": "0.34.2",
"@img/sharp-win32-x64": "0.34.2"
}
},
"node_modules/shebang-command": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz",
@@ -10921,6 +11398,21 @@
"url": "https://github.com/sponsors/isaacs"
}
},
"node_modules/simple-swizzle": {
"version": "0.2.2",
"resolved": "https://registry.npmjs.org/simple-swizzle/-/simple-swizzle-0.2.2.tgz",
"integrity": "sha512-JA//kQgZtbuY83m+xT+tXJkmJncGMTFT+C+g2h2R9uxkYIrE2yy9sgmcLhCnw57/WSD+Eh3J97FPEDFnbXnDUg==",
"license": "MIT",
"dependencies": {
"is-arrayish": "^0.3.1"
}
},
"node_modules/simple-swizzle/node_modules/is-arrayish": {
"version": "0.3.2",
"resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.3.2.tgz",
"integrity": "sha512-eVRqCvVlZbuw3GrM63ovNSNAeA1K16kaR/LRY/92w0zxQ5/1YzwblUX652i4Xs9RwAGjW9d9y6X88t8OaAJfWQ==",
"license": "MIT"
},
"node_modules/sisteransi": {
"version": "1.0.5",
"resolved": "https://registry.npmjs.org/sisteransi/-/sisteransi-1.0.5.tgz",
packages/bytebotd/package.json
+3 -2
View File
@@ -34,14 +34,15 @@
"@nestjs/serve-static": "^5.0.3",
"@nestjs/websockets": "^11.1.2",
"@nut-tree-fork/nut-js": "^4.2.6",
"@rekog/mcp-nest": "^1.6.2",
"class-transformer": "^0.5.1",
"class-validator": "^0.14.2",
"http-proxy-middleware": "^3.0.5",
"reflect-metadata": "^0.2.2",
"rxjs": "^7.8.1",
"sharp": "^0.34.2",
"socket.io": "^4.8.1",
"uiohook-napi": "^1.5.4",
"@rekog/mcp-nest": "^1.6.2"
"uiohook-napi": "^1.5.4"
},
"devDependencies": {
"@eslint/eslintrc": "^3.2.0",
packages/bytebotd/src/mcp/compressor.ts
+233
View File
@@ -0,0 +1,233 @@
import * as sharp from 'sharp';
interface CompressionOptions {
targetSizeKB?: number;
initialQuality?: number;
minQuality?: number;
format?: 'png' | 'jpeg' | 'webp';
maxIterations?: number;
}
interface CompressionResult {
base64: string;
sizeBytes: number;
sizeKB: number;
sizeMB: number;
quality: number;
format: string;
iterations: number;
}
class Base64ImageCompressor {
/**
* Compress a base64 PNG string to under specified size (default 1MB)
*/
static async compressToSize(
base64String: string,
options: CompressionOptions = {},
): Promise<CompressionResult> {
const {
targetSizeKB = 1024, // 1MB default
initialQuality = 95,
minQuality = 10,
format = 'png',
maxIterations = 10,
} = options;
// Extract base64 data (remove data URL prefix if present)
const base64Data = base64String.replace(/^data:image\/\w+;base64,/, '');
const inputBuffer = Buffer.from(base64Data, 'base64');
let quality = initialQuality;
let outputBuffer: Buffer;
let iterations = 0;
// Binary search for optimal quality
let low = minQuality;
let high = initialQuality;
let bestResult: { buffer: Buffer; quality: number } | null = null;
while (low <= high && iterations < maxIterations) {
quality = Math.floor((low + high) / 2);
outputBuffer = await this.compressBuffer(inputBuffer, quality, format);
const sizeKB = outputBuffer.length / 1024;
if (sizeKB <= targetSizeKB) {
// Size is acceptable, try higher quality
bestResult = { buffer: outputBuffer, quality };
low = quality + 1;
} else {
// Size too large, reduce quality
high = quality - 1;
}
iterations++;
}
// If no result found under target size, use lowest quality
if (!bestResult) {
outputBuffer = await this.compressBuffer(inputBuffer, minQuality, format);
quality = minQuality;
} else {
outputBuffer = bestResult.buffer;
quality = bestResult.quality;
}
// Convert back to base64
const outputBase64 = outputBuffer.toString('base64');
const sizeBytes = outputBuffer.length;
return {
base64: outputBase64,
sizeBytes,
sizeKB: sizeBytes / 1024,
sizeMB: sizeBytes / (1024 * 1024),
quality,
format,
iterations,
};
}
/**
* Compress buffer with specified quality
*/
private static async compressBuffer(
inputBuffer: Buffer,
quality: number,
format: 'png' | 'jpeg' | 'webp',
): Promise<Buffer> {
let sharpInstance = sharp(inputBuffer);
switch (format) {
case 'png':
return sharpInstance
.png({
quality,
compressionLevel: 9,
adaptiveFiltering: true,
palette: true,
})
.toBuffer();
case 'jpeg':
return sharpInstance
.jpeg({
quality,
progressive: true,
mozjpeg: true,
optimizeScans: true,
})
.toBuffer();
case 'webp':
return sharpInstance
.webp({
quality,
alphaQuality: quality,
lossless: false,
nearLossless: false,
smartSubsample: true,
})
.toBuffer();
default:
throw new Error(`Unsupported format: ${format}`);
}
}
/**
* Compress with dimension reduction if quality alone isn't enough
*/
static async compressWithResize(
base64String: string,
options: CompressionOptions & {
maxWidth?: number;
maxHeight?: number;
} = {},
): Promise<CompressionResult> {
const {
targetSizeKB = 1024,
maxWidth = 2048,
maxHeight = 2048,
...compressionOptions
} = options;
// First try compression without resizing
let result = await this.compressToSize(base64String, compressionOptions);
// If still too large, apply progressive resizing
if (result.sizeKB > targetSizeKB) {
const base64Data = base64String.replace(/^data:image\/\w+;base64,/, '');
const inputBuffer = Buffer.from(base64Data, 'base64');
const metadata = await sharp(inputBuffer).metadata();
const originalWidth = metadata.width || maxWidth;
const originalHeight = metadata.height || maxHeight;
let scale = 0.9; // Start with 90% of original size
while (result.sizeKB > targetSizeKB && scale > 0.3) {
const newWidth = Math.floor(originalWidth * scale);
const newHeight = Math.floor(originalHeight * scale);
const resizedBuffer = await sharp(inputBuffer)
.resize(newWidth, newHeight, {
fit: 'inside',
withoutEnlargement: true,
})
.toBuffer();
const resizedBase64 = resizedBuffer.toString('base64');
result = await this.compressToSize(resizedBase64, compressionOptions);
scale -= 0.1;
}
}
return result;
}
/**
* Get size information for a base64 string
*/
static getBase64SizeInfo(base64String: string): {
bytes: number;
kb: number;
mb: number;
formatted: string;
} {
const base64Data = base64String.replace(/^data:image\/\w+;base64,/, '');
const bytes = Buffer.from(base64Data, 'base64').length;
const kb = bytes / 1024;
const mb = bytes / (1024 * 1024);
let formatted: string;
if (mb >= 1) {
formatted = `${mb.toFixed(2)} MB`;
} else if (kb >= 1) {
formatted = `${kb.toFixed(2)} KB`;
} else {
formatted = `${bytes} bytes`;
}
return { bytes, kb, mb, formatted };
}
}
// Utility function for quick compression
export async function compressPngBase64Under1MB(
base64String: string,
): Promise<string> {
const result = await Base64ImageCompressor.compressToSize(base64String, {
targetSizeKB: 1024,
format: 'png',
initialQuality: 95,
minQuality: 10,
});
return result.base64;
}
// Export the class for more control
export { Base64ImageCompressor, CompressionOptions, CompressionResult };
packages/bytebotd/src/mcp/computer-use.tools.ts
+171 -66
View File
@@ -2,7 +2,7 @@ import { Injectable } from '@nestjs/common';
import { Tool } from '@rekog/mcp-nest';
import { z } from 'zod';
import { ComputerUseService } from '../computer-use/computer-use.service';
import { MessageContentType } from '@bytebot/shared';
import { compressPngBase64Under1MB } from './compressor';
@Injectable()
export class ComputerUseTools {
@@ -10,20 +10,23 @@ export class ComputerUseTools {
@Tool({
name: 'computer_move_mouse',
description: 'Move the mouse to specific coordinates.',
description: 'Moves the mouse cursor to the specified coordinates.',
parameters: z.object({
coordinates: z.object({ x: z.number(), y: z.number() }),
coordinates: z.object({
x: z.number().describe('The x-coordinate to move the mouse to.'),
y: z.number().describe('The y-coordinate to move the mouse to.'),
}),
}),
})
async moveMouse({ coordinates }: { coordinates: { x: number; y: number } }) {
try {
await this.computerUse.action({ action: 'move_mouse', coordinates });
return { content: [{ type: MessageContentType.Text, text: 'mouse moved' }] };
return { content: [{ type: 'text', text: 'mouse moved' }] };
} catch (err) {
return {
content: [
{
type: MessageContentType.Text,
type: 'text',
text: `Error moving mouse: ${(err as Error).message}`,
},
],
@@ -33,10 +36,21 @@ export class ComputerUseTools {
@Tool({
name: 'computer_trace_mouse',
description: 'Trace a path with the mouse.',
description:
'Moves the mouse cursor along a specified path of coordinates.',
parameters: z.object({
path: z.array(z.object({ x: z.number(), y: z.number() })),
holdKeys: z.array(z.string()).optional(),
path: z
.array(
z.object({
x: z.number().describe('The x-coordinate to move the mouse to.'),
y: z.number().describe('The y-coordinate to move the mouse to.'),
}),
)
.describe('An array of coordinate objects representing the path.'),
holdKeys: z
.array(z.string())
.optional()
.describe('Optional array of keys to hold during the trace.'),
}),
})
async traceMouse({
@@ -49,13 +63,13 @@ export class ComputerUseTools {
try {
await this.computerUse.action({ action: 'trace_mouse', path, holdKeys });
return {
content: [{ type: MessageContentType.Text, text: 'mouse traced' }],
content: [{ type: 'text', text: 'mouse traced' }],
};
} catch (err) {
return {
content: [
{
type: MessageContentType.Text,
type: 'text',
text: `Error tracing mouse: ${(err as Error).message}`,
},
],
@@ -65,12 +79,28 @@ export class ComputerUseTools {
@Tool({
name: 'computer_click_mouse',
description: 'Perform mouse clicks.',
description:
'Performs a mouse click at the specified coordinates or current position.',
parameters: z.object({
coordinates: z.object({ x: z.number(), y: z.number() }).optional(),
button: z.enum(['left', 'right', 'middle']),
holdKeys: z.array(z.string()).optional(),
numClicks: z.number(),
coordinates: z
.object({
x: z.number().describe('The x-coordinate to move the mouse to.'),
y: z.number().describe('The y-coordinate to move the mouse to.'),
})
.optional()
.describe(
'Optional coordinates for the click. If not provided, clicks at the current mouse position.',
),
button: z
.enum(['left', 'right', 'middle'])
.describe('The mouse button to click.'),
holdKeys: z
.array(z.string())
.optional()
.describe('Optional array of keys to hold during the click.'),
numClicks: z
.number()
.describe('Number of clicks to perform (e.g., 2 for double-click).'),
}),
})
async clickMouse({
@@ -93,13 +123,13 @@ export class ComputerUseTools {
numClicks,
});
return {
content: [{ type: MessageContentType.Text, text: 'mouse clicked' }],
content: [{ type: 'text', text: 'mouse clicked' }],
};
} catch (err) {
return {
content: [
{
type: MessageContentType.Text,
type: 'text',
text: `Error clicking mouse: ${(err as Error).message}`,
},
],
@@ -109,11 +139,24 @@ export class ComputerUseTools {
@Tool({
name: 'computer_press_mouse',
description: 'Press or release a mouse button.',
description:
'Presses or releases a specified mouse button at the given coordinates or current position.',
parameters: z.object({
coordinates: z.object({ x: z.number(), y: z.number() }).optional(),
button: z.enum(['left', 'right', 'middle']),
press: z.enum(['down', 'up']),
coordinates: z
.object({
x: z.number().describe('The x-coordinate for the mouse action.'),
y: z.number().describe('The y-coordinate for the mouse action.'),
})
.optional()
.describe(
'Optional coordinates for the mouse press/release. If not provided, uses the current mouse position.',
),
button: z
.enum(['left', 'right', 'middle'])
.describe('The mouse button to press or release.'),
press: z
.enum(['down', 'up'])
.describe('The action to perform (press or release).'),
}),
})
async pressMouse({
@@ -133,13 +176,13 @@ export class ComputerUseTools {
press,
});
return {
content: [{ type: MessageContentType.Text, text: 'mouse pressed' }],
content: [{ type: 'text', text: 'mouse pressed' }],
};
} catch (err) {
return {
content: [
{
type: MessageContentType.Text,
type: 'text',
text: `Error pressing mouse: ${(err as Error).message}`,
},
],
@@ -149,11 +192,30 @@ export class ComputerUseTools {
@Tool({
name: 'computer_drag_mouse',
description: 'Drag the mouse along a path.',
description:
'Drags the mouse from a starting point along a path while holding a specified button.',
parameters: z.object({
path: z.array(z.object({ x: z.number(), y: z.number() })),
button: z.enum(['left', 'right', 'middle']),
holdKeys: z.array(z.string()).optional(),
path: z
.array(
z.object({
x: z
.number()
.describe('The x-coordinate of a point in the drag path.'),
y: z
.number()
.describe('The y-coordinate of a point in the drag path.'),
}),
)
.describe(
'An array of coordinate objects representing the drag path. The first coordinate is the start point.',
),
button: z
.enum(['left', 'right', 'middle'])
.describe('The mouse button to hold while dragging.'),
holdKeys: z
.array(z.string())
.optional()
.describe('Optional array of keys to hold during the drag.'),
}),
})
async dragMouse({
@@ -173,13 +235,13 @@ export class ComputerUseTools {
holdKeys,
});
return {
content: [{ type: MessageContentType.Text, text: 'mouse dragged' }],
content: [{ type: 'text', text: 'mouse dragged' }],
};
} catch (err) {
return {
content: [
{
type: MessageContentType.Text,
type: 'text',
text: `Error dragging mouse: ${(err as Error).message}`,
},
],
@@ -189,12 +251,35 @@ export class ComputerUseTools {
@Tool({
name: 'computer_scroll',
description: 'Scroll the mouse wheel.',
description: 'Scrolls the mouse wheel up, down, left, or right.',
parameters: z.object({
coordinates: z.object({ x: z.number(), y: z.number() }).optional(),
direction: z.enum(['up', 'down', 'left', 'right']),
numScrolls: z.number(),
holdKeys: z.array(z.string()).optional(),
coordinates: z
.object({
x: z
.number()
.describe(
'The x-coordinate for the scroll action (if applicable).',
),
y: z
.number()
.describe(
'The y-coordinate for the scroll action (if applicable).',
),
})
.optional()
.describe(
'Coordinates for where the scroll should occur. Behavior might depend on the OS/application.',
),
direction: z
.enum(['up', 'down', 'left', 'right'])
.describe('The direction to scroll the mouse wheel.'),
numScrolls: z
.number()
.describe('The number of times to scroll the mouse wheel.'),
holdKeys: z
.array(z.string())
.optional()
.describe('Optional array of keys to hold during the scroll.'),
}),
})
async scroll({
@@ -216,12 +301,12 @@ export class ComputerUseTools {
numScrolls,
holdKeys,
});
return { content: [{ type: MessageContentType.Text, text: 'scrolled' }] };
return { content: [{ type: 'text', text: 'scrolled' }] };
} catch (err) {
return {
content: [
{
type: MessageContentType.Text,
type: 'text',
text: `Error scrolling: ${(err as Error).message}`,
},
],
@@ -231,21 +316,29 @@ export class ComputerUseTools {
@Tool({
name: 'computer_type_keys',
description: 'Press a sequence of keys.',
description:
'Simulates typing a sequence of keys, often used for shortcuts involving modifier keys (e.g., Ctrl+C). Presses and releases each key in order.',
parameters: z.object({
keys: z.array(z.string()),
delay: z.number().optional(),
keys: z
.array(z.string())
.describe(
'An array of key names to type in sequence (e.g., ["control", "c"]).',
),
delay: z
.number()
.optional()
.describe('Optional delay in milliseconds between key presses.'),
}),
})
async typeKeys({ keys, delay }: { keys: string[]; delay?: number }) {
try {
await this.computerUse.action({ action: 'type_keys', keys, delay });
return { content: [{ type: MessageContentType.Text, text: 'keys typed' }] };
return { content: [{ type: 'text', text: 'keys typed' }] };
} catch (err) {
return {
content: [
{
type: MessageContentType.Text,
type: 'text',
text: `Error typing keys: ${(err as Error).message}`,
},
],
@@ -255,21 +348,28 @@ export class ComputerUseTools {
@Tool({
name: 'computer_press_keys',
description: 'Hold or release keys.',
description:
'Simulates pressing down or releasing specific keys. Useful for holding modifier keys.',
parameters: z.object({
keys: z.array(z.string()),
press: z.enum(['down', 'up']),
keys: z
.array(z.string())
.describe(
'An array of key names to press or release (e.g., ["shift"]).',
),
press: z
.enum(['down', 'up'])
.describe('Whether to press the keys down or release them up.'),
}),
})
async pressKeys({ keys, press }: { keys: string[]; press: 'down' | 'up' }) {
try {
await this.computerUse.action({ action: 'press_keys', keys, press });
return { content: [{ type: MessageContentType.Text, text: 'keys pressed' }] };
return { content: [{ type: 'text', text: 'keys pressed' }] };
} catch (err) {
return {
content: [
{
type: MessageContentType.Text,
type: 'text',
text: `Error pressing keys: ${(err as Error).message}`,
},
],
@@ -279,21 +379,24 @@ export class ComputerUseTools {
@Tool({
name: 'computer_type_text',
description: 'Type a text string.',
description: 'Simulates typing a string of text character by character.',
parameters: z.object({
text: z.string(),
delay: z.number().optional(),
text: z.string().describe('The text string to type.'),
delay: z
.number()
.optional()
.describe('Optional delay in milliseconds between key presses.'),
}),
})
async typeText({ text, delay }: { text: string; delay?: number }) {
try {
await this.computerUse.action({ action: 'type_text', text, delay });
return { content: [{ type: MessageContentType.Text, text: 'text typed' }] };
return { content: [{ type: 'text', text: 'text typed' }] };
} catch (err) {
return {
content: [
{
type: MessageContentType.Text,
type: 'text',
text: `Error typing text: ${(err as Error).message}`,
},
],
@@ -303,18 +406,23 @@ export class ComputerUseTools {
@Tool({
name: 'computer_wait',
description: 'Wait for a period of time.',
parameters: z.object({ duration: z.number() }),
description: 'Pauses execution for a specified duration.',
parameters: z.object({
duration: z
.number()
.default(500)
.describe('The duration to wait in milliseconds.'),
}),
})
async wait({ duration }: { duration: number }) {
try {
await this.computerUse.action({ action: 'wait', duration });
return { content: [{ type: MessageContentType.Text, text: 'waiting done' }] };
return { content: [{ type: 'text', text: 'waiting done' }] };
} catch (err) {
return {
content: [
{
type: MessageContentType.Text,
type: 'text',
text: `Error waiting: ${(err as Error).message}`,
},
],
@@ -324,7 +432,7 @@ export class ComputerUseTools {
@Tool({
name: 'computer_screenshot',
description: 'Capture a screenshot of the desktop.',
description: 'Captures a screenshot of the current screen.',
})
async screenshot() {
try {
@@ -334,12 +442,9 @@ export class ComputerUseTools {
return {
content: [
{
type: MessageContentType.Image,
source: {
type: 'base64',
media_type: 'image/png',
data: shot.image,
},
type: 'image',
data: await compressPngBase64Under1MB(shot.image),
mimeType: 'image/png',
},
],
};
@@ -347,7 +452,7 @@ export class ComputerUseTools {
return {
content: [
{
type: MessageContentType.Text,
type: 'text',
text: `Error taking screenshot: ${(err as Error).message}`,
},
],
@@ -357,7 +462,7 @@ export class ComputerUseTools {
@Tool({
name: 'computer_cursor_position',
description: 'Get the current cursor position.',
description: 'Gets the current (x, y) coordinates of the mouse cursor.',
})
async cursorPosition() {
try {
@@ -367,7 +472,7 @@ export class ComputerUseTools {
return {
content: [
{
type: MessageContentType.Text,
type: 'text',
text: JSON.stringify(pos),
},
],
@@ -376,7 +481,7 @@ export class ComputerUseTools {
return {
content: [
{
type: MessageContentType.Text,
type: 'text',
text: `Error getting cursor position: ${(err as Error).message}`,
},
],