diff --git a/README.md b/README.md index 9372887..93cf7ff 100644 --- a/README.md +++ b/README.md @@ -158,6 +158,23 @@ agent_s \ --grounding_height 1080 ``` +#### Local Coding Environment (Optional) +For tasks that require code execution (e.g., data processing, file manipulation, system automation), you can enable the local coding environment: + +```bash +agent_s \ + --provider openai \ + --model gpt-5-2025-08-07 \ + --ground_provider huggingface \ + --ground_url http://localhost:8080 \ + --ground_model ui-tars-1.5-7b \ + --grounding_width 1920 \ + --grounding_height 1080 \ + --enable_local_env +``` + +⚠️ **WARNING**: The local coding environment executes arbitrary Python and Bash code locally on your machine. Only use this feature in trusted environments and with trusted inputs. + #### Required Parameters - **`--provider`**: Main generation model provider (e.g., openai, anthropic, etc.) - Default: "openai" - **`--model`**: Main generation model name (e.g., gpt-5-2025-08-07) - Default: "gpt-5-2025-08-07" @@ -181,6 +198,30 @@ The grounding width and height should match the output coordinate resolution of - **`--ground_api_key`**: API key for grounding model endpoint - Default: "" - **`--max_trajectory_length`**: Maximum number of image turns to keep in trajectory - Default: 8 - **`--enable_reflection`**: Enable reflection agent to assist the worker agent - Default: True +- **`--enable_local_env`**: Enable local coding environment for code execution (WARNING: Executes arbitrary code locally) - Default: False + +#### Local Coding Environment Details +The local coding environment enables Agent S3 to execute Python and Bash code directly on your machine. This is particularly useful for: + +- **Data Processing**: Manipulating spreadsheets, CSV files, or databases +- **File Operations**: Bulk file processing, content extraction, or file organization +- **System Automation**: Configuration changes, system setup, or automation scripts +- **Code Development**: Writing, editing, or executing code files +- **Text Processing**: Document manipulation, content editing, or formatting + +When enabled, the agent can use the `call_code_agent` action to execute code blocks for tasks that can be completed through programming rather than GUI interaction. + +**Requirements:** +- **Python**: The same Python interpreter used to run Agent S3 (automatically detected) +- **Bash**: Available at `/bin/bash` (standard on macOS and Linux) +- **System Permissions**: The agent runs with the same permissions as the user executing it + +**Security Considerations:** +- The local environment executes arbitrary code with the same permissions as the user running the agent +- Only enable this feature in trusted environments +- Be cautious when the agent generates code for system-level operations +- Consider running in a sandboxed environment for untrusted tasks +- Bash scripts are executed with a 30-second timeout to prevent hanging processes ### `gui_agents` SDK @@ -190,6 +231,7 @@ import pyautogui import io from gui_agents.s3.agents.agent_s import AgentS3 from gui_agents.s3.agents.grounding import OSWorldACI +from gui_agents.s3.utils.local_env import LocalEnv # Optional: for local coding environment # Load in your API keys. from dotenv import load_dotenv @@ -234,7 +276,12 @@ engine_params_for_grounding = { Then, we define our grounding agent and Agent S3. ```python +# Optional: Enable local coding environment +enable_local_env = False # Set to True to enable local code execution +local_env = LocalEnv() if enable_local_env else None + grounding_agent = OSWorldACI( + env=local_env, # Pass local_env for code execution capability platform=current_platform, engine_params_for_generation=engine_params, engine_params_for_grounding=engine_params_for_grounding, diff --git a/gui_agents/s3/cli_app.py b/gui_agents/s3/cli_app.py index 78ffa2f..39d5be8 100644 --- a/gui_agents/s3/cli_app.py +++ b/gui_agents/s3/cli_app.py @@ -13,6 +13,7 @@ from PIL import Image from gui_agents.s3.agents.grounding import OSWorldACI from gui_agents.s3.agents.agent_s import AgentS3 +from gui_agents.s3.utils.local_env import LocalEnv current_platform = platform.system().lower() @@ -308,6 +309,12 @@ def main(): default=True, help="Enable reflection agent to assist the worker agent", ) + parser.add_argument( + "--enable_local_env", + action="store_true", + default=False, + help="Enable local coding environment for code execution (WARNING: Executes arbitrary code locally)", + ) args = parser.parse_args() @@ -336,8 +343,16 @@ def main(): "grounding_height": args.grounding_height, } + # Initialize environment based on user preference + local_env = None + if args.enable_local_env: + print( + "⚠️ WARNING: Local coding environment enabled. This will execute arbitrary code locally!" + ) + local_env = LocalEnv() + grounding_agent = OSWorldACI( - env=None, + env=local_env, platform=current_platform, engine_params_for_generation=engine_params, engine_params_for_grounding=engine_params_for_grounding, diff --git a/gui_agents/s3/utils/local_env.py b/gui_agents/s3/utils/local_env.py new file mode 100644 index 0000000..d69b1b5 --- /dev/null +++ b/gui_agents/s3/utils/local_env.py @@ -0,0 +1,77 @@ +import subprocess +import sys +from typing import Dict + + +class LocalController: + """Minimal controller to execute bash and python code locally. + + WARNING: Executing arbitrary code is dangerous. Only enable/use this in trusted + environments and with trusted inputs. + """ + + def run_bash_script(self, code: str, timeout: int = 30) -> Dict: + try: + proc = subprocess.run( + ["/bin/bash", "-lc", code], + capture_output=True, + text=True, + timeout=timeout, + ) + output = (proc.stdout or "") + (proc.stderr or "") + + print("BASH OUTPUT =======================================") + print(output) + print("BASH OUTPUT =======================================") + + return { + "status": "ok" if proc.returncode == 0 else "error", + "returncode": proc.returncode, + "output": output, + "error": "", + } + except subprocess.TimeoutExpired as e: + return { + "status": "error", + "returncode": -1, + "output": e.stdout or "", + "error": f"TimeoutExpired: {str(e)}", + } + except Exception as e: + return { + "status": "error", + "returncode": -1, + "output": "", + "error": str(e), + } + + def run_python_script(self, code: str) -> Dict: + try: + proc = subprocess.run( + [sys.executable, "-c", code], + capture_output=True, + text=True, + ) + print("PYTHON OUTPUT =======================================") + print(proc.stdout or "") + print("PYTHON OUTPUT =======================================") + return { + "status": "ok" if proc.returncode == 0 else "error", + "return_code": proc.returncode, + "output": proc.stdout or "", + "error": proc.stderr or "", + } + except Exception as e: + return { + "status": "error", + "return_code": -1, + "output": "", + "error": str(e), + } + + +class LocalEnv: + """Simple environment that provides a controller compatible with CodeAgent.""" + + def __init__(self): + self.controller = LocalController()