add local env option for code agent

README.md

@@ -158,6 +158,23 @@ agent_s \
     --grounding_height 1080
 ```
 
+#### Local Coding Environment (Optional)
+For tasks that require code execution (e.g., data processing, file manipulation, system automation), you can enable the local coding environment:
+
+```bash
+agent_s \
+    --provider openai \
+    --model gpt-5-2025-08-07 \
+    --ground_provider huggingface \
+    --ground_url http://localhost:8080 \
+    --ground_model ui-tars-1.5-7b \
+    --grounding_width 1920 \
+    --grounding_height 1080 \
+    --enable_local_env
+```
+
+⚠️ **WARNING**: The local coding environment executes arbitrary Python and Bash code locally on your machine. Only use this feature in trusted environments and with trusted inputs.
+
 #### Required Parameters
 - **`--provider`**: Main generation model provider (e.g., openai, anthropic, etc.) - Default: "openai"
 - **`--model`**: Main generation model name (e.g., gpt-5-2025-08-07) - Default: "gpt-5-2025-08-07"
@@ -181,6 +198,30 @@ The grounding width and height should match the output coordinate resolution of
 - **`--ground_api_key`**: API key for grounding model endpoint - Default: ""
 - **`--max_trajectory_length`**: Maximum number of image turns to keep in trajectory - Default: 8
 - **`--enable_reflection`**: Enable reflection agent to assist the worker agent - Default: True
+- **`--enable_local_env`**: Enable local coding environment for code execution (WARNING: Executes arbitrary code locally) - Default: False
+
+#### Local Coding Environment Details
+The local coding environment enables Agent S3 to execute Python and Bash code directly on your machine. This is particularly useful for:
+
+- **Data Processing**: Manipulating spreadsheets, CSV files, or databases
+- **File Operations**: Bulk file processing, content extraction, or file organization
+- **System Automation**: Configuration changes, system setup, or automation scripts
+- **Code Development**: Writing, editing, or executing code files
+- **Text Processing**: Document manipulation, content editing, or formatting
+
+When enabled, the agent can use the `call_code_agent` action to execute code blocks for tasks that can be completed through programming rather than GUI interaction.
+
+**Requirements:**
+- **Python**: The same Python interpreter used to run Agent S3 (automatically detected)
+- **Bash**: Available at `/bin/bash` (standard on macOS and Linux)
+- **System Permissions**: The agent runs with the same permissions as the user executing it
+
+**Security Considerations:**
+- The local environment executes arbitrary code with the same permissions as the user running the agent
+- Only enable this feature in trusted environments
+- Be cautious when the agent generates code for system-level operations
+- Consider running in a sandboxed environment for untrusted tasks
+- Bash scripts are executed with a 30-second timeout to prevent hanging processes
 
 ### `gui_agents` SDK
 
@@ -190,6 +231,7 @@ import pyautogui
 import io
 from gui_agents.s3.agents.agent_s import AgentS3
 from gui_agents.s3.agents.grounding import OSWorldACI
+from gui_agents.s3.utils.local_env import LocalEnv  # Optional: for local coding environment
 
 # Load in your API keys.
 from dotenv import load_dotenv
@@ -234,7 +276,12 @@ engine_params_for_grounding = {
 Then, we define our grounding agent and Agent S3.
 
 ```python
+# Optional: Enable local coding environment
+enable_local_env = False  # Set to True to enable local code execution
+local_env = LocalEnv() if enable_local_env else None
+
 grounding_agent = OSWorldACI(
+    env=local_env,  # Pass local_env for code execution capability
     platform=current_platform,
     engine_params_for_generation=engine_params,
     engine_params_for_grounding=engine_params_for_grounding,

gui_agents/s3/cli_app.py

@@ -13,6 +13,7 @@ from PIL import Image
 
 from gui_agents.s3.agents.grounding import OSWorldACI
 from gui_agents.s3.agents.agent_s import AgentS3
+from gui_agents.s3.utils.local_env import LocalEnv
 
 current_platform = platform.system().lower()
 
@@ -308,6 +309,12 @@ def main():
         default=True,
         help="Enable reflection agent to assist the worker agent",
     )
+    parser.add_argument(
+        "--enable_local_env",
+        action="store_true",
+        default=False,
+        help="Enable local coding environment for code execution (WARNING: Executes arbitrary code locally)",
+    )
 
     args = parser.parse_args()
 
@@ -336,8 +343,16 @@ def main():
         "grounding_height": args.grounding_height,
     }
 
+    # Initialize environment based on user preference
+    local_env = None
+    if args.enable_local_env:
+        print(
+            "⚠️  WARNING: Local coding environment enabled. This will execute arbitrary code locally!"
+        )
+        local_env = LocalEnv()
+
     grounding_agent = OSWorldACI(
-        env=None,
+        env=local_env,
         platform=current_platform,
         engine_params_for_generation=engine_params,
         engine_params_for_grounding=engine_params_for_grounding,

gui_agents/s3/utils/local_env.py

@@ -0,0 +1,77 @@
+import subprocess
+import sys
+from typing import Dict
+
+
+class LocalController:
+    """Minimal controller to execute bash and python code locally.
+
+    WARNING: Executing arbitrary code is dangerous. Only enable/use this in trusted
+    environments and with trusted inputs.
+    """
+
+    def run_bash_script(self, code: str, timeout: int = 30) -> Dict:
+        try:
+            proc = subprocess.run(
+                ["/bin/bash", "-lc", code],
+                capture_output=True,
+                text=True,
+                timeout=timeout,
+            )
+            output = (proc.stdout or "") + (proc.stderr or "")
+
+            print("BASH OUTPUT =======================================")
+            print(output)
+            print("BASH OUTPUT =======================================")
+
+            return {
+                "status": "ok" if proc.returncode == 0 else "error",
+                "returncode": proc.returncode,
+                "output": output,
+                "error": "",
+            }
+        except subprocess.TimeoutExpired as e:
+            return {
+                "status": "error",
+                "returncode": -1,
+                "output": e.stdout or "",
+                "error": f"TimeoutExpired: {str(e)}",
+            }
+        except Exception as e:
+            return {
+                "status": "error",
+                "returncode": -1,
+                "output": "",
+                "error": str(e),
+            }
+
+    def run_python_script(self, code: str) -> Dict:
+        try:
+            proc = subprocess.run(
+                [sys.executable, "-c", code],
+                capture_output=True,
+                text=True,
+            )
+            print("PYTHON OUTPUT =======================================")
+            print(proc.stdout or "")
+            print("PYTHON OUTPUT =======================================")
+            return {
+                "status": "ok" if proc.returncode == 0 else "error",
+                "return_code": proc.returncode,
+                "output": proc.stdout or "",
+                "error": proc.stderr or "",
+            }
+        except Exception as e:
+            return {
+                "status": "error",
+                "return_code": -1,
+                "output": "",
+                "error": str(e),
+            }
+
+
+class LocalEnv:
+    """Simple environment that provides a controller compatible with CodeAgent."""
+
+    def __init__(self):
+        self.controller = LocalController()