From b48f448f30013ba7e28128d479bd27c7c13905f7 Mon Sep 17 00:00:00 2001 From: kylesimular Date: Wed, 12 Mar 2025 11:33:08 -0700 Subject: [PATCH] fixed image rescaling for claude-3.5 --- gui_agents/s2/agents/grounding.py | 26 ++++++++++++++--------- gui_agents/s2/agents/manager.py | 2 +- gui_agents/s2/cli_app.py | 34 +++++++++++++++++-------------- 3 files changed, 36 insertions(+), 26 deletions(-) diff --git a/gui_agents/s2/agents/grounding.py b/gui_agents/s2/agents/grounding.py index 3d323d0..f3db77d 100644 --- a/gui_agents/s2/agents/grounding.py +++ b/gui_agents/s2/agents/grounding.py @@ -168,8 +168,8 @@ class OSWorldACI(ACI): platform: str, engine_params_for_generation: Dict, engine_params_for_grounding: Dict, - height: int = 1080, width: int = 1920, + height: int = 1080, ): self.platform = ( platform # Dictates how the switch_applications agent action works. @@ -188,10 +188,10 @@ class OSWorldACI(ACI): # Configure the visual grounding model responsible for coordinate generation self.grounding_model = LMMAgent(engine_params_for_grounding) + self.engine_params_for_grounding = engine_params_for_grounding # Configure text grounding agent self.text_span_agent = LMMAgent( - # Swap out with your desired engine type engine_params=engine_params_for_generation, system_prompt=PROCEDURAL_MEMORY.PHRASE_TO_WORD_COORDS_PROMPT, ) @@ -317,26 +317,32 @@ class OSWorldACI(ACI): and len(args) >= 1 and args[0] != None ): - print(f"DESCRIPTION 1: {args[0]}") self.coords1 = self.generate_coords(args[0], obs) # arg0 and arg1 are descriptions elif function_name == "agent.drag_and_drop" and len(args) >= 2: - print(f"DESCRIPTION 1: {args[0]}") - print(f"DESCRIPTION 2: {args[1]}") self.coords1 = self.generate_coords(args[0], obs) self.coords2 = self.generate_coords(args[1], obs) # arg0 and arg1 are text phrases elif function_name == "agent.highlight_text_span" and len(args) >= 2: - print(f"PHRASE 1: {args[0]}") - print(f"PHRASE 2: {args[1]}") self.coords1 = self.generate_text_coords(args[0], obs, alignment="start") self.coords2 = self.generate_text_coords(args[1], obs, alignment="end") - # Resize from HF model dim (1000 x 1000) into OSWorld dim (1920 * 1080) + # Resize from grounding model dim into OSWorld dim (1920 * 1080) def resize_coordinates(self, coordinates: List[int]) -> List[int]: + # User explicitly passes the grounding model dimensions + if {"grounding_width", "grounding_height"}.issubset( + self.engine_params_for_grounding + ): + grounding_width = self.engine_params_for_grounding["grounding_width"] + grounding_height = self.engine_params_for_grounding["grounding_height"] + # Default to (1000, 1000), which is UI-TARS resizing + else: + grounding_width = 1000 + grounding_height = 1000 + return [ - round(coordinates[0] * self.width / 1000), - round(coordinates[1] * self.height / 1000), + round(coordinates[0] * self.width / grounding_width), + round(coordinates[1] * self.height / grounding_height), ] # Given a generated ACI function, returns a list of argument values, where descriptions are at the front of the list diff --git a/gui_agents/s2/agents/manager.py b/gui_agents/s2/agents/manager.py index 640f422..d9d124d 100644 --- a/gui_agents/s2/agents/manager.py +++ b/gui_agents/s2/agents/manager.py @@ -182,7 +182,7 @@ class Manager(BaseModule): else: generator_message = "Please generate the initial plan for the task.\n" - print("GENERATOR MESSAGE:\n", generator_message) + logger.info("GENERATOR MESSAGE: %s", generator_message) self.generator_agent.add_message( generator_message, diff --git a/gui_agents/s2/cli_app.py b/gui_agents/s2/cli_app.py index cfa23fa..cda1d94 100644 --- a/gui_agents/s2/cli_app.py +++ b/gui_agents/s2/cli_app.py @@ -80,23 +80,21 @@ def show_permission_dialog(code: str, action_description: str): return False -def scale_screen_dimensions(width: int, height: int): - MAX_DIMENSION_SIZE = 2400 - scale_factor = min(MAX_DIMENSION_SIZE / width, MAX_DIMENSION_SIZE / height, 1) +def scale_screen_dimensions(width: int, height: int, max_dim_size: int): + scale_factor = min(max_dim_size / width, max_dim_size / height, 1) safe_width = int(width * scale_factor) safe_height = int(height * scale_factor) return safe_width, safe_height -def run_agent(agent, instruction: str, safe_width: int, safe_height: int): +def run_agent(agent, instruction: str, scaled_width: int, scaled_height: int): obs = {} traj = "Task:\n" + instruction subtask_traj = "" for _ in range(15): - # Get screen shot using pyautogui. - # Take a screenshot + # Get screen shot using pyautogui screenshot = pyautogui.screenshot() - screenshot = screenshot.resize((safe_width, safe_height), Image.LANCZOS) + screenshot = screenshot.resize((scaled_width, scaled_height), Image.LANCZOS) # Save the screenshot to a BytesIO object buffered = io.BytesIO() @@ -164,7 +162,7 @@ def main(): "--grounding_model", type=str, default="", - help="Specify the grounding model to use (e.g., claude-3-7-sonnet)", + help="Specify the grounding model to use (e.g., claude-3-5-sonnet-20241022)", ) # Grounding model config option 2: Self-hosted endpoint based @@ -186,6 +184,12 @@ def main(): args.grounding_model or args.endpoint_url ), "Error: No grounding model was provided. Either provide an API based model, or a self-hosted HuggingFace endpoint" + # Re-scales screenshot size to ensure it fits in UI-TARS context limit + screen_width, screen_height = pyautogui.size() + scaled_width, scaled_height = scale_screen_dimensions( + screen_width, screen_height, max_dim_size=2400 + ) + # Load the general engine params if args.model.startswith("claude"): engine_params = {"engine_type": "anthropic", "model": args.model} @@ -203,30 +207,30 @@ def main(): "endpoint_url": args.endpoint_url, } elif args.grounding_model.startswith("claude"): + CLAUDE_3_5_MAX_WIDTH = 1366 engine_params_for_grounding = { "engine_type": "anthropic", "model": args.grounding_model, + "grounding_width": CLAUDE_3_5_MAX_WIDTH, + "grounding_height": screen_height * CLAUDE_3_5_MAX_WIDTH / screen_width, } elif args.grounding_model.startswith("gpt"): engine_params_for_grounding = { "engine_type": "openai", "model": args.grounding_model, + # TODO: set your image scaling for gpt here } else: raise ValueError( "Invalid grounding model specficiation. Please provide a supported model type" ) - # Re-scales screenshot size to ensure it fits in UI-TARS context limit - screen_width, screen_height = pyautogui.size() - safe_width, safe_height = scale_screen_dimensions(screen_width, screen_height) - grounding_agent = OSWorldACI( platform=current_platform, engine_params_for_generation=engine_params, engine_params_for_grounding=engine_params_for_grounding, - width=safe_width, - height=safe_height, + width=screen_width, + height=screen_height, ) agent = GraphSearchAgent( @@ -245,7 +249,7 @@ def main(): agent.reset() # Run the agent on your own device - run_agent(agent, query, safe_width, safe_height) + run_agent(agent, query, scaled_width, scaled_height) response = input("Would you like to provide another query? (y/n): ") if response.lower() != "y":