reorganize folder structure

gui_agents/s1/aci/LinuxOSACI.py

@@ -7,8 +7,8 @@ from typing import Dict, List, Optional, Tuple, Any, Sequence
 import numpy as np
 import requests
 
-from gui_agents.v1.aci.ACI import ACI
-from gui_agents.v1.utils.common_utils import box_iou
+from gui_agents.s1.aci.ACI import ACI
+from gui_agents.s1.utils.common_utils import box_iou
 
 import platform
 

gui_agents/s1/aci/MacOSACI.py

@@ -5,7 +5,7 @@ from typing import Any, Dict, List, Tuple
 import numpy as np
 import requests
 import platform
-from gui_agents.v1.utils.common_utils import box_iou
+from gui_agents.s1.utils.common_utils import box_iou
 
 if platform.system() == "Darwin":
     from AppKit import *
@@ -15,7 +15,7 @@ if platform.system() == "Darwin":
         AXUIElementCreateSystemWide,
     )
 
-from gui_agents.v1.aci.ACI import ACI, agent_action
+from gui_agents.s1.aci.ACI import ACI, agent_action
 
 
 def _normalize_key(key: str) -> str:

gui_agents/s1/aci/WindowsOSACI.py

@@ -6,7 +6,7 @@ from typing import Any, Dict, List, Tuple
 import numpy as np
 import psutil
 import requests
-from gui_agents.v1.utils.common_utils import box_iou
+from gui_agents.s1.utils.common_utils import box_iou
 
 if platform.system() == "Windows":
     import pywinauto
@@ -14,7 +14,7 @@ if platform.system() == "Windows":
     import win32gui
     import win32process
 
-from gui_agents.v1.aci.ACI import ACI, agent_action
+from gui_agents.s1.aci.ACI import ACI, agent_action
 
 
 # Helper functions

gui_agents/s1/aci/windowsagentarena/GroundingAgent.py

@@ -6,7 +6,7 @@ import xml.etree.ElementTree as ET
 from typing import Dict, List, Tuple
 import numpy as np
 import requests
-from gui_agents.v1.utils.common_utils import box_iou
+from gui_agents.s1.utils.common_utils import box_iou
 
 logger = logging.getLogger("desktopenv.agent")
 

gui_agents/s1/cli_app.py

@@ -11,17 +11,17 @@ import pyautogui
 
 if platform.system() == "Darwin":
     current_platform = "macos"
-    from gui_agents.v1.aci.MacOSACI import MacOSACI, UIElement
+    from gui_agents.s1.aci.MacOSACI import MacOSACI, UIElement
 elif platform.system() == "Windows":
     current_platform = "windows"
-    from gui_agents.v1.aci.WindowsOSACI import WindowsACI, UIElement
+    from gui_agents.s1.aci.WindowsOSACI import WindowsACI, UIElement
 elif platform.system() == "Linux":
     current_platform = "ubuntu"
-    from gui_agents.v1.aci.LinuxOSACI import LinuxACI, UIElement
+    from gui_agents.s1.aci.LinuxOSACI import LinuxACI, UIElement
 else:
     raise ValueError("Unsupported platform")
 
-from gui_agents.v1.core.AgentS import GraphSearchAgent, UIAgent
+from gui_agents.s1.core.AgentS import GraphSearchAgent, UIAgent
 
 logger = logging.getLogger()
 logger.setLevel(logging.DEBUG)

gui_agents/s1/core/AgentS.py

@@ -6,10 +6,10 @@ import time
 from dataclasses import dataclass
 from typing import Any, Dict, List, Optional, Tuple
 
-from gui_agents.v1.aci.ACI import ACI
-from gui_agents.v1.core.Manager import Manager
-from gui_agents.v1.core.Worker import Worker
-from gui_agents.v1.utils.common_utils import Node
+from gui_agents.s1.aci.ACI import ACI
+from gui_agents.s1.core.Manager import Manager
+from gui_agents.s1.core.Worker import Worker
+from gui_agents.s1.utils.common_utils import Node
 
 logger = logging.getLogger("desktopenv.agent")
 working_dir = os.path.dirname(os.path.abspath(__file__))

gui_agents/s1/core/BaseModule.py

@@ -1,6 +1,6 @@
 from typing import Dict, Optional
 
-from gui_agents.v1.mllm.MultimodalAgent import LMMAgent
+from gui_agents.s1.mllm.MultimodalAgent import LMMAgent
 
 
 class BaseModule:

gui_agents/s1/core/Knowledge.py

@@ -5,15 +5,15 @@ from typing import Dict, Tuple
 import numpy as np
 from sklearn.metrics.pairwise import cosine_similarity
 
-from gui_agents.v1.core.BaseModule import BaseModule
-from gui_agents.v1.core.ProceduralMemory import PROCEDURAL_MEMORY
-from gui_agents.v1.mllm.MultimodalEngine import OpenAIEmbeddingEngine
-from gui_agents.v1.utils.common_utils import (
+from gui_agents.s1.core.BaseModule import BaseModule
+from gui_agents.s1.core.ProceduralMemory import PROCEDURAL_MEMORY
+from gui_agents.s1.mllm.MultimodalEngine import OpenAIEmbeddingEngine
+from gui_agents.s1.utils.common_utils import (
     load_embeddings,
     load_knowledge_base,
     save_embeddings,
 )
-from gui_agents.v1.utils.query_perplexica import query_to_perplexica
+from gui_agents.s1.utils.query_perplexica import query_to_perplexica
 
 working_dir = os.path.dirname(os.path.abspath(__file__))
 

gui_agents/s1/core/Manager.py

@@ -3,11 +3,11 @@ import os
 from collections import defaultdict
 from typing import Dict, List, Optional, Tuple
 
-from gui_agents.v1.aci.ACI import ACI
-from gui_agents.v1.core.BaseModule import BaseModule
-from gui_agents.v1.core.Knowledge import KnowledgeBase
-from gui_agents.v1.core.ProceduralMemory import PROCEDURAL_MEMORY
-from gui_agents.v1.utils.common_utils import (
+from gui_agents.s1.aci.ACI import ACI
+from gui_agents.s1.core.BaseModule import BaseModule
+from gui_agents.s1.core.Knowledge import KnowledgeBase
+from gui_agents.s1.core.ProceduralMemory import PROCEDURAL_MEMORY
+from gui_agents.s1.utils.common_utils import (
     Dag,
     Node,
     calculate_tokens,

gui_agents/s1/core/Worker.py

@@ -4,13 +4,13 @@ import os
 import re
 from typing import Dict, List, Tuple
 
-from gui_agents.v1.aci.ACI import ACI
-from gui_agents.v1.core.BaseModule import BaseModule
-from gui_agents.v1.core.Knowledge import KnowledgeBase
-from gui_agents.v1.core.ProceduralMemory import PROCEDURAL_MEMORY
-from gui_agents.v1.mllm.MultimodalEngine import OpenAIEmbeddingEngine
-from gui_agents.v1.utils import common_utils
-from gui_agents.v1.utils.common_utils import Node, calculate_tokens, call_llm_safe
+from gui_agents.s1.aci.ACI import ACI
+from gui_agents.s1.core.BaseModule import BaseModule
+from gui_agents.s1.core.Knowledge import KnowledgeBase
+from gui_agents.s1.core.ProceduralMemory import PROCEDURAL_MEMORY
+from gui_agents.s1.mllm.MultimodalEngine import OpenAIEmbeddingEngine
+from gui_agents.s1.utils import common_utils
+from gui_agents.s1.utils.common_utils import Node, calculate_tokens, call_llm_safe
 
 logger = logging.getLogger("desktopenv.agent")
 working_dir = os.path.dirname(os.path.abspath(__file__))

gui_agents/s1/mllm/MultimodalAgent.py

@@ -5,7 +5,7 @@
 import base64
 import re
 
-from gui_agents.v1.mllm.MultimodalEngine import (
+from gui_agents.s1.mllm.MultimodalEngine import (
     LMMEngineAnthropic,
     LMMEngineAzureOpenAI,
     LMMEngineOpenAI,

gui_agents/s2/agents/agent_s.py

@@ -4,10 +4,10 @@ import os
 import shutil
 from typing import Any, Dict, List, Optional, Tuple
 
-from gui_agents.v2.core.grounding import ACI
-from gui_agents.v2.core.worker import Worker
-from gui_agents.v2.core.manager import Manager
-from gui_agents.v2.utils.common_utils import Node
+from gui_agents.s2.agents.grounding import ACI
+from gui_agents.s2.agents.worker import Worker
+from gui_agents.s2.agents.manager import Manager
+from gui_agents.s2.utils.common_utils import Node
 
 logger = logging.getLogger("desktopenv.agent")
 working_dir = os.path.dirname(os.path.abspath(__file__))
@@ -181,7 +181,7 @@ class GraphSearchAgent(UIAgent):
         self.step_count = 0
 
     def predict(
-        self, instruction: str, observation: Dict, info: Dict[str, Any]
+        self, instruction: str, observation: Dict
     ) -> Tuple[Dict, List[str]]:
         # Initialize the three info dictionaries
         planner_info = {}

gui_agents/s2/agents/grounding.py

@@ -15,9 +15,9 @@ from openai import OpenAI
 from PIL import Image
 from pytesseract import Output
 
-from gui_agents.v2.core.procedural_memory import PROCEDURAL_MEMORY
-from gui_agents.v2.mllm.mllm import LMMAgent
-from gui_agents.v2.utils.common_utils import (
+from gui_agents.s2.memory.procedural_memory import PROCEDURAL_MEMORY
+from gui_agents.s2.core.mllm import LMMAgent
+from gui_agents.s2.utils.common_utils import (
     call_llm_safe,
     parse_single_code_from_string,
 )

gui_agents/s2/agents/manager.py

@@ -3,11 +3,11 @@ import re
 from collections import defaultdict
 from typing import Dict, List, Optional, Tuple
 
-from gui_agents.v2.core.grounding import ACI
-from gui_agents.v2.core.base_module import BaseModule
-from gui_agents.v2.core.knowledge import KnowledgeBase
-from gui_agents.v2.core.procedural_memory import PROCEDURAL_MEMORY
-from gui_agents.v2.utils.common_utils import (
+from gui_agents.s2.agents.grounding import ACI
+from gui_agents.s2.core.module import BaseModule
+from gui_agents.s2.core.knowledge import KnowledgeBase
+from gui_agents.s2.memory.procedural_memory import PROCEDURAL_MEMORY
+from gui_agents.s2.utils.common_utils import (
     Dag,
     Node,
     calculate_tokens,

gui_agents/s2/agents/worker.py

@@ -3,12 +3,12 @@ import re
 import textwrap
 from typing import Dict, List, Tuple
 
-from gui_agents.v2.core.grounding import ACI
-from gui_agents.v2.core.base_module import BaseModule
-from gui_agents.v2.core.knowledge import KnowledgeBase
-from gui_agents.v2.core.procedural_memory import PROCEDURAL_MEMORY
-from gui_agents.v2.utils import common_utils
-from gui_agents.v2.utils.common_utils import Node, calculate_tokens, call_llm_safe
+from gui_agents.s2.agents.grounding import ACI
+from gui_agents.s2.core.module import BaseModule
+from gui_agents.s2.core.knowledge import KnowledgeBase
+from gui_agents.s2.memory.procedural_memory import PROCEDURAL_MEMORY
+from gui_agents.s2.utils import common_utils
+from gui_agents.s2.utils.common_utils import Node, calculate_tokens, call_llm_safe
 
 logger = logging.getLogger("desktopenv.agent")
 

gui_agents/s2/cli_app.py

@@ -18,8 +18,8 @@ elif platform.system() == "Windows":
 else:
     raise ValueError("Unsupported platform")
 
-from gui_agents.v2.core.grounding import OSWorldACI
-from gui_agents.v2.core.agent_s import GraphSearchAgent
+from gui_agents.s2.agents.grounding import OSWorldACI
+from gui_agents.s2.agents.agent_s import GraphSearchAgent
 
 logger = logging.getLogger()
 logger.setLevel(logging.DEBUG)
@@ -148,9 +148,24 @@ def main():
         default="gpt-4o-mini",
         help="Specify the model to use (e.g., gpt-4o)",
     )
+    parser.add_argument(
+        "--endpoint_provider",
+        type=str,
+        default="huggingface",
+        help="Specify the endpoint provider (e.g., huggingface)",
+    )
+    parser.add_argument(
+        "--endpoint_url",
+        type=str,
+        default="",
+        help="Specify the endpoint URL to your HuggingFace Inference Endpoint.",
+    )
     args = parser.parse_args()
 
-    grounding_agent = OSWorldACI()
+    grounding_agent = OSWorldACI(
+        endpoint_provider=args.endpoint_provider,
+        endpoint_url=args.endpoint_url
+    )
 
     while True:
         query = input("Query: ")

gui_agents/s2/core/knowledge.py

@@ -5,16 +5,16 @@ from typing import Dict, Tuple
 import numpy as np
 from sklearn.metrics.pairwise import cosine_similarity
 
-from gui_agents.v2.core.base_module import BaseModule
-from gui_agents.v2.core.procedural_memory import PROCEDURAL_MEMORY
-from gui_agents.v2.mllm.engine import OpenAIEmbeddingEngine
-from gui_agents.v2.utils.common_utils import (
+from gui_agents.s2.core.module import BaseModule
+from gui_agents.s2.memory.procedural_memory import PROCEDURAL_MEMORY
+from gui_agents.s2.core.engine import OpenAIEmbeddingEngine
+from gui_agents.s2.utils.common_utils import (
     call_llm_safe,
     load_embeddings,
     load_knowledge_base,
     save_embeddings,
 )
-from gui_agents.v2.utils.query_perplexica import query_to_perplexica
+from gui_agents.s2.utils.query_perplexica import query_to_perplexica
 
 
 class KnowledgeBase(BaseModule):

gui_agents/s2/core/mllm.py

@@ -2,7 +2,7 @@ import base64
 
 import numpy as np
 
-from gui_agents.v2.mllm.engine import (
+from gui_agents.s2.core.engine import (
     LMMEngineAnthropic,
     LMMEngineAzureOpenAI,
     LMMEngineOpenAI,

gui_agents/s2/core/module.py

@@ -1,5 +1,5 @@
 from typing import Dict, Optional
-from gui_agents.v2.mllm.mllm import LMMAgent
+from gui_agents.s2.core.mllm import LMMAgent
 
 
 class BaseModule:

server.py

@@ -4,7 +4,7 @@ import platform
 from fastapi import FastAPI, HTTPException
 from fastapi.responses import StreamingResponse
 from pydantic import BaseModel
-from gui_agents.v1.core.AgentS import GraphSearchAgent
+from gui_agents.s1.core.AgentS import GraphSearchAgent
 import io
 import pyautogui
 import time
@@ -13,15 +13,15 @@ from threading import Event, Lock
 # Determine the operating system and select appropriate ACI
 os_name = platform.system().lower()
 if os_name == "linux":
-    from gui_agents.v1.aci.LinuxOSACI import LinuxACI, UIElement
+    from gui_agents.s1.aci.LinuxOSACI import LinuxACI, UIElement
     grounding_agent = LinuxACI()
     platform_name = "ubuntu"
 elif os_name == "darwin":
-    from gui_agents.v1.aci.MacOSACI import MacOSACI, UIElement
+    from gui_agents.s1.aci.MacOSACI import MacOSACI, UIElement
     grounding_agent = MacOSACI()
     platform_name = "macos"
 elif os_name == "windows":
-    from gui_agents.v1.aci.WindowsOSACI import WindowsACI, UIElement
+    from gui_agents.s1.aci.WindowsOSACI import WindowsACI, UIElement
     grounding_agent = WindowsACI()
     platform_name = "windows"
 else:

tests/test_aci.py

@@ -2,7 +2,7 @@ from unittest.mock import Mock, patch
 
 import pytest
 
-from gui_agents.v1.aci.ACI import ACI, _normalize_key
+from gui_agents.s1.aci.ACI import ACI, _normalize_key
 
 
 @pytest.fixture

tests/test_app_switching.py

@@ -3,7 +3,7 @@ import time
 import pyautogui
 from AppKit import NSWorkspace
 
-from gui_agents.v1.aci.MacOSACI import MacOSACI
+from gui_agents.s1.aci.MacOSACI import MacOSACI
 
 agent = MacOSACI()
 

tests/test_uielement_macos.py

@@ -3,7 +3,7 @@ from unittest.mock import Mock, patch
 
 import pytest
 
-from gui_agents.v1.aci.MacOSACI import UIElement
+from gui_agents.s1.aci.MacOSACI import UIElement
 
 
 @pytest.fixture