refactor(macrocli): replace Gemini with generic OpenAI-compatible LLM backend

- Rename gemini_assist.py → llm_assist.py, rewrite to use OpenAI SDK
- Rewrite parameterize.py's gemini_suggest_parameters → llm_suggest_parameters
  (old name kept as alias for backwards compat)
- Update macrocli_cli.py: all Gemini references → generic LLM, env vars
  MACROCLI_MODEL / MACROCLI_API_KEY / MACROCLI_BASE_URL
- Update recorder.py UI text
- Add .gitignore rules to exclude non-gedit demo files from tracking
This commit is contained in:
haorui-harry
2026-05-03 00:41:31 +08:00
parent fb767afe0a
commit 0f301dbf4f
5 changed files with 163 additions and 113 deletions
+5
View File
@@ -226,6 +226,11 @@
!/lldb/agent-harness/
!/macrocli/agent-harness/
# Exclude non-gedit demo macros from macrocli (local only)
/macrocli/agent-harness/cli_anything/macrocli/macro_definitions/demo/flameshot*
/macrocli/agent-harness/cli_anything/macrocli/macro_definitions/demo/kolourpaint*
/macrocli/agent-harness/cli_anything/macrocli/macro_definitions/demo/snapshots/
# Step 7: Ignore build artifacts within allowed dirs
**/__pycache__/
**/*.egg-info/
@@ -1,15 +1,23 @@
"""GeminiAssist — use Gemini Vision to generate macro steps from screenshots.
"""LLMAssist — use a vision model to generate macro steps from screenshots.
This module is OPTIONAL. It requires:
pip install google-generativeai
pip install openai mss Pillow
Uses the OpenAI SDK, which is compatible with any OpenAI-compatible API
provider (OpenAI, Azure, local vLLM, Ollama, LiteLLM, etc.).
Configure via environment variables:
MACROCLI_MODEL model name (required)
MACROCLI_API_KEY API key
MACROCLI_BASE_URL base URL (only needed for non-OpenAI hosts)
How it works:
1. Capture a screenshot of the current screen (or use a provided image)
2. Send the image + user goal to Gemini Vision with a strict system prompt
3. Gemini returns a JSON array of steps (constrained action space)
2. Send the image + user goal to the vision model with a strict system prompt
3. The model returns a JSON array of steps (constrained action space)
4. Steps are validated and written as a macro YAML file
The action space Gemini is allowed to produce:
The action space the model is allowed to produce:
{"type": "click_image", "description": "...", "confidence": 0.85}
{"type": "click_relative", "window_title": "...", "x_pct": 0.5, "y_pct": 0.1}
@@ -20,7 +28,7 @@ The action space Gemini is allowed to produce:
{"type": "menu_click", "app_name": "...", "menu_path": ["File", "Export"]}
{"type": "scroll", "description": "...", "dy": -3}
Gemini is NOT allowed to:
The model is NOT allowed to:
- Produce shell commands, Python code, or arbitrary actions
- Use absolute pixel coordinates
- Output anything other than the JSON array
@@ -29,11 +37,10 @@ The "description" field in click_image / wait_image / scroll tells the user
what template image to capture with 'macro record' or 'capture_region'.
Usage:
cli-anything-macrocli macro define my_export --assist \
--goal "Export the current diagram as PNG to /tmp/out.png" \
cli-anything-macrocli macro define my_export --assist \\
--goal "Export the current diagram as PNG to /tmp/out.png" \\
--screenshot current # takes a fresh screenshot
--screenshot /path/to/img.png # use existing image
--api-key $GEMINI_API_KEY
"""
from __future__ import annotations
@@ -144,7 +151,7 @@ _REQUIRED_FIELDS = {
def _validate_steps(raw_steps: list) -> tuple[list[dict], list[str]]:
"""Validate and sanitize steps from Gemini output.
"""Validate and sanitize steps from model output.
Returns (valid_steps, error_messages).
"""
@@ -182,8 +189,8 @@ def _validate_steps(raw_steps: list) -> tuple[list[dict], list[str]]:
# ── Step → YAML step dict conversion ─────────────────────────────────────────
def _gemini_step_to_yaml_step(step: dict, index: int) -> dict:
"""Convert a validated Gemini step to a macro YAML step dict."""
def _step_to_yaml_step(step: dict, index: int) -> dict:
"""Convert a validated model step to a macro YAML step dict."""
stype = step["type"]
sid = f"step_{index:03d}_{stype}"
@@ -199,7 +206,7 @@ def _gemini_step_to_yaml_step(step: dict, index: int) -> dict:
"_template_description": step.get("description", ""),
},
"on_failure": "fail",
"_gemini_description": step.get("description", ""),
"_model_description": step.get("description", ""),
}
elif stype == "click_relative":
return {
@@ -241,7 +248,7 @@ def _gemini_step_to_yaml_step(step: dict, index: int) -> dict:
"_template_description": step.get("description", ""),
},
"on_failure": "fail",
"_gemini_description": step.get("description", ""),
"_model_description": step.get("description", ""),
}
elif stype == "wait_for_window":
return {
@@ -289,41 +296,55 @@ def generate_macro(
macro_name: str,
screenshot_source: str = "current", # "current" | path to image file
api_key: Optional[str] = None,
model: str = "gemini-1.5-flash",
model: Optional[str] = None,
base_url: Optional[str] = None,
output_path: Optional[str] = None,
) -> dict:
"""Generate a macro YAML from a user goal and screenshot using Gemini.
"""Generate a macro YAML from a user goal and screenshot using a vision model.
Args:
goal: Natural language description of what the macro should do.
macro_name: Name for the generated macro.
screenshot_source: "current" to take a fresh screenshot, or a
file path to use an existing image.
api_key: Gemini API key. Falls back to GEMINI_API_KEY env var.
model: Gemini model to use.
api_key: API key. Falls back to MACROCLI_API_KEY env var.
model: Model name. Falls back to MACROCLI_MODEL env var.
base_url: Base URL for non-OpenAI providers. Falls back to
MACROCLI_BASE_URL env var.
output_path: Where to write the YAML file. Defaults to
<macro_name>.yaml in the current directory.
Returns:
dict with keys: yaml_path, steps_count, warnings, raw_steps
"""
import base64
try:
import google.generativeai as genai
from openai import OpenAI
except ImportError:
raise ImportError(
"google-generativeai is required for Gemini assist.\n"
" pip install google-generativeai"
"openai is required for LLM assist.\n"
" pip install openai"
)
# Resolve API key
key = api_key or os.environ.get("GEMINI_API_KEY", "")
# Resolve config
resolved_model = model or os.environ.get("MACROCLI_MODEL", "")
key = api_key or os.environ.get("MACROCLI_API_KEY", "")
resolved_base_url = base_url or os.environ.get("MACROCLI_BASE_URL", "")
if not resolved_model:
raise ValueError(
"Model required. Pass --model or set MACROCLI_MODEL env var."
)
if not key:
raise ValueError(
"Gemini API key required. Pass --api-key or set GEMINI_API_KEY env var.\n"
"Get a key at: https://aistudio.google.com/app/apikey"
"API key required. Pass --api-key or set MACROCLI_API_KEY env var."
)
genai.configure(api_key=key)
client_kwargs = {"api_key": key}
if resolved_base_url:
client_kwargs["base_url"] = resolved_base_url
client = OpenAI(**client_kwargs)
# Get screenshot
if screenshot_source == "current":
@@ -333,25 +354,27 @@ def generate_macro(
raise FileNotFoundError(f"Screenshot not found: {screenshot_source}")
image_bytes = _load_image_bytes(screenshot_source)
image_b64 = base64.b64encode(image_bytes).decode("utf-8")
# Build prompt
user_prompt = (
f"Goal: {goal}\n\n"
"Generate the minimal sequence of steps to achieve this goal. "
"Output ONLY the JSON array, nothing else."
user_content = [
{"type": "text", "text": (
f"Goal: {goal}\n\n"
"Generate the minimal sequence of steps to achieve this goal. "
"Output ONLY the JSON array, nothing else."
)},
{"type": "image_url", "image_url": {"url": f"data:image/png;base64,{image_b64}"}},
]
response = client.chat.completions.create(
model=resolved_model,
messages=[
{"role": "system", "content": _SYSTEM_PROMPT},
{"role": "user", "content": user_content},
],
max_tokens=2048,
)
# Call Gemini
gemini_model = genai.GenerativeModel(
model_name=model,
system_instruction=_SYSTEM_PROMPT,
)
import PIL.Image
import io
img = PIL.Image.open(io.BytesIO(image_bytes))
response = gemini_model.generate_content([user_prompt, img])
raw_text = response.text.strip()
raw_text = response.choices[0].message.content.strip()
# Strip markdown code fences if model added them despite instructions
if raw_text.startswith("```"):
@@ -366,13 +389,13 @@ def generate_macro(
raw_steps = json.loads(raw_text)
except json.JSONDecodeError as e:
raise ValueError(
f"Gemini returned invalid JSON: {e}\n"
f"Model returned invalid JSON: {e}\n"
f"Raw response (first 500 chars):\n{raw_text[:500]}"
)
if not isinstance(raw_steps, list):
raise ValueError(
f"Gemini returned non-array JSON (expected list): {type(raw_steps)}"
f"Model returned non-array JSON (expected list): {type(raw_steps)}"
)
# Validate
@@ -380,7 +403,7 @@ def generate_macro(
# Convert to YAML step dicts
yaml_steps = [
_gemini_step_to_yaml_step(s, i + 1)
_step_to_yaml_step(s, i + 1)
for i, s in enumerate(valid_steps)
]
@@ -389,7 +412,7 @@ def generate_macro(
"name": macro_name,
"version": "1.0",
"description": goal,
"tags": ["generated", "gemini-assist"],
"tags": ["generated", "llm-assist"],
"parameters": {},
"preconditions": [],
"steps": yaml_steps,
@@ -399,8 +422,8 @@ def generate_macro(
"danger_level": "moderate",
"side_effects": ["gui_interaction"],
"reversible": False,
"generated_by": "gemini-assist",
"model": model,
"generated_by": "llm-assist",
"model": resolved_model,
},
}
@@ -409,10 +432,10 @@ def generate_macro(
{
"step_id": s["id"],
"template_path": s["params"].get("template", ""),
"description": s.get("_gemini_description", ""),
"description": s.get("_model_description", ""),
}
for s in yaml_steps
if s.get("params", {}).get("template") and s.get("_gemini_description")
if s.get("params", {}).get("template") and s.get("_model_description")
]
if templates_needed:
@@ -1,4 +1,4 @@
"""Parameterization helpers — interactive and Gemini-assisted.
"""Parameterization helpers — interactive and LLM-assisted.
Interactive flow (no external deps):
assignments = interactive_parameterize(type_steps)
@@ -8,8 +8,8 @@ Interactive flow (no external deps):
Post-hoc flow on an existing YAML file:
parameterize_yaml_file(yaml_path) # modifies in-place
Gemini-assisted flow (optional, requires google-generativeai):
assignments = gemini_suggest_parameters(type_steps, api_key=...)
LLM-assisted flow (optional, requires openai):
assignments = llm_suggest_parameters(type_steps, api_key=...)
# returns same shape as interactive_parameterize, can be passed directly
"""
@@ -202,20 +202,23 @@ def parameterize_yaml_file(yaml_path: str) -> bool:
return True
# ── Gemini-assisted parameterization ─────────────────────────────────────────
# ── LLM-assisted parameterization ─────────────────────────────────────────
def gemini_suggest_parameters(
def llm_suggest_parameters(
type_steps: list[tuple[int, object]],
api_key: Optional[str] = None,
model: str = "gemini-1.5-flash",
model: Optional[str] = None,
base_url: Optional[str] = None,
) -> dict[int, str]:
"""Use Gemini to suggest which type_text steps should be parameterized
"""Use a vision model to suggest which type_text steps should be parameterized
and what to name the parameters.
Args:
type_steps: Same format as interactive_parameterize input.
api_key: Gemini API key. Falls back to GEMINI_API_KEY env var.
model: Gemini model name.
api_key: API key. Falls back to MACROCLI_API_KEY env var.
model: Model name. Falls back to MACROCLI_MODEL env var.
base_url: Base URL for non-OpenAI providers. Falls back to
MACROCLI_BASE_URL env var.
Returns:
{list_index: suggested_param_name} — same shape as interactive output.
@@ -226,21 +229,30 @@ def gemini_suggest_parameters(
import os
try:
import google.generativeai as genai
from openai import OpenAI
except ImportError:
raise ImportError(
"google-generativeai required for auto-parameterization.\n"
" pip install google-generativeai"
"openai required for auto-parameterization.\n"
" pip install openai"
)
key = api_key or os.environ.get("GEMINI_API_KEY", "")
resolved_model = model or os.environ.get("MACROCLI_MODEL", "")
key = api_key or os.environ.get("MACROCLI_API_KEY", "")
resolved_base_url = base_url or os.environ.get("MACROCLI_BASE_URL", "")
if not resolved_model:
raise ValueError(
"Model required. Set MACROCLI_MODEL env var or pass --model."
)
if not key:
raise ValueError(
"Gemini API key required. Pass --api-key or set GEMINI_API_KEY.\n"
" https://aistudio.google.com/app/apikey"
"API key required. Pass --api-key or set MACROCLI_API_KEY."
)
genai.configure(api_key=key)
client_kwargs = {"api_key": key}
if resolved_base_url:
client_kwargs["base_url"] = resolved_base_url
client = OpenAI(**client_kwargs)
_SYSTEM = """\
You are a macro parameterization assistant. Given a list of text values
@@ -266,9 +278,15 @@ Example: {"0": "output_path", "2": "export_width"}
)
prompt = f"Typed values from the recording:\n{items}\n\nOutput JSON only."
gem = genai.GenerativeModel(model_name=model, system_instruction=_SYSTEM)
response = gem.generate_content(prompt)
raw = response.text.strip()
response = client.chat.completions.create(
model=resolved_model,
messages=[
{"role": "system", "content": _SYSTEM},
{"role": "user", "content": prompt},
],
max_tokens=1024,
)
raw = response.choices[0].message.content.strip()
# Strip markdown fences if present
if raw.startswith("```"):
@@ -281,7 +299,7 @@ Example: {"0": "output_path", "2": "export_width"}
raw_dict: dict = json.loads(raw)
except json.JSONDecodeError as e:
raise ValueError(
f"Gemini returned invalid JSON: {e}\nRaw: {raw[:300]}"
f"Model returned invalid JSON: {e}\nRaw: {raw[:300]}"
)
# Convert string keys to int, validate names
@@ -298,3 +316,7 @@ Example: {"0": "output_path", "2": "export_width"}
result[idx] = v
return result
# Keep old name as alias for backwards compatibility
gemini_suggest_parameters = llm_suggest_parameters
@@ -636,7 +636,7 @@ class MacroRecorder:
print("" * 60)
print(" Step Review — mark steps as 'fixed' or 'agent'")
print(" Enter = fixed (fast, deterministic)")
print(" a = agent step (Gemini decides at runtime)")
print(" a = agent step (vision model decides at runtime)")
print("" * 60)
for i, step in enumerate(self._steps):
@@ -440,10 +440,10 @@ def macro_define(name, output):
help="After recording, interactively choose which typed values "
"become CLI parameters.")
@click.option("--auto-parameterize", "do_auto_param", is_flag=True,
help="After recording, use Gemini to automatically suggest "
"parameter names (requires --api-key or GEMINI_API_KEY).")
@click.option("--api-key", default=None, envvar="GEMINI_API_KEY",
help="Gemini API key for --auto-parameterize.")
help="After recording, use an LLM to automatically suggest "
"parameter names (requires --api-key or MACROCLI_API_KEY).")
@click.option("--api-key", default=None, envvar="MACROCLI_API_KEY",
help="API key for --auto-parameterize.")
@handle_error
def macro_record(name, output_dir, timeout, do_agent_review,
do_parameterize, do_auto_param, api_key):
@@ -509,14 +509,14 @@ def macro_record(name, output_dir, timeout, do_agent_review,
if do_auto_param and type_steps:
try:
from cli_anything.macrocli.core.parameterize import (
gemini_suggest_parameters,
llm_suggest_parameters,
interactive_parameterize,
)
if not _json_output:
click.echo(f"\nAsking Gemini to suggest parameters...")
suggestions = gemini_suggest_parameters(type_steps, api_key=api_key)
click.echo(f"\nAsking LLM to suggest parameters...")
suggestions = llm_suggest_parameters(type_steps, api_key=api_key)
if suggestions and not _json_output:
click.echo(" Gemini suggestions:")
click.echo(" LLM suggestions:")
for idx, pname in suggestions.items():
step = recorder._steps[idx]
click.echo(f" step {idx+1} {step.text!r} → ${{{pname}}}")
@@ -527,9 +527,9 @@ def macro_record(name, output_dir, timeout, do_agent_review,
final = {**suggestions, **confirmed}
parameters = recorder.apply_parameterization(final)
elif not suggestions and not _json_output:
click.echo(" Gemini found no values to parameterize.")
click.echo(" LLM found no values to parameterize.")
except Exception as e:
click.echo(f" Warning: Gemini parameterization failed: {e}", err=True)
click.echo(f" Warning: LLM parameterization failed: {e}", err=True)
do_parameterize = True
if do_parameterize and type_steps:
@@ -568,7 +568,7 @@ def macro_record(name, output_dir, timeout, do_agent_review,
else:
click.echo(f"✓ Saved {len(recorder._steps)} steps to: {pkg_dir}/")
if agent_count:
click.echo(f" Agent steps: {agent_count} (will use Gemini at runtime)")
click.echo(f" Agent steps: {agent_count} (will use vision model at runtime)")
if parameters:
click.echo(f" Parameters: {', '.join(parameters.keys())}")
click.echo(
@@ -593,8 +593,8 @@ def macro_record(name, output_dir, timeout, do_agent_review,
# Record + interactively parameterize typed values
macro record my_export --parameterize
# Record + auto-parameterize with Gemini
macro record my_export --auto-parameterize --api-key $GEMINI_API_KEY
# Record + auto-parameterize with LLM
macro record my_export --auto-parameterize --api-key $MACROCLI_API_KEY
Requires: pip install mss Pillow pynput
"""
@@ -634,17 +634,17 @@ def macro_record(name, output_dir, timeout, do_agent_review,
if do_auto_param and type_steps:
try:
from cli_anything.macrocli.core.parameterize import (
gemini_suggest_parameters,
llm_suggest_parameters,
interactive_parameterize,
)
if not _json_output:
click.echo(f"\nAsking Gemini to suggest parameters for "
click.echo(f"\nAsking LLM to suggest parameters for "
f"{len(type_steps)} type_text step(s)...")
suggestions = gemini_suggest_parameters(
suggestions = llm_suggest_parameters(
type_steps, api_key=api_key
)
if suggestions and not _json_output:
click.echo(" Gemini suggestions:")
click.echo(" LLM suggestions:")
for idx, pname in suggestions.items():
step = recorder._steps[idx]
click.echo(f" step {idx+1} {step.text!r} → ${{{pname}}}")
@@ -654,15 +654,15 @@ def macro_record(name, output_dir, timeout, do_agent_review,
[(i, s) for i, s in type_steps if i in suggestions],
existing_params=set(),
)
# For steps Gemini suggested but user skipped, remove them
# For steps LLM suggested but user skipped, remove them
final = {i: n for i, n in suggestions.items() if i in confirmed}
# For steps user renamed, use their name
final.update(confirmed)
parameters = recorder.apply_parameterization(final)
elif not suggestions and not _json_output:
click.echo(" Gemini found no values to parameterize.")
click.echo(" LLM found no values to parameterize.")
except Exception as e:
click.echo(f" Warning: Gemini parameterization failed: {e}", err=True)
click.echo(f" Warning: LLM parameterization failed: {e}", err=True)
click.echo(" Falling back to interactive mode...")
do_parameterize = True
@@ -709,9 +709,9 @@ def macro_record(name, output_dir, timeout, do_agent_review,
@macro.command("parameterize")
@click.argument("yaml_file")
@click.option("--auto", "do_auto", is_flag=True,
help="Use Gemini to suggest parameter names automatically.")
@click.option("--api-key", default=None, envvar="GEMINI_API_KEY",
help="Gemini API key for --auto.")
help="Use an LLM to suggest parameter names automatically.")
@click.option("--api-key", default=None, envvar="MACROCLI_API_KEY",
help="API key for --auto.")
@handle_error
def macro_parameterize(yaml_file, do_auto, api_key):
"""Interactively parameterize typed values in an existing macro YAML.
@@ -723,11 +723,11 @@ def macro_parameterize(yaml_file, do_auto, api_key):
\b
Examples:
macro parameterize /tmp/recording/my_export.yaml
macro parameterize my_export.yaml --auto --api-key $GEMINI_API_KEY
macro parameterize my_export.yaml --auto --api-key $MACROCLI_API_KEY
"""
from cli_anything.macrocli.core.parameterize import (
parameterize_yaml_file,
gemini_suggest_parameters,
llm_suggest_parameters,
interactive_parameterize,
_YamlTypeStep,
)
@@ -740,7 +740,7 @@ def macro_parameterize(yaml_file, do_auto, api_key):
return
if do_auto:
# Load the file, extract type_text steps, ask Gemini, then apply
# Load the file, extract type_text steps, ask LLM, then apply
import yaml as _yaml
with open(p, encoding="utf-8") as f:
macro_dict = _yaml.safe_load(f)
@@ -759,11 +759,11 @@ def macro_parameterize(yaml_file, do_auto, api_key):
wrapped = [(i, _YamlTypeStep(i, s)) for i, s in type_steps_raw]
try:
click.echo(f"Asking Gemini to suggest parameters for "
click.echo(f"Asking LLM to suggest parameters for "
f"{len(wrapped)} type_text step(s)...")
suggestions = gemini_suggest_parameters(wrapped, api_key=api_key)
suggestions = llm_suggest_parameters(wrapped, api_key=api_key)
except Exception as e:
click.echo(f"Gemini failed: {e}\nFalling back to interactive.", err=True)
click.echo(f"LLM failed: {e}\nFalling back to interactive.", err=True)
suggestions = {}
do_auto = False
@@ -774,7 +774,7 @@ def macro_parameterize(yaml_file, do_auto, api_key):
click.echo(f" step {idx+1} {w.text!r} → ${{{pname}}}")
click.echo()
# Let user confirm (pre-fill Gemini suggestions as defaults)
# Let user confirm (pre-fill LLM suggestions as defaults)
existing = set((macro_dict.get("parameters") or {}).keys())
confirmed = interactive_parameterize(
[(i, w) for i, w in wrapped if i in suggestions],
@@ -831,35 +831,35 @@ def macro_parameterize(yaml_file, do_auto, api_key):
help="'current' to take a screenshot now, or path to an image file.")
@click.option("--output", "-o", default=None,
help="Output YAML file path (default: <name>.yaml).")
@click.option("--api-key", default=None, envvar="GEMINI_API_KEY",
help="Gemini API key (or set GEMINI_API_KEY env var).")
@click.option("--model", default="gemini-1.5-flash",
help="Gemini model name.")
@click.option("--api-key", default=None, envvar="MACROCLI_API_KEY",
help="API key (or set MACROCLI_API_KEY env var).")
@click.option("--model", default=None,
help="Model name (or set MACROCLI_MODEL env var).")
@handle_error
def macro_assist(name, goal, screenshot, output, api_key, model):
"""Generate a macro YAML from a screenshot using Gemini Vision (optional).
"""Generate a macro YAML from a screenshot using a vision model (optional).
\b
Takes a screenshot, sends it to Gemini with your goal, and generates
a macro YAML. Steps that require visual templates will include
Takes a screenshot, sends it to the configured model with your goal, and
generates a macro YAML. Steps that require visual templates will include
instructions for which template images to capture.
Requires: pip install google-generativeai mss Pillow
Requires: pip install openai mss Pillow
\b
Example:
macro assist export_png \\
--goal "Export the current diagram as PNG to /tmp/out.png" \\
--api-key $GEMINI_API_KEY
--api-key $MACROCLI_API_KEY
"""
try:
from cli_anything.macrocli.core.gemini_assist import generate_macro
from cli_anything.macrocli.core.llm_assist import generate_macro
except ImportError as e:
click.echo(f"Error: {e}", err=True)
sys.exit(1)
if not _json_output:
click.echo(f"Sending screenshot to Gemini ({model})...")
click.echo(f"Sending screenshot to model ({model or os.environ.get('MACROCLI_MODEL', 'unset')})...")
result = generate_macro(
goal=goal,