mirror of
https://github.com/HKUDS/CLI-Anything.git
synced 2026-08-28 23:27:04 +08:00
refactor(macrocli): replace Gemini with generic OpenAI-compatible LLM backend
- Rename gemini_assist.py → llm_assist.py, rewrite to use OpenAI SDK - Rewrite parameterize.py's gemini_suggest_parameters → llm_suggest_parameters (old name kept as alias for backwards compat) - Update macrocli_cli.py: all Gemini references → generic LLM, env vars MACROCLI_MODEL / MACROCLI_API_KEY / MACROCLI_BASE_URL - Update recorder.py UI text - Add .gitignore rules to exclude non-gedit demo files from tracking
This commit is contained in:
@@ -226,6 +226,11 @@
|
||||
!/lldb/agent-harness/
|
||||
!/macrocli/agent-harness/
|
||||
|
||||
# Exclude non-gedit demo macros from macrocli (local only)
|
||||
/macrocli/agent-harness/cli_anything/macrocli/macro_definitions/demo/flameshot*
|
||||
/macrocli/agent-harness/cli_anything/macrocli/macro_definitions/demo/kolourpaint*
|
||||
/macrocli/agent-harness/cli_anything/macrocli/macro_definitions/demo/snapshots/
|
||||
|
||||
# Step 7: Ignore build artifacts within allowed dirs
|
||||
**/__pycache__/
|
||||
**/*.egg-info/
|
||||
|
||||
+74
-51
@@ -1,15 +1,23 @@
|
||||
"""GeminiAssist — use Gemini Vision to generate macro steps from screenshots.
|
||||
"""LLMAssist — use a vision model to generate macro steps from screenshots.
|
||||
|
||||
This module is OPTIONAL. It requires:
|
||||
pip install google-generativeai
|
||||
pip install openai mss Pillow
|
||||
|
||||
Uses the OpenAI SDK, which is compatible with any OpenAI-compatible API
|
||||
provider (OpenAI, Azure, local vLLM, Ollama, LiteLLM, etc.).
|
||||
|
||||
Configure via environment variables:
|
||||
MACROCLI_MODEL — model name (required)
|
||||
MACROCLI_API_KEY — API key
|
||||
MACROCLI_BASE_URL — base URL (only needed for non-OpenAI hosts)
|
||||
|
||||
How it works:
|
||||
1. Capture a screenshot of the current screen (or use a provided image)
|
||||
2. Send the image + user goal to Gemini Vision with a strict system prompt
|
||||
3. Gemini returns a JSON array of steps (constrained action space)
|
||||
2. Send the image + user goal to the vision model with a strict system prompt
|
||||
3. The model returns a JSON array of steps (constrained action space)
|
||||
4. Steps are validated and written as a macro YAML file
|
||||
|
||||
The action space Gemini is allowed to produce:
|
||||
The action space the model is allowed to produce:
|
||||
|
||||
{"type": "click_image", "description": "...", "confidence": 0.85}
|
||||
{"type": "click_relative", "window_title": "...", "x_pct": 0.5, "y_pct": 0.1}
|
||||
@@ -20,7 +28,7 @@ The action space Gemini is allowed to produce:
|
||||
{"type": "menu_click", "app_name": "...", "menu_path": ["File", "Export"]}
|
||||
{"type": "scroll", "description": "...", "dy": -3}
|
||||
|
||||
Gemini is NOT allowed to:
|
||||
The model is NOT allowed to:
|
||||
- Produce shell commands, Python code, or arbitrary actions
|
||||
- Use absolute pixel coordinates
|
||||
- Output anything other than the JSON array
|
||||
@@ -29,11 +37,10 @@ The "description" field in click_image / wait_image / scroll tells the user
|
||||
what template image to capture with 'macro record' or 'capture_region'.
|
||||
|
||||
Usage:
|
||||
cli-anything-macrocli macro define my_export --assist \
|
||||
--goal "Export the current diagram as PNG to /tmp/out.png" \
|
||||
cli-anything-macrocli macro define my_export --assist \\
|
||||
--goal "Export the current diagram as PNG to /tmp/out.png" \\
|
||||
--screenshot current # takes a fresh screenshot
|
||||
--screenshot /path/to/img.png # use existing image
|
||||
--api-key $GEMINI_API_KEY
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -144,7 +151,7 @@ _REQUIRED_FIELDS = {
|
||||
|
||||
|
||||
def _validate_steps(raw_steps: list) -> tuple[list[dict], list[str]]:
|
||||
"""Validate and sanitize steps from Gemini output.
|
||||
"""Validate and sanitize steps from model output.
|
||||
|
||||
Returns (valid_steps, error_messages).
|
||||
"""
|
||||
@@ -182,8 +189,8 @@ def _validate_steps(raw_steps: list) -> tuple[list[dict], list[str]]:
|
||||
|
||||
# ── Step → YAML step dict conversion ─────────────────────────────────────────
|
||||
|
||||
def _gemini_step_to_yaml_step(step: dict, index: int) -> dict:
|
||||
"""Convert a validated Gemini step to a macro YAML step dict."""
|
||||
def _step_to_yaml_step(step: dict, index: int) -> dict:
|
||||
"""Convert a validated model step to a macro YAML step dict."""
|
||||
stype = step["type"]
|
||||
sid = f"step_{index:03d}_{stype}"
|
||||
|
||||
@@ -199,7 +206,7 @@ def _gemini_step_to_yaml_step(step: dict, index: int) -> dict:
|
||||
"_template_description": step.get("description", ""),
|
||||
},
|
||||
"on_failure": "fail",
|
||||
"_gemini_description": step.get("description", ""),
|
||||
"_model_description": step.get("description", ""),
|
||||
}
|
||||
elif stype == "click_relative":
|
||||
return {
|
||||
@@ -241,7 +248,7 @@ def _gemini_step_to_yaml_step(step: dict, index: int) -> dict:
|
||||
"_template_description": step.get("description", ""),
|
||||
},
|
||||
"on_failure": "fail",
|
||||
"_gemini_description": step.get("description", ""),
|
||||
"_model_description": step.get("description", ""),
|
||||
}
|
||||
elif stype == "wait_for_window":
|
||||
return {
|
||||
@@ -289,41 +296,55 @@ def generate_macro(
|
||||
macro_name: str,
|
||||
screenshot_source: str = "current", # "current" | path to image file
|
||||
api_key: Optional[str] = None,
|
||||
model: str = "gemini-1.5-flash",
|
||||
model: Optional[str] = None,
|
||||
base_url: Optional[str] = None,
|
||||
output_path: Optional[str] = None,
|
||||
) -> dict:
|
||||
"""Generate a macro YAML from a user goal and screenshot using Gemini.
|
||||
"""Generate a macro YAML from a user goal and screenshot using a vision model.
|
||||
|
||||
Args:
|
||||
goal: Natural language description of what the macro should do.
|
||||
macro_name: Name for the generated macro.
|
||||
screenshot_source: "current" to take a fresh screenshot, or a
|
||||
file path to use an existing image.
|
||||
api_key: Gemini API key. Falls back to GEMINI_API_KEY env var.
|
||||
model: Gemini model to use.
|
||||
api_key: API key. Falls back to MACROCLI_API_KEY env var.
|
||||
model: Model name. Falls back to MACROCLI_MODEL env var.
|
||||
base_url: Base URL for non-OpenAI providers. Falls back to
|
||||
MACROCLI_BASE_URL env var.
|
||||
output_path: Where to write the YAML file. Defaults to
|
||||
<macro_name>.yaml in the current directory.
|
||||
|
||||
Returns:
|
||||
dict with keys: yaml_path, steps_count, warnings, raw_steps
|
||||
"""
|
||||
import base64
|
||||
|
||||
try:
|
||||
import google.generativeai as genai
|
||||
from openai import OpenAI
|
||||
except ImportError:
|
||||
raise ImportError(
|
||||
"google-generativeai is required for Gemini assist.\n"
|
||||
" pip install google-generativeai"
|
||||
"openai is required for LLM assist.\n"
|
||||
" pip install openai"
|
||||
)
|
||||
|
||||
# Resolve API key
|
||||
key = api_key or os.environ.get("GEMINI_API_KEY", "")
|
||||
# Resolve config
|
||||
resolved_model = model or os.environ.get("MACROCLI_MODEL", "")
|
||||
key = api_key or os.environ.get("MACROCLI_API_KEY", "")
|
||||
resolved_base_url = base_url or os.environ.get("MACROCLI_BASE_URL", "")
|
||||
|
||||
if not resolved_model:
|
||||
raise ValueError(
|
||||
"Model required. Pass --model or set MACROCLI_MODEL env var."
|
||||
)
|
||||
if not key:
|
||||
raise ValueError(
|
||||
"Gemini API key required. Pass --api-key or set GEMINI_API_KEY env var.\n"
|
||||
"Get a key at: https://aistudio.google.com/app/apikey"
|
||||
"API key required. Pass --api-key or set MACROCLI_API_KEY env var."
|
||||
)
|
||||
|
||||
genai.configure(api_key=key)
|
||||
client_kwargs = {"api_key": key}
|
||||
if resolved_base_url:
|
||||
client_kwargs["base_url"] = resolved_base_url
|
||||
client = OpenAI(**client_kwargs)
|
||||
|
||||
# Get screenshot
|
||||
if screenshot_source == "current":
|
||||
@@ -333,25 +354,27 @@ def generate_macro(
|
||||
raise FileNotFoundError(f"Screenshot not found: {screenshot_source}")
|
||||
image_bytes = _load_image_bytes(screenshot_source)
|
||||
|
||||
image_b64 = base64.b64encode(image_bytes).decode("utf-8")
|
||||
|
||||
# Build prompt
|
||||
user_prompt = (
|
||||
f"Goal: {goal}\n\n"
|
||||
"Generate the minimal sequence of steps to achieve this goal. "
|
||||
"Output ONLY the JSON array, nothing else."
|
||||
user_content = [
|
||||
{"type": "text", "text": (
|
||||
f"Goal: {goal}\n\n"
|
||||
"Generate the minimal sequence of steps to achieve this goal. "
|
||||
"Output ONLY the JSON array, nothing else."
|
||||
)},
|
||||
{"type": "image_url", "image_url": {"url": f"data:image/png;base64,{image_b64}"}},
|
||||
]
|
||||
|
||||
response = client.chat.completions.create(
|
||||
model=resolved_model,
|
||||
messages=[
|
||||
{"role": "system", "content": _SYSTEM_PROMPT},
|
||||
{"role": "user", "content": user_content},
|
||||
],
|
||||
max_tokens=2048,
|
||||
)
|
||||
|
||||
# Call Gemini
|
||||
gemini_model = genai.GenerativeModel(
|
||||
model_name=model,
|
||||
system_instruction=_SYSTEM_PROMPT,
|
||||
)
|
||||
|
||||
import PIL.Image
|
||||
import io
|
||||
img = PIL.Image.open(io.BytesIO(image_bytes))
|
||||
|
||||
response = gemini_model.generate_content([user_prompt, img])
|
||||
raw_text = response.text.strip()
|
||||
raw_text = response.choices[0].message.content.strip()
|
||||
|
||||
# Strip markdown code fences if model added them despite instructions
|
||||
if raw_text.startswith("```"):
|
||||
@@ -366,13 +389,13 @@ def generate_macro(
|
||||
raw_steps = json.loads(raw_text)
|
||||
except json.JSONDecodeError as e:
|
||||
raise ValueError(
|
||||
f"Gemini returned invalid JSON: {e}\n"
|
||||
f"Model returned invalid JSON: {e}\n"
|
||||
f"Raw response (first 500 chars):\n{raw_text[:500]}"
|
||||
)
|
||||
|
||||
if not isinstance(raw_steps, list):
|
||||
raise ValueError(
|
||||
f"Gemini returned non-array JSON (expected list): {type(raw_steps)}"
|
||||
f"Model returned non-array JSON (expected list): {type(raw_steps)}"
|
||||
)
|
||||
|
||||
# Validate
|
||||
@@ -380,7 +403,7 @@ def generate_macro(
|
||||
|
||||
# Convert to YAML step dicts
|
||||
yaml_steps = [
|
||||
_gemini_step_to_yaml_step(s, i + 1)
|
||||
_step_to_yaml_step(s, i + 1)
|
||||
for i, s in enumerate(valid_steps)
|
||||
]
|
||||
|
||||
@@ -389,7 +412,7 @@ def generate_macro(
|
||||
"name": macro_name,
|
||||
"version": "1.0",
|
||||
"description": goal,
|
||||
"tags": ["generated", "gemini-assist"],
|
||||
"tags": ["generated", "llm-assist"],
|
||||
"parameters": {},
|
||||
"preconditions": [],
|
||||
"steps": yaml_steps,
|
||||
@@ -399,8 +422,8 @@ def generate_macro(
|
||||
"danger_level": "moderate",
|
||||
"side_effects": ["gui_interaction"],
|
||||
"reversible": False,
|
||||
"generated_by": "gemini-assist",
|
||||
"model": model,
|
||||
"generated_by": "llm-assist",
|
||||
"model": resolved_model,
|
||||
},
|
||||
}
|
||||
|
||||
@@ -409,10 +432,10 @@ def generate_macro(
|
||||
{
|
||||
"step_id": s["id"],
|
||||
"template_path": s["params"].get("template", ""),
|
||||
"description": s.get("_gemini_description", ""),
|
||||
"description": s.get("_model_description", ""),
|
||||
}
|
||||
for s in yaml_steps
|
||||
if s.get("params", {}).get("template") and s.get("_gemini_description")
|
||||
if s.get("params", {}).get("template") and s.get("_model_description")
|
||||
]
|
||||
|
||||
if templates_needed:
|
||||
@@ -1,4 +1,4 @@
|
||||
"""Parameterization helpers — interactive and Gemini-assisted.
|
||||
"""Parameterization helpers — interactive and LLM-assisted.
|
||||
|
||||
Interactive flow (no external deps):
|
||||
assignments = interactive_parameterize(type_steps)
|
||||
@@ -8,8 +8,8 @@ Interactive flow (no external deps):
|
||||
Post-hoc flow on an existing YAML file:
|
||||
parameterize_yaml_file(yaml_path) # modifies in-place
|
||||
|
||||
Gemini-assisted flow (optional, requires google-generativeai):
|
||||
assignments = gemini_suggest_parameters(type_steps, api_key=...)
|
||||
LLM-assisted flow (optional, requires openai):
|
||||
assignments = llm_suggest_parameters(type_steps, api_key=...)
|
||||
# returns same shape as interactive_parameterize, can be passed directly
|
||||
"""
|
||||
|
||||
@@ -202,20 +202,23 @@ def parameterize_yaml_file(yaml_path: str) -> bool:
|
||||
return True
|
||||
|
||||
|
||||
# ── Gemini-assisted parameterization ─────────────────────────────────────────
|
||||
# ── LLM-assisted parameterization ─────────────────────────────────────────
|
||||
|
||||
def gemini_suggest_parameters(
|
||||
def llm_suggest_parameters(
|
||||
type_steps: list[tuple[int, object]],
|
||||
api_key: Optional[str] = None,
|
||||
model: str = "gemini-1.5-flash",
|
||||
model: Optional[str] = None,
|
||||
base_url: Optional[str] = None,
|
||||
) -> dict[int, str]:
|
||||
"""Use Gemini to suggest which type_text steps should be parameterized
|
||||
"""Use a vision model to suggest which type_text steps should be parameterized
|
||||
and what to name the parameters.
|
||||
|
||||
Args:
|
||||
type_steps: Same format as interactive_parameterize input.
|
||||
api_key: Gemini API key. Falls back to GEMINI_API_KEY env var.
|
||||
model: Gemini model name.
|
||||
api_key: API key. Falls back to MACROCLI_API_KEY env var.
|
||||
model: Model name. Falls back to MACROCLI_MODEL env var.
|
||||
base_url: Base URL for non-OpenAI providers. Falls back to
|
||||
MACROCLI_BASE_URL env var.
|
||||
|
||||
Returns:
|
||||
{list_index: suggested_param_name} — same shape as interactive output.
|
||||
@@ -226,21 +229,30 @@ def gemini_suggest_parameters(
|
||||
import os
|
||||
|
||||
try:
|
||||
import google.generativeai as genai
|
||||
from openai import OpenAI
|
||||
except ImportError:
|
||||
raise ImportError(
|
||||
"google-generativeai required for auto-parameterization.\n"
|
||||
" pip install google-generativeai"
|
||||
"openai required for auto-parameterization.\n"
|
||||
" pip install openai"
|
||||
)
|
||||
|
||||
key = api_key or os.environ.get("GEMINI_API_KEY", "")
|
||||
resolved_model = model or os.environ.get("MACROCLI_MODEL", "")
|
||||
key = api_key or os.environ.get("MACROCLI_API_KEY", "")
|
||||
resolved_base_url = base_url or os.environ.get("MACROCLI_BASE_URL", "")
|
||||
|
||||
if not resolved_model:
|
||||
raise ValueError(
|
||||
"Model required. Set MACROCLI_MODEL env var or pass --model."
|
||||
)
|
||||
if not key:
|
||||
raise ValueError(
|
||||
"Gemini API key required. Pass --api-key or set GEMINI_API_KEY.\n"
|
||||
" https://aistudio.google.com/app/apikey"
|
||||
"API key required. Pass --api-key or set MACROCLI_API_KEY."
|
||||
)
|
||||
|
||||
genai.configure(api_key=key)
|
||||
client_kwargs = {"api_key": key}
|
||||
if resolved_base_url:
|
||||
client_kwargs["base_url"] = resolved_base_url
|
||||
client = OpenAI(**client_kwargs)
|
||||
|
||||
_SYSTEM = """\
|
||||
You are a macro parameterization assistant. Given a list of text values
|
||||
@@ -266,9 +278,15 @@ Example: {"0": "output_path", "2": "export_width"}
|
||||
)
|
||||
prompt = f"Typed values from the recording:\n{items}\n\nOutput JSON only."
|
||||
|
||||
gem = genai.GenerativeModel(model_name=model, system_instruction=_SYSTEM)
|
||||
response = gem.generate_content(prompt)
|
||||
raw = response.text.strip()
|
||||
response = client.chat.completions.create(
|
||||
model=resolved_model,
|
||||
messages=[
|
||||
{"role": "system", "content": _SYSTEM},
|
||||
{"role": "user", "content": prompt},
|
||||
],
|
||||
max_tokens=1024,
|
||||
)
|
||||
raw = response.choices[0].message.content.strip()
|
||||
|
||||
# Strip markdown fences if present
|
||||
if raw.startswith("```"):
|
||||
@@ -281,7 +299,7 @@ Example: {"0": "output_path", "2": "export_width"}
|
||||
raw_dict: dict = json.loads(raw)
|
||||
except json.JSONDecodeError as e:
|
||||
raise ValueError(
|
||||
f"Gemini returned invalid JSON: {e}\nRaw: {raw[:300]}"
|
||||
f"Model returned invalid JSON: {e}\nRaw: {raw[:300]}"
|
||||
)
|
||||
|
||||
# Convert string keys to int, validate names
|
||||
@@ -298,3 +316,7 @@ Example: {"0": "output_path", "2": "export_width"}
|
||||
result[idx] = v
|
||||
|
||||
return result
|
||||
|
||||
|
||||
# Keep old name as alias for backwards compatibility
|
||||
gemini_suggest_parameters = llm_suggest_parameters
|
||||
|
||||
@@ -636,7 +636,7 @@ class MacroRecorder:
|
||||
print("─" * 60)
|
||||
print(" Step Review — mark steps as 'fixed' or 'agent'")
|
||||
print(" Enter = fixed (fast, deterministic)")
|
||||
print(" a = agent step (Gemini decides at runtime)")
|
||||
print(" a = agent step (vision model decides at runtime)")
|
||||
print("─" * 60)
|
||||
|
||||
for i, step in enumerate(self._steps):
|
||||
|
||||
@@ -440,10 +440,10 @@ def macro_define(name, output):
|
||||
help="After recording, interactively choose which typed values "
|
||||
"become CLI parameters.")
|
||||
@click.option("--auto-parameterize", "do_auto_param", is_flag=True,
|
||||
help="After recording, use Gemini to automatically suggest "
|
||||
"parameter names (requires --api-key or GEMINI_API_KEY).")
|
||||
@click.option("--api-key", default=None, envvar="GEMINI_API_KEY",
|
||||
help="Gemini API key for --auto-parameterize.")
|
||||
help="After recording, use an LLM to automatically suggest "
|
||||
"parameter names (requires --api-key or MACROCLI_API_KEY).")
|
||||
@click.option("--api-key", default=None, envvar="MACROCLI_API_KEY",
|
||||
help="API key for --auto-parameterize.")
|
||||
@handle_error
|
||||
def macro_record(name, output_dir, timeout, do_agent_review,
|
||||
do_parameterize, do_auto_param, api_key):
|
||||
@@ -509,14 +509,14 @@ def macro_record(name, output_dir, timeout, do_agent_review,
|
||||
if do_auto_param and type_steps:
|
||||
try:
|
||||
from cli_anything.macrocli.core.parameterize import (
|
||||
gemini_suggest_parameters,
|
||||
llm_suggest_parameters,
|
||||
interactive_parameterize,
|
||||
)
|
||||
if not _json_output:
|
||||
click.echo(f"\nAsking Gemini to suggest parameters...")
|
||||
suggestions = gemini_suggest_parameters(type_steps, api_key=api_key)
|
||||
click.echo(f"\nAsking LLM to suggest parameters...")
|
||||
suggestions = llm_suggest_parameters(type_steps, api_key=api_key)
|
||||
if suggestions and not _json_output:
|
||||
click.echo(" Gemini suggestions:")
|
||||
click.echo(" LLM suggestions:")
|
||||
for idx, pname in suggestions.items():
|
||||
step = recorder._steps[idx]
|
||||
click.echo(f" step {idx+1} {step.text!r} → ${{{pname}}}")
|
||||
@@ -527,9 +527,9 @@ def macro_record(name, output_dir, timeout, do_agent_review,
|
||||
final = {**suggestions, **confirmed}
|
||||
parameters = recorder.apply_parameterization(final)
|
||||
elif not suggestions and not _json_output:
|
||||
click.echo(" Gemini found no values to parameterize.")
|
||||
click.echo(" LLM found no values to parameterize.")
|
||||
except Exception as e:
|
||||
click.echo(f" Warning: Gemini parameterization failed: {e}", err=True)
|
||||
click.echo(f" Warning: LLM parameterization failed: {e}", err=True)
|
||||
do_parameterize = True
|
||||
|
||||
if do_parameterize and type_steps:
|
||||
@@ -568,7 +568,7 @@ def macro_record(name, output_dir, timeout, do_agent_review,
|
||||
else:
|
||||
click.echo(f"✓ Saved {len(recorder._steps)} steps to: {pkg_dir}/")
|
||||
if agent_count:
|
||||
click.echo(f" Agent steps: {agent_count} (will use Gemini at runtime)")
|
||||
click.echo(f" Agent steps: {agent_count} (will use vision model at runtime)")
|
||||
if parameters:
|
||||
click.echo(f" Parameters: {', '.join(parameters.keys())}")
|
||||
click.echo(
|
||||
@@ -593,8 +593,8 @@ def macro_record(name, output_dir, timeout, do_agent_review,
|
||||
# Record + interactively parameterize typed values
|
||||
macro record my_export --parameterize
|
||||
|
||||
# Record + auto-parameterize with Gemini
|
||||
macro record my_export --auto-parameterize --api-key $GEMINI_API_KEY
|
||||
# Record + auto-parameterize with LLM
|
||||
macro record my_export --auto-parameterize --api-key $MACROCLI_API_KEY
|
||||
|
||||
Requires: pip install mss Pillow pynput
|
||||
"""
|
||||
@@ -634,17 +634,17 @@ def macro_record(name, output_dir, timeout, do_agent_review,
|
||||
if do_auto_param and type_steps:
|
||||
try:
|
||||
from cli_anything.macrocli.core.parameterize import (
|
||||
gemini_suggest_parameters,
|
||||
llm_suggest_parameters,
|
||||
interactive_parameterize,
|
||||
)
|
||||
if not _json_output:
|
||||
click.echo(f"\nAsking Gemini to suggest parameters for "
|
||||
click.echo(f"\nAsking LLM to suggest parameters for "
|
||||
f"{len(type_steps)} type_text step(s)...")
|
||||
suggestions = gemini_suggest_parameters(
|
||||
suggestions = llm_suggest_parameters(
|
||||
type_steps, api_key=api_key
|
||||
)
|
||||
if suggestions and not _json_output:
|
||||
click.echo(" Gemini suggestions:")
|
||||
click.echo(" LLM suggestions:")
|
||||
for idx, pname in suggestions.items():
|
||||
step = recorder._steps[idx]
|
||||
click.echo(f" step {idx+1} {step.text!r} → ${{{pname}}}")
|
||||
@@ -654,15 +654,15 @@ def macro_record(name, output_dir, timeout, do_agent_review,
|
||||
[(i, s) for i, s in type_steps if i in suggestions],
|
||||
existing_params=set(),
|
||||
)
|
||||
# For steps Gemini suggested but user skipped, remove them
|
||||
# For steps LLM suggested but user skipped, remove them
|
||||
final = {i: n for i, n in suggestions.items() if i in confirmed}
|
||||
# For steps user renamed, use their name
|
||||
final.update(confirmed)
|
||||
parameters = recorder.apply_parameterization(final)
|
||||
elif not suggestions and not _json_output:
|
||||
click.echo(" Gemini found no values to parameterize.")
|
||||
click.echo(" LLM found no values to parameterize.")
|
||||
except Exception as e:
|
||||
click.echo(f" Warning: Gemini parameterization failed: {e}", err=True)
|
||||
click.echo(f" Warning: LLM parameterization failed: {e}", err=True)
|
||||
click.echo(" Falling back to interactive mode...")
|
||||
do_parameterize = True
|
||||
|
||||
@@ -709,9 +709,9 @@ def macro_record(name, output_dir, timeout, do_agent_review,
|
||||
@macro.command("parameterize")
|
||||
@click.argument("yaml_file")
|
||||
@click.option("--auto", "do_auto", is_flag=True,
|
||||
help="Use Gemini to suggest parameter names automatically.")
|
||||
@click.option("--api-key", default=None, envvar="GEMINI_API_KEY",
|
||||
help="Gemini API key for --auto.")
|
||||
help="Use an LLM to suggest parameter names automatically.")
|
||||
@click.option("--api-key", default=None, envvar="MACROCLI_API_KEY",
|
||||
help="API key for --auto.")
|
||||
@handle_error
|
||||
def macro_parameterize(yaml_file, do_auto, api_key):
|
||||
"""Interactively parameterize typed values in an existing macro YAML.
|
||||
@@ -723,11 +723,11 @@ def macro_parameterize(yaml_file, do_auto, api_key):
|
||||
\b
|
||||
Examples:
|
||||
macro parameterize /tmp/recording/my_export.yaml
|
||||
macro parameterize my_export.yaml --auto --api-key $GEMINI_API_KEY
|
||||
macro parameterize my_export.yaml --auto --api-key $MACROCLI_API_KEY
|
||||
"""
|
||||
from cli_anything.macrocli.core.parameterize import (
|
||||
parameterize_yaml_file,
|
||||
gemini_suggest_parameters,
|
||||
llm_suggest_parameters,
|
||||
interactive_parameterize,
|
||||
_YamlTypeStep,
|
||||
)
|
||||
@@ -740,7 +740,7 @@ def macro_parameterize(yaml_file, do_auto, api_key):
|
||||
return
|
||||
|
||||
if do_auto:
|
||||
# Load the file, extract type_text steps, ask Gemini, then apply
|
||||
# Load the file, extract type_text steps, ask LLM, then apply
|
||||
import yaml as _yaml
|
||||
with open(p, encoding="utf-8") as f:
|
||||
macro_dict = _yaml.safe_load(f)
|
||||
@@ -759,11 +759,11 @@ def macro_parameterize(yaml_file, do_auto, api_key):
|
||||
wrapped = [(i, _YamlTypeStep(i, s)) for i, s in type_steps_raw]
|
||||
|
||||
try:
|
||||
click.echo(f"Asking Gemini to suggest parameters for "
|
||||
click.echo(f"Asking LLM to suggest parameters for "
|
||||
f"{len(wrapped)} type_text step(s)...")
|
||||
suggestions = gemini_suggest_parameters(wrapped, api_key=api_key)
|
||||
suggestions = llm_suggest_parameters(wrapped, api_key=api_key)
|
||||
except Exception as e:
|
||||
click.echo(f"Gemini failed: {e}\nFalling back to interactive.", err=True)
|
||||
click.echo(f"LLM failed: {e}\nFalling back to interactive.", err=True)
|
||||
suggestions = {}
|
||||
do_auto = False
|
||||
|
||||
@@ -774,7 +774,7 @@ def macro_parameterize(yaml_file, do_auto, api_key):
|
||||
click.echo(f" step {idx+1} {w.text!r} → ${{{pname}}}")
|
||||
click.echo()
|
||||
|
||||
# Let user confirm (pre-fill Gemini suggestions as defaults)
|
||||
# Let user confirm (pre-fill LLM suggestions as defaults)
|
||||
existing = set((macro_dict.get("parameters") or {}).keys())
|
||||
confirmed = interactive_parameterize(
|
||||
[(i, w) for i, w in wrapped if i in suggestions],
|
||||
@@ -831,35 +831,35 @@ def macro_parameterize(yaml_file, do_auto, api_key):
|
||||
help="'current' to take a screenshot now, or path to an image file.")
|
||||
@click.option("--output", "-o", default=None,
|
||||
help="Output YAML file path (default: <name>.yaml).")
|
||||
@click.option("--api-key", default=None, envvar="GEMINI_API_KEY",
|
||||
help="Gemini API key (or set GEMINI_API_KEY env var).")
|
||||
@click.option("--model", default="gemini-1.5-flash",
|
||||
help="Gemini model name.")
|
||||
@click.option("--api-key", default=None, envvar="MACROCLI_API_KEY",
|
||||
help="API key (or set MACROCLI_API_KEY env var).")
|
||||
@click.option("--model", default=None,
|
||||
help="Model name (or set MACROCLI_MODEL env var).")
|
||||
@handle_error
|
||||
def macro_assist(name, goal, screenshot, output, api_key, model):
|
||||
"""Generate a macro YAML from a screenshot using Gemini Vision (optional).
|
||||
"""Generate a macro YAML from a screenshot using a vision model (optional).
|
||||
|
||||
\b
|
||||
Takes a screenshot, sends it to Gemini with your goal, and generates
|
||||
a macro YAML. Steps that require visual templates will include
|
||||
Takes a screenshot, sends it to the configured model with your goal, and
|
||||
generates a macro YAML. Steps that require visual templates will include
|
||||
instructions for which template images to capture.
|
||||
|
||||
Requires: pip install google-generativeai mss Pillow
|
||||
Requires: pip install openai mss Pillow
|
||||
|
||||
\b
|
||||
Example:
|
||||
macro assist export_png \\
|
||||
--goal "Export the current diagram as PNG to /tmp/out.png" \\
|
||||
--api-key $GEMINI_API_KEY
|
||||
--api-key $MACROCLI_API_KEY
|
||||
"""
|
||||
try:
|
||||
from cli_anything.macrocli.core.gemini_assist import generate_macro
|
||||
from cli_anything.macrocli.core.llm_assist import generate_macro
|
||||
except ImportError as e:
|
||||
click.echo(f"Error: {e}", err=True)
|
||||
sys.exit(1)
|
||||
|
||||
if not _json_output:
|
||||
click.echo(f"Sending screenshot to Gemini ({model})...")
|
||||
click.echo(f"Sending screenshot to model ({model or os.environ.get('MACROCLI_MODEL', 'unset')})...")
|
||||
|
||||
result = generate_macro(
|
||||
goal=goal,
|
||||
|
||||
Reference in New Issue
Block a user