fix(openrefine): support piped REPL workflows (#385)

* fix(openrefine): support piped REPL workflows

Signed-off-by: asahikiko <1225133002@qq.com>

* fix(openrefine): keep piped REPL output ASCII-safe

Signed-off-by: asahikiko <1225133002@qq.com>

* fix(openrefine): propagate piped REPL failures

Signed-off-by: asahikiko <1225133002@qq.com>

* fix(openrefine): escape piped command output

Signed-off-by: asahikiko <1225133002@qq.com>

---------

Signed-off-by: asahikiko <1225133002@qq.com>
This commit is contained in:
asahikiko
2026-08-03 16:40:39 +08:00
committed by GitHub
parent c6c622a2eb
commit 53c56535da
11 changed files with 282 additions and 37 deletions
+6
View File
@@ -15,6 +15,12 @@ cli-anything-openrefine --help
cli-anything-openrefine
```
Replay a scripted REPL user journey in CI or a shell pipeline:
```bash
printf 'help\nexit\n' | cli-anything-openrefine
```
Start OpenRefine first for backend commands:
```bash
@@ -12,6 +12,21 @@ cli-anything-openrefine --json data export clean.csv
Run `cli-anything-openrefine` with no arguments for the REPL.
The REPL also accepts newline-separated commands from stdin, so user journeys
can be replayed in CI without an interactive terminal:
```bash
printf 'help\nexit\n' | cli-anything-openrefine
```
Interactive terminals keep the Unicode banner, prompt history, and styling;
redirected input uses ASCII-only output for cross-platform reliability,
including Windows environments configured with legacy encodings.
Unicode command payloads are preserved as ASCII backslash escapes in this
mode, so values remain identifiable without triggering encoding failures.
If any piped command fails, the REPL continues consuming the script but exits
nonzero at `exit` or EOF so CI can reject the failed user journey.
Start OpenRefine first:
```bash
@@ -1,3 +1,3 @@
"""CLI-Anything harness for OpenRefine."""
__version__ = "1.0.0"
__version__ = "1.0.1"
@@ -25,27 +25,57 @@ def _service(ctx: click.Context) -> OpenRefineService:
return OpenRefineService(OpenRefineBackend(base_url, timeout=ctx.obj["timeout"]), store)
def _emit(data: Any, as_json: bool) -> None:
def _ascii_safe(value: Any) -> str:
"""Render text as reversible ASCII escapes for legacy output streams."""
return str(value).encode("ascii", errors="backslashreplace").decode("ascii")
def _emit(data: Any, as_json: bool, ascii_safe: bool = False) -> None:
if as_json:
click.echo(json.dumps(data, indent=2, sort_keys=True))
elif isinstance(data, dict):
for key, value in data.items():
click.echo(f"{key}: {value}")
text = f"{key}: {value}"
click.echo(_ascii_safe(text) if ascii_safe else text)
else:
click.echo(str(data))
text = str(data)
click.echo(_ascii_safe(text) if ascii_safe else text)
def _handle(ctx: click.Context, func, *args, **kwargs) -> None:
try:
_emit(func(*args, **kwargs), ctx.obj["json"])
_emit(
func(*args, **kwargs),
ctx.obj["json"],
ascii_safe=ctx.obj.get("ascii_output", False),
)
except (OpenRefineError, ValueError, OSError) as exc:
if ctx.obj["json"]:
click.echo(json.dumps({"error": str(exc), "ok": False}, indent=2, sort_keys=True), err=True)
else:
click.echo(f"Error: {exc}", err=True)
message = f"Error: {exc}"
if ctx.obj.get("ascii_output", False):
message = _ascii_safe(message)
click.echo(message, err=True)
raise click.exceptions.Exit(1)
def _supports_interactive_prompt(stdin=None, stdout=None) -> bool:
"""Return whether prompt_toolkit can safely attach to the current terminal.
Redirected streams are common in CI, Click's ``CliRunner``, and shell
pipelines that replay a user workflow. On Windows prompt_toolkit raises
``NoConsoleScreenBufferError`` for those streams instead of reading the
piped commands, so keep its rich prompt for real terminals only.
"""
stdin = sys.stdin if stdin is None else stdin
stdout = sys.stdout if stdout is None else stdout
try:
return bool(stdin.isatty() and stdout.isatty())
except (AttributeError, OSError):
return False
@click.group(invoke_without_command=True)
@click.option("--base-url", default=None, help="OpenRefine URL. Defaults to OPENREFINE_URL, then session state, then http://127.0.0.1:3333.")
@click.option("--session", "session_path", type=click.Path(dir_okay=False), default=None, help="Session JSON path.")
@@ -68,8 +98,16 @@ def repl(ctx: click.Context) -> None:
"""Start the interactive REPL."""
history_file = _repl_history_file(ctx)
skin = ReplSkin("openrefine", version=__version__, history_file=history_file)
skin.print_banner()
prompt = skin.create_prompt_session()
interactive = _supports_interactive_prompt()
if interactive:
skin.print_banner()
prompt = skin.create_prompt_session()
else:
# Keep redirected workflows ASCII-only. Windows pipes and CI runners
# may expose legacy encodings such as cp1252, which cannot represent
# the skin's box-drawing banner, prompt arrow, or status icons.
prompt = None
ctx.obj["ascii_output"] = True
commands = {
"status": "Check backend and session",
"projects": "List OpenRefine projects",
@@ -80,37 +118,76 @@ def repl(ctx: click.Context) -> None:
"undo / redo": "Use OpenRefine undo-redo where possible",
"exit": "Quit",
}
had_error = False
while True:
try:
state = SessionStore(ctx.obj["session"]).load()
line = skin.get_input(prompt, project_name=state.project_name)
if interactive:
line = skin.get_input(prompt, project_name=state.project_name)
else:
line = input().strip()
except (EOFError, KeyboardInterrupt):
skin.print_goodbye()
if interactive:
skin.print_goodbye()
else:
click.echo("Goodbye!")
if had_error:
raise click.exceptions.Exit(1)
return
try:
parts = shlex.split(line)
except (IndexError, ValueError) as exc:
skin.error(str(exc))
if interactive:
skin.error(str(exc))
else:
click.echo(f"Error: {_ascii_safe(exc)}", err=True)
had_error = True
continue
if not parts:
continue
try:
args = _repl_to_args(parts)
except (IndexError, ValueError) as exc:
skin.error(str(exc))
if interactive:
skin.error(str(exc))
else:
click.echo(f"Error: {_ascii_safe(exc)}", err=True)
had_error = True
continue
if parts[0] in {"exit", "quit"}:
skin.print_goodbye()
if interactive:
skin.print_goodbye()
else:
click.echo("Goodbye!")
if had_error:
raise click.exceptions.Exit(1)
return
if parts[0] == "help":
skin.help(commands)
if interactive:
skin.help(commands)
else:
for command, description in commands.items():
click.echo(f"{command}: {description}")
continue
try:
cli.main(args=_global_args(ctx) + args, prog_name="cli-anything-openrefine", obj=ctx.obj, standalone_mode=False)
except SystemExit:
pass
result = cli.main(
args=_global_args(ctx) + args,
prog_name="cli-anything-openrefine",
obj=ctx.obj,
standalone_mode=False,
)
if isinstance(result, int) and result != 0:
had_error = True
except (SystemExit, click.exceptions.Exit) as exc:
exit_code = getattr(exc, "exit_code", getattr(exc, "code", 0))
if exit_code:
had_error = True
except Exception as exc:
skin.error(str(exc))
if interactive:
skin.error(str(exc))
else:
click.echo(f"Error: {_ascii_safe(exc)}", err=True)
had_error = True
def _repl_to_args(parts: list[str]) -> list[str]:
@@ -51,6 +51,22 @@ cli-anything-openrefine --json --session run/session.json session redo
Run `cli-anything-openrefine` with no subcommand to enter the REPL.
For automated user journeys, pipe newline-separated REPL commands through
stdin. Redirected streams automatically use an ASCII-only input/output path
while interactive terminals retain the Unicode banner, prompt history, and
styling:
```bash
printf 'help\nexit\n' | cli-anything-openrefine
```
Unicode payloads are rendered as reversible ASCII backslash escapes when the
REPL is redirected, preventing legacy Windows output encodings from turning a
successful command into a failed journey.
Piped workflows return a nonzero exit status at `exit` or EOF if any command
failed, allowing CI to detect an unsuccessful user journey.
## Error Handling
When `--json` is set, command failures write a JSON object to stderr with `ok: false`.
@@ -2,8 +2,8 @@
## Test Inventory Plan
- `test_core.py`: 76 backend-free unit and CLI tests planned.
- `test_full_e2e.py`: 12 real-backend E2E tests planned.
- `test_core.py`: 81 backend-free unit and CLI tests planned.
- `test_full_e2e.py`: 14 subprocess and real-backend E2E tests planned.
## Unit Test Plan
@@ -11,7 +11,7 @@
- `core.session`: default state, atomic save/load, record, undo, redo, empty-stack errors.
- `core.project`: service orchestration with fake backend, import/open/apply/export/rows, local and backend undo/redo behavior.
- `utils.openrefine_backend`: small pure helpers and error types.
- `openrefine_cli`: help output, default REPL entry, JSON operation builder commands, session show, REPL command mapping.
- `openrefine_cli`: help output, terminal capability detection, piped multi-step REPL journeys, JSON operation builder commands, session show, REPL command mapping.
## E2E Test Plan
@@ -24,6 +24,7 @@ It intentionally fails loudly when the backend is unavailable.
- **Cleaning operation history**: apply `core/text-transform` and verify exported CSV no longer contains padded names.
- **Normalization operation history**: apply `core/mass-edit` to city values and verify exported content.
- **Agent subprocess workflow**: run the installed or module CLI with `--json`, import data, inspect rows, export CSV, and parse exported rows with Python `csv`.
- **Piped REPL workflow**: replay newline-separated user commands through the real CLI subprocess and verify the plain-input fallback exits cleanly on Windows and CI.
- **Operation file workflow**: build an operation-history JSON file via CLI, apply it to a backend project, and verify operation count.
- **State persistence**: verify session JSON persists current project and action history across subprocess calls.
- **Undo/redo recovery**: apply a backend operation and exercise OpenRefine undo/redo endpoints.
@@ -36,9 +37,17 @@ Unit suite run:
```text
$ python -m pytest cli_anything/openrefine/tests/test_core.py -q
........................................................................ [ 94%]
.... [100%]
76 passed in 0.42s
........................................................................ [ 88%]
......... [100%]
81 passed in 0.34s
```
Backend-independent subprocess checks:
```text
$ python -m pytest cli_anything/openrefine/tests/test_full_e2e.py -q -k "piped_user_commands or failed_piped_command or cli_help_subprocess"
... [100%]
3 passed, 11 deselected in 1.18s
```
Previous full suite run with OpenRefine 3.10.1 running at `http://127.0.0.1:3333`:
@@ -123,7 +132,7 @@ Collection check:
```text
$ python -m pytest cli_anything/openrefine/tests/ --collect-only -q
88 tests collected in 0.17s
95 tests collected in 0.06s
```
Setup metadata check:
@@ -132,18 +141,18 @@ Setup metadata check:
$ python setup.py --name
cli-anything-openrefine
$ python setup.py --version
1.0.0
1.0.1
```
## Summary Statistics
- Total collected tests: 88
- Backend-free unit tests: 76 passing
- E2E tests: 12 collected and previously passing against a real OpenRefine 3.10.1 local HTTP backend
- Total collected tests: 95
- Backend-free unit tests: 81 passing
- E2E tests: 14 collected; the 3 backend-independent subprocess checks pass locally, and the original 12-test suite previously passed against a real OpenRefine 3.10.1 local HTTP backend
- Minimum validator thresholds met: 50+ pytest tests and 10+ E2E pytest tests
## Coverage Notes
- Unit tests cover operation JSON builders, session persistence, fake-backend service orchestration, CLI JSON output, and default REPL entry.
- E2E tests cover real backend import, metadata, row reads, operation application, CSV export verification, subprocess CLI workflows, session persistence, undo/redo, JSON error handling, and cleanup recovery.
- E2E tests cover piped REPL user input, real backend import, metadata, row reads, operation application, CSV export verification, subprocess CLI workflows, session persistence, undo/redo, JSON error handling, and cleanup recovery.
- Reconciliation workflows are documented as a limitation and currently require applying exported OpenRefine reconciliation operation histories.
@@ -17,7 +17,7 @@ from cli_anything.openrefine.core.operations import (
from cli_anything.openrefine.core.project import OpenRefineService, _extract_project_id
from cli_anything.openrefine.core.session import SessionState, SessionStore
from cli_anything.openrefine import openrefine_cli
from cli_anything.openrefine.openrefine_cli import _repl_to_args, cli
from cli_anything.openrefine.openrefine_cli import _ascii_safe, _repl_to_args, _supports_interactive_prompt, cli
from cli_anything.openrefine.utils.openrefine_backend import OpenRefineBackend, OpenRefineError, _coerce_json_or_text
@@ -457,8 +457,67 @@ def test_cli_session_show_json_uses_custom_path(tmp_path):
def test_cli_default_enters_repl_and_exits():
result = CliRunner().invoke(cli, input="exit\n")
assert result.exit_code == 0
assert "cli-anything" in result.output
assert "Openrefine" in result.output
assert result.output == "Goodbye!\n"
def test_prompt_toolkit_is_reserved_for_real_terminals():
class Stream:
def __init__(self, is_tty):
self.is_tty = is_tty
def isatty(self):
return self.is_tty
assert _supports_interactive_prompt(Stream(True), Stream(True))
assert not _supports_interactive_prompt(Stream(False), Stream(True))
assert not _supports_interactive_prompt(Stream(True), Stream(False))
def test_ascii_safe_output_preserves_unicode_as_escapes():
assert _ascii_safe("Project 😀 café") == r"Project \U0001f600 caf\xe9"
def test_cli_repl_accepts_piped_multi_step_user_journey(tmp_path, monkeypatch):
session = tmp_path / "session.json"
output = tmp_path / "clean export.csv"
monkeypatch.setattr(openrefine_cli, "OpenRefineBackend", FakeBackend)
commands = "\n".join(
[
"open 123",
"rows 2",
f'export "{output}" csv',
"exit",
"",
]
)
result = CliRunner().invoke(cli, ["--session", str(session)], input=commands)
assert result.exit_code == 0, result.output
assert "project_id: 123" in result.output
assert "Alice" in result.output
assert output.read_text(encoding="utf-8").startswith("name,value")
state = SessionStore(session).load()
assert state.project_id == "123"
assert state.last_export == str(output)
def test_cli_repl_piped_errors_are_ascii_safe():
result = CliRunner().invoke(cli, input='import "unterminated\nexit\n')
assert result.exit_code == 1
assert "Error: No closing quotation" in result.stderr
assert result.output.endswith("Goodbye!\n")
def test_cli_repl_propagates_piped_command_failures(tmp_path):
for script in ("rows\n", "definitely-not-a-command\n"):
session = tmp_path / f"session-{len(script)}.json"
result = CliRunner().invoke(cli, ["--session", str(session)], input=script)
assert result.exit_code == 1
assert "Error:" in result.stderr
assert result.output.endswith("Goodbye!\n")
def test_openrefine_error_is_runtime_error():
@@ -7,11 +7,10 @@ import shutil
import subprocess
import sys
import time
from pathlib import Path
import pytest
from cli_anything.openrefine.utils.openrefine_backend import INSTALL_INSTRUCTIONS, OpenRefineBackend, OpenRefineError
from cli_anything.openrefine.utils.openrefine_backend import INSTALL_INSTRUCTIONS, OpenRefineBackend
def _resolve_cli(name):
@@ -163,6 +162,54 @@ def test_e2e_cli_help_subprocess(cli_base):
assert "data" in result.stdout
def test_e2e_cli_repl_accepts_piped_user_commands(cli_base, tmp_path):
env = os.environ.copy()
env.update({"NO_COLOR": "1", "PYTHONIOENCODING": "cp1252"})
session = tmp_path / "unicode-session.json"
session.write_text(
json.dumps({"project_name": "Project 😀"}, ensure_ascii=False),
encoding="utf-8",
)
result = subprocess.run(
cli_base + ["--session", str(session)],
input="session show\nhelp\nexit\n",
capture_output=True,
text=True,
encoding="cp1252",
check=False,
timeout=30,
env=env,
)
print("STDOUT:", result.stdout)
print("STDERR:", result.stderr)
assert result.returncode == 0
assert r"Project \U0001f600" in result.stdout
assert "List OpenRefine projects" in result.stdout
assert "Goodbye!" in result.stdout
assert "NoConsoleScreenBufferError" not in result.stderr
assert "UnicodeEncodeError" not in result.stderr
def test_e2e_cli_repl_returns_nonzero_after_failed_piped_command(cli_base):
env = os.environ.copy()
env.update({"NO_COLOR": "1", "PYTHONIOENCODING": "cp1252"})
result = subprocess.run(
cli_base,
input="definitely-not-a-command\n",
capture_output=True,
text=True,
encoding="cp1252",
check=False,
timeout=30,
env=env,
)
print("STDOUT:", result.stdout)
print("STDERR:", result.stderr)
assert result.returncode == 1
assert "No such command 'definitely-not-a-command'" in result.stderr
assert result.stdout.endswith("Goodbye!\n")
def test_e2e_cli_json_import_rows_export_workflow(backend, cli_base, sample_csv, tmp_path, base_url):
session = tmp_path / "session.json"
imported = _run(cli_base, ["--json", "--base-url", base_url, "--session", str(session), "project", "import", str(sample_csv), "--name", "cli-anything-e2e-cli"])
+1 -1
View File
@@ -3,7 +3,7 @@ from setuptools import find_namespace_packages, setup
setup(
name="cli-anything-openrefine",
version="1.0.0",
version="1.0.1",
description="CLI-Anything harness for OpenRefine data wrangling workflows",
long_description="Agent-native Click CLI for OpenRefine's local HTTP API, operation histories, exports, and sessions.",
author="CLI-Anything-Team",
+1 -1
View File
@@ -27,7 +27,7 @@
{
"name": "openrefine",
"display_name": "OpenRefine",
"version": "1.0.0",
"version": "1.0.1",
"description": "Agent-native CLI for OpenRefine import, operation-history cleaning, row inspection, export, and session undo/redo through the real local HTTP API.",
"requires": "OpenRefine 3.10.x or newer running as a local web server",
"homepage": "https://openrefine.org/",
+16
View File
@@ -51,6 +51,22 @@ cli-anything-openrefine --json --session run/session.json session redo
Run `cli-anything-openrefine` with no subcommand to enter the REPL.
For automated user journeys, pipe newline-separated REPL commands through
stdin. Redirected streams automatically use an ASCII-only input/output path
while interactive terminals retain the Unicode banner, prompt history, and
styling:
```bash
printf 'help\nexit\n' | cli-anything-openrefine
```
Unicode payloads are rendered as reversible ASCII backslash escapes when the
REPL is redirected, preventing legacy Windows output encodings from turning a
successful command into a failed journey.
Piped workflows return a nonzero exit status at `exit` or EOF if any command
failed, allowing CI to detect an unsuccessful user journey.
## Error Handling
When `--json` is set, command failures write a JSON object to stderr with `ok: false`.