From e281ed2f6d3da7229c7c79fd847630b674d7c597 Mon Sep 17 00:00:00 2001 From: GatewayJ <835269233@qq.com> Date: Thu, 27 Aug 2026 22:25:33 +0800 Subject: [PATCH] test(table-catalog): generate DuckDB REST attach SQL (#6749) --- docs/architecture/s3-tables-support-matrix.md | 3 +- scripts/table-catalog/README.md | 40 +++++- scripts/table-catalog/engine_compatibility.py | 115 ++++++++++++++++-- .../test_engine_compatibility.py | 98 ++++++++++++++- 4 files changed, 246 insertions(+), 10 deletions(-) diff --git a/docs/architecture/s3-tables-support-matrix.md b/docs/architecture/s3-tables-support-matrix.md index 65eeb4be7..a49c3520d 100644 --- a/docs/architecture/s3-tables-support-matrix.md +++ b/docs/architecture/s3-tables-support-matrix.md @@ -43,7 +43,7 @@ catalog extension. | PyIceberg | Automated | Creates namespace and table, appends rows, reloads, scans, probes metadata-location, refs, views, maintenance, diagnostics, and optional catalog-vended table credentials with an exact-prefix data-plane scope check. | | Spark Iceberg REST catalog | Manual/live harness | RustFS can generate pinned Spark/Iceberg package inputs, REST catalog properties, SQL, run commands, expected `row_count=2`, and a CI opt-in gate for namespace creation, table creation, append, refresh, count, and cleanup. Live Spark execution and commit-conflict probing are still manual validation items unless explicitly enabled in the runner. | | Trino Iceberg REST catalog | Manual/live harness | RustFS can generate catalog properties and a read-only `SELECT COUNT(*)` command for a table created by PyIceberg or Spark. Write compatibility is not claimed. | -| DuckDB Iceberg | Manual/live harness | RustFS can generate `httpfs` and `iceberg` SQL using an operator-supplied current metadata location. Write and commit compatibility are not claimed. | +| DuckDB Iceberg | Manual/live harness | RustFS can generate the read-only `iceberg_scan` path using an operator-supplied current metadata location and a generic signed Iceberg REST Catalog profile for `/iceberg` or `/_iceberg`. The REST profile disables staged create, post-create metadata updates, multi-table commit, client-side file removal, and purge-on-drop. Write and commit compatibility remain not claimed until repeatable live evidence is automated. | | StarRocks Iceberg REST catalog | Documented, not automated | External catalog read-path reference only. Write compatibility is not claimed. | | Databend | Manual/live harness | RustFS can generate an S3 stage read probe for table data files. RustFS does not claim Databend Iceberg REST Catalog integration yet. | | Snowflake Open Catalog / Iceberg integrations | Generated harness | RustFS can generate an operator-adapted external volume/catalog SQL template. Live RustFS interoperability is not claimed. | @@ -250,6 +250,7 @@ python3 scripts/table-catalog/pyiceberg_smoke.py --print-vendor-profiles python3 scripts/table-catalog/pyiceberg_smoke.py --print-production-readiness python3 scripts/table-catalog/engine_compatibility.py --print-vendor-audit python3 scripts/table-catalog/engine_compatibility.py --print-spark-config +python3 scripts/table-catalog/engine_compatibility.py --print-duckdb-rest-sql python3 scripts/table-catalog/engine_compatibility.py \ --profile aws-s3tables \ --region us-east-1 \ diff --git a/scripts/table-catalog/README.md b/scripts/table-catalog/README.md index 0756c2039..cbeb08b84 100644 --- a/scripts/table-catalog/README.md +++ b/scripts/table-catalog/README.md @@ -241,6 +241,7 @@ python3 scripts/table-catalog/engine_compatibility.py \ --table-bucket analytics \ --print-spark-config python3 scripts/table-catalog/engine_compatibility.py --print-spark-sql --cleanup +python3 scripts/table-catalog/engine_compatibility.py --print-duckdb-rest-sql python3 scripts/table-catalog/engine_compatibility.py --print-live-conformance --cleanup python3 scripts/table-catalog/engine_compatibility.py --print-operations-guide ``` @@ -310,7 +311,7 @@ The smoke test also probes catalog-backed advanced Iceberg surfaces: | PyIceberg | Automated smoke target | create namespace, create table, append, reload, scan, metadata-location, refs, views, maintenance, diagnostics, optional catalog-vended table credentials with exact-prefix data-plane scope probe | | Spark Iceberg REST catalog | Manual/live harness | pinned Spark and Iceberg package inputs, configuration, SQL, run command, expected row count, and cleanup can be generated for a running RustFS endpoint; CI execution is opt-in | | Trino Iceberg REST catalog | Manual/live read probe | generated catalog properties and a read-only SELECT probe for a table created by PyIceberg or Spark; no write compatibility claim yet | -| DuckDB Iceberg | Manual/live read probe | generated httpfs/iceberg SQL using an operator-supplied current metadata location; read-path only | +| DuckDB Iceberg | Manual/live harness | generated metadata-location read SQL plus generic Iceberg REST Catalog attach SQL for `/iceberg` with `s3` signing or `/_iceberg` with `s3tables` signing; write compatibility still requires repeatable live evidence | | StarRocks Iceberg REST catalog | Documented, not automated | external catalog read-path reference only | | Databend | Manual/live S3 stage probe | generated S3 stage read probe for table data files; Iceberg REST catalog integration is not claimed | | Snowflake/Open Catalog integrations | Manual reference probe | generated external volume/catalog SQL template; live RustFS interoperability is not claimed | @@ -503,6 +504,43 @@ RUSTFS_TABLE_CATALOG_CREDENTIAL_TTL_SECONDS=900 The TTL is clamped to the supported short-lived range by the server. +## DuckDB REST Catalog Profile + +DuckDB can read an individual Iceberg table with `iceberg_scan` or attach RustFS +as a generic Iceberg REST Catalog. The metadata-location path remains read-only. +The attached catalog path is the prerequisite for DuckDB writes. + +Generate the canonical RustFS REST Catalog profile: + +```bash +python3 scripts/table-catalog/engine_compatibility.py \ + --endpoint http://127.0.0.1:9000 \ + --warehouse rustfs-s3table-smoke \ + --namespace smoke \ + --table events \ + --rest-path /iceberg \ + --rest-signing-name s3 \ + --print-duckdb-rest-sql +``` + +Generate the compatibility alias profile by changing the last three arguments: + +```bash +python3 scripts/table-catalog/engine_compatibility.py \ + --rest-path /_iceberg \ + --rest-signing-name s3tables \ + --print-duckdb-rest-sql +``` + +The generated `ATTACH` disables staged create, post-create metadata updates, +multi-table commit, client-side file removal, and purge-on-drop. These options +keep DuckDB within RustFS's claimed single-table REST surface. Do not replace +the explicit endpoint with DuckDB `ENDPOINT_TYPE S3_TABLES`; that shortcut is +for AWS S3 Tables endpoint and warehouse shapes. + +This profile is generated conformance input. It does not promote DuckDB write +compatibility until the repeatable live smoke records passing evidence. + ## Spark Manual/Live Harness Spark validation should use the same RustFS endpoint and warehouse bucket as the diff --git a/scripts/table-catalog/engine_compatibility.py b/scripts/table-catalog/engine_compatibility.py index 9f7a241fd..31a64adf4 100644 --- a/scripts/table-catalog/engine_compatibility.py +++ b/scripts/table-catalog/engine_compatibility.py @@ -17,7 +17,7 @@ DEFAULT_SPARK_VERSION = "3.5.4" DEFAULT_ICEBERG_VERSION = "1.7.1" DEFAULT_SCALA_VERSION = "2.12" DEFAULT_TRINO_VERSION = "477" -DEFAULT_DUCKDB_VERSION = "1.3.2" +DEFAULT_DUCKDB_VERSION = "1.5.5" DEFAULT_SNOWFLAKE_CLIENT_VERSION = "operator-recorded" DEFAULT_DATABEND_VERSION = "operator-recorded" DEFAULT_TRINO_SERVER = "http://127.0.0.1:8080" @@ -155,12 +155,13 @@ def engine_compatibility_matrix() -> list[dict[str, Any]]: }, { "client": "DuckDB Iceberg", - "status": "manual-live-read-probe", + "status": "manual-live-harness", "entrypoint": "scripts/table-catalog/engine_compatibility.py --print-live-conformance", "scenarios": [ scenario("metadata-read", "manual-live-probe", "read a supplied Iceberg metadata location through DuckDB iceberg_scan"), - scenario("read-table", "manual-live-probe", "read-path verification only"), - scenario("write-table", "not-claimed", "DuckDB write/commit compatibility is not claimed"), + scenario("catalog-attach", "generated-harness", "attach RustFS as a generic signed Iceberg REST catalog"), + scenario("read-table", "manual-live-probe", "read an existing table through the attached catalog"), + scenario("write-table", "manual-validation-required", "generated SQL does not promote DuckDB write compatibility"), ], }, { @@ -489,8 +490,66 @@ def duckdb_sql_probe( return "\n".join(statements) + "\n" -def duckdb_command() -> str: - return shell_join(["duckdb", "-c", ".read /tmp/rustfs-s3tables-duckdb-read.sql"]) +def duckdb_rest_catalog_sql( + *, + endpoint: str, + warehouse: str, + access_key: str, + secret_key: str, + region: str, + catalog_name: str, + namespace: str, + table: str, + rest_path: str, + rest_signing_name: str, +) -> str: + parsed = re.match(r"^(https?)://(.+)$", normalized_endpoint(endpoint)) + if not parsed: + raise ValueError("DuckDB REST catalog endpoint must include http:// or https://") + scheme, endpoint_without_scheme = parsed.groups() + rest_path = normalized_rest_path(rest_path) + catalog_identifier = quote_double_identifier(catalog_name) + table_identifier = ".".join( + [catalog_identifier, quote_double_identifier(namespace), quote_double_identifier(table)] + ) + statements = [ + "INSTALL httpfs;", + "LOAD httpfs;", + "INSTALL iceberg;", + "LOAD iceberg;", + "CREATE OR REPLACE SECRET rustfs_s3 (", + " TYPE s3,", + " PROVIDER config,", + f" KEY_ID {sql_string(access_key)},", + f" SECRET {sql_string(secret_key)},", + f" REGION {sql_string(region)},", + f" ENDPOINT {sql_string(endpoint_without_scheme)},", + " URL_STYLE 'path',", + f" USE_SSL {'true' if scheme == 'https' else 'false'},", + f" SCOPE {sql_string(f's3://{warehouse}')}", + ");", + f"ATTACH {sql_string(warehouse)} AS {catalog_identifier} (", + " TYPE iceberg,", + f" ENDPOINT {sql_string(f'{normalized_endpoint(endpoint)}{rest_path}')},", + " AUTHORIZATION_TYPE 'sigv4',", + " SECRET 'rustfs_s3',", + f" SIGV4_REGION {sql_string(region)},", + f" SIGV4_SERVICE {sql_string(rest_signing_name)},", + " ACCESS_DELEGATION_MODE 'none',", + " STAGE_CREATE_TABLES false,", + " SKIP_CREATE_TABLE_METADATA_UPDATES true,", + " DISABLE_MULTI_TABLE_COMMIT true,", + " REMOVE_FILES_ON_DELETE false,", + " PURGE_REQUESTED false,", + " SUPPORT_NESTED_NAMESPACES false", + ");", + f"SELECT COUNT(*) AS row_count FROM {table_identifier};", + ] + return "\n".join(statements) + "\n" + + +def duckdb_command(*, sql_file: str = "/tmp/rustfs-s3tables-duckdb-read.sql") -> str: + return shell_join(["duckdb", "-c", f".read {sql_file}"]) def snowflake_sql_template(*, endpoint: str, warehouse: str, rest_path: str, namespace: str, table: str) -> str: @@ -1425,6 +1484,18 @@ def live_conformance_harness( region=region, metadata_location=metadata_location, ) + duckdb_rest_sql = duckdb_rest_catalog_sql( + endpoint=endpoint, + warehouse=warehouse, + access_key=access_key, + secret_key=secret_key, + region=region, + catalog_name=catalog_name, + namespace=namespace, + table=table, + rest_path=rest_path, + rest_signing_name=rest_signing_name, + ) snowflake_sql = snowflake_sql_template( endpoint=endpoint, warehouse=warehouse, @@ -1553,13 +1624,24 @@ def live_conformance_harness( OrderedDict( [ ("name", "DuckDB Iceberg"), - ("status", "manual-live-read-probe"), + ("status", "manual-live-harness"), ("version", duckdb_version), ("metadata_location", metadata_location), ("sql_file", "/tmp/rustfs-s3tables-duckdb-read.sql"), ("sql", duckdb_sql), ("command", duckdb_command()), ("expected", "iceberg_scan returns row_count=2 when metadata_location points at the current Iceberg metadata JSON"), + ("rest_catalog_sql_file", "/tmp/rustfs-s3tables-duckdb-rest.sql"), + ("rest_catalog_sql", duckdb_rest_sql), + ( + "rest_catalog_command", + duckdb_command(sql_file="/tmp/rustfs-s3tables-duckdb-rest.sql"), + ), + ( + "rest_catalog_expected", + "generic Iceberg REST ATTACH returns row_count=2 for an existing RustFS table", + ), + ("rest_catalog_write_compatibility", "manual-live-validation-required"), ("write_compatibility", "not-claimed"), ] ), @@ -1627,6 +1709,7 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace: parser.add_argument("--print-operations-guide", action="store_true") parser.add_argument("--print-spark-config", action="store_true") parser.add_argument("--print-spark-sql", action="store_true") + parser.add_argument("--print-duckdb-rest-sql", action="store_true") return parser.parse_args(argv) @@ -1734,6 +1817,24 @@ def run(args: argparse.Namespace, output: StringIO | None = None) -> None: else: output.write(sql) printed = True + if args.print_duckdb_rest_sql: + sql = duckdb_rest_catalog_sql( + endpoint=args.endpoint, + warehouse=args.warehouse, + access_key=args.access_key, + secret_key=args.secret_key, + region=args.region, + catalog_name=args.catalog_name, + namespace=args.namespace, + table=args.table, + rest_path=args.rest_path or "/iceberg", + rest_signing_name=args.rest_signing_name or "s3", + ) + if output is None: + print(sql, end="") + else: + output.write(sql) + printed = True if not printed: print_json({"engine_compatibility": engine_compatibility_matrix()}, output) diff --git a/scripts/table-catalog/test_engine_compatibility.py b/scripts/table-catalog/test_engine_compatibility.py index 2636d1a21..0d0307bd3 100644 --- a/scripts/table-catalog/test_engine_compatibility.py +++ b/scripts/table-catalog/test_engine_compatibility.py @@ -41,6 +41,12 @@ class EngineCompatibilityTest(unittest.TestCase): self.assertEqual(trino["status"], "manual-live-read-probe") self.assertContainsScenario(trino, "catalog-load", "manual-live-probe") + duckdb = by_client["DuckDB Iceberg"] + self.assertEqual(duckdb["status"], "manual-live-harness") + self.assertContainsScenario(duckdb, "metadata-read", "manual-live-probe") + self.assertContainsScenario(duckdb, "catalog-attach", "generated-harness") + self.assertContainsScenario(duckdb, "write-table", "manual-validation-required") + def test_spark_config_uses_rustfs_rest_catalog_and_s3fileio(self) -> None: config = engine_compatibility.spark_catalog_config( endpoint="http://127.0.0.1:9000", @@ -158,6 +164,70 @@ class EngineCompatibilityTest(unittest.TestCase): table="orders", ) + def test_duckdb_rest_catalog_sql_uses_rustfs_compatibility_options(self) -> None: + sql = engine_compatibility.duckdb_rest_catalog_sql( + endpoint="http://127.0.0.1:9000", + warehouse="rustfs-s3table-smoke", + access_key="rustfsadmin", + secret_key="rustfsadmin", + region="us-east-1", + catalog_name="rustfs", + namespace="smoke", + table="events", + rest_path="/iceberg", + rest_signing_name="s3", + ) + + self.assertIn("CREATE OR REPLACE SECRET rustfs_s3", sql) + self.assertIn("ENDPOINT '127.0.0.1:9000'", sql) + self.assertIn("SCOPE 's3://rustfs-s3table-smoke'", sql) + self.assertIn("ATTACH 'rustfs-s3table-smoke' AS \"rustfs\"", sql) + self.assertIn("ENDPOINT 'http://127.0.0.1:9000/iceberg'", sql) + self.assertIn("SIGV4_SERVICE 's3'", sql) + self.assertIn("STAGE_CREATE_TABLES false", sql) + self.assertIn("SKIP_CREATE_TABLE_METADATA_UPDATES true", sql) + self.assertIn("DISABLE_MULTI_TABLE_COMMIT true", sql) + self.assertIn("REMOVE_FILES_ON_DELETE false", sql) + self.assertIn("PURGE_REQUESTED false", sql) + self.assertIn('FROM "rustfs"."smoke"."events"', sql) + self.assertNotIn("ENDPOINT_TYPE", sql) + + def test_duckdb_rest_catalog_sql_supports_s3tables_alias(self) -> None: + sql = engine_compatibility.duckdb_rest_catalog_sql( + endpoint="https://rustfs.example", + warehouse="analytics", + access_key="access'key", + secret_key="secret'key", + region="us-east-1", + catalog_name="rustfs_compat", + namespace="smoke", + table="events", + rest_path="/_iceberg", + rest_signing_name="s3tables", + ) + + self.assertIn("USE_SSL true", sql) + self.assertIn("ENDPOINT 'rustfs.example'", sql) + self.assertIn("ENDPOINT 'https://rustfs.example/_iceberg'", sql) + self.assertIn("SIGV4_SERVICE 's3tables'", sql) + self.assertIn("KEY_ID 'access''key'", sql) + self.assertIn("SECRET 'secret''key'", sql) + + def test_duckdb_rest_catalog_sql_rejects_endpoint_without_scheme(self) -> None: + with self.assertRaisesRegex(ValueError, "must include http:// or https://"): + engine_compatibility.duckdb_rest_catalog_sql( + endpoint="127.0.0.1:9000", + warehouse="analytics", + access_key="rustfsadmin", + secret_key="rustfsadmin", + region="us-east-1", + catalog_name="rustfs", + namespace="smoke", + table="events", + rest_path="/iceberg", + rest_signing_name="s3", + ) + def test_cli_prints_machine_readable_engine_matrix(self) -> None: payload = engine_compatibility.cli_json(["--print-engine-matrix"]) document = json.loads(payload) @@ -308,6 +378,27 @@ class EngineCompatibilityTest(unittest.TestCase): self.assertEqual(config["spark.sql.catalog.rustfs.uri"], "http://127.0.0.1:9000/_iceberg") self.assertEqual(config["spark.sql.catalog.rustfs.rest.signing-name"], "s3tables") + def test_cli_prints_duckdb_rest_catalog_sql(self) -> None: + sql = engine_compatibility.cli_json( + [ + "--print-duckdb-rest-sql", + "--endpoint", + "http://127.0.0.1:9000", + "--warehouse", + "analytics", + "--catalog-name", + "rustfs_compat", + "--rest-path", + "/_iceberg", + "--rest-signing-name", + "s3tables", + ] + ) + + self.assertIn("ATTACH 'analytics' AS \"rustfs_compat\"", sql) + self.assertIn("ENDPOINT 'http://127.0.0.1:9000/_iceberg'", sql) + self.assertIn("SIGV4_SERVICE 's3tables'", sql) + def test_live_conformance_harness_pins_clients_and_records_commands(self) -> None: harness = engine_compatibility.live_conformance_harness( endpoint="http://127.0.0.1:9000", @@ -359,10 +450,15 @@ class EngineCompatibilityTest(unittest.TestCase): self.assertEqual(trino["write_compatibility"], "not-claimed") duckdb = by_client["DuckDB Iceberg"] - self.assertEqual(duckdb["status"], "manual-live-read-probe") + self.assertEqual(duckdb["status"], "manual-live-harness") + self.assertEqual(duckdb["version"], "1.5.5") self.assertIn("LOAD httpfs", duckdb["sql"]) self.assertIn("LOAD iceberg", duckdb["sql"]) self.assertIn("iceberg_scan", duckdb["sql"]) + self.assertIn("ATTACH 'rustfs-s3table-smoke'", duckdb["rest_catalog_sql"]) + self.assertIn("STAGE_CREATE_TABLES false", duckdb["rest_catalog_sql"]) + self.assertIn("SKIP_CREATE_TABLE_METADATA_UPDATES true", duckdb["rest_catalog_sql"]) + self.assertEqual(duckdb["rest_catalog_write_compatibility"], "manual-live-validation-required") self.assertEqual(duckdb["write_compatibility"], "not-claimed") snowflake = by_client["Snowflake Open Catalog / Iceberg integrations"]