From ced74b4b1d140fa9f8ff7a770da47e80f40bd7cb Mon Sep 17 00:00:00 2001 From: Jax Liu Date: Thu, 24 Sep 2026 16:12:27 +0800 Subject: [PATCH] deps(core): upgrade DataFusion to 55.1 parquet 58 pins thrift ^0.17, which carries CVE-2026-43868 (fixed in thrift 0.23). parquet 59 drops the thrift dependency, and DataFusion 55 is the first release on arrow/parquet 59, so the fix requires this upgrade. API migrations: TableReference moved out of datafusion::sql, as_any() is no longer part of the UDF/TableProvider traits, Cast/TryCast carry a field instead of a data type, and ContextProvider/Expr gained higher-order function support. Behaviour kept as before: - IN lists keep the string-preferring coercion. DataFusion now coerces them numerically, which would emit `CAST(str_col AS BIGINT) IN (1, 2)` and fail in the target database on non-numeric values. - UNION and CASE WHEN use the upstream split of comparison_coercion into type_union_coercion / get_coerce_type_for_case_when, matching their prior semantics. - wren-core-py lists only scalar/aggregate/window functions, since information_schema.routines now also reports table functions. The wasm release profile now uses fat LTO with one codegen unit; without it the binary grows past the 15 MB gzip budget checked in CI. Co-Authored-By: Claude Opus 5.5 --- .claude/CLAUDE.md | 6 +- core/wren-core-py/Cargo.lock | 585 +++++++++-------- core/wren-core-py/Cargo.toml | 2 +- core/wren-core-py/src/context.rs | 3 + core/wren-core-py/tests/test_modeling_core.py | 4 +- core/wren-core-wasm/.claude/CLAUDE.md | 12 +- core/wren-core-wasm/Cargo.lock | 591 ++++++++++-------- core/wren-core-wasm/Cargo.toml | 14 +- core/wren-core/.claude/CLAUDE.md | 4 +- core/wren-core/Cargo.toml | 6 +- .../logical_plan/analyze/access_control.rs | 22 +- .../src/logical_plan/analyze/model_anlayze.rs | 2 +- .../logical_plan/analyze/model_generation.rs | 2 +- .../core/src/logical_plan/analyze/scope.rs | 2 +- .../optimize/simplify_timestamp.rs | 8 +- .../logical_plan/optimize/type_coercion.rs | 76 ++- core/wren-core/core/src/logical_plan/utils.rs | 7 +- core/wren-core/core/src/mdl/context.rs | 7 +- .../core/src/mdl/function/remote_function.rs | 13 - .../core/src/mdl/function/scalar/to_char.rs | 6 - core/wren-core/core/src/mdl/lineage.rs | 4 +- core/wren-core/core/src/mdl/mod.rs | 69 +- .../sqllogictest/test_files/model.slt | 2 +- .../sqllogictest/test_files/tpch/q22.slt.part | 2 +- 24 files changed, 837 insertions(+), 612 deletions(-) diff --git a/.claude/CLAUDE.md b/.claude/CLAUDE.md index 55a3449dd..e48f62e58 100644 --- a/.claude/CLAUDE.md +++ b/.claude/CLAUDE.md @@ -4,7 +4,7 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co ## Project Overview -Wren is an open-source semantic engine for MCP clients and AI agents. It translates SQL queries through a semantic layer (MDL — Modeling Definition Language) and executes them against 22+ data sources (PostgreSQL, BigQuery, Snowflake, Spark, etc.). The Rust engine is powered by Apache DataFusion (upstream, crates.io v53). +Wren is an open-source semantic engine for MCP clients and AI agents. It translates SQL queries through a semantic layer (MDL — Modeling Definition Language) and executes them against 22+ data sources (PostgreSQL, BigQuery, Snowflake, Spark, etc.). The Rust engine is powered by Apache DataFusion (upstream, crates.io v55). The previous WrenAI services (`wren-ui/`, `wren-ai-service/`, `wren-launcher/`, `docker/`, `deployment/`) were moved to the `legacy/v1` branch (tag `v1-final`) as of the wren-engine import. Active development is focused on the Open Context Engine. @@ -83,7 +83,7 @@ Uses `uv` (not Poetry). `pyproject.toml` uses `hatchling` as build backend. Opti SQL query → wren CLI / wren-core-py → wren-core (Rust): MDL analysis → logical plan → optimization - → DataFusion (query planning, upstream crates.io v53) + → DataFusion (query planning, upstream crates.io v55) → connector-specific SQL (Ibis / sqlglot) → native execution on the target data source ``` @@ -110,7 +110,7 @@ SQL query - **Commits**: Conventional commits (`feat:`, `fix:`, `chore:`, `refactor:`, `test:`, `docs:`, `perf:`, `deps:`). Releases are automated via release-please with independent release lines per module. - **Rust**: format with `cargo fmt`, lint with `clippy -D warnings`, TOML with `taplo`. - **Python**: format and lint with `ruff` (line-length 88, target Python 3.11). Both `core/wren-core-py` and `core/wren` use uv. -- **DataFusion**: upstream `datafusion` v53 from crates.io (no longer the Canner fork). +- **DataFusion**: upstream `datafusion` v55 from crates.io (no longer the Canner fork). - **Snapshot testing**: wren-core uses `insta` for Rust snapshot tests. - **CI**: Per-module path-filtered workflows trigger only on changes inside that module. diff --git a/core/wren-core-py/Cargo.lock b/core/wren-core-py/Cargo.lock index e35abf463..6e856b24f 100644 --- a/core/wren-core-py/Cargo.lock +++ b/core/wren-core-py/Cargo.lock @@ -140,9 +140,9 @@ checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" [[package]] name = "arrow" -version = "58.1.0" +version = "59.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d441fdda254b65f3e9025910eb2c2066b6295d9c8ed409522b8d2ace1ff8574c" +checksum = "7c14b3d39f306bc28fd639d59f06e17a0f377d0021e1b7e9054e4d6fedc98774" dependencies = [ "arrow-arith", "arrow-array", @@ -161,9 +161,9 @@ dependencies = [ [[package]] name = "arrow-arith" -version = "58.1.0" +version = "59.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ced5406f8b720cc0bc3aa9cf5758f93e8593cda5490677aa194e4b4b383f9a59" +checksum = "ce2961626677665b2195eb59242af4c7befe7b8737ca2050295389362380104e" dependencies = [ "arrow-array", "arrow-buffer", @@ -175,9 +175,9 @@ dependencies = [ [[package]] name = "arrow-array" -version = "58.1.0" +version = "59.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "772bd34cacdda8baec9418d80d23d0fb4d50ef0735685bd45158b83dfeb6e62d" +checksum = "1e5f6adeffdf587d7a31db5d2266189624b526730cd3627f9ff9fedae97ad584" dependencies = [ "ahash", "arrow-buffer", @@ -186,7 +186,7 @@ dependencies = [ "chrono", "chrono-tz", "half", - "hashbrown 0.16.1", + "hashbrown 0.17.1", "num-complex", "num-integer", "num-traits", @@ -194,21 +194,21 @@ dependencies = [ [[package]] name = "arrow-buffer" -version = "58.1.0" +version = "59.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "898f4cf1e9598fdb77f356fdf2134feedfd0ee8d5a4e0a5f573e7d0aec16baa4" +checksum = "097d193003ce7995d5d087089069ec2a6e0187faf5a6f8c9f38af2645d987182" dependencies = [ "bytes", "half", - "num-bigint", + "num-bigint 0.5.1", "num-traits", ] [[package]] name = "arrow-cast" -version = "58.1.0" +version = "59.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0127816c96533d20fc938729f48c52d3e48f99717e7a0b5ade77d742510736d" +checksum = "635c9c635668ad26adf76cce8fb276c4be7cf06e63bd516de7da514f9680ee53" dependencies = [ "arrow-array", "arrow-buffer", @@ -217,7 +217,7 @@ dependencies = [ "arrow-schema", "arrow-select", "atoi", - "base64", + "base64 0.23.1", "chrono", "comfy-table", "half", @@ -228,9 +228,9 @@ dependencies = [ [[package]] name = "arrow-csv" -version = "58.1.0" +version = "59.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca025bd0f38eeecb57c2153c0123b960494138e6a957bbda10da2b25415209fe" +checksum = "4c2ebf8d631e79b02c16cf5ae860561272c26024ec88fce389a56aaddd558e86" dependencies = [ "arrow-array", "arrow-cast", @@ -243,9 +243,9 @@ dependencies = [ [[package]] name = "arrow-data" -version = "58.1.0" +version = "59.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42d10beeab2b1c3bb0b53a00f7c944a178b622173a5c7bcabc3cb45d90238df4" +checksum = "9ba2f832eaeca24b8f26143dba750e42ee4ab51cf7d65e701ca9607cfda9f358" dependencies = [ "arrow-buffer", "arrow-schema", @@ -256,9 +256,9 @@ dependencies = [ [[package]] name = "arrow-ipc" -version = "58.1.0" +version = "59.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "609a441080e338147a84e8e6904b6da482cefb957c5cdc0f3398872f69a315d0" +checksum = "dcc41681ea80f521df14c36725b74d4c60702c47f0793af2be469c04527e2599" dependencies = [ "arrow-array", "arrow-buffer", @@ -267,19 +267,21 @@ dependencies = [ "arrow-select", "flatbuffers", "lz4_flex", + "zstd", ] [[package]] name = "arrow-json" -version = "58.1.0" +version = "59.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ead0914e4861a531be48fe05858265cf854a4880b9ed12618b1d08cba9bebc8" +checksum = "a2f57d7a81969f24ccf80809587b76c09897e6f829d2d65a5976bfb3218851f1" dependencies = [ "arrow-array", "arrow-buffer", "arrow-cast", - "arrow-data", + "arrow-ord", "arrow-schema", + "arrow-select", "chrono", "half", "indexmap 2.14.0", @@ -295,9 +297,9 @@ dependencies = [ [[package]] name = "arrow-ord" -version = "58.1.0" +version = "59.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "763a7ba279b20b52dad300e68cfc37c17efa65e68623169076855b3a9e941ca5" +checksum = "2c900759f3bd8354fd4196bc4403eee846894dc2adf66b4225472006a0bf18c5" dependencies = [ "arrow-array", "arrow-buffer", @@ -308,9 +310,9 @@ dependencies = [ [[package]] name = "arrow-row" -version = "58.1.0" +version = "59.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e14fe367802f16d7668163ff647830258e6e0aeea9a4d79aaedf273af3bdcd3e" +checksum = "f4c6425032e28266e3fc4ff680805e57e670d6ea92473043f3e65b7ed6ac79f2" dependencies = [ "arrow-array", "arrow-buffer", @@ -321,9 +323,9 @@ dependencies = [ [[package]] name = "arrow-schema" -version = "58.1.0" +version = "59.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c30a1365d7a7dc50cc847e54154e6af49e4c4b0fddc9f607b687f29212082743" +checksum = "10fab8d4563491417ba801fab29d205104d20d4bdf37bda6cd1cf425cff598cd" dependencies = [ "serde_core", "serde_json", @@ -331,9 +333,9 @@ dependencies = [ [[package]] name = "arrow-select" -version = "58.1.0" +version = "59.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78694888660a9e8ac949853db393af2a8b8fc82c19ce333132dfa2e72cc1a7fe" +checksum = "fc58569193c2525915f3cc6310edba3792f1200f65d6e9ed330aa33e691493b8" dependencies = [ "ahash", "arrow-array", @@ -345,9 +347,9 @@ dependencies = [ [[package]] name = "arrow-string" -version = "58.1.0" +version = "59.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61e04a01f8bb73ce54437514c5fd3ee2aa3e8abe4c777ee5cc55853b1652f79e" +checksum = "2e0813f3c35c1cfea65e14c20a953440f7783c088b7ad2d0db162ccdeefcec14" dependencies = [ "arrow-array", "arrow-buffer", @@ -368,7 +370,7 @@ checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -392,6 +394,12 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "base64" +version = "0.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5" + [[package]] name = "bigdecimal" version = "0.4.10" @@ -400,7 +408,7 @@ checksum = "4d6867f1565b3aad85681f1015055b087fcfd840d6aeee6eee7f2da317603695" dependencies = [ "autocfg", "libm", - "num-bigint", + "num-bigint 0.4.6", "num-integer", "num-traits", ] @@ -417,7 +425,7 @@ version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "46502ad458c9a52b69d4d4d32775c788b7a1b85e8bc9d482d92250fc0e3f8efe" dependencies = [ - "digest", + "digest 0.10.7", ] [[package]] @@ -431,7 +439,7 @@ dependencies = [ "cc", "cfg-if", "constant_time_eq", - "cpufeatures 0.3.0", + "cpufeatures", ] [[package]] @@ -443,6 +451,15 @@ dependencies = [ "generic-array", ] +[[package]] +name = "block-buffer" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa" +dependencies = [ + "hybrid-array", +] + [[package]] name = "brotli" version = "8.0.2" @@ -479,12 +496,6 @@ version = "3.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5d20789868f4b01b2f2caec9f5c4e0213b41e3e5702a50157d699ae31ced2fcb" -[[package]] -name = "byteorder" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" - [[package]] name = "bytes" version = "1.11.1" @@ -511,9 +522,9 @@ checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" [[package]] name = "chrono" -version = "0.4.44" +version = "0.4.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c673075a2e0e5f4a1dde27ce9dee1ea4558c7ffe648f576438a20ca1d2acc4b0" +checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" dependencies = [ "iana-time-zone", "num-traits", @@ -547,6 +558,12 @@ dependencies = [ "unicode-width", ] +[[package]] +name = "const-oid" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c" + [[package]] name = "const-random" version = "0.1.18" @@ -579,15 +596,6 @@ version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" -[[package]] -name = "cpufeatures" -version = "0.2.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" -dependencies = [ - "libc", -] - [[package]] name = "cpufeatures" version = "0.3.0" @@ -619,6 +627,15 @@ dependencies = [ "typenum", ] +[[package]] +name = "crypto-common" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" +dependencies = [ + "hybrid-array", +] + [[package]] name = "csv" version = "1.4.0" @@ -660,7 +677,7 @@ dependencies = [ "proc-macro2", "quote", "strsim", - "syn", + "syn 2.0.117", ] [[package]] @@ -671,14 +688,14 @@ checksum = "ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d" dependencies = [ "darling_core", "quote", - "syn", + "syn 2.0.117", ] [[package]] name = "dashmap" -version = "6.1.0" +version = "6.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5041cc499144891f3790297212f32a74fb938e5136a14943f338ef9e0ae276cf" +checksum = "e6361d5c062261c78a176addb82d4c821ae42bed6089de0e12603cd25de2059c" dependencies = [ "cfg-if", "crossbeam-utils", @@ -690,14 +707,13 @@ dependencies = [ [[package]] name = "datafusion" -version = "53.0.0" +version = "55.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de9f8117889ba9503440f1dd79ebab32ba52ccf1720bb83cd718a29d4edc0d16" +checksum = "14313430439ef858ed5f52ca9c840e9eea9844da34d3716d05a476449634da5d" dependencies = [ "arrow", "arrow-schema", "async-trait", - "bytes", "chrono", "datafusion-catalog", "datafusion-catalog-listing", @@ -725,14 +741,13 @@ dependencies = [ "datafusion-session", "datafusion-sql", "futures", - "itertools", + "indexmap 2.14.0", + "itertools 0.15.0", "log", "object_store", "parking_lot", "parquet", - "rand", - "regex", - "sqlparser 0.61.0", + "sqlparser 0.62.0", "tempfile", "tokio", "url", @@ -741,9 +756,9 @@ dependencies = [ [[package]] name = "datafusion-catalog" -version = "53.0.0" +version = "55.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be893b73a13671f310ffcc8da2c546b81efcc54c22e0382c0a28aa3537017137" +checksum = "2c0756ebc770bcebae366dfd273dad50935f6103b4d212253cb17e2e53c5427f" dependencies = [ "arrow", "async-trait", @@ -757,7 +772,7 @@ dependencies = [ "datafusion-physical-plan", "datafusion-session", "futures", - "itertools", + "itertools 0.15.0", "log", "object_store", "parking_lot", @@ -766,9 +781,9 @@ dependencies = [ [[package]] name = "datafusion-catalog-listing" -version = "53.0.0" +version = "55.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "830487b51ed83807d6b32d6325f349c3144ae0c9bf772cf2a712db180c31d5e6" +checksum = "bbde2961698705d8402f0f382f08876fc6f1126ffde790ef17833cf0ab87cd59" dependencies = [ "arrow", "async-trait", @@ -782,40 +797,43 @@ dependencies = [ "datafusion-physical-expr-common", "datafusion-physical-plan", "futures", - "itertools", + "itertools 0.15.0", "log", "object_store", + "percent-encoding", ] [[package]] name = "datafusion-common" -version = "53.0.0" +version = "55.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d7663f3af955292f8004e74bcaf8f7ea3d66cc38438749615bb84815b61a293" +checksum = "a6e06fe711ff815c8cf3143158547aea599702fd43f76f5866c70d5a5b3693a0" dependencies = [ - "ahash", "arrow", "arrow-ipc", + "arrow-schema", "chrono", + "foldhash 0.2.0", "half", - "hashbrown 0.16.1", + "hashbrown 0.17.1", "indexmap 2.14.0", - "itertools", + "itertools 0.15.0", "libc", "log", + "num-traits", "object_store", "parquet", - "paste", - "sqlparser 0.61.0", + "sqlparser 0.62.0", "tokio", + "uuid", "web-time", ] [[package]] name = "datafusion-common-runtime" -version = "53.0.0" +version = "55.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f590205c7e32fe1fea48dd53ffb406e56ae0e7a062213a3ac848db8771641bd" +checksum = "f7a5107acff4e3ed50a7732acec06bba40e74ab49df021397ed250d60f04e42a" dependencies = [ "futures", "log", @@ -824,9 +842,9 @@ dependencies = [ [[package]] name = "datafusion-datasource" -version = "53.0.0" +version = "55.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fde1e030a9dc87b743c806fbd631f5ecfa2ccaa4ffb61fa19144a07fea406b79" +checksum = "f69f45d2d700785be8df9d4b44410e1c8e4a856f10cc728b418a1066c0b594fb" dependencies = [ "arrow", "async-trait", @@ -843,9 +861,10 @@ dependencies = [ "datafusion-session", "futures", "glob", - "itertools", + "itertools 0.15.0", "log", "object_store", + "parking_lot", "rand", "tokio", "url", @@ -853,9 +872,9 @@ dependencies = [ [[package]] name = "datafusion-datasource-arrow" -version = "53.0.0" +version = "55.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "331ebae7055dc108f9b54994b93dff91f3a17445539efe5b74e89264f7b36e15" +checksum = "81650af64894ef34b46d2ef818ef244b40a81d90818f3bc642bf8fbf6758f8e0" dependencies = [ "arrow", "arrow-ipc", @@ -870,16 +889,16 @@ dependencies = [ "datafusion-physical-plan", "datafusion-session", "futures", - "itertools", + "itertools 0.15.0", "object_store", "tokio", ] [[package]] name = "datafusion-datasource-csv" -version = "53.0.0" +version = "55.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e0d475088325e2986876aa27bb30d0574f72a22955a527d202f454681d55c5c" +checksum = "b5e2519bed59a6907c79298b9c21fe807bbc34e4ff8414577d699b0741083593" dependencies = [ "arrow", "async-trait", @@ -900,9 +919,9 @@ dependencies = [ [[package]] name = "datafusion-datasource-json" -version = "53.0.0" +version = "55.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea1520d81f31770f3ad6ee98b391e75e87a68a5bb90de70064ace5e0a7182fe8" +checksum = "635e61290e171597f6e50427e0cb486eda468ab47e880fa24d15bbb0fa24ac7e" dependencies = [ "arrow", "async-trait", @@ -917,18 +936,18 @@ dependencies = [ "datafusion-session", "futures", "object_store", - "serde_json", "tokio", "tokio-stream", ] [[package]] name = "datafusion-datasource-parquet" -version = "53.0.0" +version = "55.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95be805d0742ab129720f4c51ad9242cd872599cdb076098b03f061fcdc7f946" +checksum = "07cd518f85b3a027d913047b4ab0811111a2b734fdf84a51b78234455f10d8fa" dependencies = [ "arrow", + "arrow-schema", "async-trait", "bytes", "datafusion-common", @@ -936,6 +955,7 @@ dependencies = [ "datafusion-datasource", "datafusion-execution", "datafusion-expr", + "datafusion-functions", "datafusion-functions-aggregate-common", "datafusion-physical-expr", "datafusion-physical-expr-adapter", @@ -944,7 +964,7 @@ dependencies = [ "datafusion-pruning", "datafusion-session", "futures", - "itertools", + "itertools 0.15.0", "log", "object_store", "parking_lot", @@ -954,20 +974,20 @@ dependencies = [ [[package]] name = "datafusion-doc" -version = "53.0.0" +version = "55.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c93ad9e37730d2c7196e68616f3f2dd3b04c892e03acd3a8eeca6e177f3c06a" +checksum = "d44c5919b560862dd1cb20a5c75b3e48c3d9e49719c91d78da5c38eaad5495c4" [[package]] name = "datafusion-execution" -version = "53.0.0" +version = "55.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9437d3cd5d363f9319f8122182d4d233427de79c7eb748f23054c9aaa0fdd8df" +checksum = "7620ad535af8730186bdec05d720e40d9a4584b40c7eb75f72c43aa7978e5484" dependencies = [ "arrow", "arrow-buffer", "async-trait", - "chrono", + "bytes", "dashmap", "datafusion-common", "datafusion-expr", @@ -976,18 +996,22 @@ dependencies = [ "log", "object_store", "parking_lot", + "pin-project-lite", "rand", "tempfile", + "tokio", + "tokio-util", "url", ] [[package]] name = "datafusion-expr" -version = "53.0.0" +version = "55.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67164333342b86521d6d93fa54081ee39839894fb10f7a700c099af96d7552cf" +checksum = "ea229af8b73e566fc6ebb1c8dd42ab250f8855c5304bd173f4b50ead78b64304" dependencies = [ "arrow", + "arrow-schema", "async-trait", "chrono", "datafusion-common", @@ -997,34 +1021,32 @@ dependencies = [ "datafusion-functions-window-common", "datafusion-physical-expr-common", "indexmap 2.14.0", - "itertools", - "paste", + "itertools 0.15.0", "serde_json", - "sqlparser 0.61.0", + "sqlparser 0.62.0", ] [[package]] name = "datafusion-expr-common" -version = "53.0.0" +version = "55.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab05fdd00e05d5a6ee362882546d29d6d3df43a6c55355164a7fbee12d163bc9" +checksum = "6a18baa840f77760dcc9eaad281329bc57536c5c39e65e8649596f2a51216be7" dependencies = [ "arrow", "datafusion-common", "indexmap 2.14.0", - "itertools", - "paste", + "itertools 0.15.0", ] [[package]] name = "datafusion-functions" -version = "53.0.0" +version = "55.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04fb863482d987cf938db2079e07ab0d3bb64595f28907a6c2f8671ad71cca7e" +checksum = "763bbfeec40b55a8a961d7349c0fcfcec4f3f66cc234d6295250b95371b85e34" dependencies = [ "arrow", "arrow-buffer", - "base64", + "base64 0.23.1", "blake2", "blake3", "chrono", @@ -1035,8 +1057,9 @@ dependencies = [ "datafusion-expr", "datafusion-expr-common", "datafusion-macros", + "datafusion-physical-expr-common", "hex", - "itertools", + "itertools 0.15.0", "log", "md-5", "memchr", @@ -1044,17 +1067,15 @@ dependencies = [ "rand", "regex", "sha2", - "unicode-segmentation", "uuid", ] [[package]] name = "datafusion-functions-aggregate" -version = "53.0.0" +version = "55.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "829856f4e14275fb376c104f27cbf3c3b57a9cfe24885d98677525f5e43ce8d6" +checksum = "46c0c2b2e856883ecf6a4fa8b23e7c6041183af8a68d0583dc95d1d21e4eaee0" dependencies = [ - "ahash", "arrow", "datafusion-common", "datafusion-doc", @@ -1065,18 +1086,17 @@ dependencies = [ "datafusion-physical-expr", "datafusion-physical-expr-common", "half", + "hashbrown 0.17.1", "log", "num-traits", - "paste", ] [[package]] name = "datafusion-functions-aggregate-common" -version = "53.0.0" +version = "55.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08af79cc3d2aa874a362fb97decfcbd73d687190cb096f16a6c85a7780cce311" +checksum = "e35e42c87dfbff06934ea0e99cfed3440ac614c3f3bfb8c33be35ca0587d4140" dependencies = [ - "ahash", "arrow", "datafusion-common", "datafusion-expr-common", @@ -1085,9 +1105,9 @@ dependencies = [ [[package]] name = "datafusion-functions-nested" -version = "53.0.0" +version = "55.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "465ae3368146d49c2eda3e2c0ef114424c87e8a6b509ab34c1026ace6497e790" +checksum = "08079b0604cdcc43c7d6cf006cfd5d3ed9c235ade0c1ebe91085a62f875603df" dependencies = [ "arrow", "arrow-ord", @@ -1101,34 +1121,34 @@ dependencies = [ "datafusion-functions-aggregate-common", "datafusion-macros", "datafusion-physical-expr-common", - "hashbrown 0.16.1", - "itertools", + "hashbrown 0.17.1", + "itertools 0.15.0", "itoa", "log", - "paste", + "memchr", ] [[package]] name = "datafusion-functions-table" -version = "53.0.0" +version = "55.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6156e6b22fcf1784112fc0173f3ae6e78c8fdb4d3ed0eace9543873b437e2af6" +checksum = "fe2070ecf5638a93c81c8babc7858339ef52165a9f54da1411afd69adf7d85ce" dependencies = [ "arrow", "async-trait", "datafusion-catalog", "datafusion-common", "datafusion-expr", + "datafusion-physical-expr", "datafusion-physical-plan", "parking_lot", - "paste", ] [[package]] name = "datafusion-functions-window" -version = "53.0.0" +version = "55.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca7baec14f866729012efb89011a6973f3a346dc8090c567bfcd328deff551c1" +checksum = "0e9a164ebe1998c5c3f9e15c9a0c4e6b599a0c8739b5471bfb089752441963f1" dependencies = [ "arrow", "datafusion-common", @@ -1139,14 +1159,13 @@ dependencies = [ "datafusion-physical-expr", "datafusion-physical-expr-common", "log", - "paste", ] [[package]] name = "datafusion-functions-window-common" -version = "53.0.0" +version = "55.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "159228c3280d342658466bb556dc24de30047fe1d7e559dc5d16ccc5324166f9" +checksum = "98965b183f92201c16798994820ae032649c4821e089c4c80e7d0f4485d41df5" dependencies = [ "datafusion-common", "datafusion-physical-expr-common", @@ -1154,20 +1173,20 @@ dependencies = [ [[package]] name = "datafusion-macros" -version = "53.0.0" +version = "55.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5427e5da5edca4d21ea1c7f50e1c9421775fe33d7d5726e5641a833566e7578" +checksum = "3a711bd5a06ec85d9683e142d28a98ad0679a7220669f5c1b8589e57e9b29e46" dependencies = [ "datafusion-doc", "quote", - "syn", + "syn 3.0.6", ] [[package]] name = "datafusion-optimizer" -version = "53.0.0" +version = "55.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89099eefcd5b223ec685c36a41d35c69239236310d71d339f2af0fa4383f3f46" +checksum = "ff3256b62ff44616ec83c7f04741d21124970833bb377ed56d9abc5ec6f707ec" dependencies = [ "arrow", "chrono", @@ -1176,7 +1195,7 @@ dependencies = [ "datafusion-expr-common", "datafusion-physical-expr", "indexmap 2.14.0", - "itertools", + "itertools 0.15.0", "log", "regex", "regex-syntax", @@ -1184,32 +1203,31 @@ dependencies = [ [[package]] name = "datafusion-physical-expr" -version = "53.0.0" +version = "55.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f222df5195d605d79098ef37bdd5323bff0131c9d877a24da6ec98dfca9fe36" +checksum = "c1e37a25b5aa1f3a1db114f0ee3605bf01639b83a831dcde35119ee61f0b11d6" dependencies = [ - "ahash", "arrow", + "arrow-schema", "datafusion-common", "datafusion-expr", "datafusion-expr-common", "datafusion-functions-aggregate-common", "datafusion-physical-expr-common", "half", - "hashbrown 0.16.1", + "hashbrown 0.17.1", "indexmap 2.14.0", - "itertools", + "itertools 0.15.0", "parking_lot", - "paste", "petgraph", "tokio", ] [[package]] name = "datafusion-physical-expr-adapter" -version = "53.0.0" +version = "55.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40838625d63d9c12549d81979db3dd675d159055eb9135009ba272ab0e8d0f64" +checksum = "3a4d12787d42e88b26b0d3ec006192fa23ceb832ef2a9e87439ce820a8e038cc" dependencies = [ "arrow", "datafusion-common", @@ -1217,31 +1235,31 @@ dependencies = [ "datafusion-functions", "datafusion-physical-expr", "datafusion-physical-expr-common", - "itertools", + "itertools 0.15.0", ] [[package]] name = "datafusion-physical-expr-common" -version = "53.0.0" +version = "55.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eacbcc4cfd502558184ed58fa3c72e775ec65bf077eef5fd2b3453db676f893c" +checksum = "d13bf7f50fd902f5afc46d50759f5185b5620c40b24054f80f7c648897d3e5a6" dependencies = [ - "ahash", "arrow", "chrono", "datafusion-common", "datafusion-expr-common", - "hashbrown 0.16.1", + "hashbrown 0.17.1", "indexmap 2.14.0", - "itertools", + "itertools 0.15.0", "parking_lot", + "pin-project", ] [[package]] name = "datafusion-physical-optimizer" -version = "53.0.0" +version = "55.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d501d0e1d0910f015677121601ac177ec59272ef5c9324d1147b394988f40941" +checksum = "11fbe7e6665072957d44ed9587676e17a1f9ab3acfadfc287dac23750585f18a" dependencies = [ "arrow", "datafusion-common", @@ -1252,20 +1270,23 @@ dependencies = [ "datafusion-physical-expr-common", "datafusion-physical-plan", "datafusion-pruning", - "itertools", + "datafusion-session", + "itertools 0.15.0", ] [[package]] name = "datafusion-physical-plan" -version = "53.0.0" +version = "55.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "463c88ad6f1ecab1810f4c9f046898bee035b370137eb79b2b2db925e270631d" +checksum = "1265d58e5bce07d154e642a51ff43033b576a6ae40d50a29ac2b9f311004eb52" dependencies = [ - "ahash", "arrow", + "arrow-data", + "arrow-ipc", "arrow-ord", "arrow-schema", "async-trait", + "bytes", "datafusion-common", "datafusion-common-runtime", "datafusion-execution", @@ -1277,21 +1298,22 @@ dependencies = [ "datafusion-physical-expr-common", "futures", "half", - "hashbrown 0.16.1", + "hashbrown 0.17.1", "indexmap 2.14.0", - "itertools", + "itertools 0.15.0", "log", "num-traits", "parking_lot", "pin-project-lite", + "serde_json", "tokio", ] [[package]] name = "datafusion-pruning" -version = "53.0.0" +version = "55.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2857618a0ecbd8cd0cf29826889edd3a25774ec26b2995fc3862095c95d88fc6" +checksum = "74b06b333405c05015836ed36cea02cc29d1e2ee8fb414a77532d8346fb56492" dependencies = [ "arrow", "datafusion-common", @@ -1300,16 +1322,16 @@ dependencies = [ "datafusion-physical-expr", "datafusion-physical-expr-common", "datafusion-physical-plan", - "itertools", "log", ] [[package]] name = "datafusion-session" -version = "53.0.0" +version = "55.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef8637e35022c5c775003b3ab1debc6b4a8f0eb41b069bdd5475dd3aa93f6eba" +checksum = "e7616b17b1d2fee3e9b13c651eac91b768d153265e6ba88e06e0b4af7883f69c" dependencies = [ + "arrow-schema", "async-trait", "datafusion-common", "datafusion-execution", @@ -1320,9 +1342,9 @@ dependencies = [ [[package]] name = "datafusion-sql" -version = "53.0.0" +version = "55.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12d9e9f16a1692a11c94bcc418191fa15fd2b4d72a0c1a0c607db93c0b84dd81" +checksum = "5b4848be66dd008d55adbe1dbc8cbd6a2502588ad0c0b8483bab0f43fd4495d0" dependencies = [ "arrow", "bigdecimal", @@ -1333,7 +1355,7 @@ dependencies = [ "indexmap 2.14.0", "log", "regex", - "sqlparser 0.61.0", + "sqlparser 0.62.0", ] [[package]] @@ -1352,11 +1374,22 @@ version = "0.10.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ - "block-buffer", - "crypto-common", + "block-buffer 0.10.4", + "crypto-common 0.1.7", "subtle", ] +[[package]] +name = "digest" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" +dependencies = [ + "block-buffer 0.12.1", + "const-oid", + "crypto-common 0.2.2", +] + [[package]] name = "displaydoc" version = "0.2.5" @@ -1365,7 +1398,7 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -1534,7 +1567,7 @@ checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -1659,21 +1692,15 @@ dependencies = [ [[package]] name = "hashbrown" -version = "0.16.1" +version = "0.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" dependencies = [ "allocator-api2", "equivalent", "foldhash 0.2.0", ] -[[package]] -name = "hashbrown" -version = "0.17.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f467dd6dccf739c208452f8014c75c18bb8301b050ad1cfb27153803edb0f51" - [[package]] name = "heck" version = "0.5.0" @@ -1702,6 +1729,15 @@ version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "135b12329e5e3ce057a9f972339ea52bc954fe1e9358ef27f95e89716fbc5424" +[[package]] +name = "hybrid-array" +version = "0.4.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3944cf8cf766b40e2a1a333ee5e9b563f854d5fa49d6a8ca2764e97c6eddb214" +dependencies = [ + "typenum", +] + [[package]] name = "iana-time-zone" version = "0.1.65" @@ -1859,17 +1895,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" dependencies = [ "equivalent", - "hashbrown 0.17.0", + "hashbrown 0.17.1", "serde", "serde_core", ] -[[package]] -name = "integer-encoding" -version = "3.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8bb03732005da905c88227371639bf1ad885cc712789c011c31c5fb3ab3ccf02" - [[package]] name = "is_terminal_polyfill" version = "1.70.2" @@ -1885,6 +1915,15 @@ dependencies = [ "either", ] +[[package]] +name = "itertools" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b4baf93f58d4425749ca49a51c50ebab072c5df6994d08fed93541c331481dc" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.18" @@ -1912,7 +1951,7 @@ checksum = "2a8c8b344124222efd714b73bb41f8b5120b27a7cc1c75593a6ff768d9d05aa4" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -2002,9 +2041,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.184" +version = "0.2.189" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48f5d2a454e16a5ea0f4ced81bd44e4cfc7bd3a507b61887c99fd3538b28e4af" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" [[package]] name = "libm" @@ -2041,28 +2080,28 @@ checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" [[package]] name = "lz4_flex" -version = "0.13.0" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db9a0d582c2874f68138a16ce1867e0ffde6c0bb0a0df85e1f36d04146db488a" +checksum = "ecbdfe44b1bd960b68170b417450a628c43f7cf56bb3c5317e61cb230ee7f226" dependencies = [ "twox-hash", ] [[package]] name = "md-5" -version = "0.10.6" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" +checksum = "69b6441f590336821bb897fb28fc622898ccceb1d6cea3fde5ea86b090c4de98" dependencies = [ "cfg-if", - "digest", + "digest 0.11.3", ] [[package]] name = "memchr" -version = "2.8.0" +version = "2.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" [[package]] name = "miniz_oxide" @@ -2084,6 +2123,16 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-bigint" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93e7820bc0a80a0238e650327316f929ba18d5be054b647490a3a6a339f3e7c0" +dependencies = [ + "num-integer", + "num-traits", +] + [[package]] name = "num-complex" version = "0.4.6" @@ -2141,7 +2190,7 @@ dependencies = [ "futures-util", "http", "humantime", - "itertools", + "itertools 0.14.0", "parking_lot", "percent-encoding", "thiserror", @@ -2165,15 +2214,6 @@ version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" -[[package]] -name = "ordered-float" -version = "2.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68f19d67e5a2795c94e73e0bb1cc1a7edeb2e28efd39e2e1c9b7a40c1108b11c" -dependencies = [ - "num-traits", -] - [[package]] name = "parking_lot" version = "0.12.5" @@ -2199,9 +2239,9 @@ dependencies = [ [[package]] name = "parquet" -version = "58.1.0" +version = "59.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d3f9f2205199603564127932b89695f52b62322f541d0fc7179d57c2e1c9877" +checksum = "ff322f54b1a0f9288e614ed1f2d329b380af5476420db19f46ffb865e1163d73" dependencies = [ "ahash", "arrow-array", @@ -2210,35 +2250,27 @@ dependencies = [ "arrow-ipc", "arrow-schema", "arrow-select", - "base64", + "base64 0.23.1", "brotli", "bytes", "chrono", "flate2", "futures", "half", - "hashbrown 0.16.1", + "hashbrown 0.17.1", "lz4_flex", - "num-bigint", + "num-bigint 0.5.1", "num-integer", "num-traits", "object_store", - "paste", "seq-macro", "simdutf8", "snap", - "thrift", "tokio", "twox-hash", "zstd", ] -[[package]] -name = "paste" -version = "1.0.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" - [[package]] name = "percent-encoding" version = "2.3.2" @@ -2275,6 +2307,26 @@ dependencies = [ "siphasher", ] +[[package]] +name = "pin-project" +version = "1.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2466b2336ed02bcdca6b294417127b90ec92038d1d5c4fbeac971a922e0e0924" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c96395f0a926bc13b1c17622aaddda1ecb55d49c8f1bf9777e4d877800a43f8b" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "pin-project-lite" version = "0.2.17" @@ -2333,7 +2385,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" dependencies = [ "proc-macro2", - "syn", + "syn 2.0.117", ] [[package]] @@ -2406,7 +2458,7 @@ dependencies = [ "proc-macro2", "pyo3-macros-backend", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -2418,7 +2470,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -2488,7 +2540,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "76009fbe0614077fc1a2ce255e3a1881a2e3a3527097d5dc6d8212c585e7e38b" dependencies = [ "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -2517,7 +2569,7 @@ checksum = "b7186006dcb21920990093f30e3dea63b7d6e977bf1256be20c3563a5db070da" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -2581,7 +2633,7 @@ dependencies = [ "regex", "relative-path", "rustc_version", - "syn", + "syn 2.0.117", "unicode-ident", ] @@ -2697,7 +2749,7 @@ checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -2706,6 +2758,7 @@ version = "1.0.149" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86" dependencies = [ + "indexmap 2.14.0", "itoa", "memchr", "serde", @@ -2719,7 +2772,7 @@ version = "3.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "76a5c54c7310e7b8b9577c286d7e399ddd876c3e12b3ed917a8aabc4b96e9e8c" dependencies = [ - "base64", + "base64 0.22.1", "bs58", "chrono", "hex", @@ -2742,18 +2795,18 @@ dependencies = [ "darling", "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] name = "sha2" -version = "0.10.9" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +checksum = "446ba717509524cb3f22f17ecc096f10f4822d76ab5c0b9822c5f9c284e825f4" dependencies = [ "cfg-if", - "cpufeatures 0.2.17", - "digest", + "cpufeatures", + "digest 0.11.3", ] [[package]] @@ -2811,9 +2864,9 @@ dependencies = [ [[package]] name = "sqlparser" -version = "0.61.0" +version = "0.62.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dbf5ea8d4d7c808e1af1cbabebca9a2abe603bcefc22294c5b95018d53200cb7" +checksum = "13c6d1b651dc4edf07eead2a0c6c78016ce971bc2c10da5266861b13f25e7cec" dependencies = [ "log", "sqlparser_derive 0.5.0", @@ -2827,7 +2880,7 @@ checksum = "da5fc6819faabb412da764b99d3b713bb55083c11e7e0c00144d386cd6a1939c" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -2838,7 +2891,7 @@ checksum = "a6dd45d8fc1c79299bfbb7190e42ccbbdf6a5f52e4a6ad98d92357ea965bd289" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -2883,6 +2936,17 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "syn" +version = "3.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8593e8e72159ed2257d083c7a454a85cbf854f37a0966d8d483aff8c8a3ebcee" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + [[package]] name = "synstructure" version = "0.13.2" @@ -2891,7 +2955,7 @@ checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -2930,18 +2994,7 @@ checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" dependencies = [ "proc-macro2", "quote", - "syn", -] - -[[package]] -name = "thrift" -version = "0.17.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e54bc85fc7faa8bc175c4bab5b92ba8d9a3ce893d0e9f42cc455c8ab16a9e09" -dependencies = [ - "byteorder", - "integer-encoding", - "ordered-float", + "syn 2.0.117", ] [[package]] @@ -3011,9 +3064,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.51.1" +version = "1.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f66bf9585cda4b724d3e78ab34b73fb2bbaba9011b9bfdf69dc836382ea13b8c" +checksum = "202caea871b69668250d242070849eb495be178ed697a3e98aebce5bc81a0bed" dependencies = [ "bytes", "pin-project-lite", @@ -3028,7 +3081,7 @@ checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -3105,7 +3158,7 @@ checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -3270,7 +3323,7 @@ dependencies = [ "bumpalo", "proc-macro2", "quote", - "syn", + "syn 2.0.117", "wasm-bindgen-shared", ] @@ -3357,7 +3410,7 @@ checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -3368,7 +3421,7 @@ checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -3516,7 +3569,7 @@ dependencies = [ "heck", "indexmap 2.14.0", "prettyplease", - "syn", + "syn 2.0.117", "wasm-metadata", "wit-bindgen-core", "wit-component", @@ -3532,7 +3585,7 @@ dependencies = [ "prettyplease", "proc-macro2", "quote", - "syn", + "syn 2.0.117", "wit-bindgen-core", "wit-bindgen-rust", ] @@ -3590,7 +3643,7 @@ dependencies = [ name = "wren-core-py" version = "0.1.0" dependencies = [ - "base64", + "base64 0.22.1", "csv", "datafusion-common", "env_logger", @@ -3612,7 +3665,7 @@ name = "wren-manifest-macro" version = "0.4.0" dependencies = [ "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -3623,7 +3676,7 @@ dependencies = [ "csv", "datafusion", "datafusion-session", - "itertools", + "itertools 0.14.0", "log", "parking_lot", "petgraph", @@ -3660,7 +3713,7 @@ checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", "synstructure", ] @@ -3681,7 +3734,7 @@ checksum = "70e3cd084b1788766f53af483dd21f93881ff30d7320490ec3ef7526d203bad4" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -3701,7 +3754,7 @@ checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", "synstructure", ] @@ -3735,7 +3788,7 @@ checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] diff --git a/core/wren-core-py/Cargo.toml b/core/wren-core-py/Cargo.toml index a8caabbb8..4f3260ba3 100644 --- a/core/wren-core-py/Cargo.toml +++ b/core/wren-core-py/Cargo.toml @@ -15,7 +15,7 @@ crate-type = ["cdylib"] pyo3 = { version = "0.29.0", features = ["extension-module", "abi3-py311"] } wren-core = { version = "0.4.0", path = "../wren-core/core", package = "wren-semantic-core" } wren-core-base = { path = "../wren-core-base", features = ["python-binding"] } -datafusion-common = "53.0.0" +datafusion-common = "55.1.0" base64 = "0.22.1" futures = "0.3.31" serde_json = "1.0.117" diff --git a/core/wren-core-py/src/context.rs b/core/wren-core-py/src/context.rs index b61a9eb2a..6dfb6fab0 100644 --- a/core/wren-core-py/src/context.rs +++ b/core/wren-core-py/src/context.rs @@ -635,6 +635,8 @@ impl PySessionContext { r.description FROM information_schema.routines r + WHERE + r.function_type IN ('SCALAR', 'AGGREGATE', 'WINDOW') "#; let batches = ctx .sql(sql) @@ -742,6 +744,7 @@ impl PySessionContext { inputs i ON r.specific_name = i.specific_name WHERE r.routine_name = '{}' + AND r.function_type IN ('SCALAR', 'AGGREGATE', 'WINDOW') "#, function_name ); diff --git a/core/wren-core-py/tests/test_modeling_core.py b/core/wren-core-py/tests/test_modeling_core.py index 18d2642aa..ffe84d661 100644 --- a/core/wren-core-py/tests/test_modeling_core.py +++ b/core/wren-core-py/tests/test_modeling_core.py @@ -144,7 +144,7 @@ def test_read_function_list(): path = "tests/functions.csv" session_context = SessionContext(manifest_str, path) functions = session_context.get_available_functions() - assert len(functions) == 290 + assert len(functions) == 291 rewritten_sql = session_context.transform_sql( "SELECT add_two(c_custkey, c_custkey) FROM my_catalog.my_schema.customer" @@ -156,7 +156,7 @@ def test_read_function_list(): session_context = SessionContext(manifest_str, None) functions = session_context.get_available_functions() - assert len(functions) == 283 + assert len(functions) == 284 def test_get_available_functions(): diff --git a/core/wren-core-wasm/.claude/CLAUDE.md b/core/wren-core-wasm/.claude/CLAUDE.md index 177d6d04f..d9936aca8 100644 --- a/core/wren-core-wasm/.claude/CLAUDE.md +++ b/core/wren-core-wasm/.claude/CLAUDE.md @@ -4,7 +4,7 @@ Wren Engine compiled to WebAssembly for browser-native analytics. Runs SQL queri ## Why a Separate Crate? -Uses **upstream DataFusion** (crates.io v53), not the Canner fork. The WASM version executes queries directly via DataFusion — no SQL unparser or dialect transpilation needed. Kept outside the `wren-core/` workspace to avoid dependency conflicts. +Uses **upstream DataFusion** (crates.io v55), not the Canner fork. The WASM version executes queries directly via DataFusion — no SQL unparser or dialect transpilation needed. Kept outside the `wren-core/` workspace to avoid dependency conflicts. Shared code: `wren-core-base` (manifest types, no DataFusion dependency) and `wren-core` semantic layer (MDL analysis rules). @@ -51,10 +51,10 @@ just clean # Remove pkg/, dist/, target/ ## Dependencies -- **DataFusion v53** (upstream, crates.io) — query engine, `default-features = false` + selected features -- **Arrow v58.1** — `json` feature for JSON reader -- **Parquet v58.1** — `snap` + `lz4` only (no zstd — requires C library, can't compile to WASM) -- **object_store v0.13.1** — `aws` + `http` features for URL mode (HttpStore) +- **DataFusion v55** (upstream, crates.io) — query engine, `default-features = false` + selected features +- **Arrow v59.2** — `json` feature for JSON reader +- **Parquet v59.2** — `snap` + `lz4` only (no zstd — requires C library, can't compile to WASM) +- **object_store v0.13.2** — `aws` + `http` features for URL mode (HttpStore) - **wren-core** (path: `../wren-core/core`) — semantic layer, `default-features = false` - **wren-core-base** (path: `../wren-core-base`) — shared manifest types - **wasm-bindgen / js-sys / web-sys** — WASM ↔ JS bindings @@ -69,7 +69,7 @@ just clean # Remove pkg/, dist/, target/ - **No SystemTime**: chrono `wasmbind` feature required - **getrandom**: All three major versions (0.2, 0.3, 0.4) in the dep tree need explicit WASM JS backend - **macOS build**: Needs LLVM (`brew install llvm`) for C deps — justfile handles env vars automatically -- **Binary size**: ~68 MB raw / ~14 MB gzip (target: < 15 MB gzip) +- **Binary size**: ~57 MB raw / ~14 MB gzip (target: < 15 MB gzip); the release profile uses fat LTO + `codegen-units = 1` to stay under budget ## npm Package (wren-core-wasm) diff --git a/core/wren-core-wasm/Cargo.lock b/core/wren-core-wasm/Cargo.lock index 72b8b76dd..ef01c538a 100644 --- a/core/wren-core-wasm/Cargo.lock +++ b/core/wren-core-wasm/Cargo.lock @@ -90,9 +90,9 @@ checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" [[package]] name = "arrow" -version = "58.1.0" +version = "59.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d441fdda254b65f3e9025910eb2c2066b6295d9c8ed409522b8d2ace1ff8574c" +checksum = "7c14b3d39f306bc28fd639d59f06e17a0f377d0021e1b7e9054e4d6fedc98774" dependencies = [ "arrow-arith", "arrow-array", @@ -111,9 +111,9 @@ dependencies = [ [[package]] name = "arrow-arith" -version = "58.1.0" +version = "59.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ced5406f8b720cc0bc3aa9cf5758f93e8593cda5490677aa194e4b4b383f9a59" +checksum = "ce2961626677665b2195eb59242af4c7befe7b8737ca2050295389362380104e" dependencies = [ "arrow-array", "arrow-buffer", @@ -125,9 +125,9 @@ dependencies = [ [[package]] name = "arrow-array" -version = "58.1.0" +version = "59.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "772bd34cacdda8baec9418d80d23d0fb4d50ef0735685bd45158b83dfeb6e62d" +checksum = "1e5f6adeffdf587d7a31db5d2266189624b526730cd3627f9ff9fedae97ad584" dependencies = [ "ahash", "arrow-buffer", @@ -136,7 +136,7 @@ dependencies = [ "chrono", "chrono-tz", "half", - "hashbrown 0.16.1", + "hashbrown 0.17.1", "num-complex", "num-integer", "num-traits", @@ -144,21 +144,21 @@ dependencies = [ [[package]] name = "arrow-buffer" -version = "58.1.0" +version = "59.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "898f4cf1e9598fdb77f356fdf2134feedfd0ee8d5a4e0a5f573e7d0aec16baa4" +checksum = "097d193003ce7995d5d087089069ec2a6e0187faf5a6f8c9f38af2645d987182" dependencies = [ "bytes", "half", - "num-bigint", + "num-bigint 0.5.1", "num-traits", ] [[package]] name = "arrow-cast" -version = "58.1.0" +version = "59.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0127816c96533d20fc938729f48c52d3e48f99717e7a0b5ade77d742510736d" +checksum = "635c9c635668ad26adf76cce8fb276c4be7cf06e63bd516de7da514f9680ee53" dependencies = [ "arrow-array", "arrow-buffer", @@ -167,7 +167,7 @@ dependencies = [ "arrow-schema", "arrow-select", "atoi", - "base64", + "base64 0.23.1", "chrono", "comfy-table", "half", @@ -178,9 +178,9 @@ dependencies = [ [[package]] name = "arrow-csv" -version = "58.1.0" +version = "59.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca025bd0f38eeecb57c2153c0123b960494138e6a957bbda10da2b25415209fe" +checksum = "4c2ebf8d631e79b02c16cf5ae860561272c26024ec88fce389a56aaddd558e86" dependencies = [ "arrow-array", "arrow-cast", @@ -193,9 +193,9 @@ dependencies = [ [[package]] name = "arrow-data" -version = "58.1.0" +version = "59.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42d10beeab2b1c3bb0b53a00f7c944a178b622173a5c7bcabc3cb45d90238df4" +checksum = "9ba2f832eaeca24b8f26143dba750e42ee4ab51cf7d65e701ca9607cfda9f358" dependencies = [ "arrow-buffer", "arrow-schema", @@ -206,9 +206,9 @@ dependencies = [ [[package]] name = "arrow-ipc" -version = "58.1.0" +version = "59.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "609a441080e338147a84e8e6904b6da482cefb957c5cdc0f3398872f69a315d0" +checksum = "dcc41681ea80f521df14c36725b74d4c60702c47f0793af2be469c04527e2599" dependencies = [ "arrow-array", "arrow-buffer", @@ -217,19 +217,21 @@ dependencies = [ "arrow-select", "flatbuffers", "lz4_flex", + "zstd", ] [[package]] name = "arrow-json" -version = "58.1.0" +version = "59.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ead0914e4861a531be48fe05858265cf854a4880b9ed12618b1d08cba9bebc8" +checksum = "a2f57d7a81969f24ccf80809587b76c09897e6f829d2d65a5976bfb3218851f1" dependencies = [ "arrow-array", "arrow-buffer", "arrow-cast", - "arrow-data", + "arrow-ord", "arrow-schema", + "arrow-select", "chrono", "half", "indexmap 2.14.0", @@ -245,9 +247,9 @@ dependencies = [ [[package]] name = "arrow-ord" -version = "58.1.0" +version = "59.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "763a7ba279b20b52dad300e68cfc37c17efa65e68623169076855b3a9e941ca5" +checksum = "2c900759f3bd8354fd4196bc4403eee846894dc2adf66b4225472006a0bf18c5" dependencies = [ "arrow-array", "arrow-buffer", @@ -258,9 +260,9 @@ dependencies = [ [[package]] name = "arrow-row" -version = "58.1.0" +version = "59.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e14fe367802f16d7668163ff647830258e6e0aeea9a4d79aaedf273af3bdcd3e" +checksum = "f4c6425032e28266e3fc4ff680805e57e670d6ea92473043f3e65b7ed6ac79f2" dependencies = [ "arrow-array", "arrow-buffer", @@ -271,9 +273,9 @@ dependencies = [ [[package]] name = "arrow-schema" -version = "58.1.0" +version = "59.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c30a1365d7a7dc50cc847e54154e6af49e4c4b0fddc9f607b687f29212082743" +checksum = "10fab8d4563491417ba801fab29d205104d20d4bdf37bda6cd1cf425cff598cd" dependencies = [ "serde_core", "serde_json", @@ -281,9 +283,9 @@ dependencies = [ [[package]] name = "arrow-select" -version = "58.1.0" +version = "59.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78694888660a9e8ac949853db393af2a8b8fc82c19ce333132dfa2e72cc1a7fe" +checksum = "fc58569193c2525915f3cc6310edba3792f1200f65d6e9ed330aa33e691493b8" dependencies = [ "ahash", "arrow-array", @@ -295,9 +297,9 @@ dependencies = [ [[package]] name = "arrow-string" -version = "58.1.0" +version = "59.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61e04a01f8bb73ce54437514c5fd3ee2aa3e8abe4c777ee5cc55853b1652f79e" +checksum = "2e0813f3c35c1cfea65e14c20a953440f7783c088b7ad2d0db162ccdeefcec14" dependencies = [ "arrow-array", "arrow-buffer", @@ -318,7 +320,7 @@ checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -348,6 +350,12 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "base64" +version = "0.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5" + [[package]] name = "bigdecimal" version = "0.4.10" @@ -356,7 +364,7 @@ checksum = "4d6867f1565b3aad85681f1015055b087fcfd840d6aeee6eee7f2da317603695" dependencies = [ "autocfg", "libm", - "num-bigint", + "num-bigint 0.4.6", "num-integer", "num-traits", ] @@ -373,7 +381,7 @@ version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "46502ad458c9a52b69d4d4d32775c788b7a1b85e8bc9d482d92250fc0e3f8efe" dependencies = [ - "digest", + "digest 0.10.7", ] [[package]] @@ -387,7 +395,7 @@ dependencies = [ "cc", "cfg-if", "constant_time_eq", - "cpufeatures 0.3.0", + "cpufeatures", ] [[package]] @@ -399,6 +407,15 @@ dependencies = [ "generic-array", ] +[[package]] +name = "block-buffer" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa" +dependencies = [ + "hybrid-array", +] + [[package]] name = "brotli" version = "8.0.2" @@ -435,12 +452,6 @@ version = "3.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5d20789868f4b01b2f2caec9f5c4e0213b41e3e5702a50157d699ae31ced2fcb" -[[package]] -name = "byteorder" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" - [[package]] name = "bytes" version = "1.11.1" @@ -484,15 +495,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6f8d983286843e49675a4b7a2d174efe136dc93a18d69130dd18198a6c167601" dependencies = [ "cfg-if", - "cpufeatures 0.3.0", + "cpufeatures", "rand_core 0.10.1", ] [[package]] name = "chrono" -version = "0.4.44" +version = "0.4.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c673075a2e0e5f4a1dde27ce9dee1ea4558c7ffe648f576438a20ca1d2acc4b0" +checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" dependencies = [ "iana-time-zone", "js-sys", @@ -532,6 +543,12 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "const-oid" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c" + [[package]] name = "const-random" version = "0.1.18" @@ -574,15 +591,6 @@ version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" -[[package]] -name = "cpufeatures" -version = "0.2.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" -dependencies = [ - "libc", -] - [[package]] name = "cpufeatures" version = "0.3.0" @@ -614,6 +622,15 @@ dependencies = [ "typenum", ] +[[package]] +name = "crypto-common" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" +dependencies = [ + "hybrid-array", +] + [[package]] name = "csv" version = "1.4.0" @@ -655,7 +672,7 @@ dependencies = [ "proc-macro2", "quote", "strsim", - "syn", + "syn 2.0.117", ] [[package]] @@ -666,14 +683,14 @@ checksum = "ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d" dependencies = [ "darling_core", "quote", - "syn", + "syn 2.0.117", ] [[package]] name = "dashmap" -version = "6.1.0" +version = "6.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5041cc499144891f3790297212f32a74fb938e5136a14943f338ef9e0ae276cf" +checksum = "e6361d5c062261c78a176addb82d4c821ae42bed6089de0e12603cd25de2059c" dependencies = [ "cfg-if", "crossbeam-utils", @@ -685,14 +702,13 @@ dependencies = [ [[package]] name = "datafusion" -version = "53.0.0" +version = "55.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de9f8117889ba9503440f1dd79ebab32ba52ccf1720bb83cd718a29d4edc0d16" +checksum = "14313430439ef858ed5f52ca9c840e9eea9844da34d3716d05a476449634da5d" dependencies = [ "arrow", "arrow-schema", "async-trait", - "bytes", "chrono", "datafusion-catalog", "datafusion-catalog-listing", @@ -720,14 +736,13 @@ dependencies = [ "datafusion-session", "datafusion-sql", "futures", - "itertools", + "indexmap 2.14.0", + "itertools 0.15.0", "log", "object_store", "parking_lot", "parquet", - "rand 0.9.4", - "regex", - "sqlparser 0.61.0", + "sqlparser 0.62.0", "tempfile", "tokio", "url", @@ -736,9 +751,9 @@ dependencies = [ [[package]] name = "datafusion-catalog" -version = "53.0.0" +version = "55.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be893b73a13671f310ffcc8da2c546b81efcc54c22e0382c0a28aa3537017137" +checksum = "2c0756ebc770bcebae366dfd273dad50935f6103b4d212253cb17e2e53c5427f" dependencies = [ "arrow", "async-trait", @@ -752,7 +767,7 @@ dependencies = [ "datafusion-physical-plan", "datafusion-session", "futures", - "itertools", + "itertools 0.15.0", "log", "object_store", "parking_lot", @@ -761,9 +776,9 @@ dependencies = [ [[package]] name = "datafusion-catalog-listing" -version = "53.0.0" +version = "55.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "830487b51ed83807d6b32d6325f349c3144ae0c9bf772cf2a712db180c31d5e6" +checksum = "bbde2961698705d8402f0f382f08876fc6f1126ffde790ef17833cf0ab87cd59" dependencies = [ "arrow", "async-trait", @@ -777,40 +792,43 @@ dependencies = [ "datafusion-physical-expr-common", "datafusion-physical-plan", "futures", - "itertools", + "itertools 0.15.0", "log", "object_store", + "percent-encoding", ] [[package]] name = "datafusion-common" -version = "53.0.0" +version = "55.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d7663f3af955292f8004e74bcaf8f7ea3d66cc38438749615bb84815b61a293" +checksum = "a6e06fe711ff815c8cf3143158547aea599702fd43f76f5866c70d5a5b3693a0" dependencies = [ - "ahash", "arrow", "arrow-ipc", + "arrow-schema", "chrono", + "foldhash 0.2.0", "half", - "hashbrown 0.16.1", + "hashbrown 0.17.1", "indexmap 2.14.0", - "itertools", + "itertools 0.15.0", "libc", "log", + "num-traits", "object_store", "parquet", - "paste", - "sqlparser 0.61.0", + "sqlparser 0.62.0", "tokio", + "uuid", "web-time", ] [[package]] name = "datafusion-common-runtime" -version = "53.0.0" +version = "55.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f590205c7e32fe1fea48dd53ffb406e56ae0e7a062213a3ac848db8771641bd" +checksum = "f7a5107acff4e3ed50a7732acec06bba40e74ab49df021397ed250d60f04e42a" dependencies = [ "futures", "log", @@ -819,9 +837,9 @@ dependencies = [ [[package]] name = "datafusion-datasource" -version = "53.0.0" +version = "55.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fde1e030a9dc87b743c806fbd631f5ecfa2ccaa4ffb61fa19144a07fea406b79" +checksum = "f69f45d2d700785be8df9d4b44410e1c8e4a856f10cc728b418a1066c0b594fb" dependencies = [ "arrow", "async-trait", @@ -838,9 +856,10 @@ dependencies = [ "datafusion-session", "futures", "glob", - "itertools", + "itertools 0.15.0", "log", "object_store", + "parking_lot", "rand 0.9.4", "tokio", "url", @@ -848,9 +867,9 @@ dependencies = [ [[package]] name = "datafusion-datasource-arrow" -version = "53.0.0" +version = "55.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "331ebae7055dc108f9b54994b93dff91f3a17445539efe5b74e89264f7b36e15" +checksum = "81650af64894ef34b46d2ef818ef244b40a81d90818f3bc642bf8fbf6758f8e0" dependencies = [ "arrow", "arrow-ipc", @@ -865,16 +884,16 @@ dependencies = [ "datafusion-physical-plan", "datafusion-session", "futures", - "itertools", + "itertools 0.15.0", "object_store", "tokio", ] [[package]] name = "datafusion-datasource-csv" -version = "53.0.0" +version = "55.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e0d475088325e2986876aa27bb30d0574f72a22955a527d202f454681d55c5c" +checksum = "b5e2519bed59a6907c79298b9c21fe807bbc34e4ff8414577d699b0741083593" dependencies = [ "arrow", "async-trait", @@ -895,9 +914,9 @@ dependencies = [ [[package]] name = "datafusion-datasource-json" -version = "53.0.0" +version = "55.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea1520d81f31770f3ad6ee98b391e75e87a68a5bb90de70064ace5e0a7182fe8" +checksum = "635e61290e171597f6e50427e0cb486eda468ab47e880fa24d15bbb0fa24ac7e" dependencies = [ "arrow", "async-trait", @@ -912,18 +931,18 @@ dependencies = [ "datafusion-session", "futures", "object_store", - "serde_json", "tokio", "tokio-stream", ] [[package]] name = "datafusion-datasource-parquet" -version = "53.0.0" +version = "55.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95be805d0742ab129720f4c51ad9242cd872599cdb076098b03f061fcdc7f946" +checksum = "07cd518f85b3a027d913047b4ab0811111a2b734fdf84a51b78234455f10d8fa" dependencies = [ "arrow", + "arrow-schema", "async-trait", "bytes", "datafusion-common", @@ -931,6 +950,7 @@ dependencies = [ "datafusion-datasource", "datafusion-execution", "datafusion-expr", + "datafusion-functions", "datafusion-functions-aggregate-common", "datafusion-physical-expr", "datafusion-physical-expr-adapter", @@ -939,7 +959,7 @@ dependencies = [ "datafusion-pruning", "datafusion-session", "futures", - "itertools", + "itertools 0.15.0", "log", "object_store", "parking_lot", @@ -949,20 +969,20 @@ dependencies = [ [[package]] name = "datafusion-doc" -version = "53.0.0" +version = "55.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c93ad9e37730d2c7196e68616f3f2dd3b04c892e03acd3a8eeca6e177f3c06a" +checksum = "d44c5919b560862dd1cb20a5c75b3e48c3d9e49719c91d78da5c38eaad5495c4" [[package]] name = "datafusion-execution" -version = "53.0.0" +version = "55.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9437d3cd5d363f9319f8122182d4d233427de79c7eb748f23054c9aaa0fdd8df" +checksum = "7620ad535af8730186bdec05d720e40d9a4584b40c7eb75f72c43aa7978e5484" dependencies = [ "arrow", "arrow-buffer", "async-trait", - "chrono", + "bytes", "dashmap", "datafusion-common", "datafusion-expr", @@ -971,18 +991,22 @@ dependencies = [ "log", "object_store", "parking_lot", + "pin-project-lite", "rand 0.9.4", "tempfile", + "tokio", + "tokio-util", "url", ] [[package]] name = "datafusion-expr" -version = "53.0.0" +version = "55.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67164333342b86521d6d93fa54081ee39839894fb10f7a700c099af96d7552cf" +checksum = "ea229af8b73e566fc6ebb1c8dd42ab250f8855c5304bd173f4b50ead78b64304" dependencies = [ "arrow", + "arrow-schema", "async-trait", "chrono", "datafusion-common", @@ -992,34 +1016,32 @@ dependencies = [ "datafusion-functions-window-common", "datafusion-physical-expr-common", "indexmap 2.14.0", - "itertools", - "paste", + "itertools 0.15.0", "serde_json", - "sqlparser 0.61.0", + "sqlparser 0.62.0", ] [[package]] name = "datafusion-expr-common" -version = "53.0.0" +version = "55.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab05fdd00e05d5a6ee362882546d29d6d3df43a6c55355164a7fbee12d163bc9" +checksum = "6a18baa840f77760dcc9eaad281329bc57536c5c39e65e8649596f2a51216be7" dependencies = [ "arrow", "datafusion-common", "indexmap 2.14.0", - "itertools", - "paste", + "itertools 0.15.0", ] [[package]] name = "datafusion-functions" -version = "53.0.0" +version = "55.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04fb863482d987cf938db2079e07ab0d3bb64595f28907a6c2f8671ad71cca7e" +checksum = "763bbfeec40b55a8a961d7349c0fcfcec4f3f66cc234d6295250b95371b85e34" dependencies = [ "arrow", "arrow-buffer", - "base64", + "base64 0.23.1", "blake2", "blake3", "chrono", @@ -1030,26 +1052,25 @@ dependencies = [ "datafusion-expr", "datafusion-expr-common", "datafusion-macros", + "datafusion-physical-expr-common", "hex", - "itertools", + "itertools 0.15.0", "log", - "md-5", + "md-5 0.11.0", "memchr", "num-traits", "rand 0.9.4", "regex", "sha2", - "unicode-segmentation", "uuid", ] [[package]] name = "datafusion-functions-aggregate" -version = "53.0.0" +version = "55.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "829856f4e14275fb376c104f27cbf3c3b57a9cfe24885d98677525f5e43ce8d6" +checksum = "46c0c2b2e856883ecf6a4fa8b23e7c6041183af8a68d0583dc95d1d21e4eaee0" dependencies = [ - "ahash", "arrow", "datafusion-common", "datafusion-doc", @@ -1060,18 +1081,17 @@ dependencies = [ "datafusion-physical-expr", "datafusion-physical-expr-common", "half", + "hashbrown 0.17.1", "log", "num-traits", - "paste", ] [[package]] name = "datafusion-functions-aggregate-common" -version = "53.0.0" +version = "55.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08af79cc3d2aa874a362fb97decfcbd73d687190cb096f16a6c85a7780cce311" +checksum = "e35e42c87dfbff06934ea0e99cfed3440ac614c3f3bfb8c33be35ca0587d4140" dependencies = [ - "ahash", "arrow", "datafusion-common", "datafusion-expr-common", @@ -1080,9 +1100,9 @@ dependencies = [ [[package]] name = "datafusion-functions-nested" -version = "53.0.0" +version = "55.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "465ae3368146d49c2eda3e2c0ef114424c87e8a6b509ab34c1026ace6497e790" +checksum = "08079b0604cdcc43c7d6cf006cfd5d3ed9c235ade0c1ebe91085a62f875603df" dependencies = [ "arrow", "arrow-ord", @@ -1096,34 +1116,34 @@ dependencies = [ "datafusion-functions-aggregate-common", "datafusion-macros", "datafusion-physical-expr-common", - "hashbrown 0.16.1", - "itertools", + "hashbrown 0.17.1", + "itertools 0.15.0", "itoa", "log", - "paste", + "memchr", ] [[package]] name = "datafusion-functions-table" -version = "53.0.0" +version = "55.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6156e6b22fcf1784112fc0173f3ae6e78c8fdb4d3ed0eace9543873b437e2af6" +checksum = "fe2070ecf5638a93c81c8babc7858339ef52165a9f54da1411afd69adf7d85ce" dependencies = [ "arrow", "async-trait", "datafusion-catalog", "datafusion-common", "datafusion-expr", + "datafusion-physical-expr", "datafusion-physical-plan", "parking_lot", - "paste", ] [[package]] name = "datafusion-functions-window" -version = "53.0.0" +version = "55.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca7baec14f866729012efb89011a6973f3a346dc8090c567bfcd328deff551c1" +checksum = "0e9a164ebe1998c5c3f9e15c9a0c4e6b599a0c8739b5471bfb089752441963f1" dependencies = [ "arrow", "datafusion-common", @@ -1134,14 +1154,13 @@ dependencies = [ "datafusion-physical-expr", "datafusion-physical-expr-common", "log", - "paste", ] [[package]] name = "datafusion-functions-window-common" -version = "53.0.0" +version = "55.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "159228c3280d342658466bb556dc24de30047fe1d7e559dc5d16ccc5324166f9" +checksum = "98965b183f92201c16798994820ae032649c4821e089c4c80e7d0f4485d41df5" dependencies = [ "datafusion-common", "datafusion-physical-expr-common", @@ -1149,20 +1168,20 @@ dependencies = [ [[package]] name = "datafusion-macros" -version = "53.0.0" +version = "55.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5427e5da5edca4d21ea1c7f50e1c9421775fe33d7d5726e5641a833566e7578" +checksum = "3a711bd5a06ec85d9683e142d28a98ad0679a7220669f5c1b8589e57e9b29e46" dependencies = [ "datafusion-doc", "quote", - "syn", + "syn 3.0.6", ] [[package]] name = "datafusion-optimizer" -version = "53.0.0" +version = "55.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89099eefcd5b223ec685c36a41d35c69239236310d71d339f2af0fa4383f3f46" +checksum = "ff3256b62ff44616ec83c7f04741d21124970833bb377ed56d9abc5ec6f707ec" dependencies = [ "arrow", "chrono", @@ -1171,7 +1190,7 @@ dependencies = [ "datafusion-expr-common", "datafusion-physical-expr", "indexmap 2.14.0", - "itertools", + "itertools 0.15.0", "log", "regex", "regex-syntax", @@ -1179,32 +1198,31 @@ dependencies = [ [[package]] name = "datafusion-physical-expr" -version = "53.0.0" +version = "55.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f222df5195d605d79098ef37bdd5323bff0131c9d877a24da6ec98dfca9fe36" +checksum = "c1e37a25b5aa1f3a1db114f0ee3605bf01639b83a831dcde35119ee61f0b11d6" dependencies = [ - "ahash", "arrow", + "arrow-schema", "datafusion-common", "datafusion-expr", "datafusion-expr-common", "datafusion-functions-aggregate-common", "datafusion-physical-expr-common", "half", - "hashbrown 0.16.1", + "hashbrown 0.17.1", "indexmap 2.14.0", - "itertools", + "itertools 0.15.0", "parking_lot", - "paste", "petgraph", "tokio", ] [[package]] name = "datafusion-physical-expr-adapter" -version = "53.0.0" +version = "55.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40838625d63d9c12549d81979db3dd675d159055eb9135009ba272ab0e8d0f64" +checksum = "3a4d12787d42e88b26b0d3ec006192fa23ceb832ef2a9e87439ce820a8e038cc" dependencies = [ "arrow", "datafusion-common", @@ -1212,31 +1230,31 @@ dependencies = [ "datafusion-functions", "datafusion-physical-expr", "datafusion-physical-expr-common", - "itertools", + "itertools 0.15.0", ] [[package]] name = "datafusion-physical-expr-common" -version = "53.0.0" +version = "55.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eacbcc4cfd502558184ed58fa3c72e775ec65bf077eef5fd2b3453db676f893c" +checksum = "d13bf7f50fd902f5afc46d50759f5185b5620c40b24054f80f7c648897d3e5a6" dependencies = [ - "ahash", "arrow", "chrono", "datafusion-common", "datafusion-expr-common", - "hashbrown 0.16.1", + "hashbrown 0.17.1", "indexmap 2.14.0", - "itertools", + "itertools 0.15.0", "parking_lot", + "pin-project", ] [[package]] name = "datafusion-physical-optimizer" -version = "53.0.0" +version = "55.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d501d0e1d0910f015677121601ac177ec59272ef5c9324d1147b394988f40941" +checksum = "11fbe7e6665072957d44ed9587676e17a1f9ab3acfadfc287dac23750585f18a" dependencies = [ "arrow", "datafusion-common", @@ -1247,20 +1265,23 @@ dependencies = [ "datafusion-physical-expr-common", "datafusion-physical-plan", "datafusion-pruning", - "itertools", + "datafusion-session", + "itertools 0.15.0", ] [[package]] name = "datafusion-physical-plan" -version = "53.0.0" +version = "55.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "463c88ad6f1ecab1810f4c9f046898bee035b370137eb79b2b2db925e270631d" +checksum = "1265d58e5bce07d154e642a51ff43033b576a6ae40d50a29ac2b9f311004eb52" dependencies = [ - "ahash", "arrow", + "arrow-data", + "arrow-ipc", "arrow-ord", "arrow-schema", "async-trait", + "bytes", "datafusion-common", "datafusion-common-runtime", "datafusion-execution", @@ -1272,21 +1293,22 @@ dependencies = [ "datafusion-physical-expr-common", "futures", "half", - "hashbrown 0.16.1", + "hashbrown 0.17.1", "indexmap 2.14.0", - "itertools", + "itertools 0.15.0", "log", "num-traits", "parking_lot", "pin-project-lite", + "serde_json", "tokio", ] [[package]] name = "datafusion-pruning" -version = "53.0.0" +version = "55.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2857618a0ecbd8cd0cf29826889edd3a25774ec26b2995fc3862095c95d88fc6" +checksum = "74b06b333405c05015836ed36cea02cc29d1e2ee8fb414a77532d8346fb56492" dependencies = [ "arrow", "datafusion-common", @@ -1295,16 +1317,16 @@ dependencies = [ "datafusion-physical-expr", "datafusion-physical-expr-common", "datafusion-physical-plan", - "itertools", "log", ] [[package]] name = "datafusion-session" -version = "53.0.0" +version = "55.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef8637e35022c5c775003b3ab1debc6b4a8f0eb41b069bdd5475dd3aa93f6eba" +checksum = "e7616b17b1d2fee3e9b13c651eac91b768d153265e6ba88e06e0b4af7883f69c" dependencies = [ + "arrow-schema", "async-trait", "datafusion-common", "datafusion-execution", @@ -1315,9 +1337,9 @@ dependencies = [ [[package]] name = "datafusion-sql" -version = "53.0.0" +version = "55.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12d9e9f16a1692a11c94bcc418191fa15fd2b4d72a0c1a0c607db93c0b84dd81" +checksum = "5b4848be66dd008d55adbe1dbc8cbd6a2502588ad0c0b8483bab0f43fd4495d0" dependencies = [ "arrow", "bigdecimal", @@ -1328,7 +1350,7 @@ dependencies = [ "indexmap 2.14.0", "log", "regex", - "sqlparser 0.61.0", + "sqlparser 0.62.0", ] [[package]] @@ -1347,11 +1369,22 @@ version = "0.10.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ - "block-buffer", - "crypto-common", + "block-buffer 0.10.4", + "crypto-common 0.1.7", "subtle", ] +[[package]] +name = "digest" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" +dependencies = [ + "block-buffer 0.12.1", + "const-oid", + "crypto-common 0.2.2", +] + [[package]] name = "displaydoc" version = "0.2.5" @@ -1360,7 +1393,7 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -1512,7 +1545,7 @@ checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -1657,21 +1690,15 @@ dependencies = [ [[package]] name = "hashbrown" -version = "0.16.1" +version = "0.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" dependencies = [ "allocator-api2", "equivalent", "foldhash 0.2.0", ] -[[package]] -name = "hashbrown" -version = "0.17.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f467dd6dccf739c208452f8014c75c18bb8301b050ad1cfb27153803edb0f51" - [[package]] name = "heck" version = "0.5.0" @@ -1729,6 +1756,15 @@ version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "135b12329e5e3ce057a9f972339ea52bc954fe1e9358ef27f95e89716fbc5424" +[[package]] +name = "hybrid-array" +version = "0.4.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3944cf8cf766b40e2a1a333ee5e9b563f854d5fa49d6a8ca2764e97c6eddb214" +dependencies = [ + "typenum", +] + [[package]] name = "hyper" version = "1.9.0" @@ -1772,7 +1808,7 @@ version = "0.1.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" dependencies = [ - "base64", + "base64 0.22.1", "bytes", "futures-channel", "futures-util", @@ -1946,17 +1982,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" dependencies = [ "equivalent", - "hashbrown 0.17.0", + "hashbrown 0.17.1", "serde", "serde_core", ] -[[package]] -name = "integer-encoding" -version = "3.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8bb03732005da905c88227371639bf1ad885cc712789c011c31c5fb3ab3ccf02" - [[package]] name = "ipnet" version = "2.12.0" @@ -1982,6 +2012,15 @@ dependencies = [ "either", ] +[[package]] +name = "itertools" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b4baf93f58d4425749ca49a51c50ebab072c5df6994d08fed93541c331481dc" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.18" @@ -2120,9 +2159,9 @@ checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" [[package]] name = "lz4_flex" -version = "0.13.0" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db9a0d582c2874f68138a16ce1867e0ffde6c0bb0a0df85e1f36d04146db488a" +checksum = "ecbdfe44b1bd960b68170b417450a628c43f7cf56bb3c5317e61cb230ee7f226" dependencies = [ "twox-hash", ] @@ -2134,14 +2173,24 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" dependencies = [ "cfg-if", - "digest", + "digest 0.10.7", +] + +[[package]] +name = "md-5" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69b6441f590336821bb897fb28fc622898ccceb1d6cea3fde5ea86b090c4de98" +dependencies = [ + "cfg-if", + "digest 0.11.3", ] [[package]] name = "memchr" -version = "2.8.0" +version = "2.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" [[package]] name = "minicov" @@ -2193,6 +2242,16 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-bigint" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93e7820bc0a80a0238e650327316f929ba18d5be054b647490a3a6a339f3e7c0" +dependencies = [ + "num-integer", + "num-traits", +] + [[package]] name = "num-complex" version = "0.4.6" @@ -2243,7 +2302,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "622acbc9100d3c10e2ee15804b0caa40e55c933d5aa53814cd520805b7958a49" dependencies = [ "async-trait", - "base64", + "base64 0.22.1", "bytes", "chrono", "form_urlencoded", @@ -2254,8 +2313,8 @@ dependencies = [ "http-body-util", "humantime", "hyper", - "itertools", - "md-5", + "itertools 0.14.0", + "md-5 0.10.6", "parking_lot", "percent-encoding", "quick-xml", @@ -2292,15 +2351,6 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" -[[package]] -name = "ordered-float" -version = "2.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68f19d67e5a2795c94e73e0bb1cc1a7edeb2e28efd39e2e1c9b7a40c1108b11c" -dependencies = [ - "num-traits", -] - [[package]] name = "parking_lot" version = "0.12.5" @@ -2326,9 +2376,9 @@ dependencies = [ [[package]] name = "parquet" -version = "58.1.0" +version = "59.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d3f9f2205199603564127932b89695f52b62322f541d0fc7179d57c2e1c9877" +checksum = "ff322f54b1a0f9288e614ed1f2d329b380af5476420db19f46ffb865e1163d73" dependencies = [ "ahash", "arrow-array", @@ -2337,35 +2387,27 @@ dependencies = [ "arrow-ipc", "arrow-schema", "arrow-select", - "base64", + "base64 0.23.1", "brotli", "bytes", "chrono", "flate2", "futures", "half", - "hashbrown 0.16.1", + "hashbrown 0.17.1", "lz4_flex", - "num-bigint", + "num-bigint 0.5.1", "num-integer", "num-traits", "object_store", - "paste", "seq-macro", "simdutf8", "snap", - "thrift", "tokio", "twox-hash", "zstd", ] -[[package]] -name = "paste" -version = "1.0.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" - [[package]] name = "percent-encoding" version = "2.3.2" @@ -2402,6 +2444,26 @@ dependencies = [ "siphasher", ] +[[package]] +name = "pin-project" +version = "1.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2466b2336ed02bcdca6b294417127b90ec92038d1d5c4fbeac971a922e0e0924" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c96395f0a926bc13b1c17622aaddda1ecb55d49c8f1bf9777e4d877800a43f8b" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "pin-project-lite" version = "0.2.17" @@ -2445,7 +2507,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" dependencies = [ "proc-macro2", - "syn", + "syn 2.0.117", ] [[package]] @@ -2626,7 +2688,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "76009fbe0614077fc1a2ce255e3a1881a2e3a3527097d5dc6d8212c585e7e38b" dependencies = [ "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -2655,7 +2717,7 @@ checksum = "b7186006dcb21920990093f30e3dea63b7d6e977bf1256be20c3563a5db070da" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -2693,7 +2755,7 @@ version = "0.12.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" dependencies = [ - "base64", + "base64 0.22.1", "bytes", "futures-core", "futures-util", @@ -2940,7 +3002,7 @@ checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -2949,6 +3011,7 @@ version = "1.0.149" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86" dependencies = [ + "indexmap 2.14.0", "itoa", "memchr", "serde", @@ -2974,7 +3037,7 @@ version = "3.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "76a5c54c7310e7b8b9577c286d7e399ddd876c3e12b3ed917a8aabc4b96e9e8c" dependencies = [ - "base64", + "base64 0.22.1", "bs58", "chrono", "hex", @@ -2997,18 +3060,18 @@ dependencies = [ "darling", "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] name = "sha2" -version = "0.10.9" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +checksum = "446ba717509524cb3f22f17ecc096f10f4822d76ab5c0b9822c5f9c284e825f4" dependencies = [ "cfg-if", - "cpufeatures 0.2.17", - "digest", + "cpufeatures", + "digest 0.11.3", ] [[package]] @@ -3076,9 +3139,9 @@ dependencies = [ [[package]] name = "sqlparser" -version = "0.61.0" +version = "0.62.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dbf5ea8d4d7c808e1af1cbabebca9a2abe603bcefc22294c5b95018d53200cb7" +checksum = "13c6d1b651dc4edf07eead2a0c6c78016ce971bc2c10da5266861b13f25e7cec" dependencies = [ "log", "sqlparser_derive 0.5.0", @@ -3092,7 +3155,7 @@ checksum = "da5fc6819faabb412da764b99d3b713bb55083c11e7e0c00144d386cd6a1939c" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -3103,7 +3166,7 @@ checksum = "a6dd45d8fc1c79299bfbb7190e42ccbbdf6a5f52e4a6ad98d92357ea965bd289" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -3148,6 +3211,17 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "syn" +version = "3.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8593e8e72159ed2257d083c7a454a85cbf854f37a0966d8d483aff8c8a3ebcee" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + [[package]] name = "sync_wrapper" version = "1.0.2" @@ -3165,7 +3239,7 @@ checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -3198,18 +3272,7 @@ checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" dependencies = [ "proc-macro2", "quote", - "syn", -] - -[[package]] -name = "thrift" -version = "0.17.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e54bc85fc7faa8bc175c4bab5b92ba8d9a3ce893d0e9f42cc455c8ab16a9e09" -dependencies = [ - "byteorder", - "integer-encoding", - "ordered-float", + "syn 2.0.117", ] [[package]] @@ -3279,9 +3342,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.51.1" +version = "1.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f66bf9585cda4b724d3e78ab34b73fb2bbaba9011b9bfdf69dc836382ea13b8c" +checksum = "202caea871b69668250d242070849eb495be178ed697a3e98aebce5bc81a0bed" dependencies = [ "bytes", "libc", @@ -3300,7 +3363,7 @@ checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -3402,7 +3465,7 @@ checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -3582,7 +3645,7 @@ dependencies = [ "bumpalo", "proc-macro2", "quote", - "syn", + "syn 2.0.117", "wasm-bindgen-shared", ] @@ -3625,7 +3688,7 @@ checksum = "caf0ca1bd612b988616bac1ab34c4e4290ef18f7148a1d8b7f31c150080e9295" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -3731,7 +3794,7 @@ checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -3742,7 +3805,7 @@ checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -3964,7 +4027,7 @@ dependencies = [ "heck", "indexmap 2.14.0", "prettyplease", - "syn", + "syn 2.0.117", "wasm-metadata", "wit-bindgen-core", "wit-component", @@ -3980,7 +4043,7 @@ dependencies = [ "prettyplease", "proc-macro2", "quote", - "syn", + "syn 2.0.117", "wit-bindgen-core", "wit-bindgen-rust", ] @@ -4066,7 +4129,7 @@ name = "wren-manifest-macro" version = "0.4.0" dependencies = [ "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -4077,7 +4140,7 @@ dependencies = [ "csv", "datafusion", "datafusion-session", - "itertools", + "itertools 0.14.0", "log", "parking_lot", "petgraph", @@ -4114,7 +4177,7 @@ checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", "synstructure", ] @@ -4135,7 +4198,7 @@ checksum = "70e3cd084b1788766f53af483dd21f93881ff30d7320490ec3ef7526d203bad4" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -4155,7 +4218,7 @@ checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", "synstructure", ] @@ -4195,7 +4258,7 @@ checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] diff --git a/core/wren-core-wasm/Cargo.toml b/core/wren-core-wasm/Cargo.toml index 5e6b45ae4..aa14f7ae6 100644 --- a/core/wren-core-wasm/Cargo.toml +++ b/core/wren-core-wasm/Cargo.toml @@ -19,7 +19,7 @@ path = "src/lib.rs" # DataFusion WITHOUT parquet feature — parquet's default-features=true in # DataFusion enables zstd (C library) which cannot compile to WASM. # We read Parquet bytes → Arrow RecordBatch → MemTable ourselves. -datafusion = { version = "53", default-features = false, features = [ +datafusion = { version = "55.1", default-features = false, features = [ "nested_expressions", "crypto_expressions", "datetime_expressions", @@ -30,16 +30,16 @@ datafusion = { version = "53", default-features = false, features = [ "sql", ] } -object_store = { version = "0.13.1", features = ["aws", "http"] } +object_store = { version = "0.13.2", features = ["aws", "http"] } # --- Arrow / Parquet --- -arrow = { version = "58.1", default-features = false, features = [ +arrow = { version = "59.2", default-features = false, features = [ "json", "csv", ] } # No zstd: requires C library (zstd-sys) which cannot compile to WASM. # Snappy and LZ4 are pure Rust and WASM-compatible. -parquet = { version = "58.1", default-features = false, features = [ +parquet = { version = "59.2", default-features = false, features = [ "arrow", "snap", "lz4", @@ -89,3 +89,9 @@ getrandom_04 = { package = "getrandom", version = "0.4", features = [ [dev-dependencies] wasm-bindgen-test = "0.3" + +# Fat LTO with a single codegen unit keeps the binary under the 15 MB gzip +# budget checked in CI (~14 MB vs ~16 MB without it on DataFusion 55). +[profile.release] +lto = true +codegen-units = 1 diff --git a/core/wren-core/.claude/CLAUDE.md b/core/wren-core/.claude/CLAUDE.md index 309cfe9d5..c7d5ebe80 100644 --- a/core/wren-core/.claude/CLAUDE.md +++ b/core/wren-core/.claude/CLAUDE.md @@ -1,6 +1,6 @@ # wren-core -Rust semantic engine for Wren Engine. Handles MDL analysis, query planning, logical plan optimization, and SQL generation via Apache DataFusion (upstream, crates.io v53). Published to crates.io as `wren-semantic-core` (lib name `wren_core`). +Rust semantic engine for Wren Engine. Handles MDL analysis, query planning, logical plan optimization, and SQL generation via Apache DataFusion (upstream, crates.io v55). Published to crates.io as `wren-semantic-core` (lib name `wren_core`). ## Workspace Layout @@ -38,7 +38,7 @@ taplo fmt # Format Cargo.toml file ## Dependencies -- **DataFusion**: upstream `datafusion` v53 from crates.io (no longer the Canner fork) +- **DataFusion**: upstream `datafusion` v55 from crates.io (no longer the Canner fork) - **wren-core-base**: Shared manifest types from `../wren-core-base` ## Known Limitations diff --git a/core/wren-core/Cargo.toml b/core/wren-core/Cargo.toml index cbd9000bb..720889643 100644 --- a/core/wren-core/Cargo.toml +++ b/core/wren-core/Cargo.toml @@ -9,12 +9,12 @@ homepage = "https://getwren.ai" license = "Apache-2.0" readme = "README.md" repository = "https://github.com/Canner/WrenAI" -rust-version = "1.78" +rust-version = "1.94" version = "0.4.0" [workspace.dependencies] async-trait = "0.1.89" -datafusion = { version = "53", default-features = false, features = [ +datafusion = { version = "55.1", default-features = false, features = [ "nested_expressions", "crypto_expressions", "datetime_expressions", @@ -24,7 +24,7 @@ datafusion = { version = "53", default-features = false, features = [ "parquet", "sql", ] } -datafusion-session = { version = "53", default-features = false } +datafusion-session = { version = "55.1", default-features = false } env_logger = "0.11.10" hashbrown = "0.17.0" insta = { version = "1.41.1" } diff --git a/core/wren-core/core/src/logical_plan/analyze/access_control.rs b/core/wren-core/core/src/logical_plan/analyze/access_control.rs index 2eef97731..5e1f3267f 100644 --- a/core/wren-core/core/src/logical_plan/analyze/access_control.rs +++ b/core/wren-core/core/src/logical_plan/analyze/access_control.rs @@ -8,13 +8,13 @@ use datafusion::{ arrow::datatypes::{DataType, SchemaRef}, common::{ config::ConfigOptions, file_options::file_type::FileType, plan_datafusion_err, - plan_err, Result, Spans, + plan_err, Result, Spans, TableReference, }, error::DataFusionError, logical_expr::{ builder::LogicalTableSource, planner::{ContextProvider, ExprPlanner, TypePlanner}, - AggregateUDF, ScalarUDF, TableSource, WindowUDF, + AggregateUDF, HigherOrderUDF, ScalarUDF, TableSource, WindowUDF, }, prelude::Expr, sql::{ @@ -27,7 +27,6 @@ use datafusion::{ }, dialect::GenericDialect, }, - TableReference, }, }; use wren_core_base::mdl::RowLevelAccessControl; @@ -395,6 +394,14 @@ impl ContextProvider for RlacContextProvider { .cloned() } + fn get_higher_order_meta(&self, name: &str) -> Option> { + self.session_state + .read() + .higher_order_functions() + .get(name) + .cloned() + } + fn get_aggregate_meta(&self, name: &str) -> Option> { self.session_state .read() @@ -428,6 +435,15 @@ impl ContextProvider for RlacContextProvider { .collect() } + fn higher_order_function_names(&self) -> Vec { + self.session_state + .read() + .higher_order_functions() + .keys() + .cloned() + .collect() + } + fn udaf_names(&self) -> Vec { self.session_state .read() diff --git a/core/wren-core/core/src/logical_plan/analyze/model_anlayze.rs b/core/wren-core/core/src/logical_plan/analyze/model_anlayze.rs index c282abc23..f542e75b5 100644 --- a/core/wren-core/core/src/logical_plan/analyze/model_anlayze.rs +++ b/core/wren-core/core/src/logical_plan/analyze/model_anlayze.rs @@ -7,6 +7,7 @@ use crate::mdl::{AnalyzedWrenMDL, Dataset, SessionStateRef}; use datafusion::common::tree_node::{ Transformed, TransformedResult, TreeNode, TreeNodeRecursion, TreeNodeRewriter, }; +use datafusion::common::TableReference; use datafusion::common::{internal_err, plan_err, Column, DFSchemaRef, Result, Spans}; use datafusion::config::ConfigOptions; use datafusion::error::DataFusionError; @@ -17,7 +18,6 @@ use datafusion::logical_expr::{ Window, }; use datafusion::optimizer::AnalyzerRule; -use datafusion::sql::TableReference; use std::cell::RefCell; use std::collections::HashSet; use std::fmt::Debug; diff --git a/core/wren-core/core/src/logical_plan/analyze/model_generation.rs b/core/wren-core/core/src/logical_plan/analyze/model_generation.rs index da37916ae..4e837b068 100644 --- a/core/wren-core/core/src/logical_plan/analyze/model_generation.rs +++ b/core/wren-core/core/src/logical_plan/analyze/model_generation.rs @@ -17,12 +17,12 @@ use crate::DataFusionError; use datafusion::common::alias::AliasGenerator; use datafusion::common::config::ConfigOptions; use datafusion::common::tree_node::{Transformed, TransformedResult}; +use datafusion::common::TableReference; use datafusion::common::{plan_err, Result}; use datafusion::logical_expr::{col, ident, Extension, UserDefinedLogicalNodeCore}; use datafusion::logical_expr::{Expr, LogicalPlan, LogicalPlanBuilder}; use datafusion::optimizer::analyzer::AnalyzerRule; use datafusion::physical_plan::internal_err; -use datafusion::sql::TableReference; pub const SOURCE_ALIAS: &str = "__source"; diff --git a/core/wren-core/core/src/logical_plan/analyze/scope.rs b/core/wren-core/core/src/logical_plan/analyze/scope.rs index 644e08ccc..9ee41843a 100644 --- a/core/wren-core/core/src/logical_plan/analyze/scope.rs +++ b/core/wren-core/core/src/logical_plan/analyze/scope.rs @@ -1,8 +1,8 @@ use crate::mdl::Dataset; use datafusion::common::internal_err; +use datafusion::common::TableReference; use datafusion::error::Result; use datafusion::prelude::Expr; -use datafusion::sql::TableReference; use std::collections::{HashMap, HashSet, VecDeque}; use std::fmt::{Debug, Display}; diff --git a/core/wren-core/core/src/logical_plan/optimize/simplify_timestamp.rs b/core/wren-core/core/src/logical_plan/optimize/simplify_timestamp.rs index 38ad21c5d..8ee333552 100644 --- a/core/wren-core/core/src/logical_plan/optimize/simplify_timestamp.rs +++ b/core/wren-core/core/src/logical_plan/optimize/simplify_timestamp.rs @@ -79,7 +79,7 @@ impl TimestampSimplify { } else { Arc::new(DFSchema::empty()) }; - let info = SimplifyContext::default().with_schema(schema); + let info = SimplifyContext::builder().with_schema(schema).build(); // Inputs have already been rewritten (due to bottom-up traversal handled by Optimizer) // Just need to rewrite our own expressions @@ -123,13 +123,13 @@ impl TreeNodeRewriter for ExprRewriter<'_> { match &expr { // we only simplify the cast expression for the literal value Expr::Cast(Cast { - data_type, + field, expr: sub_expr, }) | Expr::TryCast(TryCast { - data_type, + field, expr: sub_expr, - }) if is_timestamp(data_type) + }) if is_timestamp(field.data_type()) && matches!(sub_expr.as_ref(), Expr::Literal(_, _)) => { let original_name = self.name_preserver.save(&expr); diff --git a/core/wren-core/core/src/logical_plan/optimize/type_coercion.rs b/core/wren-core/core/src/logical_plan/optimize/type_coercion.rs index b69c855b0..b4f4c1151 100644 --- a/core/wren-core/core/src/logical_plan/optimize/type_coercion.rs +++ b/core/wren-core/core/src/logical_plan/optimize/type_coercion.rs @@ -25,32 +25,35 @@ use datafusion::{ common::{ exec_err, internal_err, not_impl_err, plan_datafusion_err, plan_err, tree_node::{Transformed, TreeNode, TreeNodeRewriter}, - Column, DFSchema, DFSchemaRef, + Column, DFSchema, DFSchemaRef, TableReference, }, config::ConfigOptions, error::{DataFusionError, Result}, logical_expr::{ - binary::{comparison_coercion, like_coercion, BinaryTypeCoercer}, + binary::{ + comparison_coercion, like_coercion, type_union_coercion, BinaryTypeCoercer, + }, expr::{ - self, AggregateFunction, AggregateFunctionParams, Alias, Exists, InList, - InSubquery, ScalarFunction, SetComparison, WindowFunction, - WindowFunctionParams, + self, AggregateFunction, AggregateFunctionParams, Alias, Exists, + HigherOrderFunction, InList, InSubquery, ScalarFunction, SetComparison, + WindowFunction, WindowFunctionParams, }, expr_rewriter::coerce_plan_expr_for_schema, expr_schema::cast_subquery, type_coercion::{ - functions::fields_with_udf, + functions::{ + fields_with_udf, value_fields_with_higher_order_udf_and_lambdas, + }, is_datetime, - other::{get_coerce_type_for_case_expression, get_coerce_type_for_list}, + other::{get_coerce_type_for_case_expression, get_coerce_type_for_case_when}, }, utils::merge_schema, Between, BinaryExpr, Case, ExprSchemable, Join, Like, Limit, LogicalPlan, - Operator, Projection, Subquery, Union, WindowFunctionDefinition, + Operator, Projection, Subquery, Union, ValueOrLambda, WindowFunctionDefinition, }, optimizer::{utils::NamePreserver, AnalyzerRule}, prelude::Expr, scalar::ScalarValue, - sql::TableReference, }; use datafusion::logical_expr::{ @@ -535,8 +538,15 @@ impl TreeNodeRewriter for TypeCoercionRewriter<'_> { .iter() .map(|list_expr| list_expr.get_type(self.schema)) .collect::>>()?; - let result_type = - get_coerce_type_for_list(&expr_data_type, &list_data_types); + // DataFusion coerces IN lists with the numeric-preferring + // `comparison_coercion`, which would cast a string column to a number + // (`CAST(s AS BIGINT) IN (1, 2)`) and fail in the target database on + // non-numeric values. Keep the string-preferring coercion instead. + let result_type = list_data_types + .iter() + .try_fold(expr_data_type.clone(), |left_type, right_type| { + type_union_coercion(&left_type, right_type) + }); match result_type { None => plan_err!( "Can not find compatible types to compare {expr_data_type:?} with {list_data_types:?}" @@ -641,6 +651,41 @@ impl TreeNodeRewriter for TypeCoercionRewriter<'_> { }, }))) } + Expr::HigherOrderFunction(HigherOrderFunction { func, args }) => { + let current_fields = args + .iter() + .map(|arg| match arg { + Expr::Lambda(lambda) => Ok(ValueOrLambda::Lambda( + lambda.body.to_field(self.schema)?.1, + )), + _ => Ok(ValueOrLambda::Value(arg.to_field(self.schema)?.1)), + }) + .collect::>>()?; + + let new_fields = value_fields_with_higher_order_udf_and_lambdas( + ¤t_fields, + func.as_ref(), + )?; + + let new_args = std::iter::zip(args, new_fields) + .map(|(arg, new_field)| match (&arg, new_field) { + (Expr::Lambda(_), ValueOrLambda::Lambda(_)) => Ok(arg), + (Expr::Lambda(_), ValueOrLambda::Value(_)) => internal_err!( + "value_fields_with_higher_order_udf returned a value for a lambda argument" + ), + (_, ValueOrLambda::Value(new_field)) => { + arg.cast_to(new_field.data_type(), self.schema) + } + (_, ValueOrLambda::Lambda(_)) => internal_err!( + "value_fields_with_higher_order_udf returned a lambda for a value argument" + ), + }) + .collect::>()?; + + Ok(Transformed::yes(Expr::HigherOrderFunction( + HigherOrderFunction::new(func, new_args), + ))) + } // TODO: remove the next line after `Expr::Wildcard` is removed #[expect(deprecated)] Expr::Alias(_) @@ -656,7 +701,9 @@ impl TreeNodeRewriter for TypeCoercionRewriter<'_> { | Expr::Wildcard { .. } | Expr::GroupingSet(_) | Expr::Placeholder(_) - | Expr::OuterReferenceColumn(_, _) => Ok(Transformed::no(expr)), + | Expr::OuterReferenceColumn(_, _) + | Expr::Lambda(_) + | Expr::LambdaVariable(_) => Ok(Transformed::no(expr)), } } } @@ -967,8 +1014,7 @@ fn coerce_case_expression(case: Case, schema: &DFSchema) -> Result { .iter() .map(|(when, _then)| when.get_type(schema)) .collect::>>()?; - let coerced_type = - get_coerce_type_for_case_expression(&when_types, Some(case_type)); + let coerced_type = get_coerce_type_for_case_when(&when_types, case_type); coerced_type.ok_or_else(|| { plan_datafusion_err!( "Failed to coerce case ({case_type:?}) and when ({when_types:?}) \ @@ -1072,7 +1118,7 @@ fn coerce_union_schema_with_schema( plan_schema.fields().iter() ) { let coerced_type = - comparison_coercion(union_datatype, plan_field.data_type()).ok_or_else( + type_union_coercion(union_datatype, plan_field.data_type()).ok_or_else( || { plan_datafusion_err!( "Incompatible inputs for Union: Previous inputs were \ diff --git a/core/wren-core/core/src/logical_plan/utils.rs b/core/wren-core/core/src/logical_plan/utils.rs index 1540c1447..246224d0d 100644 --- a/core/wren-core/core/src/logical_plan/utils.rs +++ b/core/wren-core/core/src/logical_plan/utils.rs @@ -13,6 +13,7 @@ use datafusion::common::types::{ logical_binary, logical_boolean, logical_date, logical_float16, logical_float32, logical_float64, logical_string, }; +use datafusion::common::TableReference; use datafusion::common::{plan_err, DFSchema, DFSchemaRef}; use datafusion::datasource::DefaultTableSource; use datafusion::error::Result; @@ -22,7 +23,6 @@ use datafusion::logical_expr::{builder::LogicalTableSource, Expr, TableSource}; use datafusion::logical_expr::{Coercion, TypeSignatureClass}; use datafusion::sql::sqlparser::ast; use datafusion::sql::sqlparser::parser::Parser; -use datafusion::sql::TableReference; use log::debug; use petgraph::dot::{Config, Dot}; use petgraph::Graph; @@ -406,7 +406,10 @@ pub fn expr_to_columns( | Expr::InSubquery(_) | Expr::ScalarSubquery(_) | Expr::Wildcard { .. } - | Expr::Placeholder(_) => {} + | Expr::Placeholder(_) + | Expr::HigherOrderFunction(_) + | Expr::Lambda(_) + | Expr::LambdaVariable(_) => {} Expr::SetComparison(_) => {} } Ok(TreeNodeRecursion::Continue) diff --git a/core/wren-core/core/src/mdl/context.rs b/core/wren-core/core/src/mdl/context.rs index 1a8ce75ec..c63c57fe7 100644 --- a/core/wren-core/core/src/mdl/context.rs +++ b/core/wren-core/core/src/mdl/context.rs @@ -1,4 +1,3 @@ -use std::any::Any; use std::collections::HashMap; use std::ops::Deref; use std::sync::Arc; @@ -19,6 +18,7 @@ use datafusion::catalog::memory::{MemoryCatalogProvider, MemoryCatalogProviderLi use datafusion::catalog::CatalogProvider; use datafusion::catalog::CatalogProviderList; use datafusion::catalog::{MemorySchemaProvider, Session}; +use datafusion::common::TableReference; use datafusion::common::{internal_err, Result}; use datafusion::datasource::{TableProvider, TableType, ViewTable}; use datafusion::execution::session_state::SessionStateBuilder; @@ -36,7 +36,6 @@ use datafusion::optimizer::{AnalyzerRule, OptimizerRule}; use datafusion::physical_plan::ExecutionPlan; use datafusion::prelude::SessionContext; use datafusion::scalar::ScalarValue; -use datafusion::sql::TableReference; use parking_lot::RwLock; pub type SessionPropertiesRef = Arc>>; @@ -429,10 +428,6 @@ impl WrenDataSource { #[async_trait] impl TableProvider for WrenDataSource { - fn as_any(&self) -> &dyn Any { - self - } - fn schema(&self) -> SchemaRef { self.schema.clone() } diff --git a/core/wren-core/core/src/mdl/function/remote_function.rs b/core/wren-core/core/src/mdl/function/remote_function.rs index 9f5bfa68c..25ecffea7 100644 --- a/core/wren-core/core/src/mdl/function/remote_function.rs +++ b/core/wren-core/core/src/mdl/function/remote_function.rs @@ -10,7 +10,6 @@ use datafusion::logical_expr::{ Signature, TypeSignature, Volatility, WindowUDFImpl, }; use serde::{Deserialize, Serialize}; -use std::any::Any; use std::fmt::Display; use std::str::FromStr; use std::sync::Arc; @@ -288,10 +287,6 @@ fn build_document(func: &RemoteFunction) -> Documentation { } impl ScalarUDFImpl for ByPassScalarUDF { - fn as_any(&self) -> &dyn Any { - self - } - fn name(&self) -> &str { &self.name } @@ -389,10 +384,6 @@ impl From for ByPassAggregateUDF { } impl AggregateUDFImpl for ByPassAggregateUDF { - fn as_any(&self) -> &dyn Any { - self - } - fn name(&self) -> &str { &self.name } @@ -470,10 +461,6 @@ impl From for ByPassWindowFunction { } impl WindowUDFImpl for ByPassWindowFunction { - fn as_any(&self) -> &dyn Any { - self - } - fn name(&self) -> &str { &self.name } diff --git a/core/wren-core/core/src/mdl/function/scalar/to_char.rs b/core/wren-core/core/src/mdl/function/scalar/to_char.rs index 3e62aa07b..4b7aa94a1 100644 --- a/core/wren-core/core/src/mdl/function/scalar/to_char.rs +++ b/core/wren-core/core/src/mdl/function/scalar/to_char.rs @@ -1,5 +1,3 @@ -use std::any::Any; - use datafusion::arrow::datatypes::DataType; use datafusion::common::not_impl_err; use datafusion::common::types::{ @@ -53,10 +51,6 @@ impl ToCharFunc { } impl ScalarUDFImpl for ToCharFunc { - fn as_any(&self) -> &dyn Any { - self - } - fn name(&self) -> &str { "to_char" } diff --git a/core/wren-core/core/src/mdl/lineage.rs b/core/wren-core/core/src/mdl/lineage.rs index 9402c04a2..96e1dd3a1 100644 --- a/core/wren-core/core/src/mdl/lineage.rs +++ b/core/wren-core/core/src/mdl/lineage.rs @@ -2,9 +2,9 @@ use std::collections::{HashMap, HashSet}; use std::fmt::Display; use std::sync::Arc; +use datafusion::common::TableReference; use datafusion::common::{internal_err, plan_err, Column}; use datafusion::error::Result; -use datafusion::sql::TableReference; use petgraph::Graph; use crate::logical_plan::utils::from_qualified_name; @@ -388,9 +388,9 @@ fn get_dataset_link_revers_if_need( #[cfg(test)] mod test { + use datafusion::common::TableReference; use datafusion::common::{Column, Spans}; use datafusion::error::Result; - use datafusion::sql::TableReference; use std::collections::{HashMap, HashSet}; use std::sync::Arc; diff --git a/core/wren-core/core/src/mdl/mod.rs b/core/wren-core/core/src/mdl/mod.rs index 1ebe0b64e..6824405c1 100644 --- a/core/wren-core/core/src/mdl/mod.rs +++ b/core/wren-core/core/src/mdl/mod.rs @@ -13,6 +13,7 @@ use crate::mdl::utils::{dequote_identifier, quoted, to_field}; use crate::DataFusionError; use context::SessionPropertiesRef; use datafusion::arrow::datatypes::Field; +use datafusion::common::TableReference; use datafusion::common::{internal_datafusion_err, plan_err}; use datafusion::datasource::TableProvider; use datafusion::error::Result; @@ -24,7 +25,6 @@ use datafusion::sql::parser::DFParser; use datafusion::sql::sqlparser::ast::{Expr, ExprWithAlias, Ident}; use datafusion::sql::sqlparser::dialect::dialect_from_str; use datafusion::sql::unparser::Unparser; -use datafusion::sql::TableReference; pub use dataset::Dataset; use dialect::WrenDialect; use log::{debug, info, warn}; @@ -944,6 +944,56 @@ mod test { Ok(()) } + #[tokio::test] + async fn test_in_list_mixed_string_numeric_prefers_string() -> Result<()> { + let manifest = ManifestBuilder::new() + .catalog("wren") + .schema("test") + .model( + ModelBuilder::new("customer") + .table_reference("customer") + .column(ColumnBuilder::new("c_custkey", "int").build()) + .column(ColumnBuilder::new("c_name", "varchar").build()) + .build(), + ) + .build(); + let analyzed_mdl = Arc::new(AnalyzedWrenMDL::analyze( + manifest, + Arc::new(HashMap::default()), + Mode::Unparse, + )?); + let ctx = create_wren_ctx(None, analyzed_mdl.wren_mdl().data_source().as_ref()); + + let sql = "select c_custkey from customer where c_name in (1, 2)"; + let actual = mdl::transform_sql_with_ctx( + &ctx, + Arc::clone(&analyzed_mdl), + &[], + Arc::new(HashMap::new()), + sql, + ) + .await?; + assert_snapshot!(actual, + @"SELECT customer.c_custkey FROM (SELECT customer.c_custkey, customer.c_name FROM \ + (SELECT __source.c_custkey AS c_custkey, __source.c_name AS c_name FROM customer AS __source) AS customer) AS customer \ + WHERE customer.c_name IN (CAST(1 AS VARCHAR), CAST(2 AS VARCHAR))"); + + let sql = "select c_custkey from customer where c_custkey in ('1', '2')"; + let actual = mdl::transform_sql_with_ctx( + &ctx, + Arc::clone(&analyzed_mdl), + &[], + Arc::new(HashMap::new()), + sql, + ) + .await?; + assert_snapshot!(actual, + @"SELECT customer.c_custkey FROM (SELECT customer.c_custkey FROM \ + (SELECT __source.c_custkey AS c_custkey FROM customer AS __source) AS customer) AS customer \ + WHERE CAST(customer.c_custkey AS VARCHAR) IN ('1', '2')"); + Ok(()) + } + #[tokio::test] async fn test_remote_function() -> Result<()> { env_logger::init(); @@ -1210,7 +1260,10 @@ mod test { .await.map_err(|e| { assert_snapshot!( e.to_string(), - @"Schema error: No field named \"名字\". Valid fields are wren.test.artist.\"串接名字\"." + @r#" + Schema error: No field named "名字". Did you mean 'wren.test.artist."串接名字"'? + Valid fields are wren.test.artist."串接名字". + "# ) }); Ok(()) @@ -3076,7 +3129,10 @@ mod test { panic!("whitelist_name is hidden, it should not be selected directly") } Err(e) => { - assert_snapshot!(e.to_string(), @"Schema error: No field named whitelist_name. Valid fields are customer.c_custkey, customer.mock_id.") + assert_snapshot!(e.to_string(), @r" + Schema error: No field named whitelist_name. + Valid fields are customer.c_custkey, customer.mock_id. + ") } } @@ -3227,7 +3283,10 @@ mod test { Err(e) => { assert_snapshot!( e.to_string(), - @"Schema error: No field named c_name. Valid fields are customer.c_custkey." + @r" + Schema error: No field named c_name. + Valid fields are customer.c_custkey. + " ) } } @@ -5235,7 +5294,7 @@ mod test { use datafusion::catalog::memory::MemoryCatalogProvider; use datafusion::catalog::CatalogProvider; use datafusion::catalog::SchemaProvider; - use datafusion::sql::TableReference; + use datafusion::common::TableReference; use std::sync::Barrier; use std::thread; diff --git a/core/wren-core/sqllogictest/test_files/model.slt b/core/wren-core/sqllogictest/test_files/model.slt index 8ba81476e..bc02c5ff2 100644 --- a/core/wren-core/sqllogictest/test_files/model.slt +++ b/core/wren-core/sqllogictest/test_files/model.slt @@ -72,7 +72,7 @@ select "Customer_id" from wrenai.public."Orders" where exists (select 1 from wre 04eafb40a16989307464f27f1fed8907 0732c0881c70ebcda536a4b14e9db106 -query RIITRTTT +query RIITRTTT rowsort select * from "Order_items" where "Order_id" in ('03c83b31dbc387f83f1b5579b53182fb', '08cbb1d4cd574b126569b208fd4b26ea') ---- 14.68 1 1 03c83b31dbc387f83f1b5579b53182fb 119.8 a04087ab6a96ffa041f8a2701a72b616 2023/1/15 7:26 CA diff --git a/core/wren-core/sqllogictest/test_files/tpch/q22.slt.part b/core/wren-core/sqllogictest/test_files/tpch/q22.slt.part index 96e679c37..deeb2c7bd 100644 --- a/core/wren-core/sqllogictest/test_files/tpch/q22.slt.part +++ b/core/wren-core/sqllogictest/test_files/tpch/q22.slt.part @@ -16,7 +16,7 @@ # specific language governing permissions and limitations # under the License. -query ?IR +query TIR select cntrycode, count(*) as numcust,