mirror of
https://github.com/Canner/WrenAI.git
synced 2026-09-01 15:34:04 +08:00
refactor(wren): drop ibis for canner/postgres/mysql/mssql/trino/clickhouse/athena (combined) (#2313)
Co-authored-by: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
+10
-1
@@ -57,11 +57,20 @@ test-postgres:
|
||||
uv run pytest tests/connectors/test_postgres.py -v -m postgres
|
||||
|
||||
test-mysql:
|
||||
uv run pytest tests/connectors/test_mysql.py -v -m mysql
|
||||
uv run pytest tests/connectors/test_mysql.py tests/connectors/test_mysql_connector.py -v -m mysql
|
||||
|
||||
test-snowflake:
|
||||
uv run pytest tests/connectors/test_snowflake.py -v -m snowflake
|
||||
|
||||
test-canner:
|
||||
uv run pytest tests/connectors/test_canner.py -v -m canner
|
||||
|
||||
test-clickhouse:
|
||||
uv run pytest tests/connectors/test_clickhouse.py -v -m clickhouse
|
||||
|
||||
test-trino:
|
||||
uv run pytest tests/connectors/test_trino.py -v -m trino
|
||||
|
||||
test-connector marker:
|
||||
uv run pytest tests/connectors/ -v -m {{ marker }}
|
||||
|
||||
|
||||
@@ -43,17 +43,17 @@ dependencies = [
|
||||
]
|
||||
|
||||
[project.optional-dependencies]
|
||||
postgres = ["psycopg>=3", "ibis-framework[postgres]"]
|
||||
mysql = ["mysqlclient>=2.2", "ibis-framework[mysql]"]
|
||||
postgres = ["psycopg[binary]>=3"]
|
||||
mysql = ["mysqlclient>=2.2"]
|
||||
bigquery = ["ibis-framework[bigquery]", "google-auth"]
|
||||
snowflake = ["snowflake-connector-python[pandas]>=3.10"]
|
||||
clickhouse = ["ibis-framework[clickhouse]"]
|
||||
trino = ["ibis-framework[trino]", "trino>=0.321"]
|
||||
mssql = ["ibis-framework[mssql]"]
|
||||
clickhouse = ["clickhouse-connect>=0.8"]
|
||||
trino = ["trino>=0.333,<1"]
|
||||
mssql = ["pyodbc>=5,<6"]
|
||||
databricks = ["databricks-sql-connector", "databricks-sdk"]
|
||||
redshift = ["redshift_connector"]
|
||||
spark = ["pyspark>=3.5"]
|
||||
athena = ["ibis-framework[athena]"]
|
||||
athena = ["pyathena[pandas]>=3"]
|
||||
oracle = ["oracledb>=2"]
|
||||
memory = ["lancedb>=0.6", "sentence-transformers>=2.2"]
|
||||
interactive = ["InquirerPy>=0.3.4"]
|
||||
|
||||
@@ -0,0 +1,338 @@
|
||||
"""Native pyathena connector — bypasses the ibis athena backend.
|
||||
|
||||
Athena's wire types are Trino-flavoured (varchar, decimal(p,s), array<T>,
|
||||
row(...), map<K,V>, ...). This module parses those type strings via sqlglot
|
||||
and materialises cursor results into PyArrow tables directly, so we no longer
|
||||
depend on ibis-framework[athena].
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import contextlib
|
||||
import datetime as dtlib
|
||||
import json
|
||||
from decimal import Decimal as PyDecimal
|
||||
from typing import Any
|
||||
|
||||
import pyarrow as pa
|
||||
|
||||
from wren.connector.base import ConnectorABC
|
||||
from wren.model.error import DIALECT_SQL, ErrorCode, ErrorPhase, WrenError
|
||||
|
||||
# Athena's DB-API cursor returns Trino-style type names. We delegate the
|
||||
# lexing to sqlglot so we get nested type support (array<row<a int, b varchar>>,
|
||||
# decimal(p, s), map<K, V>, etc.) for free.
|
||||
_TRINO_DATA_TYPE_TO_ARROW: dict = {}
|
||||
|
||||
|
||||
def _init_trino_data_type_map() -> None:
|
||||
if _TRINO_DATA_TYPE_TO_ARROW:
|
||||
return
|
||||
from sqlglot.expressions import DataType # noqa: PLC0415
|
||||
|
||||
T = DataType.Type
|
||||
_TRINO_DATA_TYPE_TO_ARROW.update(
|
||||
{
|
||||
T.BOOLEAN: pa.bool_(),
|
||||
T.TINYINT: pa.int8(),
|
||||
T.SMALLINT: pa.int16(),
|
||||
T.INT: pa.int32(),
|
||||
T.BIGINT: pa.int64(),
|
||||
T.FLOAT: pa.float32(),
|
||||
T.DOUBLE: pa.float64(),
|
||||
T.VARCHAR: pa.string(),
|
||||
T.CHAR: pa.string(),
|
||||
T.NCHAR: pa.string(),
|
||||
T.NVARCHAR: pa.string(),
|
||||
T.TEXT: pa.string(),
|
||||
T.JSON: pa.string(),
|
||||
T.UUID: pa.string(),
|
||||
T.IPADDRESS: pa.string(),
|
||||
T.HLLSKETCH: pa.string(), # hyperloglog
|
||||
T.GEOMETRY: pa.string(),
|
||||
T.VARBINARY: pa.binary(),
|
||||
T.BINARY: pa.binary(),
|
||||
T.DATE: pa.date32(),
|
||||
T.TIME: pa.time64("us"),
|
||||
T.TIMETZ: pa.time64("us"),
|
||||
T.TIMESTAMP: pa.timestamp("ms"),
|
||||
T.TIMESTAMPTZ: pa.timestamp("ms", tz="UTC"),
|
||||
T.TIMESTAMPLTZ: pa.timestamp("ms", tz="UTC"),
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _parse_athena_type(type_str: str | None) -> pa.DataType:
|
||||
"""Parse an Athena/Trino cursor type string into a PyArrow type."""
|
||||
if not type_str:
|
||||
return pa.string()
|
||||
from sqlglot import parse_one # noqa: PLC0415
|
||||
from sqlglot.expressions import DataType # noqa: PLC0415
|
||||
|
||||
try:
|
||||
parsed = parse_one(type_str, into=DataType, dialect="trino")
|
||||
except Exception:
|
||||
return pa.string()
|
||||
if parsed is None:
|
||||
return pa.string()
|
||||
return _trino_data_type_to_arrow(parsed)
|
||||
|
||||
|
||||
def _trino_data_type_to_arrow(node) -> pa.DataType:
|
||||
from sqlglot.expressions import ColumnDef, DataType # noqa: PLC0415
|
||||
|
||||
_init_trino_data_type_map()
|
||||
if not isinstance(node, DataType):
|
||||
return pa.string()
|
||||
|
||||
kind = node.this
|
||||
T = DataType.Type
|
||||
if kind in _TRINO_DATA_TYPE_TO_ARROW:
|
||||
return _TRINO_DATA_TYPE_TO_ARROW[kind]
|
||||
|
||||
if kind == T.DECIMAL:
|
||||
precision, scale = 38, 9
|
||||
params = node.expressions
|
||||
if len(params) >= 1:
|
||||
with contextlib.suppress(AttributeError, ValueError, TypeError):
|
||||
precision = min(int(params[0].this.this), 38)
|
||||
if len(params) >= 2:
|
||||
with contextlib.suppress(AttributeError, ValueError, TypeError):
|
||||
scale = min(int(params[1].this.this), precision)
|
||||
return pa.decimal128(precision, scale)
|
||||
|
||||
if kind == T.ARRAY:
|
||||
inner = node.expressions[0] if node.expressions else None
|
||||
return pa.list_(_trino_data_type_to_arrow(inner) if inner else pa.string())
|
||||
|
||||
if kind == T.MAP:
|
||||
if len(node.expressions) >= 2:
|
||||
return pa.map_(
|
||||
_trino_data_type_to_arrow(node.expressions[0]),
|
||||
_trino_data_type_to_arrow(node.expressions[1]),
|
||||
)
|
||||
return pa.string()
|
||||
|
||||
if kind == T.STRUCT:
|
||||
fields: list[pa.Field] = []
|
||||
for idx, child in enumerate(node.expressions):
|
||||
if isinstance(child, ColumnDef):
|
||||
name = child.name or f"f{idx}"
|
||||
inner = child.args.get("kind")
|
||||
fields.append(
|
||||
pa.field(
|
||||
name,
|
||||
_trino_data_type_to_arrow(inner) if inner else pa.string(),
|
||||
)
|
||||
)
|
||||
else:
|
||||
fields.append(pa.field(f"f{idx}", _trino_data_type_to_arrow(child)))
|
||||
return pa.struct(fields)
|
||||
|
||||
return pa.string()
|
||||
|
||||
|
||||
def _build_athena_column(values: list, arrow_type: pa.DataType) -> pa.Array:
|
||||
"""Coerce pyathena cursor values into a PyArrow array of arrow_type."""
|
||||
if pa.types.is_string(arrow_type):
|
||||
processed: list[Any] = []
|
||||
for v in values:
|
||||
if v is None:
|
||||
processed.append(None)
|
||||
elif isinstance(v, dict | list | tuple):
|
||||
processed.append(json.dumps(v, default=str))
|
||||
elif isinstance(v, str):
|
||||
processed.append(v)
|
||||
else:
|
||||
processed.append(str(v))
|
||||
return pa.array(processed, type=pa.string(), from_pandas=True)
|
||||
|
||||
if pa.types.is_binary(arrow_type):
|
||||
processed = [bytes(v) if isinstance(v, memoryview) else v for v in values]
|
||||
return pa.array(processed, type=arrow_type, from_pandas=True)
|
||||
|
||||
if pa.types.is_decimal(arrow_type):
|
||||
processed = [
|
||||
None
|
||||
if v is None
|
||||
else (v if isinstance(v, PyDecimal) else PyDecimal(str(v)))
|
||||
for v in values
|
||||
]
|
||||
return pa.array(processed, type=arrow_type, from_pandas=True)
|
||||
|
||||
if pa.types.is_timestamp(arrow_type):
|
||||
processed = []
|
||||
for v in values:
|
||||
if v is None or isinstance(v, dtlib.datetime):
|
||||
processed.append(v)
|
||||
else:
|
||||
try:
|
||||
processed.append(dtlib.datetime.fromisoformat(str(v)))
|
||||
except ValueError:
|
||||
processed.append(None)
|
||||
return pa.array(processed, type=arrow_type, from_pandas=True)
|
||||
|
||||
if pa.types.is_date(arrow_type):
|
||||
processed = []
|
||||
for v in values:
|
||||
if v is None or isinstance(v, dtlib.date):
|
||||
processed.append(v)
|
||||
else:
|
||||
try:
|
||||
processed.append(dtlib.date.fromisoformat(str(v)))
|
||||
except ValueError:
|
||||
processed.append(None)
|
||||
return pa.array(processed, type=arrow_type, from_pandas=True)
|
||||
|
||||
if pa.types.is_time(arrow_type):
|
||||
processed = []
|
||||
for v in values:
|
||||
if v is None or isinstance(v, dtlib.time):
|
||||
processed.append(v)
|
||||
else:
|
||||
try:
|
||||
processed.append(dtlib.time.fromisoformat(str(v)))
|
||||
except ValueError:
|
||||
processed.append(None)
|
||||
return pa.array(processed, type=arrow_type, from_pandas=True)
|
||||
|
||||
if pa.types.is_map(arrow_type):
|
||||
# pyathena returns dicts for map columns; PyArrow's map_ wants iterables
|
||||
# of (key, value) pairs.
|
||||
processed = [None if v is None else list(v.items()) for v in values]
|
||||
return pa.array(processed, type=arrow_type, from_pandas=True)
|
||||
|
||||
return pa.array(values, type=arrow_type, from_pandas=True)
|
||||
|
||||
|
||||
def _build_athena_arrow_table(cursor) -> pa.Table:
|
||||
"""Materialise a pyathena DB-API cursor into a PyArrow table."""
|
||||
if cursor.description is None:
|
||||
return pa.table({})
|
||||
|
||||
rows = cursor.fetchall()
|
||||
fields = [
|
||||
pa.field(col[0], _parse_athena_type(col[1]), nullable=True)
|
||||
for col in cursor.description
|
||||
]
|
||||
schema = pa.schema(fields)
|
||||
|
||||
if not rows:
|
||||
arrays = [pa.array([], type=field.type) for field in schema]
|
||||
else:
|
||||
arrays = [
|
||||
_build_athena_column([row[i] for row in rows], schema.field(i).type)
|
||||
for i in range(len(fields))
|
||||
]
|
||||
|
||||
return pa.table(
|
||||
dict(zip([f.name for f in fields], arrays, strict=False)),
|
||||
schema=schema,
|
||||
)
|
||||
|
||||
|
||||
def _build_connect_kwargs(connection_info) -> dict[str, Any]:
|
||||
"""Translate AthenaConnectionInfo into pyathena.connect() kwargs.
|
||||
|
||||
Resolves credentials in priority order:
|
||||
1. Web Identity Token (OIDC) → STS AssumeRoleWithWebIdentity
|
||||
2. Explicit aws_access_key_id / aws_secret_access_key (+ optional session token)
|
||||
3. Default AWS credential provider chain (env, profile, instance role, …)
|
||||
"""
|
||||
import boto3 # noqa: PLC0415
|
||||
|
||||
kwargs: dict[str, Any] = {
|
||||
"s3_staging_dir": connection_info.s3_staging_dir.get_secret_value(),
|
||||
}
|
||||
if getattr(connection_info, "region_name", None):
|
||||
kwargs["region_name"] = connection_info.region_name
|
||||
if getattr(connection_info, "schema_name", None):
|
||||
kwargs["schema_name"] = connection_info.schema_name
|
||||
|
||||
web_identity_token = getattr(connection_info, "web_identity_token", None)
|
||||
role_arn = getattr(connection_info, "role_arn", None)
|
||||
access_key = getattr(connection_info, "aws_access_key_id", None)
|
||||
secret_key = getattr(connection_info, "aws_secret_access_key", None)
|
||||
|
||||
if web_identity_token and role_arn:
|
||||
session_name = (
|
||||
getattr(connection_info, "role_session_name", None) or "wren-oidc-session"
|
||||
)
|
||||
sts = boto3.client(
|
||||
"sts", region_name=getattr(connection_info, "region_name", None)
|
||||
)
|
||||
resp = sts.assume_role_with_web_identity(
|
||||
RoleArn=role_arn.get_secret_value(),
|
||||
RoleSessionName=session_name,
|
||||
WebIdentityToken=web_identity_token.get_secret_value(),
|
||||
)
|
||||
creds = resp["Credentials"]
|
||||
kwargs["aws_access_key_id"] = creds["AccessKeyId"]
|
||||
kwargs["aws_secret_access_key"] = creds["SecretAccessKey"]
|
||||
kwargs["aws_session_token"] = creds["SessionToken"]
|
||||
elif access_key and secret_key:
|
||||
kwargs["aws_access_key_id"] = access_key.get_secret_value()
|
||||
kwargs["aws_secret_access_key"] = secret_key.get_secret_value()
|
||||
session_token = getattr(connection_info, "aws_session_token", None)
|
||||
if session_token:
|
||||
kwargs["aws_session_token"] = session_token.get_secret_value()
|
||||
# else: fall back to the boto3 default credential chain
|
||||
|
||||
user_kwargs = getattr(connection_info, "kwargs", None)
|
||||
if user_kwargs:
|
||||
kwargs.update(user_kwargs)
|
||||
kwargs.setdefault("kill_on_interrupt", True)
|
||||
return kwargs
|
||||
|
||||
|
||||
class AthenaConnector(ConnectorABC):
|
||||
def __init__(self, connection_info):
|
||||
from pyathena import connect # noqa: PLC0415
|
||||
|
||||
self.connection = connect(**_build_connect_kwargs(connection_info))
|
||||
|
||||
def query(self, sql: str, limit: int | None = None) -> pa.Table:
|
||||
try:
|
||||
with contextlib.closing(self.connection.cursor()) as cursor:
|
||||
cursor.execute(sql)
|
||||
table = _build_athena_arrow_table(cursor)
|
||||
if limit is not None:
|
||||
table = table.slice(0, limit)
|
||||
return table
|
||||
except (WrenError, TimeoutError):
|
||||
raise
|
||||
except Exception as e:
|
||||
raise WrenError(
|
||||
ErrorCode.INVALID_SQL,
|
||||
str(e),
|
||||
phase=ErrorPhase.SQL_EXECUTION,
|
||||
metadata={DIALECT_SQL: sql},
|
||||
) from e
|
||||
|
||||
def dry_run(self, sql: str) -> None:
|
||||
try:
|
||||
with contextlib.closing(self.connection.cursor()) as cursor:
|
||||
cursor.execute(f"EXPLAIN {sql}")
|
||||
except (WrenError, TimeoutError):
|
||||
raise
|
||||
except Exception as e:
|
||||
raise WrenError(
|
||||
ErrorCode.INVALID_SQL,
|
||||
str(e),
|
||||
phase=ErrorPhase.SQL_DRY_RUN,
|
||||
metadata={DIALECT_SQL: sql},
|
||||
) from e
|
||||
|
||||
def close(self) -> None:
|
||||
if self.connection is None:
|
||||
return
|
||||
try:
|
||||
self.connection.close()
|
||||
except Exception:
|
||||
pass
|
||||
finally:
|
||||
self.connection = None
|
||||
|
||||
|
||||
def create_connector(connection_info) -> AthenaConnector:
|
||||
return AthenaConnector(connection_info)
|
||||
@@ -1,13 +1,16 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import pyarrow as pa
|
||||
from ibis.expr.datatypes import Decimal
|
||||
from ibis.expr.datatypes.core import UUID
|
||||
from ibis.expr.types import Table
|
||||
from loguru import logger
|
||||
|
||||
from wren.model.data_source import DataSource
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from ibis.expr.types import Table
|
||||
|
||||
|
||||
class ConnectorABC(ABC):
|
||||
@abstractmethod
|
||||
@@ -37,6 +40,9 @@ class IbisConnector(ConnectorABC):
|
||||
return ibis_table.to_pyarrow()
|
||||
|
||||
def _handle_pyarrow_unsupported_type(self, ibis_table: Table, **kwargs) -> Table:
|
||||
from ibis.expr.datatypes import Decimal # noqa: PLC0415
|
||||
from ibis.expr.datatypes.core import UUID # noqa: PLC0415
|
||||
|
||||
result_table = ibis_table
|
||||
for name, dtype in ibis_table.schema().items():
|
||||
if isinstance(dtype, Decimal):
|
||||
@@ -55,6 +61,8 @@ class IbisConnector(ConnectorABC):
|
||||
def _round_decimal_columns(
|
||||
self, result_table: Table, col_name: str, scale: int = 9
|
||||
) -> Table:
|
||||
from ibis.expr.datatypes import Decimal # noqa: PLC0415
|
||||
|
||||
col = result_table[col_name]
|
||||
decimal_type = Decimal(precision=38, scale=scale)
|
||||
rounded_col = col.cast(decimal_type).round(scale)
|
||||
|
||||
@@ -1,78 +1,301 @@
|
||||
from contextlib import closing
|
||||
from functools import cache
|
||||
"""Canner Enterprise connector using the Postgres wire protocol via psycopg.
|
||||
|
||||
Canner Enterprise exposes a Postgres-compatible endpoint, so we connect with
|
||||
``psycopg`` directly and build Arrow tables from the cursor description instead
|
||||
of going through ibis. The OID map below covers the types canner emits for
|
||||
Trino-flavoured queries (VARCHAR, DECIMAL, ROW/ARRAY/MAP serialised through the
|
||||
postgres wire).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
from decimal import ROUND_HALF_EVEN
|
||||
from decimal import Decimal as PyDecimal
|
||||
from typing import Any
|
||||
|
||||
import ibis
|
||||
import ibis.expr.schema as sch
|
||||
import pyarrow as pa
|
||||
from ibis import BaseBackend
|
||||
from ibis.backends.sql.compilers.postgres import compiler as postgres_compiler
|
||||
from ibis.expr.datatypes import Decimal
|
||||
from ibis.expr.datatypes.core import UUID
|
||||
from ibis.expr.types import Table
|
||||
from loguru import logger
|
||||
|
||||
from wren.connector.base import ConnectorABC
|
||||
from wren.model.data_source import DataSource
|
||||
from wren.model.error import DIALECT_SQL, ErrorCode, ErrorPhase, WrenError
|
||||
|
||||
# Postgres OID → Arrow type. Canner publishes Trino-style values over the
|
||||
# Postgres wire, so VARCHAR/CHAR map to string, DECIMAL to decimal128,
|
||||
# BIGINT/INT/SMALLINT to int, BOOLEAN to bool, DATE/TIMESTAMP/TIMESTAMP_TZ to
|
||||
# Arrow date/timestamp, and complex types (ROW/ARRAY/MAP) come back as JSON
|
||||
# strings that we expose as Arrow strings.
|
||||
_PG_OID_TO_ARROW: dict[int, pa.DataType] = {
|
||||
16: pa.bool_(),
|
||||
17: pa.binary(),
|
||||
18: pa.string(),
|
||||
19: pa.string(),
|
||||
20: pa.int64(),
|
||||
21: pa.int16(),
|
||||
23: pa.int32(),
|
||||
25: pa.string(),
|
||||
26: pa.int64(),
|
||||
114: pa.string(),
|
||||
142: pa.string(),
|
||||
700: pa.float32(),
|
||||
701: pa.float64(),
|
||||
650: pa.string(),
|
||||
774: pa.string(),
|
||||
790: pa.string(),
|
||||
829: pa.string(),
|
||||
869: pa.string(),
|
||||
1040: pa.string(),
|
||||
1042: pa.string(),
|
||||
1043: pa.string(),
|
||||
1082: pa.date32(),
|
||||
1083: pa.time64("us"),
|
||||
1114: pa.timestamp("us"),
|
||||
1184: pa.timestamp("us", tz="UTC"),
|
||||
1186: pa.duration("us"),
|
||||
2950: pa.string(),
|
||||
3802: pa.string(),
|
||||
1266: pa.string(),
|
||||
3614: pa.string(),
|
||||
3615: pa.string(),
|
||||
3904: pa.string(),
|
||||
3906: pa.string(),
|
||||
3908: pa.string(),
|
||||
3910: pa.string(),
|
||||
3912: pa.string(),
|
||||
3926: pa.string(),
|
||||
199: pa.list_(pa.string()),
|
||||
1000: pa.list_(pa.bool_()),
|
||||
1003: pa.list_(pa.string()),
|
||||
1005: pa.list_(pa.int16()),
|
||||
1007: pa.list_(pa.int32()),
|
||||
1009: pa.list_(pa.string()),
|
||||
1014: pa.list_(pa.string()),
|
||||
1015: pa.list_(pa.string()),
|
||||
1016: pa.list_(pa.int64()),
|
||||
1021: pa.list_(pa.float32()),
|
||||
1022: pa.list_(pa.float64()),
|
||||
1028: pa.list_(pa.string()),
|
||||
1041: pa.list_(pa.string()),
|
||||
1115: pa.list_(pa.timestamp("us")),
|
||||
1182: pa.list_(pa.string()),
|
||||
1183: pa.list_(pa.string()),
|
||||
1185: pa.list_(pa.timestamp("us", tz="UTC")),
|
||||
1187: pa.list_(pa.string()),
|
||||
1270: pa.list_(pa.string()),
|
||||
2951: pa.list_(pa.string()),
|
||||
3807: pa.list_(pa.string()),
|
||||
}
|
||||
|
||||
|
||||
@cache
|
||||
def _get_pg_type_names(connection: BaseBackend) -> dict[int, str]:
|
||||
with closing(connection.raw_sql("SELECT oid, typname FROM pg_type")) as cur:
|
||||
return dict(cur.fetchall())
|
||||
def _decimal_type(column) -> pa.DataType | None:
|
||||
"""Pick the narrowest decimal128 that fits a NUMERIC column.
|
||||
|
||||
Returns ``None`` when the column has no explicit typmod (``scale is
|
||||
None``) — the caller falls back to ``pa.string()`` so that high-precision
|
||||
values round-trip without silent rounding via ``Decimal.quantize``.
|
||||
"""
|
||||
if column.scale is None:
|
||||
return None
|
||||
scale = max(0, min(column.scale, 38))
|
||||
precision = column.precision if column.precision is not None else 38
|
||||
if precision <= 0 or precision > 38:
|
||||
precision = 38
|
||||
precision = max(precision, scale, 1)
|
||||
precision = min(precision, 38)
|
||||
return pa.decimal128(precision, scale)
|
||||
|
||||
|
||||
def _arrow_type(column) -> pa.DataType:
|
||||
if column.type_code == 1700:
|
||||
# Unconstrained NUMERIC (no typmod) falls back to string to preserve
|
||||
# the exact textual representation — quantising to an arbitrary scale
|
||||
# would silently round high-precision values.
|
||||
return _decimal_type(column) or pa.string()
|
||||
if column.type_code == 1231:
|
||||
inner = _decimal_type(column)
|
||||
return pa.list_(inner) if inner is not None else pa.list_(pa.string())
|
||||
return _PG_OID_TO_ARROW.get(column.type_code, pa.string())
|
||||
|
||||
|
||||
_TRAILING_SEMICOLONS_RE = re.compile(r"[;\s]+\Z")
|
||||
|
||||
|
||||
def _strip_trailing_semicolon(sql: str) -> str:
|
||||
"""Strip any trailing ``;`` characters and surrounding whitespace.
|
||||
|
||||
Wrapping user SQL as ``SELECT * FROM ({sql}) AS _t LIMIT N`` breaks when
|
||||
``sql`` ends in a semicolon — Postgres/Canner reject ``SELECT 1;`` inside
|
||||
a subquery. We only strip the *terminating* run of semicolons/whitespace,
|
||||
so semicolons inside string literals (e.g. ``SELECT 'a;b' FROM t``) are
|
||||
preserved.
|
||||
"""
|
||||
return _TRAILING_SEMICOLONS_RE.sub("", sql)
|
||||
|
||||
|
||||
def _coerce_decimal(value, target_type: pa.DataType):
|
||||
if value is None or not isinstance(value, PyDecimal):
|
||||
return value
|
||||
quantize_value = PyDecimal(f"1E-{target_type.scale}")
|
||||
try:
|
||||
return value.quantize(quantize_value, rounding=ROUND_HALF_EVEN)
|
||||
except Exception:
|
||||
return value
|
||||
|
||||
|
||||
def _build_column(
|
||||
values: list, arrow_type: pa.DataType, pg_type_oid: int | None = None
|
||||
) -> pa.Array:
|
||||
if arrow_type == pa.string():
|
||||
processed: list[Any] = []
|
||||
for value in values:
|
||||
if value is None:
|
||||
# SQL NULL stays as Python None regardless of source oid; the
|
||||
# caller is responsible for distinguishing a JSON ``null``
|
||||
# literal from a SQL NULL.
|
||||
processed.append(None)
|
||||
elif isinstance(value, dict | list):
|
||||
processed.append(json.dumps(value, default=str))
|
||||
elif not isinstance(value, str):
|
||||
processed.append(str(value))
|
||||
else:
|
||||
processed.append(value)
|
||||
return pa.array(processed, type=pa.string(), from_pandas=True)
|
||||
|
||||
if pa.types.is_binary(arrow_type):
|
||||
processed = [
|
||||
bytes(value) if isinstance(value, memoryview) else value for value in values
|
||||
]
|
||||
return pa.array(processed, type=pa.binary(), from_pandas=True)
|
||||
|
||||
if pa.types.is_decimal(arrow_type):
|
||||
processed = [_coerce_decimal(value, arrow_type) for value in values]
|
||||
return pa.array(processed, type=arrow_type, from_pandas=True)
|
||||
|
||||
if pa.types.is_list(arrow_type) and pa.types.is_string(arrow_type.value_type):
|
||||
processed = []
|
||||
for value in values:
|
||||
if value is None:
|
||||
processed.append(None)
|
||||
continue
|
||||
items: list[Any] = []
|
||||
for item in value:
|
||||
if item is None:
|
||||
items.append(None)
|
||||
elif isinstance(item, dict | list):
|
||||
items.append(json.dumps(item, default=str))
|
||||
elif isinstance(item, str):
|
||||
items.append(item)
|
||||
else:
|
||||
items.append(str(item))
|
||||
processed.append(items)
|
||||
return pa.array(processed, type=arrow_type, from_pandas=True)
|
||||
|
||||
if pa.types.is_list(arrow_type) and pa.types.is_decimal(arrow_type.value_type):
|
||||
processed = []
|
||||
for value in values:
|
||||
if value is None:
|
||||
processed.append(None)
|
||||
else:
|
||||
processed.append(
|
||||
[_coerce_decimal(item, arrow_type.value_type) for item in value]
|
||||
)
|
||||
return pa.array(processed, type=arrow_type, from_pandas=True)
|
||||
|
||||
return pa.array(values, type=arrow_type, from_pandas=True)
|
||||
|
||||
|
||||
def _build_arrow_table(cursor) -> pa.Table:
|
||||
"""Convert a psycopg cursor result into a PyArrow table."""
|
||||
if cursor.description is None:
|
||||
return pa.table({})
|
||||
|
||||
rows = cursor.fetchall()
|
||||
fields = [
|
||||
pa.field(column.name, _arrow_type(column), nullable=True)
|
||||
for column in cursor.description
|
||||
]
|
||||
schema = pa.schema(fields)
|
||||
|
||||
if not rows:
|
||||
arrays = [pa.array([], type=field.type) for field in schema]
|
||||
else:
|
||||
arrays = [
|
||||
_build_column(
|
||||
[row[index] for row in rows],
|
||||
field.type,
|
||||
cursor.description[index].type_code,
|
||||
)
|
||||
for index, field in enumerate(schema)
|
||||
]
|
||||
|
||||
# Use positional construction so duplicate column names (e.g. self-joins)
|
||||
# survive — dict-based construction silently drops duplicates.
|
||||
return pa.Table.from_arrays(arrays, schema=schema)
|
||||
|
||||
|
||||
class CannerConnector(ConnectorABC):
|
||||
def __init__(self, connection_info):
|
||||
self.connection = DataSource.canner.get_connection(connection_info)
|
||||
self._closed = False
|
||||
|
||||
def query(self, sql: str, limit: int | None = None) -> pa.Table:
|
||||
schema = self._get_schema(sql)
|
||||
ibis_table = self.connection.sql(sql, schema=schema)
|
||||
import psycopg # noqa: PLC0415
|
||||
|
||||
if limit is not None:
|
||||
ibis_table = ibis_table.limit(limit)
|
||||
ibis_table = self._handle_pyarrow_unsupported_type(ibis_table)
|
||||
return ibis_table.to_pyarrow()
|
||||
sql = (
|
||||
f"SELECT * FROM ({_strip_trailing_semicolon(sql)}) AS _t LIMIT {limit}"
|
||||
)
|
||||
|
||||
def _handle_pyarrow_unsupported_type(self, ibis_table: Table, **kwargs) -> Table:
|
||||
result_table = ibis_table
|
||||
for name, dtype in ibis_table.schema().items():
|
||||
if isinstance(dtype, Decimal):
|
||||
col = result_table[name]
|
||||
decimal_type = Decimal(precision=38, scale=9)
|
||||
rounded_col = col.cast(decimal_type).round(9)
|
||||
result_table = result_table.mutate(**{name: rounded_col})
|
||||
elif isinstance(dtype, UUID):
|
||||
result_table = result_table.mutate(
|
||||
**{name: result_table[name].cast("string")}
|
||||
)
|
||||
return result_table
|
||||
try:
|
||||
with self.connection.cursor() as cursor:
|
||||
cursor.execute(sql)
|
||||
return _build_arrow_table(cursor)
|
||||
except psycopg.errors.QueryCanceled:
|
||||
raise
|
||||
except (WrenError, TimeoutError):
|
||||
raise
|
||||
except Exception as e:
|
||||
raise WrenError(
|
||||
ErrorCode.GENERIC_USER_ERROR,
|
||||
str(e),
|
||||
phase=ErrorPhase.SQL_EXECUTION,
|
||||
metadata={DIALECT_SQL: sql},
|
||||
) from e
|
||||
|
||||
def dry_run(self, sql: str) -> Any:
|
||||
return self.connection.raw_sql(f"SELECT * FROM ({sql}) LIMIT 0")
|
||||
def dry_run(self, sql: str) -> None:
|
||||
import psycopg # noqa: PLC0415
|
||||
|
||||
wrapped = f"SELECT * FROM ({_strip_trailing_semicolon(sql)}) AS _t LIMIT 0"
|
||||
try:
|
||||
with self.connection.cursor() as cursor:
|
||||
cursor.execute(wrapped)
|
||||
except psycopg.errors.QueryCanceled:
|
||||
raise
|
||||
except (WrenError, TimeoutError):
|
||||
raise
|
||||
except Exception as e:
|
||||
raise WrenError(
|
||||
ErrorCode.GENERIC_USER_ERROR,
|
||||
str(e),
|
||||
phase=ErrorPhase.SQL_DRY_RUN,
|
||||
metadata={DIALECT_SQL: sql},
|
||||
) from e
|
||||
# Explicit return to honour the ConnectorABC.dry_run() contract — the
|
||||
# cursor result must not leak out of this method.
|
||||
return None
|
||||
|
||||
def close(self) -> None:
|
||||
if self._closed or not hasattr(self, "connection") or self.connection is None:
|
||||
return
|
||||
try:
|
||||
if hasattr(self.connection, "con") and hasattr(
|
||||
self.connection.con, "close"
|
||||
):
|
||||
self.connection.con.close()
|
||||
elif hasattr(self.connection, "close"):
|
||||
self.connection.close()
|
||||
self.connection.close()
|
||||
except Exception as e:
|
||||
logger.warning(f"Error closing Canner connection: {e}")
|
||||
|
||||
def _get_schema(self, sql: str) -> sch.Schema:
|
||||
cur = self.dry_run(sql)
|
||||
type_names = _get_pg_type_names(self.connection)
|
||||
return ibis.schema(
|
||||
{
|
||||
desc.name: postgres_compiler.type_mapper.from_string(
|
||||
type_names[desc.type_code]
|
||||
)
|
||||
for desc in cur.description
|
||||
}
|
||||
)
|
||||
finally:
|
||||
self._closed = True
|
||||
self.connection = None
|
||||
|
||||
|
||||
def create_connector(connection_info) -> CannerConnector:
|
||||
|
||||
@@ -0,0 +1,383 @@
|
||||
"""Native ClickHouse connector built on ``clickhouse-connect``.
|
||||
|
||||
ClickHouse types are returned by the driver as descriptor strings such as
|
||||
``Nullable(Decimal(18, 4))`` or ``Array(LowCardinality(String))``. We parse
|
||||
those into a sqlglot ``DataType`` AST and walk it to construct a matching
|
||||
PyArrow schema. Values from ``QueryResult.result_rows`` are then coerced
|
||||
column-by-column to that schema.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
from decimal import Decimal as PyDecimal
|
||||
from typing import Any
|
||||
from urllib.parse import parse_qsl, unquote_plus, urlparse
|
||||
|
||||
import pyarrow as pa
|
||||
import sqlglot
|
||||
import sqlglot.errors
|
||||
from loguru import logger
|
||||
from sqlglot.expressions import DataType
|
||||
|
||||
from wren.connector.base import ConnectorABC
|
||||
from wren.model.error import (
|
||||
DIALECT_SQL,
|
||||
DatabaseTimeoutError,
|
||||
ErrorCode,
|
||||
ErrorPhase,
|
||||
WrenError,
|
||||
)
|
||||
|
||||
try:
|
||||
import clickhouse_connect
|
||||
|
||||
_ClickHouseDbError = clickhouse_connect.driver.exceptions.DatabaseError
|
||||
except ImportError: # pragma: no cover - optional dependency
|
||||
|
||||
class _ClickHouseDbError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Type parsing — ClickHouse type-string → PyArrow DataType
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _parse_clickhouse_type(type_str: str | None) -> pa.DataType:
|
||||
"""Map a ClickHouse type-name string to an Arrow type via sqlglot.
|
||||
|
||||
sqlglot's ClickHouse dialect strips ``Nullable(...)`` during parsing
|
||||
(the inner ``DataType`` is hoisted to the top), so we only need to peel
|
||||
``LowCardinality(...)`` ourselves.
|
||||
"""
|
||||
if type_str is None:
|
||||
return pa.string()
|
||||
try:
|
||||
parsed = sqlglot.parse_one(type_str, into=DataType, dialect="clickhouse")
|
||||
except sqlglot.errors.ParseError:
|
||||
logger.warning(f"Failed to parse ClickHouse type string: {type_str}")
|
||||
return pa.string()
|
||||
if parsed is None:
|
||||
return pa.string()
|
||||
return _clickhouse_data_type_to_arrow(parsed)
|
||||
|
||||
|
||||
_CLICKHOUSE_DATA_TYPE_TO_ARROW: dict = {}
|
||||
|
||||
|
||||
def _init_clickhouse_data_type_map() -> None:
|
||||
if _CLICKHOUSE_DATA_TYPE_TO_ARROW:
|
||||
return
|
||||
T = DataType.Type
|
||||
_CLICKHOUSE_DATA_TYPE_TO_ARROW.update(
|
||||
{
|
||||
T.BOOLEAN: pa.bool_(),
|
||||
T.TINYINT: pa.int8(),
|
||||
T.SMALLINT: pa.int16(),
|
||||
T.INT: pa.int32(),
|
||||
T.BIGINT: pa.int64(),
|
||||
T.UTINYINT: pa.uint8(),
|
||||
T.USMALLINT: pa.uint16(),
|
||||
T.UINT: pa.uint32(),
|
||||
T.UBIGINT: pa.uint64(),
|
||||
# Int128 / Int256 / UInt128 / UInt256: PyArrow tops out at 64 bits,
|
||||
# so surface the wide types as string to avoid silent truncation.
|
||||
T.INT128: pa.string(),
|
||||
T.INT256: pa.string(),
|
||||
T.UINT128: pa.string(),
|
||||
T.UINT256: pa.string(),
|
||||
T.FLOAT: pa.float32(),
|
||||
T.DOUBLE: pa.float64(),
|
||||
T.TEXT: pa.string(), # ClickHouse ``String``
|
||||
T.FIXEDSTRING: pa.string(),
|
||||
T.UUID: pa.string(),
|
||||
T.IPV4: pa.string(),
|
||||
T.IPV6: pa.string(),
|
||||
T.ENUM8: pa.string(),
|
||||
T.ENUM16: pa.string(),
|
||||
T.JSON: pa.string(),
|
||||
# ``Nothing`` is ClickHouse's type for bare NULL literals. Surface
|
||||
# as string — the column will be all-None either way.
|
||||
T.NOTHING: pa.string(),
|
||||
T.DATE: pa.date32(),
|
||||
T.DATE32: pa.date32(),
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _clickhouse_data_type_to_arrow(node: Any) -> pa.DataType:
|
||||
_init_clickhouse_data_type_map()
|
||||
if not isinstance(node, DataType):
|
||||
return pa.string()
|
||||
|
||||
kind = node.this
|
||||
T = DataType.Type
|
||||
|
||||
# Peel ``LowCardinality(...)`` — purely a storage detail.
|
||||
if kind == T.LOWCARDINALITY:
|
||||
inner = node.expressions[0] if node.expressions else None
|
||||
return _clickhouse_data_type_to_arrow(inner) if inner else pa.string()
|
||||
|
||||
if kind in _CLICKHOUSE_DATA_TYPE_TO_ARROW:
|
||||
return _CLICKHOUSE_DATA_TYPE_TO_ARROW[kind]
|
||||
|
||||
if kind in (T.DECIMAL, T.DECIMAL32, T.DECIMAL64, T.DECIMAL128, T.DECIMAL256):
|
||||
# Normalise every decimal to (38, 9) so downstream consumers do not
|
||||
# have to special-case precision/scale per column.
|
||||
return pa.decimal128(38, 9)
|
||||
|
||||
if kind in (T.DATETIME, T.DATETIME64):
|
||||
tz = _clickhouse_extract_datetime_tz(node)
|
||||
if tz:
|
||||
return pa.timestamp("ns", tz=tz)
|
||||
return pa.timestamp("ns")
|
||||
|
||||
if kind == T.ARRAY:
|
||||
inner = node.expressions[0] if node.expressions else None
|
||||
return pa.list_(_clickhouse_data_type_to_arrow(inner) if inner else pa.string())
|
||||
|
||||
if kind == T.MAP:
|
||||
if len(node.expressions) >= 2:
|
||||
return pa.map_(
|
||||
_clickhouse_data_type_to_arrow(node.expressions[0]),
|
||||
_clickhouse_data_type_to_arrow(node.expressions[1]),
|
||||
)
|
||||
return pa.string()
|
||||
|
||||
if kind == T.STRUCT:
|
||||
# ``Tuple(...)`` — flatten to JSON-encoded string.
|
||||
return pa.string()
|
||||
|
||||
return pa.string()
|
||||
|
||||
|
||||
def _clickhouse_extract_datetime_tz(node: Any) -> str | None:
|
||||
"""Pull the timezone string out of ``DateTime('tz')`` / ``DateTime64(p, 'tz')``."""
|
||||
for param in node.expressions:
|
||||
# ``DataTypeParam`` wraps a ``Literal``; we want the string-typed one.
|
||||
inner = getattr(param, "this", None)
|
||||
if inner is not None and getattr(inner, "is_string", False):
|
||||
return str(inner.this)
|
||||
return None
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Arrow table assembly from a clickhouse-connect QueryResult
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _build_clickhouse_arrow_table(query_result: Any) -> pa.Table:
|
||||
"""Convert a ``clickhouse_connect`` ``QueryResult`` into a PyArrow table."""
|
||||
column_names = list(query_result.column_names)
|
||||
column_types = list(query_result.column_types)
|
||||
rows = list(query_result.result_rows or [])
|
||||
|
||||
fields = [
|
||||
pa.field(name, _parse_clickhouse_type(ct.name), nullable=True)
|
||||
for name, ct in zip(column_names, column_types, strict=False)
|
||||
]
|
||||
schema = pa.schema(fields)
|
||||
|
||||
if not rows:
|
||||
arrays = [pa.array([], type=field.type) for field in schema]
|
||||
else:
|
||||
arrays = [
|
||||
_build_clickhouse_column([row[i] for row in rows], schema.field(i).type)
|
||||
for i in range(len(fields))
|
||||
]
|
||||
# ``dict(zip(...))`` collapses duplicate column names — build the table
|
||||
# from arrays + schema so projections like ``SELECT a, a`` are preserved.
|
||||
return pa.Table.from_arrays(arrays, schema=schema)
|
||||
|
||||
|
||||
def _build_clickhouse_column(values: list, arrow_type: pa.DataType) -> pa.Array:
|
||||
"""Convert ``clickhouse_connect`` Python values into a PyArrow array."""
|
||||
if pa.types.is_string(arrow_type):
|
||||
processed: list[Any] = []
|
||||
for v in values:
|
||||
if v is None:
|
||||
processed.append(None)
|
||||
elif isinstance(v, dict | list | tuple):
|
||||
processed.append(json.dumps(v, default=str))
|
||||
elif isinstance(v, str):
|
||||
processed.append(v)
|
||||
elif isinstance(v, bytes):
|
||||
processed.append(v.decode("utf-8", errors="replace"))
|
||||
else:
|
||||
processed.append(str(v))
|
||||
return pa.array(processed, type=pa.string(), from_pandas=True)
|
||||
|
||||
if pa.types.is_decimal(arrow_type):
|
||||
# Every decimal is normalised to ``decimal128(38, 9)``, so no
|
||||
# per-column precision/scale narrowing is needed here.
|
||||
processed = [
|
||||
None
|
||||
if v is None
|
||||
else (v if isinstance(v, PyDecimal) else PyDecimal(str(v)))
|
||||
for v in values
|
||||
]
|
||||
return pa.array(processed, type=arrow_type, from_pandas=True)
|
||||
|
||||
if pa.types.is_timestamp(arrow_type):
|
||||
# ``clickhouse_connect`` returns datetime objects (naive or tz-aware).
|
||||
return pa.array(values, type=arrow_type, from_pandas=True)
|
||||
|
||||
return pa.array(values, type=arrow_type, from_pandas=True)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Client kwargs assembly
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _build_clickhouse_client_kwargs(connection_info: Any) -> dict:
|
||||
"""Translate ``ClickHouseConnectionInfo`` / ``ConnectionUrl`` into
|
||||
``clickhouse_connect.get_client`` kwargs."""
|
||||
|
||||
# URL-based connection (``ConnectionUrl``).
|
||||
if hasattr(connection_info, "connection_url"):
|
||||
url = connection_info.connection_url.get_secret_value()
|
||||
parsed = urlparse(url)
|
||||
if parsed.scheme not in {"clickhouse", "clickhouse+http", "clickhouse+https"}:
|
||||
raise WrenError(
|
||||
ErrorCode.INVALID_CONNECTION_INFO,
|
||||
"ClickHouse connection URL must use clickhouse:// scheme",
|
||||
)
|
||||
|
||||
kwargs: dict = dict(parse_qsl(parsed.query))
|
||||
info_kwargs = getattr(connection_info, "kwargs", None)
|
||||
if info_kwargs:
|
||||
kwargs.update(info_kwargs)
|
||||
|
||||
settings: dict = (
|
||||
dict(kwargs.pop("settings", {})) if "settings" in kwargs else {}
|
||||
)
|
||||
statement_timeout = kwargs.pop("statement_timeout", None)
|
||||
if statement_timeout is not None:
|
||||
settings["max_execution_time"] = int(statement_timeout)
|
||||
|
||||
# urlparse leaves percent-encoded characters in userinfo, so decode
|
||||
# them before clickhouse-connect sees the credentials. Matches the
|
||||
# mssql / postgres URL handling elsewhere in this package.
|
||||
out: dict = {
|
||||
"host": parsed.hostname,
|
||||
"port": int(parsed.port) if parsed.port else 8123,
|
||||
"username": (
|
||||
unquote_plus(parsed.username) if parsed.username else "default"
|
||||
),
|
||||
"password": unquote_plus(parsed.password) if parsed.password else "",
|
||||
"settings": settings,
|
||||
}
|
||||
if parsed.path and parsed.path != "/":
|
||||
out["database"] = parsed.path.lstrip("/")
|
||||
if parsed.scheme == "clickhouse+https":
|
||||
out["secure"] = True
|
||||
# ``settings`` already popped above, so ``out["settings"]`` survives.
|
||||
out.update(kwargs)
|
||||
return out
|
||||
|
||||
info = connection_info # ClickHouseConnectionInfo
|
||||
settings = dict(info.settings) if info.settings else {}
|
||||
kwargs = dict(info.kwargs) if info.kwargs else {}
|
||||
statement_timeout = kwargs.pop("statement_timeout", None)
|
||||
if statement_timeout is not None:
|
||||
settings["max_execution_time"] = int(statement_timeout)
|
||||
# Merge any user-supplied ``settings`` from kwargs into the local dict
|
||||
# *before* applying the rest, otherwise ``out.update(kwargs)`` below
|
||||
# would clobber the statement_timeout-derived max_execution_time.
|
||||
extra_settings = kwargs.pop("settings", None)
|
||||
if extra_settings:
|
||||
settings.update(extra_settings)
|
||||
|
||||
out = {
|
||||
"host": info.host,
|
||||
"port": int(info.port),
|
||||
"username": info.user,
|
||||
"password": info.password.get_secret_value() if info.password else "",
|
||||
"database": info.database,
|
||||
"secure": info.secure,
|
||||
"settings": settings,
|
||||
}
|
||||
out.update(kwargs)
|
||||
return out
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Connector
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
_TRAILING_SEMICOLONS_RE = re.compile(r"[;\s]+\Z")
|
||||
|
||||
|
||||
def _strip_trailing_semicolon(sql: str) -> str:
|
||||
"""Strip the terminating run of ``;`` characters and surrounding whitespace.
|
||||
|
||||
Matches the canner helper of the same name. Wrapping user SQL as
|
||||
``SELECT * FROM ({sql}) AS _wren_sub LIMIT N`` breaks when ``sql`` ends
|
||||
in a semicolon — ClickHouse rejects ``SELECT 1;`` inside a subquery. Only
|
||||
the terminating run is removed so semicolons inside string literals
|
||||
(e.g. ``SELECT 'a;b'``) are preserved.
|
||||
"""
|
||||
return _TRAILING_SEMICOLONS_RE.sub("", sql)
|
||||
|
||||
|
||||
class ClickHouseConnector(ConnectorABC):
|
||||
"""Native ``clickhouse-connect`` connector that bypasses ``ibis-project``."""
|
||||
|
||||
def __init__(self, connection_info: Any):
|
||||
connect_kwargs = _build_clickhouse_client_kwargs(connection_info)
|
||||
self.connection = clickhouse_connect.get_client(**connect_kwargs)
|
||||
self._closed = False
|
||||
|
||||
def query(self, sql: str, limit: int | None = None) -> pa.Table:
|
||||
# Strip the terminating run of ``;`` / whitespace before wrapping —
|
||||
# ``SELECT * FROM (SELECT 1;) AS _wren_sub LIMIT N`` is invalid SQL.
|
||||
# Semicolons inside string literals are preserved.
|
||||
stripped = _strip_trailing_semicolon(sql)
|
||||
statement = stripped
|
||||
if limit is not None:
|
||||
statement = f"SELECT * FROM ({stripped}) AS _wren_sub LIMIT {limit}"
|
||||
try:
|
||||
result = self.connection.query(statement)
|
||||
except _ClickHouseDbError as e:
|
||||
if "TIMEOUT_EXCEEDED" in str(e):
|
||||
raise DatabaseTimeoutError(str(e)) from e
|
||||
raise WrenError(
|
||||
ErrorCode.INVALID_SQL,
|
||||
str(e),
|
||||
phase=ErrorPhase.SQL_EXECUTION,
|
||||
metadata={DIALECT_SQL: sql},
|
||||
) from e
|
||||
return _build_clickhouse_arrow_table(result)
|
||||
|
||||
def dry_run(self, sql: str) -> None:
|
||||
stripped = _strip_trailing_semicolon(sql)
|
||||
try:
|
||||
self.connection.query(f"SELECT * FROM ({stripped}) AS _wren_sub LIMIT 0")
|
||||
except _ClickHouseDbError as e:
|
||||
if "TIMEOUT_EXCEEDED" in str(e):
|
||||
raise DatabaseTimeoutError(str(e)) from e
|
||||
raise WrenError(
|
||||
ErrorCode.INVALID_SQL,
|
||||
str(e),
|
||||
phase=ErrorPhase.SQL_DRY_RUN,
|
||||
metadata={DIALECT_SQL: sql},
|
||||
) from e
|
||||
|
||||
def close(self) -> None:
|
||||
if self._closed or not hasattr(self, "connection") or self.connection is None:
|
||||
return
|
||||
try:
|
||||
self.connection.close()
|
||||
except Exception as e:
|
||||
logger.warning(f"Error closing ClickHouse connection: {e}")
|
||||
finally:
|
||||
self._closed = True
|
||||
self.connection = None
|
||||
|
||||
|
||||
def create_connector(connection_info: Any) -> ClickHouseConnector:
|
||||
return ClickHouseConnector(connection_info)
|
||||
@@ -19,11 +19,11 @@ _REGISTRY: dict[DataSource, str] = {
|
||||
DataSource.redshift: "wren.connector.redshift",
|
||||
DataSource.spark: "wren.connector.spark",
|
||||
DataSource.databricks: "wren.connector.databricks",
|
||||
DataSource.trino: "wren.connector.ibis",
|
||||
DataSource.clickhouse: "wren.connector.ibis",
|
||||
DataSource.trino: "wren.connector.trino",
|
||||
DataSource.clickhouse: "wren.connector.clickhouse",
|
||||
DataSource.oracle: "wren.connector.oracle",
|
||||
DataSource.snowflake: "wren.connector.snowflake",
|
||||
DataSource.athena: "wren.connector.ibis",
|
||||
DataSource.athena: "wren.connector.athena",
|
||||
}
|
||||
|
||||
# Map data sources to the correct pip extra when they share a connector module
|
||||
@@ -40,8 +40,6 @@ _NEEDS_DATA_SOURCE = {
|
||||
DataSource.mysql,
|
||||
DataSource.doris,
|
||||
DataSource.trino,
|
||||
DataSource.clickhouse,
|
||||
DataSource.athena,
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -1,103 +0,0 @@
|
||||
"""Generic ibis-backed connectors with data-source-specific error handling."""
|
||||
|
||||
import pyarrow as pa
|
||||
|
||||
from wren.connector.base import IbisConnector
|
||||
from wren.model.data_source import DataSource
|
||||
from wren.model.error import DIALECT_SQL, ErrorCode, ErrorPhase, WrenError
|
||||
|
||||
try:
|
||||
import clickhouse_connect
|
||||
|
||||
_ClickHouseDbError = clickhouse_connect.driver.exceptions.DatabaseError
|
||||
except ImportError:
|
||||
|
||||
class _ClickHouseDbError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class TrinoConnector(IbisConnector):
|
||||
def __init__(self, connection_info):
|
||||
super().__init__(DataSource.trino, connection_info)
|
||||
|
||||
def query(self, sql: str, limit: int | None = None) -> pa.Table:
|
||||
import trino # noqa: PLC0415
|
||||
|
||||
try:
|
||||
return super().query(sql, limit)
|
||||
except trino.exceptions.TrinoQueryError as e:
|
||||
if not e.error_name == "EXCEEDED_TIME_LIMIT":
|
||||
raise WrenError(
|
||||
ErrorCode.INVALID_SQL,
|
||||
str(e),
|
||||
phase=ErrorPhase.SQL_EXECUTION,
|
||||
metadata={DIALECT_SQL: sql},
|
||||
) from e
|
||||
raise
|
||||
except (WrenError, TimeoutError):
|
||||
raise
|
||||
|
||||
def dry_run(self, sql: str) -> None:
|
||||
import trino # noqa: PLC0415
|
||||
|
||||
try:
|
||||
super().dry_run(sql)
|
||||
except trino.exceptions.TrinoQueryError as e:
|
||||
if not e.error_name == "EXCEEDED_TIME_LIMIT":
|
||||
raise WrenError(
|
||||
ErrorCode.INVALID_SQL,
|
||||
str(e),
|
||||
phase=ErrorPhase.SQL_DRY_RUN,
|
||||
metadata={DIALECT_SQL: sql},
|
||||
) from e
|
||||
raise
|
||||
except (WrenError, TimeoutError):
|
||||
raise
|
||||
|
||||
|
||||
class ClickHouseConnector(IbisConnector):
|
||||
def __init__(self, connection_info):
|
||||
super().__init__(DataSource.clickhouse, connection_info)
|
||||
|
||||
def query(self, sql: str, limit: int | None = None) -> pa.Table:
|
||||
try:
|
||||
return super().query(sql, limit)
|
||||
except _ClickHouseDbError as e:
|
||||
if "TIMEOUT_EXCEEDED" not in str(e):
|
||||
raise WrenError(
|
||||
ErrorCode.INVALID_SQL,
|
||||
str(e),
|
||||
phase=ErrorPhase.SQL_EXECUTION,
|
||||
metadata={DIALECT_SQL: sql},
|
||||
) from e
|
||||
raise
|
||||
except (WrenError, TimeoutError):
|
||||
raise
|
||||
|
||||
def dry_run(self, sql: str) -> None:
|
||||
try:
|
||||
super().dry_run(sql)
|
||||
except _ClickHouseDbError as e:
|
||||
if "TIMEOUT_EXCEEDED" not in str(e):
|
||||
raise WrenError(
|
||||
ErrorCode.INVALID_SQL,
|
||||
str(e),
|
||||
phase=ErrorPhase.SQL_DRY_RUN,
|
||||
metadata={DIALECT_SQL: sql},
|
||||
) from e
|
||||
raise
|
||||
except (WrenError, TimeoutError):
|
||||
raise
|
||||
|
||||
|
||||
_DATA_SOURCE_TO_CLASS = {
|
||||
DataSource.trino: TrinoConnector,
|
||||
DataSource.clickhouse: ClickHouseConnector,
|
||||
}
|
||||
|
||||
|
||||
def create_connector(data_source: DataSource, connection_info) -> IbisConnector:
|
||||
cls = _DATA_SOURCE_TO_CLASS.get(data_source, IbisConnector)
|
||||
if cls is IbisConnector:
|
||||
return IbisConnector(data_source, connection_info)
|
||||
return cls(connection_info)
|
||||
@@ -1,52 +1,111 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import datetime as dtlib
|
||||
import json
|
||||
import uuid
|
||||
from contextlib import closing
|
||||
from decimal import Decimal as PyDecimal
|
||||
|
||||
import pyarrow as pa
|
||||
import sqlglot.expressions as sge
|
||||
from ibis.expr.datatypes import Decimal
|
||||
from ibis.expr.types import Table
|
||||
from loguru import logger
|
||||
from sqlglot import exp, parse_one
|
||||
|
||||
from wren.connector.base import IbisConnector
|
||||
from wren.connector.base import ConnectorABC
|
||||
from wren.model.data_source import DataSource
|
||||
from wren.model.error import DIALECT_SQL, ErrorCode, ErrorPhase, WrenError
|
||||
|
||||
|
||||
class MSSqlConnector(IbisConnector):
|
||||
class MSSqlConnector(ConnectorABC):
|
||||
"""Native pyodbc-backed MSSQL connector.
|
||||
|
||||
Uses a raw pyodbc cursor for execution, builds Arrow schema from
|
||||
``cursor.description`` plus value sampling, and rewrites pagination via
|
||||
sqlglot (tsql dialect) so that ``LIMIT n`` becomes
|
||||
``OFFSET 0 ROWS FETCH NEXT n ROWS ONLY``.
|
||||
"""
|
||||
|
||||
def __init__(self, connection_info):
|
||||
super().__init__(DataSource.mssql, connection_info)
|
||||
self.data_source = DataSource.mssql
|
||||
self.connection = self.data_source.get_connection(connection_info)
|
||||
self._closed = False
|
||||
|
||||
def query(self, sql: str, limit: int | None = None) -> pa.Table:
|
||||
sql = self._flatten_pagination_limit(sql)
|
||||
ibis_table = self.connection.sql(sql)
|
||||
if limit is not None:
|
||||
ibis_table = ibis_table.limit(limit)
|
||||
ibis_table = self._handle_pyarrow_unsupported_type(ibis_table)
|
||||
return self._round_decimal_columns(ibis_table)
|
||||
with closing(self.connection.cursor()) as cursor:
|
||||
cursor.execute(self._raw_cursor_sql(sql, limit))
|
||||
if cursor.description is None:
|
||||
return pa.table({})
|
||||
|
||||
def _round_decimal_columns(self, ibis_table: Table, scale: int = 9) -> pa.Table:
|
||||
def round_decimal(val):
|
||||
if val is None:
|
||||
return None
|
||||
d = PyDecimal(str(val))
|
||||
return d.quantize(PyDecimal("1." + "0" * scale))
|
||||
rows = cursor.fetchmany(limit) if limit is not None else cursor.fetchall()
|
||||
arrow_schema = self._build_mssql_arrow_schema(cursor.description, rows)
|
||||
arrays = [
|
||||
self._build_mssql_column(
|
||||
[row[index] for row in rows], arrow_schema.field(index).type
|
||||
)
|
||||
for index in range(len(cursor.description))
|
||||
]
|
||||
# ``dict(zip(...))`` collapses duplicate column names — build the
|
||||
# table from arrays + schema so projections like ``SELECT a, a``
|
||||
# are preserved.
|
||||
return pa.Table.from_arrays(arrays, schema=arrow_schema)
|
||||
|
||||
decimal_columns = [
|
||||
name
|
||||
for name, dtype in ibis_table.schema().items()
|
||||
if isinstance(dtype, Decimal)
|
||||
]
|
||||
if not decimal_columns:
|
||||
return ibis_table.to_pyarrow()
|
||||
def dry_run(self, sql: str) -> None:
|
||||
sql = self._flatten_pagination_limit(sql)
|
||||
try:
|
||||
with closing(self.connection.cursor()) as cursor:
|
||||
cursor.execute(self._raw_cursor_sql(sql, 0))
|
||||
except Exception as e:
|
||||
error_message = self._describe_sql_for_error_message(sql)
|
||||
if error_message != "Unknown reason":
|
||||
raise WrenError(
|
||||
error_code=ErrorCode.INVALID_SQL,
|
||||
message=f"The sql dry run failed. {error_message}.",
|
||||
phase=ErrorPhase.SQL_DRY_RUN,
|
||||
metadata={DIALECT_SQL: sql},
|
||||
) from e
|
||||
raise
|
||||
|
||||
pandas_df = ibis_table.to_pandas()
|
||||
for col_name in decimal_columns:
|
||||
pandas_df[col_name] = pandas_df[col_name].apply(round_decimal)
|
||||
return pa.Table.from_pandas(pandas_df)
|
||||
def close(self) -> None:
|
||||
if self._closed or not hasattr(self, "connection") or self.connection is None:
|
||||
return
|
||||
try:
|
||||
self.connection.close()
|
||||
except Exception as e:
|
||||
logger.warning(f"Error closing MSSQL connection: {e}")
|
||||
finally:
|
||||
self._closed = True
|
||||
self.connection = None
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# SQL rewriting
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
@staticmethod
|
||||
def _raw_cursor_sql(
|
||||
sql: str, limit: int | None, input_dialect: str = "tsql"
|
||||
) -> str:
|
||||
"""Inject a ``LIMIT n`` into a Select so sqlglot emits the tsql
|
||||
``OFFSET 0 ROWS FETCH NEXT n ROWS ONLY`` clause."""
|
||||
if limit is None:
|
||||
return sql
|
||||
|
||||
try:
|
||||
parsed = parse_one(sql, dialect=input_dialect)
|
||||
except Exception:
|
||||
return sql
|
||||
|
||||
if isinstance(parsed, exp.Select) and not parsed.args.get("limit"):
|
||||
parsed.set("limit", exp.Limit(expression=exp.Literal.number(limit)))
|
||||
return parsed.sql(dialect="tsql")
|
||||
|
||||
return sql
|
||||
|
||||
def _flatten_pagination_limit(
|
||||
self, sql_query: str, input_dialect: str = "tsql"
|
||||
) -> str:
|
||||
"""Collapse an outer ``LIMIT`` wrapped around a single subquery into
|
||||
the inner Select's ``LIMIT`` — undoes the v4 paginate-wrap pattern."""
|
||||
try:
|
||||
parsed = parse_one(sql_query, dialect=input_dialect)
|
||||
if not isinstance(parsed, exp.Select) or not parsed.args.get("limit"):
|
||||
@@ -78,29 +137,17 @@ class MSSqlConnector(IbisConnector):
|
||||
except Exception:
|
||||
return sql_query
|
||||
|
||||
def dry_run(self, sql: str) -> None:
|
||||
try:
|
||||
super().dry_run(sql)
|
||||
except AttributeError as e:
|
||||
if "NoneType" in str(e) and "lower" in str(e):
|
||||
error_message = self._describe_sql_for_error_message(sql)
|
||||
raise WrenError(
|
||||
error_code=ErrorCode.INVALID_SQL,
|
||||
message=f"The sql dry run failed. {error_message}.",
|
||||
phase=ErrorPhase.SQL_DRY_RUN,
|
||||
metadata={DIALECT_SQL: sql},
|
||||
) from e
|
||||
raise WrenError(
|
||||
error_code=ErrorCode.IBIS_PROJECT_ERROR,
|
||||
message=str(e),
|
||||
phase=ErrorPhase.SQL_DRY_RUN,
|
||||
) from e
|
||||
|
||||
def _describe_sql_for_error_message(self, sql: str) -> str:
|
||||
"""Surface a precise error string by asking SQL Server to describe
|
||||
the first result set of the failing query."""
|
||||
try:
|
||||
tsql = sge.convert(sql).sql("mssql")
|
||||
describe_sql = f"SELECT error_message FROM sys.dm_exec_describe_first_result_set({tsql}, NULL, 0)"
|
||||
with closing(self.connection.raw_sql(describe_sql)) as cur:
|
||||
describe_sql = (
|
||||
"SELECT error_message FROM "
|
||||
f"sys.dm_exec_describe_first_result_set({tsql}, NULL, 0)"
|
||||
)
|
||||
with closing(self.connection.cursor()) as cur:
|
||||
cur.execute(describe_sql)
|
||||
rows = cur.fetchall()
|
||||
if not rows:
|
||||
return "Unknown reason"
|
||||
@@ -108,6 +155,139 @@ class MSSqlConnector(IbisConnector):
|
||||
except Exception:
|
||||
return "Unknown reason"
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Arrow schema inference + column build
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
@staticmethod
|
||||
def _build_mssql_arrow_schema(description, rows: list[tuple]) -> pa.Schema:
|
||||
fields = []
|
||||
for index, column in enumerate(description):
|
||||
values = [row[index] for row in rows]
|
||||
fields.append(
|
||||
pa.field(
|
||||
column[0],
|
||||
MSSqlConnector._mssql_arrow_type(column, values),
|
||||
nullable=True,
|
||||
)
|
||||
)
|
||||
return pa.schema(fields)
|
||||
|
||||
@staticmethod
|
||||
def _mssql_arrow_type(column, values: list) -> pa.DataType:
|
||||
type_code = column[1] if len(column) > 1 else None
|
||||
internal_size = column[3] if len(column) > 3 else None
|
||||
precision = column[4] if len(column) > 4 else None
|
||||
sample = next((value for value in values if value is not None), None)
|
||||
|
||||
if isinstance(sample, bool) or type_code is bool:
|
||||
return pa.bool_()
|
||||
if isinstance(sample, bytes | bytearray | memoryview) or type_code in {
|
||||
bytes,
|
||||
bytearray,
|
||||
memoryview,
|
||||
}:
|
||||
return pa.binary()
|
||||
if isinstance(sample, dtlib.datetime) or type_code is dtlib.datetime:
|
||||
tz = MSSqlConnector._mssql_timezone_name(sample)
|
||||
return pa.timestamp("ns", tz=tz)
|
||||
if isinstance(sample, dtlib.date) or type_code is dtlib.date:
|
||||
return pa.date32()
|
||||
if isinstance(sample, dtlib.time) or type_code is dtlib.time:
|
||||
return pa.time64("ns")
|
||||
if isinstance(sample, float) or type_code is float:
|
||||
return pa.float32() if internal_size == 4 else pa.float64()
|
||||
if isinstance(sample, int) or type_code is int:
|
||||
return MSSqlConnector._mssql_integer_arrow_type(
|
||||
internal_size, precision, values
|
||||
)
|
||||
if isinstance(sample, PyDecimal) or type_code is PyDecimal:
|
||||
return pa.string()
|
||||
if isinstance(sample, uuid.UUID) or type_code is uuid.UUID:
|
||||
return pa.string()
|
||||
|
||||
return pa.string()
|
||||
|
||||
@staticmethod
|
||||
def _mssql_timezone_name(value: dtlib.datetime | None) -> str | None:
|
||||
if value is None or value.tzinfo is None:
|
||||
return None
|
||||
offset = value.utcoffset()
|
||||
if offset is None:
|
||||
return None
|
||||
if offset.total_seconds() == 0:
|
||||
return "UTC"
|
||||
total_minutes = int(offset.total_seconds() // 60)
|
||||
sign = "+" if total_minutes >= 0 else "-"
|
||||
hours, minutes = divmod(abs(total_minutes), 60)
|
||||
return f"{sign}{hours:02d}:{minutes:02d}"
|
||||
|
||||
@staticmethod
|
||||
def _mssql_integer_arrow_type(
|
||||
internal_size: int | None, precision: int | None, values: list
|
||||
) -> pa.DataType:
|
||||
non_negative = all(value is None or int(value) >= 0 for value in values)
|
||||
|
||||
# SQL Server TINYINT is unconditionally unsigned (0..255), so map by
|
||||
# the declared internal_size rather than sampling for sign.
|
||||
if internal_size == 1:
|
||||
return pa.uint8()
|
||||
if internal_size == 2:
|
||||
return pa.int16()
|
||||
if internal_size == 4:
|
||||
return pa.int32()
|
||||
if internal_size == 8:
|
||||
return pa.int64()
|
||||
|
||||
if precision is not None:
|
||||
if precision <= 3 and non_negative:
|
||||
return pa.uint8()
|
||||
if precision <= 5:
|
||||
return pa.int16()
|
||||
if precision <= 10:
|
||||
return pa.int32()
|
||||
return pa.int64()
|
||||
|
||||
@staticmethod
|
||||
def _build_mssql_column(values: list, arrow_type: pa.DataType) -> pa.Array:
|
||||
if pa.types.is_integer(arrow_type):
|
||||
processed = [None if value is None else int(value) for value in values]
|
||||
return pa.array(processed, type=arrow_type, from_pandas=True)
|
||||
|
||||
if pa.types.is_floating(arrow_type):
|
||||
processed = [None if value is None else float(value) for value in values]
|
||||
return pa.array(processed, type=arrow_type, from_pandas=True)
|
||||
|
||||
if pa.types.is_boolean(arrow_type):
|
||||
processed = [None if value is None else bool(value) for value in values]
|
||||
return pa.array(processed, type=arrow_type, from_pandas=True)
|
||||
|
||||
if pa.types.is_decimal(arrow_type):
|
||||
processed = [
|
||||
None
|
||||
if value is None
|
||||
else value
|
||||
if isinstance(value, PyDecimal)
|
||||
else PyDecimal(str(value))
|
||||
for value in values
|
||||
]
|
||||
return pa.array(processed, type=arrow_type, from_pandas=True)
|
||||
|
||||
if pa.types.is_string(arrow_type):
|
||||
processed = []
|
||||
for value in values:
|
||||
if value is None:
|
||||
processed.append(None)
|
||||
elif isinstance(value, dict | list):
|
||||
processed.append(json.dumps(value, default=str))
|
||||
elif isinstance(value, str):
|
||||
processed.append(value)
|
||||
else:
|
||||
processed.append(str(value))
|
||||
return pa.array(processed, type=arrow_type, from_pandas=True)
|
||||
|
||||
return pa.array(values, type=arrow_type, from_pandas=True)
|
||||
|
||||
|
||||
def create_connector(connection_info) -> MSSqlConnector:
|
||||
return MSSqlConnector(connection_info)
|
||||
|
||||
@@ -1,57 +1,553 @@
|
||||
import ibis.expr.datatypes as dt
|
||||
from ibis.expr.datatypes import Decimal
|
||||
from ibis.expr.datatypes.core import UUID
|
||||
from ibis.expr.types import Table
|
||||
"""Native MySQLdb connector for MySQL and Doris.
|
||||
|
||||
from wren.connector.base import IbisConnector
|
||||
Replaces the previous ibis-based implementation. Uses the ``mysqlclient``
|
||||
(``MySQLdb``) driver directly and builds PyArrow tables from cursor
|
||||
descriptions so no ibis backend is required.
|
||||
|
||||
Doris speaks the MySQL wire protocol and reuses the same query path; it
|
||||
only differs in how the connection is opened.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from contextlib import closing
|
||||
from decimal import Decimal as PyDecimal
|
||||
from functools import cache
|
||||
|
||||
import pyarrow as pa
|
||||
from loguru import logger
|
||||
|
||||
from wren.connector.base import ConnectorABC
|
||||
from wren.model.data_source import DataSource
|
||||
from wren.model.error import ErrorCode, WrenError
|
||||
|
||||
|
||||
class MySqlConnector(IbisConnector):
|
||||
def _apply_limit(sql: str, limit: int) -> str:
|
||||
"""Append ``LIMIT n`` to a user-supplied SQL string.
|
||||
|
||||
Strips any trailing semicolon and whitespace, then appends ``LIMIT n``.
|
||||
``limit`` MUST already be validated as a non-negative ``int`` by the caller
|
||||
— this helper does not re-validate to keep the call site explicit.
|
||||
|
||||
Wrapping the user SQL in ``SELECT * FROM (...) AS _sub LIMIT n`` was
|
||||
rejected because it fails with ``ER_DUP_FIELDNAME`` whenever the inner
|
||||
SELECT projects two columns with the same name (e.g. a join that selects
|
||||
``a.id`` and ``b.id``).
|
||||
"""
|
||||
return f"{sql.rstrip().rstrip(';').rstrip()}\nLIMIT {limit}"
|
||||
|
||||
|
||||
def _coerce_limit(limit: int | None) -> int | None:
|
||||
"""Validate and coerce a user-supplied ``limit`` to a non-negative ``int``.
|
||||
|
||||
``int(limit)`` rejects strings like ``"5 OR 1=1"`` so the value can be
|
||||
safely interpolated into SQL. Negative limits are also rejected.
|
||||
"""
|
||||
if limit is None:
|
||||
return None
|
||||
coerced = int(limit)
|
||||
if coerced < 0:
|
||||
raise ValueError(f"limit must be non-negative, got {coerced}")
|
||||
return coerced
|
||||
|
||||
|
||||
class MySqlConnector(ConnectorABC):
|
||||
"""Native MySQLdb connector that bypasses ibis-project."""
|
||||
|
||||
def __init__(self, connection_info):
|
||||
super().__init__(DataSource.mysql, connection_info)
|
||||
self._closed = False
|
||||
self.connection = DataSource.mysql.get_connection(connection_info)
|
||||
# Append ANSI_QUOTES to the server-configured sql_mode so identifiers
|
||||
# quoted as "name" (the MDL convention) are accepted. CONCAT preserves
|
||||
# the server defaults (ONLY_FULL_GROUP_BY, STRICT_TRANS_TABLES, …) —
|
||||
# overwriting them would let queries silently behave differently than
|
||||
# in the user's own MySQL session.
|
||||
#
|
||||
# If this init query fails we MUST close the connection so it isn't
|
||||
# leaked; the cursor exception would otherwise leave a live socket
|
||||
# held by the (now half-constructed) connector.
|
||||
try:
|
||||
with closing(self.connection.cursor()) as cursor:
|
||||
cursor.execute("SET sql_mode=CONCAT(@@sql_mode, ',ANSI_QUOTES')")
|
||||
except Exception:
|
||||
try:
|
||||
self.connection.close()
|
||||
except Exception as close_err:
|
||||
logger.warning(
|
||||
f"Error closing MySQL connection after init failure: {close_err}"
|
||||
)
|
||||
finally:
|
||||
self._closed = True
|
||||
raise
|
||||
|
||||
def _handle_pyarrow_unsupported_type(self, ibis_table: Table, **kwargs) -> Table:
|
||||
result_table = ibis_table
|
||||
for name, dtype in ibis_table.schema().items():
|
||||
if isinstance(dtype, Decimal):
|
||||
result_table = self._round_decimal_columns(
|
||||
result_table=result_table, col_name=name, **kwargs
|
||||
)
|
||||
elif isinstance(dtype, UUID):
|
||||
result_table = self._cast_uuid_columns(
|
||||
result_table=result_table, col_name=name
|
||||
)
|
||||
elif isinstance(dtype, dt.JSON):
|
||||
result_table = result_table.mutate(
|
||||
**{name: result_table[name].cast("string")}
|
||||
)
|
||||
return result_table
|
||||
def query(self, sql: str, limit: int | None = None) -> pa.Table:
|
||||
limit = _coerce_limit(limit)
|
||||
if limit is not None:
|
||||
sql = _apply_limit(sql, limit)
|
||||
with closing(self.connection.cursor()) as cursor:
|
||||
cursor.execute(sql)
|
||||
return _build_mysql_arrow_table(cursor)
|
||||
|
||||
def dry_run(self, sql: str) -> None:
|
||||
# ``EXPLAIN`` validates the SQL on the server (table lookup, column
|
||||
# resolution, syntax) without executing it. Prefixing instead of
|
||||
# subquery-wrapping side-steps ``ER_DUP_FIELDNAME`` for queries that
|
||||
# surface duplicate column names. We strip a trailing semicolon to
|
||||
# match the same compose-ability we use for ``query``'s LIMIT path.
|
||||
explain_sql = f"EXPLAIN {sql.rstrip().rstrip(';').rstrip()}"
|
||||
with closing(self.connection.cursor()) as cursor:
|
||||
cursor.execute(explain_sql)
|
||||
cursor.fetchall()
|
||||
|
||||
def close(self) -> None:
|
||||
if self._closed:
|
||||
return
|
||||
try:
|
||||
self.connection.close()
|
||||
except Exception as e:
|
||||
logger.warning(f"Error closing MySQL connection: {e}")
|
||||
finally:
|
||||
self._closed = True
|
||||
|
||||
|
||||
class DorisConnector(IbisConnector):
|
||||
class DorisConnector(MySqlConnector):
|
||||
"""Doris connector. Speaks MySQL protocol; routes through Doris connection."""
|
||||
|
||||
def __init__(self, connection_info):
|
||||
super().__init__(DataSource.doris, connection_info)
|
||||
|
||||
def _handle_pyarrow_unsupported_type(self, ibis_table: Table, **kwargs) -> Table:
|
||||
result_table = ibis_table
|
||||
for name, dtype in ibis_table.schema().items():
|
||||
if isinstance(dtype, Decimal):
|
||||
result_table = self._round_decimal_columns(
|
||||
result_table=result_table, col_name=name, **kwargs
|
||||
)
|
||||
elif isinstance(dtype, UUID):
|
||||
result_table = self._cast_uuid_columns(
|
||||
result_table=result_table, col_name=name
|
||||
)
|
||||
elif isinstance(dtype, dt.JSON):
|
||||
result_table = result_table.mutate(
|
||||
**{name: result_table[name].cast("string")}
|
||||
)
|
||||
return result_table
|
||||
# Skip MySqlConnector.__init__ — Doris does not accept the ANSI_QUOTES
|
||||
# init command and the connection is created via Doris routing.
|
||||
self._closed = False
|
||||
self.connection = DataSource.doris.get_connection(connection_info)
|
||||
|
||||
|
||||
def create_connector(data_source: DataSource, connection_info):
|
||||
def create_connector(data_source: DataSource, connection_info) -> MySqlConnector:
|
||||
if data_source == DataSource.doris:
|
||||
return DorisConnector(connection_info)
|
||||
return MySqlConnector(connection_info)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Arrow conversion helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# MySQL ``DECIMAL(M, D)`` allows ``M`` up to 65 and ``D`` up to 30, while
|
||||
# PyArrow's ``decimal128`` only supports precision up to 38. We clamp the
|
||||
# precision derived from ``cursor.description`` to ``38`` and the scale to
|
||||
# ``min(precision, 30)`` so PyArrow can still represent the value. A future
|
||||
# change could switch to ``decimal256`` when MySQL exceeds 38 digits.
|
||||
_ARROW_DECIMAL128_MAX_PRECISION = 38
|
||||
_MYSQL_DECIMAL_MAX_SCALE = 30
|
||||
# Fallback used when ``cursor.description`` does not carry precision/scale
|
||||
# (e.g. for the legacy ``FIELD_TYPE.DECIMAL`` code or non-MySQLdb cursors).
|
||||
_MYSQL_DECIMAL_FALLBACK_PRECISION = 38
|
||||
_MYSQL_DECIMAL_FALLBACK_SCALE = 9
|
||||
|
||||
|
||||
@cache
|
||||
def _mysql_field_type_map() -> dict[int, pa.DataType]:
|
||||
"""Build the FIELD_TYPE → Arrow map once per process.
|
||||
|
||||
Returns a fully-populated local dict, then ``functools.cache`` publishes
|
||||
the reference atomically. Concurrent callers either see the fully-built
|
||||
dict or wait on the cache's GIL-protected slot — they never observe a
|
||||
partially-populated map.
|
||||
"""
|
||||
from MySQLdb.constants import FIELD_TYPE as FT # noqa: PLC0415
|
||||
|
||||
base_map: dict[str, pa.DataType] = {
|
||||
"TINY": pa.int8(),
|
||||
"SHORT": pa.int16(),
|
||||
"LONG": pa.int32(),
|
||||
"INT24": pa.int32(),
|
||||
"LONGLONG": pa.int64(),
|
||||
"FLOAT": pa.float32(),
|
||||
"DOUBLE": pa.float64(),
|
||||
# ``DECIMAL`` / ``NEWDECIMAL`` are placeholders — the actual precision
|
||||
# and scale are read from ``cursor.description`` per-column.
|
||||
"DECIMAL": pa.decimal128(
|
||||
_MYSQL_DECIMAL_FALLBACK_PRECISION, _MYSQL_DECIMAL_FALLBACK_SCALE
|
||||
),
|
||||
"NEWDECIMAL": pa.decimal128(
|
||||
_MYSQL_DECIMAL_FALLBACK_PRECISION, _MYSQL_DECIMAL_FALLBACK_SCALE
|
||||
),
|
||||
"STRING": pa.string(),
|
||||
"VAR_STRING": pa.string(),
|
||||
"VARCHAR": pa.string(),
|
||||
"ENUM": pa.string(),
|
||||
"SET": pa.string(),
|
||||
"TINY_BLOB": pa.binary(),
|
||||
"MEDIUM_BLOB": pa.binary(),
|
||||
"LONG_BLOB": pa.binary(),
|
||||
"BLOB": pa.binary(),
|
||||
"JSON": pa.string(),
|
||||
"DATE": pa.date32(),
|
||||
"NEWDATE": pa.date32(),
|
||||
# MySQL ``TIME`` ranges ``-838:59:59`` to ``838:59:59`` and can be
|
||||
# negative — semantics PyArrow ``time64("us")`` cannot represent
|
||||
# (it only accepts 0–24h positive values). ``duration("us")`` is the
|
||||
# smallest Arrow type that captures the full MySQL range without loss.
|
||||
"TIME": pa.duration("us"),
|
||||
"DATETIME": pa.timestamp("us"),
|
||||
"TIMESTAMP": pa.timestamp("us"),
|
||||
"YEAR": pa.int16(),
|
||||
"BIT": pa.binary(),
|
||||
"GEOMETRY": pa.string(),
|
||||
"NULL": pa.null(),
|
||||
}
|
||||
result: dict[int, pa.DataType] = {}
|
||||
for name, arrow_type in base_map.items():
|
||||
code = getattr(FT, name, None)
|
||||
if code is not None:
|
||||
result[code] = arrow_type
|
||||
return result
|
||||
|
||||
|
||||
@cache
|
||||
def _mysql_unsigned_variant_map() -> dict[int, pa.DataType]:
|
||||
"""Build the FIELD_TYPE → unsigned-Arrow map once per process."""
|
||||
from MySQLdb.constants import FIELD_TYPE as FT # noqa: PLC0415
|
||||
|
||||
result: dict[int, pa.DataType] = {}
|
||||
for name, arrow_type in (
|
||||
("TINY", pa.uint8()),
|
||||
("SHORT", pa.uint16()),
|
||||
("LONG", pa.uint32()),
|
||||
("INT24", pa.uint32()),
|
||||
("LONGLONG", pa.uint64()),
|
||||
):
|
||||
code = getattr(FT, name, None)
|
||||
if code is not None:
|
||||
result[code] = arrow_type
|
||||
return result
|
||||
|
||||
|
||||
@cache
|
||||
def _mysql_blob_codes() -> frozenset[int]:
|
||||
"""Build the set of BLOB-family FIELD_TYPE codes once per process."""
|
||||
from MySQLdb.constants import FIELD_TYPE as FT # noqa: PLC0415
|
||||
|
||||
return frozenset(
|
||||
code
|
||||
for code in (
|
||||
getattr(FT, n, None)
|
||||
for n in ("BLOB", "TINY_BLOB", "MEDIUM_BLOB", "LONG_BLOB")
|
||||
)
|
||||
if code is not None
|
||||
)
|
||||
|
||||
|
||||
@cache
|
||||
def _mysql_string_codes() -> frozenset[int]:
|
||||
"""Build the set of STRING-family FIELD_TYPE codes once per process."""
|
||||
from MySQLdb.constants import FIELD_TYPE as FT # noqa: PLC0415
|
||||
|
||||
return frozenset(
|
||||
code
|
||||
for code in (getattr(FT, n, None) for n in ("STRING", "VAR_STRING", "VARCHAR"))
|
||||
if code is not None
|
||||
)
|
||||
|
||||
|
||||
@cache
|
||||
def _mysql_decimal_codes() -> frozenset[int]:
|
||||
"""Build the set of DECIMAL-family FIELD_TYPE codes once per process."""
|
||||
from MySQLdb.constants import FIELD_TYPE as FT # noqa: PLC0415
|
||||
|
||||
return frozenset(
|
||||
code
|
||||
for code in (getattr(FT, n, None) for n in ("DECIMAL", "NEWDECIMAL"))
|
||||
if code is not None
|
||||
)
|
||||
|
||||
|
||||
def _arrow_decimal_from_mysql_field(
|
||||
display_length: int | None,
|
||||
scale: int | None,
|
||||
is_unsigned: bool = False,
|
||||
) -> pa.DataType:
|
||||
"""Derive a ``pa.decimal128`` type from a MySQLdb ``cursor.description`` entry.
|
||||
|
||||
MySQLdb populates ``description[4]`` (PEP 249 ``precision``) with the
|
||||
``MYSQL_FIELD.length`` — i.e. the *display length*, which includes one
|
||||
byte for the decimal point (when ``D > 0``) and one byte for the sign
|
||||
when the column is signed. The declared ``DECIMAL(M, D)`` precision ``M``
|
||||
is recovered as::
|
||||
|
||||
M = length - (1 if unsigned else 0) - (1 if D > 0 else 0)
|
||||
|
||||
MySQL allows precision up to 65 and scale up to 30, but Arrow
|
||||
``decimal128`` caps precision at 38. We clamp precision to 38 and clamp
|
||||
scale to ``min(scale, precision, 30)`` so any value MySQL accepts (within
|
||||
the 38-digit Arrow ceiling) round-trips correctly. The previous
|
||||
hard-coded ``decimal128(38, 9)`` would silently lose digits when ``D > 9``.
|
||||
"""
|
||||
if display_length is None or display_length <= 0:
|
||||
precision = _MYSQL_DECIMAL_FALLBACK_PRECISION
|
||||
else:
|
||||
derived_scale = scale if scale is not None and scale >= 0 else 0
|
||||
sign_overhead = 0 if is_unsigned else 1
|
||||
point_overhead = 1 if derived_scale > 0 else 0
|
||||
precision = int(display_length) - sign_overhead - point_overhead
|
||||
if precision <= 0:
|
||||
precision = _MYSQL_DECIMAL_FALLBACK_PRECISION
|
||||
if scale is None or scale < 0:
|
||||
scale = _MYSQL_DECIMAL_FALLBACK_SCALE
|
||||
precision = min(int(precision), _ARROW_DECIMAL128_MAX_PRECISION)
|
||||
scale = min(int(scale), _MYSQL_DECIMAL_MAX_SCALE, precision)
|
||||
return pa.decimal128(precision, scale)
|
||||
|
||||
|
||||
def _mysql_field_arrow_type(
|
||||
type_code: int,
|
||||
flags: int = 0,
|
||||
precision: int | None = None,
|
||||
scale: int | None = None,
|
||||
) -> pa.DataType:
|
||||
from MySQLdb.constants import FLAG # noqa: PLC0415
|
||||
|
||||
field_map = _mysql_field_type_map()
|
||||
unsigned_map = _mysql_unsigned_variant_map()
|
||||
blob_codes = _mysql_blob_codes()
|
||||
string_codes = _mysql_string_codes()
|
||||
decimal_codes = _mysql_decimal_codes()
|
||||
|
||||
base = field_map.get(type_code, pa.string())
|
||||
|
||||
if flags & FLAG.UNSIGNED and type_code in unsigned_map:
|
||||
return unsigned_map[type_code]
|
||||
|
||||
# DECIMAL precision/scale come from ``cursor.description`` (PEP 249 fields
|
||||
# ``precision`` / ``scale``). MySQL ``DECIMAL(M, D)`` allows scale up to 30
|
||||
# — the previous hard-coded ``decimal128(38, 9)`` would lose digits when
|
||||
# ``D > 9``.
|
||||
if type_code in decimal_codes:
|
||||
return _arrow_decimal_from_mysql_field(
|
||||
precision, scale, is_unsigned=bool(flags & FLAG.UNSIGNED)
|
||||
)
|
||||
|
||||
# MySQL packs both TEXT and BLOB into FIELD_TYPE.*BLOB; BINARY flag is the
|
||||
# discriminator. Without BINARY they are TEXT (string); with BINARY they
|
||||
# are real BLOB (bytes — keep base type).
|
||||
if type_code in blob_codes and not (flags & FLAG.BINARY):
|
||||
return pa.string()
|
||||
|
||||
# STRING / VAR_STRING with BINARY flag is BINARY/VARBINARY (bytes).
|
||||
if type_code in string_codes and (flags & FLAG.BINARY):
|
||||
return pa.binary()
|
||||
|
||||
return base
|
||||
|
||||
|
||||
def _build_mysql_arrow_table(cursor) -> pa.Table:
|
||||
"""Convert a MySQLdb cursor result to a PyArrow table."""
|
||||
if cursor.description is None:
|
||||
return pa.table({})
|
||||
|
||||
# ``cursor.description_flags`` is a tuple of int flag bitmasks in
|
||||
# MySQLdb 2.x. Older / non-MySQLdb cursors may not provide it; in that
|
||||
# case we fall back to zero flags (BLOB → string, ignore UNSIGNED).
|
||||
flags_attr = getattr(cursor, "description_flags", None)
|
||||
if flags_attr is not None:
|
||||
flag_list = list(flags_attr)
|
||||
else:
|
||||
flag_list = [0] * len(cursor.description)
|
||||
flag_list = (flag_list + [0] * len(cursor.description))[: len(cursor.description)]
|
||||
|
||||
rows = cursor.fetchall()
|
||||
fields = []
|
||||
for i, col in enumerate(cursor.description):
|
||||
# PEP 249 ``description`` tuple:
|
||||
# (name, type_code, display_size, internal_size, precision, scale, null_ok)
|
||||
# MySQLdb populates ``precision``/``scale`` for ``NEWDECIMAL`` columns,
|
||||
# which lets us reflect the actual ``DECIMAL(M, D)`` instead of using
|
||||
# a hard-coded ``decimal128(38, 9)``.
|
||||
precision = col[4] if len(col) > 4 else None
|
||||
scale = col[5] if len(col) > 5 else None
|
||||
arrow_type = _mysql_field_arrow_type(
|
||||
col[1], flag_list[i] or 0, precision=precision, scale=scale
|
||||
)
|
||||
fields.append(pa.field(col[0], arrow_type, nullable=True))
|
||||
schema = pa.schema(fields)
|
||||
|
||||
if not rows:
|
||||
arrays = [pa.array([], type=field.type) for field in schema]
|
||||
else:
|
||||
arrays = [
|
||||
_build_mysql_column([row[i] for row in rows], schema.field(i).type)
|
||||
for i in range(len(fields))
|
||||
]
|
||||
# ``pa.table(dict(...), schema=...)`` silently drops a column when two
|
||||
# fields share the same name (the dict collapses the duplicate). Use
|
||||
# ``pa.Table.from_arrays`` so a query like
|
||||
# ``SELECT a.id, b.id FROM t a JOIN t b`` round-trips both ``id``
|
||||
# columns instead of returning a one-column table.
|
||||
return pa.Table.from_arrays(arrays, schema=schema)
|
||||
|
||||
|
||||
def _build_mysql_column(values: list, arrow_type: pa.DataType) -> pa.Array:
|
||||
"""Convert MySQLdb values into a PyArrow array of the given Arrow type."""
|
||||
if pa.types.is_string(arrow_type):
|
||||
processed = []
|
||||
for v in values:
|
||||
if v is None:
|
||||
processed.append(None)
|
||||
elif isinstance(v, bytes):
|
||||
processed.append(v.decode("utf-8", errors="replace"))
|
||||
elif isinstance(v, dict | list | tuple):
|
||||
processed.append(json.dumps(v, default=str))
|
||||
elif isinstance(v, str):
|
||||
processed.append(v)
|
||||
else:
|
||||
processed.append(str(v))
|
||||
return pa.array(processed, type=pa.string(), from_pandas=True)
|
||||
|
||||
if pa.types.is_binary(arrow_type):
|
||||
processed = [bytes(v) if isinstance(v, memoryview) else v for v in values]
|
||||
return pa.array(processed, type=arrow_type, from_pandas=True)
|
||||
|
||||
if pa.types.is_decimal(arrow_type):
|
||||
processed = [
|
||||
None
|
||||
if v is None
|
||||
else (v if isinstance(v, PyDecimal) else PyDecimal(str(v)))
|
||||
for v in values
|
||||
]
|
||||
return pa.array(processed, type=arrow_type, from_pandas=True)
|
||||
|
||||
if pa.types.is_timestamp(arrow_type):
|
||||
return pa.array(values, type=arrow_type, from_pandas=True)
|
||||
|
||||
if pa.types.is_duration(arrow_type):
|
||||
# MySQLdb returns TIME columns as ``datetime.timedelta``. PyArrow's
|
||||
# ``duration("us")`` accepts ``timedelta`` directly, but we convert to
|
||||
# signed microseconds explicitly so negative values (MySQL TIME may go
|
||||
# down to ``-838:59:59``) and values beyond 24h survive without loss.
|
||||
# ``timedelta.total_seconds() * 1e6`` would lose precision; we instead
|
||||
# combine ``days``, ``seconds`` and ``microseconds`` — all of which are
|
||||
# signed on negative ``timedelta`` values.
|
||||
import datetime # noqa: PLC0415
|
||||
|
||||
processed = [
|
||||
None
|
||||
if v is None
|
||||
else (
|
||||
v.days * 86_400_000_000 + v.seconds * 1_000_000 + v.microseconds
|
||||
if isinstance(v, datetime.timedelta)
|
||||
else v
|
||||
)
|
||||
for v in values
|
||||
]
|
||||
return pa.array(processed, type=arrow_type, from_pandas=True)
|
||||
|
||||
if pa.types.is_null(arrow_type):
|
||||
return pa.array([None] * len(values), type=pa.null())
|
||||
|
||||
return pa.array(values, type=arrow_type, from_pandas=True)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Connect kwargs helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _build_mysql_connect_kwargs(connection_info) -> dict:
|
||||
"""Translate ``MySqlConnectionInfo`` / ``ConnectionUrl`` into MySQLdb kwargs."""
|
||||
from urllib.parse import parse_qsl, unquote_plus, urlparse # noqa: PLC0415
|
||||
|
||||
if hasattr(connection_info, "connection_url"):
|
||||
url = connection_info.connection_url.get_secret_value()
|
||||
parsed = urlparse(url)
|
||||
if parsed.scheme not in {"mysql", "mysql+pymysql", "mysql+mysqldb"}:
|
||||
raise WrenError(
|
||||
ErrorCode.INVALID_CONNECTION_INFO,
|
||||
"MySQL connection URL must use mysql:// scheme",
|
||||
)
|
||||
|
||||
kwargs = dict(parse_qsl(parsed.query))
|
||||
if connection_info.kwargs:
|
||||
kwargs.update(connection_info.kwargs)
|
||||
|
||||
host = parsed.hostname or "127.0.0.1"
|
||||
if host == "localhost":
|
||||
host = "127.0.0.1"
|
||||
|
||||
out: dict = {
|
||||
"host": host,
|
||||
"port": int(parsed.port) if parsed.port else 3306,
|
||||
"user": parsed.username,
|
||||
"passwd": unquote_plus(parsed.password) if parsed.password else "",
|
||||
"db": parsed.path.lstrip("/") if parsed.path else None,
|
||||
"charset": "utf8mb4",
|
||||
"use_unicode": True,
|
||||
"autocommit": True,
|
||||
}
|
||||
out.update(kwargs)
|
||||
return out
|
||||
|
||||
info = connection_info
|
||||
kwargs = dict(info.kwargs) if info.kwargs else {}
|
||||
|
||||
# MySQLdb routes host="localhost" through a unix socket by default; force
|
||||
# TCP by normalising it to 127.0.0.1.
|
||||
host = info.host
|
||||
if host == "localhost":
|
||||
host = "127.0.0.1"
|
||||
|
||||
out = {
|
||||
"host": host,
|
||||
"port": int(info.port),
|
||||
"user": info.user,
|
||||
"passwd": (info.password.get_secret_value() if info.password else ""),
|
||||
"db": info.database,
|
||||
"charset": "utf8mb4",
|
||||
"use_unicode": True,
|
||||
"autocommit": True,
|
||||
}
|
||||
ssl = _mysql_ssl_kwargs(info)
|
||||
if ssl is not None:
|
||||
out["ssl"] = ssl
|
||||
out["ssl_mode"] = "VERIFY_CA" if "ca" in ssl else "REQUIRED"
|
||||
out.update(kwargs)
|
||||
return out
|
||||
|
||||
|
||||
def _build_doris_connect_kwargs(connection_info) -> dict:
|
||||
"""Translate ``DorisConnectionInfo`` / ``ConnectionUrl`` into MySQLdb kwargs."""
|
||||
if hasattr(connection_info, "connection_url"):
|
||||
return _build_mysql_connect_kwargs(connection_info)
|
||||
|
||||
info = connection_info
|
||||
kwargs = dict(info.kwargs) if info.kwargs else {}
|
||||
host = info.host
|
||||
if host == "localhost":
|
||||
host = "127.0.0.1"
|
||||
out = {
|
||||
"host": host,
|
||||
"port": int(info.port),
|
||||
"user": info.user,
|
||||
"passwd": (info.password.get_secret_value() if info.password else ""),
|
||||
"db": info.database,
|
||||
"charset": "utf8mb4",
|
||||
"use_unicode": True,
|
||||
"autocommit": True,
|
||||
}
|
||||
out.update(kwargs)
|
||||
return out
|
||||
|
||||
|
||||
def _mysql_ssl_kwargs(info) -> dict | None:
|
||||
"""Build the MySQLdb ``ssl`` kwarg dict from ``MySqlConnectionInfo`` SSL fields."""
|
||||
ssl_mode = info.ssl_mode if hasattr(info, "ssl_mode") and info.ssl_mode else None
|
||||
ssl_mode = ssl_mode.lower() if ssl_mode else None
|
||||
if not ssl_mode or ssl_mode == "disabled":
|
||||
return None
|
||||
if ssl_mode == "verify_ca":
|
||||
if not info.ssl_ca:
|
||||
raise WrenError(
|
||||
ErrorCode.INVALID_CONNECTION_INFO,
|
||||
"SSL CA must be provided when SSL mode is VERIFY CA",
|
||||
)
|
||||
return {"ca": info.ssl_ca.get_secret_value()}
|
||||
# 'enabled' / any other non-disabled mode: require SSL without CA verification.
|
||||
return {}
|
||||
|
||||
@@ -1,22 +1,241 @@
|
||||
from contextlib import suppress
|
||||
"""Native psycopg-based PostgreSQL connector.
|
||||
|
||||
This connector executes queries through psycopg3 directly and converts the
|
||||
cursor result into a PyArrow table using a hand-rolled OID-to-Arrow type map.
|
||||
It avoids the ibis-framework dependency entirely, which gives us:
|
||||
|
||||
* a smaller install surface for the ``postgres`` extra,
|
||||
* direct control over pg-specific type handling (numeric scale, arrays,
|
||||
intervals, jsonb), and
|
||||
* a single code path for both query execution and dry-run.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from decimal import ROUND_HALF_EVEN
|
||||
from decimal import Decimal as PyDecimal
|
||||
|
||||
import psycopg
|
||||
import pyarrow as pa
|
||||
from loguru import logger
|
||||
|
||||
from wren.connector.base import IbisConnector
|
||||
from wren.connector.base import ConnectorABC
|
||||
from wren.model.data_source import DataSource
|
||||
from wren.model.error import DIALECT_SQL, ErrorCode, ErrorPhase, WrenError
|
||||
|
||||
# Map of well-known PostgreSQL OIDs to Arrow types. OIDs that we have not
|
||||
# explicitly mapped fall back to ``pa.string()`` (see ``_get_pg_arrow_type``).
|
||||
_PG_OID_TO_ARROW: dict[int, pa.DataType] = {
|
||||
16: pa.bool_(),
|
||||
17: pa.binary(),
|
||||
18: pa.string(),
|
||||
19: pa.string(),
|
||||
20: pa.int64(),
|
||||
21: pa.int16(),
|
||||
23: pa.int32(),
|
||||
25: pa.string(),
|
||||
26: pa.int64(),
|
||||
114: pa.string(), # json
|
||||
142: pa.string(), # xml
|
||||
700: pa.float32(),
|
||||
701: pa.float64(),
|
||||
650: pa.string(),
|
||||
774: pa.string(),
|
||||
790: pa.string(),
|
||||
829: pa.string(),
|
||||
869: pa.string(),
|
||||
1040: pa.string(),
|
||||
1042: pa.string(),
|
||||
1043: pa.string(),
|
||||
1082: pa.date32(),
|
||||
1083: pa.time64("us"),
|
||||
1114: pa.timestamp("us"),
|
||||
1184: pa.timestamp("us", tz="UTC"),
|
||||
1186: pa.duration("us"),
|
||||
2950: pa.string(), # uuid
|
||||
3802: pa.string(), # jsonb
|
||||
1266: pa.string(), # timetz
|
||||
3614: pa.string(), # tsvector
|
||||
3615: pa.string(), # tsquery
|
||||
3904: pa.string(),
|
||||
3906: pa.string(),
|
||||
3908: pa.string(),
|
||||
3910: pa.string(),
|
||||
3912: pa.string(),
|
||||
3926: pa.string(),
|
||||
199: pa.list_(pa.string()), # _json
|
||||
1000: pa.list_(pa.bool_()),
|
||||
1003: pa.list_(pa.string()),
|
||||
1005: pa.list_(pa.int16()),
|
||||
1007: pa.list_(pa.int32()),
|
||||
1009: pa.list_(pa.string()),
|
||||
1014: pa.list_(pa.string()),
|
||||
1015: pa.list_(pa.string()),
|
||||
1016: pa.list_(pa.int64()),
|
||||
1021: pa.list_(pa.float32()),
|
||||
1022: pa.list_(pa.float64()),
|
||||
1028: pa.list_(pa.string()),
|
||||
1041: pa.list_(pa.string()),
|
||||
1115: pa.list_(pa.timestamp("us")),
|
||||
1182: pa.list_(pa.string()),
|
||||
1183: pa.list_(pa.string()),
|
||||
1185: pa.list_(pa.timestamp("us", tz="UTC")),
|
||||
1187: pa.list_(pa.string()),
|
||||
1270: pa.list_(pa.string()),
|
||||
2951: pa.list_(pa.string()), # _uuid
|
||||
3807: pa.list_(pa.string()), # _jsonb
|
||||
}
|
||||
|
||||
|
||||
def _get_pg_decimal_type(column) -> pa.DataType:
|
||||
"""Map a psycopg numeric column to the narrowest Arrow decimal type we can represent."""
|
||||
if column.scale is None:
|
||||
logger.debug(
|
||||
"Postgres NUMERIC column has no scale metadata; defaulting to decimal128(38, 9)"
|
||||
)
|
||||
scale = column.scale if column.scale is not None else 9
|
||||
scale = max(0, min(scale, 38))
|
||||
|
||||
precision = column.precision if column.precision is not None else 38
|
||||
if precision <= 0 or precision > 38:
|
||||
precision = 38
|
||||
precision = max(precision, scale, 1)
|
||||
precision = min(precision, 38)
|
||||
|
||||
return pa.decimal128(precision, scale)
|
||||
|
||||
|
||||
def _get_pg_arrow_type(column) -> pa.DataType:
|
||||
"""Map a psycopg cursor description column to an Arrow type."""
|
||||
if column.type_code == 1700:
|
||||
return _get_pg_decimal_type(column)
|
||||
if column.type_code == 1231:
|
||||
return pa.list_(_get_pg_decimal_type(column))
|
||||
return _PG_OID_TO_ARROW.get(column.type_code, pa.string())
|
||||
|
||||
|
||||
def _build_pg_arrow_table(cursor) -> pa.Table:
|
||||
"""Convert a psycopg3 cursor result to a PyArrow table."""
|
||||
if cursor.description is None:
|
||||
return pa.table({})
|
||||
|
||||
rows = cursor.fetchall()
|
||||
fields = [
|
||||
pa.field(column.name, _get_pg_arrow_type(column), nullable=True)
|
||||
for column in cursor.description
|
||||
]
|
||||
schema = pa.schema(fields)
|
||||
|
||||
if not rows:
|
||||
arrays = [pa.array([], type=field.type) for field in schema]
|
||||
else:
|
||||
arrays = [
|
||||
_build_pg_column(
|
||||
[row[index] for row in rows],
|
||||
field.type,
|
||||
cursor.description[index].type_code,
|
||||
)
|
||||
for index, field in enumerate(schema)
|
||||
]
|
||||
|
||||
# Build positionally — ``pa.table({...})`` silently drops duplicate column
|
||||
# names (very common in joins like ``SELECT a.id, b.id FROM t a, t b``).
|
||||
return pa.Table.from_arrays(arrays, schema=schema)
|
||||
|
||||
|
||||
def _build_pg_column(
|
||||
values: list, arrow_type: pa.DataType, pg_type_oid: int | None = None
|
||||
) -> pa.Array:
|
||||
"""Build a PyArrow column from psycopg values with PG-specific coercions."""
|
||||
|
||||
def _coerce_decimal(value: PyDecimal | None, target_type: pa.DataType):
|
||||
if value is None or not isinstance(value, PyDecimal):
|
||||
return value
|
||||
|
||||
quantize_value = PyDecimal(f"1E-{target_type.scale}")
|
||||
try:
|
||||
return value.quantize(quantize_value, rounding=ROUND_HALF_EVEN)
|
||||
except Exception:
|
||||
return value
|
||||
|
||||
if arrow_type == pa.string():
|
||||
processed = []
|
||||
for value in values:
|
||||
if value is None:
|
||||
# json / jsonb SQL NULLs come back as Python None too, but the
|
||||
# SQL value ``'null'::jsonb`` is also None at the Python level.
|
||||
# Keep both as None — callers that care about the distinction
|
||||
# should cast the column to text in SQL.
|
||||
processed.append(None)
|
||||
elif isinstance(value, dict | list):
|
||||
processed.append(json.dumps(value, default=str))
|
||||
elif not isinstance(value, str):
|
||||
processed.append(str(value))
|
||||
else:
|
||||
processed.append(value)
|
||||
return pa.array(processed, type=pa.string(), from_pandas=True)
|
||||
|
||||
if pa.types.is_binary(arrow_type):
|
||||
processed = [
|
||||
bytes(value) if isinstance(value, memoryview) else value for value in values
|
||||
]
|
||||
return pa.array(processed, type=pa.binary(), from_pandas=True)
|
||||
|
||||
if pa.types.is_decimal(arrow_type):
|
||||
processed = [_coerce_decimal(value, arrow_type) for value in values]
|
||||
return pa.array(processed, type=arrow_type, from_pandas=True)
|
||||
|
||||
if pa.types.is_list(arrow_type) and pa.types.is_string(arrow_type.value_type):
|
||||
processed = []
|
||||
for value in values:
|
||||
if value is None:
|
||||
processed.append(None)
|
||||
continue
|
||||
|
||||
items = []
|
||||
for item in value:
|
||||
if item is None:
|
||||
items.append(None)
|
||||
elif isinstance(item, dict | list):
|
||||
items.append(json.dumps(item, default=str))
|
||||
elif isinstance(item, str):
|
||||
items.append(item)
|
||||
else:
|
||||
items.append(str(item))
|
||||
processed.append(items)
|
||||
|
||||
return pa.array(processed, type=arrow_type, from_pandas=True)
|
||||
|
||||
if pa.types.is_list(arrow_type) and pa.types.is_decimal(arrow_type.value_type):
|
||||
processed = []
|
||||
for value in values:
|
||||
if value is None:
|
||||
processed.append(None)
|
||||
else:
|
||||
processed.append(
|
||||
[_coerce_decimal(item, arrow_type.value_type) for item in value]
|
||||
)
|
||||
return pa.array(processed, type=arrow_type, from_pandas=True)
|
||||
|
||||
return pa.array(values, type=arrow_type, from_pandas=True)
|
||||
|
||||
|
||||
class PostgresConnector(ConnectorABC):
|
||||
"""Native psycopg3 implementation of the Wren postgres connector."""
|
||||
|
||||
class PostgresConnector(IbisConnector):
|
||||
def __init__(self, connection_info):
|
||||
super().__init__(DataSource.postgres, connection_info)
|
||||
self.connection = DataSource.postgres.get_connection(connection_info)
|
||||
self._closed = False
|
||||
|
||||
def query(self, sql: str, limit: int | None = None) -> pa.Table:
|
||||
import psycopg # noqa: PLC0415
|
||||
if limit is not None:
|
||||
sql = f"SELECT * FROM ({sql}) AS _sub LIMIT {limit}"
|
||||
|
||||
try:
|
||||
return super().query(sql, limit)
|
||||
with self.connection.cursor() as cursor:
|
||||
cursor.execute(sql)
|
||||
return _build_pg_arrow_table(cursor)
|
||||
except psycopg.errors.QueryCanceled:
|
||||
raise
|
||||
except (WrenError, TimeoutError):
|
||||
@@ -30,10 +249,10 @@ class PostgresConnector(IbisConnector):
|
||||
) from e
|
||||
|
||||
def dry_run(self, sql: str) -> None:
|
||||
import psycopg # noqa: PLC0415
|
||||
|
||||
wrapped = f"SELECT * FROM ({sql}) AS _sub LIMIT 0"
|
||||
try:
|
||||
super().dry_run(sql)
|
||||
with self.connection.cursor() as cursor:
|
||||
cursor.execute(wrapped)
|
||||
except psycopg.errors.QueryCanceled:
|
||||
raise
|
||||
except (WrenError, TimeoutError):
|
||||
@@ -47,26 +266,17 @@ class PostgresConnector(IbisConnector):
|
||||
) from e
|
||||
|
||||
def close(self) -> None:
|
||||
if self._closed or not hasattr(self, "connection") or self.connection is None:
|
||||
if self._closed or self.connection is None:
|
||||
return
|
||||
try:
|
||||
if hasattr(self.connection, "con") and self.connection.con is not None:
|
||||
if (
|
||||
hasattr(self.connection.con, "closed")
|
||||
and not self.connection.con.closed
|
||||
):
|
||||
with suppress(Exception):
|
||||
self.connection.con.cancel()
|
||||
import time # noqa: PLC0415
|
||||
|
||||
time.sleep(0.1)
|
||||
self.connection.con.close()
|
||||
elif hasattr(self.connection, "close"):
|
||||
if not self.connection.closed:
|
||||
try:
|
||||
self.connection.cancel()
|
||||
except Exception:
|
||||
pass
|
||||
self.connection.close()
|
||||
except Exception as e:
|
||||
logger.warning(f"Error closing postgres connection: {e}")
|
||||
if hasattr(self.connection, "con"):
|
||||
self.connection.con = None
|
||||
finally:
|
||||
self._closed = True
|
||||
self.connection = None
|
||||
|
||||
@@ -0,0 +1,442 @@
|
||||
"""Native Trino connector that talks to the trino python client directly.
|
||||
|
||||
This module bypasses ibis-framework[trino]; ``cursor.description`` exposes
|
||||
Trino type strings (``array(row("a" integer, "b" varchar))`` etc.) which we
|
||||
lex with sqlglot to build an equivalent PyArrow schema.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import contextlib
|
||||
import datetime as dtlib
|
||||
import json
|
||||
from decimal import Decimal as PyDecimal
|
||||
from urllib.parse import parse_qsl, urlparse
|
||||
|
||||
import pyarrow as pa
|
||||
import sqlglot
|
||||
import sqlglot.errors
|
||||
from loguru import logger
|
||||
from sqlglot.expressions import ColumnDef, DataType
|
||||
|
||||
from wren.connector.base import ConnectorABC
|
||||
from wren.model.error import (
|
||||
DIALECT_SQL,
|
||||
ErrorCode,
|
||||
ErrorPhase,
|
||||
WrenError,
|
||||
)
|
||||
|
||||
|
||||
def _parse_trino_data_type(type_str: str | None) -> pa.DataType:
|
||||
"""Parse a Trino type string from ``cursor.description`` into an Arrow type.
|
||||
|
||||
Delegates the lexing/parsing to sqlglot and walks the resulting DataType
|
||||
AST to build the equivalent PyArrow type. sqlglot handles the awkward
|
||||
cases (anonymous row fields whose type contains whitespace, nested
|
||||
array/map/row, decimal(p,s), etc.).
|
||||
"""
|
||||
if type_str is None:
|
||||
return pa.string()
|
||||
try:
|
||||
parsed = sqlglot.parse_one(type_str, into=DataType, dialect="trino")
|
||||
except sqlglot.errors.ParseError:
|
||||
logger.warning(f"Failed to parse trino type string: {type_str}")
|
||||
return pa.string()
|
||||
if parsed is None:
|
||||
return pa.string()
|
||||
return _trino_data_type_to_arrow(parsed)
|
||||
|
||||
|
||||
_TRINO_DATA_TYPE_TO_ARROW: dict = {}
|
||||
|
||||
|
||||
def _init_trino_data_type_map() -> None:
|
||||
if _TRINO_DATA_TYPE_TO_ARROW:
|
||||
return
|
||||
|
||||
T = DataType.Type
|
||||
_TRINO_DATA_TYPE_TO_ARROW.update(
|
||||
{
|
||||
T.BOOLEAN: pa.bool_(),
|
||||
T.TINYINT: pa.int8(),
|
||||
T.SMALLINT: pa.int16(),
|
||||
T.INT: pa.int32(),
|
||||
T.BIGINT: pa.int64(),
|
||||
T.FLOAT: pa.float32(),
|
||||
T.DOUBLE: pa.float64(),
|
||||
T.VARCHAR: pa.string(),
|
||||
T.CHAR: pa.string(),
|
||||
T.NCHAR: pa.string(),
|
||||
T.NVARCHAR: pa.string(),
|
||||
T.TEXT: pa.string(),
|
||||
T.JSON: pa.string(),
|
||||
T.UUID: pa.string(),
|
||||
T.IPADDRESS: pa.string(),
|
||||
T.HLLSKETCH: pa.string(), # hyperloglog
|
||||
T.GEOMETRY: pa.string(),
|
||||
T.VARBINARY: pa.binary(),
|
||||
T.BINARY: pa.binary(),
|
||||
T.DATE: pa.date32(),
|
||||
T.TIME: pa.time64("us"),
|
||||
T.TIMETZ: pa.time64("us"),
|
||||
# Millisecond precision matches PyArrow's default and keeps round-trip
|
||||
# behaviour stable for ``timestamp(n)`` results across Trino versions.
|
||||
T.TIMESTAMP: pa.timestamp("ms"),
|
||||
T.TIMESTAMPTZ: pa.timestamp("ms", tz="UTC"),
|
||||
T.TIMESTAMPLTZ: pa.timestamp("ms", tz="UTC"),
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _trino_data_type_to_arrow(node) -> pa.DataType:
|
||||
_init_trino_data_type_map()
|
||||
if not isinstance(node, DataType):
|
||||
# e.g. Interval — fall back to string representation.
|
||||
return pa.string()
|
||||
|
||||
kind = node.this
|
||||
T = DataType.Type
|
||||
if kind in _TRINO_DATA_TYPE_TO_ARROW:
|
||||
return _TRINO_DATA_TYPE_TO_ARROW[kind]
|
||||
|
||||
if kind == T.DECIMAL:
|
||||
precision, scale = 38, 9
|
||||
params = node.expressions
|
||||
if len(params) >= 1:
|
||||
with contextlib.suppress(AttributeError, ValueError):
|
||||
precision = min(int(params[0].this.this), 38)
|
||||
if len(params) >= 2:
|
||||
with contextlib.suppress(AttributeError, ValueError):
|
||||
scale = min(int(params[1].this.this), precision)
|
||||
return pa.decimal128(precision, scale)
|
||||
|
||||
if kind == T.ARRAY:
|
||||
inner = node.expressions[0] if node.expressions else None
|
||||
return pa.list_(_trino_data_type_to_arrow(inner) if inner else pa.string())
|
||||
|
||||
if kind == T.MAP:
|
||||
if len(node.expressions) >= 2:
|
||||
return pa.map_(
|
||||
_trino_data_type_to_arrow(node.expressions[0]),
|
||||
_trino_data_type_to_arrow(node.expressions[1]),
|
||||
)
|
||||
return pa.string()
|
||||
|
||||
if kind == T.STRUCT:
|
||||
fields: list[pa.Field] = []
|
||||
for idx, child in enumerate(node.expressions):
|
||||
if isinstance(child, ColumnDef):
|
||||
# Named row field: row(a integer, b varchar)
|
||||
name = child.name or f"f{idx}"
|
||||
inner = child.args.get("kind")
|
||||
fields.append(
|
||||
pa.field(
|
||||
name,
|
||||
_trino_data_type_to_arrow(inner) if inner else pa.string(),
|
||||
)
|
||||
)
|
||||
else:
|
||||
# Anonymous row field: row(map(varchar,integer), bigint)
|
||||
fields.append(pa.field(f"f{idx}", _trino_data_type_to_arrow(child)))
|
||||
return pa.struct(fields)
|
||||
|
||||
return pa.string()
|
||||
|
||||
|
||||
def _build_trino_column(values: list, arrow_type: pa.DataType) -> pa.Array:
|
||||
"""Convert trino DB-API values to a PyArrow array of the given Arrow type."""
|
||||
if pa.types.is_string(arrow_type):
|
||||
processed = []
|
||||
for v in values:
|
||||
if v is None:
|
||||
processed.append(None)
|
||||
elif isinstance(v, dict | list | tuple):
|
||||
processed.append(json.dumps(v, default=str))
|
||||
elif isinstance(v, str):
|
||||
processed.append(v)
|
||||
else:
|
||||
processed.append(str(v))
|
||||
return pa.array(processed, type=pa.string(), from_pandas=True)
|
||||
|
||||
if pa.types.is_binary(arrow_type):
|
||||
processed = [bytes(v) if isinstance(v, memoryview) else v for v in values]
|
||||
return pa.array(processed, type=arrow_type, from_pandas=True)
|
||||
|
||||
if pa.types.is_decimal(arrow_type):
|
||||
processed = [
|
||||
None
|
||||
if v is None
|
||||
else (v if isinstance(v, PyDecimal) else PyDecimal(str(v)))
|
||||
for v in values
|
||||
]
|
||||
return pa.array(processed, type=arrow_type, from_pandas=True)
|
||||
|
||||
if pa.types.is_timestamp(arrow_type):
|
||||
# The trino driver returns either datetime objects or ISO-8601 strings
|
||||
# depending on column type and adapter settings; normalise to datetime.
|
||||
processed = []
|
||||
for v in values:
|
||||
if v is None or isinstance(v, dtlib.datetime):
|
||||
processed.append(v)
|
||||
else:
|
||||
try:
|
||||
processed.append(dtlib.datetime.fromisoformat(str(v)))
|
||||
except ValueError:
|
||||
processed.append(None)
|
||||
return pa.array(processed, type=arrow_type, from_pandas=True)
|
||||
|
||||
if pa.types.is_date(arrow_type):
|
||||
processed = []
|
||||
for v in values:
|
||||
if v is None or isinstance(v, dtlib.date):
|
||||
processed.append(v)
|
||||
else:
|
||||
try:
|
||||
processed.append(dtlib.date.fromisoformat(str(v)))
|
||||
except ValueError:
|
||||
processed.append(None)
|
||||
return pa.array(processed, type=arrow_type, from_pandas=True)
|
||||
|
||||
if pa.types.is_time(arrow_type):
|
||||
processed = []
|
||||
for v in values:
|
||||
if v is None or isinstance(v, dtlib.time):
|
||||
processed.append(v)
|
||||
else:
|
||||
try:
|
||||
processed.append(dtlib.time.fromisoformat(str(v)))
|
||||
except ValueError:
|
||||
processed.append(None)
|
||||
return pa.array(processed, type=arrow_type, from_pandas=True)
|
||||
|
||||
if pa.types.is_struct(arrow_type):
|
||||
# The trino driver returns Python tuples for row(...) values. PyArrow
|
||||
# accepts dicts keyed by field name; convert to make field order
|
||||
# mismatch (e.g. anonymous row) explicit.
|
||||
names = [f.name for f in arrow_type]
|
||||
processed: list = []
|
||||
for v in values:
|
||||
if v is None:
|
||||
processed.append(None)
|
||||
elif isinstance(v, dict):
|
||||
processed.append(v)
|
||||
else:
|
||||
processed.append(dict(zip(names, v, strict=False)))
|
||||
return pa.array(processed, type=arrow_type, from_pandas=True)
|
||||
|
||||
if pa.types.is_map(arrow_type):
|
||||
# PyArrow's map_ constructor expects an iterable of (key, value) pairs;
|
||||
# the trino driver returns Python dicts.
|
||||
processed = [None if v is None else list(v.items()) for v in values]
|
||||
return pa.array(processed, type=arrow_type, from_pandas=True)
|
||||
|
||||
return pa.array(values, type=arrow_type, from_pandas=True)
|
||||
|
||||
|
||||
def _build_trino_arrow_table(cursor) -> pa.Table:
|
||||
"""Convert a trino DB-API cursor result to a PyArrow table."""
|
||||
if cursor.description is None:
|
||||
return pa.table({})
|
||||
|
||||
rows = cursor.fetchall()
|
||||
fields = [
|
||||
pa.field(col[0], _parse_trino_data_type(col[1]), nullable=True)
|
||||
for col in cursor.description
|
||||
]
|
||||
schema = pa.schema(fields)
|
||||
|
||||
if not rows:
|
||||
arrays = [pa.array([], type=field.type) for field in schema]
|
||||
else:
|
||||
arrays = [
|
||||
_build_trino_column([row[i] for row in rows], schema.field(i).type)
|
||||
for i in range(len(fields))
|
||||
]
|
||||
|
||||
return pa.table(
|
||||
dict(zip([f.name for f in fields], arrays, strict=False)),
|
||||
schema=schema,
|
||||
)
|
||||
|
||||
|
||||
def _build_trino_connect_kwargs(connection_info) -> dict:
|
||||
"""Build kwargs for ``trino.dbapi.connect`` from either a typed
|
||||
``TrinoConnectionInfo`` or a generic ``ConnectionUrl``.
|
||||
|
||||
Returns a dict that may contain the sentinel ``_password`` key — the caller
|
||||
is expected to pop it before passing the dict to ``trino_connect``.
|
||||
"""
|
||||
if hasattr(connection_info, "connection_url"):
|
||||
url = connection_info.connection_url.get_secret_value()
|
||||
return _parse_trino_url(url, connection_info.kwargs)
|
||||
|
||||
info = connection_info # TrinoConnectionInfo
|
||||
kwargs = dict(info.kwargs) if info.kwargs else {}
|
||||
password = info.password.get_secret_value() if info.password else None
|
||||
|
||||
out: dict = {
|
||||
"host": info.host,
|
||||
"port": int(info.port),
|
||||
"user": info.user,
|
||||
"catalog": info.catalog,
|
||||
"schema": info.trino_schema,
|
||||
# Pin session timezone to UTC so CAST('...' AS TIMESTAMP WITH TIME ZONE)
|
||||
# produces deterministic results across deployments.
|
||||
"timezone": "UTC",
|
||||
"_password": password,
|
||||
}
|
||||
out.update(kwargs)
|
||||
return out
|
||||
|
||||
|
||||
def _parse_trino_url(url: str, extra_kwargs: dict | None) -> dict:
|
||||
"""Parse a ``trino://[user[:pwd]@]host[:port][/catalog[/schema]][?...]`` URL.
|
||||
|
||||
Returns the same ``_password`` sentinel-key shape as
|
||||
:func:`_build_trino_connect_kwargs`.
|
||||
"""
|
||||
parsed = urlparse(url)
|
||||
if parsed.scheme not in {"trino", "trino+https"}:
|
||||
raise WrenError(
|
||||
ErrorCode.INVALID_CONNECTION_INFO,
|
||||
"Trino connection URL must use trino:// scheme",
|
||||
)
|
||||
|
||||
if not parsed.username:
|
||||
raise WrenError(
|
||||
ErrorCode.INVALID_CONNECTION_INFO,
|
||||
"Trino connection URL must include a username",
|
||||
)
|
||||
|
||||
path_parts = parsed.path.lstrip("/").split("/")
|
||||
catalog = path_parts[0] if path_parts and path_parts[0] else None
|
||||
schema = path_parts[1] if len(path_parts) > 1 else None
|
||||
|
||||
query_kwargs = dict(parse_qsl(parsed.query))
|
||||
if extra_kwargs:
|
||||
query_kwargs.update(extra_kwargs)
|
||||
|
||||
out: dict = {
|
||||
"host": parsed.hostname,
|
||||
"port": int(parsed.port or 8080),
|
||||
"user": parsed.username,
|
||||
"catalog": catalog,
|
||||
"schema": schema,
|
||||
"timezone": "UTC",
|
||||
"_password": parsed.password,
|
||||
}
|
||||
if parsed.scheme == "trino+https":
|
||||
out["http_scheme"] = "https"
|
||||
out.update(query_kwargs)
|
||||
return out
|
||||
|
||||
|
||||
_TRINO_IMPORT_HINT = (
|
||||
"The 'trino' package is required for the Trino connector. "
|
||||
"Install it with: pip install wren-engine[trino]"
|
||||
)
|
||||
|
||||
|
||||
def _import_trino():
|
||||
"""Lazy import of the ``trino`` package with a clear install hint."""
|
||||
try:
|
||||
import trino # noqa: PLC0415
|
||||
|
||||
return trino
|
||||
except ImportError as e:
|
||||
raise WrenError(
|
||||
ErrorCode.INVALID_CONNECTION_INFO,
|
||||
f"{_TRINO_IMPORT_HINT} (original error: {e})",
|
||||
) from e
|
||||
|
||||
|
||||
def _strip_trailing_semicolon(sql: str) -> str:
|
||||
"""Strip trailing whitespace and an optional final semicolon.
|
||||
|
||||
Wrapping ``SELECT * FROM ({sql}) AS _sub LIMIT N`` breaks if ``sql`` ends
|
||||
with ``;`` because the parser sees ``...; ) AS _sub...``.
|
||||
"""
|
||||
return sql.rstrip().removesuffix(";").rstrip()
|
||||
|
||||
|
||||
class TrinoConnector(ConnectorABC):
|
||||
"""Native trino DB-API connector that bypasses ibis-project."""
|
||||
|
||||
def __init__(self, connection_info):
|
||||
trino = _import_trino()
|
||||
BasicAuthentication = trino.auth.BasicAuthentication
|
||||
JWTAuthentication = trino.auth.JWTAuthentication
|
||||
trino_connect = trino.dbapi.connect
|
||||
|
||||
connect_kwargs = _build_trino_connect_kwargs(connection_info)
|
||||
password = connect_kwargs.pop("_password", None)
|
||||
token = connect_kwargs.pop("access_token", None)
|
||||
|
||||
if connect_kwargs.get("auth") is None:
|
||||
user = connect_kwargs.get("user")
|
||||
if token:
|
||||
connect_kwargs["auth"] = JWTAuthentication(token)
|
||||
connect_kwargs.setdefault("http_scheme", "https")
|
||||
elif user and password:
|
||||
connect_kwargs["auth"] = BasicAuthentication(user, password)
|
||||
connect_kwargs.setdefault("http_scheme", "https")
|
||||
|
||||
self.connection = trino_connect(**connect_kwargs)
|
||||
self._closed = False
|
||||
|
||||
def query(self, sql: str, limit: int | None = None) -> pa.Table:
|
||||
trino = _import_trino()
|
||||
|
||||
if limit is not None:
|
||||
sql = f"SELECT * FROM ({_strip_trailing_semicolon(sql)}) AS _sub LIMIT {limit}"
|
||||
try:
|
||||
with contextlib.closing(self.connection.cursor()) as cursor:
|
||||
cursor.execute(sql)
|
||||
return _build_trino_arrow_table(cursor)
|
||||
except trino.exceptions.TrinoQueryError as e:
|
||||
if e.error_name == "EXCEEDED_TIME_LIMIT":
|
||||
raise
|
||||
raise WrenError(
|
||||
ErrorCode.INVALID_SQL,
|
||||
str(e),
|
||||
phase=ErrorPhase.SQL_EXECUTION,
|
||||
metadata={DIALECT_SQL: sql},
|
||||
) from e
|
||||
except (WrenError, TimeoutError):
|
||||
raise
|
||||
|
||||
def dry_run(self, sql: str) -> None:
|
||||
trino = _import_trino()
|
||||
|
||||
wrapped = f"SELECT * FROM ({_strip_trailing_semicolon(sql)}) AS _sub LIMIT 0"
|
||||
try:
|
||||
with contextlib.closing(self.connection.cursor()) as cursor:
|
||||
cursor.execute(wrapped)
|
||||
cursor.fetchall()
|
||||
except trino.exceptions.TrinoQueryError as e:
|
||||
if e.error_name == "EXCEEDED_TIME_LIMIT":
|
||||
raise
|
||||
raise WrenError(
|
||||
ErrorCode.INVALID_SQL,
|
||||
str(e),
|
||||
phase=ErrorPhase.SQL_DRY_RUN,
|
||||
metadata={DIALECT_SQL: sql},
|
||||
) from e
|
||||
except (WrenError, TimeoutError):
|
||||
raise
|
||||
|
||||
def close(self) -> None:
|
||||
if self._closed or self.connection is None:
|
||||
return
|
||||
try:
|
||||
self.connection.close()
|
||||
except Exception as e:
|
||||
logger.warning(f"Error closing Trino connection: {e}")
|
||||
finally:
|
||||
self._closed = True
|
||||
self.connection = None
|
||||
|
||||
|
||||
def create_connector(data_source, connection_info) -> TrinoConnector:
|
||||
return TrinoConnector(connection_info)
|
||||
@@ -1,17 +1,21 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import ssl
|
||||
import datetime as dtlib
|
||||
import urllib
|
||||
from enum import Enum, StrEnum, auto
|
||||
from json import loads
|
||||
from typing import Any
|
||||
from urllib.parse import unquote_plus
|
||||
from typing import TYPE_CHECKING, Any, Union
|
||||
from urllib.parse import unquote_plus, urlparse
|
||||
|
||||
import boto3
|
||||
import ibis
|
||||
from ibis import BaseBackend
|
||||
|
||||
try:
|
||||
import pyodbc
|
||||
except ImportError: # pragma: no cover
|
||||
pyodbc = None
|
||||
|
||||
from wren.model import (
|
||||
AthenaConnectionInfo,
|
||||
BaseConnectionInfo,
|
||||
@@ -37,12 +41,25 @@ from wren.model import (
|
||||
S3FileConnectionInfo,
|
||||
SnowflakeConnectionInfo,
|
||||
SparkConnectionInfo,
|
||||
SSLMode,
|
||||
TrinoConnectionInfo,
|
||||
)
|
||||
from wren.model.error import ErrorCode, WrenError
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import MySQLdb
|
||||
import psycopg
|
||||
from pyathena.connection import Connection as PyAthenaConnection
|
||||
|
||||
# get_connection() may return either an ibis BaseBackend (for connectors still
|
||||
# routed through ibis) or a native driver connection for connectors that have
|
||||
# dropped the ibis dependency (Athena via pyathena, MySQL/Doris via MySQLdb,
|
||||
# Postgres/Canner via psycopg).
|
||||
BackendOrConnection = Union[
|
||||
BaseBackend, "PyAthenaConnection", "MySQLdb.Connection", "psycopg.Connection"
|
||||
]
|
||||
|
||||
X_WREN_DB_STATEMENT_TIMEOUT = "x-wren-db-statement_timeout"
|
||||
MSSQL_DATETIMEOFFSET_TYPE_CODE = -155
|
||||
|
||||
|
||||
class DataSource(StrEnum):
|
||||
@@ -67,7 +84,7 @@ class DataSource(StrEnum):
|
||||
spark = auto()
|
||||
databricks = auto()
|
||||
|
||||
def get_connection(self, info: ConnectionInfo) -> BaseBackend:
|
||||
def get_connection(self, info: ConnectionInfo) -> BackendOrConnection:
|
||||
try:
|
||||
return DataSourceExtension[self].get_connection(info)
|
||||
except KeyError:
|
||||
@@ -181,7 +198,8 @@ class DataSource(StrEnum):
|
||||
def _handle_clickhouse_url(
|
||||
self, parsed: urllib.parse.ParseResult
|
||||
) -> ClickHouseConnectionInfo:
|
||||
if not parsed.scheme or parsed.scheme != "clickhouse":
|
||||
allowed_schemes = {"clickhouse", "clickhouse+http", "clickhouse+https"}
|
||||
if not parsed.scheme or parsed.scheme not in allowed_schemes:
|
||||
raise WrenError(
|
||||
ErrorCode.INVALID_CONNECTION_INFO,
|
||||
"Invalid connection URL for ClickHouse",
|
||||
@@ -201,6 +219,8 @@ class DataSource(StrEnum):
|
||||
if "secure" in parsed_kwargs:
|
||||
kwargs["secure"] = self._safe_strtobool(parsed_kwargs["secure"])
|
||||
parsed_kwargs.pop("secure")
|
||||
elif parsed.scheme == "clickhouse+https":
|
||||
kwargs["secure"] = True
|
||||
kwargs["kwargs"] = parsed_kwargs
|
||||
return ClickHouseConnectionInfo(**kwargs)
|
||||
|
||||
@@ -230,10 +250,34 @@ class DataSourceExtension(Enum):
|
||||
databricks = "databricks"
|
||||
spark = "spark"
|
||||
|
||||
def get_connection(self, info: ConnectionInfo) -> BaseBackend:
|
||||
def get_connection(self, info: ConnectionInfo) -> BackendOrConnection:
|
||||
try:
|
||||
if hasattr(info, "connection_url"):
|
||||
# MySQL / Doris use the native MySQLdb driver, not ibis.
|
||||
if self.name in {"mysql", "doris"}:
|
||||
return getattr(self, f"get_{self.name}_connection")(info)
|
||||
if self.name == "trino":
|
||||
# Trino uses the native DB-API client; the generic
|
||||
# ``ibis.connect()`` path was removed when the native
|
||||
# connector landed. Route the URL through the dedicated
|
||||
# parser so callers still get a working connection.
|
||||
from wren.connector.trino import ( # noqa: PLC0415
|
||||
_build_trino_connect_kwargs,
|
||||
)
|
||||
|
||||
trino_kwargs = _build_trino_connect_kwargs(info)
|
||||
trino_kwargs.pop("_password", None)
|
||||
trino_kwargs.pop("access_token", None)
|
||||
from trino.dbapi import ( # noqa: PLC0415
|
||||
connect as trino_connect,
|
||||
)
|
||||
|
||||
return trino_connect(**trino_kwargs)
|
||||
kwargs = info.kwargs if info.kwargs else {}
|
||||
if self.name == "mssql":
|
||||
return self.get_mssql_connection_from_url(
|
||||
info.connection_url.get_secret_value(), kwargs
|
||||
)
|
||||
return ibis.connect(info.connection_url.get_secret_value(), **kwargs)
|
||||
if self.name in {"local_file", "redshift", "spark", "duckdb", "datafusion"}:
|
||||
raise NotImplementedError(
|
||||
@@ -248,41 +292,24 @@ class DataSourceExtension(Enum):
|
||||
raise WrenError(ErrorCode.GET_CONNECTION_ERROR, f"{e!s}") from e
|
||||
|
||||
@staticmethod
|
||||
def get_athena_connection(info: AthenaConnectionInfo) -> BaseBackend:
|
||||
kwargs: dict[str, Any] = {
|
||||
"s3_staging_dir": info.s3_staging_dir.get_secret_value(),
|
||||
"schema_name": info.schema_name,
|
||||
}
|
||||
if info.region_name:
|
||||
kwargs["region_name"] = info.region_name
|
||||
def get_athena_connection(info: AthenaConnectionInfo) -> PyAthenaConnection:
|
||||
"""Open a pyathena DB-API connection.
|
||||
|
||||
if info.web_identity_token and info.role_arn:
|
||||
oidc_token = info.web_identity_token.get_secret_value()
|
||||
role_arn = info.role_arn.get_secret_value()
|
||||
session_name = info.role_session_name or "wren-oidc-session"
|
||||
region = info.region_name if info.region_name else None
|
||||
sts = boto3.client("sts", region_name=region)
|
||||
resp = sts.assume_role_with_web_identity(
|
||||
RoleArn=role_arn,
|
||||
RoleSessionName=session_name,
|
||||
WebIdentityToken=oidc_token,
|
||||
)
|
||||
creds = resp["Credentials"]
|
||||
kwargs["aws_access_key_id"] = creds["AccessKeyId"]
|
||||
kwargs["aws_secret_access_key"] = creds["SecretAccessKey"]
|
||||
kwargs["aws_session_token"] = creds["SessionToken"]
|
||||
elif info.aws_access_key_id and info.aws_secret_access_key:
|
||||
kwargs["aws_access_key_id"] = info.aws_access_key_id.get_secret_value()
|
||||
kwargs["aws_secret_access_key"] = (
|
||||
info.aws_secret_access_key.get_secret_value()
|
||||
)
|
||||
if info.aws_session_token:
|
||||
kwargs["aws_session_token"] = info.aws_session_token.get_secret_value()
|
||||
Delegates connection-kwargs construction to
|
||||
:func:`wren.connector.athena._build_connect_kwargs` so the legacy
|
||||
``data_source`` path and the native :class:`AthenaConnector` stay in
|
||||
lockstep on credential resolution, ``schema_name`` propagation,
|
||||
``kill_on_interrupt``, and user ``kwargs`` merge semantics.
|
||||
"""
|
||||
from pyathena import connect # noqa: PLC0415
|
||||
|
||||
return ibis.athena.connect(**kwargs)
|
||||
from wren.connector.athena import _build_connect_kwargs # noqa: PLC0415
|
||||
|
||||
return connect(**_build_connect_kwargs(info))
|
||||
|
||||
@staticmethod
|
||||
def get_bigquery_connection(info: BigQueryDatasetConnectionInfo) -> BaseBackend:
|
||||
import ibis # noqa: PLC0415
|
||||
from google.cloud import bigquery # noqa: PLC0415
|
||||
from google.oauth2 import service_account # noqa: PLC0415
|
||||
|
||||
@@ -305,86 +332,243 @@ class DataSourceExtension(Enum):
|
||||
return ibis.bigquery.connect(client=bq_client, credentials=credentials)
|
||||
|
||||
@staticmethod
|
||||
def get_canner_connection(info: CannerConnectionInfo) -> BaseBackend:
|
||||
return ibis.postgres.connect(
|
||||
def get_canner_connection(info: CannerConnectionInfo):
|
||||
import psycopg # noqa: PLC0415
|
||||
|
||||
return psycopg.connect(
|
||||
host=info.host,
|
||||
port=int(info.port),
|
||||
database=info.workspace,
|
||||
dbname=info.workspace,
|
||||
user=info.user,
|
||||
password=info.pat.get_secret_value(),
|
||||
autocommit=True,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def get_clickhouse_connection(info: ClickHouseConnectionInfo) -> BaseBackend:
|
||||
return ibis.clickhouse.connect(
|
||||
host=info.host,
|
||||
port=int(info.port),
|
||||
database=info.database,
|
||||
user=info.user,
|
||||
password=(info.password and info.password.get_secret_value()),
|
||||
settings=info.settings if info.settings else {},
|
||||
**info.kwargs if info.kwargs else {},
|
||||
)
|
||||
def get_clickhouse_connection(info: ClickHouseConnectionInfo):
|
||||
import clickhouse_connect # noqa: PLC0415
|
||||
|
||||
settings = dict(info.settings) if info.settings else {}
|
||||
kwargs = dict(info.kwargs) if info.kwargs else {}
|
||||
statement_timeout = kwargs.pop("statement_timeout", None)
|
||||
if statement_timeout is not None:
|
||||
settings["max_execution_time"] = int(statement_timeout)
|
||||
# Merge any user-supplied ``settings`` from kwargs into the local dict
|
||||
# *before* applying the rest, otherwise ``client_kwargs.update(kwargs)``
|
||||
# below would clobber the statement_timeout-derived max_execution_time.
|
||||
extra_settings = kwargs.pop("settings", None)
|
||||
if extra_settings:
|
||||
settings.update(extra_settings)
|
||||
|
||||
client_kwargs = {
|
||||
"host": info.host,
|
||||
"port": int(info.port),
|
||||
"database": info.database,
|
||||
"username": info.user,
|
||||
"password": info.password.get_secret_value() if info.password else "",
|
||||
"secure": info.secure,
|
||||
"settings": settings,
|
||||
}
|
||||
client_kwargs.update(kwargs)
|
||||
return clickhouse_connect.get_client(**client_kwargs)
|
||||
|
||||
@classmethod
|
||||
def get_mssql_connection(cls, info: MSSqlConnectionInfo) -> BaseBackend:
|
||||
return ibis.mssql.connect(
|
||||
def get_mssql_connection(cls, info: MSSqlConnectionInfo):
|
||||
return cls._connect_mssql_pyodbc(
|
||||
host=info.host,
|
||||
port=info.port,
|
||||
database=info.database,
|
||||
user=info.user,
|
||||
password=info.password.get_secret_value(),
|
||||
password=info.password.get_secret_value() if info.password else None,
|
||||
driver=info.driver,
|
||||
TDS_Version=info.tds_version,
|
||||
**info.kwargs if info.kwargs else {},
|
||||
kwargs={
|
||||
"TDS_Version": info.tds_version,
|
||||
**(info.kwargs if info.kwargs else {}),
|
||||
},
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def get_mysql_connection(cls, info: MySqlConnectionInfo) -> BaseBackend:
|
||||
ssl_context = cls._create_ssl_context(info)
|
||||
kwargs = {"ssl": ssl_context} if ssl_context else {}
|
||||
kwargs.setdefault("charset", "utf8mb4")
|
||||
if info.kwargs:
|
||||
kwargs.update(info.kwargs)
|
||||
return ibis.mysql.connect(
|
||||
host=info.host,
|
||||
port=int(info.port),
|
||||
database=info.database,
|
||||
user=info.user,
|
||||
password=info.password.get_secret_value() if info.password else "",
|
||||
**kwargs,
|
||||
def get_mssql_connection_from_url(
|
||||
cls, connection_url: str, base_kwargs: dict[str, Any] | None = None
|
||||
):
|
||||
parsed = urlparse(connection_url)
|
||||
if parsed.scheme != "mssql":
|
||||
raise WrenError(
|
||||
ErrorCode.INVALID_CONNECTION_INFO,
|
||||
"Invalid connection URL for MSSQL",
|
||||
)
|
||||
|
||||
if not parsed.hostname or not parsed.path or not parsed.username:
|
||||
raise WrenError(
|
||||
ErrorCode.INVALID_CONNECTION_INFO,
|
||||
"MSSQL connection URL must include user, host and database",
|
||||
)
|
||||
|
||||
kwargs = dict(base_kwargs) if base_kwargs else {}
|
||||
# parse_qsl already URL-decodes values, but we re-apply unquote_plus
|
||||
# below only to components urlparse leaves encoded (user, path, password).
|
||||
for key, value in urllib.parse.parse_qsl(parsed.query):
|
||||
kwargs[key] = value
|
||||
driver = kwargs.pop("driver", "ODBC Driver 18 for SQL Server")
|
||||
|
||||
return cls._connect_mssql_pyodbc(
|
||||
host=parsed.hostname,
|
||||
port=str(parsed.port or 1433),
|
||||
database=unquote_plus(parsed.path.lstrip("/")),
|
||||
user=unquote_plus(parsed.username),
|
||||
password=unquote_plus(parsed.password) if parsed.password else None,
|
||||
driver=driver,
|
||||
kwargs=kwargs,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def get_doris_connection(cls, info: DorisConnectionInfo) -> BaseBackend:
|
||||
kwargs: dict = {}
|
||||
kwargs.setdefault("charset", "utf8mb4")
|
||||
if info.kwargs:
|
||||
kwargs.update(info.kwargs)
|
||||
connection = ibis.mysql.connect(
|
||||
host=info.host,
|
||||
port=int(info.port),
|
||||
database=info.database,
|
||||
user=info.user,
|
||||
password=info.password.get_secret_value() if info.password else "",
|
||||
**kwargs,
|
||||
)
|
||||
connection.con.get_autocommit = lambda: True
|
||||
@staticmethod
|
||||
def _connect_mssql_pyodbc(
|
||||
host: str,
|
||||
port: str,
|
||||
database: str,
|
||||
user: str | None,
|
||||
password: str | None,
|
||||
driver: str,
|
||||
kwargs: dict[str, Any] | None = None,
|
||||
):
|
||||
if pyodbc is None: # pragma: no cover
|
||||
raise WrenError(
|
||||
ErrorCode.GET_CONNECTION_ERROR, "pyodbc is required for MSSQL"
|
||||
)
|
||||
|
||||
connect_kwargs = dict(kwargs) if kwargs else {}
|
||||
statement_timeout = connect_kwargs.pop("statement_timeout", None)
|
||||
# Validate statement_timeout before opening the connection so a bad
|
||||
# value can't leak the pyodbc connection we'd otherwise open first.
|
||||
if statement_timeout is not None:
|
||||
try:
|
||||
statement_timeout = int(statement_timeout)
|
||||
except (TypeError, ValueError) as exc:
|
||||
raise WrenError(
|
||||
ErrorCode.INVALID_CONNECTION_INFO,
|
||||
f"Invalid statement_timeout for MSSQL: {statement_timeout!r}",
|
||||
) from exc
|
||||
|
||||
connection_parts = [
|
||||
f"DRIVER={DataSourceExtension._escape_odbc_value(driver)}",
|
||||
f"SERVER={host},{port}",
|
||||
f"DATABASE={DataSourceExtension._escape_odbc_value(database)}",
|
||||
]
|
||||
if user is None and password is None:
|
||||
connection_parts.append("Trusted_Connection=yes")
|
||||
elif user is None or password is None:
|
||||
raise WrenError(
|
||||
ErrorCode.INVALID_CONNECTION_INFO,
|
||||
"MSSQL connection requires both user and password, "
|
||||
"or neither (for Trusted_Connection)",
|
||||
)
|
||||
else:
|
||||
connection_parts.append(
|
||||
f"UID={DataSourceExtension._escape_odbc_value(user)}"
|
||||
)
|
||||
connection_parts.append(
|
||||
f"PWD={DataSourceExtension._escape_odbc_value(password)}"
|
||||
)
|
||||
|
||||
for key, value in connect_kwargs.items():
|
||||
connection_parts.append(
|
||||
f"{key}={DataSourceExtension._escape_odbc_value(str(value))}"
|
||||
)
|
||||
|
||||
connection = pyodbc.connect(";".join(connection_parts))
|
||||
DataSourceExtension._register_mssql_output_converters(connection)
|
||||
|
||||
if statement_timeout is not None:
|
||||
connection.timeout = statement_timeout
|
||||
|
||||
return connection
|
||||
|
||||
@staticmethod
|
||||
def get_postgres_connection(info: PostgresConnectionInfo) -> BaseBackend:
|
||||
return ibis.postgres.connect(
|
||||
def _escape_odbc_value(value: str) -> str:
|
||||
return "{" + value.replace("}", "}}") + "}"
|
||||
|
||||
@staticmethod
|
||||
def _register_mssql_output_converters(connection) -> None:
|
||||
connection.add_output_converter(
|
||||
MSSQL_DATETIMEOFFSET_TYPE_CODE,
|
||||
DataSourceExtension._decode_mssql_datetimeoffset,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _decode_mssql_datetimeoffset(value: bytes | None) -> dtlib.datetime | None:
|
||||
if value is None:
|
||||
return None
|
||||
if len(value) != 20:
|
||||
raise ValueError(
|
||||
"unexpected mssql datetimeoffset payload length: "
|
||||
f"expected 20, got {len(value)}"
|
||||
)
|
||||
|
||||
year = int.from_bytes(value[0:2], "little")
|
||||
month = int.from_bytes(value[2:4], "little")
|
||||
day = int.from_bytes(value[4:6], "little")
|
||||
hour = int.from_bytes(value[6:8], "little")
|
||||
minute = int.from_bytes(value[8:10], "little")
|
||||
second = int.from_bytes(value[10:12], "little")
|
||||
nanoseconds = int.from_bytes(value[12:16], "little")
|
||||
offset_hours = int.from_bytes(value[16:18], "little", signed=True)
|
||||
offset_minutes = int.from_bytes(value[18:20], "little", signed=True)
|
||||
|
||||
tzinfo = dtlib.timezone(
|
||||
dtlib.timedelta(hours=offset_hours, minutes=offset_minutes)
|
||||
)
|
||||
return dtlib.datetime(
|
||||
year=year,
|
||||
month=month,
|
||||
day=day,
|
||||
hour=hour,
|
||||
minute=minute,
|
||||
second=second,
|
||||
microsecond=nanoseconds // 1000,
|
||||
tzinfo=tzinfo,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def get_mysql_connection(cls, info: MySqlConnectionInfo) -> "MySQLdb.Connection":
|
||||
import MySQLdb # noqa: PLC0415
|
||||
|
||||
from wren.connector.mysql import _build_mysql_connect_kwargs # noqa: PLC0415
|
||||
|
||||
return MySQLdb.connect(**_build_mysql_connect_kwargs(info))
|
||||
|
||||
@classmethod
|
||||
def get_doris_connection(cls, info: DorisConnectionInfo) -> "MySQLdb.Connection":
|
||||
import MySQLdb # noqa: PLC0415
|
||||
|
||||
from wren.connector.mysql import _build_doris_connect_kwargs # noqa: PLC0415
|
||||
|
||||
return MySQLdb.connect(**_build_doris_connect_kwargs(info))
|
||||
|
||||
@staticmethod
|
||||
def get_postgres_connection(
|
||||
info: PostgresConnectionInfo,
|
||||
) -> "psycopg.Connection":
|
||||
"""Open a native psycopg3 connection to PostgreSQL.
|
||||
|
||||
Returned object is a ``psycopg.Connection`` — the postgres connector
|
||||
uses raw cursors and an OID-to-Arrow mapping to convert results.
|
||||
"""
|
||||
import psycopg # noqa: PLC0415
|
||||
|
||||
kwargs: dict[str, Any] = dict(info.kwargs) if info.kwargs else {}
|
||||
return psycopg.connect(
|
||||
host=info.host,
|
||||
port=int(info.port),
|
||||
database=info.database,
|
||||
dbname=info.database,
|
||||
user=info.user,
|
||||
password=(info.password and info.password.get_secret_value()),
|
||||
**info.kwargs if info.kwargs else {},
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def get_oracle_connection(info: OracleConnectionInfo) -> BaseBackend:
|
||||
import ibis # noqa: PLC0415
|
||||
|
||||
if hasattr(info, "dsn") and info.dsn:
|
||||
return ibis.oracle.connect(
|
||||
dsn=info.dsn.get_secret_value(),
|
||||
@@ -406,51 +590,41 @@ class DataSourceExtension(Enum):
|
||||
return make_snowflake_connection(info)
|
||||
|
||||
@staticmethod
|
||||
def get_trino_connection(info: TrinoConnectionInfo) -> BaseBackend:
|
||||
return ibis.trino.connect(
|
||||
host=info.host,
|
||||
port=int(info.port),
|
||||
database=info.catalog,
|
||||
schema=info.trino_schema,
|
||||
user=info.user,
|
||||
password=(info.password and info.password.get_secret_value()),
|
||||
**info.kwargs if info.kwargs else {},
|
||||
)
|
||||
def get_trino_connection(info: TrinoConnectionInfo):
|
||||
"""Return a ``trino.dbapi.Connection`` (not an ibis backend).
|
||||
|
||||
The wren SDK calls into ``wren.connector.trino`` for trino, which
|
||||
operates directly on the DB-API cursor. ``get_connection`` only ever
|
||||
re-routes here when something outside the v4 connector code path
|
||||
explicitly requests a trino connection.
|
||||
"""
|
||||
from trino.auth import BasicAuthentication # noqa: PLC0415
|
||||
from trino.dbapi import connect as trino_connect # noqa: PLC0415
|
||||
|
||||
kwargs = dict(info.kwargs) if info.kwargs else {}
|
||||
password = info.password.get_secret_value() if info.password else None
|
||||
|
||||
connect_kwargs: dict = {
|
||||
"host": info.host,
|
||||
"port": int(info.port),
|
||||
"user": info.user,
|
||||
"catalog": info.catalog,
|
||||
"schema": info.trino_schema,
|
||||
# Pin to UTC so timestamp-with-tz casts are deterministic.
|
||||
"timezone": "UTC",
|
||||
}
|
||||
if info.user and password:
|
||||
connect_kwargs["auth"] = BasicAuthentication(info.user, password)
|
||||
connect_kwargs["http_scheme"] = "https"
|
||||
connect_kwargs.update(kwargs)
|
||||
return trino_connect(**connect_kwargs)
|
||||
|
||||
@staticmethod
|
||||
def get_databricks_connection(info: DatabricksTokenConnectionInfo) -> BaseBackend:
|
||||
import ibis # noqa: PLC0415
|
||||
|
||||
return ibis.databricks.connect(
|
||||
server_hostname=info.server_hostname,
|
||||
http_path=info.http_path,
|
||||
access_token=info.access_token.get_secret_value(),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _create_ssl_context(info: ConnectionInfo) -> ssl.SSLContext | None:
|
||||
ssl_mode = (
|
||||
info.ssl_mode if hasattr(info, "ssl_mode") and info.ssl_mode else None
|
||||
)
|
||||
|
||||
if ssl_mode == SSLMode.VERIFY_CA and not info.ssl_ca:
|
||||
raise WrenError(
|
||||
ErrorCode.INVALID_CONNECTION_INFO,
|
||||
"SSL CA must be provided when SSL mode is VERIFY CA",
|
||||
)
|
||||
|
||||
if not ssl_mode or ssl_mode == SSLMode.DISABLED:
|
||||
return None
|
||||
|
||||
ctx = ssl.create_default_context()
|
||||
ctx.check_hostname = False
|
||||
|
||||
if ssl_mode == SSLMode.ENABLED:
|
||||
ctx.verify_mode = ssl.CERT_NONE
|
||||
elif ssl_mode == SSLMode.VERIFY_CA:
|
||||
ctx.verify_mode = ssl.CERT_REQUIRED
|
||||
ctx.load_verify_locations(
|
||||
cadata=base64.b64decode(info.ssl_ca.get_secret_value()).decode("utf-8")
|
||||
if info.ssl_ca
|
||||
else None
|
||||
)
|
||||
|
||||
return ctx
|
||||
|
||||
@@ -19,3 +19,11 @@ def pytest_configure(config: pytest.Config) -> None:
|
||||
config.addinivalue_line(
|
||||
"markers", "snowflake: Snowflake connector tests — mocked, no Docker required"
|
||||
)
|
||||
config.addinivalue_line(
|
||||
"markers", "canner: Canner connector tests — requires Docker"
|
||||
)
|
||||
config.addinivalue_line(
|
||||
"markers", "clickhouse: ClickHouse connector tests — requires Docker"
|
||||
)
|
||||
config.addinivalue_line("markers", "mssql: MSSQL connector tests — requires Docker")
|
||||
config.addinivalue_line("markers", "trino: Trino connector tests — requires Docker")
|
||||
|
||||
@@ -0,0 +1,332 @@
|
||||
"""Canner connector tests.
|
||||
|
||||
Canner Enterprise speaks the Postgres wire protocol, so we point a
|
||||
``PostgresContainer`` at the connector and stand up tables with the data
|
||||
types Canner publishes (Trino-style VARCHAR/DECIMAL/ARRAY/ROW/MAP plus the
|
||||
usual postgres numeric/date/time types) to exercise the native psycopg
|
||||
Arrow builder.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from decimal import Decimal
|
||||
from urllib.parse import urlparse
|
||||
|
||||
import pyarrow as pa
|
||||
import pytest
|
||||
|
||||
from wren.connector.canner import (
|
||||
CannerConnector,
|
||||
_arrow_type,
|
||||
_build_column,
|
||||
_strip_trailing_semicolon,
|
||||
)
|
||||
from wren.model import CannerConnectionInfo
|
||||
|
||||
psycopg = pytest.importorskip("psycopg")
|
||||
testcontainers_postgres = pytest.importorskip("testcontainers.postgres")
|
||||
PostgresContainer = testcontainers_postgres.PostgresContainer
|
||||
|
||||
pytestmark = pytest.mark.canner
|
||||
|
||||
|
||||
_FIXTURE_DDL = """
|
||||
CREATE TABLE canner_demo (
|
||||
id BIGINT PRIMARY KEY,
|
||||
name VARCHAR(64) NOT NULL,
|
||||
flag BOOLEAN,
|
||||
amount DECIMAL(18, 4),
|
||||
ratio DOUBLE PRECISION,
|
||||
small SMALLINT,
|
||||
sample_date DATE,
|
||||
sample_ts TIMESTAMP,
|
||||
sample_tstz TIMESTAMPTZ,
|
||||
tags VARCHAR[],
|
||||
struct_col JSON,
|
||||
map_col JSONB
|
||||
)
|
||||
"""
|
||||
|
||||
_FIXTURE_ROWS = [
|
||||
(
|
||||
1,
|
||||
"alpha",
|
||||
True,
|
||||
Decimal("12.3400"),
|
||||
1.5,
|
||||
7,
|
||||
"2024-01-02",
|
||||
"2024-01-02 03:04:05",
|
||||
"2024-01-02 03:04:05+00",
|
||||
["a", "b"],
|
||||
'{"k": "v"}',
|
||||
'{"m": 1}',
|
||||
),
|
||||
(
|
||||
2,
|
||||
"beta",
|
||||
False,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
# ── helper-level unit tests ───────────────────────────────────────────────
|
||||
|
||||
|
||||
def _column(type_code: int, precision: int | None = None, scale: int | None = None):
|
||||
class _Col:
|
||||
pass
|
||||
|
||||
col = _Col()
|
||||
col.type_code = type_code
|
||||
col.precision = precision
|
||||
col.scale = scale
|
||||
return col
|
||||
|
||||
|
||||
def test_arrow_type_maps_canner_scalars() -> None:
|
||||
# VARCHAR / CHAR / TEXT → string
|
||||
assert _arrow_type(_column(1043)) == pa.string()
|
||||
assert _arrow_type(_column(1042)) == pa.string()
|
||||
assert _arrow_type(_column(25)) == pa.string()
|
||||
# BIGINT / INTEGER / SMALLINT → int
|
||||
assert _arrow_type(_column(20)) == pa.int64()
|
||||
assert _arrow_type(_column(23)) == pa.int32()
|
||||
assert _arrow_type(_column(21)) == pa.int16()
|
||||
# BOOLEAN → bool
|
||||
assert _arrow_type(_column(16)) == pa.bool_()
|
||||
# DOUBLE / REAL → float
|
||||
assert _arrow_type(_column(701)) == pa.float64()
|
||||
assert _arrow_type(_column(700)) == pa.float32()
|
||||
# DATE / TIMESTAMP / TIMESTAMPTZ → date/timestamp
|
||||
assert _arrow_type(_column(1082)) == pa.date32()
|
||||
assert _arrow_type(_column(1114)) == pa.timestamp("us")
|
||||
assert _arrow_type(_column(1184)) == pa.timestamp("us", tz="UTC")
|
||||
# NUMERIC honours precision/scale
|
||||
assert _arrow_type(_column(1700, precision=18, scale=4)) == pa.decimal128(18, 4)
|
||||
# JSON/JSONB (ROW/MAP) → string
|
||||
assert _arrow_type(_column(114)) == pa.string()
|
||||
assert _arrow_type(_column(3802)) == pa.string()
|
||||
# ARRAY → list
|
||||
assert _arrow_type(_column(1009)) == pa.list_(pa.string())
|
||||
|
||||
|
||||
def test_build_column_serialises_complex_values_to_json() -> None:
|
||||
array = _build_column([{"k": "v"}, [1, 2], "raw", None], pa.string(), 114)
|
||||
# SQL NULL must stay Python None — only actual JSON literals are stringified.
|
||||
assert array.to_pylist() == ['{"k": "v"}', "[1, 2]", "raw", None]
|
||||
|
||||
|
||||
def test_build_column_preserves_sql_null_for_jsonb() -> None:
|
||||
# Regression: a SQL NULL in a json (114) / jsonb (3802) column must stay
|
||||
# Python None rather than being coerced to the string "null".
|
||||
for oid in (114, 3802):
|
||||
array = _build_column([None], pa.string(), oid)
|
||||
assert array.to_pylist() == [None]
|
||||
|
||||
|
||||
def test_build_column_quantises_decimal_values() -> None:
|
||||
array = _build_column(
|
||||
[Decimal("12.345678"), None],
|
||||
pa.decimal128(18, 4),
|
||||
)
|
||||
assert array.to_pylist() == [Decimal("12.3457"), None]
|
||||
|
||||
|
||||
def test_arrow_type_for_unconstrained_numeric_falls_back_to_string() -> None:
|
||||
# NUMERIC without typmod (scale is None) must not silently quantise — we
|
||||
# surface it as a string so high-precision values round-trip intact.
|
||||
assert _arrow_type(_column(1700)) == pa.string()
|
||||
# NUMERIC[] inherits the same behaviour for its element type.
|
||||
assert _arrow_type(_column(1231)) == pa.list_(pa.string())
|
||||
|
||||
|
||||
def test_build_column_preserves_unconstrained_numeric_precision() -> None:
|
||||
# Regression: previously NUMERIC without typmod defaulted to scale=9, so
|
||||
# values past the 9th decimal were silently rounded by Decimal.quantize.
|
||||
high_precision = Decimal("12345678901234567890.123456789012345")
|
||||
array = _build_column([high_precision, None], pa.string(), 1700)
|
||||
assert array.to_pylist() == [str(high_precision), None]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("raw", "expected"),
|
||||
[
|
||||
("SELECT 1", "SELECT 1"),
|
||||
("SELECT 1;", "SELECT 1"),
|
||||
("SELECT 1 ; ", "SELECT 1"),
|
||||
("SELECT 1;;", "SELECT 1"),
|
||||
("SELECT 1;\n", "SELECT 1"),
|
||||
# Semicolons inside string literals are *not* terminators — only the
|
||||
# trailing run is stripped.
|
||||
("SELECT 'a;b' FROM t", "SELECT 'a;b' FROM t"),
|
||||
("SELECT 'a;b' FROM t;", "SELECT 'a;b' FROM t"),
|
||||
],
|
||||
)
|
||||
def test_strip_trailing_semicolon(raw: str, expected: str) -> None:
|
||||
assert _strip_trailing_semicolon(raw) == expected
|
||||
|
||||
|
||||
def test_dry_run_returns_none_contract() -> None:
|
||||
# Regression: ConnectorABC.dry_run() must return None. The cursor result
|
||||
# must not leak out of the method, even on the success path.
|
||||
class _FakeCursor:
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *_args) -> None:
|
||||
return None
|
||||
|
||||
def execute(self, _sql: str) -> "_FakeCursor":
|
||||
return self
|
||||
|
||||
class _FakeConnection:
|
||||
def cursor(self) -> _FakeCursor:
|
||||
return _FakeCursor()
|
||||
|
||||
connector = CannerConnector.__new__(CannerConnector)
|
||||
connector.connection = _FakeConnection()
|
||||
connector._closed = False
|
||||
|
||||
assert connector.dry_run("SELECT 1") is None
|
||||
|
||||
|
||||
# ── end-to-end testcontainer test ─────────────────────────────────────────
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def canner_connector():
|
||||
with PostgresContainer("postgres:16") as pg:
|
||||
url = pg.get_connection_url().replace("+psycopg2", "")
|
||||
parsed = urlparse(url)
|
||||
|
||||
with psycopg.connect(url, autocommit=True) as conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(_FIXTURE_DDL)
|
||||
cur.executemany(
|
||||
"""
|
||||
INSERT INTO canner_demo VALUES (
|
||||
%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s
|
||||
)
|
||||
""",
|
||||
_FIXTURE_ROWS,
|
||||
)
|
||||
|
||||
# CannerConnectionInfo treats `workspace` like a postgres database and
|
||||
# `pat` like a password — pass them through directly.
|
||||
connection_info = CannerConnectionInfo(
|
||||
host=parsed.hostname,
|
||||
port=str(parsed.port),
|
||||
user=parsed.username,
|
||||
pat=parsed.password,
|
||||
workspace=parsed.path.lstrip("/"),
|
||||
)
|
||||
connector = CannerConnector(connection_info)
|
||||
try:
|
||||
yield connector
|
||||
finally:
|
||||
connector.close()
|
||||
|
||||
|
||||
def test_canner_connector_query_returns_arrow_table(canner_connector) -> None:
|
||||
table = canner_connector.query("SELECT * FROM canner_demo ORDER BY id")
|
||||
|
||||
assert table.num_rows == 2
|
||||
assert table.schema.field("id").type == pa.int64()
|
||||
assert table.schema.field("name").type == pa.string()
|
||||
assert table.schema.field("flag").type == pa.bool_()
|
||||
assert table.schema.field("amount").type == pa.decimal128(18, 4)
|
||||
assert table.schema.field("ratio").type == pa.float64()
|
||||
assert table.schema.field("small").type == pa.int16()
|
||||
assert table.schema.field("sample_date").type == pa.date32()
|
||||
assert table.schema.field("sample_ts").type == pa.timestamp("us")
|
||||
assert table.schema.field("sample_tstz").type == pa.timestamp("us", tz="UTC")
|
||||
assert table.schema.field("tags").type == pa.list_(pa.string())
|
||||
assert table.schema.field("struct_col").type == pa.string()
|
||||
assert table.schema.field("map_col").type == pa.string()
|
||||
|
||||
rows = table.to_pylist()
|
||||
row = rows[0]
|
||||
assert row["id"] == 1
|
||||
assert row["name"] == "alpha"
|
||||
assert row["flag"] is True
|
||||
assert row["amount"] == Decimal("12.3400")
|
||||
assert row["tags"] == ["a", "b"]
|
||||
# complex types come back as JSON strings
|
||||
assert row["struct_col"] == '{"k": "v"}'
|
||||
assert row["map_col"] == '{"m": 1}'
|
||||
|
||||
# SQL NULL in a JSON/JSONB column stays Python None — it must not be
|
||||
# silently coerced into the string "null".
|
||||
null_row = rows[1]
|
||||
assert null_row["struct_col"] is None
|
||||
assert null_row["map_col"] is None
|
||||
|
||||
|
||||
def test_canner_connector_query_applies_limit(canner_connector) -> None:
|
||||
table = canner_connector.query("SELECT * FROM canner_demo ORDER BY id", limit=1)
|
||||
assert table.num_rows == 1
|
||||
|
||||
|
||||
def test_canner_connector_query_preserves_duplicate_column_names(
|
||||
canner_connector,
|
||||
) -> None:
|
||||
# Regression: dict-based pa.Table construction silently drops duplicate
|
||||
# column names — a self-join projecting both ``id`` columns must keep both.
|
||||
table = canner_connector.query(
|
||||
"SELECT a.id, b.id FROM canner_demo a, canner_demo b ORDER BY a.id, b.id LIMIT 1"
|
||||
)
|
||||
assert table.num_columns == 2
|
||||
assert [field.name for field in table.schema] == ["id", "id"]
|
||||
|
||||
|
||||
def test_canner_connector_dry_run_succeeds(canner_connector) -> None:
|
||||
# Returns None and must not raise on a valid statement.
|
||||
assert canner_connector.dry_run("SELECT 1 AS x") is None
|
||||
|
||||
|
||||
def test_canner_connector_dry_run_raises_for_invalid_sql(canner_connector) -> None:
|
||||
from wren.model.error import WrenError # noqa: PLC0415
|
||||
|
||||
with pytest.raises(WrenError):
|
||||
canner_connector.dry_run("SELECT * FROM no_such_table")
|
||||
|
||||
|
||||
def test_canner_connector_preserves_unconstrained_numeric_precision(
|
||||
canner_connector,
|
||||
) -> None:
|
||||
# Regression: unconstrained NUMERIC must round-trip without silent rounding.
|
||||
# The cursor description reports scale=None for an unconstrained cast, so
|
||||
# the connector falls back to pa.string() to keep the exact textual value.
|
||||
literal = "12345678901234567890.123456789012345"
|
||||
table = canner_connector.query(f"SELECT '{literal}'::numeric AS n")
|
||||
assert table.schema.field("n").type == pa.string()
|
||||
assert table.to_pylist() == [{"n": literal}]
|
||||
|
||||
|
||||
def test_canner_connector_query_wraps_sql_with_trailing_semicolon(
|
||||
canner_connector,
|
||||
) -> None:
|
||||
# Regression: a trailing semicolon on the user SQL must not break the
|
||||
# ``SELECT * FROM (...) AS _t LIMIT N`` wrap that the connector applies
|
||||
# when ``limit`` is provided.
|
||||
table = canner_connector.query("SELECT 1 AS x;", limit=1)
|
||||
assert table.num_rows == 1
|
||||
assert table.to_pylist() == [{"x": 1}]
|
||||
|
||||
|
||||
def test_canner_connector_dry_run_wraps_sql_with_trailing_semicolon(
|
||||
canner_connector,
|
||||
) -> None:
|
||||
# Same regression for dry_run, which always wraps as ``... LIMIT 0``.
|
||||
assert canner_connector.dry_run("SELECT 1 AS x;") is None
|
||||
assert canner_connector.dry_run("SELECT 1 AS x ; ") is None
|
||||
@@ -0,0 +1,291 @@
|
||||
"""ClickHouse connector tests.
|
||||
|
||||
Uses ``testcontainers`` to spin up a real ClickHouse instance. TPCH-shaped
|
||||
fixture data is fabricated inline in Python (no network downloads) and loaded
|
||||
over the native ``clickhouse-connect`` HTTP client.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import datetime as _dt
|
||||
import time
|
||||
|
||||
import orjson
|
||||
import pytest
|
||||
from testcontainers.core.container import DockerContainer
|
||||
|
||||
from tests.suite.manifests import make_tpch_manifest
|
||||
from tests.suite.query import WrenQueryTestSuite
|
||||
from wren import WrenEngine
|
||||
from wren.connector.clickhouse import (
|
||||
_build_clickhouse_client_kwargs,
|
||||
_parse_clickhouse_type,
|
||||
)
|
||||
from wren.model.data_source import DataSource
|
||||
|
||||
pytestmark = pytest.mark.clickhouse
|
||||
|
||||
_SCHEMA = "default"
|
||||
_ORDER_COUNT = 15000
|
||||
_CUSTOMER_COUNT = 1500
|
||||
_ORDER_STATUSES = ("O", "F", "P")
|
||||
_BASE_DATE = _dt.date(1992, 1, 1)
|
||||
|
||||
|
||||
def _make_fixture_rows() -> tuple[list[tuple], list[tuple]]:
|
||||
"""Fabricate TPCH-shaped orders + customer rows without network access.
|
||||
|
||||
Row counts match TPCH sf=0.01 so the shared ``WrenQueryTestSuite``
|
||||
assertions (15000 orders, 1500 customers, first orderkey == 1) hold.
|
||||
"""
|
||||
customers = [(i, f"Customer#{i:09d}") for i in range(1, _CUSTOMER_COUNT + 1)]
|
||||
orders = [
|
||||
(
|
||||
i,
|
||||
((i - 1) % _CUSTOMER_COUNT) + 1,
|
||||
_ORDER_STATUSES[i % len(_ORDER_STATUSES)],
|
||||
float(100 + i),
|
||||
_BASE_DATE + _dt.timedelta(days=i % 3650),
|
||||
)
|
||||
for i in range(1, _ORDER_COUNT + 1)
|
||||
]
|
||||
return orders, customers
|
||||
|
||||
|
||||
class _ClickHouseContainer(DockerContainer):
|
||||
"""Minimal ClickHouse container wrapper — exposes HTTP port 8123."""
|
||||
|
||||
def __init__(self, image: str = "clickhouse/clickhouse-server:24.3-alpine"):
|
||||
super().__init__(image)
|
||||
self.with_exposed_ports(8123, 9000)
|
||||
# Use the default tcp_port_secure-free defaults; no auth.
|
||||
self.with_env("CLICKHOUSE_DB", _SCHEMA)
|
||||
self.with_env("CLICKHOUSE_USER", "default")
|
||||
self.with_env("CLICKHOUSE_PASSWORD", "")
|
||||
# Allow empty password (the default user already has it; this is for
|
||||
# any user clickhouse-connect tries to authenticate as).
|
||||
self.with_env("CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT", "1")
|
||||
|
||||
def start(self): # type: ignore[override]
|
||||
super().start()
|
||||
# The alpine image redirects ClickHouse logs to file, so we cannot rely
|
||||
# on ``wait_for_logs``. ``_wait_for_http_ready`` polls the HTTP endpoint
|
||||
# instead.
|
||||
return self
|
||||
|
||||
def get_host_ip(self) -> str:
|
||||
return self.get_container_host_ip()
|
||||
|
||||
def get_http_port(self) -> int:
|
||||
return int(self.get_exposed_port(8123))
|
||||
|
||||
|
||||
def _wait_for_http_ready(host: str, port: int, timeout: float = 120.0) -> None:
|
||||
"""Poll the ClickHouse HTTP endpoint until it responds to a trivial query."""
|
||||
import clickhouse_connect # noqa: PLC0415
|
||||
|
||||
deadline = time.time() + timeout
|
||||
last_err: Exception | None = None
|
||||
while time.time() < deadline:
|
||||
client = None
|
||||
try:
|
||||
client = clickhouse_connect.get_client(
|
||||
host=host, port=port, username="default", password=""
|
||||
)
|
||||
client.query("SELECT 1")
|
||||
return
|
||||
except Exception as e: # noqa: BLE001
|
||||
last_err = e
|
||||
time.sleep(1)
|
||||
finally:
|
||||
if client is not None:
|
||||
client.close()
|
||||
raise RuntimeError(f"ClickHouse did not become ready: {last_err}")
|
||||
|
||||
|
||||
def _load_tpch(host: str, port: int) -> None:
|
||||
"""Bulk-load fabricated TPCH-shaped data into ClickHouse."""
|
||||
import clickhouse_connect # noqa: PLC0415
|
||||
|
||||
orders_rows, customer_rows = _make_fixture_rows()
|
||||
|
||||
client = clickhouse_connect.get_client(
|
||||
host=host, port=port, username="default", password="", database=_SCHEMA
|
||||
)
|
||||
try:
|
||||
client.command(
|
||||
"CREATE TABLE IF NOT EXISTS orders ("
|
||||
" o_orderkey Int32,"
|
||||
" o_custkey Int32,"
|
||||
" o_orderstatus String,"
|
||||
" o_totalprice Float64,"
|
||||
" o_orderdate Date"
|
||||
") ENGINE = MergeTree ORDER BY o_orderkey"
|
||||
)
|
||||
client.command(
|
||||
"CREATE TABLE IF NOT EXISTS customer ("
|
||||
" c_custkey Int32,"
|
||||
" c_name String"
|
||||
") ENGINE = MergeTree ORDER BY c_custkey"
|
||||
)
|
||||
client.insert(
|
||||
"orders",
|
||||
orders_rows,
|
||||
column_names=[
|
||||
"o_orderkey",
|
||||
"o_custkey",
|
||||
"o_orderstatus",
|
||||
"o_totalprice",
|
||||
"o_orderdate",
|
||||
],
|
||||
)
|
||||
client.insert("customer", customer_rows, column_names=["c_custkey", "c_name"])
|
||||
finally:
|
||||
client.close()
|
||||
|
||||
|
||||
class TestClickHouse(WrenQueryTestSuite):
|
||||
manifest = make_tpch_manifest(table_catalog=None, table_schema=_SCHEMA)
|
||||
# ClickHouse `Int32` round-trips to Arrow as ``int32`` via the native
|
||||
# connector's sqlglot-driven type mapping.
|
||||
order_id_dtype = "int32"
|
||||
|
||||
@pytest.fixture(scope="class")
|
||||
def engine(self) -> WrenEngine: # type: ignore[override]
|
||||
with _ClickHouseContainer() as ch:
|
||||
host = ch.get_host_ip()
|
||||
port = ch.get_http_port()
|
||||
_wait_for_http_ready(host, port)
|
||||
_load_tpch(host, port)
|
||||
|
||||
conn_info = {
|
||||
"host": host,
|
||||
"port": port,
|
||||
"database": _SCHEMA,
|
||||
"user": "default",
|
||||
"password": "",
|
||||
}
|
||||
manifest_str = base64.b64encode(orjson.dumps(self.manifest)).decode()
|
||||
with WrenEngine(
|
||||
manifest_str, DataSource.clickhouse, conn_info, fallback=False
|
||||
) as e:
|
||||
yield e
|
||||
|
||||
|
||||
@pytest.mark.clickhouse
|
||||
class TestClickHouseTypeParser:
|
||||
"""Pure-Python tests for the ClickHouse type-string → Arrow mapping.
|
||||
|
||||
Runs without Docker — exercises ``_parse_clickhouse_type`` directly.
|
||||
"""
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("type_str", "expected"),
|
||||
[
|
||||
("String", "string"),
|
||||
("FixedString(8)", "string"),
|
||||
("Int8", "int8"),
|
||||
("Int16", "int16"),
|
||||
("Int32", "int32"),
|
||||
("Int64", "int64"),
|
||||
("UInt8", "uint8"),
|
||||
("UInt16", "uint16"),
|
||||
("UInt32", "uint32"),
|
||||
("UInt64", "uint64"),
|
||||
("Int128", "string"),
|
||||
("Int256", "string"),
|
||||
("UInt128", "string"),
|
||||
("UInt256", "string"),
|
||||
("Float32", "float"),
|
||||
("Float64", "double"),
|
||||
("Bool", "bool"),
|
||||
("UUID", "string"),
|
||||
("IPv4", "string"),
|
||||
("IPv6", "string"),
|
||||
("Date", "date32[day]"),
|
||||
("Date32", "date32[day]"),
|
||||
("DateTime", "timestamp[ns]"),
|
||||
("Decimal(18, 4)", "decimal128(38, 9)"),
|
||||
("Nullable(Int32)", "int32"),
|
||||
("Nullable(String)", "string"),
|
||||
("LowCardinality(String)", "string"),
|
||||
("LowCardinality(Nullable(String))", "string"),
|
||||
("Array(Int32)", "list<item: int32>"),
|
||||
("Array(Nullable(String))", "list<item: string>"),
|
||||
("Map(String, Int32)", "map<string, int32>"),
|
||||
("Tuple(a Int32, b String)", "string"),
|
||||
("Enum8('a' = 1, 'b' = 2)", "string"),
|
||||
],
|
||||
)
|
||||
def test_type_parse(self, type_str: str, expected: str) -> None:
|
||||
result = _parse_clickhouse_type(type_str)
|
||||
assert str(result) == expected
|
||||
|
||||
def test_datetime64_with_tz(self) -> None:
|
||||
result = _parse_clickhouse_type("DateTime64(3, 'UTC')")
|
||||
assert str(result) == "timestamp[ns, tz=UTC]"
|
||||
|
||||
def test_datetime_with_tz(self) -> None:
|
||||
result = _parse_clickhouse_type("DateTime('Asia/Taipei')")
|
||||
assert str(result) == "timestamp[ns, tz=Asia/Taipei]"
|
||||
|
||||
def test_unknown_type_defaults_to_string(self) -> None:
|
||||
result = _parse_clickhouse_type("SomethingExotic")
|
||||
assert str(result) == "string"
|
||||
|
||||
def test_none_type_defaults_to_string(self) -> None:
|
||||
result = _parse_clickhouse_type(None)
|
||||
assert str(result) == "string"
|
||||
|
||||
|
||||
class _FakeChInfo:
|
||||
"""Stand-in for ``ClickHouseConnectionInfo`` used by the kwargs builder.
|
||||
|
||||
Pydantic enforces ``kwargs: dict[str, str]`` on the real model, so we
|
||||
bypass it here to exercise the merge logic with nested ``settings``.
|
||||
``_build_clickhouse_client_kwargs`` only reads attributes off the object.
|
||||
"""
|
||||
|
||||
def __init__(self, **attrs) -> None:
|
||||
self.host = attrs.get("host", "localhost")
|
||||
self.port = attrs.get("port", "8123")
|
||||
self.database = attrs.get("database", "default")
|
||||
self.user = attrs.get("user", "default")
|
||||
self.password = attrs.get("password")
|
||||
self.secure = attrs.get("secure", False)
|
||||
self.settings = attrs.get("settings")
|
||||
self.kwargs = attrs.get("kwargs")
|
||||
|
||||
|
||||
@pytest.mark.clickhouse
|
||||
class TestClickHouseClientKwargs:
|
||||
"""Pure-Python tests for ``_build_clickhouse_client_kwargs``.
|
||||
|
||||
Exercises the kwargs/settings merge logic without spinning up a real
|
||||
ClickHouse instance.
|
||||
"""
|
||||
|
||||
def test_statement_timeout_survives_user_settings(self) -> None:
|
||||
"""statement_timeout must merge with — not be clobbered by — user settings."""
|
||||
info = _FakeChInfo(
|
||||
kwargs={
|
||||
"statement_timeout": 10,
|
||||
"settings": {"max_threads": 4},
|
||||
},
|
||||
)
|
||||
out = _build_clickhouse_client_kwargs(info)
|
||||
assert out["settings"] == {
|
||||
"max_execution_time": 10,
|
||||
"max_threads": 4,
|
||||
}
|
||||
|
||||
def test_user_settings_only(self) -> None:
|
||||
info = _FakeChInfo(kwargs={"settings": {"max_threads": 4}})
|
||||
out = _build_clickhouse_client_kwargs(info)
|
||||
assert out["settings"] == {"max_threads": 4}
|
||||
|
||||
def test_statement_timeout_only(self) -> None:
|
||||
info = _FakeChInfo(kwargs={"statement_timeout": 10})
|
||||
out = _build_clickhouse_client_kwargs(info)
|
||||
assert out["settings"] == {"max_execution_time": 10}
|
||||
@@ -0,0 +1,305 @@
|
||||
"""MSSQL connector tests.
|
||||
|
||||
Uses testcontainers to spin up a real SQL Server instance.
|
||||
TPCH data is generated via DuckDB's built-in extension and loaded via pyodbc.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import datetime as dtlib
|
||||
from decimal import Decimal as PyDecimal
|
||||
|
||||
import duckdb
|
||||
import orjson
|
||||
import pyarrow as pa
|
||||
import pytest
|
||||
|
||||
pyodbc = pytest.importorskip("pyodbc", reason="pyodbc not installed (mssql extra)")
|
||||
testcontainers_mssql = pytest.importorskip(
|
||||
"testcontainers.mssql", reason="testcontainers[mssql] not installed"
|
||||
)
|
||||
SqlServerContainer = testcontainers_mssql.SqlServerContainer
|
||||
|
||||
from tests.suite.manifests import make_tpch_manifest # noqa: E402
|
||||
from tests.suite.query import WrenQueryTestSuite # noqa: E402
|
||||
from wren import WrenEngine # noqa: E402
|
||||
from wren.connector.mssql import MSSqlConnector # noqa: E402
|
||||
from wren.model import MSSqlConnectionInfo # noqa: E402
|
||||
from wren.model.data_source import DataSource # noqa: E402
|
||||
from wren.model.error import WrenError # noqa: E402
|
||||
|
||||
_SCHEMA = "dbo"
|
||||
_MSSQL_IMAGE = "mcr.microsoft.com/mssql/server:2022-latest"
|
||||
_DRIVER = "ODBC Driver 18 for SQL Server"
|
||||
|
||||
|
||||
def _have_mssql_driver() -> bool:
|
||||
try:
|
||||
return _DRIVER in pyodbc.drivers()
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
pytestmark = [
|
||||
pytest.mark.mssql,
|
||||
pytest.mark.skipif(
|
||||
not _have_mssql_driver(),
|
||||
reason=f"{_DRIVER} not installed",
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
def _pyodbc_connect(container: SqlServerContainer) -> pyodbc.Connection:
|
||||
host = container.get_container_host_ip()
|
||||
port = container.get_exposed_port(container.port)
|
||||
password = container.password
|
||||
user = container.username
|
||||
database = container.dbname
|
||||
conn_str = (
|
||||
f"DRIVER={{{_DRIVER}}};"
|
||||
f"SERVER={host},{port};"
|
||||
f"DATABASE={database};"
|
||||
f"UID={user};PWD={password};"
|
||||
"TrustServerCertificate=yes;"
|
||||
)
|
||||
return pyodbc.connect(conn_str)
|
||||
|
||||
|
||||
def _load_tpch(container: SqlServerContainer) -> None:
|
||||
"""Generate TPCH sf=0.01 via DuckDB and bulk-load into SQL Server."""
|
||||
duck = duckdb.connect()
|
||||
duck.execute("INSTALL tpch; LOAD tpch; CALL dbgen(sf=0.01)")
|
||||
orders_rows = duck.execute(
|
||||
"SELECT o_orderkey, o_custkey, o_orderstatus, "
|
||||
"cast(o_totalprice as double), o_orderdate FROM orders"
|
||||
).fetchall()
|
||||
customer_rows = duck.execute("SELECT c_custkey, c_name FROM customer").fetchall()
|
||||
duck.close()
|
||||
|
||||
conn = _pyodbc_connect(container)
|
||||
conn.autocommit = True
|
||||
with conn:
|
||||
cur = conn.cursor()
|
||||
cur.execute("""
|
||||
CREATE TABLE orders (
|
||||
o_orderkey INT PRIMARY KEY,
|
||||
o_custkey INT NOT NULL,
|
||||
o_orderstatus CHAR(1) NOT NULL,
|
||||
o_totalprice FLOAT NOT NULL,
|
||||
o_orderdate DATE NOT NULL
|
||||
)
|
||||
""")
|
||||
cur.fast_executemany = True
|
||||
cur.executemany(
|
||||
"INSERT INTO orders VALUES (?, ?, ?, ?, ?)",
|
||||
[(k, c, s, float(p), d) for (k, c, s, p, d) in orders_rows],
|
||||
)
|
||||
cur.execute("""
|
||||
CREATE TABLE customer (
|
||||
c_custkey INT PRIMARY KEY,
|
||||
c_name VARCHAR(25) NOT NULL
|
||||
)
|
||||
""")
|
||||
cur.executemany("INSERT INTO customer VALUES (?, ?)", customer_rows)
|
||||
cur.close()
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def mssql_container():
|
||||
with SqlServerContainer(_MSSQL_IMAGE, dialect="mssql+pyodbc") as ms:
|
||||
_load_tpch(ms)
|
||||
yield ms
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def conn_info(mssql_container: SqlServerContainer) -> dict:
|
||||
return {
|
||||
"host": mssql_container.get_container_host_ip(),
|
||||
"port": str(mssql_container.get_exposed_port(mssql_container.port)),
|
||||
"database": mssql_container.dbname,
|
||||
"user": mssql_container.username,
|
||||
"password": mssql_container.password,
|
||||
"driver": _DRIVER,
|
||||
"kwargs": {"TrustServerCertificate": "yes"},
|
||||
}
|
||||
|
||||
|
||||
class TestMSSqlEngine(WrenQueryTestSuite):
|
||||
manifest = make_tpch_manifest(table_catalog=None, table_schema=_SCHEMA)
|
||||
# SQL Server INT → Arrow int32 with our value-sampling inference
|
||||
order_id_dtype = "int32"
|
||||
|
||||
@pytest.fixture(scope="class")
|
||||
def engine(self, conn_info) -> WrenEngine: # type: ignore[override]
|
||||
manifest_str = base64.b64encode(orjson.dumps(self.manifest)).decode()
|
||||
with WrenEngine(manifest_str, DataSource.mssql, conn_info, fallback=False) as e:
|
||||
yield e
|
||||
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# Type-specific tests (no MDL — directly via the connector)
|
||||
# ----------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def types_table(mssql_container: SqlServerContainer) -> str:
|
||||
conn = _pyodbc_connect(mssql_container)
|
||||
conn.autocommit = True
|
||||
table = "mssql_types"
|
||||
with conn:
|
||||
cur = conn.cursor()
|
||||
cur.execute(f"""
|
||||
CREATE TABLE {table} (
|
||||
c_int INT,
|
||||
c_smallint SMALLINT,
|
||||
c_bigint BIGINT,
|
||||
c_tinyint TINYINT,
|
||||
c_bit BIT,
|
||||
c_varchar VARCHAR(50),
|
||||
c_decimal DECIMAL(18,4),
|
||||
c_datetime DATETIME,
|
||||
c_datetime2 DATETIME2,
|
||||
c_dto DATETIMEOFFSET,
|
||||
c_dto_utc DATETIMEOFFSET,
|
||||
c_uuid UNIQUEIDENTIFIER,
|
||||
c_varbinary VARBINARY(16)
|
||||
)
|
||||
""")
|
||||
cur.execute(
|
||||
f"INSERT INTO {table} VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
|
||||
(
|
||||
42,
|
||||
7,
|
||||
10_000_000_000,
|
||||
255,
|
||||
1,
|
||||
"hello",
|
||||
PyDecimal("123.4500"),
|
||||
dtlib.datetime(2024, 1, 2, 3, 4, 5),
|
||||
dtlib.datetime(2024, 1, 2, 3, 4, 5, 678900),
|
||||
"2024-06-15 12:00:00 +05:30",
|
||||
"2024-06-15 12:00:00 +00:00",
|
||||
"00000000-0000-0000-0000-000000000001",
|
||||
b"\x01\x02\x03",
|
||||
),
|
||||
)
|
||||
cur.close()
|
||||
return table
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def connector(conn_info) -> MSSqlConnector:
|
||||
info = MSSqlConnectionInfo.model_validate(conn_info)
|
||||
c = MSSqlConnector(info)
|
||||
try:
|
||||
yield c
|
||||
finally:
|
||||
c.close()
|
||||
|
||||
|
||||
def test_int_types(connector: MSSqlConnector, types_table: str) -> None:
|
||||
result = connector.query(
|
||||
f"SELECT c_int, c_smallint, c_bigint, c_tinyint FROM {types_table}"
|
||||
)
|
||||
assert str(result.schema.field("c_int").type) == "int32"
|
||||
assert str(result.schema.field("c_smallint").type) == "int16"
|
||||
assert str(result.schema.field("c_bigint").type) == "int64"
|
||||
# TINYINT is unsigned in SQL Server; sampled non-negative
|
||||
assert str(result.schema.field("c_tinyint").type) == "uint8"
|
||||
assert result["c_int"][0].as_py() == 42
|
||||
assert result["c_tinyint"][0].as_py() == 255
|
||||
|
||||
|
||||
def test_bit_and_varchar(connector: MSSqlConnector, types_table: str) -> None:
|
||||
result = connector.query(f"SELECT c_bit, c_varchar FROM {types_table}")
|
||||
assert result.schema.field("c_bit").type == pa.bool_()
|
||||
assert result["c_bit"][0].as_py() is True
|
||||
assert result["c_varchar"][0].as_py() == "hello"
|
||||
|
||||
|
||||
def test_decimal_as_string(connector: MSSqlConnector, types_table: str) -> None:
|
||||
result = connector.query(f"SELECT c_decimal FROM {types_table}")
|
||||
# Decimals serialise as strings to avoid arrow decimal precision pitfalls
|
||||
assert str(result.schema.field("c_decimal").type) == "string"
|
||||
assert result["c_decimal"][0].as_py() == "123.4500"
|
||||
|
||||
|
||||
def test_datetime_columns(connector: MSSqlConnector, types_table: str) -> None:
|
||||
result = connector.query(f"SELECT c_datetime, c_datetime2 FROM {types_table}")
|
||||
assert str(result.schema.field("c_datetime").type) == "timestamp[ns]"
|
||||
assert str(result.schema.field("c_datetime2").type) == "timestamp[ns]"
|
||||
|
||||
|
||||
def test_datetimeoffset_non_utc(connector: MSSqlConnector, types_table: str) -> None:
|
||||
result = connector.query(f"SELECT c_dto FROM {types_table}")
|
||||
assert str(result.schema.field("c_dto").type) == "timestamp[ns, tz=+05:30]"
|
||||
value = result["c_dto"][0].as_py()
|
||||
assert isinstance(value, dtlib.datetime)
|
||||
assert value.utcoffset() == dtlib.timedelta(hours=5, minutes=30)
|
||||
|
||||
|
||||
def test_datetimeoffset_utc(connector: MSSqlConnector, types_table: str) -> None:
|
||||
result = connector.query(f"SELECT c_dto_utc FROM {types_table}")
|
||||
assert str(result.schema.field("c_dto_utc").type) == "timestamp[ns, tz=UTC]"
|
||||
value = result["c_dto_utc"][0].as_py()
|
||||
assert value.utcoffset() == dtlib.timedelta(0)
|
||||
|
||||
|
||||
def test_uuid_as_string(connector: MSSqlConnector, types_table: str) -> None:
|
||||
result = connector.query(f"SELECT c_uuid FROM {types_table}")
|
||||
assert str(result.schema.field("c_uuid").type) == "string"
|
||||
# SQL Server returns uppercase UUIDs
|
||||
assert result["c_uuid"][0].as_py().lower() == (
|
||||
"00000000-0000-0000-0000-000000000001"
|
||||
)
|
||||
|
||||
|
||||
def test_varbinary_as_binary(connector: MSSqlConnector, types_table: str) -> None:
|
||||
result = connector.query(f"SELECT c_varbinary FROM {types_table}")
|
||||
assert str(result.schema.field("c_varbinary").type) == "binary"
|
||||
assert result["c_varbinary"][0].as_py() == b"\x01\x02\x03"
|
||||
|
||||
|
||||
def test_dry_run_invalid_column_returns_describe_error(
|
||||
connector: MSSqlConnector, types_table: str
|
||||
) -> None:
|
||||
with pytest.raises(WrenError) as exc:
|
||||
connector.dry_run(f"SELECT not_a_column FROM {types_table}")
|
||||
assert "dry run failed" in str(exc.value).lower()
|
||||
|
||||
|
||||
def test_raw_cursor_sql_injects_fetch_next() -> None:
|
||||
rewritten = MSSqlConnector._raw_cursor_sql("SELECT * FROM orders", 10)
|
||||
# sqlglot emits OFFSET 0 ROWS FETCH NEXT n ROWS ONLY for tsql with LIMIT
|
||||
lower = rewritten.lower()
|
||||
assert "fetch next 10 rows only" in lower
|
||||
assert "offset 0 rows" in lower
|
||||
|
||||
|
||||
def test_raw_cursor_sql_no_limit_passthrough() -> None:
|
||||
sql = "SELECT * FROM orders"
|
||||
assert MSSqlConnector._raw_cursor_sql(sql, None) == sql
|
||||
|
||||
|
||||
def test_url_connection(mssql_container: SqlServerContainer) -> None:
|
||||
host = mssql_container.get_container_host_ip()
|
||||
port = mssql_container.get_exposed_port(mssql_container.port)
|
||||
password = mssql_container.password
|
||||
user = mssql_container.username
|
||||
database = mssql_container.dbname
|
||||
|
||||
url = (
|
||||
f"mssql://{user}:{password}@{host}:{port}/{database}?TrustServerCertificate=yes"
|
||||
)
|
||||
info = {"connectionUrl": url}
|
||||
|
||||
parsed = DataSource.mssql.get_connection_info(info)
|
||||
conn = DataSource.mssql.get_connection(parsed)
|
||||
try:
|
||||
cur = conn.cursor()
|
||||
cur.execute("SELECT 1 AS x")
|
||||
assert cur.fetchone()[0] == 1
|
||||
cur.close()
|
||||
finally:
|
||||
conn.close()
|
||||
@@ -75,7 +75,7 @@ class TestMySQL(WrenQueryTestSuite):
|
||||
|
||||
@pytest.fixture(scope="class")
|
||||
def engine(self) -> WrenEngine: # type: ignore[override]
|
||||
with MySqlContainer("mysql:8") as mysql:
|
||||
with MySqlContainer("mysql:8.0.36") as mysql:
|
||||
url = mysql.get_connection_url()
|
||||
_load_tpch(url)
|
||||
|
||||
|
||||
@@ -0,0 +1,335 @@
|
||||
"""MySQL native connector type-coverage tests.
|
||||
|
||||
Spins up a real MySQL via testcontainers and exercises every field-type
|
||||
to Arrow conversion path that the native ``MySqlConnector`` supports.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from contextlib import closing
|
||||
from decimal import Decimal
|
||||
|
||||
import pyarrow as pa
|
||||
import pytest
|
||||
from testcontainers.mysql import MySqlContainer
|
||||
|
||||
from wren.connector.mysql import MySqlConnector
|
||||
from wren.model import MySqlConnectionInfo
|
||||
|
||||
pytestmark = pytest.mark.mysql
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def mysql_container():
|
||||
with MySqlContainer("mysql:8.0.36") as mysql:
|
||||
yield mysql
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def connector(mysql_container):
|
||||
info = MySqlConnectionInfo(
|
||||
host=mysql_container.get_container_host_ip(),
|
||||
port=mysql_container.get_exposed_port(3306),
|
||||
database=mysql_container.dbname,
|
||||
user=mysql_container.username,
|
||||
password=mysql_container.password,
|
||||
sslMode="disabled",
|
||||
)
|
||||
c = MySqlConnector(info)
|
||||
try:
|
||||
yield c
|
||||
finally:
|
||||
c.close()
|
||||
|
||||
|
||||
def _exec(connector: MySqlConnector, sql: str) -> None:
|
||||
with closing(connector.connection.cursor()) as cur:
|
||||
cur.execute(sql)
|
||||
|
||||
|
||||
def test_tinyint_signed_and_unsigned(connector: MySqlConnector) -> None:
|
||||
_exec(connector, "DROP TABLE IF EXISTS t_tiny")
|
||||
_exec(
|
||||
connector,
|
||||
"CREATE TABLE t_tiny (a TINYINT, b TINYINT UNSIGNED)",
|
||||
)
|
||||
_exec(connector, "INSERT INTO t_tiny VALUES (-1, 200)")
|
||||
tbl = connector.query("SELECT a, b FROM t_tiny")
|
||||
assert tbl.schema.field("a").type == pa.int8()
|
||||
assert tbl.schema.field("b").type == pa.uint8()
|
||||
assert tbl.column("a").to_pylist() == [-1]
|
||||
assert tbl.column("b").to_pylist() == [200]
|
||||
|
||||
|
||||
def test_smallint(connector: MySqlConnector) -> None:
|
||||
_exec(connector, "DROP TABLE IF EXISTS t_small")
|
||||
_exec(connector, "CREATE TABLE t_small (a SMALLINT, b SMALLINT UNSIGNED)")
|
||||
_exec(connector, "INSERT INTO t_small VALUES (-32000, 65000)")
|
||||
tbl = connector.query("SELECT a, b FROM t_small")
|
||||
assert tbl.schema.field("a").type == pa.int16()
|
||||
assert tbl.schema.field("b").type == pa.uint16()
|
||||
|
||||
|
||||
def test_int_and_bigint(connector: MySqlConnector) -> None:
|
||||
_exec(connector, "DROP TABLE IF EXISTS t_int")
|
||||
_exec(
|
||||
connector,
|
||||
"CREATE TABLE t_int (a INT, b INT UNSIGNED, c BIGINT, d BIGINT UNSIGNED)",
|
||||
)
|
||||
_exec(
|
||||
connector, "INSERT INTO t_int VALUES (-1, 4000000000, -1, 18000000000000000000)"
|
||||
)
|
||||
tbl = connector.query("SELECT a, b, c, d FROM t_int")
|
||||
assert tbl.schema.field("a").type == pa.int32()
|
||||
assert tbl.schema.field("b").type == pa.uint32()
|
||||
assert tbl.schema.field("c").type == pa.int64()
|
||||
assert tbl.schema.field("d").type == pa.uint64()
|
||||
|
||||
|
||||
def test_decimal(connector: MySqlConnector) -> None:
|
||||
_exec(connector, "DROP TABLE IF EXISTS t_dec")
|
||||
_exec(connector, "CREATE TABLE t_dec (a DECIMAL(12, 4))")
|
||||
_exec(connector, "INSERT INTO t_dec VALUES (1234.5678)")
|
||||
tbl = connector.query("SELECT a FROM t_dec")
|
||||
arrow_type = tbl.schema.field("a").type
|
||||
assert pa.types.is_decimal(arrow_type)
|
||||
# Precision/scale now reflect the column definition instead of the previous
|
||||
# hard-coded ``decimal128(38, 9)``.
|
||||
assert arrow_type.precision == 12
|
||||
assert arrow_type.scale == 4
|
||||
assert tbl.column("a").to_pylist()[0] == Decimal("1234.5678")
|
||||
|
||||
|
||||
def test_decimal_large_scale(connector: MySqlConnector) -> None:
|
||||
"""DECIMAL with scale > 9 must round-trip without truncating digits.
|
||||
|
||||
The previous hard-coded ``pa.decimal128(38, 9)`` silently dropped digits
|
||||
beyond the 9th decimal place. We now derive precision/scale from
|
||||
``cursor.description``.
|
||||
"""
|
||||
_exec(connector, "DROP TABLE IF EXISTS t_dec_big")
|
||||
_exec(connector, "CREATE TABLE t_dec_big (a DECIMAL(30, 15))")
|
||||
_exec(connector, "INSERT INTO t_dec_big VALUES (12345.123456789012345)")
|
||||
tbl = connector.query("SELECT a FROM t_dec_big")
|
||||
arrow_type = tbl.schema.field("a").type
|
||||
assert pa.types.is_decimal(arrow_type)
|
||||
assert arrow_type.precision == 30
|
||||
assert arrow_type.scale == 15
|
||||
assert tbl.column("a").to_pylist()[0] == Decimal("12345.123456789012345")
|
||||
|
||||
|
||||
def test_float_and_double(connector: MySqlConnector) -> None:
|
||||
_exec(connector, "DROP TABLE IF EXISTS t_real")
|
||||
_exec(connector, "CREATE TABLE t_real (a FLOAT, b DOUBLE)")
|
||||
_exec(connector, "INSERT INTO t_real VALUES (1.5, 2.5)")
|
||||
tbl = connector.query("SELECT a, b FROM t_real")
|
||||
assert tbl.schema.field("a").type == pa.float32()
|
||||
assert tbl.schema.field("b").type == pa.float64()
|
||||
|
||||
|
||||
def test_char_varchar_text(connector: MySqlConnector) -> None:
|
||||
_exec(connector, "DROP TABLE IF EXISTS t_str")
|
||||
_exec(connector, "CREATE TABLE t_str (a CHAR(8), b VARCHAR(32), c TEXT)")
|
||||
_exec(connector, "INSERT INTO t_str VALUES ('abc', 'hello', 'world')")
|
||||
tbl = connector.query("SELECT a, b, c FROM t_str")
|
||||
for col in ("a", "b", "c"):
|
||||
assert tbl.schema.field(col).type == pa.string()
|
||||
assert tbl.column("b").to_pylist() == ["hello"]
|
||||
assert tbl.column("c").to_pylist() == ["world"]
|
||||
|
||||
|
||||
def test_json(connector: MySqlConnector) -> None:
|
||||
_exec(connector, "DROP TABLE IF EXISTS t_json")
|
||||
_exec(connector, "CREATE TABLE t_json (a JSON)")
|
||||
_exec(connector, """INSERT INTO t_json VALUES ('{"k": 1}')""")
|
||||
tbl = connector.query("SELECT a FROM t_json")
|
||||
assert tbl.schema.field("a").type == pa.string()
|
||||
assert "k" in tbl.column("a").to_pylist()[0]
|
||||
|
||||
|
||||
def test_blob_binary(connector: MySqlConnector) -> None:
|
||||
_exec(connector, "DROP TABLE IF EXISTS t_blob")
|
||||
_exec(connector, "CREATE TABLE t_blob (a BLOB, b VARBINARY(16))")
|
||||
_exec(connector, "INSERT INTO t_blob VALUES (X'DEADBEEF', X'ABCD')")
|
||||
tbl = connector.query("SELECT a, b FROM t_blob")
|
||||
assert pa.types.is_binary(tbl.schema.field("a").type)
|
||||
assert pa.types.is_binary(tbl.schema.field("b").type)
|
||||
assert tbl.column("a").to_pylist()[0] == b"\xde\xad\xbe\xef"
|
||||
|
||||
|
||||
def test_datetime_and_timestamp(connector: MySqlConnector) -> None:
|
||||
_exec(connector, "DROP TABLE IF EXISTS t_dt")
|
||||
_exec(connector, "CREATE TABLE t_dt (a DATETIME, b TIMESTAMP NULL)")
|
||||
_exec(
|
||||
connector,
|
||||
"INSERT INTO t_dt VALUES ('2024-01-02 03:04:05', '2024-01-02 03:04:05')",
|
||||
)
|
||||
tbl = connector.query("SELECT a, b FROM t_dt")
|
||||
assert pa.types.is_timestamp(tbl.schema.field("a").type)
|
||||
assert pa.types.is_timestamp(tbl.schema.field("b").type)
|
||||
|
||||
|
||||
def test_date(connector: MySqlConnector) -> None:
|
||||
_exec(connector, "DROP TABLE IF EXISTS t_date")
|
||||
_exec(connector, "CREATE TABLE t_date (a DATE)")
|
||||
_exec(connector, "INSERT INTO t_date VALUES ('2024-05-14')")
|
||||
tbl = connector.query("SELECT a FROM t_date")
|
||||
assert tbl.schema.field("a").type == pa.date32()
|
||||
|
||||
|
||||
def test_time(connector: MySqlConnector) -> None:
|
||||
_exec(connector, "DROP TABLE IF EXISTS t_time")
|
||||
_exec(connector, "CREATE TABLE t_time (a TIME)")
|
||||
_exec(connector, "INSERT INTO t_time VALUES ('12:34:56')")
|
||||
tbl = connector.query("SELECT a FROM t_time")
|
||||
# MySQL TIME maps to Arrow ``duration("us")``: ``time64`` cannot represent
|
||||
# negative or >24h values that MySQL ``TIME`` permits.
|
||||
assert pa.types.is_duration(tbl.schema.field("a").type)
|
||||
|
||||
|
||||
def test_time_full_range(connector: MySqlConnector) -> None:
|
||||
"""MySQL ``TIME`` ranges ``-838:59:59`` to ``838:59:59``.
|
||||
|
||||
The previous mapping to ``pa.time64("us")`` silently corrupted negative
|
||||
values and values past 24h (``time64`` only accepts 0–24h positive). Map
|
||||
to ``duration("us")`` instead so the full MySQL range round-trips.
|
||||
"""
|
||||
import datetime # noqa: PLC0415
|
||||
|
||||
_exec(connector, "DROP TABLE IF EXISTS t_time_range")
|
||||
_exec(connector, "CREATE TABLE t_time_range (label VARCHAR(16), a TIME)")
|
||||
_exec(
|
||||
connector,
|
||||
"INSERT INTO t_time_range VALUES "
|
||||
"('neg_100', '-100:00:00'), "
|
||||
"('zero', '0:00:00'), "
|
||||
"('max', '838:59:59'), "
|
||||
"('min', '-838:59:59')",
|
||||
)
|
||||
tbl = connector.query("SELECT label, a FROM t_time_range ORDER BY label")
|
||||
arrow_type = tbl.schema.field("a").type
|
||||
assert pa.types.is_duration(arrow_type)
|
||||
|
||||
by_label = dict(
|
||||
zip(tbl.column("label").to_pylist(), tbl.column("a").to_pylist(), strict=True)
|
||||
)
|
||||
# ``duration("us")`` round-trips to ``datetime.timedelta`` in PyArrow.
|
||||
assert by_label["neg_100"] == datetime.timedelta(hours=-100)
|
||||
assert by_label["zero"] == datetime.timedelta(0)
|
||||
assert by_label["max"] == datetime.timedelta(hours=838, minutes=59, seconds=59)
|
||||
assert by_label["min"] == -datetime.timedelta(hours=838, minutes=59, seconds=59)
|
||||
|
||||
|
||||
def test_year(connector: MySqlConnector) -> None:
|
||||
_exec(connector, "DROP TABLE IF EXISTS t_year")
|
||||
_exec(connector, "CREATE TABLE t_year (a YEAR)")
|
||||
_exec(connector, "INSERT INTO t_year VALUES (2024)")
|
||||
tbl = connector.query("SELECT a FROM t_year")
|
||||
assert tbl.schema.field("a").type == pa.int16()
|
||||
assert tbl.column("a").to_pylist() == [2024]
|
||||
|
||||
|
||||
def test_bit(connector: MySqlConnector) -> None:
|
||||
_exec(connector, "DROP TABLE IF EXISTS t_bit")
|
||||
_exec(connector, "CREATE TABLE t_bit (a BIT(8))")
|
||||
_exec(connector, "INSERT INTO t_bit VALUES (b'10101010')")
|
||||
tbl = connector.query("SELECT a FROM t_bit")
|
||||
assert pa.types.is_binary(tbl.schema.field("a").type)
|
||||
|
||||
|
||||
def test_enum_and_set(connector: MySqlConnector) -> None:
|
||||
_exec(connector, "DROP TABLE IF EXISTS t_enum")
|
||||
_exec(
|
||||
connector,
|
||||
"CREATE TABLE t_enum (a ENUM('x', 'y', 'z'), b SET('p', 'q', 'r'))",
|
||||
)
|
||||
_exec(connector, "INSERT INTO t_enum VALUES ('y', 'p,r')")
|
||||
tbl = connector.query("SELECT a, b FROM t_enum")
|
||||
assert tbl.schema.field("a").type == pa.string()
|
||||
assert tbl.schema.field("b").type == pa.string()
|
||||
assert tbl.column("a").to_pylist() == ["y"]
|
||||
|
||||
|
||||
def test_null_handling(connector: MySqlConnector) -> None:
|
||||
_exec(connector, "DROP TABLE IF EXISTS t_null")
|
||||
_exec(connector, "CREATE TABLE t_null (a INT, b VARCHAR(8))")
|
||||
_exec(connector, "INSERT INTO t_null VALUES (NULL, NULL)")
|
||||
tbl = connector.query("SELECT a, b FROM t_null")
|
||||
assert tbl.column("a").to_pylist() == [None]
|
||||
assert tbl.column("b").to_pylist() == [None]
|
||||
|
||||
|
||||
def test_query_with_limit(connector: MySqlConnector) -> None:
|
||||
_exec(connector, "DROP TABLE IF EXISTS t_limit")
|
||||
_exec(connector, "CREATE TABLE t_limit (a INT)")
|
||||
_exec(connector, "INSERT INTO t_limit VALUES (1), (2), (3), (4), (5)")
|
||||
tbl = connector.query("SELECT a FROM t_limit", limit=2)
|
||||
assert tbl.num_rows == 2
|
||||
|
||||
|
||||
def test_query_limit_rejects_sql_injection(connector: MySqlConnector) -> None:
|
||||
"""``limit`` must be coerced via ``int()`` so it can be safely interpolated.
|
||||
|
||||
Passing a crafted string would previously land directly in the rendered
|
||||
SQL via an f-string. ``int()`` rejects it with ``ValueError``.
|
||||
"""
|
||||
_exec(connector, "DROP TABLE IF EXISTS t_inj")
|
||||
_exec(connector, "CREATE TABLE t_inj (a INT)")
|
||||
_exec(connector, "INSERT INTO t_inj VALUES (1), (2)")
|
||||
with pytest.raises((ValueError, TypeError)):
|
||||
connector.query("SELECT a FROM t_inj", limit="1; DROP TABLE t_inj")
|
||||
# Table must still exist — the malicious payload never reached the server.
|
||||
tbl = connector.query("SELECT a FROM t_inj")
|
||||
assert tbl.num_rows == 2
|
||||
|
||||
|
||||
def test_query_with_duplicate_column_names(connector: MySqlConnector) -> None:
|
||||
"""``SELECT * FROM (...) AS _sub`` would fail with ER_DUP_FIELDNAME on a
|
||||
join that exposes the same column name twice. Appending ``LIMIT`` to the
|
||||
user SQL avoids the subquery and so avoids the duplicate-column error.
|
||||
|
||||
Also asserts the resulting Arrow table preserves BOTH ``id`` fields —
|
||||
building the table via ``dict(zip(names, arrays))`` would silently drop
|
||||
one of the duplicate columns because the dict collapses the key.
|
||||
"""
|
||||
_exec(connector, "DROP TABLE IF EXISTS t_dup_a")
|
||||
_exec(connector, "DROP TABLE IF EXISTS t_dup_b")
|
||||
_exec(connector, "CREATE TABLE t_dup_a (id INT, val INT)")
|
||||
_exec(connector, "CREATE TABLE t_dup_b (id INT, val INT)")
|
||||
_exec(connector, "INSERT INTO t_dup_a VALUES (1, 10), (2, 20)")
|
||||
_exec(connector, "INSERT INTO t_dup_b VALUES (1, 100), (2, 200)")
|
||||
sql = "SELECT a.id, b.id FROM t_dup_a a JOIN t_dup_b b ON a.id = b.id"
|
||||
tbl = connector.query(sql, limit=10)
|
||||
assert tbl.num_rows == 2
|
||||
# Two ``id`` columns must survive — the schema is name-positional.
|
||||
assert tbl.num_columns == 2
|
||||
assert [f.name for f in tbl.schema] == ["id", "id"]
|
||||
# Both columns hold the same data (joined on ``id``), but they must each
|
||||
# exist independently in the result.
|
||||
assert tbl.column(0).to_pylist() == tbl.column(1).to_pylist()
|
||||
# dry_run should also work on duplicate-column queries.
|
||||
connector.dry_run(sql)
|
||||
|
||||
|
||||
def test_query_trailing_semicolon(connector: MySqlConnector) -> None:
|
||||
"""Trailing semicolons must be stripped before appending ``LIMIT``."""
|
||||
_exec(connector, "DROP TABLE IF EXISTS t_semi")
|
||||
_exec(connector, "CREATE TABLE t_semi (a INT)")
|
||||
_exec(connector, "INSERT INTO t_semi VALUES (1), (2), (3)")
|
||||
tbl = connector.query("SELECT a FROM t_semi;", limit=2)
|
||||
assert tbl.num_rows == 2
|
||||
|
||||
|
||||
def test_dry_run(connector: MySqlConnector) -> None:
|
||||
_exec(connector, "DROP TABLE IF EXISTS t_dry")
|
||||
_exec(connector, "CREATE TABLE t_dry (a INT)")
|
||||
connector.dry_run("SELECT a FROM t_dry") # must not raise
|
||||
|
||||
|
||||
def test_empty_result(connector: MySqlConnector) -> None:
|
||||
_exec(connector, "DROP TABLE IF EXISTS t_empty")
|
||||
_exec(connector, "CREATE TABLE t_empty (a INT, b VARCHAR(8))")
|
||||
tbl = connector.query("SELECT a, b FROM t_empty")
|
||||
assert tbl.num_rows == 0
|
||||
assert tbl.schema.field("a").type == pa.int32()
|
||||
assert tbl.schema.field("b").type == pa.string()
|
||||
@@ -7,17 +7,20 @@ TPCH data is generated via DuckDB's built-in extension and loaded via psycopg.
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
from decimal import Decimal
|
||||
from urllib.parse import urlparse
|
||||
|
||||
import duckdb
|
||||
import orjson
|
||||
import psycopg
|
||||
import pyarrow as pa
|
||||
import pytest
|
||||
from testcontainers.postgres import PostgresContainer
|
||||
|
||||
from tests.suite.manifests import make_tpch_manifest
|
||||
from tests.suite.query import WrenQueryTestSuite
|
||||
from wren import WrenEngine
|
||||
from wren.connector.postgres import PostgresConnector
|
||||
from wren.model.data_source import DataSource
|
||||
|
||||
pytestmark = pytest.mark.postgres
|
||||
@@ -85,3 +88,165 @@ class TestPostgres(WrenQueryTestSuite):
|
||||
manifest_str, DataSource.postgres, conn_info, fallback=False
|
||||
) as e:
|
||||
yield e
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Direct PostgresConnector type-coverage tests (no MDL / engine layer)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _build_type_table(conn_str: str) -> None:
|
||||
with psycopg.connect(conn_str) as pg:
|
||||
with pg.cursor() as cur:
|
||||
cur.execute("""
|
||||
CREATE TABLE type_zoo (
|
||||
c_int4 INTEGER,
|
||||
c_int8 BIGINT,
|
||||
c_numeric NUMERIC(38, 9),
|
||||
c_text TEXT,
|
||||
c_bool BOOLEAN,
|
||||
c_bytea BYTEA,
|
||||
c_uuid UUID,
|
||||
c_jsonb JSONB,
|
||||
c_ts TIMESTAMP,
|
||||
c_tstz TIMESTAMPTZ,
|
||||
c_int4_arr INTEGER[],
|
||||
c_text_arr TEXT[],
|
||||
c_numeric_arr NUMERIC(38, 9)[]
|
||||
)
|
||||
""")
|
||||
cur.execute(
|
||||
"""
|
||||
INSERT INTO type_zoo VALUES (
|
||||
%s, %s, %s, %s, %s, %s, %s, %s::jsonb,
|
||||
%s::timestamp, %s::timestamptz,
|
||||
%s::int[], %s::text[], %s::numeric[]
|
||||
)
|
||||
""",
|
||||
(
|
||||
42,
|
||||
9_000_000_000,
|
||||
Decimal("12345.123456789"),
|
||||
"hello",
|
||||
True,
|
||||
b"\x01\x02\x03",
|
||||
"00000000-0000-0000-0000-000000000001",
|
||||
'{"a": 1, "b": "two"}',
|
||||
"2024-01-02 03:04:05",
|
||||
"2024-01-02 03:04:05+00",
|
||||
[1, 2, 3],
|
||||
["a", "b", "c"],
|
||||
[Decimal("1.5"), Decimal("2.25")],
|
||||
),
|
||||
)
|
||||
cur.execute("INSERT INTO type_zoo (c_int4) VALUES (NULL)")
|
||||
pg.commit()
|
||||
|
||||
|
||||
class TestPostgresConnectorTypes:
|
||||
"""End-to-end type coverage for the native PostgresConnector."""
|
||||
|
||||
@pytest.fixture(scope="class")
|
||||
def connector(self):
|
||||
with PostgresContainer("postgres:16") as pg:
|
||||
url = pg.get_connection_url().replace("+psycopg2", "")
|
||||
_build_type_table(url)
|
||||
|
||||
parsed = urlparse(url)
|
||||
raw_info = {
|
||||
"host": parsed.hostname,
|
||||
"port": parsed.port,
|
||||
"database": parsed.path.lstrip("/"),
|
||||
"user": parsed.username,
|
||||
"password": parsed.password,
|
||||
}
|
||||
conn_info = DataSource.postgres.get_connection_info(raw_info)
|
||||
connector = PostgresConnector(conn_info)
|
||||
try:
|
||||
yield connector
|
||||
finally:
|
||||
connector.close()
|
||||
|
||||
def test_arrow_schema(self, connector: PostgresConnector) -> None:
|
||||
result = connector.query("SELECT * FROM type_zoo ORDER BY c_int4 NULLS LAST")
|
||||
assert isinstance(result, pa.Table)
|
||||
assert result.num_rows == 2
|
||||
|
||||
expected_types = {
|
||||
"c_int4": pa.int32(),
|
||||
"c_int8": pa.int64(),
|
||||
"c_numeric": pa.decimal128(38, 9),
|
||||
"c_text": pa.string(),
|
||||
"c_bool": pa.bool_(),
|
||||
"c_bytea": pa.binary(),
|
||||
"c_uuid": pa.string(),
|
||||
"c_jsonb": pa.string(),
|
||||
"c_ts": pa.timestamp("us"),
|
||||
"c_tstz": pa.timestamp("us", tz="UTC"),
|
||||
"c_int4_arr": pa.list_(pa.int32()),
|
||||
"c_text_arr": pa.list_(pa.string()),
|
||||
"c_numeric_arr": pa.list_(pa.decimal128(38, 9)),
|
||||
}
|
||||
for name, expected in expected_types.items():
|
||||
assert result.schema.field(name).type == expected, (
|
||||
f"unexpected Arrow type for {name}: {result.schema.field(name).type}"
|
||||
)
|
||||
|
||||
def test_value_round_trip(self, connector: PostgresConnector) -> None:
|
||||
result = connector.query("SELECT * FROM type_zoo WHERE c_int4 = 42")
|
||||
assert result.num_rows == 1
|
||||
row = result.to_pylist()[0]
|
||||
assert row["c_int4"] == 42
|
||||
assert row["c_int8"] == 9_000_000_000
|
||||
assert row["c_numeric"] == Decimal("12345.123456789")
|
||||
assert row["c_text"] == "hello"
|
||||
assert row["c_bool"] is True
|
||||
assert bytes(row["c_bytea"]) == b"\x01\x02\x03"
|
||||
assert row["c_uuid"] == "00000000-0000-0000-0000-000000000001"
|
||||
# jsonb comes back as JSON string
|
||||
assert '"a"' in row["c_jsonb"] and '"b"' in row["c_jsonb"]
|
||||
assert row["c_int4_arr"] == [1, 2, 3]
|
||||
assert row["c_text_arr"] == ["a", "b", "c"]
|
||||
assert row["c_numeric_arr"] == [Decimal("1.500000000"), Decimal("2.250000000")]
|
||||
|
||||
def test_nulls(self, connector: PostgresConnector) -> None:
|
||||
# Row inserted as `(NULL)` should produce a NULL in every column.
|
||||
result = connector.query("SELECT * FROM type_zoo WHERE c_int4 IS NULL")
|
||||
assert result.num_rows == 1
|
||||
row = result.to_pylist()[0]
|
||||
for col in result.column_names:
|
||||
assert row[col] is None, f"expected NULL for {col}, got {row[col]!r}"
|
||||
|
||||
def test_query_limit_parameter(self, connector: PostgresConnector) -> None:
|
||||
result = connector.query("SELECT c_int4 FROM type_zoo", limit=1)
|
||||
assert result.num_rows == 1
|
||||
|
||||
def test_dry_run(self, connector: PostgresConnector) -> None:
|
||||
# Should not raise and should not return rows.
|
||||
connector.dry_run("SELECT c_int4 FROM type_zoo")
|
||||
|
||||
def test_dry_run_invalid_sql_raises(self, connector: PostgresConnector) -> None:
|
||||
from wren.model.error import WrenError
|
||||
|
||||
with pytest.raises(WrenError):
|
||||
connector.dry_run("SELECT * FROM nope_does_not_exist")
|
||||
|
||||
def test_duplicate_column_names_preserved(
|
||||
self, connector: PostgresConnector
|
||||
) -> None:
|
||||
# ``pa.table({...})`` silently drops duplicate keys, which trashes
|
||||
# join results like ``SELECT a.id, b.id FROM t a, t b``. The
|
||||
# connector must preserve both fields positionally.
|
||||
# The previous ``test_dry_run_invalid_sql_raises`` aborts the shared
|
||||
# class-scoped connection's transaction; reset it before running.
|
||||
connector.connection.rollback()
|
||||
result = connector.query(
|
||||
"SELECT a.c_int4 AS id, b.c_int4 AS id "
|
||||
"FROM type_zoo a, type_zoo b "
|
||||
"WHERE a.c_int4 = 42 AND b.c_int4 = 42"
|
||||
)
|
||||
assert result.num_rows == 1
|
||||
assert result.num_columns == 2
|
||||
assert [field.name for field in result.schema] == ["id", "id"]
|
||||
assert result.column(0).to_pylist() == [42]
|
||||
assert result.column(1).to_pylist() == [42]
|
||||
|
||||
@@ -0,0 +1,370 @@
|
||||
"""Trino connector tests.
|
||||
|
||||
Combines:
|
||||
1. Unit-level type parser tests for the native Trino → Arrow mapping (no Docker).
|
||||
2. Integration tests against a real Trino testcontainer that load TPCH from
|
||||
the built-in ``tpch`` catalog into the in-memory ``memory.default`` schema
|
||||
so the shared ``WrenQueryTestSuite`` can run against it.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import time
|
||||
from decimal import Decimal
|
||||
|
||||
import orjson
|
||||
import pyarrow as pa
|
||||
import pytest
|
||||
from testcontainers.trino import TrinoContainer
|
||||
|
||||
from tests.suite.manifests import make_tpch_manifest
|
||||
from tests.suite.query import WrenQueryTestSuite
|
||||
from wren import WrenEngine
|
||||
from wren.connector.trino import (
|
||||
_build_trino_column,
|
||||
_parse_trino_data_type,
|
||||
_parse_trino_url,
|
||||
_strip_trailing_semicolon,
|
||||
)
|
||||
from wren.model.data_source import DataSource
|
||||
from wren.model.error import ErrorCode, WrenError
|
||||
|
||||
pytestmark = pytest.mark.trino
|
||||
|
||||
_CATALOG = "memory"
|
||||
_SCHEMA = "default"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Unit tests — type-string parser and column builder (no Docker required).
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("type_str", "expected"),
|
||||
[
|
||||
# Scalars
|
||||
("boolean", pa.bool_()),
|
||||
("tinyint", pa.int8()),
|
||||
("smallint", pa.int16()),
|
||||
("integer", pa.int32()),
|
||||
("int", pa.int32()),
|
||||
("bigint", pa.int64()),
|
||||
("real", pa.float32()),
|
||||
("double", pa.float64()),
|
||||
("varchar", pa.string()),
|
||||
("varchar(255)", pa.string()),
|
||||
("char(10)", pa.string()),
|
||||
("varbinary", pa.binary()),
|
||||
("date", pa.date32()),
|
||||
("uuid", pa.string()),
|
||||
("ipaddress", pa.string()),
|
||||
("json", pa.string()),
|
||||
# Decimal
|
||||
("decimal(10,2)", pa.decimal128(10, 2)),
|
||||
("decimal(38,9)", pa.decimal128(38, 9)),
|
||||
("decimal", pa.decimal128(38, 9)),
|
||||
# Time / timestamp
|
||||
("time", pa.time64("us")),
|
||||
("time(3)", pa.time64("us")),
|
||||
("timestamp", pa.timestamp("ms")),
|
||||
("timestamp(6)", pa.timestamp("ms")),
|
||||
("timestamp with time zone", pa.timestamp("ms", tz="UTC")),
|
||||
("timestamp(6) with time zone", pa.timestamp("ms", tz="UTC")),
|
||||
# Containers
|
||||
("array(integer)", pa.list_(pa.int32())),
|
||||
("array(decimal(10,2))", pa.list_(pa.decimal128(10, 2))),
|
||||
("array(varchar)", pa.list_(pa.string())),
|
||||
# Map
|
||||
("map(varchar,bigint)", pa.map_(pa.string(), pa.int64())),
|
||||
("map(varchar, bigint)", pa.map_(pa.string(), pa.int64())),
|
||||
# Row — named and anonymous
|
||||
(
|
||||
"row(a integer, b varchar)",
|
||||
pa.struct([pa.field("a", pa.int32()), pa.field("b", pa.string())]),
|
||||
),
|
||||
(
|
||||
"row(map(varchar, integer), bigint)",
|
||||
pa.struct(
|
||||
[
|
||||
pa.field("f0", pa.map_(pa.string(), pa.int32())),
|
||||
pa.field("f1", pa.int64()),
|
||||
]
|
||||
),
|
||||
),
|
||||
# Nested array(map(varchar, row(...)))
|
||||
(
|
||||
"array(map(varchar, row(a integer, b varchar)))",
|
||||
pa.list_(
|
||||
pa.map_(
|
||||
pa.string(),
|
||||
pa.struct(
|
||||
[
|
||||
pa.field("a", pa.int32()),
|
||||
pa.field("b", pa.string()),
|
||||
]
|
||||
),
|
||||
)
|
||||
),
|
||||
),
|
||||
# Unknown / interval — fall back to string
|
||||
("interval year to month", pa.string()),
|
||||
("interval day to second", pa.string()),
|
||||
("hyperloglog", pa.string()),
|
||||
],
|
||||
)
|
||||
def test_parse_trino_data_type(type_str: str, expected: pa.DataType) -> None:
|
||||
assert _parse_trino_data_type(type_str) == expected
|
||||
|
||||
|
||||
def test_parse_trino_data_type_handles_none() -> None:
|
||||
assert _parse_trino_data_type(None) == pa.string()
|
||||
|
||||
|
||||
def test_parse_trino_data_type_unparseable_falls_back() -> None:
|
||||
assert _parse_trino_data_type("not a real type {{") == pa.string()
|
||||
|
||||
|
||||
def test_build_trino_column_map_dict_to_pairs() -> None:
|
||||
# Trino driver returns Python dicts; PyArrow map_ wants (k, v) pairs.
|
||||
arrow_type = pa.map_(pa.string(), pa.int64())
|
||||
arr = _build_trino_column([{"a": 1, "b": 2}, None, {"x": 9}], arrow_type)
|
||||
assert arr.type == arrow_type
|
||||
assert arr.to_pylist() == [
|
||||
[("a", 1), ("b", 2)],
|
||||
None,
|
||||
[("x", 9)],
|
||||
]
|
||||
|
||||
|
||||
def test_build_trino_column_decimal_string_input() -> None:
|
||||
arr = _build_trino_column(["1.23", "4.56", None], pa.decimal128(10, 2))
|
||||
assert arr.to_pylist() == [Decimal("1.23"), Decimal("4.56"), None]
|
||||
|
||||
|
||||
def test_build_trino_column_struct_tuple_to_dict() -> None:
|
||||
arrow_type = pa.struct([pa.field("a", pa.int32()), pa.field("b", pa.string())])
|
||||
arr = _build_trino_column([(1, "x"), None, (2, "y")], arrow_type)
|
||||
assert arr.to_pylist() == [{"a": 1, "b": "x"}, None, {"a": 2, "b": "y"}]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("raw", "expected"),
|
||||
[
|
||||
("SELECT 1", "SELECT 1"),
|
||||
("SELECT 1;", "SELECT 1"),
|
||||
("SELECT 1 ; ", "SELECT 1"),
|
||||
("SELECT 1\n;\n", "SELECT 1"),
|
||||
("SELECT 1; -- trailing", "SELECT 1; -- trailing"),
|
||||
# Only the final semicolon is stripped — internal ones stay.
|
||||
("SELECT 1; SELECT 2;", "SELECT 1; SELECT 2"),
|
||||
],
|
||||
)
|
||||
def test_strip_trailing_semicolon(raw: str, expected: str) -> None:
|
||||
assert _strip_trailing_semicolon(raw) == expected
|
||||
|
||||
|
||||
def test_parse_trino_url_rejects_missing_username() -> None:
|
||||
with pytest.raises(WrenError) as exc:
|
||||
_parse_trino_url("trino://host:8080/catalog/schema", None)
|
||||
assert exc.value.error_code == ErrorCode.INVALID_CONNECTION_INFO
|
||||
assert "username" in str(exc.value).lower()
|
||||
|
||||
|
||||
def test_parse_trino_url_accepts_explicit_username() -> None:
|
||||
out = _parse_trino_url("trino://alice@host:8080/catalog/schema", None)
|
||||
assert out["user"] == "alice"
|
||||
assert out["host"] == "host"
|
||||
assert out["catalog"] == "catalog"
|
||||
assert out["schema"] == "schema"
|
||||
|
||||
|
||||
def test_parse_trino_url_rejects_bad_scheme() -> None:
|
||||
with pytest.raises(WrenError) as exc:
|
||||
_parse_trino_url("http://alice@host:8080/c/s", None)
|
||||
assert exc.value.error_code == ErrorCode.INVALID_CONNECTION_INFO
|
||||
|
||||
|
||||
def test_trino_connector_import_error_has_install_hint(monkeypatch) -> None:
|
||||
"""If ``import trino`` fails, the connector should raise a WrenError with
|
||||
a clear ``pip install wren-engine[trino]`` hint rather than a raw
|
||||
ImportError.
|
||||
"""
|
||||
import builtins # noqa: PLC0415
|
||||
import sys # noqa: PLC0415
|
||||
|
||||
from wren.connector import trino as trino_module # noqa: PLC0415
|
||||
|
||||
# Remove cached trino module so the lazy import re-runs.
|
||||
monkeypatch.setitem(sys.modules, "trino", None)
|
||||
real_import = builtins.__import__
|
||||
|
||||
def _fake_import(name, *args, **kwargs):
|
||||
if name == "trino" or name.startswith("trino."):
|
||||
raise ImportError("No module named 'trino'")
|
||||
return real_import(name, *args, **kwargs)
|
||||
|
||||
monkeypatch.setattr(builtins, "__import__", _fake_import)
|
||||
|
||||
with pytest.raises(WrenError) as exc:
|
||||
trino_module._import_trino()
|
||||
assert exc.value.error_code == ErrorCode.INVALID_CONNECTION_INFO
|
||||
msg = str(exc.value)
|
||||
assert "wren-engine[trino]" in msg
|
||||
|
||||
|
||||
def test_native_connector_does_not_import_ibis() -> None:
|
||||
# Acceptance criterion: importing the native trino connector must not
|
||||
# pull ibis into sys.modules (the new module is independent of
|
||||
# ibis-framework[trino]).
|
||||
import sys # noqa: PLC0415
|
||||
|
||||
sys.modules.pop("ibis", None)
|
||||
sys.modules.pop("wren.connector.trino", None)
|
||||
import wren.connector.trino # noqa: F401, PLC0415
|
||||
|
||||
assert "ibis" not in sys.modules
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Integration tests — testcontainer-backed query suite.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _create_tpch_tables(host: str, port: int) -> None:
|
||||
"""Materialise TPCH ``orders`` / ``customer`` into ``memory.default``.
|
||||
|
||||
Trino's bundled ``tpch`` connector provides scale-factor data on demand,
|
||||
so we just copy the rows we need into the in-memory catalog rather than
|
||||
generating them with DuckDB.
|
||||
"""
|
||||
from trino.dbapi import connect as trino_connect # noqa: PLC0415
|
||||
|
||||
conn = trino_connect(host=host, port=port, user="test")
|
||||
try:
|
||||
cur = conn.cursor()
|
||||
cur.execute(
|
||||
"CREATE TABLE memory.default.orders AS "
|
||||
"SELECT orderkey AS o_orderkey, custkey AS o_custkey, "
|
||||
"orderstatus AS o_orderstatus, "
|
||||
"CAST(totalprice AS DOUBLE) AS o_totalprice, "
|
||||
"orderdate AS o_orderdate "
|
||||
"FROM tpch.tiny.orders"
|
||||
)
|
||||
cur.fetchall()
|
||||
cur.execute(
|
||||
"CREATE TABLE memory.default.customer AS "
|
||||
"SELECT custkey AS c_custkey, name AS c_name FROM tpch.tiny.customer"
|
||||
)
|
||||
cur.fetchall()
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
class TestTrino(WrenQueryTestSuite):
|
||||
"""Run the shared connector test suite against a real Trino container."""
|
||||
|
||||
manifest = make_tpch_manifest(table_catalog=_CATALOG, table_schema=_SCHEMA)
|
||||
# tpch.tiny is sf=0.01 — same row counts as the other connector tests.
|
||||
order_count = 15000
|
||||
customer_count = 1500
|
||||
# Trino BIGINT → Arrow int64.
|
||||
order_id_dtype = "int64"
|
||||
|
||||
@pytest.fixture(scope="class")
|
||||
def engine(self) -> WrenEngine: # type: ignore[override]
|
||||
with TrinoContainer() as trino:
|
||||
host = trino.get_container_host_ip()
|
||||
port = int(trino.get_exposed_port(trino.port))
|
||||
|
||||
# Trino sometimes returns "nodes is empty" if we query before the
|
||||
# coordinator has registered a worker; brief wait avoids it.
|
||||
time.sleep(5)
|
||||
_create_tpch_tables(host, port)
|
||||
|
||||
conn_info = {
|
||||
"host": host,
|
||||
"port": port,
|
||||
"catalog": _CATALOG,
|
||||
"schema": _SCHEMA,
|
||||
"user": "test",
|
||||
}
|
||||
manifest_str = base64.b64encode(orjson.dumps(self.manifest)).decode()
|
||||
with WrenEngine(
|
||||
manifest_str, DataSource.trino, conn_info, fallback=False
|
||||
) as e:
|
||||
yield e
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Trino-specific type coverage — exercises every branch in the native
|
||||
# type parser end-to-end against a live coordinator.
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def test_scalar_types(self, engine: WrenEngine) -> None:
|
||||
result = engine.query(
|
||||
"SELECT "
|
||||
"CAST(1 AS BIGINT) AS c_bigint, "
|
||||
"CAST(1 AS INTEGER) AS c_integer, "
|
||||
"CAST(1 AS SMALLINT) AS c_smallint, "
|
||||
"CAST(1 AS TINYINT) AS c_tinyint, "
|
||||
"CAST(1.5 AS DOUBLE) AS c_double, "
|
||||
"CAST(1.5 AS REAL) AS c_real, "
|
||||
"CAST('1.23' AS DECIMAL(10,2)) AS c_decimal, "
|
||||
"CAST('abc' AS VARCHAR) AS c_varchar, "
|
||||
"CAST('abc' AS CHAR(3)) AS c_char, "
|
||||
"CAST(X'AB' AS VARBINARY) AS c_varbinary, "
|
||||
"CAST('{\"a\":1}' AS JSON) AS c_json, "
|
||||
"CAST('12151fd2-7586-11e9-8f9e-2a86e4085a59' AS UUID) AS c_uuid, "
|
||||
"CAST('1.2.3.4' AS IPADDRESS) AS c_ip"
|
||||
)
|
||||
assert result.num_rows == 1
|
||||
assert result.schema.field("c_bigint").type == pa.int64()
|
||||
assert result.schema.field("c_integer").type == pa.int32()
|
||||
assert result.schema.field("c_smallint").type == pa.int16()
|
||||
assert result.schema.field("c_tinyint").type == pa.int8()
|
||||
assert result.schema.field("c_double").type == pa.float64()
|
||||
assert result.schema.field("c_real").type == pa.float32()
|
||||
assert result.schema.field("c_decimal").type == pa.decimal128(10, 2)
|
||||
assert result.schema.field("c_varchar").type == pa.string()
|
||||
assert result.schema.field("c_char").type == pa.string()
|
||||
assert result.schema.field("c_varbinary").type == pa.binary()
|
||||
assert result.schema.field("c_json").type == pa.string()
|
||||
assert result.schema.field("c_uuid").type == pa.string()
|
||||
assert result.schema.field("c_ip").type == pa.string()
|
||||
|
||||
def test_temporal_types(self, engine: WrenEngine) -> None:
|
||||
result = engine.query(
|
||||
"SELECT "
|
||||
"DATE '2024-01-02' AS c_date, "
|
||||
"TIME '12:34:56' AS c_time, "
|
||||
"TIMESTAMP '2024-01-02 12:34:56' AS c_ts, "
|
||||
"TIMESTAMP '2024-01-02 12:34:56 UTC' AS c_tstz"
|
||||
)
|
||||
assert result.schema.field("c_date").type == pa.date32()
|
||||
assert result.schema.field("c_time").type == pa.time64("us")
|
||||
assert result.schema.field("c_ts").type == pa.timestamp("ms")
|
||||
assert result.schema.field("c_tstz").type == pa.timestamp("ms", tz="UTC")
|
||||
|
||||
def test_array_type(self, engine: WrenEngine) -> None:
|
||||
result = engine.query("SELECT ARRAY[1, 2, 3] AS c_array")
|
||||
assert result.schema.field("c_array").type == pa.list_(pa.int32())
|
||||
assert result["c_array"][0].as_py() == [1, 2, 3]
|
||||
|
||||
def test_map_type(self, engine: WrenEngine) -> None:
|
||||
result = engine.query("SELECT MAP(ARRAY['a', 'b'], ARRAY[1, 2]) AS c_map")
|
||||
assert result.schema.field("c_map").type == pa.map_(pa.string(), pa.int32())
|
||||
|
||||
def test_row_type(self, engine: WrenEngine) -> None:
|
||||
result = engine.query(
|
||||
"SELECT CAST(ROW(1, 'x') AS ROW(a INTEGER, b VARCHAR)) AS c_row"
|
||||
)
|
||||
row_type = result.schema.field("c_row").type
|
||||
assert pa.types.is_struct(row_type)
|
||||
assert {f.name for f in row_type} == {"a", "b"}
|
||||
|
||||
def test_anonymous_row_type(self, engine: WrenEngine) -> None:
|
||||
# Anonymous row(...) result — field names come back as f0/f1.
|
||||
result = engine.query("SELECT ROW(1, 'x') AS c_row")
|
||||
row_type = result.schema.field("c_row").type
|
||||
assert pa.types.is_struct(row_type)
|
||||
@@ -0,0 +1,335 @@
|
||||
"""Unit tests for the native pyathena-backed Athena connector.
|
||||
|
||||
These tests stub out :mod:`pyathena` and :mod:`boto3` so they can run without
|
||||
any AWS credentials or network access.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import datetime as dtlib
|
||||
import sys
|
||||
import types
|
||||
from decimal import Decimal as PyDecimal
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pyarrow as pa
|
||||
import pytest
|
||||
from pydantic import SecretStr
|
||||
|
||||
pytestmark = pytest.mark.unit
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# stub pyathena module before importing the connector under test
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_pyathena_connect_calls: list[dict] = []
|
||||
_pyathena_connection_mock = MagicMock(name="pyathena_connection")
|
||||
|
||||
|
||||
def _fake_pyathena_connect(**kwargs):
|
||||
_pyathena_connect_calls.append(kwargs)
|
||||
return _pyathena_connection_mock
|
||||
|
||||
|
||||
_pyathena_module = types.ModuleType("pyathena")
|
||||
_pyathena_module.connect = _fake_pyathena_connect # type: ignore[attr-defined]
|
||||
sys.modules.setdefault("pyathena", _pyathena_module)
|
||||
|
||||
from wren.connector.athena import ( # noqa: E402
|
||||
AthenaConnector,
|
||||
_build_athena_arrow_table,
|
||||
_parse_athena_type,
|
||||
)
|
||||
from wren.model import AthenaConnectionInfo # noqa: E402
|
||||
from wren.model.data_source import DataSourceExtension # noqa: E402
|
||||
from wren.model.error import ErrorCode, ErrorPhase, WrenError # noqa: E402
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _reset_connect_calls():
|
||||
_pyathena_connect_calls.clear()
|
||||
_pyathena_connection_mock.reset_mock(return_value=True, side_effect=True)
|
||||
_pyathena_connection_mock.close.reset_mock()
|
||||
yield
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# type lexer
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("type_str", "expected"),
|
||||
[
|
||||
("varchar", pa.string()),
|
||||
("integer", pa.int32()),
|
||||
("bigint", pa.int64()),
|
||||
("boolean", pa.bool_()),
|
||||
("double", pa.float64()),
|
||||
("date", pa.date32()),
|
||||
("varbinary", pa.binary()),
|
||||
],
|
||||
)
|
||||
def test_parse_athena_type_primitives(type_str, expected):
|
||||
assert _parse_athena_type(type_str) == expected
|
||||
|
||||
|
||||
def test_parse_athena_type_decimal():
|
||||
assert _parse_athena_type("decimal(12,4)") == pa.decimal128(12, 4)
|
||||
|
||||
|
||||
def test_parse_athena_type_array():
|
||||
assert _parse_athena_type("array(varchar)") == pa.list_(pa.string())
|
||||
|
||||
|
||||
def test_parse_athena_type_map():
|
||||
assert _parse_athena_type("map(varchar,bigint)") == pa.map_(pa.string(), pa.int64())
|
||||
|
||||
|
||||
def test_parse_athena_type_row():
|
||||
parsed = _parse_athena_type("row(a integer, b varchar)")
|
||||
assert pa.types.is_struct(parsed)
|
||||
fields = {
|
||||
parsed.field(i).name: parsed.field(i).type for i in range(parsed.num_fields)
|
||||
}
|
||||
assert fields["a"] == pa.int32()
|
||||
assert fields["b"] == pa.string()
|
||||
|
||||
|
||||
def test_parse_athena_type_unknown_falls_back_to_string():
|
||||
assert _parse_athena_type("totally not a type") == pa.string()
|
||||
|
||||
|
||||
def test_parse_athena_type_none_returns_string():
|
||||
assert _parse_athena_type(None) == pa.string()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# cursor → arrow table
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _make_cursor(description, rows):
|
||||
cursor = MagicMock()
|
||||
cursor.description = description
|
||||
cursor.fetchall.return_value = rows
|
||||
cursor.execute.return_value = cursor
|
||||
return cursor
|
||||
|
||||
|
||||
def test_build_athena_arrow_table_mixed_types():
|
||||
description = [
|
||||
("id", "bigint"),
|
||||
("name", "varchar"),
|
||||
("price", "decimal(10,2)"),
|
||||
("tags", "array(varchar)"),
|
||||
("ordered_at", "timestamp"),
|
||||
("ordered_on", "date"),
|
||||
]
|
||||
rows = [
|
||||
(
|
||||
1,
|
||||
"alice",
|
||||
PyDecimal("9.99"),
|
||||
["a", "b"],
|
||||
dtlib.datetime(2024, 1, 2, 3, 4, 5),
|
||||
dtlib.date(2024, 1, 2),
|
||||
),
|
||||
(2, "bob", PyDecimal("1.50"), [], None, None),
|
||||
]
|
||||
cursor = _make_cursor(description, rows)
|
||||
table = _build_athena_arrow_table(cursor)
|
||||
assert table.num_rows == 2
|
||||
assert table.schema.field("id").type == pa.int64()
|
||||
assert table.schema.field("name").type == pa.string()
|
||||
assert table.schema.field("price").type == pa.decimal128(10, 2)
|
||||
assert table.schema.field("tags").type == pa.list_(pa.string())
|
||||
assert table.schema.field("ordered_at").type == pa.timestamp("ms")
|
||||
assert table.schema.field("ordered_on").type == pa.date32()
|
||||
|
||||
|
||||
def test_build_athena_arrow_table_empty():
|
||||
cursor = _make_cursor([("a", "varchar")], [])
|
||||
table = _build_athena_arrow_table(cursor)
|
||||
assert table.num_rows == 0
|
||||
assert table.schema.field("a").type == pa.string()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# AthenaConnector — happy path & error mapping
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _info(**overrides) -> AthenaConnectionInfo:
|
||||
data = {
|
||||
"s3_staging_dir": SecretStr("s3://bucket/staging/"),
|
||||
"region_name": "us-west-2",
|
||||
"schema_name": "default",
|
||||
}
|
||||
data.update(overrides)
|
||||
return AthenaConnectionInfo(**data)
|
||||
|
||||
|
||||
def test_connector_query_passes_kwargs_and_kills_on_interrupt():
|
||||
AthenaConnector(_info())
|
||||
assert _pyathena_connect_calls, "pyathena.connect was not invoked"
|
||||
kwargs = _pyathena_connect_calls[-1]
|
||||
assert kwargs["s3_staging_dir"] == "s3://bucket/staging/"
|
||||
assert kwargs["region_name"] == "us-west-2"
|
||||
assert kwargs["schema_name"] == "default"
|
||||
assert kwargs["kill_on_interrupt"] is True
|
||||
|
||||
|
||||
def test_connector_query_returns_arrow_table_and_respects_limit():
|
||||
cursor = _make_cursor(
|
||||
[("id", "integer"), ("name", "varchar")],
|
||||
[(1, "a"), (2, "b"), (3, "c")],
|
||||
)
|
||||
_pyathena_connection_mock.cursor.return_value = cursor
|
||||
|
||||
conn = AthenaConnector(_info())
|
||||
table = conn.query("SELECT id, name FROM t", limit=2)
|
||||
assert table.num_rows == 2
|
||||
assert table.column("id").to_pylist() == [1, 2]
|
||||
cursor.execute.assert_called_once_with("SELECT id, name FROM t")
|
||||
|
||||
|
||||
def test_connector_query_wraps_driver_errors():
|
||||
cursor = MagicMock()
|
||||
cursor.execute.side_effect = RuntimeError("boom")
|
||||
_pyathena_connection_mock.cursor.return_value = cursor
|
||||
|
||||
conn = AthenaConnector(_info())
|
||||
with pytest.raises(WrenError) as exc:
|
||||
conn.query("SELECT 1")
|
||||
assert exc.value.error_code == ErrorCode.INVALID_SQL
|
||||
assert exc.value.phase == ErrorPhase.SQL_EXECUTION
|
||||
|
||||
|
||||
def test_connector_dry_run_emits_explain():
|
||||
cursor = MagicMock()
|
||||
cursor.execute.return_value = cursor
|
||||
_pyathena_connection_mock.cursor.return_value = cursor
|
||||
|
||||
conn = AthenaConnector(_info())
|
||||
conn.dry_run("SELECT 1")
|
||||
cursor.execute.assert_called_once_with("EXPLAIN SELECT 1")
|
||||
|
||||
|
||||
def test_connector_dry_run_wraps_driver_errors():
|
||||
cursor = MagicMock()
|
||||
cursor.execute.side_effect = RuntimeError("parse error")
|
||||
_pyathena_connection_mock.cursor.return_value = cursor
|
||||
|
||||
conn = AthenaConnector(_info())
|
||||
with pytest.raises(WrenError) as exc:
|
||||
conn.dry_run("SELECT bogus")
|
||||
assert exc.value.error_code == ErrorCode.INVALID_SQL
|
||||
assert exc.value.phase == ErrorPhase.SQL_DRY_RUN
|
||||
|
||||
|
||||
def test_connector_close_calls_underlying_connection():
|
||||
conn = AthenaConnector(_info())
|
||||
underlying = conn.connection
|
||||
conn.close()
|
||||
underlying.close.assert_called_once()
|
||||
assert conn.connection is None
|
||||
# Idempotent.
|
||||
conn.close()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# credential resolution
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_connector_uses_access_key_when_provided():
|
||||
AthenaConnector(
|
||||
_info(
|
||||
aws_access_key_id=SecretStr("AKIA"),
|
||||
aws_secret_access_key=SecretStr("SECRET"),
|
||||
aws_session_token=SecretStr("TOKEN"),
|
||||
)
|
||||
)
|
||||
kwargs = _pyathena_connect_calls[-1]
|
||||
assert kwargs["aws_access_key_id"] == "AKIA"
|
||||
assert kwargs["aws_secret_access_key"] == "SECRET"
|
||||
assert kwargs["aws_session_token"] == "TOKEN"
|
||||
|
||||
|
||||
def test_connector_uses_oidc_when_web_identity_token_provided(monkeypatch):
|
||||
sts_client = MagicMock()
|
||||
sts_client.assume_role_with_web_identity.return_value = {
|
||||
"Credentials": {
|
||||
"AccessKeyId": "OIDC_AK",
|
||||
"SecretAccessKey": "OIDC_SK",
|
||||
"SessionToken": "OIDC_TK",
|
||||
}
|
||||
}
|
||||
boto_module = MagicMock()
|
||||
boto_module.client.return_value = sts_client
|
||||
monkeypatch.setattr("boto3.client", boto_module.client)
|
||||
|
||||
AthenaConnector(
|
||||
_info(
|
||||
web_identity_token=SecretStr("token-xyz"),
|
||||
role_arn=SecretStr("arn:aws:iam::123:role/wren"),
|
||||
role_session_name="custom-session",
|
||||
)
|
||||
)
|
||||
|
||||
boto_module.client.assert_called_once_with("sts", region_name="us-west-2")
|
||||
sts_client.assume_role_with_web_identity.assert_called_once_with(
|
||||
RoleArn="arn:aws:iam::123:role/wren",
|
||||
RoleSessionName="custom-session",
|
||||
WebIdentityToken="token-xyz",
|
||||
)
|
||||
kwargs = _pyathena_connect_calls[-1]
|
||||
assert kwargs["aws_access_key_id"] == "OIDC_AK"
|
||||
assert kwargs["aws_secret_access_key"] == "OIDC_SK"
|
||||
assert kwargs["aws_session_token"] == "OIDC_TK"
|
||||
|
||||
|
||||
def test_connector_falls_back_to_default_chain_when_no_credentials():
|
||||
AthenaConnector(_info())
|
||||
kwargs = _pyathena_connect_calls[-1]
|
||||
assert "aws_access_key_id" not in kwargs
|
||||
assert "aws_secret_access_key" not in kwargs
|
||||
assert "aws_session_token" not in kwargs
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# data_source.get_athena_connection still works end-to-end
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_data_source_get_athena_connection_returns_pyathena_connection():
|
||||
DataSourceExtension.get_athena_connection(_info())
|
||||
assert _pyathena_connect_calls, "pyathena.connect should be called"
|
||||
|
||||
|
||||
def test_data_source_get_athena_connection_propagates_schema_and_kwargs():
|
||||
"""Regression: data_source.get_athena_connection must funnel through the
|
||||
same kwargs builder as AthenaConnector so schema_name, kill_on_interrupt,
|
||||
and any user-supplied info.kwargs reach pyathena.connect consistently.
|
||||
"""
|
||||
info = _info(schema_name="analytics")
|
||||
# AthenaConnectionInfo currently has no ``kwargs`` field, but the shared
|
||||
# builder reads it defensively. Simulate a future/user-provided value.
|
||||
object.__setattr__(info, "kwargs", {"work_group": "wg-1", "kill_on_interrupt": False})
|
||||
|
||||
DataSourceExtension.get_athena_connection(info)
|
||||
|
||||
kwargs = _pyathena_connect_calls[-1]
|
||||
assert kwargs["schema_name"] == "analytics"
|
||||
assert kwargs["work_group"] == "wg-1"
|
||||
# User-supplied kwargs override the default.
|
||||
assert kwargs["kill_on_interrupt"] is False
|
||||
|
||||
|
||||
def test_data_source_get_athena_connection_defaults_kill_on_interrupt():
|
||||
DataSourceExtension.get_athena_connection(_info())
|
||||
kwargs = _pyathena_connect_calls[-1]
|
||||
assert kwargs["kill_on_interrupt"] is True
|
||||
assert kwargs["schema_name"] == "default"
|
||||
@@ -0,0 +1,157 @@
|
||||
"""Unit tests for ``wren.connector.clickhouse`` URL parsing and query wrapping.
|
||||
|
||||
Pure-Python — no Docker, no real ClickHouse instance.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
from wren.connector.clickhouse import (
|
||||
ClickHouseConnector,
|
||||
_build_clickhouse_arrow_table,
|
||||
_build_clickhouse_client_kwargs,
|
||||
)
|
||||
|
||||
pytestmark = pytest.mark.unit
|
||||
|
||||
|
||||
class _FakeConnUrl:
|
||||
def __init__(self, url: str) -> None:
|
||||
self._url = url
|
||||
|
||||
def get_secret_value(self) -> str:
|
||||
return self._url
|
||||
|
||||
|
||||
class _FakeConnInfoFromUrl:
|
||||
"""Minimal stand-in for ConnectionUrl payloads accepted by the builder."""
|
||||
|
||||
def __init__(self, url: str, **extras) -> None:
|
||||
self.connection_url = _FakeConnUrl(url)
|
||||
self.kwargs = extras.get("kwargs")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 1. URL credentials are percent-decoded
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_clickhouse_url_decodes_username_and_password() -> None:
|
||||
"""user / password from the URL must be unquote_plus'd.
|
||||
|
||||
urlparse leaves ``%40`` (``@``) and ``%20`` (space) literal in the
|
||||
userinfo, which would otherwise reach ClickHouse verbatim and fail auth.
|
||||
"""
|
||||
info = _FakeConnInfoFromUrl(
|
||||
"clickhouse://us%40er:p%40ss%20word@clickhouse-host:9000/analytics"
|
||||
)
|
||||
|
||||
out = _build_clickhouse_client_kwargs(info)
|
||||
|
||||
assert out["username"] == "us@er"
|
||||
assert out["password"] == "p@ss word"
|
||||
assert out["host"] == "clickhouse-host"
|
||||
assert out["port"] == 9000
|
||||
assert out["database"] == "analytics"
|
||||
|
||||
|
||||
def test_clickhouse_url_defaults_username_when_omitted() -> None:
|
||||
"""When the URL has no userinfo, the default ``"default"`` user wins."""
|
||||
info = _FakeConnInfoFromUrl("clickhouse://clickhouse-host/analytics")
|
||||
|
||||
out = _build_clickhouse_client_kwargs(info)
|
||||
|
||||
assert out["username"] == "default"
|
||||
assert out["password"] == ""
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 2. Trailing semicolons in caller SQL must not break the subquery wrap
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _make_connector_with_mock_query() -> tuple[ClickHouseConnector, MagicMock]:
|
||||
"""Build a ClickHouseConnector bypassing ``__init__`` (no real client)."""
|
||||
connector = ClickHouseConnector.__new__(ClickHouseConnector)
|
||||
connector._closed = False
|
||||
connector.connection = MagicMock()
|
||||
# ``query()`` consumes the returned object via ``_build_clickhouse_arrow_table``;
|
||||
# arrange a minimal result that produces an empty Arrow table.
|
||||
fake_result = MagicMock()
|
||||
fake_result.column_names = []
|
||||
fake_result.column_types = []
|
||||
fake_result.result_columns = []
|
||||
connector.connection.query.return_value = fake_result
|
||||
return connector, connector.connection
|
||||
|
||||
|
||||
def test_clickhouse_query_strips_trailing_semicolon_before_subquery_wrap() -> None:
|
||||
"""``SELECT 1;`` must not become ``SELECT * FROM (SELECT 1;) ...``."""
|
||||
connector, mock_conn = _make_connector_with_mock_query()
|
||||
|
||||
connector.query("SELECT 1;", limit=5)
|
||||
|
||||
(sent,), _ = mock_conn.query.call_args
|
||||
assert sent == "SELECT * FROM (SELECT 1) AS _wren_sub LIMIT 5"
|
||||
|
||||
|
||||
def test_clickhouse_query_strips_multiple_trailing_semicolons_and_whitespace() -> None:
|
||||
"""Trailing whitespace and a stray ``;`` both get trimmed."""
|
||||
connector, mock_conn = _make_connector_with_mock_query()
|
||||
|
||||
connector.query("SELECT 1 ; ", limit=3)
|
||||
|
||||
(sent,), _ = mock_conn.query.call_args
|
||||
assert sent == "SELECT * FROM (SELECT 1) AS _wren_sub LIMIT 3"
|
||||
|
||||
|
||||
def test_clickhouse_query_without_limit_still_strips_semicolon() -> None:
|
||||
"""When no limit is supplied the executed SQL is also the stripped form."""
|
||||
connector, mock_conn = _make_connector_with_mock_query()
|
||||
|
||||
connector.query("SELECT 1;")
|
||||
|
||||
(sent,), _ = mock_conn.query.call_args
|
||||
assert sent == "SELECT 1"
|
||||
|
||||
|
||||
def test_clickhouse_dry_run_strips_trailing_semicolon() -> None:
|
||||
"""Same fix must apply to dry_run, which also wraps in a subquery."""
|
||||
connector, mock_conn = _make_connector_with_mock_query()
|
||||
|
||||
connector.dry_run("SELECT 1;")
|
||||
|
||||
(sent,), _ = mock_conn.query.call_args
|
||||
assert sent == "SELECT * FROM (SELECT 1) AS _wren_sub LIMIT 0"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 3. Duplicate column names survive ``_build_clickhouse_arrow_table``
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class _FakeChType:
|
||||
def __init__(self, name: str) -> None:
|
||||
self.name = name
|
||||
|
||||
|
||||
def test_clickhouse_arrow_table_preserves_duplicate_column_names() -> None:
|
||||
"""``SELECT a, a`` must yield a two-column Arrow table, not one column.
|
||||
|
||||
Earlier the table was built via ``dict(zip(names, arrays))`` which
|
||||
silently collapsed duplicate names.
|
||||
"""
|
||||
fake = MagicMock()
|
||||
fake.column_names = ["a", "a"]
|
||||
fake.column_types = [_FakeChType("Int64"), _FakeChType("Int64")]
|
||||
fake.result_rows = [[1, 2], [3, 4]]
|
||||
|
||||
table = _build_clickhouse_arrow_table(fake)
|
||||
|
||||
assert table.num_columns == 2
|
||||
assert table.column_names == ["a", "a"]
|
||||
assert table.column(0).to_pylist() == [1, 3]
|
||||
assert table.column(1).to_pylist() == [2, 4]
|
||||
@@ -0,0 +1,280 @@
|
||||
"""Unit tests for the MSSQL pyodbc connection plumbing.
|
||||
|
||||
These tests mock out ``pyodbc`` so they run without ODBC Driver 18
|
||||
installed; they cover behaviours called out in PR #2274 review:
|
||||
|
||||
1. ``mssql://`` URL components are URL-decoded round-trip.
|
||||
2. Asymmetric ``user`` / ``password`` combinations raise instead of
|
||||
leaking a half-built ODBC connection string.
|
||||
3. A non-numeric ``statement_timeout`` raises BEFORE ``pyodbc.connect``
|
||||
is called, so the connection can't be leaked.
|
||||
4. SQL Server ``TINYINT`` (``internal_size == 1``) maps unconditionally
|
||||
to ``pa.uint8()``.
|
||||
5. ``_decode_mssql_datetimeoffset`` validates payload length explicitly.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pyarrow as pa
|
||||
import pytest
|
||||
|
||||
from wren.connector.mssql import MSSqlConnector
|
||||
from wren.model.data_source import DataSourceExtension
|
||||
from wren.model.error import ErrorCode, WrenError
|
||||
|
||||
pytestmark = pytest.mark.unit
|
||||
|
||||
|
||||
class _FakePyodbc:
|
||||
"""Stand-in for the ``pyodbc`` module used by ``_connect_mssql_pyodbc``."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.connect = MagicMock(return_value=MagicMock(timeout=0))
|
||||
|
||||
|
||||
def _parse_conn_str(connect_call) -> dict[str, str]:
|
||||
"""Split an ODBC connection string from a ``connect`` call into a dict."""
|
||||
(conn_str,), _ = connect_call
|
||||
parts: dict[str, str] = {}
|
||||
for piece in conn_str.split(";"):
|
||||
if not piece:
|
||||
continue
|
||||
key, _, value = piece.partition("=")
|
||||
# Strip the {value} escaping used by _escape_odbc_value
|
||||
if value.startswith("{") and value.endswith("}"):
|
||||
value = value[1:-1].replace("}}", "}")
|
||||
parts[key] = value
|
||||
return parts
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 1. URL decoding round-trip
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_mssql_url_decodes_user_database_and_password() -> None:
|
||||
"""user, database path, and password should all be URL-decoded."""
|
||||
fake = _FakePyodbc()
|
||||
url = (
|
||||
"mssql://us%40er:p%40ss%20word@host:1433/"
|
||||
"my%20db?TrustServerCertificate=yes&app%20name=wren"
|
||||
)
|
||||
|
||||
with patch("wren.model.data_source.pyodbc", fake):
|
||||
DataSourceExtension.get_mssql_connection_from_url(url)
|
||||
|
||||
fake.connect.assert_called_once()
|
||||
parts = _parse_conn_str(fake.connect.call_args)
|
||||
assert parts["UID"] == "us@er"
|
||||
assert parts["PWD"] == "p@ss word"
|
||||
assert parts["DATABASE"] == "my db"
|
||||
assert parts["SERVER"] == "host,1433"
|
||||
# parse_qsl handles query-string decoding; the key keeps its space.
|
||||
assert parts["TrustServerCertificate"] == "yes"
|
||||
assert parts["app name"] == "wren"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 2. Auth combination validation
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_mssql_user_without_password_raises() -> None:
|
||||
fake = _FakePyodbc()
|
||||
with patch("wren.model.data_source.pyodbc", fake):
|
||||
with pytest.raises(WrenError) as exc:
|
||||
DataSourceExtension._connect_mssql_pyodbc(
|
||||
host="h",
|
||||
port="1433",
|
||||
database="db",
|
||||
user="alice",
|
||||
password=None,
|
||||
driver="ODBC Driver 18 for SQL Server",
|
||||
)
|
||||
assert exc.value.error_code == ErrorCode.INVALID_CONNECTION_INFO
|
||||
fake.connect.assert_not_called()
|
||||
|
||||
|
||||
def test_mssql_password_without_user_raises() -> None:
|
||||
fake = _FakePyodbc()
|
||||
with patch("wren.model.data_source.pyodbc", fake):
|
||||
with pytest.raises(WrenError) as exc:
|
||||
DataSourceExtension._connect_mssql_pyodbc(
|
||||
host="h",
|
||||
port="1433",
|
||||
database="db",
|
||||
user=None,
|
||||
password="secret",
|
||||
driver="ODBC Driver 18 for SQL Server",
|
||||
)
|
||||
assert exc.value.error_code == ErrorCode.INVALID_CONNECTION_INFO
|
||||
fake.connect.assert_not_called()
|
||||
|
||||
|
||||
def test_mssql_no_credentials_uses_trusted_connection() -> None:
|
||||
fake = _FakePyodbc()
|
||||
with patch("wren.model.data_source.pyodbc", fake):
|
||||
DataSourceExtension._connect_mssql_pyodbc(
|
||||
host="h",
|
||||
port="1433",
|
||||
database="db",
|
||||
user=None,
|
||||
password=None,
|
||||
driver="ODBC Driver 18 for SQL Server",
|
||||
)
|
||||
|
||||
parts = _parse_conn_str(fake.connect.call_args)
|
||||
assert parts.get("Trusted_Connection") == "yes"
|
||||
assert "UID" not in parts
|
||||
assert "PWD" not in parts
|
||||
|
||||
|
||||
def test_mssql_both_credentials_emits_uid_and_pwd() -> None:
|
||||
fake = _FakePyodbc()
|
||||
with patch("wren.model.data_source.pyodbc", fake):
|
||||
DataSourceExtension._connect_mssql_pyodbc(
|
||||
host="h",
|
||||
port="1433",
|
||||
database="db",
|
||||
user="alice",
|
||||
password="secret",
|
||||
driver="ODBC Driver 18 for SQL Server",
|
||||
)
|
||||
|
||||
parts = _parse_conn_str(fake.connect.call_args)
|
||||
assert parts["UID"] == "alice"
|
||||
assert parts["PWD"] == "secret"
|
||||
assert "Trusted_Connection" not in parts
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 3. statement_timeout validated before connect()
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_mssql_invalid_statement_timeout_does_not_leak_connection() -> None:
|
||||
fake = _FakePyodbc()
|
||||
with patch("wren.model.data_source.pyodbc", fake):
|
||||
with pytest.raises(WrenError) as exc:
|
||||
DataSourceExtension._connect_mssql_pyodbc(
|
||||
host="h",
|
||||
port="1433",
|
||||
database="db",
|
||||
user="alice",
|
||||
password="secret",
|
||||
driver="ODBC Driver 18 for SQL Server",
|
||||
kwargs={"statement_timeout": "not-a-number"},
|
||||
)
|
||||
|
||||
assert exc.value.error_code == ErrorCode.INVALID_CONNECTION_INFO
|
||||
# The crucial assertion: connect() must not be called when the
|
||||
# timeout is invalid, otherwise the connection would leak.
|
||||
fake.connect.assert_not_called()
|
||||
|
||||
|
||||
def test_mssql_valid_statement_timeout_is_applied() -> None:
|
||||
fake = _FakePyodbc()
|
||||
conn = MagicMock()
|
||||
conn.timeout = 0
|
||||
fake.connect.return_value = conn
|
||||
|
||||
with patch("wren.model.data_source.pyodbc", fake):
|
||||
result = DataSourceExtension._connect_mssql_pyodbc(
|
||||
host="h",
|
||||
port="1433",
|
||||
database="db",
|
||||
user="alice",
|
||||
password="secret",
|
||||
driver="ODBC Driver 18 for SQL Server",
|
||||
kwargs={"statement_timeout": "42"},
|
||||
)
|
||||
|
||||
fake.connect.assert_called_once()
|
||||
assert result.timeout == 42
|
||||
parts = _parse_conn_str(fake.connect.call_args)
|
||||
# statement_timeout is popped, never sent to the ODBC string
|
||||
assert "statement_timeout" not in parts
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 4. TINYINT maps unconditionally to uint8
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_mssql_tinyint_maps_to_uint8_regardless_of_sample_sign() -> None:
|
||||
"""SQL Server TINYINT is unsigned (0..255); the Arrow type must be
|
||||
``uint8`` regardless of the sampled values (which can never legitimately
|
||||
be negative, but the helper must not branch on sign)."""
|
||||
# internal_size == 1 is what pyodbc reports for TINYINT columns.
|
||||
column_desc = ("c_tinyint", int, None, 1, 3, 0, True)
|
||||
|
||||
# Non-negative sample → uint8.
|
||||
assert MSSqlConnector._mssql_arrow_type(column_desc, [0, 255]) == pa.uint8()
|
||||
# All-None sample → still uint8 (driver-declared internal_size wins).
|
||||
assert MSSqlConnector._mssql_arrow_type(column_desc, [None, None]) == pa.uint8()
|
||||
|
||||
|
||||
def test_mssql_tinyint_round_trips_through_build_column() -> None:
|
||||
"""A TINYINT column with 0 and 255 must survive the column build path."""
|
||||
arr = MSSqlConnector._build_mssql_column([0, 255, None], pa.uint8())
|
||||
assert arr.type == pa.uint8()
|
||||
assert arr.to_pylist() == [0, 255, None]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 5. datetimeoffset payload length validation
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_mssql_decode_datetimeoffset_rejects_truncated_payload() -> None:
|
||||
"""A short DATETIMEOFFSET payload must raise a clear error, not the
|
||||
cryptic ``month must be in 1..12`` that bubbles up from datetime()."""
|
||||
truncated = b"\x00" * 10
|
||||
with pytest.raises(ValueError) as exc:
|
||||
DataSourceExtension._decode_mssql_datetimeoffset(truncated)
|
||||
msg = str(exc.value)
|
||||
assert "datetimeoffset" in msg.lower()
|
||||
assert "20" in msg
|
||||
assert "10" in msg
|
||||
|
||||
|
||||
def test_mssql_decode_datetimeoffset_accepts_none() -> None:
|
||||
"""``None`` continues to pass through (NULL values from pyodbc)."""
|
||||
assert DataSourceExtension._decode_mssql_datetimeoffset(None) is None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 6. Duplicate column names survive ``query()``
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_mssql_query_preserves_duplicate_column_names() -> None:
|
||||
"""``SELECT a, a`` must yield a two-column Arrow table, not one column.
|
||||
|
||||
Earlier the result was built via ``dict(zip(names, arrays))`` which
|
||||
silently collapsed duplicate names. Build the table from arrays + schema
|
||||
directly to keep each projection.
|
||||
"""
|
||||
cursor = MagicMock()
|
||||
# internal_size 4 → int32. Two columns both named ``a`` with distinct values.
|
||||
cursor.description = [
|
||||
("a", int, None, 4, 10, 0, True),
|
||||
("a", int, None, 4, 10, 0, True),
|
||||
]
|
||||
cursor.fetchall.return_value = [(1, 2), (3, 4)]
|
||||
cursor.fetchmany.return_value = [(1, 2), (3, 4)]
|
||||
|
||||
fake_conn = MagicMock()
|
||||
fake_conn.cursor.return_value = cursor
|
||||
|
||||
connector = MSSqlConnector.__new__(MSSqlConnector)
|
||||
connector.connection = fake_conn
|
||||
|
||||
table = connector.query("SELECT a, a FROM t")
|
||||
|
||||
assert table.num_columns == 2
|
||||
assert table.column_names == ["a", "a"]
|
||||
assert table.column(0).to_pylist() == [1, 3]
|
||||
assert table.column(1).to_pylist() == [2, 4]
|
||||
@@ -0,0 +1,204 @@
|
||||
"""Unit tests for native MySQL connector helpers.
|
||||
|
||||
These tests cover the pure helpers in ``wren.connector.mysql`` — limit
|
||||
sanitisation, SQL composition, decimal type derivation — without requiring
|
||||
a live MySQL server.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pyarrow as pa
|
||||
import pytest
|
||||
|
||||
from wren.connector.mysql import (
|
||||
_apply_limit,
|
||||
_arrow_decimal_from_mysql_field,
|
||||
_build_mysql_column,
|
||||
_coerce_limit,
|
||||
_mysql_blob_codes,
|
||||
_mysql_decimal_codes,
|
||||
_mysql_field_type_map,
|
||||
_mysql_string_codes,
|
||||
_mysql_unsigned_variant_map,
|
||||
)
|
||||
|
||||
pytestmark = pytest.mark.unit
|
||||
|
||||
|
||||
# ── _coerce_limit ─────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_coerce_limit_none_passthrough() -> None:
|
||||
assert _coerce_limit(None) is None
|
||||
|
||||
|
||||
def test_coerce_limit_accepts_int() -> None:
|
||||
assert _coerce_limit(10) == 10
|
||||
|
||||
|
||||
def test_coerce_limit_accepts_numeric_string() -> None:
|
||||
# ``int()`` accepts numeric strings — keep that contract.
|
||||
assert _coerce_limit("25") == 25
|
||||
|
||||
|
||||
def test_coerce_limit_rejects_injection_string() -> None:
|
||||
"""A crafted limit value must not survive ``int()`` coercion."""
|
||||
with pytest.raises(ValueError):
|
||||
_coerce_limit("1; DROP TABLE foo")
|
||||
|
||||
|
||||
def test_coerce_limit_rejects_negative() -> None:
|
||||
with pytest.raises(ValueError):
|
||||
_coerce_limit(-1)
|
||||
|
||||
|
||||
# ── _apply_limit ──────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_apply_limit_appends_clause() -> None:
|
||||
out = _apply_limit("SELECT a FROM t", 5)
|
||||
assert out.endswith("LIMIT 5")
|
||||
assert "SELECT a FROM t" in out
|
||||
|
||||
|
||||
def test_apply_limit_strips_trailing_semicolon() -> None:
|
||||
out = _apply_limit("SELECT a FROM t;", 3)
|
||||
assert "; " not in out
|
||||
assert out.endswith("LIMIT 3")
|
||||
assert ";" not in out.split("LIMIT")[0]
|
||||
|
||||
|
||||
def test_apply_limit_zero() -> None:
|
||||
out = _apply_limit("SELECT a FROM t", 0)
|
||||
assert out.endswith("LIMIT 0")
|
||||
|
||||
|
||||
# ── _arrow_decimal_from_mysql_field ──────────────────────────────────────
|
||||
|
||||
|
||||
def test_decimal_type_passthrough() -> None:
|
||||
# DECIMAL(12, 4) signed → MySQLdb description length = 12 + 1 (sign) + 1
|
||||
# (decimal point) = 14.
|
||||
t = _arrow_decimal_from_mysql_field(14, 4, is_unsigned=False)
|
||||
assert pa.types.is_decimal(t)
|
||||
assert t.precision == 12
|
||||
assert t.scale == 4
|
||||
|
||||
|
||||
def test_decimal_type_unsigned_recovers_precision() -> None:
|
||||
# DECIMAL(12, 4) UNSIGNED → length = 12 + 0 (no sign) + 1 (point) = 13.
|
||||
t = _arrow_decimal_from_mysql_field(13, 4, is_unsigned=True)
|
||||
assert t.precision == 12
|
||||
assert t.scale == 4
|
||||
|
||||
|
||||
def test_decimal_type_zero_scale() -> None:
|
||||
# DECIMAL(10, 0) signed → length = 10 + 1 (sign) + 0 (no point) = 11.
|
||||
t = _arrow_decimal_from_mysql_field(11, 0, is_unsigned=False)
|
||||
assert t.precision == 10
|
||||
assert t.scale == 0
|
||||
|
||||
|
||||
def test_decimal_type_high_scale() -> None:
|
||||
"""MySQL allows scale up to 30 — we must not clamp below that for
|
||||
precision >= 30."""
|
||||
# DECIMAL(38, 30) signed → length = 38 + 1 + 1 = 40.
|
||||
t = _arrow_decimal_from_mysql_field(40, 30, is_unsigned=False)
|
||||
assert t.precision == 38
|
||||
assert t.scale == 30
|
||||
|
||||
|
||||
def test_decimal_type_clamps_above_arrow_max_precision() -> None:
|
||||
"""MySQL precision tops at 65; Arrow decimal128 tops at 38. We clamp."""
|
||||
# DECIMAL(65, 30) signed → length = 65 + 1 + 1 = 67.
|
||||
t = _arrow_decimal_from_mysql_field(67, 30, is_unsigned=False)
|
||||
assert t.precision == 38
|
||||
assert t.scale == 30
|
||||
|
||||
|
||||
def test_decimal_type_none_uses_fallback() -> None:
|
||||
t = _arrow_decimal_from_mysql_field(None, None)
|
||||
assert t.precision == 38
|
||||
assert t.scale == 9
|
||||
|
||||
|
||||
def test_decimal_type_scale_not_greater_than_precision() -> None:
|
||||
# Pathological case: length implies tiny precision but scale is huge.
|
||||
# Result must keep scale <= precision so PyArrow accepts the type.
|
||||
t = _arrow_decimal_from_mysql_field(7, 30, is_unsigned=False)
|
||||
assert t.scale <= t.precision
|
||||
|
||||
|
||||
# ── TIME → duration round-trip ────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_time_column_preserves_negative_and_over_24h() -> None:
|
||||
"""MySQL ``TIME`` ranges ``-838:59:59`` to ``838:59:59`` and can be
|
||||
negative. The Arrow type must be ``duration("us")``, not ``time64("us")``
|
||||
(which only accepts 0-24h positive values), and the conversion must
|
||||
preserve the sign and magnitude of MySQLdb's ``datetime.timedelta``
|
||||
values.
|
||||
"""
|
||||
import datetime # noqa: PLC0415
|
||||
|
||||
values = [
|
||||
datetime.timedelta(hours=-100),
|
||||
datetime.timedelta(0),
|
||||
datetime.timedelta(hours=838, minutes=59, seconds=59),
|
||||
-datetime.timedelta(hours=838, minutes=59, seconds=59),
|
||||
None,
|
||||
]
|
||||
arr = _build_mysql_column(values, pa.duration("us"))
|
||||
assert pa.types.is_duration(arr.type)
|
||||
out = arr.to_pylist()
|
||||
assert out[0] == datetime.timedelta(hours=-100)
|
||||
assert out[1] == datetime.timedelta(0)
|
||||
assert out[2] == datetime.timedelta(hours=838, minutes=59, seconds=59)
|
||||
assert out[3] == -datetime.timedelta(hours=838, minutes=59, seconds=59)
|
||||
assert out[4] is None
|
||||
|
||||
|
||||
# ── Thread-safe lazy init ────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_lazy_init_thread_safe() -> None:
|
||||
"""The cached FIELD_TYPE accessors must publish fully-populated results
|
||||
even when many threads hit them concurrently on a cold cache.
|
||||
|
||||
The previous in-place dict/set mutation pattern could expose a partially
|
||||
populated map to a thread that raced the initializer. ``functools.cache``
|
||||
guarantees the initializer body runs to completion before the result is
|
||||
visible to any caller.
|
||||
"""
|
||||
pytest.importorskip("MySQLdb")
|
||||
from concurrent.futures import ThreadPoolExecutor # noqa: PLC0415
|
||||
|
||||
accessors = (
|
||||
_mysql_field_type_map,
|
||||
_mysql_unsigned_variant_map,
|
||||
_mysql_blob_codes,
|
||||
_mysql_string_codes,
|
||||
_mysql_decimal_codes,
|
||||
)
|
||||
for fn in accessors:
|
||||
fn.cache_clear()
|
||||
|
||||
# Capture the expected fully-populated reference values once, single-
|
||||
# threaded, so the assertions below have a definitive ground truth.
|
||||
expected = {fn: fn() for fn in accessors}
|
||||
for fn in accessors:
|
||||
fn.cache_clear()
|
||||
|
||||
def hit_all() -> tuple:
|
||||
return tuple(fn() for fn in accessors)
|
||||
|
||||
with ThreadPoolExecutor(max_workers=16) as ex:
|
||||
results = list(ex.map(lambda _: hit_all(), range(64)))
|
||||
|
||||
for row in results:
|
||||
for fn, got in zip(accessors, row, strict=True):
|
||||
# Every thread sees the same fully-populated object.
|
||||
assert got == expected[fn]
|
||||
# Sanity: the field-type map is non-empty (MySQLdb constants exist).
|
||||
if fn is _mysql_field_type_map:
|
||||
assert len(got) > 0
|
||||
Generated
+1841
-1872
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user