refactor(wren): drop ibis for canner/postgres/mysql/mssql/trino/clickhouse/athena (combined) (#2313)

Co-authored-by: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
Jax Liu
2026-05-21 14:51:23 +08:00
committed by GitHub
parent 2ae4eeae62
commit 6f671243c2
26 changed files with 7393 additions and 2284 deletions
+10 -1
View File
@@ -57,11 +57,20 @@ test-postgres:
uv run pytest tests/connectors/test_postgres.py -v -m postgres
test-mysql:
uv run pytest tests/connectors/test_mysql.py -v -m mysql
uv run pytest tests/connectors/test_mysql.py tests/connectors/test_mysql_connector.py -v -m mysql
test-snowflake:
uv run pytest tests/connectors/test_snowflake.py -v -m snowflake
test-canner:
uv run pytest tests/connectors/test_canner.py -v -m canner
test-clickhouse:
uv run pytest tests/connectors/test_clickhouse.py -v -m clickhouse
test-trino:
uv run pytest tests/connectors/test_trino.py -v -m trino
test-connector marker:
uv run pytest tests/connectors/ -v -m {{ marker }}
+6 -6
View File
@@ -43,17 +43,17 @@ dependencies = [
]
[project.optional-dependencies]
postgres = ["psycopg>=3", "ibis-framework[postgres]"]
mysql = ["mysqlclient>=2.2", "ibis-framework[mysql]"]
postgres = ["psycopg[binary]>=3"]
mysql = ["mysqlclient>=2.2"]
bigquery = ["ibis-framework[bigquery]", "google-auth"]
snowflake = ["snowflake-connector-python[pandas]>=3.10"]
clickhouse = ["ibis-framework[clickhouse]"]
trino = ["ibis-framework[trino]", "trino>=0.321"]
mssql = ["ibis-framework[mssql]"]
clickhouse = ["clickhouse-connect>=0.8"]
trino = ["trino>=0.333,<1"]
mssql = ["pyodbc>=5,<6"]
databricks = ["databricks-sql-connector", "databricks-sdk"]
redshift = ["redshift_connector"]
spark = ["pyspark>=3.5"]
athena = ["ibis-framework[athena]"]
athena = ["pyathena[pandas]>=3"]
oracle = ["oracledb>=2"]
memory = ["lancedb>=0.6", "sentence-transformers>=2.2"]
interactive = ["InquirerPy>=0.3.4"]
+338
View File
@@ -0,0 +1,338 @@
"""Native pyathena connector — bypasses the ibis athena backend.
Athena's wire types are Trino-flavoured (varchar, decimal(p,s), array<T>,
row(...), map<K,V>, ...). This module parses those type strings via sqlglot
and materialises cursor results into PyArrow tables directly, so we no longer
depend on ibis-framework[athena].
"""
from __future__ import annotations
import contextlib
import datetime as dtlib
import json
from decimal import Decimal as PyDecimal
from typing import Any
import pyarrow as pa
from wren.connector.base import ConnectorABC
from wren.model.error import DIALECT_SQL, ErrorCode, ErrorPhase, WrenError
# Athena's DB-API cursor returns Trino-style type names. We delegate the
# lexing to sqlglot so we get nested type support (array<row<a int, b varchar>>,
# decimal(p, s), map<K, V>, etc.) for free.
_TRINO_DATA_TYPE_TO_ARROW: dict = {}
def _init_trino_data_type_map() -> None:
if _TRINO_DATA_TYPE_TO_ARROW:
return
from sqlglot.expressions import DataType # noqa: PLC0415
T = DataType.Type
_TRINO_DATA_TYPE_TO_ARROW.update(
{
T.BOOLEAN: pa.bool_(),
T.TINYINT: pa.int8(),
T.SMALLINT: pa.int16(),
T.INT: pa.int32(),
T.BIGINT: pa.int64(),
T.FLOAT: pa.float32(),
T.DOUBLE: pa.float64(),
T.VARCHAR: pa.string(),
T.CHAR: pa.string(),
T.NCHAR: pa.string(),
T.NVARCHAR: pa.string(),
T.TEXT: pa.string(),
T.JSON: pa.string(),
T.UUID: pa.string(),
T.IPADDRESS: pa.string(),
T.HLLSKETCH: pa.string(), # hyperloglog
T.GEOMETRY: pa.string(),
T.VARBINARY: pa.binary(),
T.BINARY: pa.binary(),
T.DATE: pa.date32(),
T.TIME: pa.time64("us"),
T.TIMETZ: pa.time64("us"),
T.TIMESTAMP: pa.timestamp("ms"),
T.TIMESTAMPTZ: pa.timestamp("ms", tz="UTC"),
T.TIMESTAMPLTZ: pa.timestamp("ms", tz="UTC"),
}
)
def _parse_athena_type(type_str: str | None) -> pa.DataType:
"""Parse an Athena/Trino cursor type string into a PyArrow type."""
if not type_str:
return pa.string()
from sqlglot import parse_one # noqa: PLC0415
from sqlglot.expressions import DataType # noqa: PLC0415
try:
parsed = parse_one(type_str, into=DataType, dialect="trino")
except Exception:
return pa.string()
if parsed is None:
return pa.string()
return _trino_data_type_to_arrow(parsed)
def _trino_data_type_to_arrow(node) -> pa.DataType:
from sqlglot.expressions import ColumnDef, DataType # noqa: PLC0415
_init_trino_data_type_map()
if not isinstance(node, DataType):
return pa.string()
kind = node.this
T = DataType.Type
if kind in _TRINO_DATA_TYPE_TO_ARROW:
return _TRINO_DATA_TYPE_TO_ARROW[kind]
if kind == T.DECIMAL:
precision, scale = 38, 9
params = node.expressions
if len(params) >= 1:
with contextlib.suppress(AttributeError, ValueError, TypeError):
precision = min(int(params[0].this.this), 38)
if len(params) >= 2:
with contextlib.suppress(AttributeError, ValueError, TypeError):
scale = min(int(params[1].this.this), precision)
return pa.decimal128(precision, scale)
if kind == T.ARRAY:
inner = node.expressions[0] if node.expressions else None
return pa.list_(_trino_data_type_to_arrow(inner) if inner else pa.string())
if kind == T.MAP:
if len(node.expressions) >= 2:
return pa.map_(
_trino_data_type_to_arrow(node.expressions[0]),
_trino_data_type_to_arrow(node.expressions[1]),
)
return pa.string()
if kind == T.STRUCT:
fields: list[pa.Field] = []
for idx, child in enumerate(node.expressions):
if isinstance(child, ColumnDef):
name = child.name or f"f{idx}"
inner = child.args.get("kind")
fields.append(
pa.field(
name,
_trino_data_type_to_arrow(inner) if inner else pa.string(),
)
)
else:
fields.append(pa.field(f"f{idx}", _trino_data_type_to_arrow(child)))
return pa.struct(fields)
return pa.string()
def _build_athena_column(values: list, arrow_type: pa.DataType) -> pa.Array:
"""Coerce pyathena cursor values into a PyArrow array of arrow_type."""
if pa.types.is_string(arrow_type):
processed: list[Any] = []
for v in values:
if v is None:
processed.append(None)
elif isinstance(v, dict | list | tuple):
processed.append(json.dumps(v, default=str))
elif isinstance(v, str):
processed.append(v)
else:
processed.append(str(v))
return pa.array(processed, type=pa.string(), from_pandas=True)
if pa.types.is_binary(arrow_type):
processed = [bytes(v) if isinstance(v, memoryview) else v for v in values]
return pa.array(processed, type=arrow_type, from_pandas=True)
if pa.types.is_decimal(arrow_type):
processed = [
None
if v is None
else (v if isinstance(v, PyDecimal) else PyDecimal(str(v)))
for v in values
]
return pa.array(processed, type=arrow_type, from_pandas=True)
if pa.types.is_timestamp(arrow_type):
processed = []
for v in values:
if v is None or isinstance(v, dtlib.datetime):
processed.append(v)
else:
try:
processed.append(dtlib.datetime.fromisoformat(str(v)))
except ValueError:
processed.append(None)
return pa.array(processed, type=arrow_type, from_pandas=True)
if pa.types.is_date(arrow_type):
processed = []
for v in values:
if v is None or isinstance(v, dtlib.date):
processed.append(v)
else:
try:
processed.append(dtlib.date.fromisoformat(str(v)))
except ValueError:
processed.append(None)
return pa.array(processed, type=arrow_type, from_pandas=True)
if pa.types.is_time(arrow_type):
processed = []
for v in values:
if v is None or isinstance(v, dtlib.time):
processed.append(v)
else:
try:
processed.append(dtlib.time.fromisoformat(str(v)))
except ValueError:
processed.append(None)
return pa.array(processed, type=arrow_type, from_pandas=True)
if pa.types.is_map(arrow_type):
# pyathena returns dicts for map columns; PyArrow's map_ wants iterables
# of (key, value) pairs.
processed = [None if v is None else list(v.items()) for v in values]
return pa.array(processed, type=arrow_type, from_pandas=True)
return pa.array(values, type=arrow_type, from_pandas=True)
def _build_athena_arrow_table(cursor) -> pa.Table:
"""Materialise a pyathena DB-API cursor into a PyArrow table."""
if cursor.description is None:
return pa.table({})
rows = cursor.fetchall()
fields = [
pa.field(col[0], _parse_athena_type(col[1]), nullable=True)
for col in cursor.description
]
schema = pa.schema(fields)
if not rows:
arrays = [pa.array([], type=field.type) for field in schema]
else:
arrays = [
_build_athena_column([row[i] for row in rows], schema.field(i).type)
for i in range(len(fields))
]
return pa.table(
dict(zip([f.name for f in fields], arrays, strict=False)),
schema=schema,
)
def _build_connect_kwargs(connection_info) -> dict[str, Any]:
"""Translate AthenaConnectionInfo into pyathena.connect() kwargs.
Resolves credentials in priority order:
1. Web Identity Token (OIDC) → STS AssumeRoleWithWebIdentity
2. Explicit aws_access_key_id / aws_secret_access_key (+ optional session token)
3. Default AWS credential provider chain (env, profile, instance role, …)
"""
import boto3 # noqa: PLC0415
kwargs: dict[str, Any] = {
"s3_staging_dir": connection_info.s3_staging_dir.get_secret_value(),
}
if getattr(connection_info, "region_name", None):
kwargs["region_name"] = connection_info.region_name
if getattr(connection_info, "schema_name", None):
kwargs["schema_name"] = connection_info.schema_name
web_identity_token = getattr(connection_info, "web_identity_token", None)
role_arn = getattr(connection_info, "role_arn", None)
access_key = getattr(connection_info, "aws_access_key_id", None)
secret_key = getattr(connection_info, "aws_secret_access_key", None)
if web_identity_token and role_arn:
session_name = (
getattr(connection_info, "role_session_name", None) or "wren-oidc-session"
)
sts = boto3.client(
"sts", region_name=getattr(connection_info, "region_name", None)
)
resp = sts.assume_role_with_web_identity(
RoleArn=role_arn.get_secret_value(),
RoleSessionName=session_name,
WebIdentityToken=web_identity_token.get_secret_value(),
)
creds = resp["Credentials"]
kwargs["aws_access_key_id"] = creds["AccessKeyId"]
kwargs["aws_secret_access_key"] = creds["SecretAccessKey"]
kwargs["aws_session_token"] = creds["SessionToken"]
elif access_key and secret_key:
kwargs["aws_access_key_id"] = access_key.get_secret_value()
kwargs["aws_secret_access_key"] = secret_key.get_secret_value()
session_token = getattr(connection_info, "aws_session_token", None)
if session_token:
kwargs["aws_session_token"] = session_token.get_secret_value()
# else: fall back to the boto3 default credential chain
user_kwargs = getattr(connection_info, "kwargs", None)
if user_kwargs:
kwargs.update(user_kwargs)
kwargs.setdefault("kill_on_interrupt", True)
return kwargs
class AthenaConnector(ConnectorABC):
def __init__(self, connection_info):
from pyathena import connect # noqa: PLC0415
self.connection = connect(**_build_connect_kwargs(connection_info))
def query(self, sql: str, limit: int | None = None) -> pa.Table:
try:
with contextlib.closing(self.connection.cursor()) as cursor:
cursor.execute(sql)
table = _build_athena_arrow_table(cursor)
if limit is not None:
table = table.slice(0, limit)
return table
except (WrenError, TimeoutError):
raise
except Exception as e:
raise WrenError(
ErrorCode.INVALID_SQL,
str(e),
phase=ErrorPhase.SQL_EXECUTION,
metadata={DIALECT_SQL: sql},
) from e
def dry_run(self, sql: str) -> None:
try:
with contextlib.closing(self.connection.cursor()) as cursor:
cursor.execute(f"EXPLAIN {sql}")
except (WrenError, TimeoutError):
raise
except Exception as e:
raise WrenError(
ErrorCode.INVALID_SQL,
str(e),
phase=ErrorPhase.SQL_DRY_RUN,
metadata={DIALECT_SQL: sql},
) from e
def close(self) -> None:
if self.connection is None:
return
try:
self.connection.close()
except Exception:
pass
finally:
self.connection = None
def create_connector(connection_info) -> AthenaConnector:
return AthenaConnector(connection_info)
+11 -3
View File
@@ -1,13 +1,16 @@
from __future__ import annotations
from abc import ABC, abstractmethod
from typing import TYPE_CHECKING
import pyarrow as pa
from ibis.expr.datatypes import Decimal
from ibis.expr.datatypes.core import UUID
from ibis.expr.types import Table
from loguru import logger
from wren.model.data_source import DataSource
if TYPE_CHECKING:
from ibis.expr.types import Table
class ConnectorABC(ABC):
@abstractmethod
@@ -37,6 +40,9 @@ class IbisConnector(ConnectorABC):
return ibis_table.to_pyarrow()
def _handle_pyarrow_unsupported_type(self, ibis_table: Table, **kwargs) -> Table:
from ibis.expr.datatypes import Decimal # noqa: PLC0415
from ibis.expr.datatypes.core import UUID # noqa: PLC0415
result_table = ibis_table
for name, dtype in ibis_table.schema().items():
if isinstance(dtype, Decimal):
@@ -55,6 +61,8 @@ class IbisConnector(ConnectorABC):
def _round_decimal_columns(
self, result_table: Table, col_name: str, scale: int = 9
) -> Table:
from ibis.expr.datatypes import Decimal # noqa: PLC0415
col = result_table[col_name]
decimal_type = Decimal(precision=38, scale=scale)
rounded_col = col.cast(decimal_type).round(scale)
+274 -51
View File
@@ -1,78 +1,301 @@
from contextlib import closing
from functools import cache
"""Canner Enterprise connector using the Postgres wire protocol via psycopg.
Canner Enterprise exposes a Postgres-compatible endpoint, so we connect with
``psycopg`` directly and build Arrow tables from the cursor description instead
of going through ibis. The OID map below covers the types canner emits for
Trino-flavoured queries (VARCHAR, DECIMAL, ROW/ARRAY/MAP serialised through the
postgres wire).
"""
from __future__ import annotations
import json
import re
from decimal import ROUND_HALF_EVEN
from decimal import Decimal as PyDecimal
from typing import Any
import ibis
import ibis.expr.schema as sch
import pyarrow as pa
from ibis import BaseBackend
from ibis.backends.sql.compilers.postgres import compiler as postgres_compiler
from ibis.expr.datatypes import Decimal
from ibis.expr.datatypes.core import UUID
from ibis.expr.types import Table
from loguru import logger
from wren.connector.base import ConnectorABC
from wren.model.data_source import DataSource
from wren.model.error import DIALECT_SQL, ErrorCode, ErrorPhase, WrenError
# Postgres OID → Arrow type. Canner publishes Trino-style values over the
# Postgres wire, so VARCHAR/CHAR map to string, DECIMAL to decimal128,
# BIGINT/INT/SMALLINT to int, BOOLEAN to bool, DATE/TIMESTAMP/TIMESTAMP_TZ to
# Arrow date/timestamp, and complex types (ROW/ARRAY/MAP) come back as JSON
# strings that we expose as Arrow strings.
_PG_OID_TO_ARROW: dict[int, pa.DataType] = {
16: pa.bool_(),
17: pa.binary(),
18: pa.string(),
19: pa.string(),
20: pa.int64(),
21: pa.int16(),
23: pa.int32(),
25: pa.string(),
26: pa.int64(),
114: pa.string(),
142: pa.string(),
700: pa.float32(),
701: pa.float64(),
650: pa.string(),
774: pa.string(),
790: pa.string(),
829: pa.string(),
869: pa.string(),
1040: pa.string(),
1042: pa.string(),
1043: pa.string(),
1082: pa.date32(),
1083: pa.time64("us"),
1114: pa.timestamp("us"),
1184: pa.timestamp("us", tz="UTC"),
1186: pa.duration("us"),
2950: pa.string(),
3802: pa.string(),
1266: pa.string(),
3614: pa.string(),
3615: pa.string(),
3904: pa.string(),
3906: pa.string(),
3908: pa.string(),
3910: pa.string(),
3912: pa.string(),
3926: pa.string(),
199: pa.list_(pa.string()),
1000: pa.list_(pa.bool_()),
1003: pa.list_(pa.string()),
1005: pa.list_(pa.int16()),
1007: pa.list_(pa.int32()),
1009: pa.list_(pa.string()),
1014: pa.list_(pa.string()),
1015: pa.list_(pa.string()),
1016: pa.list_(pa.int64()),
1021: pa.list_(pa.float32()),
1022: pa.list_(pa.float64()),
1028: pa.list_(pa.string()),
1041: pa.list_(pa.string()),
1115: pa.list_(pa.timestamp("us")),
1182: pa.list_(pa.string()),
1183: pa.list_(pa.string()),
1185: pa.list_(pa.timestamp("us", tz="UTC")),
1187: pa.list_(pa.string()),
1270: pa.list_(pa.string()),
2951: pa.list_(pa.string()),
3807: pa.list_(pa.string()),
}
@cache
def _get_pg_type_names(connection: BaseBackend) -> dict[int, str]:
with closing(connection.raw_sql("SELECT oid, typname FROM pg_type")) as cur:
return dict(cur.fetchall())
def _decimal_type(column) -> pa.DataType | None:
"""Pick the narrowest decimal128 that fits a NUMERIC column.
Returns ``None`` when the column has no explicit typmod (``scale is
None``) — the caller falls back to ``pa.string()`` so that high-precision
values round-trip without silent rounding via ``Decimal.quantize``.
"""
if column.scale is None:
return None
scale = max(0, min(column.scale, 38))
precision = column.precision if column.precision is not None else 38
if precision <= 0 or precision > 38:
precision = 38
precision = max(precision, scale, 1)
precision = min(precision, 38)
return pa.decimal128(precision, scale)
def _arrow_type(column) -> pa.DataType:
if column.type_code == 1700:
# Unconstrained NUMERIC (no typmod) falls back to string to preserve
# the exact textual representation — quantising to an arbitrary scale
# would silently round high-precision values.
return _decimal_type(column) or pa.string()
if column.type_code == 1231:
inner = _decimal_type(column)
return pa.list_(inner) if inner is not None else pa.list_(pa.string())
return _PG_OID_TO_ARROW.get(column.type_code, pa.string())
_TRAILING_SEMICOLONS_RE = re.compile(r"[;\s]+\Z")
def _strip_trailing_semicolon(sql: str) -> str:
"""Strip any trailing ``;`` characters and surrounding whitespace.
Wrapping user SQL as ``SELECT * FROM ({sql}) AS _t LIMIT N`` breaks when
``sql`` ends in a semicolon — Postgres/Canner reject ``SELECT 1;`` inside
a subquery. We only strip the *terminating* run of semicolons/whitespace,
so semicolons inside string literals (e.g. ``SELECT 'a;b' FROM t``) are
preserved.
"""
return _TRAILING_SEMICOLONS_RE.sub("", sql)
def _coerce_decimal(value, target_type: pa.DataType):
if value is None or not isinstance(value, PyDecimal):
return value
quantize_value = PyDecimal(f"1E-{target_type.scale}")
try:
return value.quantize(quantize_value, rounding=ROUND_HALF_EVEN)
except Exception:
return value
def _build_column(
values: list, arrow_type: pa.DataType, pg_type_oid: int | None = None
) -> pa.Array:
if arrow_type == pa.string():
processed: list[Any] = []
for value in values:
if value is None:
# SQL NULL stays as Python None regardless of source oid; the
# caller is responsible for distinguishing a JSON ``null``
# literal from a SQL NULL.
processed.append(None)
elif isinstance(value, dict | list):
processed.append(json.dumps(value, default=str))
elif not isinstance(value, str):
processed.append(str(value))
else:
processed.append(value)
return pa.array(processed, type=pa.string(), from_pandas=True)
if pa.types.is_binary(arrow_type):
processed = [
bytes(value) if isinstance(value, memoryview) else value for value in values
]
return pa.array(processed, type=pa.binary(), from_pandas=True)
if pa.types.is_decimal(arrow_type):
processed = [_coerce_decimal(value, arrow_type) for value in values]
return pa.array(processed, type=arrow_type, from_pandas=True)
if pa.types.is_list(arrow_type) and pa.types.is_string(arrow_type.value_type):
processed = []
for value in values:
if value is None:
processed.append(None)
continue
items: list[Any] = []
for item in value:
if item is None:
items.append(None)
elif isinstance(item, dict | list):
items.append(json.dumps(item, default=str))
elif isinstance(item, str):
items.append(item)
else:
items.append(str(item))
processed.append(items)
return pa.array(processed, type=arrow_type, from_pandas=True)
if pa.types.is_list(arrow_type) and pa.types.is_decimal(arrow_type.value_type):
processed = []
for value in values:
if value is None:
processed.append(None)
else:
processed.append(
[_coerce_decimal(item, arrow_type.value_type) for item in value]
)
return pa.array(processed, type=arrow_type, from_pandas=True)
return pa.array(values, type=arrow_type, from_pandas=True)
def _build_arrow_table(cursor) -> pa.Table:
"""Convert a psycopg cursor result into a PyArrow table."""
if cursor.description is None:
return pa.table({})
rows = cursor.fetchall()
fields = [
pa.field(column.name, _arrow_type(column), nullable=True)
for column in cursor.description
]
schema = pa.schema(fields)
if not rows:
arrays = [pa.array([], type=field.type) for field in schema]
else:
arrays = [
_build_column(
[row[index] for row in rows],
field.type,
cursor.description[index].type_code,
)
for index, field in enumerate(schema)
]
# Use positional construction so duplicate column names (e.g. self-joins)
# survive — dict-based construction silently drops duplicates.
return pa.Table.from_arrays(arrays, schema=schema)
class CannerConnector(ConnectorABC):
def __init__(self, connection_info):
self.connection = DataSource.canner.get_connection(connection_info)
self._closed = False
def query(self, sql: str, limit: int | None = None) -> pa.Table:
schema = self._get_schema(sql)
ibis_table = self.connection.sql(sql, schema=schema)
import psycopg # noqa: PLC0415
if limit is not None:
ibis_table = ibis_table.limit(limit)
ibis_table = self._handle_pyarrow_unsupported_type(ibis_table)
return ibis_table.to_pyarrow()
sql = (
f"SELECT * FROM ({_strip_trailing_semicolon(sql)}) AS _t LIMIT {limit}"
)
def _handle_pyarrow_unsupported_type(self, ibis_table: Table, **kwargs) -> Table:
result_table = ibis_table
for name, dtype in ibis_table.schema().items():
if isinstance(dtype, Decimal):
col = result_table[name]
decimal_type = Decimal(precision=38, scale=9)
rounded_col = col.cast(decimal_type).round(9)
result_table = result_table.mutate(**{name: rounded_col})
elif isinstance(dtype, UUID):
result_table = result_table.mutate(
**{name: result_table[name].cast("string")}
)
return result_table
try:
with self.connection.cursor() as cursor:
cursor.execute(sql)
return _build_arrow_table(cursor)
except psycopg.errors.QueryCanceled:
raise
except (WrenError, TimeoutError):
raise
except Exception as e:
raise WrenError(
ErrorCode.GENERIC_USER_ERROR,
str(e),
phase=ErrorPhase.SQL_EXECUTION,
metadata={DIALECT_SQL: sql},
) from e
def dry_run(self, sql: str) -> Any:
return self.connection.raw_sql(f"SELECT * FROM ({sql}) LIMIT 0")
def dry_run(self, sql: str) -> None:
import psycopg # noqa: PLC0415
wrapped = f"SELECT * FROM ({_strip_trailing_semicolon(sql)}) AS _t LIMIT 0"
try:
with self.connection.cursor() as cursor:
cursor.execute(wrapped)
except psycopg.errors.QueryCanceled:
raise
except (WrenError, TimeoutError):
raise
except Exception as e:
raise WrenError(
ErrorCode.GENERIC_USER_ERROR,
str(e),
phase=ErrorPhase.SQL_DRY_RUN,
metadata={DIALECT_SQL: sql},
) from e
# Explicit return to honour the ConnectorABC.dry_run() contract — the
# cursor result must not leak out of this method.
return None
def close(self) -> None:
if self._closed or not hasattr(self, "connection") or self.connection is None:
return
try:
if hasattr(self.connection, "con") and hasattr(
self.connection.con, "close"
):
self.connection.con.close()
elif hasattr(self.connection, "close"):
self.connection.close()
self.connection.close()
except Exception as e:
logger.warning(f"Error closing Canner connection: {e}")
def _get_schema(self, sql: str) -> sch.Schema:
cur = self.dry_run(sql)
type_names = _get_pg_type_names(self.connection)
return ibis.schema(
{
desc.name: postgres_compiler.type_mapper.from_string(
type_names[desc.type_code]
)
for desc in cur.description
}
)
finally:
self._closed = True
self.connection = None
def create_connector(connection_info) -> CannerConnector:
+383
View File
@@ -0,0 +1,383 @@
"""Native ClickHouse connector built on ``clickhouse-connect``.
ClickHouse types are returned by the driver as descriptor strings such as
``Nullable(Decimal(18, 4))`` or ``Array(LowCardinality(String))``. We parse
those into a sqlglot ``DataType`` AST and walk it to construct a matching
PyArrow schema. Values from ``QueryResult.result_rows`` are then coerced
column-by-column to that schema.
"""
from __future__ import annotations
import json
import re
from decimal import Decimal as PyDecimal
from typing import Any
from urllib.parse import parse_qsl, unquote_plus, urlparse
import pyarrow as pa
import sqlglot
import sqlglot.errors
from loguru import logger
from sqlglot.expressions import DataType
from wren.connector.base import ConnectorABC
from wren.model.error import (
DIALECT_SQL,
DatabaseTimeoutError,
ErrorCode,
ErrorPhase,
WrenError,
)
try:
import clickhouse_connect
_ClickHouseDbError = clickhouse_connect.driver.exceptions.DatabaseError
except ImportError: # pragma: no cover - optional dependency
class _ClickHouseDbError(Exception):
pass
# --------------------------------------------------------------------------
# Type parsing — ClickHouse type-string → PyArrow DataType
# --------------------------------------------------------------------------
def _parse_clickhouse_type(type_str: str | None) -> pa.DataType:
"""Map a ClickHouse type-name string to an Arrow type via sqlglot.
sqlglot's ClickHouse dialect strips ``Nullable(...)`` during parsing
(the inner ``DataType`` is hoisted to the top), so we only need to peel
``LowCardinality(...)`` ourselves.
"""
if type_str is None:
return pa.string()
try:
parsed = sqlglot.parse_one(type_str, into=DataType, dialect="clickhouse")
except sqlglot.errors.ParseError:
logger.warning(f"Failed to parse ClickHouse type string: {type_str}")
return pa.string()
if parsed is None:
return pa.string()
return _clickhouse_data_type_to_arrow(parsed)
_CLICKHOUSE_DATA_TYPE_TO_ARROW: dict = {}
def _init_clickhouse_data_type_map() -> None:
if _CLICKHOUSE_DATA_TYPE_TO_ARROW:
return
T = DataType.Type
_CLICKHOUSE_DATA_TYPE_TO_ARROW.update(
{
T.BOOLEAN: pa.bool_(),
T.TINYINT: pa.int8(),
T.SMALLINT: pa.int16(),
T.INT: pa.int32(),
T.BIGINT: pa.int64(),
T.UTINYINT: pa.uint8(),
T.USMALLINT: pa.uint16(),
T.UINT: pa.uint32(),
T.UBIGINT: pa.uint64(),
# Int128 / Int256 / UInt128 / UInt256: PyArrow tops out at 64 bits,
# so surface the wide types as string to avoid silent truncation.
T.INT128: pa.string(),
T.INT256: pa.string(),
T.UINT128: pa.string(),
T.UINT256: pa.string(),
T.FLOAT: pa.float32(),
T.DOUBLE: pa.float64(),
T.TEXT: pa.string(), # ClickHouse ``String``
T.FIXEDSTRING: pa.string(),
T.UUID: pa.string(),
T.IPV4: pa.string(),
T.IPV6: pa.string(),
T.ENUM8: pa.string(),
T.ENUM16: pa.string(),
T.JSON: pa.string(),
# ``Nothing`` is ClickHouse's type for bare NULL literals. Surface
# as string — the column will be all-None either way.
T.NOTHING: pa.string(),
T.DATE: pa.date32(),
T.DATE32: pa.date32(),
}
)
def _clickhouse_data_type_to_arrow(node: Any) -> pa.DataType:
_init_clickhouse_data_type_map()
if not isinstance(node, DataType):
return pa.string()
kind = node.this
T = DataType.Type
# Peel ``LowCardinality(...)`` — purely a storage detail.
if kind == T.LOWCARDINALITY:
inner = node.expressions[0] if node.expressions else None
return _clickhouse_data_type_to_arrow(inner) if inner else pa.string()
if kind in _CLICKHOUSE_DATA_TYPE_TO_ARROW:
return _CLICKHOUSE_DATA_TYPE_TO_ARROW[kind]
if kind in (T.DECIMAL, T.DECIMAL32, T.DECIMAL64, T.DECIMAL128, T.DECIMAL256):
# Normalise every decimal to (38, 9) so downstream consumers do not
# have to special-case precision/scale per column.
return pa.decimal128(38, 9)
if kind in (T.DATETIME, T.DATETIME64):
tz = _clickhouse_extract_datetime_tz(node)
if tz:
return pa.timestamp("ns", tz=tz)
return pa.timestamp("ns")
if kind == T.ARRAY:
inner = node.expressions[0] if node.expressions else None
return pa.list_(_clickhouse_data_type_to_arrow(inner) if inner else pa.string())
if kind == T.MAP:
if len(node.expressions) >= 2:
return pa.map_(
_clickhouse_data_type_to_arrow(node.expressions[0]),
_clickhouse_data_type_to_arrow(node.expressions[1]),
)
return pa.string()
if kind == T.STRUCT:
# ``Tuple(...)`` — flatten to JSON-encoded string.
return pa.string()
return pa.string()
def _clickhouse_extract_datetime_tz(node: Any) -> str | None:
"""Pull the timezone string out of ``DateTime('tz')`` / ``DateTime64(p, 'tz')``."""
for param in node.expressions:
# ``DataTypeParam`` wraps a ``Literal``; we want the string-typed one.
inner = getattr(param, "this", None)
if inner is not None and getattr(inner, "is_string", False):
return str(inner.this)
return None
# --------------------------------------------------------------------------
# Arrow table assembly from a clickhouse-connect QueryResult
# --------------------------------------------------------------------------
def _build_clickhouse_arrow_table(query_result: Any) -> pa.Table:
"""Convert a ``clickhouse_connect`` ``QueryResult`` into a PyArrow table."""
column_names = list(query_result.column_names)
column_types = list(query_result.column_types)
rows = list(query_result.result_rows or [])
fields = [
pa.field(name, _parse_clickhouse_type(ct.name), nullable=True)
for name, ct in zip(column_names, column_types, strict=False)
]
schema = pa.schema(fields)
if not rows:
arrays = [pa.array([], type=field.type) for field in schema]
else:
arrays = [
_build_clickhouse_column([row[i] for row in rows], schema.field(i).type)
for i in range(len(fields))
]
# ``dict(zip(...))`` collapses duplicate column names — build the table
# from arrays + schema so projections like ``SELECT a, a`` are preserved.
return pa.Table.from_arrays(arrays, schema=schema)
def _build_clickhouse_column(values: list, arrow_type: pa.DataType) -> pa.Array:
"""Convert ``clickhouse_connect`` Python values into a PyArrow array."""
if pa.types.is_string(arrow_type):
processed: list[Any] = []
for v in values:
if v is None:
processed.append(None)
elif isinstance(v, dict | list | tuple):
processed.append(json.dumps(v, default=str))
elif isinstance(v, str):
processed.append(v)
elif isinstance(v, bytes):
processed.append(v.decode("utf-8", errors="replace"))
else:
processed.append(str(v))
return pa.array(processed, type=pa.string(), from_pandas=True)
if pa.types.is_decimal(arrow_type):
# Every decimal is normalised to ``decimal128(38, 9)``, so no
# per-column precision/scale narrowing is needed here.
processed = [
None
if v is None
else (v if isinstance(v, PyDecimal) else PyDecimal(str(v)))
for v in values
]
return pa.array(processed, type=arrow_type, from_pandas=True)
if pa.types.is_timestamp(arrow_type):
# ``clickhouse_connect`` returns datetime objects (naive or tz-aware).
return pa.array(values, type=arrow_type, from_pandas=True)
return pa.array(values, type=arrow_type, from_pandas=True)
# --------------------------------------------------------------------------
# Client kwargs assembly
# --------------------------------------------------------------------------
def _build_clickhouse_client_kwargs(connection_info: Any) -> dict:
"""Translate ``ClickHouseConnectionInfo`` / ``ConnectionUrl`` into
``clickhouse_connect.get_client`` kwargs."""
# URL-based connection (``ConnectionUrl``).
if hasattr(connection_info, "connection_url"):
url = connection_info.connection_url.get_secret_value()
parsed = urlparse(url)
if parsed.scheme not in {"clickhouse", "clickhouse+http", "clickhouse+https"}:
raise WrenError(
ErrorCode.INVALID_CONNECTION_INFO,
"ClickHouse connection URL must use clickhouse:// scheme",
)
kwargs: dict = dict(parse_qsl(parsed.query))
info_kwargs = getattr(connection_info, "kwargs", None)
if info_kwargs:
kwargs.update(info_kwargs)
settings: dict = (
dict(kwargs.pop("settings", {})) if "settings" in kwargs else {}
)
statement_timeout = kwargs.pop("statement_timeout", None)
if statement_timeout is not None:
settings["max_execution_time"] = int(statement_timeout)
# urlparse leaves percent-encoded characters in userinfo, so decode
# them before clickhouse-connect sees the credentials. Matches the
# mssql / postgres URL handling elsewhere in this package.
out: dict = {
"host": parsed.hostname,
"port": int(parsed.port) if parsed.port else 8123,
"username": (
unquote_plus(parsed.username) if parsed.username else "default"
),
"password": unquote_plus(parsed.password) if parsed.password else "",
"settings": settings,
}
if parsed.path and parsed.path != "/":
out["database"] = parsed.path.lstrip("/")
if parsed.scheme == "clickhouse+https":
out["secure"] = True
# ``settings`` already popped above, so ``out["settings"]`` survives.
out.update(kwargs)
return out
info = connection_info # ClickHouseConnectionInfo
settings = dict(info.settings) if info.settings else {}
kwargs = dict(info.kwargs) if info.kwargs else {}
statement_timeout = kwargs.pop("statement_timeout", None)
if statement_timeout is not None:
settings["max_execution_time"] = int(statement_timeout)
# Merge any user-supplied ``settings`` from kwargs into the local dict
# *before* applying the rest, otherwise ``out.update(kwargs)`` below
# would clobber the statement_timeout-derived max_execution_time.
extra_settings = kwargs.pop("settings", None)
if extra_settings:
settings.update(extra_settings)
out = {
"host": info.host,
"port": int(info.port),
"username": info.user,
"password": info.password.get_secret_value() if info.password else "",
"database": info.database,
"secure": info.secure,
"settings": settings,
}
out.update(kwargs)
return out
# --------------------------------------------------------------------------
# Connector
# --------------------------------------------------------------------------
_TRAILING_SEMICOLONS_RE = re.compile(r"[;\s]+\Z")
def _strip_trailing_semicolon(sql: str) -> str:
"""Strip the terminating run of ``;`` characters and surrounding whitespace.
Matches the canner helper of the same name. Wrapping user SQL as
``SELECT * FROM ({sql}) AS _wren_sub LIMIT N`` breaks when ``sql`` ends
in a semicolon — ClickHouse rejects ``SELECT 1;`` inside a subquery. Only
the terminating run is removed so semicolons inside string literals
(e.g. ``SELECT 'a;b'``) are preserved.
"""
return _TRAILING_SEMICOLONS_RE.sub("", sql)
class ClickHouseConnector(ConnectorABC):
"""Native ``clickhouse-connect`` connector that bypasses ``ibis-project``."""
def __init__(self, connection_info: Any):
connect_kwargs = _build_clickhouse_client_kwargs(connection_info)
self.connection = clickhouse_connect.get_client(**connect_kwargs)
self._closed = False
def query(self, sql: str, limit: int | None = None) -> pa.Table:
# Strip the terminating run of ``;`` / whitespace before wrapping —
# ``SELECT * FROM (SELECT 1;) AS _wren_sub LIMIT N`` is invalid SQL.
# Semicolons inside string literals are preserved.
stripped = _strip_trailing_semicolon(sql)
statement = stripped
if limit is not None:
statement = f"SELECT * FROM ({stripped}) AS _wren_sub LIMIT {limit}"
try:
result = self.connection.query(statement)
except _ClickHouseDbError as e:
if "TIMEOUT_EXCEEDED" in str(e):
raise DatabaseTimeoutError(str(e)) from e
raise WrenError(
ErrorCode.INVALID_SQL,
str(e),
phase=ErrorPhase.SQL_EXECUTION,
metadata={DIALECT_SQL: sql},
) from e
return _build_clickhouse_arrow_table(result)
def dry_run(self, sql: str) -> None:
stripped = _strip_trailing_semicolon(sql)
try:
self.connection.query(f"SELECT * FROM ({stripped}) AS _wren_sub LIMIT 0")
except _ClickHouseDbError as e:
if "TIMEOUT_EXCEEDED" in str(e):
raise DatabaseTimeoutError(str(e)) from e
raise WrenError(
ErrorCode.INVALID_SQL,
str(e),
phase=ErrorPhase.SQL_DRY_RUN,
metadata={DIALECT_SQL: sql},
) from e
def close(self) -> None:
if self._closed or not hasattr(self, "connection") or self.connection is None:
return
try:
self.connection.close()
except Exception as e:
logger.warning(f"Error closing ClickHouse connection: {e}")
finally:
self._closed = True
self.connection = None
def create_connector(connection_info: Any) -> ClickHouseConnector:
return ClickHouseConnector(connection_info)
+3 -5
View File
@@ -19,11 +19,11 @@ _REGISTRY: dict[DataSource, str] = {
DataSource.redshift: "wren.connector.redshift",
DataSource.spark: "wren.connector.spark",
DataSource.databricks: "wren.connector.databricks",
DataSource.trino: "wren.connector.ibis",
DataSource.clickhouse: "wren.connector.ibis",
DataSource.trino: "wren.connector.trino",
DataSource.clickhouse: "wren.connector.clickhouse",
DataSource.oracle: "wren.connector.oracle",
DataSource.snowflake: "wren.connector.snowflake",
DataSource.athena: "wren.connector.ibis",
DataSource.athena: "wren.connector.athena",
}
# Map data sources to the correct pip extra when they share a connector module
@@ -40,8 +40,6 @@ _NEEDS_DATA_SOURCE = {
DataSource.mysql,
DataSource.doris,
DataSource.trino,
DataSource.clickhouse,
DataSource.athena,
}
-103
View File
@@ -1,103 +0,0 @@
"""Generic ibis-backed connectors with data-source-specific error handling."""
import pyarrow as pa
from wren.connector.base import IbisConnector
from wren.model.data_source import DataSource
from wren.model.error import DIALECT_SQL, ErrorCode, ErrorPhase, WrenError
try:
import clickhouse_connect
_ClickHouseDbError = clickhouse_connect.driver.exceptions.DatabaseError
except ImportError:
class _ClickHouseDbError(Exception):
pass
class TrinoConnector(IbisConnector):
def __init__(self, connection_info):
super().__init__(DataSource.trino, connection_info)
def query(self, sql: str, limit: int | None = None) -> pa.Table:
import trino # noqa: PLC0415
try:
return super().query(sql, limit)
except trino.exceptions.TrinoQueryError as e:
if not e.error_name == "EXCEEDED_TIME_LIMIT":
raise WrenError(
ErrorCode.INVALID_SQL,
str(e),
phase=ErrorPhase.SQL_EXECUTION,
metadata={DIALECT_SQL: sql},
) from e
raise
except (WrenError, TimeoutError):
raise
def dry_run(self, sql: str) -> None:
import trino # noqa: PLC0415
try:
super().dry_run(sql)
except trino.exceptions.TrinoQueryError as e:
if not e.error_name == "EXCEEDED_TIME_LIMIT":
raise WrenError(
ErrorCode.INVALID_SQL,
str(e),
phase=ErrorPhase.SQL_DRY_RUN,
metadata={DIALECT_SQL: sql},
) from e
raise
except (WrenError, TimeoutError):
raise
class ClickHouseConnector(IbisConnector):
def __init__(self, connection_info):
super().__init__(DataSource.clickhouse, connection_info)
def query(self, sql: str, limit: int | None = None) -> pa.Table:
try:
return super().query(sql, limit)
except _ClickHouseDbError as e:
if "TIMEOUT_EXCEEDED" not in str(e):
raise WrenError(
ErrorCode.INVALID_SQL,
str(e),
phase=ErrorPhase.SQL_EXECUTION,
metadata={DIALECT_SQL: sql},
) from e
raise
except (WrenError, TimeoutError):
raise
def dry_run(self, sql: str) -> None:
try:
super().dry_run(sql)
except _ClickHouseDbError as e:
if "TIMEOUT_EXCEEDED" not in str(e):
raise WrenError(
ErrorCode.INVALID_SQL,
str(e),
phase=ErrorPhase.SQL_DRY_RUN,
metadata={DIALECT_SQL: sql},
) from e
raise
except (WrenError, TimeoutError):
raise
_DATA_SOURCE_TO_CLASS = {
DataSource.trino: TrinoConnector,
DataSource.clickhouse: ClickHouseConnector,
}
def create_connector(data_source: DataSource, connection_info) -> IbisConnector:
cls = _DATA_SOURCE_TO_CLASS.get(data_source, IbisConnector)
if cls is IbisConnector:
return IbisConnector(data_source, connection_info)
return cls(connection_info)
+227 -47
View File
@@ -1,52 +1,111 @@
from __future__ import annotations
import datetime as dtlib
import json
import uuid
from contextlib import closing
from decimal import Decimal as PyDecimal
import pyarrow as pa
import sqlglot.expressions as sge
from ibis.expr.datatypes import Decimal
from ibis.expr.types import Table
from loguru import logger
from sqlglot import exp, parse_one
from wren.connector.base import IbisConnector
from wren.connector.base import ConnectorABC
from wren.model.data_source import DataSource
from wren.model.error import DIALECT_SQL, ErrorCode, ErrorPhase, WrenError
class MSSqlConnector(IbisConnector):
class MSSqlConnector(ConnectorABC):
"""Native pyodbc-backed MSSQL connector.
Uses a raw pyodbc cursor for execution, builds Arrow schema from
``cursor.description`` plus value sampling, and rewrites pagination via
sqlglot (tsql dialect) so that ``LIMIT n`` becomes
``OFFSET 0 ROWS FETCH NEXT n ROWS ONLY``.
"""
def __init__(self, connection_info):
super().__init__(DataSource.mssql, connection_info)
self.data_source = DataSource.mssql
self.connection = self.data_source.get_connection(connection_info)
self._closed = False
def query(self, sql: str, limit: int | None = None) -> pa.Table:
sql = self._flatten_pagination_limit(sql)
ibis_table = self.connection.sql(sql)
if limit is not None:
ibis_table = ibis_table.limit(limit)
ibis_table = self._handle_pyarrow_unsupported_type(ibis_table)
return self._round_decimal_columns(ibis_table)
with closing(self.connection.cursor()) as cursor:
cursor.execute(self._raw_cursor_sql(sql, limit))
if cursor.description is None:
return pa.table({})
def _round_decimal_columns(self, ibis_table: Table, scale: int = 9) -> pa.Table:
def round_decimal(val):
if val is None:
return None
d = PyDecimal(str(val))
return d.quantize(PyDecimal("1." + "0" * scale))
rows = cursor.fetchmany(limit) if limit is not None else cursor.fetchall()
arrow_schema = self._build_mssql_arrow_schema(cursor.description, rows)
arrays = [
self._build_mssql_column(
[row[index] for row in rows], arrow_schema.field(index).type
)
for index in range(len(cursor.description))
]
# ``dict(zip(...))`` collapses duplicate column names — build the
# table from arrays + schema so projections like ``SELECT a, a``
# are preserved.
return pa.Table.from_arrays(arrays, schema=arrow_schema)
decimal_columns = [
name
for name, dtype in ibis_table.schema().items()
if isinstance(dtype, Decimal)
]
if not decimal_columns:
return ibis_table.to_pyarrow()
def dry_run(self, sql: str) -> None:
sql = self._flatten_pagination_limit(sql)
try:
with closing(self.connection.cursor()) as cursor:
cursor.execute(self._raw_cursor_sql(sql, 0))
except Exception as e:
error_message = self._describe_sql_for_error_message(sql)
if error_message != "Unknown reason":
raise WrenError(
error_code=ErrorCode.INVALID_SQL,
message=f"The sql dry run failed. {error_message}.",
phase=ErrorPhase.SQL_DRY_RUN,
metadata={DIALECT_SQL: sql},
) from e
raise
pandas_df = ibis_table.to_pandas()
for col_name in decimal_columns:
pandas_df[col_name] = pandas_df[col_name].apply(round_decimal)
return pa.Table.from_pandas(pandas_df)
def close(self) -> None:
if self._closed or not hasattr(self, "connection") or self.connection is None:
return
try:
self.connection.close()
except Exception as e:
logger.warning(f"Error closing MSSQL connection: {e}")
finally:
self._closed = True
self.connection = None
# ------------------------------------------------------------------
# SQL rewriting
# ------------------------------------------------------------------
@staticmethod
def _raw_cursor_sql(
sql: str, limit: int | None, input_dialect: str = "tsql"
) -> str:
"""Inject a ``LIMIT n`` into a Select so sqlglot emits the tsql
``OFFSET 0 ROWS FETCH NEXT n ROWS ONLY`` clause."""
if limit is None:
return sql
try:
parsed = parse_one(sql, dialect=input_dialect)
except Exception:
return sql
if isinstance(parsed, exp.Select) and not parsed.args.get("limit"):
parsed.set("limit", exp.Limit(expression=exp.Literal.number(limit)))
return parsed.sql(dialect="tsql")
return sql
def _flatten_pagination_limit(
self, sql_query: str, input_dialect: str = "tsql"
) -> str:
"""Collapse an outer ``LIMIT`` wrapped around a single subquery into
the inner Select's ``LIMIT`` — undoes the v4 paginate-wrap pattern."""
try:
parsed = parse_one(sql_query, dialect=input_dialect)
if not isinstance(parsed, exp.Select) or not parsed.args.get("limit"):
@@ -78,29 +137,17 @@ class MSSqlConnector(IbisConnector):
except Exception:
return sql_query
def dry_run(self, sql: str) -> None:
try:
super().dry_run(sql)
except AttributeError as e:
if "NoneType" in str(e) and "lower" in str(e):
error_message = self._describe_sql_for_error_message(sql)
raise WrenError(
error_code=ErrorCode.INVALID_SQL,
message=f"The sql dry run failed. {error_message}.",
phase=ErrorPhase.SQL_DRY_RUN,
metadata={DIALECT_SQL: sql},
) from e
raise WrenError(
error_code=ErrorCode.IBIS_PROJECT_ERROR,
message=str(e),
phase=ErrorPhase.SQL_DRY_RUN,
) from e
def _describe_sql_for_error_message(self, sql: str) -> str:
"""Surface a precise error string by asking SQL Server to describe
the first result set of the failing query."""
try:
tsql = sge.convert(sql).sql("mssql")
describe_sql = f"SELECT error_message FROM sys.dm_exec_describe_first_result_set({tsql}, NULL, 0)"
with closing(self.connection.raw_sql(describe_sql)) as cur:
describe_sql = (
"SELECT error_message FROM "
f"sys.dm_exec_describe_first_result_set({tsql}, NULL, 0)"
)
with closing(self.connection.cursor()) as cur:
cur.execute(describe_sql)
rows = cur.fetchall()
if not rows:
return "Unknown reason"
@@ -108,6 +155,139 @@ class MSSqlConnector(IbisConnector):
except Exception:
return "Unknown reason"
# ------------------------------------------------------------------
# Arrow schema inference + column build
# ------------------------------------------------------------------
@staticmethod
def _build_mssql_arrow_schema(description, rows: list[tuple]) -> pa.Schema:
fields = []
for index, column in enumerate(description):
values = [row[index] for row in rows]
fields.append(
pa.field(
column[0],
MSSqlConnector._mssql_arrow_type(column, values),
nullable=True,
)
)
return pa.schema(fields)
@staticmethod
def _mssql_arrow_type(column, values: list) -> pa.DataType:
type_code = column[1] if len(column) > 1 else None
internal_size = column[3] if len(column) > 3 else None
precision = column[4] if len(column) > 4 else None
sample = next((value for value in values if value is not None), None)
if isinstance(sample, bool) or type_code is bool:
return pa.bool_()
if isinstance(sample, bytes | bytearray | memoryview) or type_code in {
bytes,
bytearray,
memoryview,
}:
return pa.binary()
if isinstance(sample, dtlib.datetime) or type_code is dtlib.datetime:
tz = MSSqlConnector._mssql_timezone_name(sample)
return pa.timestamp("ns", tz=tz)
if isinstance(sample, dtlib.date) or type_code is dtlib.date:
return pa.date32()
if isinstance(sample, dtlib.time) or type_code is dtlib.time:
return pa.time64("ns")
if isinstance(sample, float) or type_code is float:
return pa.float32() if internal_size == 4 else pa.float64()
if isinstance(sample, int) or type_code is int:
return MSSqlConnector._mssql_integer_arrow_type(
internal_size, precision, values
)
if isinstance(sample, PyDecimal) or type_code is PyDecimal:
return pa.string()
if isinstance(sample, uuid.UUID) or type_code is uuid.UUID:
return pa.string()
return pa.string()
@staticmethod
def _mssql_timezone_name(value: dtlib.datetime | None) -> str | None:
if value is None or value.tzinfo is None:
return None
offset = value.utcoffset()
if offset is None:
return None
if offset.total_seconds() == 0:
return "UTC"
total_minutes = int(offset.total_seconds() // 60)
sign = "+" if total_minutes >= 0 else "-"
hours, minutes = divmod(abs(total_minutes), 60)
return f"{sign}{hours:02d}:{minutes:02d}"
@staticmethod
def _mssql_integer_arrow_type(
internal_size: int | None, precision: int | None, values: list
) -> pa.DataType:
non_negative = all(value is None or int(value) >= 0 for value in values)
# SQL Server TINYINT is unconditionally unsigned (0..255), so map by
# the declared internal_size rather than sampling for sign.
if internal_size == 1:
return pa.uint8()
if internal_size == 2:
return pa.int16()
if internal_size == 4:
return pa.int32()
if internal_size == 8:
return pa.int64()
if precision is not None:
if precision <= 3 and non_negative:
return pa.uint8()
if precision <= 5:
return pa.int16()
if precision <= 10:
return pa.int32()
return pa.int64()
@staticmethod
def _build_mssql_column(values: list, arrow_type: pa.DataType) -> pa.Array:
if pa.types.is_integer(arrow_type):
processed = [None if value is None else int(value) for value in values]
return pa.array(processed, type=arrow_type, from_pandas=True)
if pa.types.is_floating(arrow_type):
processed = [None if value is None else float(value) for value in values]
return pa.array(processed, type=arrow_type, from_pandas=True)
if pa.types.is_boolean(arrow_type):
processed = [None if value is None else bool(value) for value in values]
return pa.array(processed, type=arrow_type, from_pandas=True)
if pa.types.is_decimal(arrow_type):
processed = [
None
if value is None
else value
if isinstance(value, PyDecimal)
else PyDecimal(str(value))
for value in values
]
return pa.array(processed, type=arrow_type, from_pandas=True)
if pa.types.is_string(arrow_type):
processed = []
for value in values:
if value is None:
processed.append(None)
elif isinstance(value, dict | list):
processed.append(json.dumps(value, default=str))
elif isinstance(value, str):
processed.append(value)
else:
processed.append(str(value))
return pa.array(processed, type=arrow_type, from_pandas=True)
return pa.array(values, type=arrow_type, from_pandas=True)
def create_connector(connection_info) -> MSSqlConnector:
return MSSqlConnector(connection_info)
+539 -43
View File
@@ -1,57 +1,553 @@
import ibis.expr.datatypes as dt
from ibis.expr.datatypes import Decimal
from ibis.expr.datatypes.core import UUID
from ibis.expr.types import Table
"""Native MySQLdb connector for MySQL and Doris.
from wren.connector.base import IbisConnector
Replaces the previous ibis-based implementation. Uses the ``mysqlclient``
(``MySQLdb``) driver directly and builds PyArrow tables from cursor
descriptions so no ibis backend is required.
Doris speaks the MySQL wire protocol and reuses the same query path; it
only differs in how the connection is opened.
"""
from __future__ import annotations
import json
from contextlib import closing
from decimal import Decimal as PyDecimal
from functools import cache
import pyarrow as pa
from loguru import logger
from wren.connector.base import ConnectorABC
from wren.model.data_source import DataSource
from wren.model.error import ErrorCode, WrenError
class MySqlConnector(IbisConnector):
def _apply_limit(sql: str, limit: int) -> str:
"""Append ``LIMIT n`` to a user-supplied SQL string.
Strips any trailing semicolon and whitespace, then appends ``LIMIT n``.
``limit`` MUST already be validated as a non-negative ``int`` by the caller
— this helper does not re-validate to keep the call site explicit.
Wrapping the user SQL in ``SELECT * FROM (...) AS _sub LIMIT n`` was
rejected because it fails with ``ER_DUP_FIELDNAME`` whenever the inner
SELECT projects two columns with the same name (e.g. a join that selects
``a.id`` and ``b.id``).
"""
return f"{sql.rstrip().rstrip(';').rstrip()}\nLIMIT {limit}"
def _coerce_limit(limit: int | None) -> int | None:
"""Validate and coerce a user-supplied ``limit`` to a non-negative ``int``.
``int(limit)`` rejects strings like ``"5 OR 1=1"`` so the value can be
safely interpolated into SQL. Negative limits are also rejected.
"""
if limit is None:
return None
coerced = int(limit)
if coerced < 0:
raise ValueError(f"limit must be non-negative, got {coerced}")
return coerced
class MySqlConnector(ConnectorABC):
"""Native MySQLdb connector that bypasses ibis-project."""
def __init__(self, connection_info):
super().__init__(DataSource.mysql, connection_info)
self._closed = False
self.connection = DataSource.mysql.get_connection(connection_info)
# Append ANSI_QUOTES to the server-configured sql_mode so identifiers
# quoted as "name" (the MDL convention) are accepted. CONCAT preserves
# the server defaults (ONLY_FULL_GROUP_BY, STRICT_TRANS_TABLES, …) —
# overwriting them would let queries silently behave differently than
# in the user's own MySQL session.
#
# If this init query fails we MUST close the connection so it isn't
# leaked; the cursor exception would otherwise leave a live socket
# held by the (now half-constructed) connector.
try:
with closing(self.connection.cursor()) as cursor:
cursor.execute("SET sql_mode=CONCAT(@@sql_mode, ',ANSI_QUOTES')")
except Exception:
try:
self.connection.close()
except Exception as close_err:
logger.warning(
f"Error closing MySQL connection after init failure: {close_err}"
)
finally:
self._closed = True
raise
def _handle_pyarrow_unsupported_type(self, ibis_table: Table, **kwargs) -> Table:
result_table = ibis_table
for name, dtype in ibis_table.schema().items():
if isinstance(dtype, Decimal):
result_table = self._round_decimal_columns(
result_table=result_table, col_name=name, **kwargs
)
elif isinstance(dtype, UUID):
result_table = self._cast_uuid_columns(
result_table=result_table, col_name=name
)
elif isinstance(dtype, dt.JSON):
result_table = result_table.mutate(
**{name: result_table[name].cast("string")}
)
return result_table
def query(self, sql: str, limit: int | None = None) -> pa.Table:
limit = _coerce_limit(limit)
if limit is not None:
sql = _apply_limit(sql, limit)
with closing(self.connection.cursor()) as cursor:
cursor.execute(sql)
return _build_mysql_arrow_table(cursor)
def dry_run(self, sql: str) -> None:
# ``EXPLAIN`` validates the SQL on the server (table lookup, column
# resolution, syntax) without executing it. Prefixing instead of
# subquery-wrapping side-steps ``ER_DUP_FIELDNAME`` for queries that
# surface duplicate column names. We strip a trailing semicolon to
# match the same compose-ability we use for ``query``'s LIMIT path.
explain_sql = f"EXPLAIN {sql.rstrip().rstrip(';').rstrip()}"
with closing(self.connection.cursor()) as cursor:
cursor.execute(explain_sql)
cursor.fetchall()
def close(self) -> None:
if self._closed:
return
try:
self.connection.close()
except Exception as e:
logger.warning(f"Error closing MySQL connection: {e}")
finally:
self._closed = True
class DorisConnector(IbisConnector):
class DorisConnector(MySqlConnector):
"""Doris connector. Speaks MySQL protocol; routes through Doris connection."""
def __init__(self, connection_info):
super().__init__(DataSource.doris, connection_info)
def _handle_pyarrow_unsupported_type(self, ibis_table: Table, **kwargs) -> Table:
result_table = ibis_table
for name, dtype in ibis_table.schema().items():
if isinstance(dtype, Decimal):
result_table = self._round_decimal_columns(
result_table=result_table, col_name=name, **kwargs
)
elif isinstance(dtype, UUID):
result_table = self._cast_uuid_columns(
result_table=result_table, col_name=name
)
elif isinstance(dtype, dt.JSON):
result_table = result_table.mutate(
**{name: result_table[name].cast("string")}
)
return result_table
# Skip MySqlConnector.__init__ — Doris does not accept the ANSI_QUOTES
# init command and the connection is created via Doris routing.
self._closed = False
self.connection = DataSource.doris.get_connection(connection_info)
def create_connector(data_source: DataSource, connection_info):
def create_connector(data_source: DataSource, connection_info) -> MySqlConnector:
if data_source == DataSource.doris:
return DorisConnector(connection_info)
return MySqlConnector(connection_info)
# ---------------------------------------------------------------------------
# Arrow conversion helpers
# ---------------------------------------------------------------------------
# MySQL ``DECIMAL(M, D)`` allows ``M`` up to 65 and ``D`` up to 30, while
# PyArrow's ``decimal128`` only supports precision up to 38. We clamp the
# precision derived from ``cursor.description`` to ``38`` and the scale to
# ``min(precision, 30)`` so PyArrow can still represent the value. A future
# change could switch to ``decimal256`` when MySQL exceeds 38 digits.
_ARROW_DECIMAL128_MAX_PRECISION = 38
_MYSQL_DECIMAL_MAX_SCALE = 30
# Fallback used when ``cursor.description`` does not carry precision/scale
# (e.g. for the legacy ``FIELD_TYPE.DECIMAL`` code or non-MySQLdb cursors).
_MYSQL_DECIMAL_FALLBACK_PRECISION = 38
_MYSQL_DECIMAL_FALLBACK_SCALE = 9
@cache
def _mysql_field_type_map() -> dict[int, pa.DataType]:
"""Build the FIELD_TYPE → Arrow map once per process.
Returns a fully-populated local dict, then ``functools.cache`` publishes
the reference atomically. Concurrent callers either see the fully-built
dict or wait on the cache's GIL-protected slot — they never observe a
partially-populated map.
"""
from MySQLdb.constants import FIELD_TYPE as FT # noqa: PLC0415
base_map: dict[str, pa.DataType] = {
"TINY": pa.int8(),
"SHORT": pa.int16(),
"LONG": pa.int32(),
"INT24": pa.int32(),
"LONGLONG": pa.int64(),
"FLOAT": pa.float32(),
"DOUBLE": pa.float64(),
# ``DECIMAL`` / ``NEWDECIMAL`` are placeholders — the actual precision
# and scale are read from ``cursor.description`` per-column.
"DECIMAL": pa.decimal128(
_MYSQL_DECIMAL_FALLBACK_PRECISION, _MYSQL_DECIMAL_FALLBACK_SCALE
),
"NEWDECIMAL": pa.decimal128(
_MYSQL_DECIMAL_FALLBACK_PRECISION, _MYSQL_DECIMAL_FALLBACK_SCALE
),
"STRING": pa.string(),
"VAR_STRING": pa.string(),
"VARCHAR": pa.string(),
"ENUM": pa.string(),
"SET": pa.string(),
"TINY_BLOB": pa.binary(),
"MEDIUM_BLOB": pa.binary(),
"LONG_BLOB": pa.binary(),
"BLOB": pa.binary(),
"JSON": pa.string(),
"DATE": pa.date32(),
"NEWDATE": pa.date32(),
# MySQL ``TIME`` ranges ``-838:59:59`` to ``838:59:59`` and can be
# negative — semantics PyArrow ``time64("us")`` cannot represent
# (it only accepts 024h positive values). ``duration("us")`` is the
# smallest Arrow type that captures the full MySQL range without loss.
"TIME": pa.duration("us"),
"DATETIME": pa.timestamp("us"),
"TIMESTAMP": pa.timestamp("us"),
"YEAR": pa.int16(),
"BIT": pa.binary(),
"GEOMETRY": pa.string(),
"NULL": pa.null(),
}
result: dict[int, pa.DataType] = {}
for name, arrow_type in base_map.items():
code = getattr(FT, name, None)
if code is not None:
result[code] = arrow_type
return result
@cache
def _mysql_unsigned_variant_map() -> dict[int, pa.DataType]:
"""Build the FIELD_TYPE → unsigned-Arrow map once per process."""
from MySQLdb.constants import FIELD_TYPE as FT # noqa: PLC0415
result: dict[int, pa.DataType] = {}
for name, arrow_type in (
("TINY", pa.uint8()),
("SHORT", pa.uint16()),
("LONG", pa.uint32()),
("INT24", pa.uint32()),
("LONGLONG", pa.uint64()),
):
code = getattr(FT, name, None)
if code is not None:
result[code] = arrow_type
return result
@cache
def _mysql_blob_codes() -> frozenset[int]:
"""Build the set of BLOB-family FIELD_TYPE codes once per process."""
from MySQLdb.constants import FIELD_TYPE as FT # noqa: PLC0415
return frozenset(
code
for code in (
getattr(FT, n, None)
for n in ("BLOB", "TINY_BLOB", "MEDIUM_BLOB", "LONG_BLOB")
)
if code is not None
)
@cache
def _mysql_string_codes() -> frozenset[int]:
"""Build the set of STRING-family FIELD_TYPE codes once per process."""
from MySQLdb.constants import FIELD_TYPE as FT # noqa: PLC0415
return frozenset(
code
for code in (getattr(FT, n, None) for n in ("STRING", "VAR_STRING", "VARCHAR"))
if code is not None
)
@cache
def _mysql_decimal_codes() -> frozenset[int]:
"""Build the set of DECIMAL-family FIELD_TYPE codes once per process."""
from MySQLdb.constants import FIELD_TYPE as FT # noqa: PLC0415
return frozenset(
code
for code in (getattr(FT, n, None) for n in ("DECIMAL", "NEWDECIMAL"))
if code is not None
)
def _arrow_decimal_from_mysql_field(
display_length: int | None,
scale: int | None,
is_unsigned: bool = False,
) -> pa.DataType:
"""Derive a ``pa.decimal128`` type from a MySQLdb ``cursor.description`` entry.
MySQLdb populates ``description[4]`` (PEP 249 ``precision``) with the
``MYSQL_FIELD.length`` — i.e. the *display length*, which includes one
byte for the decimal point (when ``D > 0``) and one byte for the sign
when the column is signed. The declared ``DECIMAL(M, D)`` precision ``M``
is recovered as::
M = length - (1 if unsigned else 0) - (1 if D > 0 else 0)
MySQL allows precision up to 65 and scale up to 30, but Arrow
``decimal128`` caps precision at 38. We clamp precision to 38 and clamp
scale to ``min(scale, precision, 30)`` so any value MySQL accepts (within
the 38-digit Arrow ceiling) round-trips correctly. The previous
hard-coded ``decimal128(38, 9)`` would silently lose digits when ``D > 9``.
"""
if display_length is None or display_length <= 0:
precision = _MYSQL_DECIMAL_FALLBACK_PRECISION
else:
derived_scale = scale if scale is not None and scale >= 0 else 0
sign_overhead = 0 if is_unsigned else 1
point_overhead = 1 if derived_scale > 0 else 0
precision = int(display_length) - sign_overhead - point_overhead
if precision <= 0:
precision = _MYSQL_DECIMAL_FALLBACK_PRECISION
if scale is None or scale < 0:
scale = _MYSQL_DECIMAL_FALLBACK_SCALE
precision = min(int(precision), _ARROW_DECIMAL128_MAX_PRECISION)
scale = min(int(scale), _MYSQL_DECIMAL_MAX_SCALE, precision)
return pa.decimal128(precision, scale)
def _mysql_field_arrow_type(
type_code: int,
flags: int = 0,
precision: int | None = None,
scale: int | None = None,
) -> pa.DataType:
from MySQLdb.constants import FLAG # noqa: PLC0415
field_map = _mysql_field_type_map()
unsigned_map = _mysql_unsigned_variant_map()
blob_codes = _mysql_blob_codes()
string_codes = _mysql_string_codes()
decimal_codes = _mysql_decimal_codes()
base = field_map.get(type_code, pa.string())
if flags & FLAG.UNSIGNED and type_code in unsigned_map:
return unsigned_map[type_code]
# DECIMAL precision/scale come from ``cursor.description`` (PEP 249 fields
# ``precision`` / ``scale``). MySQL ``DECIMAL(M, D)`` allows scale up to 30
# — the previous hard-coded ``decimal128(38, 9)`` would lose digits when
# ``D > 9``.
if type_code in decimal_codes:
return _arrow_decimal_from_mysql_field(
precision, scale, is_unsigned=bool(flags & FLAG.UNSIGNED)
)
# MySQL packs both TEXT and BLOB into FIELD_TYPE.*BLOB; BINARY flag is the
# discriminator. Without BINARY they are TEXT (string); with BINARY they
# are real BLOB (bytes — keep base type).
if type_code in blob_codes and not (flags & FLAG.BINARY):
return pa.string()
# STRING / VAR_STRING with BINARY flag is BINARY/VARBINARY (bytes).
if type_code in string_codes and (flags & FLAG.BINARY):
return pa.binary()
return base
def _build_mysql_arrow_table(cursor) -> pa.Table:
"""Convert a MySQLdb cursor result to a PyArrow table."""
if cursor.description is None:
return pa.table({})
# ``cursor.description_flags`` is a tuple of int flag bitmasks in
# MySQLdb 2.x. Older / non-MySQLdb cursors may not provide it; in that
# case we fall back to zero flags (BLOB → string, ignore UNSIGNED).
flags_attr = getattr(cursor, "description_flags", None)
if flags_attr is not None:
flag_list = list(flags_attr)
else:
flag_list = [0] * len(cursor.description)
flag_list = (flag_list + [0] * len(cursor.description))[: len(cursor.description)]
rows = cursor.fetchall()
fields = []
for i, col in enumerate(cursor.description):
# PEP 249 ``description`` tuple:
# (name, type_code, display_size, internal_size, precision, scale, null_ok)
# MySQLdb populates ``precision``/``scale`` for ``NEWDECIMAL`` columns,
# which lets us reflect the actual ``DECIMAL(M, D)`` instead of using
# a hard-coded ``decimal128(38, 9)``.
precision = col[4] if len(col) > 4 else None
scale = col[5] if len(col) > 5 else None
arrow_type = _mysql_field_arrow_type(
col[1], flag_list[i] or 0, precision=precision, scale=scale
)
fields.append(pa.field(col[0], arrow_type, nullable=True))
schema = pa.schema(fields)
if not rows:
arrays = [pa.array([], type=field.type) for field in schema]
else:
arrays = [
_build_mysql_column([row[i] for row in rows], schema.field(i).type)
for i in range(len(fields))
]
# ``pa.table(dict(...), schema=...)`` silently drops a column when two
# fields share the same name (the dict collapses the duplicate). Use
# ``pa.Table.from_arrays`` so a query like
# ``SELECT a.id, b.id FROM t a JOIN t b`` round-trips both ``id``
# columns instead of returning a one-column table.
return pa.Table.from_arrays(arrays, schema=schema)
def _build_mysql_column(values: list, arrow_type: pa.DataType) -> pa.Array:
"""Convert MySQLdb values into a PyArrow array of the given Arrow type."""
if pa.types.is_string(arrow_type):
processed = []
for v in values:
if v is None:
processed.append(None)
elif isinstance(v, bytes):
processed.append(v.decode("utf-8", errors="replace"))
elif isinstance(v, dict | list | tuple):
processed.append(json.dumps(v, default=str))
elif isinstance(v, str):
processed.append(v)
else:
processed.append(str(v))
return pa.array(processed, type=pa.string(), from_pandas=True)
if pa.types.is_binary(arrow_type):
processed = [bytes(v) if isinstance(v, memoryview) else v for v in values]
return pa.array(processed, type=arrow_type, from_pandas=True)
if pa.types.is_decimal(arrow_type):
processed = [
None
if v is None
else (v if isinstance(v, PyDecimal) else PyDecimal(str(v)))
for v in values
]
return pa.array(processed, type=arrow_type, from_pandas=True)
if pa.types.is_timestamp(arrow_type):
return pa.array(values, type=arrow_type, from_pandas=True)
if pa.types.is_duration(arrow_type):
# MySQLdb returns TIME columns as ``datetime.timedelta``. PyArrow's
# ``duration("us")`` accepts ``timedelta`` directly, but we convert to
# signed microseconds explicitly so negative values (MySQL TIME may go
# down to ``-838:59:59``) and values beyond 24h survive without loss.
# ``timedelta.total_seconds() * 1e6`` would lose precision; we instead
# combine ``days``, ``seconds`` and ``microseconds`` — all of which are
# signed on negative ``timedelta`` values.
import datetime # noqa: PLC0415
processed = [
None
if v is None
else (
v.days * 86_400_000_000 + v.seconds * 1_000_000 + v.microseconds
if isinstance(v, datetime.timedelta)
else v
)
for v in values
]
return pa.array(processed, type=arrow_type, from_pandas=True)
if pa.types.is_null(arrow_type):
return pa.array([None] * len(values), type=pa.null())
return pa.array(values, type=arrow_type, from_pandas=True)
# ---------------------------------------------------------------------------
# Connect kwargs helpers
# ---------------------------------------------------------------------------
def _build_mysql_connect_kwargs(connection_info) -> dict:
"""Translate ``MySqlConnectionInfo`` / ``ConnectionUrl`` into MySQLdb kwargs."""
from urllib.parse import parse_qsl, unquote_plus, urlparse # noqa: PLC0415
if hasattr(connection_info, "connection_url"):
url = connection_info.connection_url.get_secret_value()
parsed = urlparse(url)
if parsed.scheme not in {"mysql", "mysql+pymysql", "mysql+mysqldb"}:
raise WrenError(
ErrorCode.INVALID_CONNECTION_INFO,
"MySQL connection URL must use mysql:// scheme",
)
kwargs = dict(parse_qsl(parsed.query))
if connection_info.kwargs:
kwargs.update(connection_info.kwargs)
host = parsed.hostname or "127.0.0.1"
if host == "localhost":
host = "127.0.0.1"
out: dict = {
"host": host,
"port": int(parsed.port) if parsed.port else 3306,
"user": parsed.username,
"passwd": unquote_plus(parsed.password) if parsed.password else "",
"db": parsed.path.lstrip("/") if parsed.path else None,
"charset": "utf8mb4",
"use_unicode": True,
"autocommit": True,
}
out.update(kwargs)
return out
info = connection_info
kwargs = dict(info.kwargs) if info.kwargs else {}
# MySQLdb routes host="localhost" through a unix socket by default; force
# TCP by normalising it to 127.0.0.1.
host = info.host
if host == "localhost":
host = "127.0.0.1"
out = {
"host": host,
"port": int(info.port),
"user": info.user,
"passwd": (info.password.get_secret_value() if info.password else ""),
"db": info.database,
"charset": "utf8mb4",
"use_unicode": True,
"autocommit": True,
}
ssl = _mysql_ssl_kwargs(info)
if ssl is not None:
out["ssl"] = ssl
out["ssl_mode"] = "VERIFY_CA" if "ca" in ssl else "REQUIRED"
out.update(kwargs)
return out
def _build_doris_connect_kwargs(connection_info) -> dict:
"""Translate ``DorisConnectionInfo`` / ``ConnectionUrl`` into MySQLdb kwargs."""
if hasattr(connection_info, "connection_url"):
return _build_mysql_connect_kwargs(connection_info)
info = connection_info
kwargs = dict(info.kwargs) if info.kwargs else {}
host = info.host
if host == "localhost":
host = "127.0.0.1"
out = {
"host": host,
"port": int(info.port),
"user": info.user,
"passwd": (info.password.get_secret_value() if info.password else ""),
"db": info.database,
"charset": "utf8mb4",
"use_unicode": True,
"autocommit": True,
}
out.update(kwargs)
return out
def _mysql_ssl_kwargs(info) -> dict | None:
"""Build the MySQLdb ``ssl`` kwarg dict from ``MySqlConnectionInfo`` SSL fields."""
ssl_mode = info.ssl_mode if hasattr(info, "ssl_mode") and info.ssl_mode else None
ssl_mode = ssl_mode.lower() if ssl_mode else None
if not ssl_mode or ssl_mode == "disabled":
return None
if ssl_mode == "verify_ca":
if not info.ssl_ca:
raise WrenError(
ErrorCode.INVALID_CONNECTION_INFO,
"SSL CA must be provided when SSL mode is VERIFY CA",
)
return {"ca": info.ssl_ca.get_secret_value()}
# 'enabled' / any other non-disabled mode: require SSL without CA verification.
return {}
+234 -24
View File
@@ -1,22 +1,241 @@
from contextlib import suppress
"""Native psycopg-based PostgreSQL connector.
This connector executes queries through psycopg3 directly and converts the
cursor result into a PyArrow table using a hand-rolled OID-to-Arrow type map.
It avoids the ibis-framework dependency entirely, which gives us:
* a smaller install surface for the ``postgres`` extra,
* direct control over pg-specific type handling (numeric scale, arrays,
intervals, jsonb), and
* a single code path for both query execution and dry-run.
"""
from __future__ import annotations
import json
from decimal import ROUND_HALF_EVEN
from decimal import Decimal as PyDecimal
import psycopg
import pyarrow as pa
from loguru import logger
from wren.connector.base import IbisConnector
from wren.connector.base import ConnectorABC
from wren.model.data_source import DataSource
from wren.model.error import DIALECT_SQL, ErrorCode, ErrorPhase, WrenError
# Map of well-known PostgreSQL OIDs to Arrow types. OIDs that we have not
# explicitly mapped fall back to ``pa.string()`` (see ``_get_pg_arrow_type``).
_PG_OID_TO_ARROW: dict[int, pa.DataType] = {
16: pa.bool_(),
17: pa.binary(),
18: pa.string(),
19: pa.string(),
20: pa.int64(),
21: pa.int16(),
23: pa.int32(),
25: pa.string(),
26: pa.int64(),
114: pa.string(), # json
142: pa.string(), # xml
700: pa.float32(),
701: pa.float64(),
650: pa.string(),
774: pa.string(),
790: pa.string(),
829: pa.string(),
869: pa.string(),
1040: pa.string(),
1042: pa.string(),
1043: pa.string(),
1082: pa.date32(),
1083: pa.time64("us"),
1114: pa.timestamp("us"),
1184: pa.timestamp("us", tz="UTC"),
1186: pa.duration("us"),
2950: pa.string(), # uuid
3802: pa.string(), # jsonb
1266: pa.string(), # timetz
3614: pa.string(), # tsvector
3615: pa.string(), # tsquery
3904: pa.string(),
3906: pa.string(),
3908: pa.string(),
3910: pa.string(),
3912: pa.string(),
3926: pa.string(),
199: pa.list_(pa.string()), # _json
1000: pa.list_(pa.bool_()),
1003: pa.list_(pa.string()),
1005: pa.list_(pa.int16()),
1007: pa.list_(pa.int32()),
1009: pa.list_(pa.string()),
1014: pa.list_(pa.string()),
1015: pa.list_(pa.string()),
1016: pa.list_(pa.int64()),
1021: pa.list_(pa.float32()),
1022: pa.list_(pa.float64()),
1028: pa.list_(pa.string()),
1041: pa.list_(pa.string()),
1115: pa.list_(pa.timestamp("us")),
1182: pa.list_(pa.string()),
1183: pa.list_(pa.string()),
1185: pa.list_(pa.timestamp("us", tz="UTC")),
1187: pa.list_(pa.string()),
1270: pa.list_(pa.string()),
2951: pa.list_(pa.string()), # _uuid
3807: pa.list_(pa.string()), # _jsonb
}
def _get_pg_decimal_type(column) -> pa.DataType:
"""Map a psycopg numeric column to the narrowest Arrow decimal type we can represent."""
if column.scale is None:
logger.debug(
"Postgres NUMERIC column has no scale metadata; defaulting to decimal128(38, 9)"
)
scale = column.scale if column.scale is not None else 9
scale = max(0, min(scale, 38))
precision = column.precision if column.precision is not None else 38
if precision <= 0 or precision > 38:
precision = 38
precision = max(precision, scale, 1)
precision = min(precision, 38)
return pa.decimal128(precision, scale)
def _get_pg_arrow_type(column) -> pa.DataType:
"""Map a psycopg cursor description column to an Arrow type."""
if column.type_code == 1700:
return _get_pg_decimal_type(column)
if column.type_code == 1231:
return pa.list_(_get_pg_decimal_type(column))
return _PG_OID_TO_ARROW.get(column.type_code, pa.string())
def _build_pg_arrow_table(cursor) -> pa.Table:
"""Convert a psycopg3 cursor result to a PyArrow table."""
if cursor.description is None:
return pa.table({})
rows = cursor.fetchall()
fields = [
pa.field(column.name, _get_pg_arrow_type(column), nullable=True)
for column in cursor.description
]
schema = pa.schema(fields)
if not rows:
arrays = [pa.array([], type=field.type) for field in schema]
else:
arrays = [
_build_pg_column(
[row[index] for row in rows],
field.type,
cursor.description[index].type_code,
)
for index, field in enumerate(schema)
]
# Build positionally — ``pa.table({...})`` silently drops duplicate column
# names (very common in joins like ``SELECT a.id, b.id FROM t a, t b``).
return pa.Table.from_arrays(arrays, schema=schema)
def _build_pg_column(
values: list, arrow_type: pa.DataType, pg_type_oid: int | None = None
) -> pa.Array:
"""Build a PyArrow column from psycopg values with PG-specific coercions."""
def _coerce_decimal(value: PyDecimal | None, target_type: pa.DataType):
if value is None or not isinstance(value, PyDecimal):
return value
quantize_value = PyDecimal(f"1E-{target_type.scale}")
try:
return value.quantize(quantize_value, rounding=ROUND_HALF_EVEN)
except Exception:
return value
if arrow_type == pa.string():
processed = []
for value in values:
if value is None:
# json / jsonb SQL NULLs come back as Python None too, but the
# SQL value ``'null'::jsonb`` is also None at the Python level.
# Keep both as None — callers that care about the distinction
# should cast the column to text in SQL.
processed.append(None)
elif isinstance(value, dict | list):
processed.append(json.dumps(value, default=str))
elif not isinstance(value, str):
processed.append(str(value))
else:
processed.append(value)
return pa.array(processed, type=pa.string(), from_pandas=True)
if pa.types.is_binary(arrow_type):
processed = [
bytes(value) if isinstance(value, memoryview) else value for value in values
]
return pa.array(processed, type=pa.binary(), from_pandas=True)
if pa.types.is_decimal(arrow_type):
processed = [_coerce_decimal(value, arrow_type) for value in values]
return pa.array(processed, type=arrow_type, from_pandas=True)
if pa.types.is_list(arrow_type) and pa.types.is_string(arrow_type.value_type):
processed = []
for value in values:
if value is None:
processed.append(None)
continue
items = []
for item in value:
if item is None:
items.append(None)
elif isinstance(item, dict | list):
items.append(json.dumps(item, default=str))
elif isinstance(item, str):
items.append(item)
else:
items.append(str(item))
processed.append(items)
return pa.array(processed, type=arrow_type, from_pandas=True)
if pa.types.is_list(arrow_type) and pa.types.is_decimal(arrow_type.value_type):
processed = []
for value in values:
if value is None:
processed.append(None)
else:
processed.append(
[_coerce_decimal(item, arrow_type.value_type) for item in value]
)
return pa.array(processed, type=arrow_type, from_pandas=True)
return pa.array(values, type=arrow_type, from_pandas=True)
class PostgresConnector(ConnectorABC):
"""Native psycopg3 implementation of the Wren postgres connector."""
class PostgresConnector(IbisConnector):
def __init__(self, connection_info):
super().__init__(DataSource.postgres, connection_info)
self.connection = DataSource.postgres.get_connection(connection_info)
self._closed = False
def query(self, sql: str, limit: int | None = None) -> pa.Table:
import psycopg # noqa: PLC0415
if limit is not None:
sql = f"SELECT * FROM ({sql}) AS _sub LIMIT {limit}"
try:
return super().query(sql, limit)
with self.connection.cursor() as cursor:
cursor.execute(sql)
return _build_pg_arrow_table(cursor)
except psycopg.errors.QueryCanceled:
raise
except (WrenError, TimeoutError):
@@ -30,10 +249,10 @@ class PostgresConnector(IbisConnector):
) from e
def dry_run(self, sql: str) -> None:
import psycopg # noqa: PLC0415
wrapped = f"SELECT * FROM ({sql}) AS _sub LIMIT 0"
try:
super().dry_run(sql)
with self.connection.cursor() as cursor:
cursor.execute(wrapped)
except psycopg.errors.QueryCanceled:
raise
except (WrenError, TimeoutError):
@@ -47,26 +266,17 @@ class PostgresConnector(IbisConnector):
) from e
def close(self) -> None:
if self._closed or not hasattr(self, "connection") or self.connection is None:
if self._closed or self.connection is None:
return
try:
if hasattr(self.connection, "con") and self.connection.con is not None:
if (
hasattr(self.connection.con, "closed")
and not self.connection.con.closed
):
with suppress(Exception):
self.connection.con.cancel()
import time # noqa: PLC0415
time.sleep(0.1)
self.connection.con.close()
elif hasattr(self.connection, "close"):
if not self.connection.closed:
try:
self.connection.cancel()
except Exception:
pass
self.connection.close()
except Exception as e:
logger.warning(f"Error closing postgres connection: {e}")
if hasattr(self.connection, "con"):
self.connection.con = None
finally:
self._closed = True
self.connection = None
+442
View File
@@ -0,0 +1,442 @@
"""Native Trino connector that talks to the trino python client directly.
This module bypasses ibis-framework[trino]; ``cursor.description`` exposes
Trino type strings (``array(row("a" integer, "b" varchar))`` etc.) which we
lex with sqlglot to build an equivalent PyArrow schema.
"""
from __future__ import annotations
import contextlib
import datetime as dtlib
import json
from decimal import Decimal as PyDecimal
from urllib.parse import parse_qsl, urlparse
import pyarrow as pa
import sqlglot
import sqlglot.errors
from loguru import logger
from sqlglot.expressions import ColumnDef, DataType
from wren.connector.base import ConnectorABC
from wren.model.error import (
DIALECT_SQL,
ErrorCode,
ErrorPhase,
WrenError,
)
def _parse_trino_data_type(type_str: str | None) -> pa.DataType:
"""Parse a Trino type string from ``cursor.description`` into an Arrow type.
Delegates the lexing/parsing to sqlglot and walks the resulting DataType
AST to build the equivalent PyArrow type. sqlglot handles the awkward
cases (anonymous row fields whose type contains whitespace, nested
array/map/row, decimal(p,s), etc.).
"""
if type_str is None:
return pa.string()
try:
parsed = sqlglot.parse_one(type_str, into=DataType, dialect="trino")
except sqlglot.errors.ParseError:
logger.warning(f"Failed to parse trino type string: {type_str}")
return pa.string()
if parsed is None:
return pa.string()
return _trino_data_type_to_arrow(parsed)
_TRINO_DATA_TYPE_TO_ARROW: dict = {}
def _init_trino_data_type_map() -> None:
if _TRINO_DATA_TYPE_TO_ARROW:
return
T = DataType.Type
_TRINO_DATA_TYPE_TO_ARROW.update(
{
T.BOOLEAN: pa.bool_(),
T.TINYINT: pa.int8(),
T.SMALLINT: pa.int16(),
T.INT: pa.int32(),
T.BIGINT: pa.int64(),
T.FLOAT: pa.float32(),
T.DOUBLE: pa.float64(),
T.VARCHAR: pa.string(),
T.CHAR: pa.string(),
T.NCHAR: pa.string(),
T.NVARCHAR: pa.string(),
T.TEXT: pa.string(),
T.JSON: pa.string(),
T.UUID: pa.string(),
T.IPADDRESS: pa.string(),
T.HLLSKETCH: pa.string(), # hyperloglog
T.GEOMETRY: pa.string(),
T.VARBINARY: pa.binary(),
T.BINARY: pa.binary(),
T.DATE: pa.date32(),
T.TIME: pa.time64("us"),
T.TIMETZ: pa.time64("us"),
# Millisecond precision matches PyArrow's default and keeps round-trip
# behaviour stable for ``timestamp(n)`` results across Trino versions.
T.TIMESTAMP: pa.timestamp("ms"),
T.TIMESTAMPTZ: pa.timestamp("ms", tz="UTC"),
T.TIMESTAMPLTZ: pa.timestamp("ms", tz="UTC"),
}
)
def _trino_data_type_to_arrow(node) -> pa.DataType:
_init_trino_data_type_map()
if not isinstance(node, DataType):
# e.g. Interval — fall back to string representation.
return pa.string()
kind = node.this
T = DataType.Type
if kind in _TRINO_DATA_TYPE_TO_ARROW:
return _TRINO_DATA_TYPE_TO_ARROW[kind]
if kind == T.DECIMAL:
precision, scale = 38, 9
params = node.expressions
if len(params) >= 1:
with contextlib.suppress(AttributeError, ValueError):
precision = min(int(params[0].this.this), 38)
if len(params) >= 2:
with contextlib.suppress(AttributeError, ValueError):
scale = min(int(params[1].this.this), precision)
return pa.decimal128(precision, scale)
if kind == T.ARRAY:
inner = node.expressions[0] if node.expressions else None
return pa.list_(_trino_data_type_to_arrow(inner) if inner else pa.string())
if kind == T.MAP:
if len(node.expressions) >= 2:
return pa.map_(
_trino_data_type_to_arrow(node.expressions[0]),
_trino_data_type_to_arrow(node.expressions[1]),
)
return pa.string()
if kind == T.STRUCT:
fields: list[pa.Field] = []
for idx, child in enumerate(node.expressions):
if isinstance(child, ColumnDef):
# Named row field: row(a integer, b varchar)
name = child.name or f"f{idx}"
inner = child.args.get("kind")
fields.append(
pa.field(
name,
_trino_data_type_to_arrow(inner) if inner else pa.string(),
)
)
else:
# Anonymous row field: row(map(varchar,integer), bigint)
fields.append(pa.field(f"f{idx}", _trino_data_type_to_arrow(child)))
return pa.struct(fields)
return pa.string()
def _build_trino_column(values: list, arrow_type: pa.DataType) -> pa.Array:
"""Convert trino DB-API values to a PyArrow array of the given Arrow type."""
if pa.types.is_string(arrow_type):
processed = []
for v in values:
if v is None:
processed.append(None)
elif isinstance(v, dict | list | tuple):
processed.append(json.dumps(v, default=str))
elif isinstance(v, str):
processed.append(v)
else:
processed.append(str(v))
return pa.array(processed, type=pa.string(), from_pandas=True)
if pa.types.is_binary(arrow_type):
processed = [bytes(v) if isinstance(v, memoryview) else v for v in values]
return pa.array(processed, type=arrow_type, from_pandas=True)
if pa.types.is_decimal(arrow_type):
processed = [
None
if v is None
else (v if isinstance(v, PyDecimal) else PyDecimal(str(v)))
for v in values
]
return pa.array(processed, type=arrow_type, from_pandas=True)
if pa.types.is_timestamp(arrow_type):
# The trino driver returns either datetime objects or ISO-8601 strings
# depending on column type and adapter settings; normalise to datetime.
processed = []
for v in values:
if v is None or isinstance(v, dtlib.datetime):
processed.append(v)
else:
try:
processed.append(dtlib.datetime.fromisoformat(str(v)))
except ValueError:
processed.append(None)
return pa.array(processed, type=arrow_type, from_pandas=True)
if pa.types.is_date(arrow_type):
processed = []
for v in values:
if v is None or isinstance(v, dtlib.date):
processed.append(v)
else:
try:
processed.append(dtlib.date.fromisoformat(str(v)))
except ValueError:
processed.append(None)
return pa.array(processed, type=arrow_type, from_pandas=True)
if pa.types.is_time(arrow_type):
processed = []
for v in values:
if v is None or isinstance(v, dtlib.time):
processed.append(v)
else:
try:
processed.append(dtlib.time.fromisoformat(str(v)))
except ValueError:
processed.append(None)
return pa.array(processed, type=arrow_type, from_pandas=True)
if pa.types.is_struct(arrow_type):
# The trino driver returns Python tuples for row(...) values. PyArrow
# accepts dicts keyed by field name; convert to make field order
# mismatch (e.g. anonymous row) explicit.
names = [f.name for f in arrow_type]
processed: list = []
for v in values:
if v is None:
processed.append(None)
elif isinstance(v, dict):
processed.append(v)
else:
processed.append(dict(zip(names, v, strict=False)))
return pa.array(processed, type=arrow_type, from_pandas=True)
if pa.types.is_map(arrow_type):
# PyArrow's map_ constructor expects an iterable of (key, value) pairs;
# the trino driver returns Python dicts.
processed = [None if v is None else list(v.items()) for v in values]
return pa.array(processed, type=arrow_type, from_pandas=True)
return pa.array(values, type=arrow_type, from_pandas=True)
def _build_trino_arrow_table(cursor) -> pa.Table:
"""Convert a trino DB-API cursor result to a PyArrow table."""
if cursor.description is None:
return pa.table({})
rows = cursor.fetchall()
fields = [
pa.field(col[0], _parse_trino_data_type(col[1]), nullable=True)
for col in cursor.description
]
schema = pa.schema(fields)
if not rows:
arrays = [pa.array([], type=field.type) for field in schema]
else:
arrays = [
_build_trino_column([row[i] for row in rows], schema.field(i).type)
for i in range(len(fields))
]
return pa.table(
dict(zip([f.name for f in fields], arrays, strict=False)),
schema=schema,
)
def _build_trino_connect_kwargs(connection_info) -> dict:
"""Build kwargs for ``trino.dbapi.connect`` from either a typed
``TrinoConnectionInfo`` or a generic ``ConnectionUrl``.
Returns a dict that may contain the sentinel ``_password`` key — the caller
is expected to pop it before passing the dict to ``trino_connect``.
"""
if hasattr(connection_info, "connection_url"):
url = connection_info.connection_url.get_secret_value()
return _parse_trino_url(url, connection_info.kwargs)
info = connection_info # TrinoConnectionInfo
kwargs = dict(info.kwargs) if info.kwargs else {}
password = info.password.get_secret_value() if info.password else None
out: dict = {
"host": info.host,
"port": int(info.port),
"user": info.user,
"catalog": info.catalog,
"schema": info.trino_schema,
# Pin session timezone to UTC so CAST('...' AS TIMESTAMP WITH TIME ZONE)
# produces deterministic results across deployments.
"timezone": "UTC",
"_password": password,
}
out.update(kwargs)
return out
def _parse_trino_url(url: str, extra_kwargs: dict | None) -> dict:
"""Parse a ``trino://[user[:pwd]@]host[:port][/catalog[/schema]][?...]`` URL.
Returns the same ``_password`` sentinel-key shape as
:func:`_build_trino_connect_kwargs`.
"""
parsed = urlparse(url)
if parsed.scheme not in {"trino", "trino+https"}:
raise WrenError(
ErrorCode.INVALID_CONNECTION_INFO,
"Trino connection URL must use trino:// scheme",
)
if not parsed.username:
raise WrenError(
ErrorCode.INVALID_CONNECTION_INFO,
"Trino connection URL must include a username",
)
path_parts = parsed.path.lstrip("/").split("/")
catalog = path_parts[0] if path_parts and path_parts[0] else None
schema = path_parts[1] if len(path_parts) > 1 else None
query_kwargs = dict(parse_qsl(parsed.query))
if extra_kwargs:
query_kwargs.update(extra_kwargs)
out: dict = {
"host": parsed.hostname,
"port": int(parsed.port or 8080),
"user": parsed.username,
"catalog": catalog,
"schema": schema,
"timezone": "UTC",
"_password": parsed.password,
}
if parsed.scheme == "trino+https":
out["http_scheme"] = "https"
out.update(query_kwargs)
return out
_TRINO_IMPORT_HINT = (
"The 'trino' package is required for the Trino connector. "
"Install it with: pip install wren-engine[trino]"
)
def _import_trino():
"""Lazy import of the ``trino`` package with a clear install hint."""
try:
import trino # noqa: PLC0415
return trino
except ImportError as e:
raise WrenError(
ErrorCode.INVALID_CONNECTION_INFO,
f"{_TRINO_IMPORT_HINT} (original error: {e})",
) from e
def _strip_trailing_semicolon(sql: str) -> str:
"""Strip trailing whitespace and an optional final semicolon.
Wrapping ``SELECT * FROM ({sql}) AS _sub LIMIT N`` breaks if ``sql`` ends
with ``;`` because the parser sees ``...; ) AS _sub...``.
"""
return sql.rstrip().removesuffix(";").rstrip()
class TrinoConnector(ConnectorABC):
"""Native trino DB-API connector that bypasses ibis-project."""
def __init__(self, connection_info):
trino = _import_trino()
BasicAuthentication = trino.auth.BasicAuthentication
JWTAuthentication = trino.auth.JWTAuthentication
trino_connect = trino.dbapi.connect
connect_kwargs = _build_trino_connect_kwargs(connection_info)
password = connect_kwargs.pop("_password", None)
token = connect_kwargs.pop("access_token", None)
if connect_kwargs.get("auth") is None:
user = connect_kwargs.get("user")
if token:
connect_kwargs["auth"] = JWTAuthentication(token)
connect_kwargs.setdefault("http_scheme", "https")
elif user and password:
connect_kwargs["auth"] = BasicAuthentication(user, password)
connect_kwargs.setdefault("http_scheme", "https")
self.connection = trino_connect(**connect_kwargs)
self._closed = False
def query(self, sql: str, limit: int | None = None) -> pa.Table:
trino = _import_trino()
if limit is not None:
sql = f"SELECT * FROM ({_strip_trailing_semicolon(sql)}) AS _sub LIMIT {limit}"
try:
with contextlib.closing(self.connection.cursor()) as cursor:
cursor.execute(sql)
return _build_trino_arrow_table(cursor)
except trino.exceptions.TrinoQueryError as e:
if e.error_name == "EXCEEDED_TIME_LIMIT":
raise
raise WrenError(
ErrorCode.INVALID_SQL,
str(e),
phase=ErrorPhase.SQL_EXECUTION,
metadata={DIALECT_SQL: sql},
) from e
except (WrenError, TimeoutError):
raise
def dry_run(self, sql: str) -> None:
trino = _import_trino()
wrapped = f"SELECT * FROM ({_strip_trailing_semicolon(sql)}) AS _sub LIMIT 0"
try:
with contextlib.closing(self.connection.cursor()) as cursor:
cursor.execute(wrapped)
cursor.fetchall()
except trino.exceptions.TrinoQueryError as e:
if e.error_name == "EXCEEDED_TIME_LIMIT":
raise
raise WrenError(
ErrorCode.INVALID_SQL,
str(e),
phase=ErrorPhase.SQL_DRY_RUN,
metadata={DIALECT_SQL: sql},
) from e
except (WrenError, TimeoutError):
raise
def close(self) -> None:
if self._closed or self.connection is None:
return
try:
self.connection.close()
except Exception as e:
logger.warning(f"Error closing Trino connection: {e}")
finally:
self._closed = True
self.connection = None
def create_connector(data_source, connection_info) -> TrinoConnector:
return TrinoConnector(connection_info)
+302 -128
View File
@@ -1,17 +1,21 @@
from __future__ import annotations
import base64
import ssl
import datetime as dtlib
import urllib
from enum import Enum, StrEnum, auto
from json import loads
from typing import Any
from urllib.parse import unquote_plus
from typing import TYPE_CHECKING, Any, Union
from urllib.parse import unquote_plus, urlparse
import boto3
import ibis
from ibis import BaseBackend
try:
import pyodbc
except ImportError: # pragma: no cover
pyodbc = None
from wren.model import (
AthenaConnectionInfo,
BaseConnectionInfo,
@@ -37,12 +41,25 @@ from wren.model import (
S3FileConnectionInfo,
SnowflakeConnectionInfo,
SparkConnectionInfo,
SSLMode,
TrinoConnectionInfo,
)
from wren.model.error import ErrorCode, WrenError
if TYPE_CHECKING:
import MySQLdb
import psycopg
from pyathena.connection import Connection as PyAthenaConnection
# get_connection() may return either an ibis BaseBackend (for connectors still
# routed through ibis) or a native driver connection for connectors that have
# dropped the ibis dependency (Athena via pyathena, MySQL/Doris via MySQLdb,
# Postgres/Canner via psycopg).
BackendOrConnection = Union[
BaseBackend, "PyAthenaConnection", "MySQLdb.Connection", "psycopg.Connection"
]
X_WREN_DB_STATEMENT_TIMEOUT = "x-wren-db-statement_timeout"
MSSQL_DATETIMEOFFSET_TYPE_CODE = -155
class DataSource(StrEnum):
@@ -67,7 +84,7 @@ class DataSource(StrEnum):
spark = auto()
databricks = auto()
def get_connection(self, info: ConnectionInfo) -> BaseBackend:
def get_connection(self, info: ConnectionInfo) -> BackendOrConnection:
try:
return DataSourceExtension[self].get_connection(info)
except KeyError:
@@ -181,7 +198,8 @@ class DataSource(StrEnum):
def _handle_clickhouse_url(
self, parsed: urllib.parse.ParseResult
) -> ClickHouseConnectionInfo:
if not parsed.scheme or parsed.scheme != "clickhouse":
allowed_schemes = {"clickhouse", "clickhouse+http", "clickhouse+https"}
if not parsed.scheme or parsed.scheme not in allowed_schemes:
raise WrenError(
ErrorCode.INVALID_CONNECTION_INFO,
"Invalid connection URL for ClickHouse",
@@ -201,6 +219,8 @@ class DataSource(StrEnum):
if "secure" in parsed_kwargs:
kwargs["secure"] = self._safe_strtobool(parsed_kwargs["secure"])
parsed_kwargs.pop("secure")
elif parsed.scheme == "clickhouse+https":
kwargs["secure"] = True
kwargs["kwargs"] = parsed_kwargs
return ClickHouseConnectionInfo(**kwargs)
@@ -230,10 +250,34 @@ class DataSourceExtension(Enum):
databricks = "databricks"
spark = "spark"
def get_connection(self, info: ConnectionInfo) -> BaseBackend:
def get_connection(self, info: ConnectionInfo) -> BackendOrConnection:
try:
if hasattr(info, "connection_url"):
# MySQL / Doris use the native MySQLdb driver, not ibis.
if self.name in {"mysql", "doris"}:
return getattr(self, f"get_{self.name}_connection")(info)
if self.name == "trino":
# Trino uses the native DB-API client; the generic
# ``ibis.connect()`` path was removed when the native
# connector landed. Route the URL through the dedicated
# parser so callers still get a working connection.
from wren.connector.trino import ( # noqa: PLC0415
_build_trino_connect_kwargs,
)
trino_kwargs = _build_trino_connect_kwargs(info)
trino_kwargs.pop("_password", None)
trino_kwargs.pop("access_token", None)
from trino.dbapi import ( # noqa: PLC0415
connect as trino_connect,
)
return trino_connect(**trino_kwargs)
kwargs = info.kwargs if info.kwargs else {}
if self.name == "mssql":
return self.get_mssql_connection_from_url(
info.connection_url.get_secret_value(), kwargs
)
return ibis.connect(info.connection_url.get_secret_value(), **kwargs)
if self.name in {"local_file", "redshift", "spark", "duckdb", "datafusion"}:
raise NotImplementedError(
@@ -248,41 +292,24 @@ class DataSourceExtension(Enum):
raise WrenError(ErrorCode.GET_CONNECTION_ERROR, f"{e!s}") from e
@staticmethod
def get_athena_connection(info: AthenaConnectionInfo) -> BaseBackend:
kwargs: dict[str, Any] = {
"s3_staging_dir": info.s3_staging_dir.get_secret_value(),
"schema_name": info.schema_name,
}
if info.region_name:
kwargs["region_name"] = info.region_name
def get_athena_connection(info: AthenaConnectionInfo) -> PyAthenaConnection:
"""Open a pyathena DB-API connection.
if info.web_identity_token and info.role_arn:
oidc_token = info.web_identity_token.get_secret_value()
role_arn = info.role_arn.get_secret_value()
session_name = info.role_session_name or "wren-oidc-session"
region = info.region_name if info.region_name else None
sts = boto3.client("sts", region_name=region)
resp = sts.assume_role_with_web_identity(
RoleArn=role_arn,
RoleSessionName=session_name,
WebIdentityToken=oidc_token,
)
creds = resp["Credentials"]
kwargs["aws_access_key_id"] = creds["AccessKeyId"]
kwargs["aws_secret_access_key"] = creds["SecretAccessKey"]
kwargs["aws_session_token"] = creds["SessionToken"]
elif info.aws_access_key_id and info.aws_secret_access_key:
kwargs["aws_access_key_id"] = info.aws_access_key_id.get_secret_value()
kwargs["aws_secret_access_key"] = (
info.aws_secret_access_key.get_secret_value()
)
if info.aws_session_token:
kwargs["aws_session_token"] = info.aws_session_token.get_secret_value()
Delegates connection-kwargs construction to
:func:`wren.connector.athena._build_connect_kwargs` so the legacy
``data_source`` path and the native :class:`AthenaConnector` stay in
lockstep on credential resolution, ``schema_name`` propagation,
``kill_on_interrupt``, and user ``kwargs`` merge semantics.
"""
from pyathena import connect # noqa: PLC0415
return ibis.athena.connect(**kwargs)
from wren.connector.athena import _build_connect_kwargs # noqa: PLC0415
return connect(**_build_connect_kwargs(info))
@staticmethod
def get_bigquery_connection(info: BigQueryDatasetConnectionInfo) -> BaseBackend:
import ibis # noqa: PLC0415
from google.cloud import bigquery # noqa: PLC0415
from google.oauth2 import service_account # noqa: PLC0415
@@ -305,86 +332,243 @@ class DataSourceExtension(Enum):
return ibis.bigquery.connect(client=bq_client, credentials=credentials)
@staticmethod
def get_canner_connection(info: CannerConnectionInfo) -> BaseBackend:
return ibis.postgres.connect(
def get_canner_connection(info: CannerConnectionInfo):
import psycopg # noqa: PLC0415
return psycopg.connect(
host=info.host,
port=int(info.port),
database=info.workspace,
dbname=info.workspace,
user=info.user,
password=info.pat.get_secret_value(),
autocommit=True,
)
@staticmethod
def get_clickhouse_connection(info: ClickHouseConnectionInfo) -> BaseBackend:
return ibis.clickhouse.connect(
host=info.host,
port=int(info.port),
database=info.database,
user=info.user,
password=(info.password and info.password.get_secret_value()),
settings=info.settings if info.settings else {},
**info.kwargs if info.kwargs else {},
)
def get_clickhouse_connection(info: ClickHouseConnectionInfo):
import clickhouse_connect # noqa: PLC0415
settings = dict(info.settings) if info.settings else {}
kwargs = dict(info.kwargs) if info.kwargs else {}
statement_timeout = kwargs.pop("statement_timeout", None)
if statement_timeout is not None:
settings["max_execution_time"] = int(statement_timeout)
# Merge any user-supplied ``settings`` from kwargs into the local dict
# *before* applying the rest, otherwise ``client_kwargs.update(kwargs)``
# below would clobber the statement_timeout-derived max_execution_time.
extra_settings = kwargs.pop("settings", None)
if extra_settings:
settings.update(extra_settings)
client_kwargs = {
"host": info.host,
"port": int(info.port),
"database": info.database,
"username": info.user,
"password": info.password.get_secret_value() if info.password else "",
"secure": info.secure,
"settings": settings,
}
client_kwargs.update(kwargs)
return clickhouse_connect.get_client(**client_kwargs)
@classmethod
def get_mssql_connection(cls, info: MSSqlConnectionInfo) -> BaseBackend:
return ibis.mssql.connect(
def get_mssql_connection(cls, info: MSSqlConnectionInfo):
return cls._connect_mssql_pyodbc(
host=info.host,
port=info.port,
database=info.database,
user=info.user,
password=info.password.get_secret_value(),
password=info.password.get_secret_value() if info.password else None,
driver=info.driver,
TDS_Version=info.tds_version,
**info.kwargs if info.kwargs else {},
kwargs={
"TDS_Version": info.tds_version,
**(info.kwargs if info.kwargs else {}),
},
)
@classmethod
def get_mysql_connection(cls, info: MySqlConnectionInfo) -> BaseBackend:
ssl_context = cls._create_ssl_context(info)
kwargs = {"ssl": ssl_context} if ssl_context else {}
kwargs.setdefault("charset", "utf8mb4")
if info.kwargs:
kwargs.update(info.kwargs)
return ibis.mysql.connect(
host=info.host,
port=int(info.port),
database=info.database,
user=info.user,
password=info.password.get_secret_value() if info.password else "",
**kwargs,
def get_mssql_connection_from_url(
cls, connection_url: str, base_kwargs: dict[str, Any] | None = None
):
parsed = urlparse(connection_url)
if parsed.scheme != "mssql":
raise WrenError(
ErrorCode.INVALID_CONNECTION_INFO,
"Invalid connection URL for MSSQL",
)
if not parsed.hostname or not parsed.path or not parsed.username:
raise WrenError(
ErrorCode.INVALID_CONNECTION_INFO,
"MSSQL connection URL must include user, host and database",
)
kwargs = dict(base_kwargs) if base_kwargs else {}
# parse_qsl already URL-decodes values, but we re-apply unquote_plus
# below only to components urlparse leaves encoded (user, path, password).
for key, value in urllib.parse.parse_qsl(parsed.query):
kwargs[key] = value
driver = kwargs.pop("driver", "ODBC Driver 18 for SQL Server")
return cls._connect_mssql_pyodbc(
host=parsed.hostname,
port=str(parsed.port or 1433),
database=unquote_plus(parsed.path.lstrip("/")),
user=unquote_plus(parsed.username),
password=unquote_plus(parsed.password) if parsed.password else None,
driver=driver,
kwargs=kwargs,
)
@classmethod
def get_doris_connection(cls, info: DorisConnectionInfo) -> BaseBackend:
kwargs: dict = {}
kwargs.setdefault("charset", "utf8mb4")
if info.kwargs:
kwargs.update(info.kwargs)
connection = ibis.mysql.connect(
host=info.host,
port=int(info.port),
database=info.database,
user=info.user,
password=info.password.get_secret_value() if info.password else "",
**kwargs,
)
connection.con.get_autocommit = lambda: True
@staticmethod
def _connect_mssql_pyodbc(
host: str,
port: str,
database: str,
user: str | None,
password: str | None,
driver: str,
kwargs: dict[str, Any] | None = None,
):
if pyodbc is None: # pragma: no cover
raise WrenError(
ErrorCode.GET_CONNECTION_ERROR, "pyodbc is required for MSSQL"
)
connect_kwargs = dict(kwargs) if kwargs else {}
statement_timeout = connect_kwargs.pop("statement_timeout", None)
# Validate statement_timeout before opening the connection so a bad
# value can't leak the pyodbc connection we'd otherwise open first.
if statement_timeout is not None:
try:
statement_timeout = int(statement_timeout)
except (TypeError, ValueError) as exc:
raise WrenError(
ErrorCode.INVALID_CONNECTION_INFO,
f"Invalid statement_timeout for MSSQL: {statement_timeout!r}",
) from exc
connection_parts = [
f"DRIVER={DataSourceExtension._escape_odbc_value(driver)}",
f"SERVER={host},{port}",
f"DATABASE={DataSourceExtension._escape_odbc_value(database)}",
]
if user is None and password is None:
connection_parts.append("Trusted_Connection=yes")
elif user is None or password is None:
raise WrenError(
ErrorCode.INVALID_CONNECTION_INFO,
"MSSQL connection requires both user and password, "
"or neither (for Trusted_Connection)",
)
else:
connection_parts.append(
f"UID={DataSourceExtension._escape_odbc_value(user)}"
)
connection_parts.append(
f"PWD={DataSourceExtension._escape_odbc_value(password)}"
)
for key, value in connect_kwargs.items():
connection_parts.append(
f"{key}={DataSourceExtension._escape_odbc_value(str(value))}"
)
connection = pyodbc.connect(";".join(connection_parts))
DataSourceExtension._register_mssql_output_converters(connection)
if statement_timeout is not None:
connection.timeout = statement_timeout
return connection
@staticmethod
def get_postgres_connection(info: PostgresConnectionInfo) -> BaseBackend:
return ibis.postgres.connect(
def _escape_odbc_value(value: str) -> str:
return "{" + value.replace("}", "}}") + "}"
@staticmethod
def _register_mssql_output_converters(connection) -> None:
connection.add_output_converter(
MSSQL_DATETIMEOFFSET_TYPE_CODE,
DataSourceExtension._decode_mssql_datetimeoffset,
)
@staticmethod
def _decode_mssql_datetimeoffset(value: bytes | None) -> dtlib.datetime | None:
if value is None:
return None
if len(value) != 20:
raise ValueError(
"unexpected mssql datetimeoffset payload length: "
f"expected 20, got {len(value)}"
)
year = int.from_bytes(value[0:2], "little")
month = int.from_bytes(value[2:4], "little")
day = int.from_bytes(value[4:6], "little")
hour = int.from_bytes(value[6:8], "little")
minute = int.from_bytes(value[8:10], "little")
second = int.from_bytes(value[10:12], "little")
nanoseconds = int.from_bytes(value[12:16], "little")
offset_hours = int.from_bytes(value[16:18], "little", signed=True)
offset_minutes = int.from_bytes(value[18:20], "little", signed=True)
tzinfo = dtlib.timezone(
dtlib.timedelta(hours=offset_hours, minutes=offset_minutes)
)
return dtlib.datetime(
year=year,
month=month,
day=day,
hour=hour,
minute=minute,
second=second,
microsecond=nanoseconds // 1000,
tzinfo=tzinfo,
)
@classmethod
def get_mysql_connection(cls, info: MySqlConnectionInfo) -> "MySQLdb.Connection":
import MySQLdb # noqa: PLC0415
from wren.connector.mysql import _build_mysql_connect_kwargs # noqa: PLC0415
return MySQLdb.connect(**_build_mysql_connect_kwargs(info))
@classmethod
def get_doris_connection(cls, info: DorisConnectionInfo) -> "MySQLdb.Connection":
import MySQLdb # noqa: PLC0415
from wren.connector.mysql import _build_doris_connect_kwargs # noqa: PLC0415
return MySQLdb.connect(**_build_doris_connect_kwargs(info))
@staticmethod
def get_postgres_connection(
info: PostgresConnectionInfo,
) -> "psycopg.Connection":
"""Open a native psycopg3 connection to PostgreSQL.
Returned object is a ``psycopg.Connection`` — the postgres connector
uses raw cursors and an OID-to-Arrow mapping to convert results.
"""
import psycopg # noqa: PLC0415
kwargs: dict[str, Any] = dict(info.kwargs) if info.kwargs else {}
return psycopg.connect(
host=info.host,
port=int(info.port),
database=info.database,
dbname=info.database,
user=info.user,
password=(info.password and info.password.get_secret_value()),
**info.kwargs if info.kwargs else {},
**kwargs,
)
@staticmethod
def get_oracle_connection(info: OracleConnectionInfo) -> BaseBackend:
import ibis # noqa: PLC0415
if hasattr(info, "dsn") and info.dsn:
return ibis.oracle.connect(
dsn=info.dsn.get_secret_value(),
@@ -406,51 +590,41 @@ class DataSourceExtension(Enum):
return make_snowflake_connection(info)
@staticmethod
def get_trino_connection(info: TrinoConnectionInfo) -> BaseBackend:
return ibis.trino.connect(
host=info.host,
port=int(info.port),
database=info.catalog,
schema=info.trino_schema,
user=info.user,
password=(info.password and info.password.get_secret_value()),
**info.kwargs if info.kwargs else {},
)
def get_trino_connection(info: TrinoConnectionInfo):
"""Return a ``trino.dbapi.Connection`` (not an ibis backend).
The wren SDK calls into ``wren.connector.trino`` for trino, which
operates directly on the DB-API cursor. ``get_connection`` only ever
re-routes here when something outside the v4 connector code path
explicitly requests a trino connection.
"""
from trino.auth import BasicAuthentication # noqa: PLC0415
from trino.dbapi import connect as trino_connect # noqa: PLC0415
kwargs = dict(info.kwargs) if info.kwargs else {}
password = info.password.get_secret_value() if info.password else None
connect_kwargs: dict = {
"host": info.host,
"port": int(info.port),
"user": info.user,
"catalog": info.catalog,
"schema": info.trino_schema,
# Pin to UTC so timestamp-with-tz casts are deterministic.
"timezone": "UTC",
}
if info.user and password:
connect_kwargs["auth"] = BasicAuthentication(info.user, password)
connect_kwargs["http_scheme"] = "https"
connect_kwargs.update(kwargs)
return trino_connect(**connect_kwargs)
@staticmethod
def get_databricks_connection(info: DatabricksTokenConnectionInfo) -> BaseBackend:
import ibis # noqa: PLC0415
return ibis.databricks.connect(
server_hostname=info.server_hostname,
http_path=info.http_path,
access_token=info.access_token.get_secret_value(),
)
@staticmethod
def _create_ssl_context(info: ConnectionInfo) -> ssl.SSLContext | None:
ssl_mode = (
info.ssl_mode if hasattr(info, "ssl_mode") and info.ssl_mode else None
)
if ssl_mode == SSLMode.VERIFY_CA and not info.ssl_ca:
raise WrenError(
ErrorCode.INVALID_CONNECTION_INFO,
"SSL CA must be provided when SSL mode is VERIFY CA",
)
if not ssl_mode or ssl_mode == SSLMode.DISABLED:
return None
ctx = ssl.create_default_context()
ctx.check_hostname = False
if ssl_mode == SSLMode.ENABLED:
ctx.verify_mode = ssl.CERT_NONE
elif ssl_mode == SSLMode.VERIFY_CA:
ctx.verify_mode = ssl.CERT_REQUIRED
ctx.load_verify_locations(
cadata=base64.b64decode(info.ssl_ca.get_secret_value()).decode("utf-8")
if info.ssl_ca
else None
)
return ctx
+8
View File
@@ -19,3 +19,11 @@ def pytest_configure(config: pytest.Config) -> None:
config.addinivalue_line(
"markers", "snowflake: Snowflake connector tests — mocked, no Docker required"
)
config.addinivalue_line(
"markers", "canner: Canner connector tests — requires Docker"
)
config.addinivalue_line(
"markers", "clickhouse: ClickHouse connector tests — requires Docker"
)
config.addinivalue_line("markers", "mssql: MSSQL connector tests — requires Docker")
config.addinivalue_line("markers", "trino: Trino connector tests — requires Docker")
+332
View File
@@ -0,0 +1,332 @@
"""Canner connector tests.
Canner Enterprise speaks the Postgres wire protocol, so we point a
``PostgresContainer`` at the connector and stand up tables with the data
types Canner publishes (Trino-style VARCHAR/DECIMAL/ARRAY/ROW/MAP plus the
usual postgres numeric/date/time types) to exercise the native psycopg
Arrow builder.
"""
from __future__ import annotations
from decimal import Decimal
from urllib.parse import urlparse
import pyarrow as pa
import pytest
from wren.connector.canner import (
CannerConnector,
_arrow_type,
_build_column,
_strip_trailing_semicolon,
)
from wren.model import CannerConnectionInfo
psycopg = pytest.importorskip("psycopg")
testcontainers_postgres = pytest.importorskip("testcontainers.postgres")
PostgresContainer = testcontainers_postgres.PostgresContainer
pytestmark = pytest.mark.canner
_FIXTURE_DDL = """
CREATE TABLE canner_demo (
id BIGINT PRIMARY KEY,
name VARCHAR(64) NOT NULL,
flag BOOLEAN,
amount DECIMAL(18, 4),
ratio DOUBLE PRECISION,
small SMALLINT,
sample_date DATE,
sample_ts TIMESTAMP,
sample_tstz TIMESTAMPTZ,
tags VARCHAR[],
struct_col JSON,
map_col JSONB
)
"""
_FIXTURE_ROWS = [
(
1,
"alpha",
True,
Decimal("12.3400"),
1.5,
7,
"2024-01-02",
"2024-01-02 03:04:05",
"2024-01-02 03:04:05+00",
["a", "b"],
'{"k": "v"}',
'{"m": 1}',
),
(
2,
"beta",
False,
None,
None,
None,
None,
None,
None,
None,
None,
None,
),
]
# ── helper-level unit tests ───────────────────────────────────────────────
def _column(type_code: int, precision: int | None = None, scale: int | None = None):
class _Col:
pass
col = _Col()
col.type_code = type_code
col.precision = precision
col.scale = scale
return col
def test_arrow_type_maps_canner_scalars() -> None:
# VARCHAR / CHAR / TEXT → string
assert _arrow_type(_column(1043)) == pa.string()
assert _arrow_type(_column(1042)) == pa.string()
assert _arrow_type(_column(25)) == pa.string()
# BIGINT / INTEGER / SMALLINT → int
assert _arrow_type(_column(20)) == pa.int64()
assert _arrow_type(_column(23)) == pa.int32()
assert _arrow_type(_column(21)) == pa.int16()
# BOOLEAN → bool
assert _arrow_type(_column(16)) == pa.bool_()
# DOUBLE / REAL → float
assert _arrow_type(_column(701)) == pa.float64()
assert _arrow_type(_column(700)) == pa.float32()
# DATE / TIMESTAMP / TIMESTAMPTZ → date/timestamp
assert _arrow_type(_column(1082)) == pa.date32()
assert _arrow_type(_column(1114)) == pa.timestamp("us")
assert _arrow_type(_column(1184)) == pa.timestamp("us", tz="UTC")
# NUMERIC honours precision/scale
assert _arrow_type(_column(1700, precision=18, scale=4)) == pa.decimal128(18, 4)
# JSON/JSONB (ROW/MAP) → string
assert _arrow_type(_column(114)) == pa.string()
assert _arrow_type(_column(3802)) == pa.string()
# ARRAY → list
assert _arrow_type(_column(1009)) == pa.list_(pa.string())
def test_build_column_serialises_complex_values_to_json() -> None:
array = _build_column([{"k": "v"}, [1, 2], "raw", None], pa.string(), 114)
# SQL NULL must stay Python None — only actual JSON literals are stringified.
assert array.to_pylist() == ['{"k": "v"}', "[1, 2]", "raw", None]
def test_build_column_preserves_sql_null_for_jsonb() -> None:
# Regression: a SQL NULL in a json (114) / jsonb (3802) column must stay
# Python None rather than being coerced to the string "null".
for oid in (114, 3802):
array = _build_column([None], pa.string(), oid)
assert array.to_pylist() == [None]
def test_build_column_quantises_decimal_values() -> None:
array = _build_column(
[Decimal("12.345678"), None],
pa.decimal128(18, 4),
)
assert array.to_pylist() == [Decimal("12.3457"), None]
def test_arrow_type_for_unconstrained_numeric_falls_back_to_string() -> None:
# NUMERIC without typmod (scale is None) must not silently quantise — we
# surface it as a string so high-precision values round-trip intact.
assert _arrow_type(_column(1700)) == pa.string()
# NUMERIC[] inherits the same behaviour for its element type.
assert _arrow_type(_column(1231)) == pa.list_(pa.string())
def test_build_column_preserves_unconstrained_numeric_precision() -> None:
# Regression: previously NUMERIC without typmod defaulted to scale=9, so
# values past the 9th decimal were silently rounded by Decimal.quantize.
high_precision = Decimal("12345678901234567890.123456789012345")
array = _build_column([high_precision, None], pa.string(), 1700)
assert array.to_pylist() == [str(high_precision), None]
@pytest.mark.parametrize(
("raw", "expected"),
[
("SELECT 1", "SELECT 1"),
("SELECT 1;", "SELECT 1"),
("SELECT 1 ; ", "SELECT 1"),
("SELECT 1;;", "SELECT 1"),
("SELECT 1;\n", "SELECT 1"),
# Semicolons inside string literals are *not* terminators — only the
# trailing run is stripped.
("SELECT 'a;b' FROM t", "SELECT 'a;b' FROM t"),
("SELECT 'a;b' FROM t;", "SELECT 'a;b' FROM t"),
],
)
def test_strip_trailing_semicolon(raw: str, expected: str) -> None:
assert _strip_trailing_semicolon(raw) == expected
def test_dry_run_returns_none_contract() -> None:
# Regression: ConnectorABC.dry_run() must return None. The cursor result
# must not leak out of the method, even on the success path.
class _FakeCursor:
def __enter__(self):
return self
def __exit__(self, *_args) -> None:
return None
def execute(self, _sql: str) -> "_FakeCursor":
return self
class _FakeConnection:
def cursor(self) -> _FakeCursor:
return _FakeCursor()
connector = CannerConnector.__new__(CannerConnector)
connector.connection = _FakeConnection()
connector._closed = False
assert connector.dry_run("SELECT 1") is None
# ── end-to-end testcontainer test ─────────────────────────────────────────
@pytest.fixture(scope="module")
def canner_connector():
with PostgresContainer("postgres:16") as pg:
url = pg.get_connection_url().replace("+psycopg2", "")
parsed = urlparse(url)
with psycopg.connect(url, autocommit=True) as conn:
with conn.cursor() as cur:
cur.execute(_FIXTURE_DDL)
cur.executemany(
"""
INSERT INTO canner_demo VALUES (
%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s
)
""",
_FIXTURE_ROWS,
)
# CannerConnectionInfo treats `workspace` like a postgres database and
# `pat` like a password — pass them through directly.
connection_info = CannerConnectionInfo(
host=parsed.hostname,
port=str(parsed.port),
user=parsed.username,
pat=parsed.password,
workspace=parsed.path.lstrip("/"),
)
connector = CannerConnector(connection_info)
try:
yield connector
finally:
connector.close()
def test_canner_connector_query_returns_arrow_table(canner_connector) -> None:
table = canner_connector.query("SELECT * FROM canner_demo ORDER BY id")
assert table.num_rows == 2
assert table.schema.field("id").type == pa.int64()
assert table.schema.field("name").type == pa.string()
assert table.schema.field("flag").type == pa.bool_()
assert table.schema.field("amount").type == pa.decimal128(18, 4)
assert table.schema.field("ratio").type == pa.float64()
assert table.schema.field("small").type == pa.int16()
assert table.schema.field("sample_date").type == pa.date32()
assert table.schema.field("sample_ts").type == pa.timestamp("us")
assert table.schema.field("sample_tstz").type == pa.timestamp("us", tz="UTC")
assert table.schema.field("tags").type == pa.list_(pa.string())
assert table.schema.field("struct_col").type == pa.string()
assert table.schema.field("map_col").type == pa.string()
rows = table.to_pylist()
row = rows[0]
assert row["id"] == 1
assert row["name"] == "alpha"
assert row["flag"] is True
assert row["amount"] == Decimal("12.3400")
assert row["tags"] == ["a", "b"]
# complex types come back as JSON strings
assert row["struct_col"] == '{"k": "v"}'
assert row["map_col"] == '{"m": 1}'
# SQL NULL in a JSON/JSONB column stays Python None — it must not be
# silently coerced into the string "null".
null_row = rows[1]
assert null_row["struct_col"] is None
assert null_row["map_col"] is None
def test_canner_connector_query_applies_limit(canner_connector) -> None:
table = canner_connector.query("SELECT * FROM canner_demo ORDER BY id", limit=1)
assert table.num_rows == 1
def test_canner_connector_query_preserves_duplicate_column_names(
canner_connector,
) -> None:
# Regression: dict-based pa.Table construction silently drops duplicate
# column names — a self-join projecting both ``id`` columns must keep both.
table = canner_connector.query(
"SELECT a.id, b.id FROM canner_demo a, canner_demo b ORDER BY a.id, b.id LIMIT 1"
)
assert table.num_columns == 2
assert [field.name for field in table.schema] == ["id", "id"]
def test_canner_connector_dry_run_succeeds(canner_connector) -> None:
# Returns None and must not raise on a valid statement.
assert canner_connector.dry_run("SELECT 1 AS x") is None
def test_canner_connector_dry_run_raises_for_invalid_sql(canner_connector) -> None:
from wren.model.error import WrenError # noqa: PLC0415
with pytest.raises(WrenError):
canner_connector.dry_run("SELECT * FROM no_such_table")
def test_canner_connector_preserves_unconstrained_numeric_precision(
canner_connector,
) -> None:
# Regression: unconstrained NUMERIC must round-trip without silent rounding.
# The cursor description reports scale=None for an unconstrained cast, so
# the connector falls back to pa.string() to keep the exact textual value.
literal = "12345678901234567890.123456789012345"
table = canner_connector.query(f"SELECT '{literal}'::numeric AS n")
assert table.schema.field("n").type == pa.string()
assert table.to_pylist() == [{"n": literal}]
def test_canner_connector_query_wraps_sql_with_trailing_semicolon(
canner_connector,
) -> None:
# Regression: a trailing semicolon on the user SQL must not break the
# ``SELECT * FROM (...) AS _t LIMIT N`` wrap that the connector applies
# when ``limit`` is provided.
table = canner_connector.query("SELECT 1 AS x;", limit=1)
assert table.num_rows == 1
assert table.to_pylist() == [{"x": 1}]
def test_canner_connector_dry_run_wraps_sql_with_trailing_semicolon(
canner_connector,
) -> None:
# Same regression for dry_run, which always wraps as ``... LIMIT 0``.
assert canner_connector.dry_run("SELECT 1 AS x;") is None
assert canner_connector.dry_run("SELECT 1 AS x ; ") is None
@@ -0,0 +1,291 @@
"""ClickHouse connector tests.
Uses ``testcontainers`` to spin up a real ClickHouse instance. TPCH-shaped
fixture data is fabricated inline in Python (no network downloads) and loaded
over the native ``clickhouse-connect`` HTTP client.
"""
from __future__ import annotations
import base64
import datetime as _dt
import time
import orjson
import pytest
from testcontainers.core.container import DockerContainer
from tests.suite.manifests import make_tpch_manifest
from tests.suite.query import WrenQueryTestSuite
from wren import WrenEngine
from wren.connector.clickhouse import (
_build_clickhouse_client_kwargs,
_parse_clickhouse_type,
)
from wren.model.data_source import DataSource
pytestmark = pytest.mark.clickhouse
_SCHEMA = "default"
_ORDER_COUNT = 15000
_CUSTOMER_COUNT = 1500
_ORDER_STATUSES = ("O", "F", "P")
_BASE_DATE = _dt.date(1992, 1, 1)
def _make_fixture_rows() -> tuple[list[tuple], list[tuple]]:
"""Fabricate TPCH-shaped orders + customer rows without network access.
Row counts match TPCH sf=0.01 so the shared ``WrenQueryTestSuite``
assertions (15000 orders, 1500 customers, first orderkey == 1) hold.
"""
customers = [(i, f"Customer#{i:09d}") for i in range(1, _CUSTOMER_COUNT + 1)]
orders = [
(
i,
((i - 1) % _CUSTOMER_COUNT) + 1,
_ORDER_STATUSES[i % len(_ORDER_STATUSES)],
float(100 + i),
_BASE_DATE + _dt.timedelta(days=i % 3650),
)
for i in range(1, _ORDER_COUNT + 1)
]
return orders, customers
class _ClickHouseContainer(DockerContainer):
"""Minimal ClickHouse container wrapper — exposes HTTP port 8123."""
def __init__(self, image: str = "clickhouse/clickhouse-server:24.3-alpine"):
super().__init__(image)
self.with_exposed_ports(8123, 9000)
# Use the default tcp_port_secure-free defaults; no auth.
self.with_env("CLICKHOUSE_DB", _SCHEMA)
self.with_env("CLICKHOUSE_USER", "default")
self.with_env("CLICKHOUSE_PASSWORD", "")
# Allow empty password (the default user already has it; this is for
# any user clickhouse-connect tries to authenticate as).
self.with_env("CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT", "1")
def start(self): # type: ignore[override]
super().start()
# The alpine image redirects ClickHouse logs to file, so we cannot rely
# on ``wait_for_logs``. ``_wait_for_http_ready`` polls the HTTP endpoint
# instead.
return self
def get_host_ip(self) -> str:
return self.get_container_host_ip()
def get_http_port(self) -> int:
return int(self.get_exposed_port(8123))
def _wait_for_http_ready(host: str, port: int, timeout: float = 120.0) -> None:
"""Poll the ClickHouse HTTP endpoint until it responds to a trivial query."""
import clickhouse_connect # noqa: PLC0415
deadline = time.time() + timeout
last_err: Exception | None = None
while time.time() < deadline:
client = None
try:
client = clickhouse_connect.get_client(
host=host, port=port, username="default", password=""
)
client.query("SELECT 1")
return
except Exception as e: # noqa: BLE001
last_err = e
time.sleep(1)
finally:
if client is not None:
client.close()
raise RuntimeError(f"ClickHouse did not become ready: {last_err}")
def _load_tpch(host: str, port: int) -> None:
"""Bulk-load fabricated TPCH-shaped data into ClickHouse."""
import clickhouse_connect # noqa: PLC0415
orders_rows, customer_rows = _make_fixture_rows()
client = clickhouse_connect.get_client(
host=host, port=port, username="default", password="", database=_SCHEMA
)
try:
client.command(
"CREATE TABLE IF NOT EXISTS orders ("
" o_orderkey Int32,"
" o_custkey Int32,"
" o_orderstatus String,"
" o_totalprice Float64,"
" o_orderdate Date"
") ENGINE = MergeTree ORDER BY o_orderkey"
)
client.command(
"CREATE TABLE IF NOT EXISTS customer ("
" c_custkey Int32,"
" c_name String"
") ENGINE = MergeTree ORDER BY c_custkey"
)
client.insert(
"orders",
orders_rows,
column_names=[
"o_orderkey",
"o_custkey",
"o_orderstatus",
"o_totalprice",
"o_orderdate",
],
)
client.insert("customer", customer_rows, column_names=["c_custkey", "c_name"])
finally:
client.close()
class TestClickHouse(WrenQueryTestSuite):
manifest = make_tpch_manifest(table_catalog=None, table_schema=_SCHEMA)
# ClickHouse `Int32` round-trips to Arrow as ``int32`` via the native
# connector's sqlglot-driven type mapping.
order_id_dtype = "int32"
@pytest.fixture(scope="class")
def engine(self) -> WrenEngine: # type: ignore[override]
with _ClickHouseContainer() as ch:
host = ch.get_host_ip()
port = ch.get_http_port()
_wait_for_http_ready(host, port)
_load_tpch(host, port)
conn_info = {
"host": host,
"port": port,
"database": _SCHEMA,
"user": "default",
"password": "",
}
manifest_str = base64.b64encode(orjson.dumps(self.manifest)).decode()
with WrenEngine(
manifest_str, DataSource.clickhouse, conn_info, fallback=False
) as e:
yield e
@pytest.mark.clickhouse
class TestClickHouseTypeParser:
"""Pure-Python tests for the ClickHouse type-string → Arrow mapping.
Runs without Docker — exercises ``_parse_clickhouse_type`` directly.
"""
@pytest.mark.parametrize(
("type_str", "expected"),
[
("String", "string"),
("FixedString(8)", "string"),
("Int8", "int8"),
("Int16", "int16"),
("Int32", "int32"),
("Int64", "int64"),
("UInt8", "uint8"),
("UInt16", "uint16"),
("UInt32", "uint32"),
("UInt64", "uint64"),
("Int128", "string"),
("Int256", "string"),
("UInt128", "string"),
("UInt256", "string"),
("Float32", "float"),
("Float64", "double"),
("Bool", "bool"),
("UUID", "string"),
("IPv4", "string"),
("IPv6", "string"),
("Date", "date32[day]"),
("Date32", "date32[day]"),
("DateTime", "timestamp[ns]"),
("Decimal(18, 4)", "decimal128(38, 9)"),
("Nullable(Int32)", "int32"),
("Nullable(String)", "string"),
("LowCardinality(String)", "string"),
("LowCardinality(Nullable(String))", "string"),
("Array(Int32)", "list<item: int32>"),
("Array(Nullable(String))", "list<item: string>"),
("Map(String, Int32)", "map<string, int32>"),
("Tuple(a Int32, b String)", "string"),
("Enum8('a' = 1, 'b' = 2)", "string"),
],
)
def test_type_parse(self, type_str: str, expected: str) -> None:
result = _parse_clickhouse_type(type_str)
assert str(result) == expected
def test_datetime64_with_tz(self) -> None:
result = _parse_clickhouse_type("DateTime64(3, 'UTC')")
assert str(result) == "timestamp[ns, tz=UTC]"
def test_datetime_with_tz(self) -> None:
result = _parse_clickhouse_type("DateTime('Asia/Taipei')")
assert str(result) == "timestamp[ns, tz=Asia/Taipei]"
def test_unknown_type_defaults_to_string(self) -> None:
result = _parse_clickhouse_type("SomethingExotic")
assert str(result) == "string"
def test_none_type_defaults_to_string(self) -> None:
result = _parse_clickhouse_type(None)
assert str(result) == "string"
class _FakeChInfo:
"""Stand-in for ``ClickHouseConnectionInfo`` used by the kwargs builder.
Pydantic enforces ``kwargs: dict[str, str]`` on the real model, so we
bypass it here to exercise the merge logic with nested ``settings``.
``_build_clickhouse_client_kwargs`` only reads attributes off the object.
"""
def __init__(self, **attrs) -> None:
self.host = attrs.get("host", "localhost")
self.port = attrs.get("port", "8123")
self.database = attrs.get("database", "default")
self.user = attrs.get("user", "default")
self.password = attrs.get("password")
self.secure = attrs.get("secure", False)
self.settings = attrs.get("settings")
self.kwargs = attrs.get("kwargs")
@pytest.mark.clickhouse
class TestClickHouseClientKwargs:
"""Pure-Python tests for ``_build_clickhouse_client_kwargs``.
Exercises the kwargs/settings merge logic without spinning up a real
ClickHouse instance.
"""
def test_statement_timeout_survives_user_settings(self) -> None:
"""statement_timeout must merge with — not be clobbered by — user settings."""
info = _FakeChInfo(
kwargs={
"statement_timeout": 10,
"settings": {"max_threads": 4},
},
)
out = _build_clickhouse_client_kwargs(info)
assert out["settings"] == {
"max_execution_time": 10,
"max_threads": 4,
}
def test_user_settings_only(self) -> None:
info = _FakeChInfo(kwargs={"settings": {"max_threads": 4}})
out = _build_clickhouse_client_kwargs(info)
assert out["settings"] == {"max_threads": 4}
def test_statement_timeout_only(self) -> None:
info = _FakeChInfo(kwargs={"statement_timeout": 10})
out = _build_clickhouse_client_kwargs(info)
assert out["settings"] == {"max_execution_time": 10}
+305
View File
@@ -0,0 +1,305 @@
"""MSSQL connector tests.
Uses testcontainers to spin up a real SQL Server instance.
TPCH data is generated via DuckDB's built-in extension and loaded via pyodbc.
"""
from __future__ import annotations
import base64
import datetime as dtlib
from decimal import Decimal as PyDecimal
import duckdb
import orjson
import pyarrow as pa
import pytest
pyodbc = pytest.importorskip("pyodbc", reason="pyodbc not installed (mssql extra)")
testcontainers_mssql = pytest.importorskip(
"testcontainers.mssql", reason="testcontainers[mssql] not installed"
)
SqlServerContainer = testcontainers_mssql.SqlServerContainer
from tests.suite.manifests import make_tpch_manifest # noqa: E402
from tests.suite.query import WrenQueryTestSuite # noqa: E402
from wren import WrenEngine # noqa: E402
from wren.connector.mssql import MSSqlConnector # noqa: E402
from wren.model import MSSqlConnectionInfo # noqa: E402
from wren.model.data_source import DataSource # noqa: E402
from wren.model.error import WrenError # noqa: E402
_SCHEMA = "dbo"
_MSSQL_IMAGE = "mcr.microsoft.com/mssql/server:2022-latest"
_DRIVER = "ODBC Driver 18 for SQL Server"
def _have_mssql_driver() -> bool:
try:
return _DRIVER in pyodbc.drivers()
except Exception:
return False
pytestmark = [
pytest.mark.mssql,
pytest.mark.skipif(
not _have_mssql_driver(),
reason=f"{_DRIVER} not installed",
),
]
def _pyodbc_connect(container: SqlServerContainer) -> pyodbc.Connection:
host = container.get_container_host_ip()
port = container.get_exposed_port(container.port)
password = container.password
user = container.username
database = container.dbname
conn_str = (
f"DRIVER={{{_DRIVER}}};"
f"SERVER={host},{port};"
f"DATABASE={database};"
f"UID={user};PWD={password};"
"TrustServerCertificate=yes;"
)
return pyodbc.connect(conn_str)
def _load_tpch(container: SqlServerContainer) -> None:
"""Generate TPCH sf=0.01 via DuckDB and bulk-load into SQL Server."""
duck = duckdb.connect()
duck.execute("INSTALL tpch; LOAD tpch; CALL dbgen(sf=0.01)")
orders_rows = duck.execute(
"SELECT o_orderkey, o_custkey, o_orderstatus, "
"cast(o_totalprice as double), o_orderdate FROM orders"
).fetchall()
customer_rows = duck.execute("SELECT c_custkey, c_name FROM customer").fetchall()
duck.close()
conn = _pyodbc_connect(container)
conn.autocommit = True
with conn:
cur = conn.cursor()
cur.execute("""
CREATE TABLE orders (
o_orderkey INT PRIMARY KEY,
o_custkey INT NOT NULL,
o_orderstatus CHAR(1) NOT NULL,
o_totalprice FLOAT NOT NULL,
o_orderdate DATE NOT NULL
)
""")
cur.fast_executemany = True
cur.executemany(
"INSERT INTO orders VALUES (?, ?, ?, ?, ?)",
[(k, c, s, float(p), d) for (k, c, s, p, d) in orders_rows],
)
cur.execute("""
CREATE TABLE customer (
c_custkey INT PRIMARY KEY,
c_name VARCHAR(25) NOT NULL
)
""")
cur.executemany("INSERT INTO customer VALUES (?, ?)", customer_rows)
cur.close()
@pytest.fixture(scope="module")
def mssql_container():
with SqlServerContainer(_MSSQL_IMAGE, dialect="mssql+pyodbc") as ms:
_load_tpch(ms)
yield ms
@pytest.fixture(scope="module")
def conn_info(mssql_container: SqlServerContainer) -> dict:
return {
"host": mssql_container.get_container_host_ip(),
"port": str(mssql_container.get_exposed_port(mssql_container.port)),
"database": mssql_container.dbname,
"user": mssql_container.username,
"password": mssql_container.password,
"driver": _DRIVER,
"kwargs": {"TrustServerCertificate": "yes"},
}
class TestMSSqlEngine(WrenQueryTestSuite):
manifest = make_tpch_manifest(table_catalog=None, table_schema=_SCHEMA)
# SQL Server INT → Arrow int32 with our value-sampling inference
order_id_dtype = "int32"
@pytest.fixture(scope="class")
def engine(self, conn_info) -> WrenEngine: # type: ignore[override]
manifest_str = base64.b64encode(orjson.dumps(self.manifest)).decode()
with WrenEngine(manifest_str, DataSource.mssql, conn_info, fallback=False) as e:
yield e
# ----------------------------------------------------------------------
# Type-specific tests (no MDL — directly via the connector)
# ----------------------------------------------------------------------
@pytest.fixture(scope="module")
def types_table(mssql_container: SqlServerContainer) -> str:
conn = _pyodbc_connect(mssql_container)
conn.autocommit = True
table = "mssql_types"
with conn:
cur = conn.cursor()
cur.execute(f"""
CREATE TABLE {table} (
c_int INT,
c_smallint SMALLINT,
c_bigint BIGINT,
c_tinyint TINYINT,
c_bit BIT,
c_varchar VARCHAR(50),
c_decimal DECIMAL(18,4),
c_datetime DATETIME,
c_datetime2 DATETIME2,
c_dto DATETIMEOFFSET,
c_dto_utc DATETIMEOFFSET,
c_uuid UNIQUEIDENTIFIER,
c_varbinary VARBINARY(16)
)
""")
cur.execute(
f"INSERT INTO {table} VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
(
42,
7,
10_000_000_000,
255,
1,
"hello",
PyDecimal("123.4500"),
dtlib.datetime(2024, 1, 2, 3, 4, 5),
dtlib.datetime(2024, 1, 2, 3, 4, 5, 678900),
"2024-06-15 12:00:00 +05:30",
"2024-06-15 12:00:00 +00:00",
"00000000-0000-0000-0000-000000000001",
b"\x01\x02\x03",
),
)
cur.close()
return table
@pytest.fixture(scope="module")
def connector(conn_info) -> MSSqlConnector:
info = MSSqlConnectionInfo.model_validate(conn_info)
c = MSSqlConnector(info)
try:
yield c
finally:
c.close()
def test_int_types(connector: MSSqlConnector, types_table: str) -> None:
result = connector.query(
f"SELECT c_int, c_smallint, c_bigint, c_tinyint FROM {types_table}"
)
assert str(result.schema.field("c_int").type) == "int32"
assert str(result.schema.field("c_smallint").type) == "int16"
assert str(result.schema.field("c_bigint").type) == "int64"
# TINYINT is unsigned in SQL Server; sampled non-negative
assert str(result.schema.field("c_tinyint").type) == "uint8"
assert result["c_int"][0].as_py() == 42
assert result["c_tinyint"][0].as_py() == 255
def test_bit_and_varchar(connector: MSSqlConnector, types_table: str) -> None:
result = connector.query(f"SELECT c_bit, c_varchar FROM {types_table}")
assert result.schema.field("c_bit").type == pa.bool_()
assert result["c_bit"][0].as_py() is True
assert result["c_varchar"][0].as_py() == "hello"
def test_decimal_as_string(connector: MSSqlConnector, types_table: str) -> None:
result = connector.query(f"SELECT c_decimal FROM {types_table}")
# Decimals serialise as strings to avoid arrow decimal precision pitfalls
assert str(result.schema.field("c_decimal").type) == "string"
assert result["c_decimal"][0].as_py() == "123.4500"
def test_datetime_columns(connector: MSSqlConnector, types_table: str) -> None:
result = connector.query(f"SELECT c_datetime, c_datetime2 FROM {types_table}")
assert str(result.schema.field("c_datetime").type) == "timestamp[ns]"
assert str(result.schema.field("c_datetime2").type) == "timestamp[ns]"
def test_datetimeoffset_non_utc(connector: MSSqlConnector, types_table: str) -> None:
result = connector.query(f"SELECT c_dto FROM {types_table}")
assert str(result.schema.field("c_dto").type) == "timestamp[ns, tz=+05:30]"
value = result["c_dto"][0].as_py()
assert isinstance(value, dtlib.datetime)
assert value.utcoffset() == dtlib.timedelta(hours=5, minutes=30)
def test_datetimeoffset_utc(connector: MSSqlConnector, types_table: str) -> None:
result = connector.query(f"SELECT c_dto_utc FROM {types_table}")
assert str(result.schema.field("c_dto_utc").type) == "timestamp[ns, tz=UTC]"
value = result["c_dto_utc"][0].as_py()
assert value.utcoffset() == dtlib.timedelta(0)
def test_uuid_as_string(connector: MSSqlConnector, types_table: str) -> None:
result = connector.query(f"SELECT c_uuid FROM {types_table}")
assert str(result.schema.field("c_uuid").type) == "string"
# SQL Server returns uppercase UUIDs
assert result["c_uuid"][0].as_py().lower() == (
"00000000-0000-0000-0000-000000000001"
)
def test_varbinary_as_binary(connector: MSSqlConnector, types_table: str) -> None:
result = connector.query(f"SELECT c_varbinary FROM {types_table}")
assert str(result.schema.field("c_varbinary").type) == "binary"
assert result["c_varbinary"][0].as_py() == b"\x01\x02\x03"
def test_dry_run_invalid_column_returns_describe_error(
connector: MSSqlConnector, types_table: str
) -> None:
with pytest.raises(WrenError) as exc:
connector.dry_run(f"SELECT not_a_column FROM {types_table}")
assert "dry run failed" in str(exc.value).lower()
def test_raw_cursor_sql_injects_fetch_next() -> None:
rewritten = MSSqlConnector._raw_cursor_sql("SELECT * FROM orders", 10)
# sqlglot emits OFFSET 0 ROWS FETCH NEXT n ROWS ONLY for tsql with LIMIT
lower = rewritten.lower()
assert "fetch next 10 rows only" in lower
assert "offset 0 rows" in lower
def test_raw_cursor_sql_no_limit_passthrough() -> None:
sql = "SELECT * FROM orders"
assert MSSqlConnector._raw_cursor_sql(sql, None) == sql
def test_url_connection(mssql_container: SqlServerContainer) -> None:
host = mssql_container.get_container_host_ip()
port = mssql_container.get_exposed_port(mssql_container.port)
password = mssql_container.password
user = mssql_container.username
database = mssql_container.dbname
url = (
f"mssql://{user}:{password}@{host}:{port}/{database}?TrustServerCertificate=yes"
)
info = {"connectionUrl": url}
parsed = DataSource.mssql.get_connection_info(info)
conn = DataSource.mssql.get_connection(parsed)
try:
cur = conn.cursor()
cur.execute("SELECT 1 AS x")
assert cur.fetchone()[0] == 1
cur.close()
finally:
conn.close()
+1 -1
View File
@@ -75,7 +75,7 @@ class TestMySQL(WrenQueryTestSuite):
@pytest.fixture(scope="class")
def engine(self) -> WrenEngine: # type: ignore[override]
with MySqlContainer("mysql:8") as mysql:
with MySqlContainer("mysql:8.0.36") as mysql:
url = mysql.get_connection_url()
_load_tpch(url)
@@ -0,0 +1,335 @@
"""MySQL native connector type-coverage tests.
Spins up a real MySQL via testcontainers and exercises every field-type
to Arrow conversion path that the native ``MySqlConnector`` supports.
"""
from __future__ import annotations
from contextlib import closing
from decimal import Decimal
import pyarrow as pa
import pytest
from testcontainers.mysql import MySqlContainer
from wren.connector.mysql import MySqlConnector
from wren.model import MySqlConnectionInfo
pytestmark = pytest.mark.mysql
@pytest.fixture(scope="module")
def mysql_container():
with MySqlContainer("mysql:8.0.36") as mysql:
yield mysql
@pytest.fixture(scope="module")
def connector(mysql_container):
info = MySqlConnectionInfo(
host=mysql_container.get_container_host_ip(),
port=mysql_container.get_exposed_port(3306),
database=mysql_container.dbname,
user=mysql_container.username,
password=mysql_container.password,
sslMode="disabled",
)
c = MySqlConnector(info)
try:
yield c
finally:
c.close()
def _exec(connector: MySqlConnector, sql: str) -> None:
with closing(connector.connection.cursor()) as cur:
cur.execute(sql)
def test_tinyint_signed_and_unsigned(connector: MySqlConnector) -> None:
_exec(connector, "DROP TABLE IF EXISTS t_tiny")
_exec(
connector,
"CREATE TABLE t_tiny (a TINYINT, b TINYINT UNSIGNED)",
)
_exec(connector, "INSERT INTO t_tiny VALUES (-1, 200)")
tbl = connector.query("SELECT a, b FROM t_tiny")
assert tbl.schema.field("a").type == pa.int8()
assert tbl.schema.field("b").type == pa.uint8()
assert tbl.column("a").to_pylist() == [-1]
assert tbl.column("b").to_pylist() == [200]
def test_smallint(connector: MySqlConnector) -> None:
_exec(connector, "DROP TABLE IF EXISTS t_small")
_exec(connector, "CREATE TABLE t_small (a SMALLINT, b SMALLINT UNSIGNED)")
_exec(connector, "INSERT INTO t_small VALUES (-32000, 65000)")
tbl = connector.query("SELECT a, b FROM t_small")
assert tbl.schema.field("a").type == pa.int16()
assert tbl.schema.field("b").type == pa.uint16()
def test_int_and_bigint(connector: MySqlConnector) -> None:
_exec(connector, "DROP TABLE IF EXISTS t_int")
_exec(
connector,
"CREATE TABLE t_int (a INT, b INT UNSIGNED, c BIGINT, d BIGINT UNSIGNED)",
)
_exec(
connector, "INSERT INTO t_int VALUES (-1, 4000000000, -1, 18000000000000000000)"
)
tbl = connector.query("SELECT a, b, c, d FROM t_int")
assert tbl.schema.field("a").type == pa.int32()
assert tbl.schema.field("b").type == pa.uint32()
assert tbl.schema.field("c").type == pa.int64()
assert tbl.schema.field("d").type == pa.uint64()
def test_decimal(connector: MySqlConnector) -> None:
_exec(connector, "DROP TABLE IF EXISTS t_dec")
_exec(connector, "CREATE TABLE t_dec (a DECIMAL(12, 4))")
_exec(connector, "INSERT INTO t_dec VALUES (1234.5678)")
tbl = connector.query("SELECT a FROM t_dec")
arrow_type = tbl.schema.field("a").type
assert pa.types.is_decimal(arrow_type)
# Precision/scale now reflect the column definition instead of the previous
# hard-coded ``decimal128(38, 9)``.
assert arrow_type.precision == 12
assert arrow_type.scale == 4
assert tbl.column("a").to_pylist()[0] == Decimal("1234.5678")
def test_decimal_large_scale(connector: MySqlConnector) -> None:
"""DECIMAL with scale > 9 must round-trip without truncating digits.
The previous hard-coded ``pa.decimal128(38, 9)`` silently dropped digits
beyond the 9th decimal place. We now derive precision/scale from
``cursor.description``.
"""
_exec(connector, "DROP TABLE IF EXISTS t_dec_big")
_exec(connector, "CREATE TABLE t_dec_big (a DECIMAL(30, 15))")
_exec(connector, "INSERT INTO t_dec_big VALUES (12345.123456789012345)")
tbl = connector.query("SELECT a FROM t_dec_big")
arrow_type = tbl.schema.field("a").type
assert pa.types.is_decimal(arrow_type)
assert arrow_type.precision == 30
assert arrow_type.scale == 15
assert tbl.column("a").to_pylist()[0] == Decimal("12345.123456789012345")
def test_float_and_double(connector: MySqlConnector) -> None:
_exec(connector, "DROP TABLE IF EXISTS t_real")
_exec(connector, "CREATE TABLE t_real (a FLOAT, b DOUBLE)")
_exec(connector, "INSERT INTO t_real VALUES (1.5, 2.5)")
tbl = connector.query("SELECT a, b FROM t_real")
assert tbl.schema.field("a").type == pa.float32()
assert tbl.schema.field("b").type == pa.float64()
def test_char_varchar_text(connector: MySqlConnector) -> None:
_exec(connector, "DROP TABLE IF EXISTS t_str")
_exec(connector, "CREATE TABLE t_str (a CHAR(8), b VARCHAR(32), c TEXT)")
_exec(connector, "INSERT INTO t_str VALUES ('abc', 'hello', 'world')")
tbl = connector.query("SELECT a, b, c FROM t_str")
for col in ("a", "b", "c"):
assert tbl.schema.field(col).type == pa.string()
assert tbl.column("b").to_pylist() == ["hello"]
assert tbl.column("c").to_pylist() == ["world"]
def test_json(connector: MySqlConnector) -> None:
_exec(connector, "DROP TABLE IF EXISTS t_json")
_exec(connector, "CREATE TABLE t_json (a JSON)")
_exec(connector, """INSERT INTO t_json VALUES ('{"k": 1}')""")
tbl = connector.query("SELECT a FROM t_json")
assert tbl.schema.field("a").type == pa.string()
assert "k" in tbl.column("a").to_pylist()[0]
def test_blob_binary(connector: MySqlConnector) -> None:
_exec(connector, "DROP TABLE IF EXISTS t_blob")
_exec(connector, "CREATE TABLE t_blob (a BLOB, b VARBINARY(16))")
_exec(connector, "INSERT INTO t_blob VALUES (X'DEADBEEF', X'ABCD')")
tbl = connector.query("SELECT a, b FROM t_blob")
assert pa.types.is_binary(tbl.schema.field("a").type)
assert pa.types.is_binary(tbl.schema.field("b").type)
assert tbl.column("a").to_pylist()[0] == b"\xde\xad\xbe\xef"
def test_datetime_and_timestamp(connector: MySqlConnector) -> None:
_exec(connector, "DROP TABLE IF EXISTS t_dt")
_exec(connector, "CREATE TABLE t_dt (a DATETIME, b TIMESTAMP NULL)")
_exec(
connector,
"INSERT INTO t_dt VALUES ('2024-01-02 03:04:05', '2024-01-02 03:04:05')",
)
tbl = connector.query("SELECT a, b FROM t_dt")
assert pa.types.is_timestamp(tbl.schema.field("a").type)
assert pa.types.is_timestamp(tbl.schema.field("b").type)
def test_date(connector: MySqlConnector) -> None:
_exec(connector, "DROP TABLE IF EXISTS t_date")
_exec(connector, "CREATE TABLE t_date (a DATE)")
_exec(connector, "INSERT INTO t_date VALUES ('2024-05-14')")
tbl = connector.query("SELECT a FROM t_date")
assert tbl.schema.field("a").type == pa.date32()
def test_time(connector: MySqlConnector) -> None:
_exec(connector, "DROP TABLE IF EXISTS t_time")
_exec(connector, "CREATE TABLE t_time (a TIME)")
_exec(connector, "INSERT INTO t_time VALUES ('12:34:56')")
tbl = connector.query("SELECT a FROM t_time")
# MySQL TIME maps to Arrow ``duration("us")``: ``time64`` cannot represent
# negative or >24h values that MySQL ``TIME`` permits.
assert pa.types.is_duration(tbl.schema.field("a").type)
def test_time_full_range(connector: MySqlConnector) -> None:
"""MySQL ``TIME`` ranges ``-838:59:59`` to ``838:59:59``.
The previous mapping to ``pa.time64("us")`` silently corrupted negative
values and values past 24h (``time64`` only accepts 024h positive). Map
to ``duration("us")`` instead so the full MySQL range round-trips.
"""
import datetime # noqa: PLC0415
_exec(connector, "DROP TABLE IF EXISTS t_time_range")
_exec(connector, "CREATE TABLE t_time_range (label VARCHAR(16), a TIME)")
_exec(
connector,
"INSERT INTO t_time_range VALUES "
"('neg_100', '-100:00:00'), "
"('zero', '0:00:00'), "
"('max', '838:59:59'), "
"('min', '-838:59:59')",
)
tbl = connector.query("SELECT label, a FROM t_time_range ORDER BY label")
arrow_type = tbl.schema.field("a").type
assert pa.types.is_duration(arrow_type)
by_label = dict(
zip(tbl.column("label").to_pylist(), tbl.column("a").to_pylist(), strict=True)
)
# ``duration("us")`` round-trips to ``datetime.timedelta`` in PyArrow.
assert by_label["neg_100"] == datetime.timedelta(hours=-100)
assert by_label["zero"] == datetime.timedelta(0)
assert by_label["max"] == datetime.timedelta(hours=838, minutes=59, seconds=59)
assert by_label["min"] == -datetime.timedelta(hours=838, minutes=59, seconds=59)
def test_year(connector: MySqlConnector) -> None:
_exec(connector, "DROP TABLE IF EXISTS t_year")
_exec(connector, "CREATE TABLE t_year (a YEAR)")
_exec(connector, "INSERT INTO t_year VALUES (2024)")
tbl = connector.query("SELECT a FROM t_year")
assert tbl.schema.field("a").type == pa.int16()
assert tbl.column("a").to_pylist() == [2024]
def test_bit(connector: MySqlConnector) -> None:
_exec(connector, "DROP TABLE IF EXISTS t_bit")
_exec(connector, "CREATE TABLE t_bit (a BIT(8))")
_exec(connector, "INSERT INTO t_bit VALUES (b'10101010')")
tbl = connector.query("SELECT a FROM t_bit")
assert pa.types.is_binary(tbl.schema.field("a").type)
def test_enum_and_set(connector: MySqlConnector) -> None:
_exec(connector, "DROP TABLE IF EXISTS t_enum")
_exec(
connector,
"CREATE TABLE t_enum (a ENUM('x', 'y', 'z'), b SET('p', 'q', 'r'))",
)
_exec(connector, "INSERT INTO t_enum VALUES ('y', 'p,r')")
tbl = connector.query("SELECT a, b FROM t_enum")
assert tbl.schema.field("a").type == pa.string()
assert tbl.schema.field("b").type == pa.string()
assert tbl.column("a").to_pylist() == ["y"]
def test_null_handling(connector: MySqlConnector) -> None:
_exec(connector, "DROP TABLE IF EXISTS t_null")
_exec(connector, "CREATE TABLE t_null (a INT, b VARCHAR(8))")
_exec(connector, "INSERT INTO t_null VALUES (NULL, NULL)")
tbl = connector.query("SELECT a, b FROM t_null")
assert tbl.column("a").to_pylist() == [None]
assert tbl.column("b").to_pylist() == [None]
def test_query_with_limit(connector: MySqlConnector) -> None:
_exec(connector, "DROP TABLE IF EXISTS t_limit")
_exec(connector, "CREATE TABLE t_limit (a INT)")
_exec(connector, "INSERT INTO t_limit VALUES (1), (2), (3), (4), (5)")
tbl = connector.query("SELECT a FROM t_limit", limit=2)
assert tbl.num_rows == 2
def test_query_limit_rejects_sql_injection(connector: MySqlConnector) -> None:
"""``limit`` must be coerced via ``int()`` so it can be safely interpolated.
Passing a crafted string would previously land directly in the rendered
SQL via an f-string. ``int()`` rejects it with ``ValueError``.
"""
_exec(connector, "DROP TABLE IF EXISTS t_inj")
_exec(connector, "CREATE TABLE t_inj (a INT)")
_exec(connector, "INSERT INTO t_inj VALUES (1), (2)")
with pytest.raises((ValueError, TypeError)):
connector.query("SELECT a FROM t_inj", limit="1; DROP TABLE t_inj")
# Table must still exist — the malicious payload never reached the server.
tbl = connector.query("SELECT a FROM t_inj")
assert tbl.num_rows == 2
def test_query_with_duplicate_column_names(connector: MySqlConnector) -> None:
"""``SELECT * FROM (...) AS _sub`` would fail with ER_DUP_FIELDNAME on a
join that exposes the same column name twice. Appending ``LIMIT`` to the
user SQL avoids the subquery and so avoids the duplicate-column error.
Also asserts the resulting Arrow table preserves BOTH ``id`` fields —
building the table via ``dict(zip(names, arrays))`` would silently drop
one of the duplicate columns because the dict collapses the key.
"""
_exec(connector, "DROP TABLE IF EXISTS t_dup_a")
_exec(connector, "DROP TABLE IF EXISTS t_dup_b")
_exec(connector, "CREATE TABLE t_dup_a (id INT, val INT)")
_exec(connector, "CREATE TABLE t_dup_b (id INT, val INT)")
_exec(connector, "INSERT INTO t_dup_a VALUES (1, 10), (2, 20)")
_exec(connector, "INSERT INTO t_dup_b VALUES (1, 100), (2, 200)")
sql = "SELECT a.id, b.id FROM t_dup_a a JOIN t_dup_b b ON a.id = b.id"
tbl = connector.query(sql, limit=10)
assert tbl.num_rows == 2
# Two ``id`` columns must survive — the schema is name-positional.
assert tbl.num_columns == 2
assert [f.name for f in tbl.schema] == ["id", "id"]
# Both columns hold the same data (joined on ``id``), but they must each
# exist independently in the result.
assert tbl.column(0).to_pylist() == tbl.column(1).to_pylist()
# dry_run should also work on duplicate-column queries.
connector.dry_run(sql)
def test_query_trailing_semicolon(connector: MySqlConnector) -> None:
"""Trailing semicolons must be stripped before appending ``LIMIT``."""
_exec(connector, "DROP TABLE IF EXISTS t_semi")
_exec(connector, "CREATE TABLE t_semi (a INT)")
_exec(connector, "INSERT INTO t_semi VALUES (1), (2), (3)")
tbl = connector.query("SELECT a FROM t_semi;", limit=2)
assert tbl.num_rows == 2
def test_dry_run(connector: MySqlConnector) -> None:
_exec(connector, "DROP TABLE IF EXISTS t_dry")
_exec(connector, "CREATE TABLE t_dry (a INT)")
connector.dry_run("SELECT a FROM t_dry") # must not raise
def test_empty_result(connector: MySqlConnector) -> None:
_exec(connector, "DROP TABLE IF EXISTS t_empty")
_exec(connector, "CREATE TABLE t_empty (a INT, b VARCHAR(8))")
tbl = connector.query("SELECT a, b FROM t_empty")
assert tbl.num_rows == 0
assert tbl.schema.field("a").type == pa.int32()
assert tbl.schema.field("b").type == pa.string()
+165
View File
@@ -7,17 +7,20 @@ TPCH data is generated via DuckDB's built-in extension and loaded via psycopg.
from __future__ import annotations
import base64
from decimal import Decimal
from urllib.parse import urlparse
import duckdb
import orjson
import psycopg
import pyarrow as pa
import pytest
from testcontainers.postgres import PostgresContainer
from tests.suite.manifests import make_tpch_manifest
from tests.suite.query import WrenQueryTestSuite
from wren import WrenEngine
from wren.connector.postgres import PostgresConnector
from wren.model.data_source import DataSource
pytestmark = pytest.mark.postgres
@@ -85,3 +88,165 @@ class TestPostgres(WrenQueryTestSuite):
manifest_str, DataSource.postgres, conn_info, fallback=False
) as e:
yield e
# ---------------------------------------------------------------------------
# Direct PostgresConnector type-coverage tests (no MDL / engine layer)
# ---------------------------------------------------------------------------
def _build_type_table(conn_str: str) -> None:
with psycopg.connect(conn_str) as pg:
with pg.cursor() as cur:
cur.execute("""
CREATE TABLE type_zoo (
c_int4 INTEGER,
c_int8 BIGINT,
c_numeric NUMERIC(38, 9),
c_text TEXT,
c_bool BOOLEAN,
c_bytea BYTEA,
c_uuid UUID,
c_jsonb JSONB,
c_ts TIMESTAMP,
c_tstz TIMESTAMPTZ,
c_int4_arr INTEGER[],
c_text_arr TEXT[],
c_numeric_arr NUMERIC(38, 9)[]
)
""")
cur.execute(
"""
INSERT INTO type_zoo VALUES (
%s, %s, %s, %s, %s, %s, %s, %s::jsonb,
%s::timestamp, %s::timestamptz,
%s::int[], %s::text[], %s::numeric[]
)
""",
(
42,
9_000_000_000,
Decimal("12345.123456789"),
"hello",
True,
b"\x01\x02\x03",
"00000000-0000-0000-0000-000000000001",
'{"a": 1, "b": "two"}',
"2024-01-02 03:04:05",
"2024-01-02 03:04:05+00",
[1, 2, 3],
["a", "b", "c"],
[Decimal("1.5"), Decimal("2.25")],
),
)
cur.execute("INSERT INTO type_zoo (c_int4) VALUES (NULL)")
pg.commit()
class TestPostgresConnectorTypes:
"""End-to-end type coverage for the native PostgresConnector."""
@pytest.fixture(scope="class")
def connector(self):
with PostgresContainer("postgres:16") as pg:
url = pg.get_connection_url().replace("+psycopg2", "")
_build_type_table(url)
parsed = urlparse(url)
raw_info = {
"host": parsed.hostname,
"port": parsed.port,
"database": parsed.path.lstrip("/"),
"user": parsed.username,
"password": parsed.password,
}
conn_info = DataSource.postgres.get_connection_info(raw_info)
connector = PostgresConnector(conn_info)
try:
yield connector
finally:
connector.close()
def test_arrow_schema(self, connector: PostgresConnector) -> None:
result = connector.query("SELECT * FROM type_zoo ORDER BY c_int4 NULLS LAST")
assert isinstance(result, pa.Table)
assert result.num_rows == 2
expected_types = {
"c_int4": pa.int32(),
"c_int8": pa.int64(),
"c_numeric": pa.decimal128(38, 9),
"c_text": pa.string(),
"c_bool": pa.bool_(),
"c_bytea": pa.binary(),
"c_uuid": pa.string(),
"c_jsonb": pa.string(),
"c_ts": pa.timestamp("us"),
"c_tstz": pa.timestamp("us", tz="UTC"),
"c_int4_arr": pa.list_(pa.int32()),
"c_text_arr": pa.list_(pa.string()),
"c_numeric_arr": pa.list_(pa.decimal128(38, 9)),
}
for name, expected in expected_types.items():
assert result.schema.field(name).type == expected, (
f"unexpected Arrow type for {name}: {result.schema.field(name).type}"
)
def test_value_round_trip(self, connector: PostgresConnector) -> None:
result = connector.query("SELECT * FROM type_zoo WHERE c_int4 = 42")
assert result.num_rows == 1
row = result.to_pylist()[0]
assert row["c_int4"] == 42
assert row["c_int8"] == 9_000_000_000
assert row["c_numeric"] == Decimal("12345.123456789")
assert row["c_text"] == "hello"
assert row["c_bool"] is True
assert bytes(row["c_bytea"]) == b"\x01\x02\x03"
assert row["c_uuid"] == "00000000-0000-0000-0000-000000000001"
# jsonb comes back as JSON string
assert '"a"' in row["c_jsonb"] and '"b"' in row["c_jsonb"]
assert row["c_int4_arr"] == [1, 2, 3]
assert row["c_text_arr"] == ["a", "b", "c"]
assert row["c_numeric_arr"] == [Decimal("1.500000000"), Decimal("2.250000000")]
def test_nulls(self, connector: PostgresConnector) -> None:
# Row inserted as `(NULL)` should produce a NULL in every column.
result = connector.query("SELECT * FROM type_zoo WHERE c_int4 IS NULL")
assert result.num_rows == 1
row = result.to_pylist()[0]
for col in result.column_names:
assert row[col] is None, f"expected NULL for {col}, got {row[col]!r}"
def test_query_limit_parameter(self, connector: PostgresConnector) -> None:
result = connector.query("SELECT c_int4 FROM type_zoo", limit=1)
assert result.num_rows == 1
def test_dry_run(self, connector: PostgresConnector) -> None:
# Should not raise and should not return rows.
connector.dry_run("SELECT c_int4 FROM type_zoo")
def test_dry_run_invalid_sql_raises(self, connector: PostgresConnector) -> None:
from wren.model.error import WrenError
with pytest.raises(WrenError):
connector.dry_run("SELECT * FROM nope_does_not_exist")
def test_duplicate_column_names_preserved(
self, connector: PostgresConnector
) -> None:
# ``pa.table({...})`` silently drops duplicate keys, which trashes
# join results like ``SELECT a.id, b.id FROM t a, t b``. The
# connector must preserve both fields positionally.
# The previous ``test_dry_run_invalid_sql_raises`` aborts the shared
# class-scoped connection's transaction; reset it before running.
connector.connection.rollback()
result = connector.query(
"SELECT a.c_int4 AS id, b.c_int4 AS id "
"FROM type_zoo a, type_zoo b "
"WHERE a.c_int4 = 42 AND b.c_int4 = 42"
)
assert result.num_rows == 1
assert result.num_columns == 2
assert [field.name for field in result.schema] == ["id", "id"]
assert result.column(0).to_pylist() == [42]
assert result.column(1).to_pylist() == [42]
+370
View File
@@ -0,0 +1,370 @@
"""Trino connector tests.
Combines:
1. Unit-level type parser tests for the native Trino → Arrow mapping (no Docker).
2. Integration tests against a real Trino testcontainer that load TPCH from
the built-in ``tpch`` catalog into the in-memory ``memory.default`` schema
so the shared ``WrenQueryTestSuite`` can run against it.
"""
from __future__ import annotations
import base64
import time
from decimal import Decimal
import orjson
import pyarrow as pa
import pytest
from testcontainers.trino import TrinoContainer
from tests.suite.manifests import make_tpch_manifest
from tests.suite.query import WrenQueryTestSuite
from wren import WrenEngine
from wren.connector.trino import (
_build_trino_column,
_parse_trino_data_type,
_parse_trino_url,
_strip_trailing_semicolon,
)
from wren.model.data_source import DataSource
from wren.model.error import ErrorCode, WrenError
pytestmark = pytest.mark.trino
_CATALOG = "memory"
_SCHEMA = "default"
# ---------------------------------------------------------------------------
# Unit tests — type-string parser and column builder (no Docker required).
# ---------------------------------------------------------------------------
@pytest.mark.parametrize(
("type_str", "expected"),
[
# Scalars
("boolean", pa.bool_()),
("tinyint", pa.int8()),
("smallint", pa.int16()),
("integer", pa.int32()),
("int", pa.int32()),
("bigint", pa.int64()),
("real", pa.float32()),
("double", pa.float64()),
("varchar", pa.string()),
("varchar(255)", pa.string()),
("char(10)", pa.string()),
("varbinary", pa.binary()),
("date", pa.date32()),
("uuid", pa.string()),
("ipaddress", pa.string()),
("json", pa.string()),
# Decimal
("decimal(10,2)", pa.decimal128(10, 2)),
("decimal(38,9)", pa.decimal128(38, 9)),
("decimal", pa.decimal128(38, 9)),
# Time / timestamp
("time", pa.time64("us")),
("time(3)", pa.time64("us")),
("timestamp", pa.timestamp("ms")),
("timestamp(6)", pa.timestamp("ms")),
("timestamp with time zone", pa.timestamp("ms", tz="UTC")),
("timestamp(6) with time zone", pa.timestamp("ms", tz="UTC")),
# Containers
("array(integer)", pa.list_(pa.int32())),
("array(decimal(10,2))", pa.list_(pa.decimal128(10, 2))),
("array(varchar)", pa.list_(pa.string())),
# Map
("map(varchar,bigint)", pa.map_(pa.string(), pa.int64())),
("map(varchar, bigint)", pa.map_(pa.string(), pa.int64())),
# Row — named and anonymous
(
"row(a integer, b varchar)",
pa.struct([pa.field("a", pa.int32()), pa.field("b", pa.string())]),
),
(
"row(map(varchar, integer), bigint)",
pa.struct(
[
pa.field("f0", pa.map_(pa.string(), pa.int32())),
pa.field("f1", pa.int64()),
]
),
),
# Nested array(map(varchar, row(...)))
(
"array(map(varchar, row(a integer, b varchar)))",
pa.list_(
pa.map_(
pa.string(),
pa.struct(
[
pa.field("a", pa.int32()),
pa.field("b", pa.string()),
]
),
)
),
),
# Unknown / interval — fall back to string
("interval year to month", pa.string()),
("interval day to second", pa.string()),
("hyperloglog", pa.string()),
],
)
def test_parse_trino_data_type(type_str: str, expected: pa.DataType) -> None:
assert _parse_trino_data_type(type_str) == expected
def test_parse_trino_data_type_handles_none() -> None:
assert _parse_trino_data_type(None) == pa.string()
def test_parse_trino_data_type_unparseable_falls_back() -> None:
assert _parse_trino_data_type("not a real type {{") == pa.string()
def test_build_trino_column_map_dict_to_pairs() -> None:
# Trino driver returns Python dicts; PyArrow map_ wants (k, v) pairs.
arrow_type = pa.map_(pa.string(), pa.int64())
arr = _build_trino_column([{"a": 1, "b": 2}, None, {"x": 9}], arrow_type)
assert arr.type == arrow_type
assert arr.to_pylist() == [
[("a", 1), ("b", 2)],
None,
[("x", 9)],
]
def test_build_trino_column_decimal_string_input() -> None:
arr = _build_trino_column(["1.23", "4.56", None], pa.decimal128(10, 2))
assert arr.to_pylist() == [Decimal("1.23"), Decimal("4.56"), None]
def test_build_trino_column_struct_tuple_to_dict() -> None:
arrow_type = pa.struct([pa.field("a", pa.int32()), pa.field("b", pa.string())])
arr = _build_trino_column([(1, "x"), None, (2, "y")], arrow_type)
assert arr.to_pylist() == [{"a": 1, "b": "x"}, None, {"a": 2, "b": "y"}]
@pytest.mark.parametrize(
("raw", "expected"),
[
("SELECT 1", "SELECT 1"),
("SELECT 1;", "SELECT 1"),
("SELECT 1 ; ", "SELECT 1"),
("SELECT 1\n;\n", "SELECT 1"),
("SELECT 1; -- trailing", "SELECT 1; -- trailing"),
# Only the final semicolon is stripped — internal ones stay.
("SELECT 1; SELECT 2;", "SELECT 1; SELECT 2"),
],
)
def test_strip_trailing_semicolon(raw: str, expected: str) -> None:
assert _strip_trailing_semicolon(raw) == expected
def test_parse_trino_url_rejects_missing_username() -> None:
with pytest.raises(WrenError) as exc:
_parse_trino_url("trino://host:8080/catalog/schema", None)
assert exc.value.error_code == ErrorCode.INVALID_CONNECTION_INFO
assert "username" in str(exc.value).lower()
def test_parse_trino_url_accepts_explicit_username() -> None:
out = _parse_trino_url("trino://alice@host:8080/catalog/schema", None)
assert out["user"] == "alice"
assert out["host"] == "host"
assert out["catalog"] == "catalog"
assert out["schema"] == "schema"
def test_parse_trino_url_rejects_bad_scheme() -> None:
with pytest.raises(WrenError) as exc:
_parse_trino_url("http://alice@host:8080/c/s", None)
assert exc.value.error_code == ErrorCode.INVALID_CONNECTION_INFO
def test_trino_connector_import_error_has_install_hint(monkeypatch) -> None:
"""If ``import trino`` fails, the connector should raise a WrenError with
a clear ``pip install wren-engine[trino]`` hint rather than a raw
ImportError.
"""
import builtins # noqa: PLC0415
import sys # noqa: PLC0415
from wren.connector import trino as trino_module # noqa: PLC0415
# Remove cached trino module so the lazy import re-runs.
monkeypatch.setitem(sys.modules, "trino", None)
real_import = builtins.__import__
def _fake_import(name, *args, **kwargs):
if name == "trino" or name.startswith("trino."):
raise ImportError("No module named 'trino'")
return real_import(name, *args, **kwargs)
monkeypatch.setattr(builtins, "__import__", _fake_import)
with pytest.raises(WrenError) as exc:
trino_module._import_trino()
assert exc.value.error_code == ErrorCode.INVALID_CONNECTION_INFO
msg = str(exc.value)
assert "wren-engine[trino]" in msg
def test_native_connector_does_not_import_ibis() -> None:
# Acceptance criterion: importing the native trino connector must not
# pull ibis into sys.modules (the new module is independent of
# ibis-framework[trino]).
import sys # noqa: PLC0415
sys.modules.pop("ibis", None)
sys.modules.pop("wren.connector.trino", None)
import wren.connector.trino # noqa: F401, PLC0415
assert "ibis" not in sys.modules
# ---------------------------------------------------------------------------
# Integration tests — testcontainer-backed query suite.
# ---------------------------------------------------------------------------
def _create_tpch_tables(host: str, port: int) -> None:
"""Materialise TPCH ``orders`` / ``customer`` into ``memory.default``.
Trino's bundled ``tpch`` connector provides scale-factor data on demand,
so we just copy the rows we need into the in-memory catalog rather than
generating them with DuckDB.
"""
from trino.dbapi import connect as trino_connect # noqa: PLC0415
conn = trino_connect(host=host, port=port, user="test")
try:
cur = conn.cursor()
cur.execute(
"CREATE TABLE memory.default.orders AS "
"SELECT orderkey AS o_orderkey, custkey AS o_custkey, "
"orderstatus AS o_orderstatus, "
"CAST(totalprice AS DOUBLE) AS o_totalprice, "
"orderdate AS o_orderdate "
"FROM tpch.tiny.orders"
)
cur.fetchall()
cur.execute(
"CREATE TABLE memory.default.customer AS "
"SELECT custkey AS c_custkey, name AS c_name FROM tpch.tiny.customer"
)
cur.fetchall()
finally:
conn.close()
class TestTrino(WrenQueryTestSuite):
"""Run the shared connector test suite against a real Trino container."""
manifest = make_tpch_manifest(table_catalog=_CATALOG, table_schema=_SCHEMA)
# tpch.tiny is sf=0.01 — same row counts as the other connector tests.
order_count = 15000
customer_count = 1500
# Trino BIGINT → Arrow int64.
order_id_dtype = "int64"
@pytest.fixture(scope="class")
def engine(self) -> WrenEngine: # type: ignore[override]
with TrinoContainer() as trino:
host = trino.get_container_host_ip()
port = int(trino.get_exposed_port(trino.port))
# Trino sometimes returns "nodes is empty" if we query before the
# coordinator has registered a worker; brief wait avoids it.
time.sleep(5)
_create_tpch_tables(host, port)
conn_info = {
"host": host,
"port": port,
"catalog": _CATALOG,
"schema": _SCHEMA,
"user": "test",
}
manifest_str = base64.b64encode(orjson.dumps(self.manifest)).decode()
with WrenEngine(
manifest_str, DataSource.trino, conn_info, fallback=False
) as e:
yield e
# ------------------------------------------------------------------
# Trino-specific type coverage — exercises every branch in the native
# type parser end-to-end against a live coordinator.
# ------------------------------------------------------------------
def test_scalar_types(self, engine: WrenEngine) -> None:
result = engine.query(
"SELECT "
"CAST(1 AS BIGINT) AS c_bigint, "
"CAST(1 AS INTEGER) AS c_integer, "
"CAST(1 AS SMALLINT) AS c_smallint, "
"CAST(1 AS TINYINT) AS c_tinyint, "
"CAST(1.5 AS DOUBLE) AS c_double, "
"CAST(1.5 AS REAL) AS c_real, "
"CAST('1.23' AS DECIMAL(10,2)) AS c_decimal, "
"CAST('abc' AS VARCHAR) AS c_varchar, "
"CAST('abc' AS CHAR(3)) AS c_char, "
"CAST(X'AB' AS VARBINARY) AS c_varbinary, "
"CAST('{\"a\":1}' AS JSON) AS c_json, "
"CAST('12151fd2-7586-11e9-8f9e-2a86e4085a59' AS UUID) AS c_uuid, "
"CAST('1.2.3.4' AS IPADDRESS) AS c_ip"
)
assert result.num_rows == 1
assert result.schema.field("c_bigint").type == pa.int64()
assert result.schema.field("c_integer").type == pa.int32()
assert result.schema.field("c_smallint").type == pa.int16()
assert result.schema.field("c_tinyint").type == pa.int8()
assert result.schema.field("c_double").type == pa.float64()
assert result.schema.field("c_real").type == pa.float32()
assert result.schema.field("c_decimal").type == pa.decimal128(10, 2)
assert result.schema.field("c_varchar").type == pa.string()
assert result.schema.field("c_char").type == pa.string()
assert result.schema.field("c_varbinary").type == pa.binary()
assert result.schema.field("c_json").type == pa.string()
assert result.schema.field("c_uuid").type == pa.string()
assert result.schema.field("c_ip").type == pa.string()
def test_temporal_types(self, engine: WrenEngine) -> None:
result = engine.query(
"SELECT "
"DATE '2024-01-02' AS c_date, "
"TIME '12:34:56' AS c_time, "
"TIMESTAMP '2024-01-02 12:34:56' AS c_ts, "
"TIMESTAMP '2024-01-02 12:34:56 UTC' AS c_tstz"
)
assert result.schema.field("c_date").type == pa.date32()
assert result.schema.field("c_time").type == pa.time64("us")
assert result.schema.field("c_ts").type == pa.timestamp("ms")
assert result.schema.field("c_tstz").type == pa.timestamp("ms", tz="UTC")
def test_array_type(self, engine: WrenEngine) -> None:
result = engine.query("SELECT ARRAY[1, 2, 3] AS c_array")
assert result.schema.field("c_array").type == pa.list_(pa.int32())
assert result["c_array"][0].as_py() == [1, 2, 3]
def test_map_type(self, engine: WrenEngine) -> None:
result = engine.query("SELECT MAP(ARRAY['a', 'b'], ARRAY[1, 2]) AS c_map")
assert result.schema.field("c_map").type == pa.map_(pa.string(), pa.int32())
def test_row_type(self, engine: WrenEngine) -> None:
result = engine.query(
"SELECT CAST(ROW(1, 'x') AS ROW(a INTEGER, b VARCHAR)) AS c_row"
)
row_type = result.schema.field("c_row").type
assert pa.types.is_struct(row_type)
assert {f.name for f in row_type} == {"a", "b"}
def test_anonymous_row_type(self, engine: WrenEngine) -> None:
# Anonymous row(...) result — field names come back as f0/f1.
result = engine.query("SELECT ROW(1, 'x') AS c_row")
row_type = result.schema.field("c_row").type
assert pa.types.is_struct(row_type)
@@ -0,0 +1,335 @@
"""Unit tests for the native pyathena-backed Athena connector.
These tests stub out :mod:`pyathena` and :mod:`boto3` so they can run without
any AWS credentials or network access.
"""
from __future__ import annotations
import datetime as dtlib
import sys
import types
from decimal import Decimal as PyDecimal
from unittest.mock import MagicMock
import pyarrow as pa
import pytest
from pydantic import SecretStr
pytestmark = pytest.mark.unit
# ---------------------------------------------------------------------------
# stub pyathena module before importing the connector under test
# ---------------------------------------------------------------------------
_pyathena_connect_calls: list[dict] = []
_pyathena_connection_mock = MagicMock(name="pyathena_connection")
def _fake_pyathena_connect(**kwargs):
_pyathena_connect_calls.append(kwargs)
return _pyathena_connection_mock
_pyathena_module = types.ModuleType("pyathena")
_pyathena_module.connect = _fake_pyathena_connect # type: ignore[attr-defined]
sys.modules.setdefault("pyathena", _pyathena_module)
from wren.connector.athena import ( # noqa: E402
AthenaConnector,
_build_athena_arrow_table,
_parse_athena_type,
)
from wren.model import AthenaConnectionInfo # noqa: E402
from wren.model.data_source import DataSourceExtension # noqa: E402
from wren.model.error import ErrorCode, ErrorPhase, WrenError # noqa: E402
@pytest.fixture(autouse=True)
def _reset_connect_calls():
_pyathena_connect_calls.clear()
_pyathena_connection_mock.reset_mock(return_value=True, side_effect=True)
_pyathena_connection_mock.close.reset_mock()
yield
# ---------------------------------------------------------------------------
# type lexer
# ---------------------------------------------------------------------------
@pytest.mark.parametrize(
("type_str", "expected"),
[
("varchar", pa.string()),
("integer", pa.int32()),
("bigint", pa.int64()),
("boolean", pa.bool_()),
("double", pa.float64()),
("date", pa.date32()),
("varbinary", pa.binary()),
],
)
def test_parse_athena_type_primitives(type_str, expected):
assert _parse_athena_type(type_str) == expected
def test_parse_athena_type_decimal():
assert _parse_athena_type("decimal(12,4)") == pa.decimal128(12, 4)
def test_parse_athena_type_array():
assert _parse_athena_type("array(varchar)") == pa.list_(pa.string())
def test_parse_athena_type_map():
assert _parse_athena_type("map(varchar,bigint)") == pa.map_(pa.string(), pa.int64())
def test_parse_athena_type_row():
parsed = _parse_athena_type("row(a integer, b varchar)")
assert pa.types.is_struct(parsed)
fields = {
parsed.field(i).name: parsed.field(i).type for i in range(parsed.num_fields)
}
assert fields["a"] == pa.int32()
assert fields["b"] == pa.string()
def test_parse_athena_type_unknown_falls_back_to_string():
assert _parse_athena_type("totally not a type") == pa.string()
def test_parse_athena_type_none_returns_string():
assert _parse_athena_type(None) == pa.string()
# ---------------------------------------------------------------------------
# cursor → arrow table
# ---------------------------------------------------------------------------
def _make_cursor(description, rows):
cursor = MagicMock()
cursor.description = description
cursor.fetchall.return_value = rows
cursor.execute.return_value = cursor
return cursor
def test_build_athena_arrow_table_mixed_types():
description = [
("id", "bigint"),
("name", "varchar"),
("price", "decimal(10,2)"),
("tags", "array(varchar)"),
("ordered_at", "timestamp"),
("ordered_on", "date"),
]
rows = [
(
1,
"alice",
PyDecimal("9.99"),
["a", "b"],
dtlib.datetime(2024, 1, 2, 3, 4, 5),
dtlib.date(2024, 1, 2),
),
(2, "bob", PyDecimal("1.50"), [], None, None),
]
cursor = _make_cursor(description, rows)
table = _build_athena_arrow_table(cursor)
assert table.num_rows == 2
assert table.schema.field("id").type == pa.int64()
assert table.schema.field("name").type == pa.string()
assert table.schema.field("price").type == pa.decimal128(10, 2)
assert table.schema.field("tags").type == pa.list_(pa.string())
assert table.schema.field("ordered_at").type == pa.timestamp("ms")
assert table.schema.field("ordered_on").type == pa.date32()
def test_build_athena_arrow_table_empty():
cursor = _make_cursor([("a", "varchar")], [])
table = _build_athena_arrow_table(cursor)
assert table.num_rows == 0
assert table.schema.field("a").type == pa.string()
# ---------------------------------------------------------------------------
# AthenaConnector — happy path & error mapping
# ---------------------------------------------------------------------------
def _info(**overrides) -> AthenaConnectionInfo:
data = {
"s3_staging_dir": SecretStr("s3://bucket/staging/"),
"region_name": "us-west-2",
"schema_name": "default",
}
data.update(overrides)
return AthenaConnectionInfo(**data)
def test_connector_query_passes_kwargs_and_kills_on_interrupt():
AthenaConnector(_info())
assert _pyathena_connect_calls, "pyathena.connect was not invoked"
kwargs = _pyathena_connect_calls[-1]
assert kwargs["s3_staging_dir"] == "s3://bucket/staging/"
assert kwargs["region_name"] == "us-west-2"
assert kwargs["schema_name"] == "default"
assert kwargs["kill_on_interrupt"] is True
def test_connector_query_returns_arrow_table_and_respects_limit():
cursor = _make_cursor(
[("id", "integer"), ("name", "varchar")],
[(1, "a"), (2, "b"), (3, "c")],
)
_pyathena_connection_mock.cursor.return_value = cursor
conn = AthenaConnector(_info())
table = conn.query("SELECT id, name FROM t", limit=2)
assert table.num_rows == 2
assert table.column("id").to_pylist() == [1, 2]
cursor.execute.assert_called_once_with("SELECT id, name FROM t")
def test_connector_query_wraps_driver_errors():
cursor = MagicMock()
cursor.execute.side_effect = RuntimeError("boom")
_pyathena_connection_mock.cursor.return_value = cursor
conn = AthenaConnector(_info())
with pytest.raises(WrenError) as exc:
conn.query("SELECT 1")
assert exc.value.error_code == ErrorCode.INVALID_SQL
assert exc.value.phase == ErrorPhase.SQL_EXECUTION
def test_connector_dry_run_emits_explain():
cursor = MagicMock()
cursor.execute.return_value = cursor
_pyathena_connection_mock.cursor.return_value = cursor
conn = AthenaConnector(_info())
conn.dry_run("SELECT 1")
cursor.execute.assert_called_once_with("EXPLAIN SELECT 1")
def test_connector_dry_run_wraps_driver_errors():
cursor = MagicMock()
cursor.execute.side_effect = RuntimeError("parse error")
_pyathena_connection_mock.cursor.return_value = cursor
conn = AthenaConnector(_info())
with pytest.raises(WrenError) as exc:
conn.dry_run("SELECT bogus")
assert exc.value.error_code == ErrorCode.INVALID_SQL
assert exc.value.phase == ErrorPhase.SQL_DRY_RUN
def test_connector_close_calls_underlying_connection():
conn = AthenaConnector(_info())
underlying = conn.connection
conn.close()
underlying.close.assert_called_once()
assert conn.connection is None
# Idempotent.
conn.close()
# ---------------------------------------------------------------------------
# credential resolution
# ---------------------------------------------------------------------------
def test_connector_uses_access_key_when_provided():
AthenaConnector(
_info(
aws_access_key_id=SecretStr("AKIA"),
aws_secret_access_key=SecretStr("SECRET"),
aws_session_token=SecretStr("TOKEN"),
)
)
kwargs = _pyathena_connect_calls[-1]
assert kwargs["aws_access_key_id"] == "AKIA"
assert kwargs["aws_secret_access_key"] == "SECRET"
assert kwargs["aws_session_token"] == "TOKEN"
def test_connector_uses_oidc_when_web_identity_token_provided(monkeypatch):
sts_client = MagicMock()
sts_client.assume_role_with_web_identity.return_value = {
"Credentials": {
"AccessKeyId": "OIDC_AK",
"SecretAccessKey": "OIDC_SK",
"SessionToken": "OIDC_TK",
}
}
boto_module = MagicMock()
boto_module.client.return_value = sts_client
monkeypatch.setattr("boto3.client", boto_module.client)
AthenaConnector(
_info(
web_identity_token=SecretStr("token-xyz"),
role_arn=SecretStr("arn:aws:iam::123:role/wren"),
role_session_name="custom-session",
)
)
boto_module.client.assert_called_once_with("sts", region_name="us-west-2")
sts_client.assume_role_with_web_identity.assert_called_once_with(
RoleArn="arn:aws:iam::123:role/wren",
RoleSessionName="custom-session",
WebIdentityToken="token-xyz",
)
kwargs = _pyathena_connect_calls[-1]
assert kwargs["aws_access_key_id"] == "OIDC_AK"
assert kwargs["aws_secret_access_key"] == "OIDC_SK"
assert kwargs["aws_session_token"] == "OIDC_TK"
def test_connector_falls_back_to_default_chain_when_no_credentials():
AthenaConnector(_info())
kwargs = _pyathena_connect_calls[-1]
assert "aws_access_key_id" not in kwargs
assert "aws_secret_access_key" not in kwargs
assert "aws_session_token" not in kwargs
# ---------------------------------------------------------------------------
# data_source.get_athena_connection still works end-to-end
# ---------------------------------------------------------------------------
def test_data_source_get_athena_connection_returns_pyathena_connection():
DataSourceExtension.get_athena_connection(_info())
assert _pyathena_connect_calls, "pyathena.connect should be called"
def test_data_source_get_athena_connection_propagates_schema_and_kwargs():
"""Regression: data_source.get_athena_connection must funnel through the
same kwargs builder as AthenaConnector so schema_name, kill_on_interrupt,
and any user-supplied info.kwargs reach pyathena.connect consistently.
"""
info = _info(schema_name="analytics")
# AthenaConnectionInfo currently has no ``kwargs`` field, but the shared
# builder reads it defensively. Simulate a future/user-provided value.
object.__setattr__(info, "kwargs", {"work_group": "wg-1", "kill_on_interrupt": False})
DataSourceExtension.get_athena_connection(info)
kwargs = _pyathena_connect_calls[-1]
assert kwargs["schema_name"] == "analytics"
assert kwargs["work_group"] == "wg-1"
# User-supplied kwargs override the default.
assert kwargs["kill_on_interrupt"] is False
def test_data_source_get_athena_connection_defaults_kill_on_interrupt():
DataSourceExtension.get_athena_connection(_info())
kwargs = _pyathena_connect_calls[-1]
assert kwargs["kill_on_interrupt"] is True
assert kwargs["schema_name"] == "default"
@@ -0,0 +1,157 @@
"""Unit tests for ``wren.connector.clickhouse`` URL parsing and query wrapping.
Pure-Python — no Docker, no real ClickHouse instance.
"""
from __future__ import annotations
from unittest.mock import MagicMock
import pytest
from wren.connector.clickhouse import (
ClickHouseConnector,
_build_clickhouse_arrow_table,
_build_clickhouse_client_kwargs,
)
pytestmark = pytest.mark.unit
class _FakeConnUrl:
def __init__(self, url: str) -> None:
self._url = url
def get_secret_value(self) -> str:
return self._url
class _FakeConnInfoFromUrl:
"""Minimal stand-in for ConnectionUrl payloads accepted by the builder."""
def __init__(self, url: str, **extras) -> None:
self.connection_url = _FakeConnUrl(url)
self.kwargs = extras.get("kwargs")
# ---------------------------------------------------------------------------
# 1. URL credentials are percent-decoded
# ---------------------------------------------------------------------------
def test_clickhouse_url_decodes_username_and_password() -> None:
"""user / password from the URL must be unquote_plus'd.
urlparse leaves ``%40`` (``@``) and ``%20`` (space) literal in the
userinfo, which would otherwise reach ClickHouse verbatim and fail auth.
"""
info = _FakeConnInfoFromUrl(
"clickhouse://us%40er:p%40ss%20word@clickhouse-host:9000/analytics"
)
out = _build_clickhouse_client_kwargs(info)
assert out["username"] == "us@er"
assert out["password"] == "p@ss word"
assert out["host"] == "clickhouse-host"
assert out["port"] == 9000
assert out["database"] == "analytics"
def test_clickhouse_url_defaults_username_when_omitted() -> None:
"""When the URL has no userinfo, the default ``"default"`` user wins."""
info = _FakeConnInfoFromUrl("clickhouse://clickhouse-host/analytics")
out = _build_clickhouse_client_kwargs(info)
assert out["username"] == "default"
assert out["password"] == ""
# ---------------------------------------------------------------------------
# 2. Trailing semicolons in caller SQL must not break the subquery wrap
# ---------------------------------------------------------------------------
def _make_connector_with_mock_query() -> tuple[ClickHouseConnector, MagicMock]:
"""Build a ClickHouseConnector bypassing ``__init__`` (no real client)."""
connector = ClickHouseConnector.__new__(ClickHouseConnector)
connector._closed = False
connector.connection = MagicMock()
# ``query()`` consumes the returned object via ``_build_clickhouse_arrow_table``;
# arrange a minimal result that produces an empty Arrow table.
fake_result = MagicMock()
fake_result.column_names = []
fake_result.column_types = []
fake_result.result_columns = []
connector.connection.query.return_value = fake_result
return connector, connector.connection
def test_clickhouse_query_strips_trailing_semicolon_before_subquery_wrap() -> None:
"""``SELECT 1;`` must not become ``SELECT * FROM (SELECT 1;) ...``."""
connector, mock_conn = _make_connector_with_mock_query()
connector.query("SELECT 1;", limit=5)
(sent,), _ = mock_conn.query.call_args
assert sent == "SELECT * FROM (SELECT 1) AS _wren_sub LIMIT 5"
def test_clickhouse_query_strips_multiple_trailing_semicolons_and_whitespace() -> None:
"""Trailing whitespace and a stray ``;`` both get trimmed."""
connector, mock_conn = _make_connector_with_mock_query()
connector.query("SELECT 1 ; ", limit=3)
(sent,), _ = mock_conn.query.call_args
assert sent == "SELECT * FROM (SELECT 1) AS _wren_sub LIMIT 3"
def test_clickhouse_query_without_limit_still_strips_semicolon() -> None:
"""When no limit is supplied the executed SQL is also the stripped form."""
connector, mock_conn = _make_connector_with_mock_query()
connector.query("SELECT 1;")
(sent,), _ = mock_conn.query.call_args
assert sent == "SELECT 1"
def test_clickhouse_dry_run_strips_trailing_semicolon() -> None:
"""Same fix must apply to dry_run, which also wraps in a subquery."""
connector, mock_conn = _make_connector_with_mock_query()
connector.dry_run("SELECT 1;")
(sent,), _ = mock_conn.query.call_args
assert sent == "SELECT * FROM (SELECT 1) AS _wren_sub LIMIT 0"
# ---------------------------------------------------------------------------
# 3. Duplicate column names survive ``_build_clickhouse_arrow_table``
# ---------------------------------------------------------------------------
class _FakeChType:
def __init__(self, name: str) -> None:
self.name = name
def test_clickhouse_arrow_table_preserves_duplicate_column_names() -> None:
"""``SELECT a, a`` must yield a two-column Arrow table, not one column.
Earlier the table was built via ``dict(zip(names, arrays))`` which
silently collapsed duplicate names.
"""
fake = MagicMock()
fake.column_names = ["a", "a"]
fake.column_types = [_FakeChType("Int64"), _FakeChType("Int64")]
fake.result_rows = [[1, 2], [3, 4]]
table = _build_clickhouse_arrow_table(fake)
assert table.num_columns == 2
assert table.column_names == ["a", "a"]
assert table.column(0).to_pylist() == [1, 3]
assert table.column(1).to_pylist() == [2, 4]
@@ -0,0 +1,280 @@
"""Unit tests for the MSSQL pyodbc connection plumbing.
These tests mock out ``pyodbc`` so they run without ODBC Driver 18
installed; they cover behaviours called out in PR #2274 review:
1. ``mssql://`` URL components are URL-decoded round-trip.
2. Asymmetric ``user`` / ``password`` combinations raise instead of
leaking a half-built ODBC connection string.
3. A non-numeric ``statement_timeout`` raises BEFORE ``pyodbc.connect``
is called, so the connection can't be leaked.
4. SQL Server ``TINYINT`` (``internal_size == 1``) maps unconditionally
to ``pa.uint8()``.
5. ``_decode_mssql_datetimeoffset`` validates payload length explicitly.
"""
from __future__ import annotations
from unittest.mock import MagicMock, patch
import pyarrow as pa
import pytest
from wren.connector.mssql import MSSqlConnector
from wren.model.data_source import DataSourceExtension
from wren.model.error import ErrorCode, WrenError
pytestmark = pytest.mark.unit
class _FakePyodbc:
"""Stand-in for the ``pyodbc`` module used by ``_connect_mssql_pyodbc``."""
def __init__(self) -> None:
self.connect = MagicMock(return_value=MagicMock(timeout=0))
def _parse_conn_str(connect_call) -> dict[str, str]:
"""Split an ODBC connection string from a ``connect`` call into a dict."""
(conn_str,), _ = connect_call
parts: dict[str, str] = {}
for piece in conn_str.split(";"):
if not piece:
continue
key, _, value = piece.partition("=")
# Strip the {value} escaping used by _escape_odbc_value
if value.startswith("{") and value.endswith("}"):
value = value[1:-1].replace("}}", "}")
parts[key] = value
return parts
# ---------------------------------------------------------------------------
# 1. URL decoding round-trip
# ---------------------------------------------------------------------------
def test_mssql_url_decodes_user_database_and_password() -> None:
"""user, database path, and password should all be URL-decoded."""
fake = _FakePyodbc()
url = (
"mssql://us%40er:p%40ss%20word@host:1433/"
"my%20db?TrustServerCertificate=yes&app%20name=wren"
)
with patch("wren.model.data_source.pyodbc", fake):
DataSourceExtension.get_mssql_connection_from_url(url)
fake.connect.assert_called_once()
parts = _parse_conn_str(fake.connect.call_args)
assert parts["UID"] == "us@er"
assert parts["PWD"] == "p@ss word"
assert parts["DATABASE"] == "my db"
assert parts["SERVER"] == "host,1433"
# parse_qsl handles query-string decoding; the key keeps its space.
assert parts["TrustServerCertificate"] == "yes"
assert parts["app name"] == "wren"
# ---------------------------------------------------------------------------
# 2. Auth combination validation
# ---------------------------------------------------------------------------
def test_mssql_user_without_password_raises() -> None:
fake = _FakePyodbc()
with patch("wren.model.data_source.pyodbc", fake):
with pytest.raises(WrenError) as exc:
DataSourceExtension._connect_mssql_pyodbc(
host="h",
port="1433",
database="db",
user="alice",
password=None,
driver="ODBC Driver 18 for SQL Server",
)
assert exc.value.error_code == ErrorCode.INVALID_CONNECTION_INFO
fake.connect.assert_not_called()
def test_mssql_password_without_user_raises() -> None:
fake = _FakePyodbc()
with patch("wren.model.data_source.pyodbc", fake):
with pytest.raises(WrenError) as exc:
DataSourceExtension._connect_mssql_pyodbc(
host="h",
port="1433",
database="db",
user=None,
password="secret",
driver="ODBC Driver 18 for SQL Server",
)
assert exc.value.error_code == ErrorCode.INVALID_CONNECTION_INFO
fake.connect.assert_not_called()
def test_mssql_no_credentials_uses_trusted_connection() -> None:
fake = _FakePyodbc()
with patch("wren.model.data_source.pyodbc", fake):
DataSourceExtension._connect_mssql_pyodbc(
host="h",
port="1433",
database="db",
user=None,
password=None,
driver="ODBC Driver 18 for SQL Server",
)
parts = _parse_conn_str(fake.connect.call_args)
assert parts.get("Trusted_Connection") == "yes"
assert "UID" not in parts
assert "PWD" not in parts
def test_mssql_both_credentials_emits_uid_and_pwd() -> None:
fake = _FakePyodbc()
with patch("wren.model.data_source.pyodbc", fake):
DataSourceExtension._connect_mssql_pyodbc(
host="h",
port="1433",
database="db",
user="alice",
password="secret",
driver="ODBC Driver 18 for SQL Server",
)
parts = _parse_conn_str(fake.connect.call_args)
assert parts["UID"] == "alice"
assert parts["PWD"] == "secret"
assert "Trusted_Connection" not in parts
# ---------------------------------------------------------------------------
# 3. statement_timeout validated before connect()
# ---------------------------------------------------------------------------
def test_mssql_invalid_statement_timeout_does_not_leak_connection() -> None:
fake = _FakePyodbc()
with patch("wren.model.data_source.pyodbc", fake):
with pytest.raises(WrenError) as exc:
DataSourceExtension._connect_mssql_pyodbc(
host="h",
port="1433",
database="db",
user="alice",
password="secret",
driver="ODBC Driver 18 for SQL Server",
kwargs={"statement_timeout": "not-a-number"},
)
assert exc.value.error_code == ErrorCode.INVALID_CONNECTION_INFO
# The crucial assertion: connect() must not be called when the
# timeout is invalid, otherwise the connection would leak.
fake.connect.assert_not_called()
def test_mssql_valid_statement_timeout_is_applied() -> None:
fake = _FakePyodbc()
conn = MagicMock()
conn.timeout = 0
fake.connect.return_value = conn
with patch("wren.model.data_source.pyodbc", fake):
result = DataSourceExtension._connect_mssql_pyodbc(
host="h",
port="1433",
database="db",
user="alice",
password="secret",
driver="ODBC Driver 18 for SQL Server",
kwargs={"statement_timeout": "42"},
)
fake.connect.assert_called_once()
assert result.timeout == 42
parts = _parse_conn_str(fake.connect.call_args)
# statement_timeout is popped, never sent to the ODBC string
assert "statement_timeout" not in parts
# ---------------------------------------------------------------------------
# 4. TINYINT maps unconditionally to uint8
# ---------------------------------------------------------------------------
def test_mssql_tinyint_maps_to_uint8_regardless_of_sample_sign() -> None:
"""SQL Server TINYINT is unsigned (0..255); the Arrow type must be
``uint8`` regardless of the sampled values (which can never legitimately
be negative, but the helper must not branch on sign)."""
# internal_size == 1 is what pyodbc reports for TINYINT columns.
column_desc = ("c_tinyint", int, None, 1, 3, 0, True)
# Non-negative sample → uint8.
assert MSSqlConnector._mssql_arrow_type(column_desc, [0, 255]) == pa.uint8()
# All-None sample → still uint8 (driver-declared internal_size wins).
assert MSSqlConnector._mssql_arrow_type(column_desc, [None, None]) == pa.uint8()
def test_mssql_tinyint_round_trips_through_build_column() -> None:
"""A TINYINT column with 0 and 255 must survive the column build path."""
arr = MSSqlConnector._build_mssql_column([0, 255, None], pa.uint8())
assert arr.type == pa.uint8()
assert arr.to_pylist() == [0, 255, None]
# ---------------------------------------------------------------------------
# 5. datetimeoffset payload length validation
# ---------------------------------------------------------------------------
def test_mssql_decode_datetimeoffset_rejects_truncated_payload() -> None:
"""A short DATETIMEOFFSET payload must raise a clear error, not the
cryptic ``month must be in 1..12`` that bubbles up from datetime()."""
truncated = b"\x00" * 10
with pytest.raises(ValueError) as exc:
DataSourceExtension._decode_mssql_datetimeoffset(truncated)
msg = str(exc.value)
assert "datetimeoffset" in msg.lower()
assert "20" in msg
assert "10" in msg
def test_mssql_decode_datetimeoffset_accepts_none() -> None:
"""``None`` continues to pass through (NULL values from pyodbc)."""
assert DataSourceExtension._decode_mssql_datetimeoffset(None) is None
# ---------------------------------------------------------------------------
# 6. Duplicate column names survive ``query()``
# ---------------------------------------------------------------------------
def test_mssql_query_preserves_duplicate_column_names() -> None:
"""``SELECT a, a`` must yield a two-column Arrow table, not one column.
Earlier the result was built via ``dict(zip(names, arrays))`` which
silently collapsed duplicate names. Build the table from arrays + schema
directly to keep each projection.
"""
cursor = MagicMock()
# internal_size 4 → int32. Two columns both named ``a`` with distinct values.
cursor.description = [
("a", int, None, 4, 10, 0, True),
("a", int, None, 4, 10, 0, True),
]
cursor.fetchall.return_value = [(1, 2), (3, 4)]
cursor.fetchmany.return_value = [(1, 2), (3, 4)]
fake_conn = MagicMock()
fake_conn.cursor.return_value = cursor
connector = MSSqlConnector.__new__(MSSqlConnector)
connector.connection = fake_conn
table = connector.query("SELECT a, a FROM t")
assert table.num_columns == 2
assert table.column_names == ["a", "a"]
assert table.column(0).to_pylist() == [1, 3]
assert table.column(1).to_pylist() == [2, 4]
+204
View File
@@ -0,0 +1,204 @@
"""Unit tests for native MySQL connector helpers.
These tests cover the pure helpers in ``wren.connector.mysql`` — limit
sanitisation, SQL composition, decimal type derivation — without requiring
a live MySQL server.
"""
from __future__ import annotations
import pyarrow as pa
import pytest
from wren.connector.mysql import (
_apply_limit,
_arrow_decimal_from_mysql_field,
_build_mysql_column,
_coerce_limit,
_mysql_blob_codes,
_mysql_decimal_codes,
_mysql_field_type_map,
_mysql_string_codes,
_mysql_unsigned_variant_map,
)
pytestmark = pytest.mark.unit
# ── _coerce_limit ─────────────────────────────────────────────────────────
def test_coerce_limit_none_passthrough() -> None:
assert _coerce_limit(None) is None
def test_coerce_limit_accepts_int() -> None:
assert _coerce_limit(10) == 10
def test_coerce_limit_accepts_numeric_string() -> None:
# ``int()`` accepts numeric strings — keep that contract.
assert _coerce_limit("25") == 25
def test_coerce_limit_rejects_injection_string() -> None:
"""A crafted limit value must not survive ``int()`` coercion."""
with pytest.raises(ValueError):
_coerce_limit("1; DROP TABLE foo")
def test_coerce_limit_rejects_negative() -> None:
with pytest.raises(ValueError):
_coerce_limit(-1)
# ── _apply_limit ──────────────────────────────────────────────────────────
def test_apply_limit_appends_clause() -> None:
out = _apply_limit("SELECT a FROM t", 5)
assert out.endswith("LIMIT 5")
assert "SELECT a FROM t" in out
def test_apply_limit_strips_trailing_semicolon() -> None:
out = _apply_limit("SELECT a FROM t;", 3)
assert "; " not in out
assert out.endswith("LIMIT 3")
assert ";" not in out.split("LIMIT")[0]
def test_apply_limit_zero() -> None:
out = _apply_limit("SELECT a FROM t", 0)
assert out.endswith("LIMIT 0")
# ── _arrow_decimal_from_mysql_field ──────────────────────────────────────
def test_decimal_type_passthrough() -> None:
# DECIMAL(12, 4) signed → MySQLdb description length = 12 + 1 (sign) + 1
# (decimal point) = 14.
t = _arrow_decimal_from_mysql_field(14, 4, is_unsigned=False)
assert pa.types.is_decimal(t)
assert t.precision == 12
assert t.scale == 4
def test_decimal_type_unsigned_recovers_precision() -> None:
# DECIMAL(12, 4) UNSIGNED → length = 12 + 0 (no sign) + 1 (point) = 13.
t = _arrow_decimal_from_mysql_field(13, 4, is_unsigned=True)
assert t.precision == 12
assert t.scale == 4
def test_decimal_type_zero_scale() -> None:
# DECIMAL(10, 0) signed → length = 10 + 1 (sign) + 0 (no point) = 11.
t = _arrow_decimal_from_mysql_field(11, 0, is_unsigned=False)
assert t.precision == 10
assert t.scale == 0
def test_decimal_type_high_scale() -> None:
"""MySQL allows scale up to 30 — we must not clamp below that for
precision >= 30."""
# DECIMAL(38, 30) signed → length = 38 + 1 + 1 = 40.
t = _arrow_decimal_from_mysql_field(40, 30, is_unsigned=False)
assert t.precision == 38
assert t.scale == 30
def test_decimal_type_clamps_above_arrow_max_precision() -> None:
"""MySQL precision tops at 65; Arrow decimal128 tops at 38. We clamp."""
# DECIMAL(65, 30) signed → length = 65 + 1 + 1 = 67.
t = _arrow_decimal_from_mysql_field(67, 30, is_unsigned=False)
assert t.precision == 38
assert t.scale == 30
def test_decimal_type_none_uses_fallback() -> None:
t = _arrow_decimal_from_mysql_field(None, None)
assert t.precision == 38
assert t.scale == 9
def test_decimal_type_scale_not_greater_than_precision() -> None:
# Pathological case: length implies tiny precision but scale is huge.
# Result must keep scale <= precision so PyArrow accepts the type.
t = _arrow_decimal_from_mysql_field(7, 30, is_unsigned=False)
assert t.scale <= t.precision
# ── TIME → duration round-trip ────────────────────────────────────────────
def test_time_column_preserves_negative_and_over_24h() -> None:
"""MySQL ``TIME`` ranges ``-838:59:59`` to ``838:59:59`` and can be
negative. The Arrow type must be ``duration("us")``, not ``time64("us")``
(which only accepts 0-24h positive values), and the conversion must
preserve the sign and magnitude of MySQLdb's ``datetime.timedelta``
values.
"""
import datetime # noqa: PLC0415
values = [
datetime.timedelta(hours=-100),
datetime.timedelta(0),
datetime.timedelta(hours=838, minutes=59, seconds=59),
-datetime.timedelta(hours=838, minutes=59, seconds=59),
None,
]
arr = _build_mysql_column(values, pa.duration("us"))
assert pa.types.is_duration(arr.type)
out = arr.to_pylist()
assert out[0] == datetime.timedelta(hours=-100)
assert out[1] == datetime.timedelta(0)
assert out[2] == datetime.timedelta(hours=838, minutes=59, seconds=59)
assert out[3] == -datetime.timedelta(hours=838, minutes=59, seconds=59)
assert out[4] is None
# ── Thread-safe lazy init ────────────────────────────────────────────────
def test_lazy_init_thread_safe() -> None:
"""The cached FIELD_TYPE accessors must publish fully-populated results
even when many threads hit them concurrently on a cold cache.
The previous in-place dict/set mutation pattern could expose a partially
populated map to a thread that raced the initializer. ``functools.cache``
guarantees the initializer body runs to completion before the result is
visible to any caller.
"""
pytest.importorskip("MySQLdb")
from concurrent.futures import ThreadPoolExecutor # noqa: PLC0415
accessors = (
_mysql_field_type_map,
_mysql_unsigned_variant_map,
_mysql_blob_codes,
_mysql_string_codes,
_mysql_decimal_codes,
)
for fn in accessors:
fn.cache_clear()
# Capture the expected fully-populated reference values once, single-
# threaded, so the assertions below have a definitive ground truth.
expected = {fn: fn() for fn in accessors}
for fn in accessors:
fn.cache_clear()
def hit_all() -> tuple:
return tuple(fn() for fn in accessors)
with ThreadPoolExecutor(max_workers=16) as ex:
results = list(ex.map(lambda _: hit_all(), range(64)))
for row in results:
for fn, got in zip(accessors, row, strict=True):
# Every thread sees the same fully-populated object.
assert got == expected[fn]
# Sanity: the field-type map is non-empty (MySQLdb constants exist).
if fn is _mysql_field_type_map:
assert len(got) > 0
+1841 -1872
View File
File diff suppressed because it is too large Load Diff