diff --git a/core/cml-calcite/pom.xml b/core/cml-calcite/pom.xml index ff6b00d62..301a7f777 100644 --- a/core/cml-calcite/pom.xml +++ b/core/cml-calcite/pom.xml @@ -25,6 +25,11 @@ guava + + io.cml + cml-spi + + io.cml trino-parser diff --git a/core/cml-calcite/src/main/java/io/cml/calcite/CmlSchema.java b/core/cml-calcite/src/main/java/io/cml/calcite/CmlSchema.java new file mode 100644 index 000000000..e5f35da81 --- /dev/null +++ b/core/cml-calcite/src/main/java/io/cml/calcite/CmlSchema.java @@ -0,0 +1,39 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.cml.calcite; + +import org.apache.calcite.schema.Table; +import org.apache.calcite.schema.impl.AbstractSchema; + +import java.util.Map; + +import static java.util.Objects.requireNonNull; + +class CmlSchema + extends AbstractSchema +{ + private final Map tableMap; + + public CmlSchema(Map tableMap) + { + this.tableMap = requireNonNull(tableMap, "tableMap is null"); + } + + @Override + protected Map getTableMap() + { + return tableMap; + } +} diff --git a/core/cml-calcite/src/main/java/io/cml/calcite/CmlSchemaUtil.java b/core/cml-calcite/src/main/java/io/cml/calcite/CmlSchemaUtil.java new file mode 100644 index 000000000..adf23461f --- /dev/null +++ b/core/cml-calcite/src/main/java/io/cml/calcite/CmlSchemaUtil.java @@ -0,0 +1,148 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.cml.calcite; + +import com.google.common.collect.ImmutableList; +import io.cml.spi.metadata.ColumnMetadata; +import io.cml.spi.metadata.TableMetadata; +import io.cml.spi.type.PGType; +import io.trino.sql.parser.ParsingOptions; +import io.trino.sql.parser.SqlParser; +import io.trino.sql.tree.Statement; +import org.apache.calcite.adapter.java.JavaTypeFactory; +import org.apache.calcite.config.CalciteConnectionConfigImpl; +import org.apache.calcite.jdbc.CalciteSchema; +import org.apache.calcite.jdbc.JavaTypeFactoryImpl; +import org.apache.calcite.plan.ConventionTraitDef; +import org.apache.calcite.plan.RelOptCluster; +import org.apache.calcite.plan.RelOptPlanner; +import org.apache.calcite.plan.volcano.VolcanoPlanner; +import org.apache.calcite.prepare.CalciteCatalogReader; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rel.type.RelDataTypeFactory; +import org.apache.calcite.rex.RexBuilder; +import org.apache.calcite.schema.SchemaPlus; +import org.apache.calcite.sql.SqlDialect; +import org.apache.calcite.sql.dialect.BigQuerySqlDialect; +import org.apache.calcite.tools.Frameworks; + +import java.util.List; + +import static com.google.common.collect.ImmutableMap.toImmutableMap; +import static io.cml.spi.type.BigIntType.BIGINT; +import static io.cml.spi.type.BooleanType.BOOLEAN; +import static io.cml.spi.type.DoubleType.DOUBLE; +import static io.cml.spi.type.IntegerType.INTEGER; +import static io.cml.spi.type.VarcharType.VARCHAR; + +public final class CmlSchemaUtil +{ + private CmlSchemaUtil() {} + + public enum Dialect + { + BIGQUERY(BigQuerySqlDialect.DEFAULT); + + private final SqlDialect sqlDialect; + + Dialect(SqlDialect sqlDialect) + { + this.sqlDialect = sqlDialect; + } + } + + public static String convertQuery(Dialect dialect, SchemaPlusInfo schemaPlusInfo, String sql) + { + SqlParser sqlParser = new SqlParser(); + Statement stmt = sqlParser.createStatement(sql, new ParsingOptions()); + RelOptCluster cluster = newCluster(); + + SchemaPlus schemaPlus = get(schemaPlusInfo); + CalciteCatalogReader reader = new CalciteCatalogReader( + CalciteSchema.from(schemaPlus), + ImmutableList.of(), + cluster.getTypeFactory(), + CalciteConnectionConfigImpl.DEFAULT); + + // TODO: uncomment this when CalciteRelConverter finished + // RelNode relNode = CalciteRelConverter.convert(cluster, reader, stmt); + // RelToSqlConverter relToSqlConverter = new RelToSqlConverter(dialect.sqlDialect); + // SqlNode sqlNode = relToSqlConverter.visitRoot(relNode).asStatement(); + + // SqlPrettyWriter sqlPrettyWriter = new SqlPrettyWriter(); + // return sqlPrettyWriter.format(sqlNode); + return ""; + } + + private static RelOptCluster newCluster() + { + RelDataTypeFactory typeFactory = new JavaTypeFactoryImpl(); + RelOptPlanner planner = new VolcanoPlanner(); + planner.addRelTraitDef(ConventionTraitDef.INSTANCE); + return RelOptCluster.create(planner, new RexBuilder(typeFactory)); + } + + private static SchemaPlus get(SchemaPlusInfo schemaPlusInfo) + { + SchemaPlus rootSchema = Frameworks.createRootSchema(true); + schemaPlusInfo.getSchemaTableMap() + .forEach((schema, tables) -> rootSchema.add(schema, toCmlSchema(tables))); + + return rootSchema; + } + + private static CmlTable toCmlTable(TableMetadata tableMetadata) + { + JavaTypeFactoryImpl typeFactory = new JavaTypeFactoryImpl(); + RelDataTypeFactory.Builder builder = new RelDataTypeFactory.Builder(typeFactory); + for (ColumnMetadata columnMetadata : tableMetadata.getColumns()) { + builder.add(columnMetadata.getName(), toRelDataType(typeFactory, columnMetadata.getType())); + } + + return new CmlTable(builder.build()); + } + + private static CmlSchema toCmlSchema(List tables) + { + return new CmlSchema(tables.stream().collect( + + toImmutableMap( + table -> table.getTable().getTableName(), + CmlSchemaUtil::toCmlTable, + // TODO: handle case sensitive table name + (a, b) -> a))); + } + + // TODO: handle nested types + private static RelDataType toRelDataType(JavaTypeFactory typeFactory, PGType pgType) + { + if (pgType.equals(BOOLEAN)) { + return typeFactory.createJavaType(Boolean.class); + } + if (pgType.equals(INTEGER)) { + return typeFactory.createJavaType(Integer.class); + } + if (pgType.equals(BIGINT)) { + return typeFactory.createJavaType(Long.class); + } + if (pgType.equals(VARCHAR)) { + return typeFactory.createJavaType(String.class); + } + if (pgType.equals(DOUBLE)) { + return typeFactory.createJavaType(Double.class); + } + throw new UnsupportedOperationException(pgType.type() + " not supported yet"); + } +} diff --git a/core/cml-calcite/src/main/java/io/cml/calcite/CmlTable.java b/core/cml-calcite/src/main/java/io/cml/calcite/CmlTable.java new file mode 100644 index 000000000..e641f1d52 --- /dev/null +++ b/core/cml-calcite/src/main/java/io/cml/calcite/CmlTable.java @@ -0,0 +1,36 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.cml.calcite; + +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rel.type.RelDataTypeFactory; +import org.apache.calcite.schema.impl.AbstractTable; + +class CmlTable + extends AbstractTable +{ + private final RelDataType rowType; + + CmlTable(RelDataType relDataType) + { + this.rowType = relDataType; + } + + @Override + public RelDataType getRowType(RelDataTypeFactory typeFactory) + { + return rowType; + } +} diff --git a/core/cml-calcite/src/main/java/io/cml/calcite/SchemaPlusInfo.java b/core/cml-calcite/src/main/java/io/cml/calcite/SchemaPlusInfo.java new file mode 100644 index 000000000..a43b69695 --- /dev/null +++ b/core/cml-calcite/src/main/java/io/cml/calcite/SchemaPlusInfo.java @@ -0,0 +1,35 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.cml.calcite; + +import io.cml.spi.metadata.TableMetadata; + +import java.util.List; +import java.util.Map; + +public class SchemaPlusInfo +{ + private final Map> schemaTableMap; + + public SchemaPlusInfo(Map> schemaTableMap) + { + this.schemaTableMap = schemaTableMap; + } + + public Map> getSchemaTableMap() + { + return schemaTableMap; + } +} diff --git a/core/cml-main/pom.xml b/core/cml-main/pom.xml index bd3ddc86e..43688c32c 100644 --- a/core/cml-main/pom.xml +++ b/core/cml-main/pom.xml @@ -73,6 +73,11 @@ guice + + io.cml + cml-calcite + + io.cml cml-spi diff --git a/core/cml-main/src/main/java/io/cml/connector/bigquery/BigQuerySqlConverter.java b/core/cml-main/src/main/java/io/cml/connector/bigquery/BigQuerySqlConverter.java new file mode 100644 index 000000000..5b3a01bfd --- /dev/null +++ b/core/cml-main/src/main/java/io/cml/connector/bigquery/BigQuerySqlConverter.java @@ -0,0 +1,51 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.cml.connector.bigquery; + +import io.cml.calcite.CmlSchemaUtil; +import io.cml.calcite.SchemaPlusInfo; +import io.cml.spi.connector.Connector; +import io.cml.spi.metadata.TableMetadata; + +import javax.inject.Inject; + +import java.util.List; +import java.util.Map; + +import static com.google.common.collect.ImmutableMap.toImmutableMap; +import static java.util.Objects.requireNonNull; +import static java.util.function.Function.identity; + +public class BigQuerySqlConverter +{ + private final SchemaPlusInfo schemaPlusInfo; + + @Inject + public BigQuerySqlConverter(Connector bigQueryConnector) + { + final Connector connector = requireNonNull(bigQueryConnector, "BigQueryConnector is null"); + + List schemas = connector.listSchemas(); + Map> schemaTableMap = schemas.stream() + .collect(toImmutableMap(identity(), schema -> connector.listTables(schema))); + + this.schemaPlusInfo = new SchemaPlusInfo(schemaTableMap); + } + + public String convertSql(String sql) + { + return CmlSchemaUtil.convertQuery(CmlSchemaUtil.Dialect.BIGQUERY, schemaPlusInfo, sql); + } +} diff --git a/core/cml-server/src/main/java/io/cml/server/module/BigQueryConnectorModule.java b/core/cml-server/src/main/java/io/cml/server/module/BigQueryConnectorModule.java index fa40a4dd4..70136ce29 100644 --- a/core/cml-server/src/main/java/io/cml/server/module/BigQueryConnectorModule.java +++ b/core/cml-server/src/main/java/io/cml/server/module/BigQueryConnectorModule.java @@ -28,6 +28,7 @@ import io.airlift.configuration.AbstractConfigurationAwareModule; import io.cml.connector.bigquery.BigQueryConfig; import io.cml.connector.bigquery.BigQueryConnector; import io.cml.connector.bigquery.BigQueryCredentialsSupplier; +import io.cml.connector.bigquery.BigQuerySqlConverter; import io.cml.pgcatalog.builder.BigQueryPgCatalogTableBuilder; import io.cml.pgcatalog.builder.BigQueryPgFunctionBuilder; import io.cml.pgcatalog.builder.PgCatalogTableBuilder; @@ -50,6 +51,7 @@ public class BigQueryConnectorModule binder.bind(PgCatalogTableBuilder.class).to(BigQueryPgCatalogTableBuilder.class).in(Scopes.SINGLETON); binder.bind(PgFunctionBuilder.class).to(BigQueryPgFunctionBuilder.class).in(Scopes.SINGLETON); binder.bind(PgMetadata.class).to(BigQueryPgMetadata.class).in(Scopes.SINGLETON); + binder.bind(BigQuerySqlConverter.class).in(Scopes.SINGLETON); configBinder(binder).bindConfig(BigQueryConfig.class); } diff --git a/pom.xml b/pom.xml index 37a00f8ac..f8063fc17 100644 --- a/pom.xml +++ b/pom.xml @@ -188,6 +188,12 @@ 1.15 + + io.cml + cml-calcite + ${project.version} + + io.cml cml-main @@ -245,10 +251,18 @@ commons-logging commons-logging + + org.apache.commons + commons-lang3 + org.slf4j slf4j-api + + org.yaml + snakeyaml +