Feature: Support ClickHouse data source (#481)

* add ClickHouse data source

* remove unused interface

* support clickhouse UI

* update ibis base url

wren-ui/public/images/dataSource/clickhouse.svg

@@ -0,0 +1,11 @@
+<svg width="40" height="40" viewBox="0 0 40 40" fill="none" xmlns="http://www.w3.org/2000/svg">
+<g clip-path="url(#clip0_102_61)">
+<path d="M5.00146 30.1818H8.33446V33.4935H5.00146V30.1818Z" fill="#FF0000"/>
+<path d="M5.00146 7H8.33446V30.1818H5.00146V7ZM11.6675 7H15.0005V33.4935H11.6675V7ZM18.3335 7H21.6665V33.4935H18.3335V7ZM24.9995 7H28.3325V33.4935H24.9995V7ZM31.6655 17.763H34.9985V22.7305H31.6655V17.763Z" fill="#FFCC00"/>
+</g>
+<defs>
+<clipPath id="clip0_102_61">
+<rect width="30" height="26.4935" fill="white" transform="translate(5 7)"/>
+</clipPath>
+</defs>
+</svg>

wren-ui/src/apollo/client/graphql/__types__.ts

@@ -135,6 +135,7 @@ export type DataSourceInput = {
 
 export enum DataSourceName {
   BIG_QUERY = 'BIG_QUERY',
+  CLICK_HOUSE = 'CLICK_HOUSE',
   DUCKDB = 'DUCKDB',
   MSSQL = 'MSSQL',
   MYSQL = 'MYSQL',
@@ -498,6 +499,11 @@ export type MutationDeleteViewArgs = {
 };
 
 
+export type MutationDeployArgs = {
+  force?: InputMaybe<Scalars['Boolean']>;
+};
+
+
 export type MutationPreviewDataArgs = {
   where: PreviewDataInput;
 };

wren-ui/src/apollo/server/adaptors/ibisAdaptor.ts

@@ -36,22 +36,6 @@ export type IbisPostgresConnectionInfo =
   | UrlBasedConnectionInfo
   | HostBasedConnectionInfo;
 
-export interface IbisMySQLConnectionInfo {
-  host: string;
-  port: number;
-  database: string;
-  user: string;
-  password: string;
-}
-
-export interface IbisSqlServerConnectionInfo {
-  host: string;
-  port: number;
-  database: string;
-  user: string;
-  password: string;
-}
-
 export interface IbisBigQueryConnectionInfo {
   project_id: string;
   dataset_id: string;
@@ -59,10 +43,10 @@ export interface IbisBigQueryConnectionInfo {
 }
 
 export type IbisConnectionInfo =
+  | UrlBasedConnectionInfo
+  | HostBasedConnectionInfo
   | IbisPostgresConnectionInfo
-  | IbisMySQLConnectionInfo
-  | IbisBigQueryConnectionInfo
-  | IbisSqlServerConnectionInfo;
+  | IbisBigQueryConnectionInfo;
 
 export enum SupportedDataSource {
   POSTGRES = 'POSTGRES',
@@ -70,6 +54,7 @@ export enum SupportedDataSource {
   SNOWFLAKE = 'SNOWFLAKE',
   MYSQL = 'MYSQL',
   MSSQL = 'MSSQL',
+  CLICK_HOUSE = 'CLICK_HOUSE',
 }
 
 const dataSourceUrlMap: Record<SupportedDataSource, string> = {
@@ -78,6 +63,7 @@ const dataSourceUrlMap: Record<SupportedDataSource, string> = {
   [SupportedDataSource.SNOWFLAKE]: 'snowflake',
   [SupportedDataSource.MYSQL]: 'mysql',
   [SupportedDataSource.MSSQL]: 'mssql',
+  [SupportedDataSource.CLICK_HOUSE]: 'clickhouse',
 };
 export interface TableResponse {
   tables: CompactTable[];
@@ -132,10 +118,10 @@ export interface IbisQueryResponse {
 }
 
 export class IbisAdaptor implements IIbisAdaptor {
-  private ibisServerEndpoint: string;
+  private ibisServerBaseUrl: string;
 
   constructor({ ibisServerEndpoint }: { ibisServerEndpoint: string }) {
-    this.ibisServerEndpoint = ibisServerEndpoint;
+    this.ibisServerBaseUrl = `${ibisServerEndpoint}/v2/connector`;
   }
 
   public async query(
@@ -153,7 +139,7 @@ export class IbisAdaptor implements IIbisAdaptor {
     logger.debug(`Querying ibis with body: ${JSON.stringify(body, null, 2)}`);
     try {
       const res = await axios.post(
-        `${this.ibisServerEndpoint}/v2/ibis/${dataSourceUrlMap[dataSource]}/query`,
+        `${this.ibisServerBaseUrl}/${dataSourceUrlMap[dataSource]}/query`,
         body,
         {
           params: {
@@ -188,7 +174,7 @@ export class IbisAdaptor implements IIbisAdaptor {
     logger.debug(`Dry run sql from ibis with body:`);
     try {
       await axios.post(
-        `${this.ibisServerEndpoint}/v2/ibis/${dataSourceUrlMap[dataSource]}/query?dryRun=true`,
+        `${this.ibisServerBaseUrl}/${dataSourceUrlMap[dataSource]}/query?dryRun=true`,
         body,
       );
       logger.debug(`Ibis server Dry run success`);
@@ -214,7 +200,7 @@ export class IbisAdaptor implements IIbisAdaptor {
     try {
       logger.debug(`Getting tables from ibis`);
       const res: AxiosResponse<CompactTable[]> = await axios.post(
-        `${this.ibisServerEndpoint}/v2/ibis/${dataSourceUrlMap[dataSource]}/metadata/tables`,
+        `${this.ibisServerBaseUrl}/${dataSourceUrlMap[dataSource]}/metadata/tables`,
         body,
       );
       return res.data;
@@ -241,7 +227,7 @@ export class IbisAdaptor implements IIbisAdaptor {
     try {
       logger.debug(`Getting constraint from ibis`);
       const res: AxiosResponse<RecommendConstraint[]> = await axios.post(
-        `${this.ibisServerEndpoint}/v2/ibis/${dataSourceUrlMap[dataSource]}/metadata/constraints`,
+        `${this.ibisServerBaseUrl}/${dataSourceUrlMap[dataSource]}/metadata/constraints`,
         body,
       );
       return res.data;
@@ -273,7 +259,7 @@ export class IbisAdaptor implements IIbisAdaptor {
     try {
       logger.debug(`Run validation rule "${validationRule}" with ibis`);
       await axios.post(
-        `${this.ibisServerEndpoint}/v2/ibis/${dataSourceUrlMap[dataSource]}/validate/${snakeCase(validationRule)}`,
+        `${this.ibisServerBaseUrl}/${dataSourceUrlMap[dataSource]}/validate/${snakeCase(validationRule)}`,
         body,
       );
       return { valid: true, message: null };

wren-ui/src/apollo/server/adaptors/tests/ibisAdaptor.test.ts

@@ -4,6 +4,7 @@ import { DataSourceName } from '../../types';
 import { Manifest } from '../../mdl/type';
 import {
   BIG_QUERY_CONNECTION_INFO,
+  CLICK_HOUSE_CONNECTION_INFO,
   MS_SQL_CONNECTION_INFO,
   MYSQL_CONNECTION_INFO,
   POSTGRES_CONNECTION_INFO,
@@ -45,6 +46,15 @@ describe('IbisAdaptor', () => {
     password: 'my-password',
     ssl: true,
   };
+
+  const mockClickHouseConnectionInfo: CLICK_HOUSE_CONNECTION_INFO = {
+    host: 'localhost',
+    port: 8443,
+    database: 'my-database',
+    user: 'my-user',
+    password: 'my-password',
+    ssl: true,
+  };
   const { host, port, database, user, password } = mockPostgresConnectionInfo;
   const postgresConnectionUrl = `postgresql://${user}:${password}@${host}:${port}/${database}?sslmode=require`;
 
@@ -148,6 +158,29 @@ describe('IbisAdaptor', () => {
     );
   });
 
+  it('should get click house constraints', async () => {
+    const mockResponse = { data: [] };
+    mockedAxios.post.mockResolvedValue(mockResponse);
+    // mock decrypt method in Encryptor to return the same password
+    mockedEncryptor.prototype.decrypt.mockReturnValue(
+      JSON.stringify({ password: mockClickHouseConnectionInfo.password }),
+    );
+
+    const result = await ibisAdaptor.getConstraints(
+      DataSourceName.CLICK_HOUSE,
+      mockClickHouseConnectionInfo,
+    );
+    const expectConnectionInfo = Object.entries(
+      mockClickHouseConnectionInfo,
+    ).reduce((acc, [key, value]) => ((acc[snakeCase(key)] = value), acc), {});
+
+    expect(result).toEqual([]);
+    expect(mockedAxios.post).toHaveBeenCalledWith(
+      `${ibisServerEndpoint}/v2/ibis/clickhouse/metadata/constraints`,
+      { connectionInfo: expectConnectionInfo },
+    );
+  });
+
   it('should get postgres constraints', async () => {
     const mockResponse = { data: [] };
     mockedAxios.post.mockResolvedValue(mockResponse);

wren-ui/src/apollo/server/config.ts

@@ -68,7 +68,7 @@ const defaultConfig = {
   wrenAIEndpoint: 'http://localhost:5555',
 
   // ibis server
-  ibisServerEndpoint: 'http://localhost:8000',
+  ibisServerEndpoint: 'http://127.0.0.1:8000',
 
   // encryption
   encryptionPassword: 'sementic',

wren-ui/src/apollo/server/dataSource.ts

@@ -1,8 +1,8 @@
 import {
   IbisBigQueryConnectionInfo,
-  IbisMySQLConnectionInfo,
   IbisPostgresConnectionInfo,
-  IbisSqlServerConnectionInfo,
+  HostBasedConnectionInfo,
+  UrlBasedConnectionInfo,
 } from './adaptors/ibisAdaptor';
 import {
   BIG_QUERY_CONNECTION_INFO,
@@ -11,6 +11,7 @@ import {
   POSTGRES_CONNECTION_INFO,
   MS_SQL_CONNECTION_INFO,
   WREN_AI_CONNECTION_INFO,
+  CLICK_HOUSE_CONNECTION_INFO,
 } from './repositories';
 import { DataSourceName } from './types';
 import { getConfig } from './config';
@@ -110,7 +111,7 @@ const dataSource = {
     },
   } as IDataSourceConnectionInfo<
     MYSQL_CONNECTION_INFO,
-    IbisMySQLConnectionInfo
+    HostBasedConnectionInfo
   >,
 
   // SQL Server
@@ -127,7 +128,28 @@ const dataSource = {
     },
   } as IDataSourceConnectionInfo<
     MS_SQL_CONNECTION_INFO,
-    IbisSqlServerConnectionInfo
+    HostBasedConnectionInfo
+  >,
+
+  // Click House
+  [DataSourceName.CLICK_HOUSE]: {
+    sensitiveProps: ['password'],
+    toIbisConnectionInfo(connectionInfo) {
+      const decryptedConnectionInfo = decryptConnectionInfo(
+        DataSourceName.CLICK_HOUSE,
+        connectionInfo,
+      );
+      const { host, port, database, user, password, ssl } =
+        decryptedConnectionInfo as CLICK_HOUSE_CONNECTION_INFO;
+      let connectionUrl = `clickhouse://${user}:${password}@${host}:${port}/${database}?`;
+      if (ssl) {
+        connectionUrl += 'secure=1';
+      }
+      return { connectionUrl };
+    },
+  } as IDataSourceConnectionInfo<
+    CLICK_HOUSE_CONNECTION_INFO,
+    UrlBasedConnectionInfo
   >,
 
   // DuckDB

wren-ui/src/apollo/server/repositories/projectRepository.ts

@@ -40,6 +40,15 @@ export interface MS_SQL_CONNECTION_INFO {
   database: string;
 }
 
+export interface CLICK_HOUSE_CONNECTION_INFO {
+  host: string;
+  port: number;
+  user: string;
+  password: string;
+  database: string;
+  ssl: boolean;
+}
+
 export interface DUCKDB_CONNECTION_INFO {
   initSql: string;
   extensions: Array<string>;
@@ -51,7 +60,8 @@ export type WREN_AI_CONNECTION_INFO =
   | POSTGRES_CONNECTION_INFO
   | MYSQL_CONNECTION_INFO
   | DUCKDB_CONNECTION_INFO
-  | MS_SQL_CONNECTION_INFO;
+  | MS_SQL_CONNECTION_INFO
+  | CLICK_HOUSE_CONNECTION_INFO;
 
 export interface Project {
   id: number; // ID

wren-ui/src/apollo/server/schema.ts

@@ -9,6 +9,7 @@ export const typeDefs = gql`
     POSTGRES
     MYSQL
     MSSQL
+    CLICK_HOUSE
   }
 
   enum ExpressionName {

wren-ui/src/apollo/server/services/queryService.ts

@@ -5,8 +5,6 @@ import {
   SupportedDataSource,
   IIbisAdaptor,
   IbisQueryResponse,
-  IbisPostgresConnectionInfo,
-  IbisBigQueryConnectionInfo,
   ValidationRules,
 } from '../adaptors/ibisAdaptor';
 import { getLogger } from '@server/utils';
@@ -46,11 +44,6 @@ export interface SqlValidateOptions {
   modelingOnly?: boolean;
 }
 
-export interface ComposeConnectionInfoResult {
-  datasource: DataSourceName;
-  connectionInfo?: IbisPostgresConnectionInfo | IbisBigQueryConnectionInfo;
-}
-
 export interface ValidateResponse {
   valid: boolean;
   message?: string;

wren-ui/src/apollo/server/types/dataSource.ts

@@ -4,6 +4,7 @@ export enum DataSourceName {
   POSTGRES = 'POSTGRES',
   MYSQL = 'MYSQL',
   MSSQL = 'MSSQL',
+  CLICK_HOUSE = 'CLICK_HOUSE',
 }
 
 export interface DataSource {

wren-ui/src/components/pages/setup/dataSources/ClickHouseProperties.tsx

@@ -0,0 +1,99 @@
+import { Form, Input, Switch } from 'antd';
+import { ERROR_TEXTS } from '@/utils/error';
+import { FORM_MODE } from '@/utils/enum';
+
+interface Props {
+  mode?: FORM_MODE;
+}
+
+export default function ClickHouseProperties(props: Props) {
+  const { mode } = props;
+  const isEditMode = mode === FORM_MODE.EDIT;
+  return (
+    <>
+      <Form.Item
+        label="Display name"
+        name="displayName"
+        required
+        rules={[
+          {
+            required: true,
+            message: ERROR_TEXTS.CONNECTION.DISPLAY_NAME.REQUIRED,
+          },
+        ]}
+      >
+        <Input />
+      </Form.Item>
+      <Form.Item
+        label="Host"
+        name="host"
+        required
+        rules={[
+          {
+            required: true,
+            message: ERROR_TEXTS.CONNECTION.HOST.REQUIRED,
+          },
+        ]}
+      >
+        <Input
+          placeholder="<your_click_house_account>.clickhouse.cloud"
+          disabled={isEditMode}
+        />
+      </Form.Item>
+      <Form.Item
+        label="Port"
+        name="port"
+        required
+        rules={[
+          {
+            required: true,
+            message: ERROR_TEXTS.CONNECTION.PORT.REQUIRED,
+          },
+        ]}
+      >
+        <Input placeholder="8443" disabled={isEditMode} />
+      </Form.Item>
+      <Form.Item
+        label="Username"
+        name="user"
+        rules={[
+          {
+            required: true,
+            message: ERROR_TEXTS.CONNECTION.USERNAME.REQUIRED,
+          },
+        ]}
+      >
+        <Input />
+      </Form.Item>
+      <Form.Item
+        label="Password"
+        name="password"
+        required
+        rules={[
+          {
+            required: true,
+            message: ERROR_TEXTS.CONNECTION.PASSWORD.REQUIRED,
+          },
+        ]}
+      >
+        <Input.Password placeholder="input password" />
+      </Form.Item>
+      <Form.Item
+        label="Database name"
+        name="database"
+        required
+        rules={[
+          {
+            required: true,
+            message: ERROR_TEXTS.CONNECTION.DATABASE.REQUIRED,
+          },
+        ]}
+      >
+        <Input placeholder="ClickHouse database name" disabled={isEditMode} />
+      </Form.Item>
+      <Form.Item label="Use SSL" name="ssl" valuePropName="checked">
+        <Switch />
+      </Form.Item>
+    </>
+  );
+}

wren-ui/src/components/pages/setup/utils.tsx

@@ -12,6 +12,7 @@ import DuckDBProperties from './dataSources/DuckDBProperties';
 import MySQLProperties from './dataSources/MySQLProperties';
 import PostgreSQLProperties from './dataSources/PostgreSQLProperties';
 import SQLServerProperties from './dataSources/SQLServerProperties';
+import ClickHouseProperties from './dataSources/ClickHouseProperties';
 import { SampleDatasetName } from '@/apollo/client/graphql/__types__';
 import { ERROR_CODES } from '@/utils/errorHandler';
 
@@ -88,6 +89,12 @@ export const DATA_SOURCE_OPTIONS = {
     guide: 'https://docs.getwren.ai/guide/connect/sqlserver',
     disabled: false,
   },
+  [DATA_SOURCES.CLICK_HOUSE]: {
+    label: 'ClickHouse',
+    logo: '/images/dataSource/clickhouse.svg',
+    guide: 'https://docs.getwren.ai/guide/connect/clickhouse',
+    disabled: false,
+  },
 } as { [key: string]: ButtonOption };
 
 export const DATA_SOURCE_FORM = {
@@ -96,6 +103,7 @@ export const DATA_SOURCE_FORM = {
   [DATA_SOURCES.PG_SQL]: { component: PostgreSQLProperties },
   [DATA_SOURCES.MYSQL]: { component: MySQLProperties },
   [DATA_SOURCES.MSSQL]: { component: SQLServerProperties },
+  [DATA_SOURCES.CLICK_HOUSE]: { component: ClickHouseProperties },
 };
 
 export const TEMPLATE_OPTIONS = {
@@ -140,6 +148,10 @@ export const getDataSource = (dataSource: DATA_SOURCES) => {
         DATA_SOURCE_OPTIONS[DATA_SOURCES.MSSQL],
         DATA_SOURCE_FORM[DATA_SOURCES.MSSQL],
       ),
+      [DATA_SOURCES.CLICK_HOUSE]: merge(
+        DATA_SOURCE_OPTIONS[DATA_SOURCES.CLICK_HOUSE],
+        DATA_SOURCE_FORM[DATA_SOURCES.CLICK_HOUSE],
+      ),
     }[dataSource] || defaultDataSource
   );
 };

wren-ui/src/utils/enum/dataSources.ts

@@ -4,4 +4,5 @@ export enum DATA_SOURCES {
   PG_SQL = 'POSTGRES',
   MYSQL = 'MYSQL',
   MSSQL = 'MSSQL',
+  CLICK_HOUSE = 'CLICK_HOUSE',
 }