mirror of
https://github.com/Canner/WrenAI.git
synced 2026-10-24 02:25:59 +08:00
feat(wren-ui): Support schema change phase 2 (#442)
* chore(wren-ui/api): update getSchemaChange function * feat(wren-ui/api): detect affected relationships and calculated fields * feat(wren-ui/api): handle resolve schema change * feat(wren-ui): schema change Modal UI * fix(wren-ui): update model columns issue * chore(wren-ui/api): separate getAffectedCalculatedFieldsAndRelationships as new function * chore(wren-ui): improve operating DB code * fixup! feat(wren-ui): schema change Modal UI * chore(wren-ui): use NodeType instead of ResourceTypes * chore(wren-ui): update tip message * fixup! chore(wren-ui/api): separate getAffectedCalculatedFieldsAndRelationships as new function * fixup! fix(wren-ui): update model columns issue * fix(wren-ui/api): Remove unnecessary code as its data will be deleted along with it * feat(wren-ui/api): update delete model function * fix(wren-ui/api): fix find data logic * fixup! fix(wren-ui/api): fix find data logic * feat(wren-ui): get latest model list data when resolving schema change * fix(wren-ui/api): fix logic and update comments * fixup! fix(wren-ui/api): fix logic and update comments * chore(wren-ui): improve function logic * add TODO comment * fix(wren-ui): fix update model logic for CF --------- Co-authored-by: onlyjackfrost <onlyjackfrost@gmail.com>
This commit is contained in:
Freda Lai
co-authored by
onlyjackfrost
onlyjackfrost
parent
311ae6f3bd
commit
00759947f4
@@ -148,6 +148,19 @@ export type DetailStep = {
|
||||
summary: Scalars['String'];
|
||||
};
|
||||
|
||||
export type DetailedAffectedCalculatedFields = {
|
||||
__typename?: 'DetailedAffectedCalculatedFields';
|
||||
displayName: Scalars['String'];
|
||||
referenceName: Scalars['String'];
|
||||
type: Scalars['String'];
|
||||
};
|
||||
|
||||
export type DetailedAffectedRelationships = {
|
||||
__typename?: 'DetailedAffectedRelationships';
|
||||
displayName: Scalars['String'];
|
||||
referenceName: Scalars['String'];
|
||||
};
|
||||
|
||||
export type DetailedChangeColumn = {
|
||||
__typename?: 'DetailedChangeColumn';
|
||||
displayName: Scalars['String'];
|
||||
@@ -157,8 +170,10 @@ export type DetailedChangeColumn = {
|
||||
|
||||
export type DetailedChangeTable = {
|
||||
__typename?: 'DetailedChangeTable';
|
||||
calculatedFields: Array<DetailedAffectedCalculatedFields>;
|
||||
columns: Array<DetailedChangeColumn>;
|
||||
displayName: Scalars['String'];
|
||||
relationships: Array<DetailedAffectedRelationships>;
|
||||
sourceTableName: Scalars['String'];
|
||||
};
|
||||
|
||||
@@ -342,10 +357,6 @@ export type FieldInfo = {
|
||||
type?: Maybe<Scalars['String']>;
|
||||
};
|
||||
|
||||
export type GetMdlInput = {
|
||||
hash: Scalars['String'];
|
||||
};
|
||||
|
||||
export type GetMdlResult = {
|
||||
__typename?: 'GetMDLResult';
|
||||
hash: Scalars['String'];
|
||||
@@ -398,7 +409,6 @@ export type Mutation = {
|
||||
deleteThread: Scalars['Boolean'];
|
||||
deleteView: Scalars['Boolean'];
|
||||
deploy: Scalars['JSON'];
|
||||
getMDL: GetMdlResult;
|
||||
previewData: Scalars['JSON'];
|
||||
previewModelData: Scalars['JSON'];
|
||||
previewSql: Scalars['JSON'];
|
||||
@@ -488,11 +498,6 @@ export type MutationDeleteViewArgs = {
|
||||
};
|
||||
|
||||
|
||||
export type MutationGetMdlArgs = {
|
||||
data?: InputMaybe<GetMdlInput>;
|
||||
};
|
||||
|
||||
|
||||
export type MutationPreviewDataArgs = {
|
||||
where: PreviewDataInput;
|
||||
};
|
||||
@@ -633,6 +638,7 @@ export type Query = {
|
||||
autoGenerateRelation: Array<RecommendRelations>;
|
||||
connectionInfo: ConnectionInfo;
|
||||
diagram: Diagram;
|
||||
getMDL: GetMdlResult;
|
||||
listDataSourceTables: Array<CompactTable>;
|
||||
listModels: Array<ModelInfo>;
|
||||
listViews: Array<ViewInfo>;
|
||||
@@ -655,6 +661,11 @@ export type QueryAskingTaskArgs = {
|
||||
};
|
||||
|
||||
|
||||
export type QueryGetMdlArgs = {
|
||||
hash: Scalars['String'];
|
||||
};
|
||||
|
||||
|
||||
export type QueryModelArgs = {
|
||||
where: ModelWhereInput;
|
||||
};
|
||||
|
||||
@@ -51,7 +51,7 @@ export type SaveRelationsMutation = { __typename?: 'Mutation', saveRelations: an
|
||||
export type SchemaChangeQueryVariables = Types.Exact<{ [key: string]: never; }>;
|
||||
|
||||
|
||||
export type SchemaChangeQuery = { __typename?: 'Query', schemaChange: { __typename?: 'SchemaChange', lastSchemaChangeTime?: string | null, deletedTables?: Array<{ __typename?: 'DetailedChangeTable', sourceTableName: string, displayName: string, columns: Array<{ __typename?: 'DetailedChangeColumn', sourceColumnName: string, displayName: string, type: string }> }> | null, deletedColumns?: Array<{ __typename?: 'DetailedChangeTable', sourceTableName: string, displayName: string, columns: Array<{ __typename?: 'DetailedChangeColumn', sourceColumnName: string, displayName: string, type: string }> }> | null, modifiedColumns?: Array<{ __typename?: 'DetailedChangeTable', sourceTableName: string, displayName: string, columns: Array<{ __typename?: 'DetailedChangeColumn', sourceColumnName: string, displayName: string, type: string }> }> | null } };
|
||||
export type SchemaChangeQuery = { __typename?: 'Query', schemaChange: { __typename?: 'SchemaChange', lastSchemaChangeTime?: string | null, deletedTables?: Array<{ __typename?: 'DetailedChangeTable', sourceTableName: string, displayName: string, columns: Array<{ __typename?: 'DetailedChangeColumn', sourceColumnName: string, displayName: string, type: string }>, relationships: Array<{ __typename?: 'DetailedAffectedRelationships', displayName: string, referenceName: string }>, calculatedFields: Array<{ __typename?: 'DetailedAffectedCalculatedFields', displayName: string, referenceName: string, type: string }> }> | null, deletedColumns?: Array<{ __typename?: 'DetailedChangeTable', sourceTableName: string, displayName: string, columns: Array<{ __typename?: 'DetailedChangeColumn', sourceColumnName: string, displayName: string, type: string }>, relationships: Array<{ __typename?: 'DetailedAffectedRelationships', displayName: string, referenceName: string }>, calculatedFields: Array<{ __typename?: 'DetailedAffectedCalculatedFields', displayName: string, referenceName: string, type: string }> }> | null, modifiedColumns?: Array<{ __typename?: 'DetailedChangeTable', sourceTableName: string, displayName: string, columns: Array<{ __typename?: 'DetailedChangeColumn', sourceColumnName: string, displayName: string, type: string }>, relationships: Array<{ __typename?: 'DetailedAffectedRelationships', displayName: string, referenceName: string }>, calculatedFields: Array<{ __typename?: 'DetailedAffectedCalculatedFields', displayName: string, referenceName: string, type: string }> }> | null } };
|
||||
|
||||
export type TriggerDataSourceDetectionMutationVariables = Types.Exact<{ [key: string]: never; }>;
|
||||
|
||||
@@ -324,6 +324,15 @@ export const SchemaChangeDocument = gql`
|
||||
displayName
|
||||
type
|
||||
}
|
||||
relationships {
|
||||
displayName
|
||||
referenceName
|
||||
}
|
||||
calculatedFields {
|
||||
displayName
|
||||
referenceName
|
||||
type
|
||||
}
|
||||
}
|
||||
deletedColumns {
|
||||
sourceTableName
|
||||
@@ -333,6 +342,15 @@ export const SchemaChangeDocument = gql`
|
||||
displayName
|
||||
type
|
||||
}
|
||||
relationships {
|
||||
displayName
|
||||
referenceName
|
||||
}
|
||||
calculatedFields {
|
||||
displayName
|
||||
referenceName
|
||||
type
|
||||
}
|
||||
}
|
||||
modifiedColumns {
|
||||
sourceTableName
|
||||
@@ -342,6 +360,15 @@ export const SchemaChangeDocument = gql`
|
||||
displayName
|
||||
type
|
||||
}
|
||||
relationships {
|
||||
displayName
|
||||
referenceName
|
||||
}
|
||||
calculatedFields {
|
||||
displayName
|
||||
referenceName
|
||||
type
|
||||
}
|
||||
}
|
||||
lastSchemaChangeTime
|
||||
}
|
||||
|
||||
@@ -81,6 +81,15 @@ export const GET_SCHEMA_CHANGE = gql`
|
||||
displayName
|
||||
type
|
||||
}
|
||||
relationships {
|
||||
displayName
|
||||
referenceName
|
||||
}
|
||||
calculatedFields {
|
||||
displayName
|
||||
referenceName
|
||||
type
|
||||
}
|
||||
}
|
||||
deletedColumns {
|
||||
sourceTableName
|
||||
@@ -90,6 +99,15 @@ export const GET_SCHEMA_CHANGE = gql`
|
||||
displayName
|
||||
type
|
||||
}
|
||||
relationships {
|
||||
displayName
|
||||
referenceName
|
||||
}
|
||||
calculatedFields {
|
||||
displayName
|
||||
referenceName
|
||||
type
|
||||
}
|
||||
}
|
||||
modifiedColumns {
|
||||
sourceTableName
|
||||
@@ -99,6 +117,15 @@ export const GET_SCHEMA_CHANGE = gql`
|
||||
displayName
|
||||
type
|
||||
}
|
||||
relationships {
|
||||
displayName
|
||||
referenceName
|
||||
}
|
||||
calculatedFields {
|
||||
displayName
|
||||
referenceName
|
||||
type
|
||||
}
|
||||
}
|
||||
lastSchemaChangeTime
|
||||
}
|
||||
|
||||
@@ -1,7 +1,8 @@
|
||||
import { camelCase, differenceWith, isEmpty, isEqual } from 'lodash';
|
||||
import { camelCase, differenceWith, isEmpty, isEqual, uniqBy } from 'lodash';
|
||||
import { IContext } from '@server/types';
|
||||
import { getLogger } from 'log4js';
|
||||
import { SchemaChange } from '@server/repositories/schemaChangeRepository';
|
||||
import { Model, ModelColumn, RelationInfo } from '../repositories';
|
||||
|
||||
const logger = getLogger('DataSourceSchemaDetector');
|
||||
logger.level = 'debug';
|
||||
@@ -100,8 +101,70 @@ export default class DataSourceSchemaDetector
|
||||
);
|
||||
}
|
||||
|
||||
const models = await this.ctx.modelRepository.findAllBy({
|
||||
projectId: this.projectId,
|
||||
});
|
||||
|
||||
const modelColumns =
|
||||
await this.ctx.modelColumnRepository.findColumnsByModelIds(
|
||||
models.map((model) => model.id),
|
||||
);
|
||||
|
||||
const affectedModels = models.filter(
|
||||
(model) =>
|
||||
changes.findIndex((table) => table.name === model.sourceTableName) !==
|
||||
-1,
|
||||
);
|
||||
|
||||
const allCalculatedFields = modelColumns.filter(
|
||||
(column) => column.isCalculated,
|
||||
);
|
||||
|
||||
const modelRelationships =
|
||||
await this.ctx.relationRepository.findRelationInfoBy({
|
||||
columnIds: modelColumns.map((column) => column.id),
|
||||
});
|
||||
|
||||
// Handle resolve deleted tables
|
||||
if (schemaChangeType === SchemaChangeType.DELETED_TABLES) {
|
||||
await Promise.all(
|
||||
affectedModels.map(async (model) => {
|
||||
const affectedColumns = changes.find(
|
||||
(table) => table.name === model.sourceTableName,
|
||||
).columns;
|
||||
|
||||
// Get affected calculated fields and affected relationships
|
||||
const affectedMaterials = await this.getAffectedResources(
|
||||
model,
|
||||
affectedColumns,
|
||||
{
|
||||
models,
|
||||
modelColumns,
|
||||
allCalculatedFields,
|
||||
modelRelationships,
|
||||
},
|
||||
);
|
||||
|
||||
// delete calculated fields
|
||||
const affectedCalculatedFields = uniqBy(
|
||||
affectedMaterials.calculatedFields,
|
||||
'id',
|
||||
);
|
||||
|
||||
logger.debug(
|
||||
`Start to remove all affected calculated fields "${affectedCalculatedFields.map(
|
||||
(column) => `${column.displayName} (${column.referenceName})`,
|
||||
)}".`,
|
||||
);
|
||||
|
||||
const columnIds = affectedCalculatedFields.map((column) => column.id);
|
||||
return await this.ctx.modelColumnRepository.deleteAllByColumnIds(
|
||||
columnIds,
|
||||
);
|
||||
}),
|
||||
);
|
||||
|
||||
// delete models
|
||||
const affectedTableNames = changes.map((table) => table.name);
|
||||
logger.debug(
|
||||
`Start to remove tables "${affectedTableNames}" from models.`,
|
||||
@@ -116,22 +179,47 @@ export default class DataSourceSchemaDetector
|
||||
|
||||
// Handle resolve deleted table columns
|
||||
if (schemaChangeType === SchemaChangeType.DELETED_COLUMNS) {
|
||||
const models = await this.ctx.modelRepository.findAllBy({
|
||||
projectId: this.projectId,
|
||||
});
|
||||
const affectedModels = models.filter(
|
||||
(model) =>
|
||||
changes.findIndex((table) => table.name === model.sourceTableName) !==
|
||||
-1,
|
||||
);
|
||||
await Promise.all(
|
||||
affectedModels.map(async (model) => {
|
||||
const affectedColumnNames = changes
|
||||
.find((table) => table.name === model.sourceTableName)
|
||||
.columns.map((column) => column.name);
|
||||
const affectedColumns = changes.find(
|
||||
(table) => table.name === model.sourceTableName,
|
||||
).columns;
|
||||
|
||||
const affectedMaterials = await this.getAffectedResources(
|
||||
model,
|
||||
affectedColumns,
|
||||
{
|
||||
models,
|
||||
modelColumns,
|
||||
allCalculatedFields,
|
||||
modelRelationships,
|
||||
},
|
||||
);
|
||||
|
||||
// delete calculated fields
|
||||
const affectedCalculatedFields = uniqBy(
|
||||
affectedMaterials.calculatedFields,
|
||||
'id',
|
||||
);
|
||||
|
||||
logger.debug(
|
||||
`Start to remove all affected calculated fields "${affectedCalculatedFields.map(
|
||||
(column) => `${column.displayName} (${column.referenceName})`,
|
||||
)}".`,
|
||||
);
|
||||
|
||||
const columnIds = affectedCalculatedFields.map((column) => column.id);
|
||||
await this.ctx.modelColumnRepository.deleteAllByColumnIds(columnIds);
|
||||
|
||||
// delete columns
|
||||
const affectedColumnNames = affectedColumns.map(
|
||||
(column) => column.name,
|
||||
);
|
||||
|
||||
logger.debug(
|
||||
`Start to remove columns "${affectedColumnNames}" from model "${model.referenceName}".`,
|
||||
);
|
||||
|
||||
return await this.ctx.modelColumnRepository.deleteAllBySourceColumnNames(
|
||||
model.id,
|
||||
affectedColumnNames,
|
||||
@@ -303,4 +391,101 @@ export default class DataSourceSchemaDetector
|
||||
});
|
||||
logger.info(`Schema change "${schemaChangeTypes}" resolved successfully.`);
|
||||
}
|
||||
|
||||
private async getAffectedResources(
|
||||
model: Model,
|
||||
affectedColumns: DataSourceSchema['columns'],
|
||||
{
|
||||
models,
|
||||
modelColumns,
|
||||
allCalculatedFields,
|
||||
modelRelationships,
|
||||
}: {
|
||||
models: Model[];
|
||||
modelColumns: ModelColumn[];
|
||||
allCalculatedFields: ModelColumn[];
|
||||
modelRelationships: RelationInfo[];
|
||||
},
|
||||
): Promise<{
|
||||
relationships: Array<{
|
||||
id: number;
|
||||
displayName: string;
|
||||
referenceName: string;
|
||||
}>;
|
||||
calculatedFields: ModelColumn[];
|
||||
}> {
|
||||
const columns: ModelColumn[] = affectedColumns.reduce((result, column) => {
|
||||
const affectedColumn = modelColumns.find(
|
||||
(modelColumn) =>
|
||||
modelColumn.sourceColumnName === column.name &&
|
||||
modelColumn.modelId === model.id,
|
||||
);
|
||||
if (affectedColumn) {
|
||||
result.push(affectedColumn);
|
||||
}
|
||||
return result;
|
||||
}, []);
|
||||
|
||||
return columns.reduce(
|
||||
(result, column) => {
|
||||
// collect affected calculated fields if it's target column
|
||||
const affectedCalculatedFieldsByColumnId = allCalculatedFields.filter(
|
||||
(calculatedField) => {
|
||||
const lineage = JSON.parse(calculatedField.lineage);
|
||||
return lineage && lineage[lineage.length - 1] === column.id;
|
||||
},
|
||||
);
|
||||
|
||||
result.calculatedFields.push(...affectedCalculatedFieldsByColumnId);
|
||||
|
||||
// collect affected relationships
|
||||
const affectedRelationships = modelRelationships
|
||||
.map((relationship) =>
|
||||
[relationship.fromColumnId, relationship.toColumnId].includes(
|
||||
column.id,
|
||||
)
|
||||
? relationship
|
||||
: null,
|
||||
)
|
||||
.filter((relationship) => !!relationship);
|
||||
|
||||
affectedRelationships.forEach((relationship) => {
|
||||
const referenceName =
|
||||
model.referenceName === relationship.fromModelName
|
||||
? relationship.toModelName
|
||||
: relationship.fromModelName;
|
||||
|
||||
const displayName = models.find(
|
||||
(model) => model.referenceName === referenceName,
|
||||
)?.displayName;
|
||||
|
||||
result.relationships.push({
|
||||
displayName,
|
||||
id: relationship.id,
|
||||
referenceName,
|
||||
});
|
||||
|
||||
// collect affected calculated fields if the relationship is in use
|
||||
const affectedCalculatedFieldsByRelationshipId =
|
||||
allCalculatedFields.filter((calculatedField) => {
|
||||
const lineage = JSON.parse(calculatedField.lineage);
|
||||
|
||||
// pop the column ID from the lineage
|
||||
lineage.pop();
|
||||
return lineage && lineage.includes(relationship.id);
|
||||
});
|
||||
|
||||
result.calculatedFields.push(
|
||||
...affectedCalculatedFieldsByRelationshipId,
|
||||
);
|
||||
});
|
||||
|
||||
return result;
|
||||
},
|
||||
{
|
||||
relationships: [],
|
||||
calculatedFields: [],
|
||||
},
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -41,6 +41,10 @@ export interface IModelColumnRepository extends IBasicRepository<ModelColumn> {
|
||||
sourceColumnNames: string[],
|
||||
queryOptions?: IQueryOptions,
|
||||
): Promise<number>;
|
||||
deleteAllByColumnIds(
|
||||
columnIds: number[],
|
||||
queryOptions?: IQueryOptions,
|
||||
): Promise<void>;
|
||||
}
|
||||
|
||||
export class ModelColumnRepository
|
||||
@@ -114,4 +118,14 @@ export class ModelColumnRepository
|
||||
.delete();
|
||||
return await builder;
|
||||
}
|
||||
|
||||
public async deleteAllByColumnIds(
|
||||
columnIds: number[],
|
||||
queryOptions?: IQueryOptions,
|
||||
): Promise<void> {
|
||||
const executer = queryOptions?.tx ? queryOptions.tx : this.knex;
|
||||
await executer<ModelColumn>(this.tableName)
|
||||
.whereIn('id', columnIds)
|
||||
.delete();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -307,6 +307,7 @@ export class ModelResolver {
|
||||
const model = await ctx.modelRepository.findOneBy({ id: args.where.id });
|
||||
const existingColumns = await ctx.modelColumnRepository.findAllBy({
|
||||
modelId: model.id,
|
||||
isCalculated: false,
|
||||
});
|
||||
const { sourceTableName } = model;
|
||||
this.validateTableExist(sourceTableName, dataSourceTables);
|
||||
@@ -315,15 +316,20 @@ export class ModelResolver {
|
||||
const sourceTableColumns = dataSourceTables.find(
|
||||
(table) => table.name === sourceTableName,
|
||||
)?.columns;
|
||||
const { toDeleteColumnIds, toCreateColumns } = findColumnsToUpdate(
|
||||
fields,
|
||||
existingColumns,
|
||||
);
|
||||
const { toDeleteColumnIds, toCreateColumns, toUpdateColumns } =
|
||||
findColumnsToUpdate(fields, existingColumns, sourceTableColumns);
|
||||
await updateModelPrimaryKey(
|
||||
ctx.modelColumnRepository,
|
||||
model.id,
|
||||
primaryKey,
|
||||
);
|
||||
|
||||
// delete columns
|
||||
if (toDeleteColumnIds.length) {
|
||||
await ctx.modelColumnRepository.deleteMany(toDeleteColumnIds);
|
||||
}
|
||||
|
||||
// create columns
|
||||
if (toCreateColumns.length) {
|
||||
const columnValues = toCreateColumns.map((columnName) => {
|
||||
const sourceTableColumn = sourceTableColumns.find(
|
||||
@@ -346,10 +352,15 @@ export class ModelResolver {
|
||||
});
|
||||
await ctx.modelColumnRepository.createMany(columnValues);
|
||||
}
|
||||
if (toDeleteColumnIds.length) {
|
||||
await ctx.modelColumnRepository.deleteMany(toDeleteColumnIds);
|
||||
|
||||
// update columns
|
||||
if (toUpdateColumns.length) {
|
||||
for (const { id, type } of toUpdateColumns) {
|
||||
await ctx.modelColumnRepository.updateOne(id, { type });
|
||||
}
|
||||
}
|
||||
logger.info(`Model created: ${JSON.stringify(model)}`);
|
||||
|
||||
logger.info(`Model updated: ${JSON.stringify(model)}`);
|
||||
|
||||
return model;
|
||||
}
|
||||
@@ -361,13 +372,8 @@ export class ModelResolver {
|
||||
if (!model) {
|
||||
throw new Error('Model not found');
|
||||
}
|
||||
const modelColumns = await ctx.modelColumnRepository.findColumnsByModelIds([
|
||||
model.id,
|
||||
]);
|
||||
logger.debug('find columns');
|
||||
const columnIds = modelColumns.map((c) => c.id);
|
||||
await ctx.relationRepository.deleteRelationsByColumnIds(columnIds);
|
||||
await ctx.modelColumnRepository.deleteMany(columnIds);
|
||||
|
||||
// related columns and relationships will be deleted in cascade
|
||||
await ctx.modelRepository.deleteOne(modelId);
|
||||
return true;
|
||||
}
|
||||
|
||||
@@ -22,7 +22,7 @@ import {
|
||||
getRelations,
|
||||
sampleDatasets,
|
||||
} from '@server/data';
|
||||
import { snakeCase, flatMap } from 'lodash';
|
||||
import { snakeCase, flatMap, uniqBy } from 'lodash';
|
||||
import { CompactTable, ProjectData } from '../services';
|
||||
import { replaceAllowableSyntax } from '../utils/regex';
|
||||
import { DuckDBPrepareOptions } from '@server/adaptors/wrenEngineAdaptor';
|
||||
@@ -440,33 +440,133 @@ export class ProjectResolver {
|
||||
const modelColumns =
|
||||
await ctx.modelColumnRepository.findColumnsByModelIds(modelIds);
|
||||
|
||||
// Mapping with affected models and columns data into schame change
|
||||
const modelRelationships = await ctx.relationRepository.findRelationInfoBy({
|
||||
modelIds,
|
||||
});
|
||||
|
||||
const allCalculatedFields = modelColumns.filter(
|
||||
(column) => column.isCalculated,
|
||||
);
|
||||
|
||||
// TODO: the same logic with getAffectedResources
|
||||
/**
|
||||
* According to affected models and column data, we also need to find affected resources, including calculated fields and relationships.
|
||||
*
|
||||
* Find all affected resources include:
|
||||
* - columns (called "affected column")
|
||||
* - relationships (called "affected relationship")
|
||||
* - calculated fields:
|
||||
* - calculated fields which were affected by affected columns
|
||||
* - calculated fields which were affected by affected relationships
|
||||
*/
|
||||
const mappingAffectedToSchemaChange = (changes: DataSourceSchema[]) => {
|
||||
const affecteds = flatMap(changes, (change) => {
|
||||
const affectedModel = models.find(
|
||||
(model) => model.sourceTableName === change.name,
|
||||
);
|
||||
return affectedModel
|
||||
? {
|
||||
sourceTableName: change.name,
|
||||
displayName: affectedModel.displayName,
|
||||
columns: flatMap(change.columns, (column) => {
|
||||
const affectedColumn = modelColumns.find(
|
||||
(modelColumn) =>
|
||||
modelColumn.sourceColumnName === column.name &&
|
||||
modelColumn.modelId === affectedModel.id,
|
||||
|
||||
if (!affectedModel) return [];
|
||||
|
||||
// collect all columns and affected relationships and affected calculated fields affected by relationships
|
||||
const affectedMaterials = change.columns.reduce(
|
||||
(result, column) => {
|
||||
const affectedColumn = modelColumns.find(
|
||||
(modelColumn) =>
|
||||
modelColumn.sourceColumnName === column.name &&
|
||||
modelColumn.modelId === affectedModel.id,
|
||||
);
|
||||
|
||||
// collect column
|
||||
if (affectedColumn) {
|
||||
result.columns.push({
|
||||
sourceColumnName: column.name,
|
||||
displayName: affectedColumn.displayName,
|
||||
type: column.type,
|
||||
});
|
||||
|
||||
// collect affected calculated fields if it's target column
|
||||
const affectedCalculatedFieldsByColumnId =
|
||||
allCalculatedFields.filter((calculatedField) => {
|
||||
const lineage = JSON.parse(calculatedField.lineage);
|
||||
return (
|
||||
lineage && lineage[lineage.length - 1] === affectedColumn.id
|
||||
);
|
||||
});
|
||||
|
||||
result.calculatedFields.push(
|
||||
...affectedCalculatedFieldsByColumnId,
|
||||
);
|
||||
|
||||
// collect affected relationships
|
||||
const affectedRelationships = modelRelationships
|
||||
.map((relationship) =>
|
||||
[relationship.fromColumnId, relationship.toColumnId].includes(
|
||||
affectedColumn.id,
|
||||
)
|
||||
? relationship
|
||||
: null,
|
||||
)
|
||||
.filter((relationship) => !!relationship);
|
||||
|
||||
affectedRelationships.forEach((relationship) => {
|
||||
const referenceName =
|
||||
affectedModel.referenceName === relationship.fromModelName
|
||||
? relationship.toModelName
|
||||
: relationship.fromModelName;
|
||||
|
||||
const displayName = models.find(
|
||||
(model) => model.referenceName === referenceName,
|
||||
)?.displayName;
|
||||
|
||||
result.relationships.push({
|
||||
displayName,
|
||||
referenceName,
|
||||
});
|
||||
|
||||
// collect affected calculated fields if the relationship is in use
|
||||
const affectedCalculatedFieldsByRelationshipId =
|
||||
allCalculatedFields.filter((calculatedField) => {
|
||||
const lineage = JSON.parse(calculatedField.lineage);
|
||||
|
||||
// pop the column ID from the lineage
|
||||
lineage.pop();
|
||||
return lineage && lineage.includes(relationship.id);
|
||||
});
|
||||
|
||||
result.calculatedFields.push(
|
||||
...affectedCalculatedFieldsByRelationshipId,
|
||||
);
|
||||
return affectedColumn
|
||||
? {
|
||||
sourceColumnName: column.name,
|
||||
displayName: affectedColumn?.displayName,
|
||||
type: column.type,
|
||||
}
|
||||
: [];
|
||||
}),
|
||||
});
|
||||
}
|
||||
: [];
|
||||
|
||||
return result;
|
||||
},
|
||||
{
|
||||
columns: [],
|
||||
relationships: [],
|
||||
calculatedFields: [],
|
||||
},
|
||||
);
|
||||
|
||||
// unique calculated fields by id since it can be duplicated
|
||||
const calculatedFields = uniqBy(
|
||||
affectedMaterials.calculatedFields,
|
||||
'id',
|
||||
).map((calculatedField) => ({
|
||||
displayName: calculatedField.displayName,
|
||||
referenceName: calculatedField.referenceName,
|
||||
type: calculatedField.type,
|
||||
}));
|
||||
|
||||
return {
|
||||
sourceTableName: change.name,
|
||||
displayName: affectedModel.displayName,
|
||||
columns: affectedMaterials.columns,
|
||||
calculatedFields,
|
||||
relationships: affectedMaterials.relationships,
|
||||
};
|
||||
});
|
||||
|
||||
return affecteds.length ? affecteds : null;
|
||||
};
|
||||
|
||||
|
||||
@@ -630,6 +630,8 @@ export const typeDefs = gql`
|
||||
sourceTableName: String!
|
||||
displayName: String!
|
||||
columns: [DetailedChangeColumn!]!
|
||||
calculatedFields: [DetailedAffectedCalculatedFields!]!
|
||||
relationships: [DetailedAffectedRelationships!]!
|
||||
}
|
||||
|
||||
type DetailedChangeColumn {
|
||||
@@ -638,6 +640,17 @@ export const typeDefs = gql`
|
||||
type: String!
|
||||
}
|
||||
|
||||
type DetailedAffectedCalculatedFields {
|
||||
displayName: String!
|
||||
referenceName: String!
|
||||
type: String!
|
||||
}
|
||||
|
||||
type DetailedAffectedRelationships {
|
||||
displayName: String!
|
||||
referenceName: String!
|
||||
}
|
||||
|
||||
input ResolveSchemaChangeWhereInput {
|
||||
type: SchemaChangeType!
|
||||
}
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import { IModelColumnRepository, ModelColumn } from '@server/repositories';
|
||||
import { replaceAllowableSyntax } from './regex';
|
||||
import { CompactColumn } from '@server/services/metadataService';
|
||||
|
||||
export function transformInvalidColumnName(columnName: string) {
|
||||
let referenceName = replaceAllowableSyntax(columnName);
|
||||
@@ -19,9 +20,14 @@ export function replaceInvalidReferenceName(referenceName: string) {
|
||||
export function findColumnsToUpdate(
|
||||
columns: string[],
|
||||
existingColumns: ModelColumn[],
|
||||
sourceTableColumns: CompactColumn[],
|
||||
): {
|
||||
toDeleteColumnIds: number[];
|
||||
toCreateColumns: string[];
|
||||
toUpdateColumns: Array<{
|
||||
id: number;
|
||||
type: string;
|
||||
}>;
|
||||
} {
|
||||
const toDeleteColumnIds = existingColumns
|
||||
.map(({ id, sourceColumnName }) => {
|
||||
@@ -35,9 +41,31 @@ export function findColumnsToUpdate(
|
||||
const toCreateColumns = columns.filter(
|
||||
(columnName) => !existColumnNames.includes(columnName),
|
||||
);
|
||||
|
||||
const toUpdateColumns = sourceTableColumns.reduce((acc, sourceColumn) => {
|
||||
const existingColumn = existingColumns.find(
|
||||
(col) => col.sourceColumnName === sourceColumn.name,
|
||||
);
|
||||
if (!existingColumn) return acc;
|
||||
|
||||
const columnName = columns.find((col) => col === sourceColumn.name);
|
||||
if (!columnName) return acc;
|
||||
|
||||
if (sourceColumn.type === existingColumn.type) return acc;
|
||||
|
||||
return [
|
||||
...acc,
|
||||
{
|
||||
id: existingColumn.id,
|
||||
type: sourceColumn.type || 'string',
|
||||
},
|
||||
];
|
||||
}, []);
|
||||
|
||||
return {
|
||||
toDeleteColumnIds,
|
||||
toCreateColumns,
|
||||
toUpdateColumns,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { useMemo } from 'react';
|
||||
import {
|
||||
Modal,
|
||||
Button,
|
||||
@@ -16,6 +17,10 @@ import FileDoneOutlined from '@ant-design/icons/FileDoneOutlined';
|
||||
import { ModalAction } from '@/hooks/useModalAction';
|
||||
import EllipsisWrapper from '@/components/EllipsisWrapper';
|
||||
import {
|
||||
DetailedChangeTable,
|
||||
DetailedAffectedCalculatedFields,
|
||||
DetailedAffectedRelationships,
|
||||
NodeType,
|
||||
SchemaChange,
|
||||
SchemaChangeType,
|
||||
} from '@/apollo/client/graphql/__types__';
|
||||
@@ -41,6 +46,18 @@ const StyledTable = styled(Table)`
|
||||
.ant-table {
|
||||
border: none;
|
||||
border-radius: 0;
|
||||
|
||||
.non-expandable {
|
||||
.ant-table-row-expand-icon {
|
||||
display: none;
|
||||
}
|
||||
}
|
||||
|
||||
.ant-table-expanded-row {
|
||||
.ant-table-cell {
|
||||
background-color: white;
|
||||
}
|
||||
}
|
||||
}
|
||||
`;
|
||||
|
||||
@@ -52,6 +69,34 @@ type Props = ModalAction<SchemaChange> & {
|
||||
};
|
||||
};
|
||||
|
||||
const nestedColumns = [
|
||||
{
|
||||
title: 'Affected Resource',
|
||||
dataIndex: 'resourceType',
|
||||
width: 200,
|
||||
render: (resourceType: NodeType) => {
|
||||
if (resourceType === NodeType.CALCULATED_FIELD) {
|
||||
return <Tag className="ant-tag--geekblue">Calculated Field</Tag>;
|
||||
}
|
||||
|
||||
if (resourceType === NodeType.RELATION) {
|
||||
return <Tag className="ant-tag--citrus">Relationship</Tag>;
|
||||
}
|
||||
|
||||
return null;
|
||||
},
|
||||
},
|
||||
{
|
||||
title: 'Name',
|
||||
dataIndex: 'displayName',
|
||||
},
|
||||
];
|
||||
|
||||
const checkIsExpandable = (record: DetailedChangeTable) =>
|
||||
record.calculatedFields.length + record.relationships.length > 0
|
||||
? ''
|
||||
: 'non-expandable';
|
||||
|
||||
const PanelHeader = (props) => {
|
||||
const { title, count, onResolve, isResolving } = props;
|
||||
const resolve = (event) => {
|
||||
@@ -95,11 +140,90 @@ const PanelHeader = (props) => {
|
||||
);
|
||||
};
|
||||
|
||||
interface ExpandedRowsProps {
|
||||
record: DetailedChangeTable & {
|
||||
resources: Array<
|
||||
DetailedAffectedCalculatedFields | DetailedAffectedRelationships
|
||||
>;
|
||||
};
|
||||
tipMessage: string;
|
||||
}
|
||||
|
||||
const ExpandedRows = ({ record, tipMessage }: ExpandedRowsProps) => {
|
||||
if (record.resources.length === 0) return null;
|
||||
|
||||
return (
|
||||
<div className="pl-12">
|
||||
<Alert
|
||||
showIcon
|
||||
icon={<WarningOutlined className="orange-5" />}
|
||||
className="gray-6 ml-2 bg-gray-1 pl-0"
|
||||
style={{ border: 'none' }}
|
||||
message={tipMessage}
|
||||
/>
|
||||
<Table
|
||||
columns={nestedColumns}
|
||||
dataSource={record.resources || []}
|
||||
pagination={{
|
||||
hideOnSinglePage: true,
|
||||
size: 'small',
|
||||
pageSize: 10,
|
||||
}}
|
||||
rowKey="rowKey"
|
||||
size="small"
|
||||
className="adm-nested-table"
|
||||
/>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
|
||||
export default function SchemaChangeModal(props: Props) {
|
||||
const { visible, onClose, defaultValue: schemaChange, payload } = props;
|
||||
const { deletedTables, deletedColumns, modifiedColumns } = schemaChange || {};
|
||||
const { onResolveSchemaChange, isResolving } = payload || {};
|
||||
|
||||
const { deletedTables, deletedColumns, modifiedColumns } = useMemo(() => {
|
||||
const { deletedTables, deletedColumns, modifiedColumns } =
|
||||
schemaChange || {};
|
||||
|
||||
if (!schemaChange)
|
||||
return {
|
||||
deletedTables,
|
||||
deletedColumns,
|
||||
modifiedColumns,
|
||||
};
|
||||
|
||||
// transform data to render UI
|
||||
const transformData = (tables: DetailedChangeTable) => ({
|
||||
...tables,
|
||||
resources: [
|
||||
...tables.calculatedFields.map(
|
||||
(
|
||||
calculatedField: DetailedAffectedCalculatedFields,
|
||||
index: number,
|
||||
) => ({
|
||||
...calculatedField,
|
||||
resourceType: NodeType.CALCULATED_FIELD,
|
||||
rowKey: `${tables.sourceTableName}-${calculatedField.referenceName}-${index}`,
|
||||
}),
|
||||
),
|
||||
...tables.relationships.map(
|
||||
(relationship: DetailedAffectedRelationships, index: number) => ({
|
||||
...relationship,
|
||||
resourceType: NodeType.RELATION,
|
||||
rowKey: `${tables.sourceTableName}-${relationship.referenceName}-${index}`,
|
||||
}),
|
||||
),
|
||||
],
|
||||
rowKey: tables.sourceTableName,
|
||||
});
|
||||
|
||||
return {
|
||||
deletedTables: deletedTables?.map(transformData),
|
||||
deletedColumns: deletedColumns?.map(transformData),
|
||||
modifiedColumns: modifiedColumns?.map(transformData),
|
||||
};
|
||||
}, [schemaChange]);
|
||||
|
||||
const columnsOfDeleteTables = [
|
||||
{ title: 'Affected model', width: 200, dataIndex: 'displayName' },
|
||||
{ title: 'Source table name', dataIndex: 'sourceTableName' },
|
||||
@@ -187,7 +311,7 @@ export default function SchemaChangeModal(props: Props) {
|
||||
key="deleteTables"
|
||||
>
|
||||
<StyledTable
|
||||
rowKey={(record: any) => record.sourceTableName}
|
||||
rowKey="rowKey"
|
||||
columns={columnsOfDeleteTables}
|
||||
dataSource={deletedTables}
|
||||
size="small"
|
||||
@@ -196,6 +320,15 @@ export default function SchemaChangeModal(props: Props) {
|
||||
size: 'small',
|
||||
pageSize: 10,
|
||||
}}
|
||||
rowClassName={checkIsExpandable}
|
||||
expandable={{
|
||||
expandedRowRender: (record: ExpandedRowsProps['record']) => (
|
||||
<ExpandedRows
|
||||
record={record}
|
||||
tipMessage="The following table shows resources affected by this model and will be deleted when resolving."
|
||||
/>
|
||||
),
|
||||
}}
|
||||
/>
|
||||
</Collapse.Panel>
|
||||
)}
|
||||
@@ -214,7 +347,7 @@ export default function SchemaChangeModal(props: Props) {
|
||||
key="deleteColumns"
|
||||
>
|
||||
<StyledTable
|
||||
rowKey={(record: any) => record.sourceTableName}
|
||||
rowKey="rowKey"
|
||||
columns={columnsOfDeletedColumns}
|
||||
dataSource={deletedColumns}
|
||||
size="small"
|
||||
@@ -223,6 +356,15 @@ export default function SchemaChangeModal(props: Props) {
|
||||
size: 'small',
|
||||
pageSize: 10,
|
||||
}}
|
||||
rowClassName={checkIsExpandable}
|
||||
expandable={{
|
||||
expandedRowRender: (record: ExpandedRowsProps['record']) => (
|
||||
<ExpandedRows
|
||||
record={record}
|
||||
tipMessage="The following table shows resources affected by this column of the model and will be deleted when resolving."
|
||||
/>
|
||||
),
|
||||
}}
|
||||
/>
|
||||
</Collapse.Panel>
|
||||
)}
|
||||
@@ -237,7 +379,7 @@ export default function SchemaChangeModal(props: Props) {
|
||||
key="modifiedColumns"
|
||||
>
|
||||
<StyledTable
|
||||
rowKey={(record: any) => record.sourceTableName}
|
||||
rowKey="rowKey"
|
||||
columns={columnsOfModifiedColumns}
|
||||
dataSource={modifiedColumns}
|
||||
size="small"
|
||||
@@ -246,6 +388,15 @@ export default function SchemaChangeModal(props: Props) {
|
||||
size: 'small',
|
||||
pageSize: 10,
|
||||
}}
|
||||
rowClassName={checkIsExpandable}
|
||||
expandable={{
|
||||
expandedRowRender: (record: ExpandedRowsProps['record']) => (
|
||||
<ExpandedRows
|
||||
record={record}
|
||||
tipMessage="The following table shows the resources utilized by this column of the model. Please review each resource and manually update the relevant ones if any changes are required."
|
||||
/>
|
||||
),
|
||||
}}
|
||||
/>
|
||||
</Collapse.Panel>
|
||||
)}
|
||||
|
||||
@@ -79,39 +79,6 @@ export default function ModelForm(props: Props) {
|
||||
}
|
||||
}, [sourceTableFieldValue]);
|
||||
|
||||
useEffect(() => {
|
||||
if (defaultValue) {
|
||||
const fields: string[] = defaultValue.fields.map(
|
||||
(field: DiagramModelField) => field.referenceName,
|
||||
);
|
||||
|
||||
const primaryKeyField = defaultValue.fields.find(
|
||||
(field: DiagramModelField) => field.isPrimaryKey,
|
||||
);
|
||||
|
||||
form.setFieldsValue({
|
||||
[FormFieldKey.COLUMNS]: fields,
|
||||
[FormFieldKey.PRIMARY_KEY]: primaryKeyField?.referenceName,
|
||||
});
|
||||
|
||||
setSourceTableName(defaultValue.sourceTableName);
|
||||
setSelectedColumns(fields);
|
||||
}
|
||||
}, [defaultValue, form]);
|
||||
|
||||
const tableOptions: JSX.Element[] = dataSourceTables.map(
|
||||
(table: CompactTable) => {
|
||||
const disabled = inUsedModelList.includes(table.name);
|
||||
const option = {
|
||||
disabled,
|
||||
children: table.name,
|
||||
value: table.name,
|
||||
};
|
||||
|
||||
return <Option {...option} key={option.value} />;
|
||||
},
|
||||
);
|
||||
|
||||
const columns: Array<{
|
||||
key: string;
|
||||
name: string;
|
||||
@@ -130,6 +97,39 @@ export default function ModelForm(props: Props) {
|
||||
}));
|
||||
}, [dataSourceTables, sourceTableName]);
|
||||
|
||||
useEffect(() => {
|
||||
if (defaultValue) {
|
||||
const fields: string[] = defaultValue.fields
|
||||
.map((field: DiagramModelField) => field.referenceName)
|
||||
.filter((col) => columns.find((c) => c.name === col));
|
||||
|
||||
const primaryKeyField = defaultValue.fields.find(
|
||||
(field: DiagramModelField) => field.isPrimaryKey,
|
||||
);
|
||||
|
||||
form.setFieldsValue({
|
||||
[FormFieldKey.COLUMNS]: fields,
|
||||
[FormFieldKey.PRIMARY_KEY]: primaryKeyField?.referenceName,
|
||||
});
|
||||
|
||||
setSourceTableName(defaultValue.sourceTableName);
|
||||
setSelectedColumns(fields);
|
||||
}
|
||||
}, [defaultValue, form, columns]);
|
||||
|
||||
const tableOptions: JSX.Element[] = dataSourceTables.map(
|
||||
(table: CompactTable) => {
|
||||
const disabled = inUsedModelList.includes(table.name);
|
||||
const option = {
|
||||
disabled,
|
||||
children: table.name,
|
||||
value: table.name,
|
||||
};
|
||||
|
||||
return <Option {...option} key={option.value} />;
|
||||
},
|
||||
);
|
||||
|
||||
const onChangeColumns = (newKeys: string[]) => setSelectedColumns(newKeys);
|
||||
|
||||
const dataSourceTablesLoading = fetching || listModelsQueryLoading;
|
||||
|
||||
@@ -24,6 +24,7 @@ import {
|
||||
useTriggerDataSourceDetectionMutation,
|
||||
} from '@/apollo/client/graphql/dataSource.generated';
|
||||
import { DIAGRAM } from '@/apollo/client/graphql/diagram';
|
||||
import { LIST_MODELS } from '@/apollo/client/graphql/model';
|
||||
|
||||
dayJs.extend(utc);
|
||||
dayJs.extend(relativeTime);
|
||||
@@ -75,7 +76,7 @@ export default function ModelTree(props: Props) {
|
||||
schemaChangeModal.closeModal();
|
||||
}
|
||||
},
|
||||
refetchQueries: [{ query: DIAGRAM }],
|
||||
refetchQueries: [{ query: DIAGRAM }, { query: LIST_MODELS }],
|
||||
});
|
||||
const { data: schemaChangeData, refetch: refetchSchemaChange } =
|
||||
useSchemaChangeQuery({
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
.ant-alert {
|
||||
align-items: flex-start;
|
||||
.anticon-exclamation-circle {
|
||||
.anticon-exclamation-circle, .anticon-warning {
|
||||
margin-top: 4px;
|
||||
}
|
||||
.ant-alert-message {
|
||||
|
||||
@@ -21,6 +21,22 @@
|
||||
background-color: @gray-3;
|
||||
}
|
||||
}
|
||||
|
||||
.adm-nested-table {
|
||||
.ant-table {
|
||||
border: none;
|
||||
border-radius: 0;
|
||||
|
||||
.ant-table-thead > tr > th {
|
||||
background: @gray-2;
|
||||
}
|
||||
|
||||
.ant-table-tbody > tr.ant-table-row:hover > td,
|
||||
.ant-table-tbody > tr > td.ant-table-cell-row-hover {
|
||||
background: @gray-2;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
.ant-table-wrapper:not(.ant-table-has-header) {
|
||||
|
||||
@@ -4,4 +4,9 @@
|
||||
background: @geekblue-1;
|
||||
border-color: @geekblue-6;
|
||||
}
|
||||
&--citrus {
|
||||
color: @citrus-6;
|
||||
background: @citrus-1;
|
||||
border-color: @citrus-6;
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user