diff --git a/packages/core/actions/src/actions/list.ts b/packages/core/actions/src/actions/list.ts index 0fcf556b14e..a99c4900ef5 100644 --- a/packages/core/actions/src/actions/list.ts +++ b/packages/core/actions/src/actions/list.ts @@ -9,7 +9,8 @@ import { Context } from '..'; import { getRepositoryFromParams, pageArgsToLimitArgs } from '../utils'; -import { DEFAULT_PAGE, DEFAULT_PER_PAGE } from '../constants'; +import { DEFAULT_PAGE, DEFAULT_PER_PAGE, SIMPLE_PAGINATION_LIMIT } from '../constants'; +import _ from 'lodash'; function totalPage(total, pageSize): number { return Math.ceil(total / pageSize); @@ -33,7 +34,7 @@ async function listWithPagination(ctx: Context) { const repository = getRepositoryFromParams(ctx); - const { simplePaginate } = repository.collection?.options || {}; + let { simplePaginate } = repository.collection?.options || {}; const options = { context: ctx, @@ -47,6 +48,22 @@ async function listWithPagination(ctx: Context) { } }); + if (_.isUndefined(simplePaginate) && _.isFunction(repository['getEstimatedRowCount'])) { + const count = await repository['getEstimatedRowCount'](); + if (count > SIMPLE_PAGINATION_LIMIT && repository.collection.options.loadedFromCollectionManager) { + await ctx.app.db.getRepository('collections').update({ + filter: { name: repository.collection.name }, + values: { + options: { + ...repository.collection?.options, + simplePaginate: true, + }, + }, + }); + simplePaginate = true; + } + } + if (simplePaginate) { options.limit = options.limit + 1; diff --git a/packages/core/actions/src/constants.ts b/packages/core/actions/src/constants.ts index 51b2e297b85..12251a67865 100644 --- a/packages/core/actions/src/constants.ts +++ b/packages/core/actions/src/constants.ts @@ -9,3 +9,4 @@ export const DEFAULT_PAGE = 1; export const DEFAULT_PER_PAGE = 20; +export const SIMPLE_PAGINATION_LIMIT = 300000; diff --git a/packages/core/data-source-manager/src/default-actions/list.ts b/packages/core/data-source-manager/src/default-actions/list.ts index 50874f9f127..9ca20bdb010 100644 --- a/packages/core/data-source-manager/src/default-actions/list.ts +++ b/packages/core/data-source-manager/src/default-actions/list.ts @@ -7,9 +7,10 @@ * For more information, please refer to: https://www.nocobase.com/agreement. */ -import { Context } from '@nocobase/actions'; +import { Context, SIMPLE_PAGINATION_LIMIT } from '@nocobase/actions'; import { assign, isValidFilter } from '@nocobase/utils'; import { pageArgsToLimitArgs } from './utils'; +import _ from 'lodash'; function totalPage(total, pageSize): number { return Math.ceil(total / pageSize); @@ -42,7 +43,7 @@ async function listWithPagination(ctx: Context) { const repository = ctx.getCurrentRepository(); - const { simplePaginate } = repository.collection?.options || {}; + let { simplePaginate } = repository.collection?.options || {}; const options = { context: ctx, @@ -56,6 +57,26 @@ async function listWithPagination(ctx: Context) { } }); + if (_.isUndefined(simplePaginate) && _.isFunction(repository['getEstimatedRowCount'])) { + const count = await repository.getEstimatedRowCount(); + if (count > SIMPLE_PAGINATION_LIMIT) { + const resourceName = ctx.action.resourceName; + const collection = ctx.dataSource.collectionManager.getCollection(resourceName); + await ctx.app.db.getRepository('dataSourcesCollections').updateOrCreate({ + filterKeys: ['name', 'dataSourceKey'], + values: { + name: collection.name, + dataSourceKey: ctx.dataSource.options.name, + options: { + ...repository.collection?.options, + simplePaginate: true, + }, + }, + }); + simplePaginate = true; + } + } + if (simplePaginate) { options.limit = options.limit + 1; diff --git a/packages/core/database/src/repository.ts b/packages/core/database/src/repository.ts index b1b9cc5f871..82763521a74 100644 --- a/packages/core/database/src/repository.ts +++ b/packages/core/database/src/repository.ts @@ -14,6 +14,7 @@ import { BulkCreateOptions, ModelStatic, Op, + QueryTypes, Sequelize, FindAndCountOptions as SequelizeAndCountOptions, CountOptions as SequelizeCountOptions, @@ -302,6 +303,83 @@ export class Repository { + const [result] = await this.database.sequelize.query(`SELECT USER FROM DUAL`, { + type: QueryTypes.SELECT, + }); + return result?.['USER'] ?? ''; + } + async aggregate(options: AggregateOptions & { optionsTransformer?: (options: any) => any }): Promise { const { method, field } = options; diff --git a/packages/plugins/@nocobase/plugin-data-source-main/src/server/__tests__/action.test.ts b/packages/plugins/@nocobase/plugin-data-source-main/src/server/__tests__/action.test.ts index f36b80b8b3d..c488591ae2b 100644 --- a/packages/plugins/@nocobase/plugin-data-source-main/src/server/__tests__/action.test.ts +++ b/packages/plugins/@nocobase/plugin-data-source-main/src/server/__tests__/action.test.ts @@ -61,4 +61,55 @@ describe('action test', () => { 'x-uid': 'test', }); }); + + test('enable auto simple pagination for large datasets', async () => { + const collectionRepository = app.db.getCollection('collections').repository; + await collectionRepository.create({ + values: { + name: 'posts', + }, + }); + // @ts-ignore + await db.getRepository('collections').load(); + await db.sync(); + const repository = app.db.getRepository('posts'); + vi.spyOn(repository, 'getEstimatedRowCount').mockResolvedValue(1000000); + await app + .agent() + .resource('posts') + .list({ + fields: ['id'], + pageSize: 1, + page: 2, + sort: ['id'], + }); + + const posts = await collectionRepository.findOne({ where: { name: 'posts' } }); + expect(posts.options.simplePaginate).toBeTruthy(); + + await collectionRepository.update({ + filter: { name: 'posts' }, + values: { + options: { + ...posts.options, + simplePaginate: false, + }, + }, + }); + // @ts-ignore + await db.getRepository('collections').load(); + + await app + .agent() + .resource('posts') + .list({ + fields: ['id'], + pageSize: 1, + page: 2, + sort: ['id'], + }); + const updatedPosts = await collectionRepository.findOne({ where: { name: 'posts' } }); + expect(updatedPosts.options.simplePaginate).toBeFalsy(); + vi.restoreAllMocks(); + }); });