perf: enable auto simple pagination for large datasets (#7227)

packages/core/actions/src/actions/list.ts

@@ -9,7 +9,8 @@
 
 import { Context } from '..';
 import { getRepositoryFromParams, pageArgsToLimitArgs } from '../utils';
-import { DEFAULT_PAGE, DEFAULT_PER_PAGE } from '../constants';
+import { DEFAULT_PAGE, DEFAULT_PER_PAGE, SIMPLE_PAGINATION_LIMIT } from '../constants';
+import _ from 'lodash';
 
 function totalPage(total, pageSize): number {
   return Math.ceil(total / pageSize);
@@ -33,7 +34,7 @@ async function listWithPagination(ctx: Context) {
 
   const repository = getRepositoryFromParams(ctx);
 
-  const { simplePaginate } = repository.collection?.options || {};
+  let { simplePaginate } = repository.collection?.options || {};
 
   const options = {
     context: ctx,
@@ -47,6 +48,22 @@ async function listWithPagination(ctx: Context) {
     }
   });
 
+  if (_.isUndefined(simplePaginate) && _.isFunction(repository['getEstimatedRowCount'])) {
+    const count = await repository['getEstimatedRowCount']();
+    if (count > SIMPLE_PAGINATION_LIMIT && repository.collection.options.loadedFromCollectionManager) {
+      await ctx.app.db.getRepository('collections').update({
+        filter: { name: repository.collection.name },
+        values: {
+          options: {
+            ...repository.collection?.options,
+            simplePaginate: true,
+          },
+        },
+      });
+      simplePaginate = true;
+    }
+  }
+
   if (simplePaginate) {
     options.limit = options.limit + 1;
 

packages/core/actions/src/constants.ts

@@ -9,3 +9,4 @@
 
 export const DEFAULT_PAGE = 1;
 export const DEFAULT_PER_PAGE = 20;
+export const SIMPLE_PAGINATION_LIMIT = 300000;

packages/core/data-source-manager/src/default-actions/list.ts

@@ -7,9 +7,10 @@
  * For more information, please refer to: https://www.nocobase.com/agreement.
  */
 
-import { Context } from '@nocobase/actions';
+import { Context, SIMPLE_PAGINATION_LIMIT } from '@nocobase/actions';
 import { assign, isValidFilter } from '@nocobase/utils';
 import { pageArgsToLimitArgs } from './utils';
+import _ from 'lodash';
 
 function totalPage(total, pageSize): number {
   return Math.ceil(total / pageSize);
@@ -42,7 +43,7 @@ async function listWithPagination(ctx: Context) {
 
   const repository = ctx.getCurrentRepository();
 
-  const { simplePaginate } = repository.collection?.options || {};
+  let { simplePaginate } = repository.collection?.options || {};
 
   const options = {
     context: ctx,
@@ -56,6 +57,26 @@ async function listWithPagination(ctx: Context) {
     }
   });
 
+  if (_.isUndefined(simplePaginate) && _.isFunction(repository['getEstimatedRowCount'])) {
+    const count = await repository.getEstimatedRowCount();
+    if (count > SIMPLE_PAGINATION_LIMIT) {
+      const resourceName = ctx.action.resourceName;
+      const collection = ctx.dataSource.collectionManager.getCollection(resourceName);
+      await ctx.app.db.getRepository('dataSourcesCollections').updateOrCreate({
+        filterKeys: ['name', 'dataSourceKey'],
+        values: {
+          name: collection.name,
+          dataSourceKey: ctx.dataSource.options.name,
+          options: {
+            ...repository.collection?.options,
+            simplePaginate: true,
+          },
+        },
+      });
+      simplePaginate = true;
+    }
+  }
+
   if (simplePaginate) {
     options.limit = options.limit + 1;
 

packages/core/database/src/repository.ts

@@ -14,6 +14,7 @@ import {
   BulkCreateOptions,
   ModelStatic,
   Op,
+  QueryTypes,
   Sequelize,
   FindAndCountOptions as SequelizeAndCountOptions,
   CountOptions as SequelizeCountOptions,
@@ -302,6 +303,83 @@ export class Repository<TModelAttributes extends {} = any, TCreationAttributes e
     });
   }
 
+  async getEstimatedRowCount() {
+    if (_.isFunction(this.collection['isSql']) && this.collection['isSql']()) {
+      return 0;
+    }
+    if (_.isFunction(this.collection['isView']) && this.collection['isView']()) {
+      return 0;
+    }
+    if (this.database.isMySQLCompatibleDialect()) {
+      await this.database.sequelize.query(`ANALYZE TABLE ${this.collection.getTableNameWithSchema()}`);
+      const results: any[] = await this.database.sequelize.query(
+        `
+        SELECT table_rows FROM information_schema.tables
+        WHERE table_schema = DATABASE()
+          AND table_name = ?
+      `,
+        { replacements: [this.collection.name], type: QueryTypes.SELECT },
+      );
+      return Number(results?.[0]?.table_rows ?? 0);
+    }
+    if (this.database.isPostgresCompatibleDialect()) {
+      await this.database.sequelize.query(`ANALYZE ${this.collection.getTableNameWithSchema()}`);
+      const results: any[] = await this.database.sequelize.query(
+        `
+        SELECT reltuples::BIGINT AS estimate
+        FROM pg_class c JOIN pg_namespace n ON c.relnamespace = n.oid
+        WHERE c.relname = ? AND n.nspname = current_schema();
+      `,
+        { replacements: [this.collection.name], type: QueryTypes.SELECT },
+      );
+      return Number(results?.[0]?.estimate ?? 0);
+    }
+
+    if (this.database.sequelize.getDialect() === 'mssql') {
+      const results: any[] = await this.database.sequelize.query(
+        `
+        SELECT SUM(row_count) AS estimate
+        FROM sys.dm_db_partition_stats
+        WHERE object_id = OBJECT_ID(?) AND (index_id = 0 OR index_id = 1)
+      `,
+        { replacements: [this.collection.name], type: QueryTypes.SELECT },
+      );
+      return Number(results?.[0]?.estimate ?? 0);
+    }
+
+    if (this.database.sequelize.getDialect() === 'oracle') {
+      const tableName = this.collection.name.toUpperCase();
+      const schemaName = (await this.getOracleSchema()).toUpperCase();
+
+      await this.database.sequelize.query(`BEGIN DBMS_STATS.GATHER_TABLE_STATS(:schema, :table); END;`, {
+        replacements: { schema: schemaName, table: tableName },
+        type: QueryTypes.RAW,
+      });
+
+      const results: any[] = await this.database.sequelize.query(
+        `
+      SELECT NUM_ROWS AS "estimate"
+      FROM ALL_TABLES
+      WHERE TABLE_NAME = :table AND OWNER = :schema
+      `,
+        {
+          replacements: { table: tableName, schema: schemaName },
+          type: QueryTypes.SELECT,
+        },
+      );
+      return Number(results?.[0]?.estimate ?? 0);
+    }
+
+    return 0;
+  }
+
+  private async getOracleSchema(): Promise<string> {
+    const [result] = await this.database.sequelize.query(`SELECT USER FROM DUAL`, {
+      type: QueryTypes.SELECT,
+    });
+    return result?.['USER'] ?? '';
+  }
+
   async aggregate(options: AggregateOptions & { optionsTransformer?: (options: any) => any }): Promise<any> {
     const { method, field } = options;
 

packages/plugins/@nocobase/plugin-data-source-main/src/server/__tests__/action.test.ts

@@ -61,4 +61,55 @@ describe('action test', () => {
       'x-uid': 'test',
     });
   });
+
+  test('enable auto simple pagination for large datasets', async () => {
+    const collectionRepository = app.db.getCollection('collections').repository;
+    await collectionRepository.create({
+      values: {
+        name: 'posts',
+      },
+    });
+    // @ts-ignore
+    await db.getRepository('collections').load();
+    await db.sync();
+    const repository = app.db.getRepository('posts');
+    vi.spyOn(repository, 'getEstimatedRowCount').mockResolvedValue(1000000);
+    await app
+      .agent()
+      .resource('posts')
+      .list({
+        fields: ['id'],
+        pageSize: 1,
+        page: 2,
+        sort: ['id'],
+      });
+
+    const posts = await collectionRepository.findOne({ where: { name: 'posts' } });
+    expect(posts.options.simplePaginate).toBeTruthy();
+
+    await collectionRepository.update({
+      filter: { name: 'posts' },
+      values: {
+        options: {
+          ...posts.options,
+          simplePaginate: false,
+        },
+      },
+    });
+    // @ts-ignore
+    await db.getRepository('collections').load();
+
+    await app
+      .agent()
+      .resource('posts')
+      .list({
+        fields: ['id'],
+        pageSize: 1,
+        page: 2,
+        sort: ['id'],
+      });
+    const updatedPosts = await collectionRepository.findOne({ where: { name: 'posts' } });
+    expect(updatedPosts.options.simplePaginate).toBeFalsy();
+    vi.restoreAllMocks();
+  });
 });