mirror of
https://github.com/nocobase/nocobase.git
synced 2026-09-24 16:02:20 +08:00
fix(plugin-backups): preserve tables during PostgreSQL restore (#9827)
* fix(plugin-backups): support preserving tables during restore * fix(plugin-backups): skip schema toc when preserving tables * feat(plugin-backups): add backup archive compression utility
This commit is contained in:
+106
@@ -58,6 +58,8 @@ vi.mock('fs/promises', async (importOriginal) => {
|
||||
return {
|
||||
...actual,
|
||||
copyFile: vi.fn(),
|
||||
writeFile: vi.fn().mockResolvedValue(undefined),
|
||||
unlink: vi.fn().mockResolvedValue(undefined),
|
||||
};
|
||||
});
|
||||
|
||||
@@ -166,6 +168,27 @@ describe('DatabaseAdapter', () => {
|
||||
expect(command).toContain(`--schema=test_version_control`);
|
||||
});
|
||||
|
||||
it('backup function should exclude owned sequences for excluded tables', async () => {
|
||||
const mockedExec = cp.exec as unknown as Mock;
|
||||
mockedExec.mockClear();
|
||||
mockedExec.mockImplementation((command, _options, callback) => {
|
||||
if (command.includes('pg_depend')) {
|
||||
callback(null, 'public.logs_id_seq\n', '');
|
||||
return;
|
||||
}
|
||||
callback(null, 'done', '');
|
||||
});
|
||||
const adapter = getDBAdapter(dbOpts);
|
||||
const dir = os.tmpdir();
|
||||
await adapter.backup({
|
||||
dir,
|
||||
excludeTables: ['logs'],
|
||||
});
|
||||
const command = mockedExec.mock.lastCall[0];
|
||||
expect(command).toContain(`-T '"logs"'`);
|
||||
expect(command).toContain(`-T '"public"."logs_id_seq"'`);
|
||||
});
|
||||
|
||||
it('restore function', async () => {
|
||||
const mockedExec = cp.exec as unknown as Mock;
|
||||
mockedExec.mockImplementation((_command, _options, callback) => {
|
||||
@@ -195,6 +218,51 @@ describe('DatabaseAdapter', () => {
|
||||
expect(commands.some((command) => command.includes('DROP TRIGGER IF EXISTS'))).toBe(false);
|
||||
});
|
||||
|
||||
it('restore function should preserve tables and drop views when restoreMode is preserveTables', async () => {
|
||||
const mockedExec = cp.exec as unknown as Mock;
|
||||
const mockedWriteFile = fsPromises.writeFile as Mock;
|
||||
const mockedUnlink = fsPromises.unlink as Mock;
|
||||
mockedExec.mockClear();
|
||||
mockedWriteFile.mockClear();
|
||||
mockedUnlink.mockClear();
|
||||
mockedExec.mockImplementation((command, _options, callback) => {
|
||||
if (String(command).includes('--list')) {
|
||||
callback(null, {
|
||||
stdout: [
|
||||
'; Archive created at 2026-06-18 00:00:00',
|
||||
'123; 2615 2200 SCHEMA - public test',
|
||||
'124; 0 0 COMMENT - SCHEMA public test',
|
||||
'125; 0 0 ACL - SCHEMA public test',
|
||||
'126; 1259 2201 TABLE public users test',
|
||||
].join('\n'),
|
||||
});
|
||||
return;
|
||||
}
|
||||
callback(null, { stdout: 'done' });
|
||||
});
|
||||
const adapter = getDBAdapter(dbOpts);
|
||||
const filePath = os.tmpdir();
|
||||
await adapter.restore({ filePath, restoreMode: 'preserveTables' });
|
||||
|
||||
const commands = mockedExec.mock.calls.map(([command]) => command);
|
||||
expect(commands).toHaveLength(3);
|
||||
expect(commands[0]).toContain('DROP VIEW IF EXISTS');
|
||||
expect(commands[0]).toContain('DROP MATERIALIZED VIEW IF EXISTS');
|
||||
expect(commands[0]).not.toContain('DROP TABLE IF EXISTS');
|
||||
expect(commands[0]).not.toContain('DROP SEQUENCE IF EXISTS');
|
||||
expect(commands[0]).not.toContain('DROP TRIGGER IF EXISTS');
|
||||
expect(commands[1]).toContain('pg_restore');
|
||||
expect(commands[1]).toContain('--list');
|
||||
expect(commands[2]).toContain('pg_restore');
|
||||
expect(commands[2]).toContain('-L');
|
||||
expect(mockedWriteFile).toHaveBeenCalledTimes(1);
|
||||
expect(mockedWriteFile.mock.calls[0][1]).not.toContain('SCHEMA - public');
|
||||
expect(mockedWriteFile.mock.calls[0][1]).toContain('COMMENT - SCHEMA public');
|
||||
expect(mockedWriteFile.mock.calls[0][1]).toContain('ACL - SCHEMA public');
|
||||
expect(mockedWriteFile.mock.calls[0][1]).toContain('TABLE public users');
|
||||
expect(mockedUnlink).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('restore function should sync collection schema metadata when schema is renamed', async () => {
|
||||
const mockedExec = cp.exec as unknown as Mock;
|
||||
mockedExec.mockClear();
|
||||
@@ -456,6 +524,25 @@ describe('DatabaseAdapter', () => {
|
||||
expect(commands[0]).toContain(` < ${filePath}`);
|
||||
expect(commands.some((command) => command.includes('drop_all_tables_and_triggers'))).toBe(false);
|
||||
});
|
||||
|
||||
it('restore function should preserve tables and drop views when restoreMode is preserveTables', async () => {
|
||||
const mockedExec = cp.exec as unknown as Mock;
|
||||
mockedExec.mockClear();
|
||||
mockedExec.mockImplementation((_command, _options, callback) => {
|
||||
callback(null, { stdout: 'done' });
|
||||
});
|
||||
const adapter = getDBAdapter(dbOpts);
|
||||
const filePath = os.tmpdir();
|
||||
await adapter.restore({ filePath, restoreMode: 'preserveTables' });
|
||||
|
||||
const commands = mockedExec.mock.calls.map(([command]) => command);
|
||||
expect(commands).toHaveLength(2);
|
||||
expect(commands[0]).toContain('drop_all_views');
|
||||
expect(commands[0]).toContain('DROP VIEW IF EXISTS');
|
||||
expect(commands[0]).not.toContain('drop_all_tables_and_triggers');
|
||||
expect(commands[1]).toContain('mysql');
|
||||
expect(commands[1]).toContain(` < ${filePath}`);
|
||||
});
|
||||
});
|
||||
|
||||
describe('MariaDBAdapter', () => {
|
||||
@@ -559,6 +646,25 @@ describe('DatabaseAdapter', () => {
|
||||
expect(commands[0]).toContain(` < ${filePath}`);
|
||||
expect(commands.some((command) => command.includes('drop_all_tables_and_triggers'))).toBe(false);
|
||||
});
|
||||
|
||||
it('restore function should preserve tables and drop views when restoreMode is preserveTables', async () => {
|
||||
const mockedExec = cp.exec as unknown as Mock;
|
||||
mockedExec.mockClear();
|
||||
mockedExec.mockImplementation((_command, _options, callback) => {
|
||||
callback(null, { stdout: 'done' });
|
||||
});
|
||||
const adapter = getDBAdapter(dbOpts);
|
||||
const filePath = os.tmpdir();
|
||||
await adapter.restore({ filePath, restoreMode: 'preserveTables' });
|
||||
|
||||
const commands = mockedExec.mock.calls.map(([command]) => command);
|
||||
expect(commands).toHaveLength(2);
|
||||
expect(commands[0]).toContain('drop_all_views');
|
||||
expect(commands[0]).toContain('DROP VIEW IF EXISTS');
|
||||
expect(commands[0]).not.toContain('drop_all_tables_and_triggers');
|
||||
expect(commands[1]).toContain('mysql');
|
||||
expect(commands[1]).toContain(` < ${filePath}`);
|
||||
});
|
||||
});
|
||||
|
||||
describe('SQLiteAdapter', () => {
|
||||
|
||||
@@ -17,6 +17,7 @@ import {
|
||||
toMajorVersion,
|
||||
EscapeQuoteTransform,
|
||||
Extractor,
|
||||
compressDirectoryToBackupArchive,
|
||||
resolvePathWithinBase,
|
||||
} from '../utils';
|
||||
import { Readable, pipeline } from 'stream';
|
||||
@@ -120,6 +121,33 @@ describe('Extractor', () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe('compressDirectoryToBackupArchive', () => {
|
||||
const pipelineAsync = promisify(pipeline);
|
||||
|
||||
it('should compress a directory that Extractor can read', async () => {
|
||||
const sourceDir = await fs.mkdtemp(path.join(os.tmpdir(), 'backups-compress-source-'));
|
||||
const extractDir = await fs.mkdtemp(path.join(os.tmpdir(), 'backups-compress-extract-'));
|
||||
const backupFilePath = path.join(os.tmpdir(), `backups-compress-${Date.now()}.nbdata`);
|
||||
|
||||
try {
|
||||
await fs.mkdir(path.join(sourceDir, 'data'), { recursive: true });
|
||||
await fs.writeFile(path.join(sourceDir, 'data', 'dump.sql'), 'database dump');
|
||||
|
||||
await compressDirectoryToBackupArchive(sourceDir, backupFilePath);
|
||||
|
||||
const archiveBuffer = await fs.readFile(backupFilePath);
|
||||
const extractor = new Extractor({ path: extractDir });
|
||||
await pipelineAsync(Readable.from([archiveBuffer]), extractor);
|
||||
|
||||
await expect(fs.readFile(path.join(extractDir, 'data', 'dump.sql'), 'utf8')).resolves.toBe('database dump');
|
||||
} finally {
|
||||
await fs.rm(sourceDir, { recursive: true, force: true });
|
||||
await fs.rm(extractDir, { recursive: true, force: true });
|
||||
await fs.rm(backupFilePath, { force: true });
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('EscapeQuoteTransform', () => {
|
||||
const pipelineAsync = promisify(pipeline);
|
||||
|
||||
|
||||
@@ -25,6 +25,9 @@ const STREAM_BUFFER_SIZE = 2 * 1024 * 1024; // 2MB buffer for better IO performa
|
||||
export type DBBackupOptions = {
|
||||
dir: string;
|
||||
skipFdw?: boolean;
|
||||
/**
|
||||
* @deprecated Prefer excludeTables. includeTables may miss dependent database objects.
|
||||
*/
|
||||
includeTables?: string[];
|
||||
excludeTables?: string[];
|
||||
};
|
||||
@@ -33,6 +36,7 @@ export type DBRestoreOptions = {
|
||||
filePath: string;
|
||||
schema?: string;
|
||||
skipDropAllTables?: boolean;
|
||||
restoreMode?: 'preserveTables';
|
||||
toolchain?: DBBackupToolchain;
|
||||
};
|
||||
|
||||
@@ -49,8 +53,17 @@ export interface DBAdapter {
|
||||
|
||||
const run = async (command: string, envVars: NodeJS.ProcessEnv = {}) => {
|
||||
try {
|
||||
const { stdout } = await exec(command, { env: { ...process.env, ...envVars } });
|
||||
return stdout;
|
||||
const result = (await exec(command, { env: { ...process.env, ...envVars } })) as unknown;
|
||||
if (typeof result === 'string') {
|
||||
return result;
|
||||
}
|
||||
|
||||
if (result && typeof result === 'object' && 'stdout' in result) {
|
||||
const { stdout } = result as { stdout?: unknown };
|
||||
return typeof stdout === 'string' ? stdout : String(stdout ?? '');
|
||||
}
|
||||
|
||||
return '';
|
||||
} catch (error) {
|
||||
throw new Error(`${error.message}`);
|
||||
}
|
||||
@@ -66,6 +79,21 @@ const escapeStringLiteral = (value: string) => String(value).replace(/'/g, "''")
|
||||
const quotePgIdentifier = (value: string) => `"${String(value).replace(/"/g, '""')}"`;
|
||||
const quoteShellArg = (value: string) => `'${String(value).replace(/'/g, "'\\''")}'`;
|
||||
const quotePgTablePattern = (table: string) => quoteShellArg(String(table).split('.').map(quotePgIdentifier).join('.'));
|
||||
const isPgRestoreSchemaTocEntry = (line: string) => /^\d+;\s+\d+\s+\d+\s+SCHEMA\s+-\s+/.test(line);
|
||||
const parsePgTableReference = (table: string, defaultSchema?: string) => {
|
||||
const parts = String(table).split('.');
|
||||
if (parts.length > 1) {
|
||||
return {
|
||||
schema: parts[0],
|
||||
table: parts.slice(1).join('.'),
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
schema: defaultSchema,
|
||||
table: parts[0],
|
||||
};
|
||||
};
|
||||
const qualifyPgTablePattern = (table: string, schema?: string) => {
|
||||
const tablePattern = String(table);
|
||||
if (!schema || tablePattern.includes('.')) {
|
||||
@@ -165,6 +193,7 @@ class MySQLAdapter extends BaseDBAdapter {
|
||||
'--set-gtid-purged=OFF',
|
||||
'--routines',
|
||||
'--triggers',
|
||||
'--events',
|
||||
...(version && version > 7 ? ['--column-statistics=0'] : []),
|
||||
database,
|
||||
];
|
||||
@@ -292,10 +321,51 @@ class MySQLAdapter extends BaseDBAdapter {
|
||||
}
|
||||
}
|
||||
|
||||
async restore({ filePath, skipDropAllTables = false }: DBRestoreOptions): Promise<void> {
|
||||
async restore({ filePath, skipDropAllTables = false, restoreMode }: DBRestoreOptions): Promise<void> {
|
||||
const { username, host, port, database, password } = this.dbOpts;
|
||||
|
||||
if (!skipDropAllTables) {
|
||||
if (restoreMode === 'preserveTables') {
|
||||
const dropViewsCommand = `mysql -u ${username} -h ${host} ${
|
||||
port ? `-P ${port}` : ''
|
||||
} --protocol=tcp -D ${database} -e "
|
||||
DELIMITER $$
|
||||
DROP PROCEDURE IF EXISTS drop_all_views$$
|
||||
CREATE PROCEDURE drop_all_views()
|
||||
BEGIN
|
||||
DECLARE _done INT DEFAULT FALSE;
|
||||
DECLARE _viewName VARCHAR(255);
|
||||
|
||||
DECLARE _cursor CURSOR FOR
|
||||
SELECT table_name
|
||||
FROM information_schema.VIEWS
|
||||
WHERE table_schema = SCHEMA();
|
||||
|
||||
DECLARE CONTINUE HANDLER FOR NOT FOUND SET _done = TRUE;
|
||||
|
||||
OPEN _cursor;
|
||||
|
||||
REPEAT
|
||||
FETCH _cursor INTO _viewName;
|
||||
|
||||
IF NOT _done THEN
|
||||
SET @stmt_sql = CONCAT('DROP VIEW IF EXISTS ', _viewName);
|
||||
PREPARE stmt FROM @stmt_sql;
|
||||
EXECUTE stmt;
|
||||
DEALLOCATE PREPARE stmt;
|
||||
END IF;
|
||||
|
||||
UNTIL _done END REPEAT;
|
||||
|
||||
CLOSE _cursor;
|
||||
END$$
|
||||
|
||||
CALL drop_all_views()$$
|
||||
DROP PROCEDURE drop_all_views$$
|
||||
DELIMITER ;
|
||||
"`;
|
||||
|
||||
await run(dropViewsCommand, { MYSQL_PWD: password });
|
||||
} else if (!skipDropAllTables) {
|
||||
const dropDataCommand = `mysql -u ${username} -h ${host} ${
|
||||
port ? `-P ${port}` : ''
|
||||
} --protocol=tcp -D ${database} -e "
|
||||
@@ -443,18 +513,20 @@ class PostgresAdapter extends BaseDBAdapter {
|
||||
const filePath = `${dir}/data`;
|
||||
const backupSchema = this.getBackupSchema();
|
||||
const schemaOption = backupSchema ? `--schema=${backupSchema}` : '';
|
||||
const expandedExcludeTables = Array.isArray(excludeTables)
|
||||
? [...new Set([...excludeTables, ...(await this.getOwnedSequenceTables(excludeTables, backupSchema))])]
|
||||
: [];
|
||||
const includeOption =
|
||||
Array.isArray(includeTables) && includeTables.length
|
||||
? includeTables
|
||||
.map((table) => `-t ${quotePgTablePattern(qualifyPgTablePattern(table, backupSchema))}`)
|
||||
.join(' ')
|
||||
: '';
|
||||
const excludeOption =
|
||||
Array.isArray(excludeTables) && excludeTables.length
|
||||
? excludeTables
|
||||
.map((table) => `-T ${quotePgTablePattern(qualifyPgTablePattern(table, backupSchema))}`)
|
||||
.join(' ')
|
||||
: '';
|
||||
const excludeOption = expandedExcludeTables.length
|
||||
? expandedExcludeTables
|
||||
.map((table) => `-T ${quotePgTablePattern(qualifyPgTablePattern(table, backupSchema))}`)
|
||||
.join(' ')
|
||||
: '';
|
||||
// set the password in the environment variable, so we don't need to pass it in the command
|
||||
const command = `${this.getBackupCommandName()} ${includeOption} ${excludeOption} -U ${username} -h ${host} ${
|
||||
port ? `-p ${port}` : ''
|
||||
@@ -462,10 +534,56 @@ class PostgresAdapter extends BaseDBAdapter {
|
||||
await run(command, this.getPasswordEnvVars(password));
|
||||
}
|
||||
|
||||
protected async getOwnedSequenceTables(
|
||||
excludeTables: string[] | undefined,
|
||||
backupSchema?: string,
|
||||
): Promise<string[]> {
|
||||
if (!Array.isArray(excludeTables) || !excludeTables.length) {
|
||||
return [];
|
||||
}
|
||||
|
||||
const { username, host, port, database, password } = this.dbOpts;
|
||||
const tableRefs = excludeTables
|
||||
.map((table) => parsePgTableReference(table, backupSchema))
|
||||
.filter((ref) => ref.table);
|
||||
if (!tableRefs.length) {
|
||||
return [];
|
||||
}
|
||||
|
||||
const values = tableRefs
|
||||
.map((ref) => {
|
||||
const schemaValue = ref.schema == null ? 'NULL' : `'${escapeStringLiteral(ref.schema)}'`;
|
||||
return `(${schemaValue}, '${escapeStringLiteral(ref.table)}')`;
|
||||
})
|
||||
.join(',');
|
||||
const query = `
|
||||
WITH excluded(schema_name, table_name) AS (VALUES ${values})
|
||||
SELECT seq_ns.nspname || '.' || seq.relname
|
||||
FROM excluded
|
||||
JOIN pg_class tbl ON tbl.relname = excluded.table_name
|
||||
JOIN pg_namespace tbl_ns ON tbl_ns.oid = tbl.relnamespace
|
||||
JOIN pg_depend dep ON dep.refobjid = tbl.oid
|
||||
JOIN pg_class seq ON seq.oid = dep.objid AND seq.relkind = 'S'
|
||||
JOIN pg_namespace seq_ns ON seq_ns.oid = seq.relnamespace
|
||||
WHERE tbl.relkind IN ('r', 'p')
|
||||
AND dep.deptype IN ('a', 'i')
|
||||
AND (excluded.schema_name IS NULL OR tbl_ns.nspname = excluded.schema_name)
|
||||
`;
|
||||
const command = `${this.getSqlCommandName()} -U ${username} -h ${host} ${
|
||||
port ? `-p ${port}` : ''
|
||||
} -d ${database} -At -c ${quoteShellArg(query)}`;
|
||||
const output = String(await run(command, this.getPasswordEnvVars(password)));
|
||||
return output
|
||||
.split('\n')
|
||||
.map((line) => line.trim())
|
||||
.filter(Boolean);
|
||||
}
|
||||
|
||||
async restore({
|
||||
filePath,
|
||||
schema,
|
||||
skipDropAllTables = false,
|
||||
restoreMode,
|
||||
toolchain = this.backupToolchain,
|
||||
}: DBRestoreOptions): Promise<void> {
|
||||
const { username, host, port, database, password } = this.dbOpts;
|
||||
@@ -487,7 +605,31 @@ class PostgresAdapter extends BaseDBAdapter {
|
||||
const relnamespaceCondition = schemaOption
|
||||
? `WHERE relnamespace = '${schemaOption}'::regnamespace`
|
||||
: `WHERE tgrelid IN (SELECT oid FROM pg_class WHERE relnamespace NOT IN (SELECT oid FROM pg_catalog.pg_namespace WHERE nspname IN ('pg_catalog', 'information_schema')))`;
|
||||
if (!skipDropAllTables) {
|
||||
if (restoreMode === 'preserveTables') {
|
||||
const dropViewsCommand = `${this.getSqlCommandName(toolchain)} -U ${username} -h ${host} ${
|
||||
port ? `-p ${port}` : ''
|
||||
} -d ${database} -c "
|
||||
DO ${D$$} DECLARE r RECORD;
|
||||
BEGIN
|
||||
FOR r IN (
|
||||
SELECT schemaname, viewname, false AS materialized FROM pg_views ${schemaNameCondition}
|
||||
UNION ALL
|
||||
SELECT schemaname, matviewname AS viewname, true AS materialized FROM pg_matviews ${schemaNameCondition}
|
||||
) LOOP
|
||||
BEGIN
|
||||
IF r.materialized THEN
|
||||
EXECUTE 'DROP MATERIALIZED VIEW IF EXISTS ' || quote_ident(r.schemaname) || '.' || quote_ident(r.viewname) || ' CASCADE';
|
||||
ELSE
|
||||
EXECUTE 'DROP VIEW IF EXISTS ' || quote_ident(r.schemaname) || '.' || quote_ident(r.viewname) || ' CASCADE';
|
||||
END IF;
|
||||
EXCEPTION
|
||||
WHEN OTHERS THEN
|
||||
END;
|
||||
END LOOP;
|
||||
END ${D$$};"`.replace(/\n/g, ' ');
|
||||
|
||||
await run(dropViewsCommand, this.getPasswordEnvVars(password, toolchain));
|
||||
} else if (!skipDropAllTables) {
|
||||
const dropDataCommand = `${this.getSqlCommandName(toolchain)} -U ${username} -h ${host} ${
|
||||
port ? `-p ${port}` : ''
|
||||
} -d ${database} -c "
|
||||
@@ -495,7 +637,7 @@ class PostgresAdapter extends BaseDBAdapter {
|
||||
BEGIN
|
||||
FOR r IN (SELECT viewname,schemaname FROM pg_views ${schemaNameCondition}) LOOP
|
||||
BEGIN
|
||||
EXECUTE 'DROP VIEW IF EXISTS ' || quote_ident(r.schemaname) || '.' || quote_ident(r.tablename) || ' CASCADE';
|
||||
EXECUTE 'DROP VIEW IF EXISTS ' || quote_ident(r.schemaname) || '.' || quote_ident(r.viewname) || ' CASCADE';
|
||||
EXCEPTION
|
||||
WHEN OTHERS THEN
|
||||
END;
|
||||
@@ -511,7 +653,7 @@ class PostgresAdapter extends BaseDBAdapter {
|
||||
|
||||
FOR r IN (SELECT sequencename,schemaname FROM pg_sequences ${schemaNameCondition}) LOOP
|
||||
BEGIN
|
||||
EXECUTE 'DROP SEQUENCE IF EXISTS ' || quote_ident(r.schemaname) || '.' || quote_ident(r.tablename) || ' CASCADE';
|
||||
EXECUTE 'DROP SEQUENCE IF EXISTS ' || quote_ident(r.schemaname) || '.' || quote_ident(r.sequencename) || ' CASCADE';
|
||||
EXCEPTION
|
||||
WHEN OTHERS THEN
|
||||
END;
|
||||
@@ -537,10 +679,27 @@ class PostgresAdapter extends BaseDBAdapter {
|
||||
|
||||
if (schema === schemaOption || !schemaOption) {
|
||||
// current schema is the same as the backup schema
|
||||
// In preserveTables mode, excluded tables must stay in the target schema.
|
||||
// pg_restore --clean would otherwise try to drop/create the schema itself
|
||||
// because the schema object is part of the archive TOC, and that fails as
|
||||
// long as preserved tables still depend on the schema. Use a filtered TOC
|
||||
// list to skip only the SCHEMA entry while keeping --clean for tables,
|
||||
// views, sequences, indexes, constraints, and other archived objects.
|
||||
// Risk: the target schema must already exist, and the filter must stay
|
||||
// narrow enough to avoid removing COMMENT/ACL/TABLE entries that mention
|
||||
// SCHEMA.
|
||||
const restoreList =
|
||||
restoreMode === 'preserveTables' ? await this.createRestoreListWithoutSchema(filePath, toolchain) : undefined;
|
||||
const pgRestoreCommand = `${this.getRestoreCommandName(toolchain)} -U ${username} -h ${host} ${
|
||||
port ? `-p ${port}` : ''
|
||||
} -d ${database} --clean --if-exists --no-owner -j ${j} ${filePath}`;
|
||||
await run(pgRestoreCommand, this.getPasswordEnvVars(password, toolchain));
|
||||
} -d ${database} --clean --if-exists --no-owner -j ${j} ${restoreList?.option ?? ''} ${filePath}`;
|
||||
try {
|
||||
await run(pgRestoreCommand, this.getPasswordEnvVars(password, toolchain));
|
||||
} finally {
|
||||
if (restoreList) {
|
||||
await fsPromises.unlink(restoreList.filePath).catch(() => {});
|
||||
}
|
||||
}
|
||||
} else {
|
||||
const srcSchema = schema || 'public';
|
||||
const pgRestoreCommand = this.buildSchemaRestoreCommand(srcSchema, schemaOption, filePath, j, toolchain);
|
||||
@@ -548,6 +707,26 @@ class PostgresAdapter extends BaseDBAdapter {
|
||||
}
|
||||
}
|
||||
|
||||
protected async createRestoreListWithoutSchema(
|
||||
filePath: string,
|
||||
toolchain: DBBackupToolchain = this.backupToolchain,
|
||||
): Promise<{ option: string; filePath: string }> {
|
||||
const listOutput = String(await run(`${this.getRestoreCommandName(toolchain)} --list ${quoteShellArg(filePath)}`));
|
||||
const filteredList = listOutput
|
||||
.split('\n')
|
||||
.filter((line) => !isPgRestoreSchemaTocEntry(line))
|
||||
.join('\n');
|
||||
const listFilePath = path.join(
|
||||
os.tmpdir(),
|
||||
`nocobase-pg-restore-${process.pid}-${Date.now()}-${Math.random().toString(16).slice(2)}.list`,
|
||||
);
|
||||
await fsPromises.writeFile(listFilePath, filteredList);
|
||||
return {
|
||||
option: `-L ${quoteShellArg(listFilePath)}`,
|
||||
filePath: listFilePath,
|
||||
};
|
||||
}
|
||||
|
||||
protected buildSchemaRestoreCommand(
|
||||
srcSchema: string,
|
||||
_targetSchema: string,
|
||||
|
||||
@@ -10,5 +10,5 @@
|
||||
export { BackupManager } from './managers/backup';
|
||||
export type { BackupSettings, BackupCreator, BackupTaskResult } from './managers/backup';
|
||||
export { RestoreManager, RestoreOptions } from './managers/restore';
|
||||
export { BACKUP_EXTENSION } from './utils';
|
||||
export { BACKUP_EXTENSION, METADATA_EXTENSION, Extractor, compressDirectoryToBackupArchive } from './utils';
|
||||
export { default } from './plugin';
|
||||
|
||||
@@ -33,7 +33,7 @@ import {
|
||||
resolvePathWithinBase,
|
||||
} from '../utils';
|
||||
|
||||
const BACKUP_METADATA_VERSION = 1;
|
||||
const BACKUP_METADATA_VERSION = 2;
|
||||
|
||||
export interface BackupSettings {
|
||||
storageId?: string;
|
||||
@@ -42,10 +42,14 @@ export interface BackupSettings {
|
||||
keep?: number;
|
||||
scheduled: boolean;
|
||||
cron: string;
|
||||
/**
|
||||
* @deprecated Prefer excludeTables. includeTables may miss dependent database objects.
|
||||
*/
|
||||
includeTables?: string[];
|
||||
excludeTables?: string[];
|
||||
description?: string;
|
||||
createdBy?: BackupCreator;
|
||||
metadata?: Record<string, unknown>;
|
||||
}
|
||||
|
||||
export interface BackupFile {
|
||||
@@ -239,6 +243,7 @@ export class BackupManager {
|
||||
});
|
||||
|
||||
const metadata = {
|
||||
...(opts.metadata ?? {}),
|
||||
metadataVersion: BACKUP_METADATA_VERSION,
|
||||
enableFilesBackup: opts.enableFilesBackup,
|
||||
version: await this.app.version.get(),
|
||||
|
||||
@@ -30,6 +30,8 @@ import {
|
||||
toMajorVersion,
|
||||
} from '../utils';
|
||||
interface Metadata {
|
||||
metadataVersion?: number;
|
||||
partialBackupMode?: string;
|
||||
version: string;
|
||||
database: {
|
||||
dialect: string;
|
||||
@@ -51,6 +53,7 @@ interface Metadata {
|
||||
export interface RestoreOptions {
|
||||
forceSchemaRestore?: boolean;
|
||||
skipDropAllTables?: boolean;
|
||||
restoreMode?: 'preserveTables';
|
||||
}
|
||||
|
||||
const RESTORE_STEPS = {
|
||||
@@ -192,6 +195,7 @@ export class RestoreManager {
|
||||
filePath: path.join(extractedDir, dbFile),
|
||||
schema: metadata.database.schema,
|
||||
skipDropAllTables: options?.skipDropAllTables === true,
|
||||
restoreMode: options?.restoreMode,
|
||||
toolchain: this.#resolveRestoreToolchain(metadata),
|
||||
});
|
||||
this.ctx.logger.info('Database restored successfully', { module: BACKUPS });
|
||||
@@ -499,6 +503,7 @@ export class RestoreManager {
|
||||
filePath: path.join(extractedDir, dbFile),
|
||||
schema: metadata.database.schema,
|
||||
skipDropAllTables: options?.skipDropAllTables === true,
|
||||
restoreMode: options?.restoreMode,
|
||||
toolchain: this.#resolveRestoreToolchain(metadata),
|
||||
});
|
||||
this.ctx.logger.info('Database restored successfully', { module: BACKUPS });
|
||||
|
||||
@@ -18,6 +18,7 @@ export const RESTORE_TASKS_CACHE_NAME = 'restore-task-results';
|
||||
export const RESTORE_TASKS_CACHE_TTL = 24 * 60 * 60 * 1000;
|
||||
|
||||
import { Writable, Transform, TransformCallback } from 'stream';
|
||||
import archiver from 'archiver';
|
||||
import yauzl from 'yauzl';
|
||||
import fs from 'fs-extra';
|
||||
import path from 'path';
|
||||
@@ -181,6 +182,33 @@ export class Extractor extends Writable {
|
||||
}
|
||||
}
|
||||
|
||||
export async function compressDirectoryToBackupArchive(sourceDir: string, backupFilePath: string): Promise<void> {
|
||||
const archive = archiver('zip', { zlib: { level: 9 } });
|
||||
const output = fs.createWriteStream(backupFilePath);
|
||||
|
||||
const outputClosed = new Promise<void>((resolve, reject) => {
|
||||
output.on('close', resolve);
|
||||
output.on('error', reject);
|
||||
archive.on('error', reject);
|
||||
archive.on('warning', (error) => {
|
||||
if (error.code === 'ENOENT') {
|
||||
return;
|
||||
}
|
||||
reject(error);
|
||||
});
|
||||
});
|
||||
|
||||
try {
|
||||
archive.pipe(output);
|
||||
archive.directory(sourceDir, false);
|
||||
await archive.finalize();
|
||||
await outputClosed;
|
||||
} catch (error) {
|
||||
await fs.remove(backupFilePath);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
export function humanFileSize(bytes: number, si = false, dp = 1): string {
|
||||
const thresh = si ? 1000 : 1024;
|
||||
const units = si
|
||||
|
||||
Reference in New Issue
Block a user