diff --git a/packages/ui/certd-server/src/plugins/plugin-aws/libs/aws-client.test.ts b/packages/ui/certd-server/src/plugins/plugin-aws/libs/aws-client.test.ts index 46f4f8157..a211fbd98 100644 --- a/packages/ui/certd-server/src/plugins/plugin-aws/libs/aws-client.test.ts +++ b/packages/ui/certd-server/src/plugins/plugin-aws/libs/aws-client.test.ts @@ -35,7 +35,7 @@ async function createAwsClient(records: any[]) { ChangeResourceRecordSetsCommand, }), } as any, - logger: { info() {}, error() {} } as any, + logger: { info() {}, warn() {}, error() {} } as any, region: "us-east-1", }); return { client, sentChanges }; @@ -92,3 +92,124 @@ describe("AwsClient.route53ChangeRecord", () => { assert.deepEqual(change.ResourceRecordSet.ResourceRecords, [{ Value: '"wildcard-token"' }]); }); }); + +function throttle(name: string, extra: Record = {}) { + const err: any = new Error(name); + err.name = name; + return Object.assign(err, extra); +} + +describe("AwsClient.withRetry", () => { + it("retries a throttled call and then resolves", async () => { + const { client } = await createAwsClient([]); + let calls = 0; + const result = await client.withRetry(async () => { + calls++; + if (calls < 3) { + throw throttle("ThrottlingException"); + } + return "ok"; + }); + assert.equal(result, "ok"); + assert.equal(calls, 3); + }); + + it("retries Route53 PriorRequestNotComplete", async () => { + const { client } = await createAwsClient([]); + let calls = 0; + const result = await client.withRetry(async () => { + calls++; + if (calls < 2) { + throw throttle("PriorRequestNotComplete"); + } + return "done"; + }); + assert.equal(result, "done"); + assert.equal(calls, 2); + }); + + it("retries on HTTP 429 without a known error code", async () => { + const { client } = await createAwsClient([]); + let calls = 0; + await client.withRetry(async () => { + calls++; + if (calls < 2) { + const err: any = new Error("too many requests"); + err.$metadata = { httpStatusCode: 429 }; + throw err; + } + return null; + }); + assert.equal(calls, 2); + }); + + it("retries transient HTTP 5xx", async () => { + const { client } = await createAwsClient([]); + let calls = 0; + await client.withRetry(async () => { + calls++; + if (calls < 2) { + const err: any = new Error("service unavailable"); + err.$metadata = { httpStatusCode: 503 }; + throw err; + } + return null; + }); + assert.equal(calls, 2); + }); + + it("retries when the SDK marks the error $retryable", async () => { + const { client } = await createAwsClient([]); + let calls = 0; + await client.withRetry(async () => { + calls++; + if (calls < 2) { + const err: any = new Error("internal failure"); + err.name = "InternalFailure"; + err.$retryable = {}; + throw err; + } + return null; + }); + assert.equal(calls, 2); + }); + + it("retries connection-level errors", async () => { + const { client } = await createAwsClient([]); + let calls = 0; + await client.withRetry(async () => { + calls++; + if (calls < 2) { + throw throttle("Error", { code: "ECONNRESET" }); + } + return null; + }); + assert.equal(calls, 2); + }); + + it("does not retry non-throttling errors", async () => { + const { client } = await createAwsClient([]); + let calls = 0; + await assert.rejects( + client.withRetry(async () => { + calls++; + throw throttle("AccessDeniedException"); + }), + /AccessDeniedException/ + ); + assert.equal(calls, 1); + }); + + it("stops after maxAttempts and rethrows the throttling error", async () => { + const { client } = await createAwsClient([]); + let calls = 0; + await assert.rejects( + client.withRetry(async () => { + calls++; + throw throttle("Throttling", { message: "Rate exceeded" }); + }, 3), + /Throttling|Rate exceeded/ + ); + assert.equal(calls, 3); + }); +}); diff --git a/packages/ui/certd-server/src/plugins/plugin-aws/libs/aws-client.ts b/packages/ui/certd-server/src/plugins/plugin-aws/libs/aws-client.ts index a0ec768fb..d49659954 100644 --- a/packages/ui/certd-server/src/plugins/plugin-aws/libs/aws-client.ts +++ b/packages/ui/certd-server/src/plugins/plugin-aws/libs/aws-client.ts @@ -28,6 +28,8 @@ export class AwsClient { accessKeyId: this.access.accessKeyId, // 从环境变量中读取 secretAccessKey: this.access.secretAccessKey, }, + // Disable the SDK's own retries; withRetry/doRequest is the single retry authority. + maxAttempts: 1, }); // Split the full PEM chain: first block is the leaf cert, the rest is the intermediate chain @@ -57,6 +59,7 @@ export class AwsClient { accessKeyId: this.access.accessKeyId, // 从环境变量中读取 secretAccessKey: this.access.secretAccessKey, }, + maxAttempts: 1, }); const command = new GetCallerIdentityCommand({}); @@ -75,6 +78,7 @@ export class AwsClient { accessKeyId: this.access.accessKeyId, // 从环境变量中读取 secretAccessKey: this.access.secretAccessKey, }, + maxAttempts: 1, }); } @@ -202,10 +206,107 @@ export class AwsClient { async doRequest(call: () => Promise): Promise { try { - return await call(); - } catch (err) { - this.logger.error(`调用接口失败:${err.Error?.Message || err.message},requestId:${err.requestId}`); + return await this.withRetry(call); + } catch (err: any) { + this.logger.error(`请求失败:${err.Error?.Message || err.message},requestId:${err.requestId}`); throw err; } } + + /** + * Retries an AWS SDK call with exponential backoff + jitter while the error is + * retryable. See {@link AwsClient.isRetryableAwsError}. + * + * `T` defaults to `any` so that call sites passing an untyped SDK client + * (`@aws-sdk/*` is a lazyDependency and has no types at build time) don't + * collapse the result to `unknown`. + */ + async withRetry(call: () => Promise, maxAttempts = 5, baseDelayMs = 2000): Promise { + for (let attempt = 1; attempt <= maxAttempts; attempt++) { + try { + return await call(); + } catch (err: any) { + if (attempt >= maxAttempts || !AwsClient.isRetryableAwsError(err)) { + throw err; + } + const code = err?.name || err?.Code || err?.code || err?.$metadata?.httpStatusCode || "unknown"; + const ceiling = baseDelayMs * Math.pow(2, attempt - 1); // 2s, 4s, 8s, 16s … + const delay = Math.round(ceiling / 2 + Math.random() * (ceiling / 2)); // equal jitter: ~1-2s, ~2-4s, ~4-8s, ~8-16s + this.logger.warn(`AWS request failed (${code}), attempt ${attempt}/${maxAttempts}, retrying in ${delay}ms…`); + await utils.sleep(delay); + } + } + throw new Error("Unreachable"); + } + + /** + * Detects AWS errors that are safe to retry. Because we disable the SDK's own + * retries (maxAttempts: 1 on every client), this is the single source of truth + * and covers what the SDK's `standard` strategy would have retried: + * - throttling / rate-limit codes across ACM / STS / CloudFront / Route53, + * including Route53's PriorRequestNotComplete concurrent-change limit; + * - SDK v3 retry hints (`$retryable`); + * - transient HTTP 429 and 5xx; + * - connection-level errors (ECONNRESET, ETIMEDOUT, EAI_AGAIN, …). + */ + static isRetryableAwsError(err: any): boolean { + if (!err) { + return false; + } + const retryableCodes = new Set([ + "TooManyRequestsException", + "ThrottlingException", + "ThrottledException", + "RequestLimitExceeded", + "RequestThrottled", + "Throttling", + "SlowDown", + "PriorRequestNotComplete", + "RequestTimeout", + "RequestTimeoutException", + "TimeoutError", + ]); + const connectionCodes = new Set(["ECONNRESET", "ECONNREFUSED", "EPIPE", "ETIMEDOUT", "EAI_AGAIN", "ENOTFOUND", "EHOSTUNREACH", "ENETUNREACH"]); + const code = err.name || err.Code || err.code || ""; + const httpStatus = err.$metadata?.httpStatusCode; + return ( + retryableCodes.has(code) || + connectionCodes.has(err.code) || + err.$retryable != null || + httpStatus === 429 || + (typeof httpStatus === "number" && httpStatus >= 500) || + (typeof err.message === "string" && err.message.toLowerCase().includes("rate exceeded")) + ); + } + + /** + * Polls a CloudFront distribution until its Status becomes "Deployed". + * CloudFront propagates changes globally and can take several minutes. + */ + async waitForDistributionDeployed(cloudFrontClient: any, distributionId: string, timeoutMs = 600_000, pollIntervalMs = 15_000): Promise { + const { GetDistributionCommand } = await this.access.importRuntime("@aws-sdk/client-cloudfront"); + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + let status: string | undefined; + try { + const res: any = await this.withRetry(() => cloudFrontClient.send(new GetDistributionCommand({ Id: distributionId }))); + status = res?.Distribution?.Status; + } catch (err: any) { + // A retryable error that outlives withRetry's budget shouldn't abort the whole deploy; + // keep polling until the deadline and let genuine (non-retryable) errors propagate. + if (!AwsClient.isRetryableAwsError(err)) { + throw err; + } + this.logger.warn(`CloudFront status check failed (${err?.name || err?.code}), will retry on next poll…`); + await utils.sleep(pollIntervalMs); + continue; + } + this.logger.info(`CloudFront distribution ${distributionId} status: ${status}`); + if (status === "Deployed") { + return; + } + await utils.sleep(pollIntervalMs); + } + throw new Error(`Timed out waiting for CloudFront distribution ${distributionId} to reach Deployed status`); + } } diff --git a/packages/ui/certd-server/src/plugins/plugin-aws/plugins/plugin-deploy-to-cloudfront.ts b/packages/ui/certd-server/src/plugins/plugin-aws/plugins/plugin-deploy-to-cloudfront.ts index 0b6cd3899..d06c75983 100644 --- a/packages/ui/certd-server/src/plugins/plugin-aws/plugins/plugin-deploy-to-cloudfront.ts +++ b/packages/ui/certd-server/src/plugins/plugin-aws/plugins/plugin-deploy-to-cloudfront.ts @@ -67,15 +67,35 @@ export class AwsDeployToCloudFront extends AbstractTaskPlugin { ) distributionIds!: string[]; + @TaskInput({ + title: "等待部署完成", + value: false, + helper: + "开启后,会轮询等待每个CloudFront分配状态变为Deployed(通常需要几分钟)再结束任务。" + + "关闭时,提交证书更新后立即结束——CloudFront会在后台自行完成部署。" + + "仅当后续任务依赖证书已全球生效时才需要开启。", + component: { + name: "a-switch", + vModel: "checked", + }, + }) + waitForDeployed = false; + async onInstance() {} async execute(): Promise { const access = await this.getAccess(this.accessId); + const acmClient = new AwsClient({ + access, + region: this.region, + logger: this.logger, + }); + let certId = this.cert as string; if (typeof this.cert !== "string") { //先上传 - certId = await this.uploadToACM(access, this.cert); + certId = await this.uploadToACM(acmClient, this.cert); } //部署到CloudFront @@ -86,42 +106,49 @@ export class AwsDeployToCloudFront extends AbstractTaskPlugin { accessKeyId: access.accessKeyId, secretAccessKey: access.secretAccessKey, }, + // Disable the SDK's own retries; acmClient.withRetry is the single retry authority. + maxAttempts: 1, }); // update-distribution for (const distributionId of this.distributionIds) { - // get-distribution-config - const getDistributionConfigCommand = new GetDistributionConfigCommand({ - Id: distributionId, - }); + // get-distribution-config (with retry for throttling) + const configData: any = await acmClient.withRetry(() => + cloudFrontClient.send(new GetDistributionConfigCommand({ Id: distributionId })) + ); - const configData = await cloudFrontClient.send(getDistributionConfigCommand); + await acmClient.withRetry(() => + cloudFrontClient.send( + new UpdateDistributionCommand({ + DistributionConfig: { + ...configData.DistributionConfig, + ViewerCertificate: { + ...configData.DistributionConfig.ViewerCertificate, + CloudFrontDefaultCertificate: false, + ACMCertificateArn: certId, + }, + }, + Id: distributionId, + IfMatch: configData.ETag, + }) + ) + ); - const updateDistributionCommand = new UpdateDistributionCommand({ - DistributionConfig: { - ...configData.DistributionConfig, - ViewerCertificate: { - ...configData.DistributionConfig.ViewerCertificate, - CloudFrontDefaultCertificate: false, - ACMCertificateArn: certId, - }, - }, - Id: distributionId, - IfMatch: configData.ETag, - }); - await cloudFrontClient.send(updateDistributionCommand); - this.logger.info(`部署${distributionId}完成:`); + if (this.waitForDeployed) { + this.logger.info(`证书已提交到 ${distributionId},等待全局部署完成…`); + // CloudFront propagates globally in a few minutes; only block when the + // user opts in (e.g. a downstream task needs the cert already live). + await acmClient.waitForDistributionDeployed(cloudFrontClient, distributionId); + this.logger.info(`部署 ${distributionId} 完成`); + } else { + this.logger.info(`证书已提交到 ${distributionId},CloudFront 将在后台完成部署`); + } } this.logger.info("部署完成"); } - private async uploadToACM(access: AwsAccess, cert: CertInfo) { - const acmClient = new AwsClient({ - access, - region: this.region, - logger: this.logger, - }); - const awsCertARN = await acmClient.importCertificate(cert); + private async uploadToACM(acmClient: AwsClient, cert: CertInfo) { + const awsCertARN = await acmClient.withRetry(() => acmClient.importCertificate(cert)); this.logger.info("证书上传成功,id=", awsCertARN); return awsCertARN; } @@ -140,6 +167,7 @@ export class AwsDeployToCloudFront extends AbstractTaskPlugin { accessKeyId: access.accessKeyId, secretAccessKey: access.secretAccessKey, }, + maxAttempts: 1, }); // list-distributions const listDistributionsCommand = new ListDistributionsCommand({}); diff --git a/packages/ui/certd-server/src/plugins/plugin-aws/plugins/plugin-upload-to-acm.ts b/packages/ui/certd-server/src/plugins/plugin-aws/plugins/plugin-upload-to-acm.ts index ea9cedc18..0ee3f8869 100644 --- a/packages/ui/certd-server/src/plugins/plugin-aws/plugins/plugin-upload-to-acm.ts +++ b/packages/ui/certd-server/src/plugins/plugin-aws/plugins/plugin-upload-to-acm.ts @@ -65,7 +65,7 @@ export class AwsUploadToACM extends AbstractTaskPlugin { region, logger: this.logger, }); - this.awsCertARN = await acmClient.importCertificate(cert); + this.awsCertARN = await acmClient.withRetry(() => acmClient.importCertificate(cert)); this.logger.info("证书上传成功,id=", this.awsCertARN); } }