From 1b882a2ca41b6d7fd5af97814be334c3125572eb Mon Sep 17 00:00:00 2001 From: Steven Zhu Date: Sun, 6 Sep 2026 04:19:47 -0400 Subject: [PATCH] AWS ratelimit fix (#796) @szhu25 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Parse PEM chain and import certificate chain Split the PEM in certInfo.crt into a leaf certificate and intermediate chain (using a lookbehind regex), trim the blocks, and pass the chain to ImportCertificateCommand only when present. Replace console.log with this.logger.info and log the returned CertificateArn. This ensures the leaf cert is uploaded separately from its chain and avoids sending an empty CertificateChain. * Add AWS retry & CloudFront deployment wait Introduce robust retry and polling helpers to handle AWS throttling and CloudFront propagation. Added AwsClient.withRetry (exponential backoff, handles common throttling errors, default 5 attempts/base 2s) and waitForDistributionDeployed (polls until distribution Status is "Deployed", default 10min timeout/15s interval). Update deploy-to-cloudfront plugin to use withRetry for Get/UpdateDistribution and importCertificate, pass AwsClient into uploadToACM, and wait for each distribution to finish deploying before continuing to avoid PreconditionFailed errors. Improves reliability when facing rate limits and global CloudFront propagation delays; adds informative logging for retry and deployment status. * fix(aws): resolve rate limit and runtime import bugs * fix(aws): broaden retry detection and harden CloudFront poll - withRetry: retry on Route53 PriorRequestNotComplete, SlowDown, SDK v3 $retryable.throttling hints and HTTP 429/503; extract detection into isRetryableAwsError - withRetry: add equal jitter to backoff so concurrent pipelines don't retry in lockstep - waitForDistributionDeployed: a throttle that outlives withRetry's budget keeps polling until the deadline instead of aborting the deploy - test: cover withRetry retry/exhaustion/non-throttle paths Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_015dRf6EbAmwb37cpnvgBDrh * fix(aws): fix build break and make withRetry the sole retry authority - withRetry: default the type param so call sites passing an untyped SDK client (@aws-sdk/* is a lazyDependency with no build-time types) don't collapse the result to `unknown`. Also annotate the two result consumers as `any`. Fixes the TS2339 errors that broke the Docker build (aws-client.ts, plugin-deploy-to-cloudfront.ts). - Set maxAttempts: 1 on every AWS SDK client (ACM/STS/Route53/CloudFront) so the SDK's own retry strategy no longer stacks on top of withRetry. - isRetryableAwsError: broaden to cover what the SDK's `standard` strategy used to handle now that it's disabled - $retryable hint, HTTP 429 + any 5xx, connection-level errors (ECONNRESET/ETIMEDOUT/EAI_AGAIN/...). - waitForDistributionDeployed: a transient error that outlives withRetry's budget now retries on the next poll instead of aborting the whole deploy. - test: cover 5xx, $retryable, and connection-error retry paths. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01BrMVmQyvQjes6t4DGjuBaD * feat(aws): make CloudFront deploy-wait opt-in (default off) Polling GetDistribution until Status=Deployed adds 3-8min of pipeline wall-time per run for no functional gain - CloudFront finishes the rollout on its own once UpdateDistribution returns 200. Gate it behind a new "等待部署完成" switch, default false. Retry/backoff on the actual mutating calls (GetDistributionConfig / UpdateDistribution / ACM import / Route53) is unaffected - it wraps those sends directly, not the poll. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01BrMVmQyvQjes6t4DGjuBaD --------- Co-authored-by: Claude Sonnet 5 --- .../plugin-aws/libs/aws-client.test.ts | 123 +++++++++++++++++- .../src/plugins/plugin-aws/libs/aws-client.ts | 107 ++++++++++++++- .../plugins/plugin-deploy-to-cloudfront.ts | 82 ++++++++---- .../plugins/plugin-upload-to-acm.ts | 2 +- 4 files changed, 282 insertions(+), 32 deletions(-) diff --git a/packages/ui/certd-server/src/plugins/plugin-aws/libs/aws-client.test.ts b/packages/ui/certd-server/src/plugins/plugin-aws/libs/aws-client.test.ts index 46f4f8157..a211fbd98 100644 --- a/packages/ui/certd-server/src/plugins/plugin-aws/libs/aws-client.test.ts +++ b/packages/ui/certd-server/src/plugins/plugin-aws/libs/aws-client.test.ts @@ -35,7 +35,7 @@ async function createAwsClient(records: any[]) { ChangeResourceRecordSetsCommand, }), } as any, - logger: { info() {}, error() {} } as any, + logger: { info() {}, warn() {}, error() {} } as any, region: "us-east-1", }); return { client, sentChanges }; @@ -92,3 +92,124 @@ describe("AwsClient.route53ChangeRecord", () => { assert.deepEqual(change.ResourceRecordSet.ResourceRecords, [{ Value: '"wildcard-token"' }]); }); }); + +function throttle(name: string, extra: Record = {}) { + const err: any = new Error(name); + err.name = name; + return Object.assign(err, extra); +} + +describe("AwsClient.withRetry", () => { + it("retries a throttled call and then resolves", async () => { + const { client } = await createAwsClient([]); + let calls = 0; + const result = await client.withRetry(async () => { + calls++; + if (calls < 3) { + throw throttle("ThrottlingException"); + } + return "ok"; + }); + assert.equal(result, "ok"); + assert.equal(calls, 3); + }); + + it("retries Route53 PriorRequestNotComplete", async () => { + const { client } = await createAwsClient([]); + let calls = 0; + const result = await client.withRetry(async () => { + calls++; + if (calls < 2) { + throw throttle("PriorRequestNotComplete"); + } + return "done"; + }); + assert.equal(result, "done"); + assert.equal(calls, 2); + }); + + it("retries on HTTP 429 without a known error code", async () => { + const { client } = await createAwsClient([]); + let calls = 0; + await client.withRetry(async () => { + calls++; + if (calls < 2) { + const err: any = new Error("too many requests"); + err.$metadata = { httpStatusCode: 429 }; + throw err; + } + return null; + }); + assert.equal(calls, 2); + }); + + it("retries transient HTTP 5xx", async () => { + const { client } = await createAwsClient([]); + let calls = 0; + await client.withRetry(async () => { + calls++; + if (calls < 2) { + const err: any = new Error("service unavailable"); + err.$metadata = { httpStatusCode: 503 }; + throw err; + } + return null; + }); + assert.equal(calls, 2); + }); + + it("retries when the SDK marks the error $retryable", async () => { + const { client } = await createAwsClient([]); + let calls = 0; + await client.withRetry(async () => { + calls++; + if (calls < 2) { + const err: any = new Error("internal failure"); + err.name = "InternalFailure"; + err.$retryable = {}; + throw err; + } + return null; + }); + assert.equal(calls, 2); + }); + + it("retries connection-level errors", async () => { + const { client } = await createAwsClient([]); + let calls = 0; + await client.withRetry(async () => { + calls++; + if (calls < 2) { + throw throttle("Error", { code: "ECONNRESET" }); + } + return null; + }); + assert.equal(calls, 2); + }); + + it("does not retry non-throttling errors", async () => { + const { client } = await createAwsClient([]); + let calls = 0; + await assert.rejects( + client.withRetry(async () => { + calls++; + throw throttle("AccessDeniedException"); + }), + /AccessDeniedException/ + ); + assert.equal(calls, 1); + }); + + it("stops after maxAttempts and rethrows the throttling error", async () => { + const { client } = await createAwsClient([]); + let calls = 0; + await assert.rejects( + client.withRetry(async () => { + calls++; + throw throttle("Throttling", { message: "Rate exceeded" }); + }, 3), + /Throttling|Rate exceeded/ + ); + assert.equal(calls, 3); + }); +}); diff --git a/packages/ui/certd-server/src/plugins/plugin-aws/libs/aws-client.ts b/packages/ui/certd-server/src/plugins/plugin-aws/libs/aws-client.ts index a0ec768fb..d49659954 100644 --- a/packages/ui/certd-server/src/plugins/plugin-aws/libs/aws-client.ts +++ b/packages/ui/certd-server/src/plugins/plugin-aws/libs/aws-client.ts @@ -28,6 +28,8 @@ export class AwsClient { accessKeyId: this.access.accessKeyId, // 从环境变量中读取 secretAccessKey: this.access.secretAccessKey, }, + // Disable the SDK's own retries; withRetry/doRequest is the single retry authority. + maxAttempts: 1, }); // Split the full PEM chain: first block is the leaf cert, the rest is the intermediate chain @@ -57,6 +59,7 @@ export class AwsClient { accessKeyId: this.access.accessKeyId, // 从环境变量中读取 secretAccessKey: this.access.secretAccessKey, }, + maxAttempts: 1, }); const command = new GetCallerIdentityCommand({}); @@ -75,6 +78,7 @@ export class AwsClient { accessKeyId: this.access.accessKeyId, // 从环境变量中读取 secretAccessKey: this.access.secretAccessKey, }, + maxAttempts: 1, }); } @@ -202,10 +206,107 @@ export class AwsClient { async doRequest(call: () => Promise): Promise { try { - return await call(); - } catch (err) { - this.logger.error(`调用接口失败:${err.Error?.Message || err.message},requestId:${err.requestId}`); + return await this.withRetry(call); + } catch (err: any) { + this.logger.error(`请求失败:${err.Error?.Message || err.message},requestId:${err.requestId}`); throw err; } } + + /** + * Retries an AWS SDK call with exponential backoff + jitter while the error is + * retryable. See {@link AwsClient.isRetryableAwsError}. + * + * `T` defaults to `any` so that call sites passing an untyped SDK client + * (`@aws-sdk/*` is a lazyDependency and has no types at build time) don't + * collapse the result to `unknown`. + */ + async withRetry(call: () => Promise, maxAttempts = 5, baseDelayMs = 2000): Promise { + for (let attempt = 1; attempt <= maxAttempts; attempt++) { + try { + return await call(); + } catch (err: any) { + if (attempt >= maxAttempts || !AwsClient.isRetryableAwsError(err)) { + throw err; + } + const code = err?.name || err?.Code || err?.code || err?.$metadata?.httpStatusCode || "unknown"; + const ceiling = baseDelayMs * Math.pow(2, attempt - 1); // 2s, 4s, 8s, 16s … + const delay = Math.round(ceiling / 2 + Math.random() * (ceiling / 2)); // equal jitter: ~1-2s, ~2-4s, ~4-8s, ~8-16s + this.logger.warn(`AWS request failed (${code}), attempt ${attempt}/${maxAttempts}, retrying in ${delay}ms…`); + await utils.sleep(delay); + } + } + throw new Error("Unreachable"); + } + + /** + * Detects AWS errors that are safe to retry. Because we disable the SDK's own + * retries (maxAttempts: 1 on every client), this is the single source of truth + * and covers what the SDK's `standard` strategy would have retried: + * - throttling / rate-limit codes across ACM / STS / CloudFront / Route53, + * including Route53's PriorRequestNotComplete concurrent-change limit; + * - SDK v3 retry hints (`$retryable`); + * - transient HTTP 429 and 5xx; + * - connection-level errors (ECONNRESET, ETIMEDOUT, EAI_AGAIN, …). + */ + static isRetryableAwsError(err: any): boolean { + if (!err) { + return false; + } + const retryableCodes = new Set([ + "TooManyRequestsException", + "ThrottlingException", + "ThrottledException", + "RequestLimitExceeded", + "RequestThrottled", + "Throttling", + "SlowDown", + "PriorRequestNotComplete", + "RequestTimeout", + "RequestTimeoutException", + "TimeoutError", + ]); + const connectionCodes = new Set(["ECONNRESET", "ECONNREFUSED", "EPIPE", "ETIMEDOUT", "EAI_AGAIN", "ENOTFOUND", "EHOSTUNREACH", "ENETUNREACH"]); + const code = err.name || err.Code || err.code || ""; + const httpStatus = err.$metadata?.httpStatusCode; + return ( + retryableCodes.has(code) || + connectionCodes.has(err.code) || + err.$retryable != null || + httpStatus === 429 || + (typeof httpStatus === "number" && httpStatus >= 500) || + (typeof err.message === "string" && err.message.toLowerCase().includes("rate exceeded")) + ); + } + + /** + * Polls a CloudFront distribution until its Status becomes "Deployed". + * CloudFront propagates changes globally and can take several minutes. + */ + async waitForDistributionDeployed(cloudFrontClient: any, distributionId: string, timeoutMs = 600_000, pollIntervalMs = 15_000): Promise { + const { GetDistributionCommand } = await this.access.importRuntime("@aws-sdk/client-cloudfront"); + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + let status: string | undefined; + try { + const res: any = await this.withRetry(() => cloudFrontClient.send(new GetDistributionCommand({ Id: distributionId }))); + status = res?.Distribution?.Status; + } catch (err: any) { + // A retryable error that outlives withRetry's budget shouldn't abort the whole deploy; + // keep polling until the deadline and let genuine (non-retryable) errors propagate. + if (!AwsClient.isRetryableAwsError(err)) { + throw err; + } + this.logger.warn(`CloudFront status check failed (${err?.name || err?.code}), will retry on next poll…`); + await utils.sleep(pollIntervalMs); + continue; + } + this.logger.info(`CloudFront distribution ${distributionId} status: ${status}`); + if (status === "Deployed") { + return; + } + await utils.sleep(pollIntervalMs); + } + throw new Error(`Timed out waiting for CloudFront distribution ${distributionId} to reach Deployed status`); + } } diff --git a/packages/ui/certd-server/src/plugins/plugin-aws/plugins/plugin-deploy-to-cloudfront.ts b/packages/ui/certd-server/src/plugins/plugin-aws/plugins/plugin-deploy-to-cloudfront.ts index 0b6cd3899..d06c75983 100644 --- a/packages/ui/certd-server/src/plugins/plugin-aws/plugins/plugin-deploy-to-cloudfront.ts +++ b/packages/ui/certd-server/src/plugins/plugin-aws/plugins/plugin-deploy-to-cloudfront.ts @@ -67,15 +67,35 @@ export class AwsDeployToCloudFront extends AbstractTaskPlugin { ) distributionIds!: string[]; + @TaskInput({ + title: "等待部署完成", + value: false, + helper: + "开启后，会轮询等待每个CloudFront分配状态变为Deployed（通常需要几分钟）再结束任务。" + + "关闭时，提交证书更新后立即结束——CloudFront会在后台自行完成部署。" + + "仅当后续任务依赖证书已全球生效时才需要开启。", + component: { + name: "a-switch", + vModel: "checked", + }, + }) + waitForDeployed = false; + async onInstance() {} async execute(): Promise { const access = await this.getAccess(this.accessId); + const acmClient = new AwsClient({ + access, + region: this.region, + logger: this.logger, + }); + let certId = this.cert as string; if (typeof this.cert !== "string") { //先上传 - certId = await this.uploadToACM(access, this.cert); + certId = await this.uploadToACM(acmClient, this.cert); } //部署到CloudFront @@ -86,42 +106,49 @@ export class AwsDeployToCloudFront extends AbstractTaskPlugin { accessKeyId: access.accessKeyId, secretAccessKey: access.secretAccessKey, }, + // Disable the SDK's own retries; acmClient.withRetry is the single retry authority. + maxAttempts: 1, }); // update-distribution for (const distributionId of this.distributionIds) { - // get-distribution-config - const getDistributionConfigCommand = new GetDistributionConfigCommand({ - Id: distributionId, - }); + // get-distribution-config (with retry for throttling) + const configData: any = await acmClient.withRetry(() => + cloudFrontClient.send(new GetDistributionConfigCommand({ Id: distributionId })) + ); - const configData = await cloudFrontClient.send(getDistributionConfigCommand); + await acmClient.withRetry(() => + cloudFrontClient.send( + new UpdateDistributionCommand({ + DistributionConfig: { + ...configData.DistributionConfig, + ViewerCertificate: { + ...configData.DistributionConfig.ViewerCertificate, + CloudFrontDefaultCertificate: false, + ACMCertificateArn: certId, + }, + }, + Id: distributionId, + IfMatch: configData.ETag, + }) + ) + ); - const updateDistributionCommand = new UpdateDistributionCommand({ - DistributionConfig: { - ...configData.DistributionConfig, - ViewerCertificate: { - ...configData.DistributionConfig.ViewerCertificate, - CloudFrontDefaultCertificate: false, - ACMCertificateArn: certId, - }, - }, - Id: distributionId, - IfMatch: configData.ETag, - }); - await cloudFrontClient.send(updateDistributionCommand); - this.logger.info(`部署${distributionId}完成:`); + if (this.waitForDeployed) { + this.logger.info(`证书已提交到 ${distributionId}，等待全局部署完成…`); + // CloudFront propagates globally in a few minutes; only block when the + // user opts in (e.g. a downstream task needs the cert already live). + await acmClient.waitForDistributionDeployed(cloudFrontClient, distributionId); + this.logger.info(`部署 ${distributionId} 完成`); + } else { + this.logger.info(`证书已提交到 ${distributionId}，CloudFront 将在后台完成部署`); + } } this.logger.info("部署完成"); } - private async uploadToACM(access: AwsAccess, cert: CertInfo) { - const acmClient = new AwsClient({ - access, - region: this.region, - logger: this.logger, - }); - const awsCertARN = await acmClient.importCertificate(cert); + private async uploadToACM(acmClient: AwsClient, cert: CertInfo) { + const awsCertARN = await acmClient.withRetry(() => acmClient.importCertificate(cert)); this.logger.info("证书上传成功,id=", awsCertARN); return awsCertARN; } @@ -140,6 +167,7 @@ export class AwsDeployToCloudFront extends AbstractTaskPlugin { accessKeyId: access.accessKeyId, secretAccessKey: access.secretAccessKey, }, + maxAttempts: 1, }); // list-distributions const listDistributionsCommand = new ListDistributionsCommand({}); diff --git a/packages/ui/certd-server/src/plugins/plugin-aws/plugins/plugin-upload-to-acm.ts b/packages/ui/certd-server/src/plugins/plugin-aws/plugins/plugin-upload-to-acm.ts index ea9cedc18..0ee3f8869 100644 --- a/packages/ui/certd-server/src/plugins/plugin-aws/plugins/plugin-upload-to-acm.ts +++ b/packages/ui/certd-server/src/plugins/plugin-aws/plugins/plugin-upload-to-acm.ts @@ -65,7 +65,7 @@ export class AwsUploadToACM extends AbstractTaskPlugin { region, logger: this.logger, }); - this.awsCertARN = await acmClient.importCertificate(cert); + this.awsCertARN = await acmClient.withRetry(() => acmClient.importCertificate(cert)); this.logger.info("证书上传成功,id=", this.awsCertARN); } }