fix(api): add in-flight timeout for blue-green rollover (#68825)

Co-authored-by: Mrugesh Mohapatra <1884376+raisedadead@users.noreply.github.com>

api/src/server.ts

@@ -8,7 +8,8 @@ import {
   DEPLOYMENT_VERSION,
   HOST,
   PORT,
-  SENTRY_SERVER_NAME
+  SENTRY_SERVER_NAME,
+  FCC_DRAIN_TIMEOUT_MS
 } from './utils/env.js';
 
 const start = async () => {
@@ -20,7 +21,17 @@ const start = async () => {
     const stop = async (signal: NodeJS.Signals) => {
       fastify!.log.info({ signal }, 'Received signal, shutting down');
 
-      fastify!.server.closeAllConnections();
+      // Safety net: if in-flight requests do not finish in time, hard-close
+      // whatever is left so Swarm's SIGKILL never fires mid-write.
+      const forceClose = setTimeout(() => {
+        fastify!.log.warn(
+          { signal, timeoutMs: FCC_DRAIN_TIMEOUT_MS },
+          'Drain timeout exceeded, force-closing connections'
+        );
+        fastify!.server.closeAllConnections();
+      }, FCC_DRAIN_TIMEOUT_MS);
+      forceClose.unref();
+
       await new Promise<void>(resolve => {
         fastify!.server.close(() => resolve());
       });
@@ -30,6 +41,7 @@ const start = async () => {
       await new Promise<void>(resolve => setImmediate(resolve));
 
       await fastify!.close();
+      clearTimeout(forceClose);
       Sentry.metrics.count('server.shutdown_completed', 1, {
         attributes: { signal }
       });

api/src/utils/env.ts

@@ -3,6 +3,7 @@ import path from 'node:path';
 import { fileURLToPath } from 'node:url';
 import { config } from 'dotenv';
 import { LogLevel } from 'fastify';
+import { parseBool, parseInt } from './validation.js';
 
 const __dirname = path.dirname(fileURLToPath(import.meta.url));
 const envPath = path.resolve(__dirname, '../../../.env');
@@ -200,21 +201,29 @@ export const PORT = process.env.PORT || '3000';
 // container.
 export const HOST = process.env.HOST || '0.0.0.0';
 export const API_LOCATION = process.env.API_LOCATION;
-export const FCC_ENABLE_SWAGGER_UI = undefinedOrBool(
-  process.env.FCC_ENABLE_SWAGGER_UI
+export const FCC_ENABLE_SWAGGER_UI = parseWith<boolean | undefined>(
+  'FCC_ENABLE_SWAGGER_UI',
+  undefined,
+  parseBool
 );
 export const FCC_ENABLE_DEV_LOGIN_MODE =
   process.env.FCC_ENABLE_DEV_LOGIN_MODE === 'true';
 export const FCC_API_LOG_LEVEL = _FCC_API_LOG_LEVEL;
 export const FCC_API_LOG_TRANSPORT = _FCC_API_LOG_TRANSPORT;
-export const FCC_ENABLE_SHADOW_CAPTURE = undefinedOrBool(
-  process.env.FCC_ENABLE_SHADOW_CAPTURE
+export const FCC_ENABLE_SHADOW_CAPTURE = parseWith<boolean | undefined>(
+  'FCC_ENABLE_SHADOW_CAPTURE',
+  undefined,
+  parseBool
 );
-export const FCC_ENABLE_SENTRY_ROUTES = undefinedOrBool(
-  process.env.FCC_ENABLE_SENTRY_ROUTES
+export const FCC_ENABLE_SENTRY_ROUTES = parseWith<boolean | undefined>(
+  'FCC_ENABLE_SENTRY_ROUTES',
+  undefined,
+  parseBool
 );
-export const FCC_ENABLE_CLASSROOM = undefinedOrBool(
-  process.env.FCC_ENABLE_CLASSROOM
+export const FCC_ENABLE_CLASSROOM = parseWith<boolean | undefined>(
+  'FCC_ENABLE_CLASSROOM',
+  undefined,
+  parseBool
 );
 export const FREECODECAMP_NODE_ENV = _FREECODECAMP_NODE_ENV;
 export const DEPLOYMENT_ENV = process.env.DEPLOYMENT_ENV;
@@ -270,12 +279,28 @@ export const GROWTHBOOK_FASTIFY_CLIENT_KEY =
 export const SOCRATES_API_KEY = process.env.SOCRATES_API_KEY;
 export const SOCRATES_ENDPOINT = process.env.SOCRATES_ENDPOINT;
 export const TPA_API_BEARER_TOKEN = process.env.TPA_API_BEARER_TOKEN;
+/** Server grace timeout before force closing in-flight requests. */
+export const FCC_DRAIN_TIMEOUT_MS = parseWith(
+  'FCC_DRAIN_TIMEOUT_MS',
+  20_000,
+  parseInt
+);
 
-function undefinedOrBool(val: string | undefined): undefined | boolean {
-  if (!val) {
-    return undefined;
-  }
-
-  return val === 'true';
-}
 export const DEPLOYMENT_VERSION = process.env.DEPLOYMENT_VERSION || 'unknown';
+
+function parseWith<T>(
+  name: string,
+  fallback: T,
+  parserFunction: (str: string) => T
+): T {
+  const str = process.env[name];
+  if (str === undefined || str === null || str?.trim() === '') return fallback;
+  try {
+    return parserFunction(str);
+  } catch (e) {
+    if (e instanceof Error) {
+      throw new Error(`Failed to parse ${name}: ${e}`);
+    }
+    throw new Error(`Unhandled error parsing '${name}'`);
+  }
+}

api/src/utils/validation.ts

@@ -1,4 +1,5 @@
 import { ObjectId } from 'bson';
+import assert from 'node:assert';
 
 // This is trivial, but makes it simple to refactor if we swap monogodb for
 // bson, say.
@@ -45,3 +46,19 @@ export const trimTags = (value: string): string => {
 
   return value.replace(/</g, '&lt;');
 };
+
+/**
+ * Parses and asserts input string to integer.
+ */
+export function parseInt(str: string): number {
+  const n = Number(str);
+  assert.ok(Number.isInteger(n), `expected '${str}' to be an integer`);
+  return n;
+}
+
+/**
+ * Parses input string into boolean. `true` if `"true"`, otherwise `false`.
+ */
+export function parseBool(str: string): boolean {
+  return str === 'true';
+}

api/turbo.json

@@ -21,6 +21,7 @@
         "FCC_ENABLE_SHADOW_CAPTURE",
         "FCC_ENABLE_SWAGGER_UI",
         "FCC_ENABLE_TEST_LOGGING",
+        "FCC_DRAIN_TIMEOUT_MS",
         "FREECODECAMP_NODE_ENV",
         "GROWTHBOOK_FASTIFY_API_HOST",
         "GROWTHBOOK_FASTIFY_CLIENT_KEY",

sample.env

@@ -77,6 +77,7 @@ FCC_ENABLE_SENTRY_ROUTES=false
 FCC_ENABLE_CLASSROOM=false
 FCC_API_LOG_LEVEL=info
 FCC_API_LOG_TRANSPORT=pretty
+FCC_DRAIN_TIMEOUT_MS=20000
 
 # Email
 # use ses in production, nodemailer for local development (with Mailpit)