feat: support afterAll hook in challenge tests (#61256)

client/gatsby-node.js

@@ -384,6 +384,7 @@ exports.createSchemaCustomization = ({ actions }) => {
       beforeEach: String
       afterEach: String
       beforeAll: String
+      afterAll: String
     }
   `;
   createTypes(typeDefs);

client/src/redux/prop-types.ts

@@ -243,6 +243,7 @@ export interface Hooks {
   beforeAll?: string;
   beforeEach?: string;
   afterEach?: string;
+  afterAll?: string;
 }
 
 export type PageContext = {

client/src/templates/Challenges/classic/show.tsx

@@ -591,6 +591,7 @@ export const query = graphql`
           beforeAll
           beforeEach
           afterEach
+          afterAll
         }
         fields {
           blockName

client/src/templates/Challenges/redux/execute-challenge-saga.js

@@ -1,5 +1,5 @@
 import i18next from 'i18next';
-import { escape } from 'lodash-es';
+import { escape, isEmpty } from 'lodash-es';
 import { channel } from 'redux-saga';
 import {
   call,
@@ -185,18 +185,17 @@ function* buildChallengeData(challengeData, options) {
 }
 
 function* executeTests(testRunner, tests, testTimeout = 5000) {
+  const testStrings = tests.map(test => test.testString);
+  const rawResults = yield call(testRunner, testStrings, testTimeout);
+
   const testResults = [];
-  for (let i = 0; i < tests.length; i++) {
+  for (let i = 0; i < rawResults.length; i++) {
     const { text, testString } = tests[i];
     const newTest = { text, testString, running: false };
     // only the first test outputs console.logs to avoid log duplication.
     const firstTest = i === 0;
     try {
-      const {
-        pass,
-        err,
-        logs = []
-      } = yield call(testRunner, testString, testTimeout);
+      const { pass, err, logs = [] } = rawResults[i] || {};
 
       const logString = logs.map(log => log.msg).join('\n');
       if (firstTest && logString) {
@@ -300,11 +299,13 @@ export function* previewChallengeSaga(action) {
         }
       } else if (isJavaScriptChallenge(challengeData)) {
         const runUserCode = yield call(getTestRunner, buildData);
-        // without a testString the testRunner just evaluates the user's code
-        const out = yield call(runUserCode, null, previewTimeout);
 
-        if (out) {
-          const logs = out.logs?.filter(
+        // Without an empty testString the testRunner just evaluates the user's
+        // code allowing us to get the console logs.
+        const results = yield call(runUserCode, [''], previewTimeout);
+
+        if (!isEmpty(results)) {
+          const logs = results[0].logs?.filter(
             log => !LOGS_TO_IGNORE.some(msg => log.msg === msg)
           );
           yield put(updateConsole(logs?.map(log => log.msg).join('\n')));

client/src/templates/Challenges/utils/build.ts

@@ -9,7 +9,7 @@ import {
   getMultifileJSXTransformers
 } from '../rechallenge/transformers';
 import {
-  runTestInTestFrame,
+  runTestsInTestFrame,
   createMainPreviewFramer,
   createProjectPreviewFramer,
   ProxyLogger,
@@ -159,8 +159,8 @@ export async function getTestRunner(buildData: BuildChallengeData) {
   }
   await prepTestRunner({ ...buildData, type });
 
-  return (testString: string, testTimeout: number) =>
-    runTestInTestFrame(testString, testTimeout, type);
+  return (testStrings: string[], testTimeout: number) =>
+    runTestsInTestFrame(testStrings, testTimeout, type);
 }
 
 type BuildResult = {

client/src/templates/Challenges/utils/frame.ts

@@ -169,19 +169,14 @@ function getContentDocument<T extends Document = FrameDocument>(
   return frameDocument as T;
 }
 
-export const runTestInTestFrame = async function (
-  test: string,
+export const runTestsInTestFrame = async function (
+  tests: string[],
   timeout: number,
   type: 'dom' | 'javascript' | 'python'
-): Promise<TestResult | undefined> {
+): Promise<TestResult[] | undefined> {
   const runner = window?.FCCTestRunner.getRunner(type);
 
-  return await Promise.race([
-    new Promise<
-      { pass: boolean } | { err: { message: string; stack?: string } }
-    >((_, reject) => setTimeout(() => reject(Error('timeout')), timeout)),
-    runner?.runTest(test)
-  ]);
+  return runner?.runAllTests(tests, timeout);
 };
 
 export const prepTestRunner = async ({

curriculum/challenges/english/blocks/lab-one-time-password-generator/67c562286b29447da020d407.md

@@ -36,6 +36,12 @@ In this lab, you will generate a 6-digit OTP (One-Time Password) and display it
 let clock = __FakeTimers.install();
 ```
 
+# --after-all--
+
+```js
+clock.uninstall();
+```
+
 # --hints--
 
 You should not remove the existing `const { useState, useEffect, useRef } = React;` assignment from the code.
@@ -234,14 +240,6 @@ You should export the `OTPGenerator` component.
 assert.isFunction(window.index.OTPGenerator);
 ```
 
-Uninstall the clock, ignore this test
-
-```js
-async () => {
-clock.uninstall()
-}
-```
-
 # --seed--
 
 ## --seed-contents--

curriculum/challenges/english/blocks/lab-tic-tac-toe/67e3a6b7f60b4085588189e6.md

@@ -43,6 +43,12 @@ function getInnerTextExcept(removingSelector) {
 }
 ```
 
+# --after-all--
+
+```js
+clock.uninstall();
+```
+
 # --hints--
 
 You should export a `Board` component.

curriculum/schema/challenge-schema.js

@@ -306,7 +306,8 @@ const schema = Joi.object().keys({
   hooks: Joi.object().keys({
     beforeAll: Joi.string().allow(''),
     beforeEach: Joi.string().allow(''),
-    afterEach: Joi.string().allow('')
+    afterEach: Joi.string().allow(''),
+    afterAll: Joi.string().allow('')
   }),
   tests: Joi.array()
     .items(

curriculum/test/test-challenges.js

@@ -412,13 +412,10 @@ function populateTestsForLang({ lang, challenges, meta, superBlocks }) {
                     fails = true;
                   }
                   if (!fails) {
-                    for (const test of tests) {
-                      try {
-                        await testRunner(test);
-                      } catch {
-                        fails = true;
-                        break;
-                      }
+                    try {
+                      await testRunner(tests);
+                    } catch {
+                      fails = true;
                     }
                   }
                   console.error = oldConsoleError;
@@ -513,9 +510,8 @@ seed goes here
                         buildChallenge,
                         solutionFromNext
                       );
-                      for (const test of tests) {
-                        await testRunner(test);
-                      }
+
+                      await testRunner(tests);
                     });
                   });
                 });
@@ -557,24 +553,26 @@ async function createTestRunner(
     hooks: challenge.hooks
   });
 
-  return async ({ text, testString }) => {
-    try {
-      const { pass, err } = await evaluator.evaluate(testString, 5000);
-      if (!pass) {
+  return async tests => {
+    const testStrings = tests.map(test => test.testString);
+
+    const results = await evaluator.evaluate(testStrings, 5000);
+    for (let i = 0; i < results.length; i++) {
+      const { err } = results[i];
+      let { text } = tests[i];
+      if (err) {
+        text = 'Test text: ' + text;
+        const newMessage = solutionFromNext
+          ? 'Check next step for solution!\n' + text
+          : text;
+        // if the stack is missing, the message should be included. Otherwise it
+        // is redundant.
+        err.message = err.stack
+          ? newMessage
+          : `${newMessage}
+      ${err.message}`;
         throw err;
       }
-    } catch (err) {
-      text = 'Test text: ' + text;
-      const newMessage = solutionFromNext
-        ? 'Check next step for solution!\n' + text
-        : text;
-      // if the stack is missing, the message should be included. Otherwise it
-      // is redundant.
-      err.message = err.stack
-        ? newMessage
-        : `${newMessage}
-      ${err.message}`;
-      throw err;
     }
   };
 }
@@ -604,31 +602,18 @@ async function getContextEvaluator(config) {
   await initializeTestRunner(config);
 
   return {
-    evaluate: async (testString, timeout) =>
-      Promise.race([
-        new Promise((_, reject) =>
-          setTimeout(
-            () =>
-              reject(
-                Error(`timeout in challenge
-${config.challengeId}
-while evaluating test:
-${testString}
-`)
-              ),
+    evaluate: async (testStrings, timeout) =>
+      await page.evaluate(
+        async (type, testStrings, timeout) => {
+          return await window.FCCTestRunner.getRunner(type).runAllTests(
+            testStrings,
             timeout
-          )
-        ),
-        await page.evaluate(
-          async (testString, type) => {
-            return await window.FCCTestRunner.getRunner(type).runTest(
-              testString
-            );
-          },
-          testString,
-          config.type
-        )
-      ])
+          );
+        },
+        config.type,
+        testStrings,
+        timeout
+      )
   };
 }
 

tools/challenge-parser/parser/__fixtures__/with-after-all-hook.md

@@ -0,0 +1,40 @@
+# --description--
+
+Paragraph 1
+
+```html
+code example
+```
+
+# --after-all--
+
+```js
+// after all code
+function teardown() {
+  return 'cleaned up';
+}
+teardown();
+```
+
+# --hints--
+
+First hint
+
+```js
+// test code
+```
+
+Second hint with <code>code</code>
+
+```js
+// more test code
+```
+
+Third *hint* with <code>code</code> and `inline code`
+
+```js
+// more test code
+if(let x of xs) {
+  console.log(x);
+}
+```

tools/challenge-parser/parser/__fixtures__/with-another-invalid-after-all-hook.md

@@ -0,0 +1,34 @@
+# --description--
+
+Paragraph 1
+
+```html
+code example
+```
+
+# --after-all--
+
+gubbins
+
+# --hints--
+
+First hint
+
+```js
+// test code
+```
+
+Second hint with <code>code</code>
+
+```js
+// more test code
+```
+
+Third *hint* with <code>code</code> and `inline code`
+
+```js
+// more test code
+if(let x of xs) {
+  console.log(x);
+}
+```

tools/challenge-parser/parser/__fixtures__/with-invalid-after-all-hook.md

@@ -0,0 +1,42 @@
+# --description--
+
+Paragraph 1
+
+```html
+code example
+```
+
+# --after-all--
+
+```js
+// after all code
+function teardown() {
+  return 'cleaned up';
+}
+teardown();
+```
+
+gubbins
+
+# --hints--
+
+First hint
+
+```js
+// test code
+```
+
+Second hint with <code>code</code>
+
+```js
+// more test code
+```
+
+Third *hint* with <code>code</code> and `inline code`
+
+```js
+// more test code
+if(let x of xs) {
+  console.log(x);
+}
+```

tools/challenge-parser/parser/__fixtures__/with-non-js-after-all-hook.md

@@ -0,0 +1,40 @@
+# --description--
+
+Paragraph 1
+
+```html
+code example
+```
+
+# --after-all--
+
+```ts
+// after all code
+function teardown() {
+  return 'cleaned up';
+}
+teardown();
+```
+
+# --hints--
+
+First hint
+
+```js
+// test code
+```
+
+Second hint with <code>code</code>
+
+```js
+// more test code
+```
+
+Third *hint* with <code>code</code> and `inline code`
+
+```js
+// more test code
+if(let x of xs) {
+  console.log(x);
+}
+```

tools/challenge-parser/parser/plugins/add-hooks.js

@@ -7,13 +7,15 @@ function plugin() {
     const beforeAll = getHook(tree, '--before-all--');
     const beforeEach = getHook(tree, '--before-each--');
     const afterEach = getHook(tree, '--after-each--');
+    const afterAll = getHook(tree, '--after-all--');
 
-    if (!beforeAll && !beforeEach && !afterEach) return;
+    if (!beforeAll && !beforeEach && !afterEach && !afterAll) return;
 
     file.data.hooks = file.data.hooks = {
       ...(beforeAll && { beforeAll }),
       ...(beforeEach && { beforeEach }),
-      ...(afterEach && { afterEach })
+      ...(afterEach && { afterEach }),
+      ...(afterAll && { afterAll })
     };
   }
 }

tools/challenge-parser/parser/plugins/add-hooks.test.js

@@ -14,7 +14,11 @@ describe('add-before-hook plugin', () => {
     withAfterEachHookAST,
     withInvalidAfterEachHookAST,
     withAnotherInvalidAfterEachHookAST,
-    withNonJSAfterEachHookAST;
+    withNonJSAfterEachHookAST,
+    withAfterAllHookAST,
+    withInvalidAfterAllHookAST,
+    withAnotherInvalidAfterAllHookAST,
+    withNonJSAfterAllHookAST;
 
   const plugin = addBeforeHook();
   let file = { data: {} };
@@ -46,6 +50,16 @@ describe('add-before-hook plugin', () => {
     withNonJSAfterEachHookAST = await parseFixture(
       'with-non-js-after-each-hook.md'
     );
+    withAfterAllHookAST = await parseFixture('with-after-all-hook.md');
+    withInvalidAfterAllHookAST = await parseFixture(
+      'with-invalid-after-all-hook.md'
+    );
+    withAnotherInvalidAfterAllHookAST = await parseFixture(
+      'with-another-invalid-after-all-hook.md'
+    );
+    withNonJSAfterAllHookAST = await parseFixture(
+      'with-non-js-after-all-hook.md'
+    );
   });
 
   beforeEach(() => {
@@ -146,4 +160,31 @@ cleanup();`);
       `# --after-each-- hook must be written in JavaScript`
     );
   });
+
+  it('populates `hooks.afterAll` with the contents of the code block', () => {
+    plugin(withAfterAllHookAST, file);
+    expect(file.data.hooks.afterAll).toBe(`// after all code
+function teardown() {
+  return 'cleaned up';
+}
+teardown();`);
+  });
+
+  it('should throw an error if the afterAll section has more than one child', () => {
+    expect(() => plugin(withInvalidAfterAllHookAST, file)).toThrow(
+      `# --after-all-- section must only contain a single code block`
+    );
+  });
+
+  it('should throw an error if the afterAll section does not contain a code block', () => {
+    expect(() => plugin(withAnotherInvalidAfterAllHookAST, file)).toThrow(
+      `# --after-all-- section must contain a code block`
+    );
+  });
+
+  it('should throw an error if the afterAll code language is not javascript', () => {
+    expect(() => plugin(withNonJSAfterAllHookAST, file)).toThrow(
+      `# --after-all-- hook must be written in JavaScript`
+    );
+  });
 });