From 967a3810f4ccecb965423e4e978e5d4ba1a8fa40 Mon Sep 17 00:00:00 2001 From: Jaayden Halko Date: Wed, 29 Jan 2025 15:57:22 +0000 Subject: [PATCH] fix: fix flaky IDP e2e tests (#16331) resolves coder/internal#325 --- site/e2e/tests/deployment/idpOrgSync.spec.ts | 13 +++++-- .../tests/organizations/idpGroupSync.spec.ts | 15 ++++++-- .../tests/organizations/idpRoleSync.spec.ts | 34 ++++++++++++------- 3 files changed, 44 insertions(+), 18 deletions(-) diff --git a/site/e2e/tests/deployment/idpOrgSync.spec.ts b/site/e2e/tests/deployment/idpOrgSync.spec.ts index c8e6b5a33b..a5162d4055 100644 --- a/site/e2e/tests/deployment/idpOrgSync.spec.ts +++ b/site/e2e/tests/deployment/idpOrgSync.spec.ts @@ -16,6 +16,8 @@ test.beforeEach(async ({ page }) => { }); test.describe("IdpOrgSyncPage", () => { + test.describe.configure({ retries: 1 }); + test("show empty table when no org mappings are present", async ({ page, }) => { @@ -149,7 +151,6 @@ test.describe("IdpOrgSyncPage", () => { }); const idpOrgInput = page.getByLabel("IdP organization name"); - const orgSelector = page.getByPlaceholder("Select organization"); const addButton = page.getByRole("button", { name: /Add IdP organization/i, }); @@ -159,8 +160,16 @@ test.describe("IdpOrgSyncPage", () => { await idpOrgInput.fill("new-idp-org"); // Select Coder organization from combobox + const orgSelector = page.getByPlaceholder("Select organization"); + await expect(orgSelector).toBeAttached(); + await expect(orgSelector).toBeVisible(); await orgSelector.click(); - await page.getByRole("option", { name: orgName }).click(); + await page.waitForTimeout(1000); + + const option = await page.getByRole("option", { name: orgName }); + await expect(option).toBeAttached({ timeout: 30000 }); + await expect(option).toBeVisible(); + await option.click(); // Add button should now be enabled await expect(addButton).toBeEnabled(); diff --git a/site/e2e/tests/organizations/idpGroupSync.spec.ts b/site/e2e/tests/organizations/idpGroupSync.spec.ts index 2ea9d02388..a612825334 100644 --- a/site/e2e/tests/organizations/idpGroupSync.spec.ts +++ b/site/e2e/tests/organizations/idpGroupSync.spec.ts @@ -16,6 +16,8 @@ test.beforeEach(async ({ page }) => { }); test.describe("IdpGroupSyncPage", () => { + test.describe.configure({ retries: 1 }); + test("show empty table when no group mappings are present", async ({ page, }) => { @@ -149,7 +151,6 @@ test.describe("IdpGroupSyncPage", () => { }); const idpOrgInput = page.getByLabel("IdP group name"); - const orgSelector = page.getByPlaceholder("Select group"); const addButton = page.getByRole("button", { name: /Add IdP group/i, }); @@ -159,8 +160,16 @@ test.describe("IdpGroupSyncPage", () => { await idpOrgInput.fill("new-idp-group"); // Select Coder organization from combobox - await orgSelector.click(); - await page.getByRole("option", { name: /Everyone/i }).click(); + const groupSelector = page.getByPlaceholder("Select group"); + await expect(groupSelector).toBeAttached(); + await expect(groupSelector).toBeVisible(); + await groupSelector.click(); + await page.waitForTimeout(1000); + + const option = await page.getByRole("option", { name: /Everyone/i }); + await expect(option).toBeAttached({ timeout: 30000 }); + await expect(option).toBeVisible(); + await option.click(); // Add button should now be enabled await expect(addButton).toBeEnabled(); diff --git a/site/e2e/tests/organizations/idpRoleSync.spec.ts b/site/e2e/tests/organizations/idpRoleSync.spec.ts index 3374151a85..a889591026 100644 --- a/site/e2e/tests/organizations/idpRoleSync.spec.ts +++ b/site/e2e/tests/organizations/idpRoleSync.spec.ts @@ -10,16 +10,18 @@ import { login } from "../../helpers"; import { beforeCoderTest } from "../../hooks"; test.beforeEach(async ({ page }) => { + requiresLicense(); beforeCoderTest(page); await login(page); await setupApiCalls(page); }); test.describe("IdpRoleSyncPage", () => { + test.describe.configure({ retries: 1 }); + test("show empty table when no role mappings are present", async ({ page, }) => { - requiresLicense(); const org = await createOrganizationWithName(randomName()); await page.goto(`/organizations/${org.name}/idp-sync?tab=roles`, { waitUntil: "domcontentloaded", @@ -36,7 +38,6 @@ test.describe("IdpRoleSyncPage", () => { }); test("add new IdP role mapping with API", async ({ page }) => { - requiresLicense(); const org = await createOrganizationWithName(randomName()); await createRoleSyncSettings(org.id); @@ -58,7 +59,6 @@ test.describe("IdpRoleSyncPage", () => { }); test("delete a IdP role to coder role mapping row", async ({ page }) => { - requiresLicense(); const org = await createOrganizationWithName(randomName()); await createRoleSyncSettings(org.id); @@ -79,7 +79,6 @@ test.describe("IdpRoleSyncPage", () => { }); test("update sync field", async ({ page }) => { - requiresLicense(); const org = await createOrganizationWithName(randomName()); await page.goto(`/organizations/${org.name}/idp-sync?tab=roles`, { waitUntil: "domcontentloaded", @@ -107,7 +106,6 @@ test.describe("IdpRoleSyncPage", () => { test("export policy button is enabled when sync settings are present", async ({ page, }) => { - requiresLicense(); const org = await createOrganizationWithName(randomName()); await page.goto(`/organizations/${org.name}/idp-sync?tab=roles`, { waitUntil: "domcontentloaded", @@ -121,7 +119,6 @@ test.describe("IdpRoleSyncPage", () => { }); test("add new IdP role mapping with UI", async ({ page }) => { - requiresLicense(); const orgName = randomName(); await createOrganizationWithName(orgName); @@ -130,18 +127,31 @@ test.describe("IdpRoleSyncPage", () => { }); const idpOrgInput = page.getByLabel("IdP role name"); - const roleSelector = page.getByPlaceholder("Select role"); const addButton = page.getByRole("button", { name: /Add IdP role/i, }); await expect(addButton).toBeDisabled(); - await idpOrgInput.fill("new-idp-role"); + const idpRoleName = randomName(); + await idpOrgInput.fill(idpRoleName); // Select Coder role from combobox + const roleSelector = page.getByPlaceholder("Select role"); + await expect(roleSelector).toBeAttached(); + await expect(roleSelector).toBeVisible(); await roleSelector.click(); - await page.getByRole("option", { name: /Organization Admin/i }).click(); + + await page.getByRole("combobox").click(); + await page.waitForTimeout(1000); + + const option = await page.getByRole("option", { + name: /Organization Admin/i, + }); + + await expect(option).toBeAttached({ timeout: 30000 }); + await expect(option).toBeVisible(); + await option.click(); // Add button should now be enabled await expect(addButton).toBeEnabled(); @@ -149,11 +159,9 @@ test.describe("IdpRoleSyncPage", () => { await addButton.click(); // Verify new mapping appears in table - const newRow = page.getByTestId("role-new-idp-role"); + const newRow = page.getByTestId(`role-${idpRoleName}`); await expect(newRow).toBeVisible(); - await expect( - newRow.getByRole("cell", { name: "new-idp-role" }), - ).toBeVisible(); + await expect(newRow.getByRole("cell", { name: idpRoleName })).toBeVisible(); await expect( newRow.getByRole("cell", { name: "organization-admin" }), ).toBeVisible();