test(e2e): improve auth coverage and authoring support (#34920)

e2e/AGENTS.md

@@ -165,3 +165,132 @@ Open the HTML report locally with:
 ```bash
 open cucumber-report/report.html
 ```
+
+## Writing new scenarios
+
+### Workflow
+
+1. Create a `.feature` file under `features/<capability>/`
+2. Add step definitions under `features/step-definitions/<capability>/`
+3. Reuse existing steps from `common/` and other definition files before writing new ones
+4. Run with `pnpm -C e2e e2e -- --tags @your-tag` to verify
+5. Run `pnpm -C e2e check` before committing
+
+### Feature file conventions
+
+Tag every feature or scenario with a capability tag. Add auth tags only when they clarify intent or change the browser session behavior:
+
+```gherkin
+@datasets @authenticated
+Feature: Create dataset
+  Scenario: Create a new empty dataset
+    Given I am signed in as the default E2E admin
+    When I open the datasets page
+    ...
+```
+
+- Capability tags (`@apps`, `@auth`, `@datasets`, …) group related scenarios for selective runs
+- Auth/session tags:
+  - default behavior — scenarios run with the shared authenticated storageState unless marked otherwise
+  - `@unauthenticated` — uses a clean BrowserContext with no cookies or storage
+  - `@authenticated` — optional intent tag for readability or selective runs; it does not currently change hook behavior on its own
+- `@fresh` — only runs in `e2e:full` mode (requires uninitialized instance)
+- `@skip` — excluded from all runs
+
+Keep scenarios short and declarative. Each step should describe **what** the user does, not **how** the UI works.
+
+### Step definition conventions
+
+```typescript
+import { When, Then } from '@cucumber/cucumber'
+import { expect } from '@playwright/test'
+import type { DifyWorld } from '../../support/world'
+
+When('I open the datasets page', async function (this: DifyWorld) {
+  await this.getPage().goto('/datasets')
+})
+```
+
+Rules:
+
+- Always type `this` as `DifyWorld` for proper context access
+- Use `async function` (not arrow functions — Cucumber binds `this`)
+- One step = one user-visible action or one assertion
+- Keep steps stateless across scenarios; use `DifyWorld` properties for in-scenario state
+
+### Locator priority
+
+Follow the Playwright recommended locator strategy, in order of preference:
+
+| Priority | Locator            | Example                                   | When to use                               |
+| -------- | ------------------ | ----------------------------------------- | ----------------------------------------- |
+| 1        | `getByRole`        | `getByRole('button', { name: 'Create' })` | Default choice — accessible and resilient |
+| 2        | `getByLabel`       | `getByLabel('App name')`                  | Form inputs with visible labels           |
+| 3        | `getByPlaceholder` | `getByPlaceholder('Enter name')`          | Inputs without visible labels             |
+| 4        | `getByText`        | `getByText('Welcome')`                    | Static text content                       |
+| 5        | `getByTestId`      | `getByTestId('workflow-canvas')`          | Only when no semantic locator works       |
+
+Avoid raw CSS/XPath selectors. They break when the DOM structure changes.
+
+### Assertions
+
+Use `@playwright/test` `expect` — it auto-waits and retries until the condition is met or the timeout expires:
+
+```typescript
+// URL assertion
+await expect(page).toHaveURL(/\/datasets\/[a-f0-9-]+\/documents/)
+
+// Element visibility
+await expect(page.getByRole('button', { name: 'Save' })).toBeVisible()
+
+// Element state
+await expect(page.getByRole('button', { name: 'Submit' })).toBeEnabled()
+
+// Negation
+await expect(page.getByText('Loading')).not.toBeVisible()
+```
+
+Do not use manual `waitForTimeout` or polling loops. If you need a longer wait for a specific assertion, pass `{ timeout: 30_000 }` to the assertion.
+
+### Cucumber expressions
+
+Use Cucumber expression parameter types to extract values from Gherkin steps:
+
+| Type       | Pattern       | Example step                       |
+| ---------- | ------------- | ---------------------------------- |
+| `{string}` | Quoted string | `I select the "Workflow" app type` |
+| `{int}`    | Integer       | `I should see {int} items`         |
+| `{float}`  | Decimal       | `the progress is {float} percent`  |
+| `{word}`   | Single word   | `I click the {word} tab`           |
+
+Prefer `{string}` for UI labels, names, and text content — it maps naturally to Gherkin's quoted values.
+
+### Scoping locators
+
+When the page has multiple similar elements, scope locators to a container:
+
+```typescript
+When('I fill in the app name in the dialog', async function (this: DifyWorld) {
+  const dialog = this.getPage().getByRole('dialog')
+  await dialog.getByPlaceholder('Give your app a name').fill('My App')
+})
+```
+
+### Failure diagnostics
+
+The `After` hook automatically captures on failure:
+
+- Full-page screenshot (PNG)
+- Page HTML dump
+- Console errors and page errors
+
+Artifacts are saved to `cucumber-report/artifacts/` and attached to the HTML report. No extra code needed in step definitions.
+
+## Reusing existing steps
+
+Before writing a new step definition, inspect the existing step definition files first. Reuse a matching step when the wording and behavior already fit, and only add a new step when the scenario needs a genuinely new user action or assertion. Steps in `common/` are designed for broad reuse across all features.
+
+Or browse the step definition files directly:
+
+- `features/step-definitions/common/` — auth guards and navigation assertions shared by all features
+- `features/step-definitions/<capability>/` — domain-specific steps scoped to a single feature area

e2e/features/auth/sign-out.feature

@@ -6,3 +6,13 @@ Feature: Sign out
     And I open the account menu
     And I sign out
     Then I should be on the sign-in page
+
+  Scenario: Redirect back to sign-in when reopening the apps console after signing out
+    Given I am signed in as the default E2E admin
+    When I open the apps console
+    And I open the account menu
+    And I sign out
+    Then I should be on the sign-in page
+    When I open the apps console
+    Then I should be redirected to the signin page
+    And I should see the "Sign in" button

e2e/features/support/hooks.ts

@@ -3,7 +3,7 @@ import { chromium, type Browser } from '@playwright/test'
 import { mkdir, writeFile } from 'node:fs/promises'
 import path from 'node:path'
 import { fileURLToPath } from 'node:url'
-import { ensureAuthenticatedState } from '../../fixtures/auth'
+import { AUTH_BOOTSTRAP_TIMEOUT_MS, ensureAuthenticatedState } from '../../fixtures/auth'
 import { baseURL, cucumberHeadless, cucumberSlowMo } from '../../test-env'
 import type { DifyWorld } from './world'
 
@@ -31,7 +31,7 @@ const writeArtifact = async (
   return artifactPath
 }
 
-BeforeAll(async () => {
+BeforeAll({ timeout: AUTH_BOOTSTRAP_TIMEOUT_MS }, async () => {
   await mkdir(artifactsDir, { recursive: true })
 
   browser = await chromium.launch({

e2e/fixtures/auth.ts

@@ -12,7 +12,7 @@ export type AuthSessionMetadata = {
   usedInitPassword: boolean
 }
 
-const WAIT_TIMEOUT_MS = 120_000
+export const AUTH_BOOTSTRAP_TIMEOUT_MS = 120_000
 const e2eRoot = fileURLToPath(new URL('..', import.meta.url))
 
 export const authDir = path.join(e2eRoot, '.auth')
@@ -39,40 +39,54 @@ const escapeRegex = (value: string) => value.replaceAll(/[.*+?^${}()|[\]\\]/g, '
 
 const appURL = (baseURL: string, pathname: string) => new URL(pathname, baseURL).toString()
 
-const waitForPageState = async (page: Page) => {
+type AuthPageState = 'install' | 'login' | 'init'
+
+const getRemainingTimeout = (deadline: number) => Math.max(deadline - Date.now(), 1)
+
+const waitForPageState = async (page: Page, deadline: number): Promise<AuthPageState> => {
   const installHeading = page.getByRole('heading', { name: 'Setting up an admin account' })
   const signInButton = page.getByRole('button', { name: 'Sign in' })
   const initPasswordField = page.getByLabel('Admin initialization password')
 
-  const deadline = Date.now() + WAIT_TIMEOUT_MS
-
-  while (Date.now() < deadline) {
-    if (await installHeading.isVisible().catch(() => false)) return 'install' as const
-    if (await signInButton.isVisible().catch(() => false)) return 'login' as const
-    if (await initPasswordField.isVisible().catch(() => false)) return 'init' as const
-
-    await page.waitForTimeout(1_000)
+  try {
+    return await Promise.any<AuthPageState>([
+      installHeading
+        .waitFor({ state: 'visible', timeout: getRemainingTimeout(deadline) })
+        .then(() => 'install'),
+      signInButton
+        .waitFor({ state: 'visible', timeout: getRemainingTimeout(deadline) })
+        .then(() => 'login'),
+      initPasswordField
+        .waitFor({ state: 'visible', timeout: getRemainingTimeout(deadline) })
+        .then(() => 'init'),
+    ])
+  } catch {
+    throw new Error(`Unable to determine auth page state for ${page.url()}`)
   }
-
-  throw new Error(`Unable to determine auth page state for ${page.url()}`)
 }
 
-const completeInitPasswordIfNeeded = async (page: Page) => {
+const completeInitPasswordIfNeeded = async (page: Page, deadline: number) => {
   const initPasswordField = page.getByLabel('Admin initialization password')
-  if (!(await initPasswordField.isVisible({ timeout: 3_000 }).catch(() => false))) return false
+
+  const needsInitPassword = await initPasswordField
+    .waitFor({ state: 'visible', timeout: Math.min(getRemainingTimeout(deadline), 3_000) })
+    .then(() => true)
+    .catch(() => false)
+
+  if (!needsInitPassword) return false
 
   await initPasswordField.fill(initPassword)
   await page.getByRole('button', { name: 'Validate' }).click()
   await expect(page.getByRole('heading', { name: 'Setting up an admin account' })).toBeVisible({
-    timeout: WAIT_TIMEOUT_MS,
+    timeout: getRemainingTimeout(deadline),
   })
 
   return true
 }
 
-const completeInstall = async (page: Page, baseURL: string) => {
+const completeInstall = async (page: Page, baseURL: string, deadline: number) => {
   await expect(page.getByRole('heading', { name: 'Setting up an admin account' })).toBeVisible({
-    timeout: WAIT_TIMEOUT_MS,
+    timeout: getRemainingTimeout(deadline),
   })
 
   await page.getByLabel('Email address').fill(adminCredentials.email)
@@ -81,13 +95,13 @@ const completeInstall = async (page: Page, baseURL: string) => {
   await page.getByRole('button', { name: 'Set up' }).click()
 
   await expect(page).toHaveURL(new RegExp(`^${escapeRegex(baseURL)}/apps(?:\\?.*)?$`), {
-    timeout: WAIT_TIMEOUT_MS,
+    timeout: getRemainingTimeout(deadline),
   })
 }
 
-const completeLogin = async (page: Page, baseURL: string) => {
+const completeLogin = async (page: Page, baseURL: string, deadline: number) => {
   await expect(page.getByRole('button', { name: 'Sign in' })).toBeVisible({
-    timeout: WAIT_TIMEOUT_MS,
+    timeout: getRemainingTimeout(deadline),
   })
 
   await page.getByLabel('Email address').fill(adminCredentials.email)
@@ -95,12 +109,13 @@ const completeLogin = async (page: Page, baseURL: string) => {
   await page.getByRole('button', { name: 'Sign in' }).click()
 
   await expect(page).toHaveURL(new RegExp(`^${escapeRegex(baseURL)}/apps(?:\\?.*)?$`), {
-    timeout: WAIT_TIMEOUT_MS,
+    timeout: getRemainingTimeout(deadline),
   })
 }
 
 export const ensureAuthenticatedState = async (browser: Browser, configuredBaseURL?: string) => {
   const baseURL = resolveBaseURL(configuredBaseURL)
+  const deadline = Date.now() + AUTH_BOOTSTRAP_TIMEOUT_MS
 
   await mkdir(authDir, { recursive: true })
 
@@ -111,25 +126,28 @@ export const ensureAuthenticatedState = async (browser: Browser, configuredBaseU
   const page = await context.newPage()
 
   try {
-    await page.goto(appURL(baseURL, '/install'), { waitUntil: 'networkidle' })
+    await page.goto(appURL(baseURL, '/install'), {
+      timeout: getRemainingTimeout(deadline),
+      waitUntil: 'domcontentloaded',
+    })
 
-    let usedInitPassword = await completeInitPasswordIfNeeded(page)
-    let pageState = await waitForPageState(page)
+    let usedInitPassword = await completeInitPasswordIfNeeded(page, deadline)
+    let pageState = await waitForPageState(page, deadline)
 
     while (pageState === 'init') {
-      const completedInitPassword = await completeInitPasswordIfNeeded(page)
+      const completedInitPassword = await completeInitPasswordIfNeeded(page, deadline)
       if (!completedInitPassword)
         throw new Error(`Unable to validate initialization password for ${page.url()}`)
 
       usedInitPassword = true
-      pageState = await waitForPageState(page)
+      pageState = await waitForPageState(page, deadline)
     }
 
-    if (pageState === 'install') await completeInstall(page, baseURL)
-    else await completeLogin(page, baseURL)
+    if (pageState === 'install') await completeInstall(page, baseURL, deadline)
+    else await completeLogin(page, baseURL, deadline)
 
     await expect(page.getByRole('button', { name: 'Create from Blank' })).toBeVisible({
-      timeout: WAIT_TIMEOUT_MS,
+      timeout: getRemainingTimeout(deadline),
     })
 
     await context.storageState({ path: authStatePath })

e2e/scripts/common.ts

@@ -1,4 +1,5 @@
 import { spawn, type ChildProcess } from 'node:child_process'
+import { createHash } from 'node:crypto'
 import { access, copyFile, readFile, writeFile } from 'node:fs/promises'
 import net from 'node:net'
 import path from 'node:path'
@@ -38,6 +39,10 @@ export const middlewareEnvExampleFile = path.join(dockerDir, 'middleware.env.exa
 export const webEnvLocalFile = path.join(webDir, '.env.local')
 export const webEnvExampleFile = path.join(webDir, '.env.example')
 export const apiEnvExampleFile = path.join(apiDir, 'tests', 'integration_tests', '.env.example')
+export const e2eWebEnvOverrides = {
+  NEXT_PUBLIC_API_PREFIX: 'http://127.0.0.1:5001/console/api',
+  NEXT_PUBLIC_PUBLIC_API_PREFIX: 'http://127.0.0.1:5001/api',
+} satisfies Record<string, string>
 
 const formatCommand = (command: string, args: string[]) => [command, ...args].join(' ')
 
@@ -166,13 +171,16 @@ export const ensureLineInFile = async (filePath: string, line: string) => {
   await writeFile(filePath, `${normalizedContent}${line}\n`, 'utf8')
 }
 
-export const ensureWebEnvLocal = async () => {
-  await ensureFileExists(webEnvLocalFile, webEnvExampleFile)
-
-  const fileContent = await readFile(webEnvLocalFile, 'utf8')
-  const nextContent = fileContent.replaceAll('http://localhost:5001', 'http://127.0.0.1:5001')
-
-  if (nextContent !== fileContent) await writeFile(webEnvLocalFile, nextContent, 'utf8')
+export const getWebEnvLocalHash = async () => {
+  const fileContent = await readFile(webEnvLocalFile, 'utf8').catch(() => '')
+  return createHash('sha256')
+    .update(
+      JSON.stringify({
+        envLocal: fileContent,
+        overrides: e2eWebEnvOverrides,
+      }),
+    )
+    .digest('hex')
 }
 
 export const readSimpleDotenv = async (filePath: string) => {

e2e/scripts/setup.ts

@@ -1,14 +1,15 @@
-import { access, mkdir, rm } from 'node:fs/promises'
+import { access, mkdir, readFile, rm, writeFile } from 'node:fs/promises'
 import path from 'node:path'
 import { waitForUrl } from '../support/process'
 import {
   apiDir,
   apiEnvExampleFile,
   dockerDir,
+  e2eWebEnvOverrides,
   e2eDir,
   ensureFileExists,
   ensureLineInFile,
-  ensureWebEnvLocal,
+  getWebEnvLocalHash,
   isMainModule,
   isTcpPortReachable,
   middlewareComposeFile,
@@ -23,6 +24,7 @@ import {
 } from './common'
 
 const buildIdPath = path.join(webDir, '.next', 'BUILD_ID')
+const webBuildEnvStampPath = path.join(webDir, '.next', 'e2e-web-env.sha256')
 
 const middlewareDataPaths = [
   path.join(dockerDir, 'volumes', 'db', 'data'),
@@ -110,27 +112,47 @@ const waitForDependency = async ({
 }
 
 export const ensureWebBuild = async () => {
-  await ensureWebEnvLocal()
+  const envHash = await getWebEnvLocalHash()
+  const buildEnv = {
+    ...e2eWebEnvOverrides,
+  }
 
   if (process.env.E2E_FORCE_WEB_BUILD === '1') {
     await runCommandOrThrow({
       command: 'pnpm',
       args: ['run', 'build'],
       cwd: webDir,
+      env: buildEnv,
     })
+    await writeFile(webBuildEnvStampPath, `${envHash}\n`, 'utf8')
     return
   }
 
   try {
-    await access(buildIdPath)
-    console.log('Reusing existing web build artifact.')
+    const [buildExists, previousEnvHash] = await Promise.all([
+      access(buildIdPath)
+        .then(() => true)
+        .catch(() => false),
+      readFile(webBuildEnvStampPath, 'utf8')
+        .then((value) => value.trim())
+        .catch(() => ''),
+    ])
+
+    if (buildExists && previousEnvHash === envHash) {
+      console.log('Reusing existing web build artifact.')
+      return
+    }
   } catch {
-    await runCommandOrThrow({
-      command: 'pnpm',
-      args: ['run', 'build'],
-      cwd: webDir,
-    })
+    // Fall through to rebuild when the existing build cannot be verified.
   }
+
+  await runCommandOrThrow({
+    command: 'pnpm',
+    args: ['run', 'build'],
+    cwd: webDir,
+    env: buildEnv,
+  })
+  await writeFile(webBuildEnvStampPath, `${envHash}\n`, 'utf8')
 }
 
 export const startWeb = async () => {
@@ -141,6 +163,7 @@ export const startWeb = async () => {
     args: ['run', 'start'],
     cwd: webDir,
     env: {
+      ...e2eWebEnvOverrides,
       HOSTNAME: '127.0.0.1',
       PORT: '3000',
     },
@@ -152,14 +175,25 @@ export const startApi = async () => {
 
   await runCommandOrThrow({
     command: 'uv',
-    args: ['run', '--project', '.', 'flask', 'upgrade-db'],
+    args: ['run', '--project', '.', '--no-sync', 'flask', 'upgrade-db'],
     cwd: apiDir,
     env,
   })
 
   await runForegroundProcess({
     command: 'uv',
-    args: ['run', '--project', '.', 'flask', 'run', '--host', '127.0.0.1', '--port', '5001'],
+    args: [
+      'run',
+      '--project',
+      '.',
+      '--no-sync',
+      'flask',
+      'run',
+      '--host',
+      '127.0.0.1',
+      '--port',
+      '5001',
+    ],
     cwd: apiDir,
     env,
   })