mirror of
https://github.com/coder/coder.git
synced 2026-09-24 15:04:27 +08:00
feat: add harness engineering layer for agent workflows (#24791)
This PR adds an opinionated harness-engineering layer for agent-driven workflows: a small set of agent-readable docs, mechanical structure checks, structured CI failure summaries, an architecture-lint umbrella, and per-worktree dev-server isolation. The goal is to make local dev, tests, and CI mechanically inspectable by agents without changing app runtime behavior. ## What landed **Agent docs and navigation** - `.claude/docs/OBSERVABILITY.md`, `.claude/docs/DEV_ISOLATION.md`, `.claude/docs/AGENT_FAILURES.md`: task-oriented guides for logs, tracing, Prometheus, dev-server isolation, and a seeded failure catalog. - `AGENTS.md`: added an `Agent navigation` block, then trimmed the file from 375 to 229 lines by migrating duplicated detail into `WORKFLOWS.md`, `GO.md`, `TESTING.md`, and `DATABASE.md`. The user-managed custom-instructions block is preserved. - `.agents/docs`: symlink mirror of `.claude/docs` for agent runtimes that look under `.agents`. **Mechanical checks** - `scripts/check_agents_structure.sh`: validates `@...` references in tracked `AGENTS.md` files and warns when root grows past 600 lines. Wired as `make lint/agents` and into `make lint`. - `scripts/audit-agent-readiness.sh`: report-first audit of harness readiness. Currently `10 ok, 0 warn, 0 fail`. - `scripts/check_architecture.sh` / `make lint/architecture`: umbrella architecture-lint target. Consolidates the existing `check_enterprise_imports.sh` and `check_codersdk_imports.sh` so they run exactly once via the umbrella. Slot is open for new high-confidence rules. **Structured CI failure summaries** - `scripts/playwright-failure-summary.sh`: parses `site/test-results/results.json` and writes Markdown to `$GITHUB_STEP_SUMMARY` on failure. Wired into the `test-e2e` matrix job. - `scripts/go-test-failure-summary.sh`: parses `go test -json` line-delimited output the same way. Wired into `test-go-pg`, `test-go-pg-17`, and `test-go-race-pg` by injecting `gotestsum --jsonfile` in the workflow without touching `Makefile`. JSON also uploaded as a CI artifact on failure. - `site/e2e/playwright.config.ts`: enables `screenshot: only-on-failure`, `trace: retain-on-failure`, JSON reporter, and HTML reporter alongside existing reporters. - `.github/workflows/ci.yaml`: failure artifact uploads for Playwright now use `if: failure()` and predictable names (`playwright-artifacts-<variant>-<sha>`). **Per-worktree dev-server isolation** (`scripts/develop/main.go`) - Deterministic FNV-64a hash of the worktree path produces a port offset in `[0, 1000)` (50 buckets, step 20 to avoid API/proxy overlap across adjacent buckets). - Offset is applied only to defaults; both env vars (`CODER_DEV_PORT`, `CODER_DEV_WEB_PORT`, `CODER_DEV_PROXY_PORT`, `CODER_DEV_PROMETHEUS_PORT`) and CLI flags retain priority. - Hardcoded ports `9090` (embedded Prometheus UI) and `12345` (Delve) are unchanged by design. - Startup banner shows each port's source: `default`, `offset`, or `explicit`. - Unit tests in `scripts/develop/main_test.go` cover determinism, bounds, no-overlap across the four ports, and explicit-skip behavior. - State (`.coderv2/`) was already worktree-isolated via `os.Getwd()`, so no state-dir changes were needed. ## Validation `make lint/agents`, `make lint/architecture`, `make lint/emdash`, `bash scripts/audit-agent-readiness.sh` (10 ok, 0 warn, 0 fail), `shellcheck` on all 5 new scripts, `go test ./scripts/develop/...`, and `js-yaml` parse of `ci.yaml` all pass. Synthetic fixtures verify both failure-summary scripts handle empty/missing input (silent exit 0), ANSI-stripped output, and parent/subtest formatting. ## Known follow-ups (deferred) - Frontend Storybook/Vitest failure summary: lowest-leverage slice of the failure-summary work. Skipping until observed pain. - Architecture lint currently only delegates to existing import checks; new rules (`InTx` outer-store detection, swagger-annotation lint) plug in as needed. - 50 port-offset buckets means two worktree paths can occasionally collide. The DEV_ISOLATION doc tells users to set the relevant env var when this happens. > Mux opened this PR on Mike's behalf.
This commit is contained in:
Michael Suchacz
@@ -146,6 +146,127 @@ func TestShellBool(t *testing.T) {
|
||||
assert.Equal(t, "0", shellBool(false))
|
||||
}
|
||||
|
||||
func TestPortOffset(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
root := "/tmp/coder/worktree-a"
|
||||
offset := portOffset(root)
|
||||
assert.Equal(t, offset, portOffset(root))
|
||||
assert.GreaterOrEqual(t, offset, 0)
|
||||
assert.Less(t, offset, 1000)
|
||||
assert.Equal(t, 0, offset%10)
|
||||
|
||||
var foundDifferent bool
|
||||
for _, otherRoot := range []string{
|
||||
"/tmp/coder/worktree-b",
|
||||
"/tmp/coder/worktree-c",
|
||||
"/tmp/coder/worktree-d",
|
||||
} {
|
||||
if portOffset(otherRoot) != offset {
|
||||
foundDifferent = true
|
||||
break
|
||||
}
|
||||
}
|
||||
assert.True(t, foundDifferent, "expected typical worktree paths to use different offsets")
|
||||
}
|
||||
|
||||
func TestApplyPortOffsetSkipsExplicitPorts(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
projectRoot := "/tmp/coder/worktree-offset"
|
||||
for i := range 100 {
|
||||
candidate := fmt.Sprintf("/tmp/coder/worktree-offset-%d", i)
|
||||
if portOffset(candidate) != 0 {
|
||||
projectRoot = candidate
|
||||
break
|
||||
}
|
||||
}
|
||||
offset := portOffset(projectRoot)
|
||||
require.NotZero(t, offset)
|
||||
|
||||
cfg := &devConfig{
|
||||
apiPort: 3000,
|
||||
webPort: 8080,
|
||||
proxyPort: 3010,
|
||||
coderMetricsPort: 2114,
|
||||
portOffsetEnabled: true,
|
||||
projectRoot: projectRoot,
|
||||
portExplicit: portExplicit{
|
||||
web: true,
|
||||
metrics: true,
|
||||
},
|
||||
}
|
||||
cfg.applyPortOffset()
|
||||
|
||||
assert.Equal(t, int64(3000+offset), cfg.apiPort)
|
||||
assert.Equal(t, int64(8080), cfg.webPort)
|
||||
assert.Equal(t, int64(3010+offset), cfg.proxyPort)
|
||||
assert.Equal(t, int64(2114), cfg.coderMetricsPort)
|
||||
assert.Equal(t, portSourceOffset, cfg.apiPortSource)
|
||||
assert.Equal(t, portSourceExplicit, cfg.webPortSource)
|
||||
assert.Equal(t, portSourceOffset, cfg.proxyPortSource)
|
||||
assert.Equal(t, portSourceExplicit, cfg.metricsPortSource)
|
||||
}
|
||||
|
||||
func TestApplyPortOffsetDisabledUsesDefaultPorts(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
projectRoot := "/tmp/coder/worktree-offset"
|
||||
for i := range 100 {
|
||||
candidate := fmt.Sprintf("/tmp/coder/worktree-offset-disabled-%d", i)
|
||||
if portOffset(candidate) != 0 {
|
||||
projectRoot = candidate
|
||||
break
|
||||
}
|
||||
}
|
||||
require.NotZero(t, portOffset(projectRoot))
|
||||
|
||||
cfg := &devConfig{
|
||||
apiPort: 3000,
|
||||
webPort: 8080,
|
||||
proxyPort: 3010,
|
||||
coderMetricsPort: 2114,
|
||||
projectRoot: projectRoot,
|
||||
}
|
||||
cfg.applyPortOffset()
|
||||
|
||||
assert.Equal(t, int64(3000), cfg.apiPort)
|
||||
assert.Equal(t, int64(8080), cfg.webPort)
|
||||
assert.Equal(t, int64(3010), cfg.proxyPort)
|
||||
assert.Equal(t, int64(2114), cfg.coderMetricsPort)
|
||||
assert.Zero(t, cfg.portOffset)
|
||||
assert.Empty(t, cfg.apiPortSource)
|
||||
assert.Empty(t, cfg.webPortSource)
|
||||
assert.Empty(t, cfg.proxyPortSource)
|
||||
assert.Empty(t, cfg.metricsPortSource)
|
||||
assert.Equal(t, "API: 3000", portBannerLine("API", cfg.apiPort, cfg.apiPortSource, cfg.portOffset))
|
||||
}
|
||||
|
||||
func TestPortOffsetDefaultPortsDoNotOverlap(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
ports := []struct {
|
||||
name string
|
||||
base int
|
||||
}{
|
||||
{name: "API", base: 3000},
|
||||
{name: "Web UI", base: 8080},
|
||||
{name: "Proxy", base: 3010},
|
||||
{name: "Coder metrics", base: 2114},
|
||||
}
|
||||
seen := make(map[int]string)
|
||||
for bucket := range portOffsetBuckets {
|
||||
offset := bucket * portOffsetStep
|
||||
for _, port := range ports {
|
||||
effective := port.base + offset
|
||||
if other, ok := seen[effective]; ok {
|
||||
t.Fatalf("%s collides with %s on port %d", port.name, other, effective)
|
||||
}
|
||||
seen[effective] = fmt.Sprintf("%s with offset %d", port.name, offset)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestDevelopInCoder(t *testing.T) {
|
||||
t.Run("DEVELOP_IN_CODER", func(t *testing.T) {
|
||||
t.Setenv("DEVELOP_IN_CODER", "1")
|
||||
@@ -431,15 +552,17 @@ func TestDevConfigResolveEnv(t *testing.T) {
|
||||
t.Setenv("CODER_SESSION_TOKEN", "leaked")
|
||||
t.Setenv("CODER_URL", "https://leaked.example.com")
|
||||
|
||||
wd, _ := os.Getwd()
|
||||
cfg := &devConfig{apiPort: 3000, accessURL: defaultAccessURL}
|
||||
require.NoError(t, cfg.resolveEnv())
|
||||
|
||||
wd, _ := os.Getwd()
|
||||
assert.Equal(t, wd, cfg.projectRoot)
|
||||
assert.Equal(t, filepath.Join(wd, "build",
|
||||
fmt.Sprintf("coder_%s_%s", runtime.GOOS, runtime.GOARCH)), cfg.binaryPath)
|
||||
assert.Equal(t, filepath.Join(wd, ".coderv2"), cfg.configDir)
|
||||
assert.Equal(t, "http://127.0.0.1:3000", cfg.accessURL)
|
||||
assert.Equal(t, int64(3000), cfg.apiPort)
|
||||
assert.Zero(t, cfg.portOffset)
|
||||
|
||||
// Should have unset leaked env vars.
|
||||
assert.Empty(t, os.Getenv("CODER_SESSION_TOKEN"))
|
||||
@@ -454,11 +577,92 @@ func TestDevConfigResolveEnv(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestDevConfigResolveEnvUsesDefaultPortsWithoutPortOffset(t *testing.T) {
|
||||
t.Setenv("CODER_SESSION_TOKEN", "")
|
||||
t.Setenv("CODER_URL", "")
|
||||
|
||||
baseRoot := t.TempDir()
|
||||
projectRoot := filepath.Join(baseRoot, "worktree")
|
||||
for i := range 100 {
|
||||
candidate := filepath.Join(baseRoot, fmt.Sprintf("worktree-default-%d", i))
|
||||
if portOffset(candidate) != 0 {
|
||||
projectRoot = candidate
|
||||
break
|
||||
}
|
||||
}
|
||||
require.NotZero(t, portOffset(projectRoot))
|
||||
require.NoError(t, os.MkdirAll(projectRoot, 0o755))
|
||||
t.Chdir(projectRoot)
|
||||
|
||||
cfg := &devConfig{
|
||||
apiPort: 3000,
|
||||
webPort: 8080,
|
||||
proxyPort: 3010,
|
||||
coderMetricsPort: 2114,
|
||||
accessURL: defaultAccessURL,
|
||||
}
|
||||
require.NoError(t, cfg.resolveEnv())
|
||||
|
||||
assert.Equal(t, projectRoot, cfg.projectRoot)
|
||||
assert.Equal(t, int64(3000), cfg.apiPort)
|
||||
assert.Equal(t, int64(8080), cfg.webPort)
|
||||
assert.Equal(t, int64(3010), cfg.proxyPort)
|
||||
assert.Equal(t, int64(2114), cfg.coderMetricsPort)
|
||||
assert.Zero(t, cfg.portOffset)
|
||||
assert.Empty(t, cfg.apiPortSource)
|
||||
assert.Empty(t, cfg.webPortSource)
|
||||
assert.Empty(t, cfg.proxyPortSource)
|
||||
assert.Empty(t, cfg.metricsPortSource)
|
||||
assert.Equal(t, "http://127.0.0.1:3000", cfg.accessURL)
|
||||
}
|
||||
|
||||
func TestDevConfigResolveEnvAppliesPortOffsetWhenEnabled(t *testing.T) {
|
||||
t.Setenv("CODER_SESSION_TOKEN", "")
|
||||
t.Setenv("CODER_URL", "")
|
||||
|
||||
baseRoot := t.TempDir()
|
||||
projectRoot := filepath.Join(baseRoot, "worktree")
|
||||
for i := range 100 {
|
||||
candidate := filepath.Join(baseRoot, fmt.Sprintf("worktree-%d", i))
|
||||
if portOffset(candidate) != 0 {
|
||||
projectRoot = candidate
|
||||
break
|
||||
}
|
||||
}
|
||||
require.NotZero(t, portOffset(projectRoot))
|
||||
require.NoError(t, os.MkdirAll(projectRoot, 0o755))
|
||||
t.Chdir(projectRoot)
|
||||
|
||||
cfg := &devConfig{
|
||||
apiPort: 3000,
|
||||
webPort: 8080,
|
||||
proxyPort: 3010,
|
||||
coderMetricsPort: 2114,
|
||||
portOffsetEnabled: true,
|
||||
accessURL: defaultAccessURL,
|
||||
}
|
||||
require.NoError(t, cfg.resolveEnv())
|
||||
|
||||
offset := portOffset(projectRoot)
|
||||
assert.Equal(t, projectRoot, cfg.projectRoot)
|
||||
assert.Equal(t, int64(3000+offset), cfg.apiPort)
|
||||
assert.Equal(t, int64(8080+offset), cfg.webPort)
|
||||
assert.Equal(t, int64(3010+offset), cfg.proxyPort)
|
||||
assert.Equal(t, int64(2114+offset), cfg.coderMetricsPort)
|
||||
assert.Equal(t, offset, cfg.portOffset)
|
||||
assert.Equal(t, portSourceOffset, cfg.apiPortSource)
|
||||
assert.Equal(t, fmt.Sprintf("http://127.0.0.1:%d", 3000+offset), cfg.accessURL)
|
||||
}
|
||||
|
||||
func TestDevConfigResolveEnvExplicitAccessURL(t *testing.T) {
|
||||
t.Setenv("CODER_SESSION_TOKEN", "")
|
||||
t.Setenv("CODER_URL", "")
|
||||
|
||||
cfg := &devConfig{apiPort: 5000, accessURL: "http://myhost:5000"}
|
||||
cfg := &devConfig{
|
||||
apiPort: 5000,
|
||||
accessURL: "http://myhost:5000",
|
||||
portExplicit: portExplicit{api: true},
|
||||
}
|
||||
require.NoError(t, cfg.resolveEnv())
|
||||
assert.Equal(t, "http://myhost:5000", cfg.accessURL)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user