feat: report clicks on the workspace build debug action to telemetry (#29853)

This commit is contained in:
david-fraley
2026-09-25 13:59:53 -05:00
committed by GitHub
parent 6f0c4474ee
commit 282365b23a
13 changed files with 500 additions and 8 deletions
+56
View File
@@ -15100,6 +15100,49 @@ const docTemplate = `{
]
}
},
"/api/v2/workspacebuilds/{workspacebuild}/debug-events": {
"post": {
"consumes": [
"application/json"
],
"tags": [
"Builds"
],
"summary": "Report a workspace build debug click",
"operationId": "report-a-workspace-build-debug-click",
"parameters": [
{
"type": "string",
"description": "Workspace build ID",
"name": "workspacebuild",
"in": "path",
"required": true
},
{
"description": "Debug event",
"name": "request",
"in": "body",
"required": true,
"schema": {
"$ref": "#/definitions/codersdk.WorkspaceBuildDebugEventRequest"
}
}
],
"responses": {
"204": {
"description": "No Content"
}
},
"security": [
{
"CoderSessionToken": []
}
],
"x-apidocgen": {
"skip": true
}
}
},
"/api/v2/workspacebuilds/{workspacebuild}/logs": {
"get": {
"produces": [
@@ -32476,6 +32519,19 @@ const docTemplate = `{
}
}
},
"codersdk.WorkspaceBuildDebugEventRequest": {
"type": "object",
"required": [
"id"
],
"properties": {
"id": {
"description": "ID identifies this click so a later step of the funnel can be\nattributed to it.",
"type": "string",
"format": "uuid"
}
}
},
"codersdk.WorkspaceBuildParameter": {
"type": "object",
"properties": {
+50
View File
@@ -13433,6 +13433,45 @@
]
}
},
"/api/v2/workspacebuilds/{workspacebuild}/debug-events": {
"post": {
"consumes": ["application/json"],
"tags": ["Builds"],
"summary": "Report a workspace build debug click",
"operationId": "report-a-workspace-build-debug-click",
"parameters": [
{
"type": "string",
"description": "Workspace build ID",
"name": "workspacebuild",
"in": "path",
"required": true
},
{
"description": "Debug event",
"name": "request",
"in": "body",
"required": true,
"schema": {
"$ref": "#/definitions/codersdk.WorkspaceBuildDebugEventRequest"
}
}
],
"responses": {
"204": {
"description": "No Content"
}
},
"security": [
{
"CoderSessionToken": []
}
],
"x-apidocgen": {
"skip": true
}
}
},
"/api/v2/workspacebuilds/{workspacebuild}/logs": {
"get": {
"produces": ["application/json"],
@@ -29982,6 +30021,17 @@
}
}
},
"codersdk.WorkspaceBuildDebugEventRequest": {
"type": "object",
"required": ["id"],
"properties": {
"id": {
"description": "ID identifies this click so a later step of the funnel can be\nattributed to it.",
"type": "string",
"format": "uuid"
}
}
},
"codersdk.WorkspaceBuildParameter": {
"type": "object",
"properties": {
+1
View File
@@ -1853,6 +1853,7 @@ func New(options *Options) *API {
)
r.Get("/", api.workspaceBuild)
r.Patch("/cancel", api.patchCancelWorkspaceBuild)
r.Post("/debug-events", api.postWorkspaceBuildDebugEvent)
r.Get("/logs", api.workspaceBuildLogs)
r.Get("/parameters", api.workspaceBuildParameters)
r.Get("/resources", api.workspaceBuildResourcesDeprecated)
+22
View File
@@ -1508,6 +1508,7 @@ type Snapshot struct {
UserSecretsSummary *UserSecretsSummary `json:"user_secrets_summary"`
TemplateBuilderSessions []TemplateBuilderSession `json:"template_builder_sessions"`
PremiumFunnelEvents []PremiumFunnelEvent `json:"premium_funnel_events"`
WorkspaceBuildDebugEvents []WorkspaceBuildDebugEvent `json:"workspace_build_debug_events"`
}
// Deployment contains information about the host running Coder.
@@ -2542,6 +2543,27 @@ type PremiumFunnelEvent struct {
CreatedAt time.Time `json:"created_at"`
}
// Steps of the failed workspace build debug funnel. These are set by coderd
// rather than the client so that a step cannot be forged.
const (
WorkspaceBuildDebugEventClick = "click"
)
// WorkspaceBuildDebugEvent tracks a click on the "Debug with Coder Agents"
// action shown for a failed workspace build. The workspace and build fields
// come from the build itself so the workspace's later builds can be joined to
// the click.
type WorkspaceBuildDebugEvent struct {
ID uuid.UUID `json:"id"`
EventType string `json:"event_type"`
UserID uuid.UUID `json:"user_id"`
WorkspaceID uuid.UUID `json:"workspace_id"`
WorkspaceBuildID uuid.UUID `json:"workspace_build_id"`
Transition string `json:"transition"`
Reason string `json:"reason"`
CreatedAt time.Time `json:"created_at"`
}
func ConvertAIBridgeInterceptionsSummary(endTime time.Time, provider, model, client string, summary database.CalculateAIBridgeInterceptionsTelemetrySummaryRow) AIBridgeInterceptionsSummary {
return AIBridgeInterceptionsSummary{
ID: uuid.New(),
+166
View File
@@ -0,0 +1,166 @@
package coderd_test
import (
"context"
"net/http"
"testing"
"github.com/google/uuid"
"github.com/stretchr/testify/require"
"golang.org/x/xerrors"
"github.com/coder/coder/v2/coderd/coderdtest"
"github.com/coder/coder/v2/coderd/database"
"github.com/coder/coder/v2/coderd/database/dbfake"
"github.com/coder/coder/v2/coderd/rbac"
"github.com/coder/coder/v2/coderd/rbac/policy"
"github.com/coder/coder/v2/coderd/telemetry"
"github.com/coder/coder/v2/codersdk"
"github.com/coder/coder/v2/testutil"
)
func TestWorkspaceBuildDebugEvent(t *testing.T) {
t.Parallel()
t.Run("AdminClicksMembersFailedBuild", func(t *testing.T) {
t.Parallel()
ctx := testutil.Context(t, testutil.WaitMedium)
fTelemetry := newFakeTelemetryReporter(ctx, t, 10)
client, db := coderdtest.NewWithDatabase(t, &coderdtest.Options{
TelemetryReporter: fTelemetry,
DeploymentValues: coderdtest.DeploymentValues(t, func(values *codersdk.DeploymentValues) {
values.Experiments = []string{string(codersdk.ExperimentEnableAIWorkspaceDebug)}
}),
})
user := coderdtest.CreateFirstUser(t, client)
_, member := coderdtest.CreateAnotherUser(t, client, user.OrganizationID)
r := dbfake.WorkspaceBuild(t, db, database.WorkspaceTable{
OrganizationID: user.OrganizationID,
OwnerID: member.ID,
}).Seed(database.WorkspaceBuild{
Transition: database.WorkspaceTransitionStop,
Reason: database.BuildReasonAutostop,
}).Failed().Do()
eventID := uuid.New()
err := client.ReportWorkspaceBuildDebugClick(ctx, r.Build.ID, codersdk.WorkspaceBuildDebugEventRequest{
ID: eventID,
})
require.NoError(t, err)
event := receiveWorkspaceBuildDebugEvent(ctx, t, fTelemetry)
require.Equal(t, eventID, event.ID)
require.Equal(t, telemetry.WorkspaceBuildDebugEventClick, event.EventType)
require.Equal(t, user.UserID, event.UserID)
require.Equal(t, r.Workspace.ID, event.WorkspaceID)
require.Equal(t, r.Build.ID, event.WorkspaceBuildID)
require.Equal(t, string(database.WorkspaceTransitionStop), event.Transition)
require.Equal(t, string(database.BuildReasonAutostop), event.Reason)
require.False(t, event.CreatedAt.IsZero())
})
for _, tt := range []struct {
name string
experimentDisabled bool
chatDenied bool
otherUsersBuild bool
missingID bool
jobStatus database.ProvisionerJobStatus
wantStatus int
}{
{name: "MissingID", missingID: true, jobStatus: database.ProvisionerJobStatusFailed, wantStatus: http.StatusBadRequest},
{name: "ExperimentDisabled", experimentDisabled: true, jobStatus: database.ProvisionerJobStatusFailed, wantStatus: http.StatusNotFound},
{name: "ChatCreateDenied", chatDenied: true, jobStatus: database.ProvisionerJobStatusFailed, wantStatus: http.StatusForbidden},
{name: "OtherUsersBuild", otherUsersBuild: true, jobStatus: database.ProvisionerJobStatusFailed, wantStatus: http.StatusNotFound},
{name: "SucceededBuild", jobStatus: database.ProvisionerJobStatusSucceeded, wantStatus: http.StatusBadRequest},
{name: "RunningBuild", jobStatus: database.ProvisionerJobStatusRunning, wantStatus: http.StatusBadRequest},
{name: "PendingBuild", jobStatus: database.ProvisionerJobStatusPending, wantStatus: http.StatusBadRequest},
{name: "CanceledBuild", jobStatus: database.ProvisionerJobStatusCanceled, wantStatus: http.StatusBadRequest},
} {
t.Run(tt.name, func(t *testing.T) {
t.Parallel()
ctx := testutil.Context(t, testutil.WaitMedium)
fTelemetry := newFakeTelemetryReporter(ctx, t, 10)
options := &coderdtest.Options{
TelemetryReporter: fTelemetry,
DeploymentValues: coderdtest.DeploymentValues(t, func(values *codersdk.DeploymentValues) {
if !tt.experimentDisabled {
values.Experiments = []string{string(codersdk.ExperimentEnableAIWorkspaceDebug)}
}
}),
}
if tt.chatDenied {
options.Authorizer = &coderdtest.FakeAuthorizer{
ConditionalReturn: func(_ context.Context, _ rbac.Subject, action policy.Action, object rbac.Object) error {
if action == policy.ActionCreate && object.Type == rbac.ResourceChat.Type {
return xerrors.New("chat creation denied")
}
return nil
},
}
}
client, db := coderdtest.NewWithDatabase(t, options)
user := coderdtest.CreateFirstUser(t, client)
build := dbfake.WorkspaceBuild(t, db, database.WorkspaceTable{
OrganizationID: user.OrganizationID,
OwnerID: user.UserID,
})
switch tt.jobStatus {
case database.ProvisionerJobStatusFailed:
build = build.Failed()
case database.ProvisionerJobStatusSucceeded:
build = build.Succeeded()
case database.ProvisionerJobStatusRunning:
build = build.Starting()
case database.ProvisionerJobStatusPending, database.ProvisionerJobStatusCanceled:
build = build.Pending()
default:
t.Fatalf("unexpected job status %q", tt.jobStatus)
}
r := build.Do()
if tt.jobStatus == database.ProvisionerJobStatusCanceled {
require.NoError(t, client.CancelWorkspaceBuild(ctx, r.Build.ID, codersdk.CancelWorkspaceBuildParams{}))
}
if tt.otherUsersBuild {
client, _ = coderdtest.CreateAnotherUser(t, client, user.OrganizationID)
}
req := codersdk.WorkspaceBuildDebugEventRequest{ID: uuid.New()}
if tt.missingID {
req.ID = uuid.Nil
}
err := client.ReportWorkspaceBuildDebugClick(ctx, r.Build.ID, req)
require.Error(t, err)
var sdkErr *codersdk.Error
require.ErrorAs(t, err, &sdkErr)
require.Equal(t, tt.wantStatus, sdkErr.StatusCode())
// Reports are synchronous, so any event emitted by this request
// is already buffered when the response arrives.
for {
select {
case snapshot := <-fTelemetry.snapshots:
require.Empty(t, snapshot.WorkspaceBuildDebugEvents)
default:
return
}
}
})
}
}
// receiveWorkspaceBuildDebugEvent drains snapshots until one carries a debug
// event. Creating the first user reports its own snapshots on the same channel.
func receiveWorkspaceBuildDebugEvent(ctx context.Context, t *testing.T, reporter *fakeTelemetryReporter) telemetry.WorkspaceBuildDebugEvent {
t.Helper()
for {
snapshot := testutil.TryReceive(ctx, t, reporter.snapshots)
if len(snapshot.WorkspaceBuildDebugEvents) > 0 {
require.Len(t, snapshot.WorkspaceBuildDebugEvents, 1)
return snapshot.WorkspaceBuildDebugEvents[0]
}
}
}
+63
View File
@@ -33,6 +33,7 @@ import (
"github.com/coder/coder/v2/coderd/provisionerdserver"
"github.com/coder/coder/v2/coderd/rbac"
"github.com/coder/coder/v2/coderd/rbac/policy"
"github.com/coder/coder/v2/coderd/telemetry"
"github.com/coder/coder/v2/coderd/wsbuilder"
"github.com/coder/coder/v2/coderd/wspubsub"
"github.com/coder/coder/v2/codersdk"
@@ -774,6 +775,68 @@ func (api *API) notifyWorkspaceUpdated(
}
}
// @Summary Report a workspace build debug click
// @ID report-a-workspace-build-debug-click
// @Security CoderSessionToken
// @Accept json
// @Tags Builds
// @Param workspacebuild path string true "Workspace build ID"
// @Param request body codersdk.WorkspaceBuildDebugEventRequest true "Debug event"
// @Success 204
// @Router /api/v2/workspacebuilds/{workspacebuild}/debug-events [post]
// @x-apidocgen {"skip": true}
func (api *API) postWorkspaceBuildDebugEvent(rw http.ResponseWriter, r *http.Request) {
ctx := r.Context()
apiKey := httpmw.APIKey(r)
workspaceBuild := httpmw.WorkspaceBuildParam(r)
if !api.Experiments.Enabled(codersdk.ExperimentEnableAIWorkspaceDebug) {
httpapi.ResourceNotFound(rw)
return
}
if !api.Authorize(r, policy.ActionCreate, rbac.ResourceChat.WithOwner(apiKey.UserID.String()).AnyOrganization()) {
httpapi.Forbidden(rw)
return
}
job, err := api.Database.GetProvisionerJobByID(ctx, workspaceBuild.JobID)
if err != nil {
httpapi.Write(ctx, rw, http.StatusInternalServerError, codersdk.Response{
Message: "Internal error fetching provisioner job.",
Detail: err.Error(),
})
return
}
if job.JobStatus != database.ProvisionerJobStatusFailed {
httpapi.Write(ctx, rw, http.StatusBadRequest, codersdk.Response{
Message: "Only failed workspace builds can be debugged.",
})
return
}
var req codersdk.WorkspaceBuildDebugEventRequest
if !httpapi.Read(ctx, rw, r, &req) {
return
}
api.Telemetry.Report(&telemetry.Snapshot{
WorkspaceBuildDebugEvents: []telemetry.WorkspaceBuildDebugEvent{
{
ID: req.ID,
EventType: telemetry.WorkspaceBuildDebugEventClick,
UserID: apiKey.UserID,
WorkspaceID: workspaceBuild.WorkspaceID,
WorkspaceBuildID: workspaceBuild.ID,
Transition: string(workspaceBuild.Transition),
Reason: string(workspaceBuild.Reason),
CreatedAt: dbtime.Now(),
},
},
})
rw.WriteHeader(http.StatusNoContent)
}
// @Summary Cancel workspace build
// @ID cancel-workspace-build
// @Security CoderSessionToken
+23
View File
@@ -138,6 +138,29 @@ func (c *Client) WorkspaceBuild(ctx context.Context, id uuid.UUID) (WorkspaceBui
return workspaceBuild, ReadBodyAsJSON(res, &workspaceBuild)
}
// WorkspaceBuildDebugEventRequest is the request body for
// POST /api/v2/workspacebuilds/{workspacebuild}/debug-events.
type WorkspaceBuildDebugEventRequest struct {
// ID identifies this click so a later step of the funnel can be
// attributed to it.
ID uuid.UUID `json:"id" format:"uuid" validate:"required"`
}
// ReportWorkspaceBuildDebugClick reports a click on the "Debug with Coder
// Agents" action of a failed workspace build for telemetry purposes. It is a
// no-op on deployments with telemetry disabled.
func (c *Client) ReportWorkspaceBuildDebugClick(ctx context.Context, buildID uuid.UUID, req WorkspaceBuildDebugEventRequest) error {
res, err := c.Request(ctx, http.MethodPost, fmt.Sprintf("/api/v2/workspacebuilds/%s/debug-events", buildID), req)
if err != nil {
return err
}
defer res.Body.Close()
if res.StatusCode != http.StatusNoContent {
return ReadBodyAsError(res)
}
return nil
}
type CancelWorkspaceBuildStatus string
const (
+14
View File
@@ -19174,6 +19174,20 @@ If the schedule is empty, the user will be updated to use the default schedule.|
| `status` | `canceled`, `canceling`, `deleted`, `deleting`, `failed`, `pending`, `running`, `starting`, `stopped`, `stopping` |
| `transition` | `delete`, `start`, `stop` |
## codersdk.WorkspaceBuildDebugEventRequest
```json
{
"id": "497f6eca-6276-4993-bfeb-53cbbbba6f08"
}
```
### Properties
| Name | Type | Required | Restrictions | Description |
|------|--------|----------|--------------|---------------------------------------------------------------------------------|
| `id` | string | true | | ID identifies this click so a later step of the funnel can be attributed to it. |
## codersdk.WorkspaceBuildParameter
```json
+10
View File
@@ -1328,6 +1328,16 @@ class ApiMethods {
return response.data;
};
reportWorkspaceBuildDebugClick = async (
buildId: string,
req: TypesGen.WorkspaceBuildDebugEventRequest,
): Promise<void> => {
await this.axios.post(
`/api/v2/workspacebuilds/${buildId}/debug-events`,
req,
);
};
waitForBuild = (build: TypesGen.WorkspaceBuild) => {
return new Promise<TypesGen.ProvisionerJob | undefined>((res, reject) => {
void (async () => {
+11
View File
@@ -8,6 +8,7 @@ import { API } from "#/api/api";
import type {
ProvisionerJobLog,
WorkspaceBuild,
WorkspaceBuildDebugEventRequest,
WorkspaceBuildParameter,
WorkspaceBuildsRequest,
} from "#/api/typesGenerated";
@@ -44,6 +45,16 @@ export const workspaceBuildById = (workspaceBuildId: string) =>
queryFn: () => API.getWorkspaceBuild(workspaceBuildId),
});
type ReportWorkspaceBuildDebugClickArgs = {
workspaceBuildId: string;
req: WorkspaceBuildDebugEventRequest;
};
export const reportWorkspaceBuildDebugClick = () => ({
mutationFn: ({ workspaceBuildId, req }: ReportWorkspaceBuildDebugClickArgs) =>
API.reportWorkspaceBuildDebugClick(workspaceBuildId, req),
});
export const workspaceBuildsKey = (workspaceId: string) => [
"workspaceBuilds",
workspaceId,
+13
View File
@@ -11880,6 +11880,19 @@ export interface WorkspaceBuild {
readonly has_external_agent?: boolean;
}
// From codersdk/workspacebuilds.go
/**
* WorkspaceBuildDebugEventRequest is the request body for
* POST /api/v2/workspacebuilds/{workspacebuild}/debug-events.
*/
export interface WorkspaceBuildDebugEventRequest {
/**
* ID identifies this click so a later step of the funnel can be
* attributed to it.
*/
readonly id: string;
}
// From codersdk/workspacebuilds.go
/**
* WorkspaceBuildParameter represents a parameter specific for a workspace build.
@@ -1,30 +1,79 @@
import { screen } from "@testing-library/react";
import { screen, waitFor } from "@testing-library/react";
import userEvent from "@testing-library/user-event";
import { HttpResponse, http } from "msw";
import { describe, expect, it } from "vitest";
import { afterEach, describe, expect, it, vi } from "vitest";
import { API } from "#/api/api";
import { MockFailedWorkspaceBuild } from "#/testHelpers/entities";
import { renderWithAuth } from "#/testHelpers/renderHelpers";
import { server } from "#/testHelpers/server";
import { isUUID } from "#/utils/uuid";
import { WorkspaceBuildFailedAlert } from "./WorkspaceBuildFailedAlert";
afterEach(() => {
vi.restoreAllMocks();
});
const failedBuild = MockFailedWorkspaceBuild();
describe("WorkspaceBuildFailedAlert", () => {
it("links to the agents page for the failed build in a new tab", async () => {
it.each(["primary click", "middle click", "keyboard"])(
"links to the agents page in a new tab and reports %s",
async (activation) => {
server.use(
http.get("/api/v2/experiments", () =>
HttpResponse.json(["enable-ai-workspace-debug"]),
),
);
const reportClick = vi
.spyOn(API, "reportWorkspaceBuildDebugClick")
.mockResolvedValue();
const user = userEvent.setup();
renderWithAuth(<WorkspaceBuildFailedAlert build={failedBuild} />);
const link = await screen.findByRole("link", {
name: "Debug with Coder Agents",
});
expect(link).toHaveAttribute(
"href",
`/agents?debug_workspace_build=${failedBuild.id}`,
);
expect(link).toHaveAttribute("target", "_blank");
if (activation === "middle click") {
await user.pointer({ target: link, keys: "[MouseMiddle]" });
} else if (activation === "keyboard") {
link.focus();
await user.keyboard("{Enter}");
} else {
await user.click(link);
}
await waitFor(() => expect(reportClick).toHaveBeenCalledTimes(1));
const [buildId, request] = reportClick.mock.calls[0];
expect(buildId).toBe(failedBuild.id);
expect(isUUID(request.id)).toBe(true);
},
);
it("does not report a right click", async () => {
server.use(
http.get("/api/v2/experiments", () =>
HttpResponse.json(["enable-ai-workspace-debug"]),
),
);
const reportClick = vi
.spyOn(API, "reportWorkspaceBuildDebugClick")
.mockResolvedValue();
const user = userEvent.setup();
renderWithAuth(<WorkspaceBuildFailedAlert build={failedBuild} />);
const link = await screen.findByRole("link", {
name: "Debug with Coder Agents",
});
expect(link).toHaveAttribute(
"href",
`/agents?debug_workspace_build=${failedBuild.id}`,
);
expect(link).toHaveAttribute("target", "_blank");
await user.pointer({ target: link, keys: "[MouseRight]" });
expect(reportClick).not.toHaveBeenCalled();
});
});
@@ -1,11 +1,14 @@
import { SquareArrowOutUpRightIcon } from "lucide-react";
import type { FC } from "react";
import { useMutation } from "react-query";
import { reportWorkspaceBuildDebugClick } from "#/api/queries/workspaceBuilds";
import type { WorkspaceBuild } from "#/api/typesGenerated";
import { Alert, AlertDescription, AlertTitle } from "#/components/Alert/Alert";
import { Button } from "#/components/Button/Button";
import { useAuthenticated } from "#/hooks/useAuthenticated";
import { useDashboard } from "#/modules/dashboard/useDashboard";
import { buildDebugWorkspaceBuildPath } from "#/modules/workspaces/workspaceBuildDebugLink";
import { generateUUID } from "#/utils/random";
type WorkspaceBuildFailedAlertProps = {
build: WorkspaceBuild;
@@ -16,9 +19,14 @@ export const WorkspaceBuildFailedAlert: FC<WorkspaceBuildFailedAlertProps> = ({
}) => {
const { permissions } = useAuthenticated();
const { experiments } = useDashboard();
const { mutate: reportClick } = useMutation(reportWorkspaceBuildDebugClick());
const canDebugWithAgents =
experiments.includes("enable-ai-workspace-debug") && permissions.createChat;
const handleDebugClick = () => {
reportClick({ workspaceBuildId: build.id, req: { id: generateUUID() } });
};
return (
<Alert
severity="error"
@@ -30,6 +38,12 @@ export const WorkspaceBuildFailedAlert: FC<WorkspaceBuildFailedAlertProps> = ({
href={buildDebugWorkspaceBuildPath(build.id)}
target="_blank"
rel="noreferrer"
onClick={handleDebugClick}
onAuxClick={(event) => {
if (event.button === 1) {
handleDebugClick();
}
}}
>
Debug with Coder Agents
<SquareArrowOutUpRightIcon />