From e4dc2d941879683af0d5021af6a7bac6b20191a0 Mon Sep 17 00:00:00 2001 From: Benjamin Peinhardt <61021968+bcpeinhardt@users.noreply.github.com> Date: Wed, 30 Jul 2025 19:09:53 -0500 Subject: [PATCH] fix: add constraint and runtime check for provisioner logs size limit (#18893) This PR sets a constraint of 1MB on the provisioner job logs written to the database. This is consistent with the constraint we place on workspace agent logs: https://github.com/coder/coder/blob/4ac6be6d835dc36c242e35a26b584b784040bf28/coderd/database/dump.sql#L2030 It also adds a message printed to the front end about the provisioner log overflow, and updates the message printed to the front end when workspace startup logs exceed the max, as it was causing some customers to think their startup script had failed to run. --- cli/testdata/coder_list_--output_json.golden | 3 +- .../coder_provisioner_jobs_list_--help.golden | 2 +- ...provisioner_jobs_list_--output_json.golden | 2 + coderd/apidoc/docs.go | 3 + coderd/apidoc/swagger.json | 3 + coderd/database/dbauthz/dbauthz.go | 16 +++ coderd/database/dbauthz/dbauthz_test.go | 14 ++ coderd/database/dbfake/dbfake.go | 1 + coderd/database/dbgen/dbgen.go | 1 + coderd/database/dbmetrics/querymetrics.go | 14 ++ coderd/database/dbmock/dbmock.go | 28 ++++ coderd/database/dump.sql | 9 +- coderd/database/errors.go | 8 ++ ...5_add_provisioner_logs_overflowed.down.sql | 2 + ...355_add_provisioner_logs_overflowed.up.sql | 6 + coderd/database/models.go | 4 + coderd/database/querier.go | 2 + coderd/database/queries.sql.go | 79 ++++++++-- .../database/queries/provisionerjoblogs.sql | 16 +++ coderd/database/queries/provisionerjobs.sql | 5 +- .../provisionerdserver/provisionerdserver.go | 85 +++++++++-- .../provisionerdserver_test.go | 135 ++++++++++++++++++ coderd/provisionerjobs.go | 1 + coderd/templateversions.go | 2 + coderd/wsbuilder/wsbuilder.go | 1 + codersdk/provisionerdaemons.go | 1 + docs/reference/api/builds.md | 6 + docs/reference/api/organizations.md | 3 + docs/reference/api/schemas.md | 6 + docs/reference/api/templates.md | 11 ++ docs/reference/api/workspaces.md | 6 + docs/reference/cli/provisioner_jobs_list.md | 8 +- .../coder_provisioner_jobs_list_--help.golden | 2 +- site/src/api/typesGenerated.ts | 1 + site/src/modules/resources/AgentRow.tsx | 3 +- .../WorkspaceBuildLogs/WorkspaceBuildLogs.tsx | 26 +++- .../WorkspaceBuildPageView.tsx | 25 +++- site/src/testHelpers/entities.ts | 1 + 38 files changed, 506 insertions(+), 35 deletions(-) create mode 100644 coderd/database/migrations/000355_add_provisioner_logs_overflowed.down.sql create mode 100644 coderd/database/migrations/000355_add_provisioner_logs_overflowed.up.sql diff --git a/cli/testdata/coder_list_--output_json.golden b/cli/testdata/coder_list_--output_json.golden index 822998329b..ba560a39f5 100644 --- a/cli/testdata/coder_list_--output_json.golden +++ b/cli/testdata/coder_list_--output_json.golden @@ -55,7 +55,8 @@ "template_name": "", "template_display_name": "", "template_icon": "" - } + }, + "logs_overflowed": false }, "reason": "initiator", "resources": [], diff --git a/cli/testdata/coder_provisioner_jobs_list_--help.golden b/cli/testdata/coder_provisioner_jobs_list_--help.golden index f380a03348..8e22f78e97 100644 --- a/cli/testdata/coder_provisioner_jobs_list_--help.golden +++ b/cli/testdata/coder_provisioner_jobs_list_--help.golden @@ -11,7 +11,7 @@ OPTIONS: -O, --org string, $CODER_ORGANIZATION Select which organization (uuid or name) to use. - -c, --column [id|created at|started at|completed at|canceled at|error|error code|status|worker id|worker name|file id|tags|queue position|queue size|organization id|template version id|workspace build id|type|available workers|template version name|template id|template name|template display name|template icon|workspace id|workspace name|organization|queue] (default: created at,id,type,template display name,status,queue,tags) + -c, --column [id|created at|started at|completed at|canceled at|error|error code|status|worker id|worker name|file id|tags|queue position|queue size|organization id|template version id|workspace build id|type|available workers|template version name|template id|template name|template display name|template icon|workspace id|workspace name|logs overflowed|organization|queue] (default: created at,id,type,template display name,status,queue,tags) Columns to display in table output. -l, --limit int, $CODER_PROVISIONER_JOB_LIST_LIMIT (default: 50) diff --git a/cli/testdata/coder_provisioner_jobs_list_--output_json.golden b/cli/testdata/coder_provisioner_jobs_list_--output_json.golden index e36723765b..6ccf672360 100644 --- a/cli/testdata/coder_provisioner_jobs_list_--output_json.golden +++ b/cli/testdata/coder_provisioner_jobs_list_--output_json.golden @@ -26,6 +26,7 @@ "template_display_name": "", "template_icon": "" }, + "logs_overflowed": false, "organization_name": "Coder" }, { @@ -57,6 +58,7 @@ "workspace_id": "===========[workspace ID]===========", "workspace_name": "test-workspace" }, + "logs_overflowed": false, "organization_name": "Coder" } ] diff --git a/coderd/apidoc/docs.go b/coderd/apidoc/docs.go index 7c723994d3..7087b5dcc9 100644 --- a/coderd/apidoc/docs.go +++ b/coderd/apidoc/docs.go @@ -15261,6 +15261,9 @@ const docTemplate = `{ "input": { "$ref": "#/definitions/codersdk.ProvisionerJobInput" }, + "logs_overflowed": { + "type": "boolean" + }, "metadata": { "$ref": "#/definitions/codersdk.ProvisionerJobMetadata" }, diff --git a/coderd/apidoc/swagger.json b/coderd/apidoc/swagger.json index 28a38ffd32..3a1443cd76 100644 --- a/coderd/apidoc/swagger.json +++ b/coderd/apidoc/swagger.json @@ -13852,6 +13852,9 @@ "input": { "$ref": "#/definitions/codersdk.ProvisionerJobInput" }, + "logs_overflowed": { + "type": "boolean" + }, "metadata": { "$ref": "#/definitions/codersdk.ProvisionerJobMetadata" }, diff --git a/coderd/database/dbauthz/dbauthz.go b/coderd/database/dbauthz/dbauthz.go index 257cbc6e6b..72489ea92d 100644 --- a/coderd/database/dbauthz/dbauthz.go +++ b/coderd/database/dbauthz/dbauthz.go @@ -4489,6 +4489,22 @@ func (q *querier) UpdateProvisionerJobByID(ctx context.Context, arg database.Upd return q.db.UpdateProvisionerJobByID(ctx, arg) } +func (q *querier) UpdateProvisionerJobLogsLength(ctx context.Context, arg database.UpdateProvisionerJobLogsLengthParams) error { + // Not sure what the rbac should be here, going with this for now + if err := q.authorizeContext(ctx, policy.ActionUpdate, rbac.ResourceProvisionerJobs); err != nil { + return err + } + return q.db.UpdateProvisionerJobLogsLength(ctx, arg) +} + +func (q *querier) UpdateProvisionerJobLogsOverflowed(ctx context.Context, arg database.UpdateProvisionerJobLogsOverflowedParams) error { + // Not sure what the rbac should be here, going with this for now + if err := q.authorizeContext(ctx, policy.ActionUpdate, rbac.ResourceProvisionerJobs); err != nil { + return err + } + return q.db.UpdateProvisionerJobLogsOverflowed(ctx, arg) +} + func (q *querier) UpdateProvisionerJobWithCancelByID(ctx context.Context, arg database.UpdateProvisionerJobWithCancelByIDParams) error { // TODO: Remove this once we have a proper rbac check for provisioner jobs. // Details in https://github.com/coder/coder/issues/16160 diff --git a/coderd/database/dbauthz/dbauthz_test.go b/coderd/database/dbauthz/dbauthz_test.go index dc86d59861..14ab09bada 100644 --- a/coderd/database/dbauthz/dbauthz_test.go +++ b/coderd/database/dbauthz/dbauthz_test.go @@ -4341,6 +4341,20 @@ func (s *MethodTestSuite) TestSystemFunctions() { UpdatedAt: time.Now(), }).Asserts(rbac.ResourceProvisionerJobs, policy.ActionUpdate) })) + s.Run("UpdateProvisionerJobLogsLength", s.Subtest(func(db database.Store, check *expects) { + j := dbgen.ProvisionerJob(s.T(), db, nil, database.ProvisionerJob{}) + check.Args(database.UpdateProvisionerJobLogsLengthParams{ + ID: j.ID, + LogsLength: 100, + }).Asserts(rbac.ResourceProvisionerJobs, policy.ActionUpdate) + })) + s.Run("UpdateProvisionerJobLogsOverflowed", s.Subtest(func(db database.Store, check *expects) { + j := dbgen.ProvisionerJob(s.T(), db, nil, database.ProvisionerJob{}) + check.Args(database.UpdateProvisionerJobLogsOverflowedParams{ + ID: j.ID, + LogsOverflowed: true, + }).Asserts(rbac.ResourceProvisionerJobs, policy.ActionUpdate) + })) s.Run("InsertProvisionerJob", s.Subtest(func(db database.Store, check *expects) { dbtestutil.DisableForeignKeysAndTriggers(s.T(), db) check.Args(database.InsertProvisionerJobParams{ diff --git a/coderd/database/dbfake/dbfake.go b/coderd/database/dbfake/dbfake.go index 99d3c72ab4..e7c00255d4 100644 --- a/coderd/database/dbfake/dbfake.go +++ b/coderd/database/dbfake/dbfake.go @@ -179,6 +179,7 @@ func (b WorkspaceBuildBuilder) Do() WorkspaceResponse { Input: payload, Tags: map[string]string{}, TraceMetadata: pqtype.NullRawMessage{}, + LogsOverflowed: false, }) require.NoError(b.t, err, "insert job") diff --git a/coderd/database/dbgen/dbgen.go b/coderd/database/dbgen/dbgen.go index 71a86c329a..81d9efd1cd 100644 --- a/coderd/database/dbgen/dbgen.go +++ b/coderd/database/dbgen/dbgen.go @@ -775,6 +775,7 @@ func ProvisionerJob(t testing.TB, db database.Store, ps pubsub.Pubsub, orig data Input: takeFirstSlice(orig.Input, []byte("{}")), Tags: tags, TraceMetadata: pqtype.NullRawMessage{}, + LogsOverflowed: false, }) require.NoError(t, err, "insert job") if ps != nil { diff --git a/coderd/database/dbmetrics/querymetrics.go b/coderd/database/dbmetrics/querymetrics.go index 811d945ac7..3fffb29966 100644 --- a/coderd/database/dbmetrics/querymetrics.go +++ b/coderd/database/dbmetrics/querymetrics.go @@ -2784,6 +2784,20 @@ func (m queryMetricsStore) UpdateProvisionerJobByID(ctx context.Context, arg dat return err } +func (m queryMetricsStore) UpdateProvisionerJobLogsLength(ctx context.Context, arg database.UpdateProvisionerJobLogsLengthParams) error { + start := time.Now() + r0 := m.s.UpdateProvisionerJobLogsLength(ctx, arg) + m.queryLatencies.WithLabelValues("UpdateProvisionerJobLogsLength").Observe(time.Since(start).Seconds()) + return r0 +} + +func (m queryMetricsStore) UpdateProvisionerJobLogsOverflowed(ctx context.Context, arg database.UpdateProvisionerJobLogsOverflowedParams) error { + start := time.Now() + r0 := m.s.UpdateProvisionerJobLogsOverflowed(ctx, arg) + m.queryLatencies.WithLabelValues("UpdateProvisionerJobLogsOverflowed").Observe(time.Since(start).Seconds()) + return r0 +} + func (m queryMetricsStore) UpdateProvisionerJobWithCancelByID(ctx context.Context, arg database.UpdateProvisionerJobWithCancelByIDParams) error { start := time.Now() err := m.s.UpdateProvisionerJobWithCancelByID(ctx, arg) diff --git a/coderd/database/dbmock/dbmock.go b/coderd/database/dbmock/dbmock.go index b20c3d0620..20bc17117e 100644 --- a/coderd/database/dbmock/dbmock.go +++ b/coderd/database/dbmock/dbmock.go @@ -5958,6 +5958,34 @@ func (mr *MockStoreMockRecorder) UpdateProvisionerJobByID(ctx, arg any) *gomock. return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateProvisionerJobByID", reflect.TypeOf((*MockStore)(nil).UpdateProvisionerJobByID), ctx, arg) } +// UpdateProvisionerJobLogsLength mocks base method. +func (m *MockStore) UpdateProvisionerJobLogsLength(ctx context.Context, arg database.UpdateProvisionerJobLogsLengthParams) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "UpdateProvisionerJobLogsLength", ctx, arg) + ret0, _ := ret[0].(error) + return ret0 +} + +// UpdateProvisionerJobLogsLength indicates an expected call of UpdateProvisionerJobLogsLength. +func (mr *MockStoreMockRecorder) UpdateProvisionerJobLogsLength(ctx, arg any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateProvisionerJobLogsLength", reflect.TypeOf((*MockStore)(nil).UpdateProvisionerJobLogsLength), ctx, arg) +} + +// UpdateProvisionerJobLogsOverflowed mocks base method. +func (m *MockStore) UpdateProvisionerJobLogsOverflowed(ctx context.Context, arg database.UpdateProvisionerJobLogsOverflowedParams) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "UpdateProvisionerJobLogsOverflowed", ctx, arg) + ret0, _ := ret[0].(error) + return ret0 +} + +// UpdateProvisionerJobLogsOverflowed indicates an expected call of UpdateProvisionerJobLogsOverflowed. +func (mr *MockStoreMockRecorder) UpdateProvisionerJobLogsOverflowed(ctx, arg any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateProvisionerJobLogsOverflowed", reflect.TypeOf((*MockStore)(nil).UpdateProvisionerJobLogsOverflowed), ctx, arg) +} + // UpdateProvisionerJobWithCancelByID mocks base method. func (m *MockStore) UpdateProvisionerJobWithCancelByID(ctx context.Context, arg database.UpdateProvisionerJobWithCancelByIDParams) error { m.ctrl.T.Helper() diff --git a/coderd/database/dump.sql b/coderd/database/dump.sql index c6c147e2f0..053b5302d3 100644 --- a/coderd/database/dump.sql +++ b/coderd/database/dump.sql @@ -1419,11 +1419,18 @@ CASE WHEN (started_at IS NULL) THEN 'pending'::provisioner_job_status ELSE 'running'::provisioner_job_status END -END) STORED NOT NULL +END) STORED NOT NULL, + logs_length integer DEFAULT 0 NOT NULL, + logs_overflowed boolean DEFAULT false NOT NULL, + CONSTRAINT max_provisioner_logs_length CHECK ((logs_length <= 1048576)) ); COMMENT ON COLUMN provisioner_jobs.job_status IS 'Computed column to track the status of the job.'; +COMMENT ON COLUMN provisioner_jobs.logs_length IS 'Total length of provisioner logs'; + +COMMENT ON COLUMN provisioner_jobs.logs_overflowed IS 'Whether the provisioner logs overflowed in length'; + CREATE TABLE provisioner_keys ( id uuid NOT NULL, created_at timestamp with time zone NOT NULL, diff --git a/coderd/database/errors.go b/coderd/database/errors.go index 66c702de24..0388ea2cbf 100644 --- a/coderd/database/errors.go +++ b/coderd/database/errors.go @@ -79,3 +79,11 @@ func IsWorkspaceAgentLogsLimitError(err error) bool { return false } + +func IsProvisionerJobLogsLimitError(err error) bool { + var pqErr *pq.Error + if errors.As(err, &pqErr) { + return pqErr.Constraint == "max_provisioner_logs_length" && pqErr.Table == "provisioner_jobs" + } + return false +} diff --git a/coderd/database/migrations/000355_add_provisioner_logs_overflowed.down.sql b/coderd/database/migrations/000355_add_provisioner_logs_overflowed.down.sql new file mode 100644 index 0000000000..39f34a2b49 --- /dev/null +++ b/coderd/database/migrations/000355_add_provisioner_logs_overflowed.down.sql @@ -0,0 +1,2 @@ +ALTER TABLE provisioner_jobs DROP COLUMN logs_length; +ALTER TABLE provisioner_jobs DROP COLUMN logs_overflowed; \ No newline at end of file diff --git a/coderd/database/migrations/000355_add_provisioner_logs_overflowed.up.sql b/coderd/database/migrations/000355_add_provisioner_logs_overflowed.up.sql new file mode 100644 index 0000000000..80f58cf5c6 --- /dev/null +++ b/coderd/database/migrations/000355_add_provisioner_logs_overflowed.up.sql @@ -0,0 +1,6 @@ + -- Add logs length tracking and overflow flag, similar to workspace agents + ALTER TABLE provisioner_jobs ADD COLUMN logs_length integer NOT NULL DEFAULT 0 CONSTRAINT max_provisioner_logs_length CHECK (logs_length <= 1048576); + ALTER TABLE provisioner_jobs ADD COLUMN logs_overflowed boolean NOT NULL DEFAULT false; + + COMMENT ON COLUMN provisioner_jobs.logs_length IS 'Total length of provisioner logs'; + COMMENT ON COLUMN provisioner_jobs.logs_overflowed IS 'Whether the provisioner logs overflowed in length'; diff --git a/coderd/database/models.go b/coderd/database/models.go index 8eed09f97b..8b13c8a8af 100644 --- a/coderd/database/models.go +++ b/coderd/database/models.go @@ -3384,6 +3384,10 @@ type ProvisionerJob struct { TraceMetadata pqtype.NullRawMessage `db:"trace_metadata" json:"trace_metadata"` // Computed column to track the status of the job. JobStatus ProvisionerJobStatus `db:"job_status" json:"job_status"` + // Total length of provisioner logs + LogsLength int32 `db:"logs_length" json:"logs_length"` + // Whether the provisioner logs overflowed in length + LogsOverflowed bool `db:"logs_overflowed" json:"logs_overflowed"` } type ProvisionerJobLog struct { diff --git a/coderd/database/querier.go b/coderd/database/querier.go index baa5d8590b..a2c6cda1af 100644 --- a/coderd/database/querier.go +++ b/coderd/database/querier.go @@ -593,6 +593,8 @@ type sqlcQuerier interface { UpdatePresetPrebuildStatus(ctx context.Context, arg UpdatePresetPrebuildStatusParams) error UpdateProvisionerDaemonLastSeenAt(ctx context.Context, arg UpdateProvisionerDaemonLastSeenAtParams) error UpdateProvisionerJobByID(ctx context.Context, arg UpdateProvisionerJobByIDParams) error + UpdateProvisionerJobLogsLength(ctx context.Context, arg UpdateProvisionerJobLogsLengthParams) error + UpdateProvisionerJobLogsOverflowed(ctx context.Context, arg UpdateProvisionerJobLogsOverflowedParams) error UpdateProvisionerJobWithCancelByID(ctx context.Context, arg UpdateProvisionerJobWithCancelByIDParams) error UpdateProvisionerJobWithCompleteByID(ctx context.Context, arg UpdateProvisionerJobWithCompleteByIDParams) error UpdateProvisionerJobWithCompleteWithStartedAtByID(ctx context.Context, arg UpdateProvisionerJobWithCompleteWithStartedAtByIDParams) error diff --git a/coderd/database/queries.sql.go b/coderd/database/queries.sql.go index 5c06119e80..6033ab7280 100644 --- a/coderd/database/queries.sql.go +++ b/coderd/database/queries.sql.go @@ -8514,6 +8514,44 @@ func (q *sqlQuerier) InsertProvisionerJobLogs(ctx context.Context, arg InsertPro return items, nil } +const updateProvisionerJobLogsLength = `-- name: UpdateProvisionerJobLogsLength :exec +UPDATE + provisioner_jobs +SET + logs_length = logs_length + $2 +WHERE + id = $1 +` + +type UpdateProvisionerJobLogsLengthParams struct { + ID uuid.UUID `db:"id" json:"id"` + LogsLength int32 `db:"logs_length" json:"logs_length"` +} + +func (q *sqlQuerier) UpdateProvisionerJobLogsLength(ctx context.Context, arg UpdateProvisionerJobLogsLengthParams) error { + _, err := q.db.ExecContext(ctx, updateProvisionerJobLogsLength, arg.ID, arg.LogsLength) + return err +} + +const updateProvisionerJobLogsOverflowed = `-- name: UpdateProvisionerJobLogsOverflowed :exec +UPDATE + provisioner_jobs +SET + logs_overflowed = $2 +WHERE + id = $1 +` + +type UpdateProvisionerJobLogsOverflowedParams struct { + ID uuid.UUID `db:"id" json:"id"` + LogsOverflowed bool `db:"logs_overflowed" json:"logs_overflowed"` +} + +func (q *sqlQuerier) UpdateProvisionerJobLogsOverflowed(ctx context.Context, arg UpdateProvisionerJobLogsOverflowedParams) error { + _, err := q.db.ExecContext(ctx, updateProvisionerJobLogsOverflowed, arg.ID, arg.LogsOverflowed) + return err +} + const acquireProvisionerJob = `-- name: AcquireProvisionerJob :one UPDATE provisioner_jobs @@ -8543,7 +8581,7 @@ WHERE SKIP LOCKED LIMIT 1 - ) RETURNING id, created_at, updated_at, started_at, canceled_at, completed_at, error, organization_id, initiator_id, provisioner, storage_method, type, input, worker_id, file_id, tags, error_code, trace_metadata, job_status + ) RETURNING id, created_at, updated_at, started_at, canceled_at, completed_at, error, organization_id, initiator_id, provisioner, storage_method, type, input, worker_id, file_id, tags, error_code, trace_metadata, job_status, logs_length, logs_overflowed ` type AcquireProvisionerJobParams struct { @@ -8589,13 +8627,15 @@ func (q *sqlQuerier) AcquireProvisionerJob(ctx context.Context, arg AcquireProvi &i.ErrorCode, &i.TraceMetadata, &i.JobStatus, + &i.LogsLength, + &i.LogsOverflowed, ) return i, err } const getProvisionerJobByID = `-- name: GetProvisionerJobByID :one SELECT - id, created_at, updated_at, started_at, canceled_at, completed_at, error, organization_id, initiator_id, provisioner, storage_method, type, input, worker_id, file_id, tags, error_code, trace_metadata, job_status + id, created_at, updated_at, started_at, canceled_at, completed_at, error, organization_id, initiator_id, provisioner, storage_method, type, input, worker_id, file_id, tags, error_code, trace_metadata, job_status, logs_length, logs_overflowed FROM provisioner_jobs WHERE @@ -8625,13 +8665,15 @@ func (q *sqlQuerier) GetProvisionerJobByID(ctx context.Context, id uuid.UUID) (P &i.ErrorCode, &i.TraceMetadata, &i.JobStatus, + &i.LogsLength, + &i.LogsOverflowed, ) return i, err } const getProvisionerJobByIDForUpdate = `-- name: GetProvisionerJobByIDForUpdate :one SELECT - id, created_at, updated_at, started_at, canceled_at, completed_at, error, organization_id, initiator_id, provisioner, storage_method, type, input, worker_id, file_id, tags, error_code, trace_metadata, job_status + id, created_at, updated_at, started_at, canceled_at, completed_at, error, organization_id, initiator_id, provisioner, storage_method, type, input, worker_id, file_id, tags, error_code, trace_metadata, job_status, logs_length, logs_overflowed FROM provisioner_jobs WHERE @@ -8665,6 +8707,8 @@ func (q *sqlQuerier) GetProvisionerJobByIDForUpdate(ctx context.Context, id uuid &i.ErrorCode, &i.TraceMetadata, &i.JobStatus, + &i.LogsLength, + &i.LogsOverflowed, ) return i, err } @@ -8708,7 +8752,7 @@ func (q *sqlQuerier) GetProvisionerJobTimingsByJobID(ctx context.Context, jobID const getProvisionerJobsByIDs = `-- name: GetProvisionerJobsByIDs :many SELECT - id, created_at, updated_at, started_at, canceled_at, completed_at, error, organization_id, initiator_id, provisioner, storage_method, type, input, worker_id, file_id, tags, error_code, trace_metadata, job_status + id, created_at, updated_at, started_at, canceled_at, completed_at, error, organization_id, initiator_id, provisioner, storage_method, type, input, worker_id, file_id, tags, error_code, trace_metadata, job_status, logs_length, logs_overflowed FROM provisioner_jobs WHERE @@ -8744,6 +8788,8 @@ func (q *sqlQuerier) GetProvisionerJobsByIDs(ctx context.Context, ids []uuid.UUI &i.ErrorCode, &i.TraceMetadata, &i.JobStatus, + &i.LogsLength, + &i.LogsOverflowed, ); err != nil { return nil, err } @@ -8811,7 +8857,7 @@ SELECT -- Step 5: Final SELECT with INNER JOIN provisioner_jobs fj.id, fj.created_at, - pj.id, pj.created_at, pj.updated_at, pj.started_at, pj.canceled_at, pj.completed_at, pj.error, pj.organization_id, pj.initiator_id, pj.provisioner, pj.storage_method, pj.type, pj.input, pj.worker_id, pj.file_id, pj.tags, pj.error_code, pj.trace_metadata, pj.job_status, + pj.id, pj.created_at, pj.updated_at, pj.started_at, pj.canceled_at, pj.completed_at, pj.error, pj.organization_id, pj.initiator_id, pj.provisioner, pj.storage_method, pj.type, pj.input, pj.worker_id, pj.file_id, pj.tags, pj.error_code, pj.trace_metadata, pj.job_status, pj.logs_length, pj.logs_overflowed, fj.queue_position, fj.queue_size FROM @@ -8867,6 +8913,8 @@ func (q *sqlQuerier) GetProvisionerJobsByIDsWithQueuePosition(ctx context.Contex &i.ProvisionerJob.ErrorCode, &i.ProvisionerJob.TraceMetadata, &i.ProvisionerJob.JobStatus, + &i.ProvisionerJob.LogsLength, + &i.ProvisionerJob.LogsOverflowed, &i.QueuePosition, &i.QueueSize, ); err != nil { @@ -8909,7 +8957,7 @@ queue_size AS ( SELECT COUNT(*) AS count FROM pending_jobs ) SELECT - pj.id, pj.created_at, pj.updated_at, pj.started_at, pj.canceled_at, pj.completed_at, pj.error, pj.organization_id, pj.initiator_id, pj.provisioner, pj.storage_method, pj.type, pj.input, pj.worker_id, pj.file_id, pj.tags, pj.error_code, pj.trace_metadata, pj.job_status, + pj.id, pj.created_at, pj.updated_at, pj.started_at, pj.canceled_at, pj.completed_at, pj.error, pj.organization_id, pj.initiator_id, pj.provisioner, pj.storage_method, pj.type, pj.input, pj.worker_id, pj.file_id, pj.tags, pj.error_code, pj.trace_metadata, pj.job_status, pj.logs_length, pj.logs_overflowed, COALESCE(qp.queue_position, 0) AS queue_position, COALESCE(qs.count, 0) AS queue_size, -- Use subquery to utilize ORDER BY in array_agg since it cannot be @@ -9045,6 +9093,8 @@ func (q *sqlQuerier) GetProvisionerJobsByOrganizationAndStatusWithQueuePositionA &i.ProvisionerJob.ErrorCode, &i.ProvisionerJob.TraceMetadata, &i.ProvisionerJob.JobStatus, + &i.ProvisionerJob.LogsLength, + &i.ProvisionerJob.LogsOverflowed, &i.QueuePosition, &i.QueueSize, pq.Array(&i.AvailableWorkers), @@ -9071,7 +9121,7 @@ func (q *sqlQuerier) GetProvisionerJobsByOrganizationAndStatusWithQueuePositionA } const getProvisionerJobsCreatedAfter = `-- name: GetProvisionerJobsCreatedAfter :many -SELECT id, created_at, updated_at, started_at, canceled_at, completed_at, error, organization_id, initiator_id, provisioner, storage_method, type, input, worker_id, file_id, tags, error_code, trace_metadata, job_status FROM provisioner_jobs WHERE created_at > $1 +SELECT id, created_at, updated_at, started_at, canceled_at, completed_at, error, organization_id, initiator_id, provisioner, storage_method, type, input, worker_id, file_id, tags, error_code, trace_metadata, job_status, logs_length, logs_overflowed FROM provisioner_jobs WHERE created_at > $1 ` func (q *sqlQuerier) GetProvisionerJobsCreatedAfter(ctx context.Context, createdAt time.Time) ([]ProvisionerJob, error) { @@ -9103,6 +9153,8 @@ func (q *sqlQuerier) GetProvisionerJobsCreatedAfter(ctx context.Context, created &i.ErrorCode, &i.TraceMetadata, &i.JobStatus, + &i.LogsLength, + &i.LogsOverflowed, ); err != nil { return nil, err } @@ -9119,7 +9171,7 @@ func (q *sqlQuerier) GetProvisionerJobsCreatedAfter(ctx context.Context, created const getProvisionerJobsToBeReaped = `-- name: GetProvisionerJobsToBeReaped :many SELECT - id, created_at, updated_at, started_at, canceled_at, completed_at, error, organization_id, initiator_id, provisioner, storage_method, type, input, worker_id, file_id, tags, error_code, trace_metadata, job_status + id, created_at, updated_at, started_at, canceled_at, completed_at, error, organization_id, initiator_id, provisioner, storage_method, type, input, worker_id, file_id, tags, error_code, trace_metadata, job_status, logs_length, logs_overflowed FROM provisioner_jobs WHERE @@ -9176,6 +9228,8 @@ func (q *sqlQuerier) GetProvisionerJobsToBeReaped(ctx context.Context, arg GetPr &i.ErrorCode, &i.TraceMetadata, &i.JobStatus, + &i.LogsLength, + &i.LogsOverflowed, ); err != nil { return nil, err } @@ -9204,10 +9258,11 @@ INSERT INTO "type", "input", tags, - trace_metadata + trace_metadata, + logs_overflowed ) VALUES - ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12) RETURNING id, created_at, updated_at, started_at, canceled_at, completed_at, error, organization_id, initiator_id, provisioner, storage_method, type, input, worker_id, file_id, tags, error_code, trace_metadata, job_status + ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13) RETURNING id, created_at, updated_at, started_at, canceled_at, completed_at, error, organization_id, initiator_id, provisioner, storage_method, type, input, worker_id, file_id, tags, error_code, trace_metadata, job_status, logs_length, logs_overflowed ` type InsertProvisionerJobParams struct { @@ -9223,6 +9278,7 @@ type InsertProvisionerJobParams struct { Input json.RawMessage `db:"input" json:"input"` Tags StringMap `db:"tags" json:"tags"` TraceMetadata pqtype.NullRawMessage `db:"trace_metadata" json:"trace_metadata"` + LogsOverflowed bool `db:"logs_overflowed" json:"logs_overflowed"` } func (q *sqlQuerier) InsertProvisionerJob(ctx context.Context, arg InsertProvisionerJobParams) (ProvisionerJob, error) { @@ -9239,6 +9295,7 @@ func (q *sqlQuerier) InsertProvisionerJob(ctx context.Context, arg InsertProvisi arg.Input, arg.Tags, arg.TraceMetadata, + arg.LogsOverflowed, ) var i ProvisionerJob err := row.Scan( @@ -9261,6 +9318,8 @@ func (q *sqlQuerier) InsertProvisionerJob(ctx context.Context, arg InsertProvisi &i.ErrorCode, &i.TraceMetadata, &i.JobStatus, + &i.LogsLength, + &i.LogsOverflowed, ) return i, err } diff --git a/coderd/database/queries/provisionerjoblogs.sql b/coderd/database/queries/provisionerjoblogs.sql index b98cf471f0..c0ef188bdd 100644 --- a/coderd/database/queries/provisionerjoblogs.sql +++ b/coderd/database/queries/provisionerjoblogs.sql @@ -19,3 +19,19 @@ SELECT unnest(@level :: log_level [ ]) AS LEVEL, unnest(@stage :: VARCHAR(128) [ ]) AS stage, unnest(@output :: VARCHAR(1024) [ ]) AS output RETURNING *; + +-- name: UpdateProvisionerJobLogsOverflowed :exec +UPDATE + provisioner_jobs +SET + logs_overflowed = $2 +WHERE + id = $1; + +-- name: UpdateProvisionerJobLogsLength :exec +UPDATE + provisioner_jobs +SET + logs_length = logs_length + $2 +WHERE + id = $1; diff --git a/coderd/database/queries/provisionerjobs.sql b/coderd/database/queries/provisionerjobs.sql index fcf348e089..3ba5816466 100644 --- a/coderd/database/queries/provisionerjobs.sql +++ b/coderd/database/queries/provisionerjobs.sql @@ -247,10 +247,11 @@ INSERT INTO "type", "input", tags, - trace_metadata + trace_metadata, + logs_overflowed ) VALUES - ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12) RETURNING *; + ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13) RETURNING *; -- name: UpdateProvisionerJobByID :exec UPDATE diff --git a/coderd/provisionerdserver/provisionerdserver.go b/coderd/provisionerdserver/provisionerdserver.go index 518b48d2fe..94173703c4 100644 --- a/coderd/provisionerdserver/provisionerdserver.go +++ b/coderd/provisionerdserver/provisionerdserver.go @@ -902,29 +902,93 @@ func (s *server) UpdateJob(ctx context.Context, request *proto.UpdateJobRequest) return nil, xerrors.Errorf("update job: %w", err) } - if len(request.Logs) > 0 { + if len(request.Logs) > 0 && !job.LogsOverflowed { //nolint:exhaustruct // We append to the additional fields below. insertParams := database.InsertProvisionerJobLogsParams{ JobID: parsedID, } + + newLogSize := 0 + overflowedErrorMsg := "Provisioner logs exceeded the max size of 1MB. Will not continue to write provisioner logs for workspace build." + lenErrMsg := len(overflowedErrorMsg) + + var ( + createdAt time.Time + level database.LogLevel + stage string + source database.LogSource + output string + ) + for _, log := range request.Logs { - logLevel, err := convertLogLevel(log.Level) + // Build our log params + level, err = convertLogLevel(log.Level) if err != nil { return nil, xerrors.Errorf("convert log level: %w", err) } - logSource, err := convertLogSource(log.Source) + source, err = convertLogSource(log.Source) if err != nil { return nil, xerrors.Errorf("convert log source: %w", err) } - insertParams.CreatedAt = append(insertParams.CreatedAt, time.UnixMilli(log.CreatedAt)) - insertParams.Level = append(insertParams.Level, logLevel) - insertParams.Stage = append(insertParams.Stage, log.Stage) - insertParams.Source = append(insertParams.Source, logSource) - insertParams.Output = append(insertParams.Output, log.Output) + createdAt = time.UnixMilli(log.CreatedAt) + stage = log.Stage + output = log.Output + + // Check if we would overflow the job logs (not leaving enough room for the error message) + willOverflow := int64(job.LogsLength)+int64(newLogSize)+int64(lenErrMsg)+int64(len(output)) > 1048576 + if willOverflow { + s.Logger.Debug(ctx, "provisioner job logs overflowed 1MB size limit in database", slog.F("job_id", parsedID)) + err = s.Database.UpdateProvisionerJobLogsOverflowed(ctx, database.UpdateProvisionerJobLogsOverflowedParams{ + ID: parsedID, + LogsOverflowed: true, + }) + if err != nil { + s.Logger.Error(ctx, "failed to set logs overflowed flag", slog.F("job_id", parsedID), slog.Error(err)) + } + + level = database.LogLevelWarn + output = overflowedErrorMsg + } + + newLogSize += len(output) + + insertParams.CreatedAt = append(insertParams.CreatedAt, createdAt) + insertParams.Level = append(insertParams.Level, level) + insertParams.Stage = append(insertParams.Stage, stage) + insertParams.Source = append(insertParams.Source, source) + insertParams.Output = append(insertParams.Output, output) s.Logger.Debug(ctx, "job log", slog.F("job_id", parsedID), - slog.F("stage", log.Stage), - slog.F("output", log.Output)) + slog.F("stage", stage), + slog.F("output", output)) + + // Don't write any more logs because there's no room. + if willOverflow { + break + } + } + + err = s.Database.UpdateProvisionerJobLogsLength(ctx, database.UpdateProvisionerJobLogsLengthParams{ + ID: parsedID, + LogsLength: int32(newLogSize), // #nosec G115 - Log output length is limited to 1MB (2^20) which fits in an int32. + }) + if err != nil { + // Even though we do the runtime check for the overflow, we still check for the database error + // as well. + if database.IsProvisionerJobLogsLimitError(err) { + err = s.Database.UpdateProvisionerJobLogsOverflowed(ctx, database.UpdateProvisionerJobLogsOverflowedParams{ + ID: parsedID, + LogsOverflowed: true, + }) + if err != nil { + s.Logger.Error(ctx, "failed to set logs overflowed flag", slog.F("job_id", parsedID), slog.Error(err)) + } + return &proto.UpdateJobResponse{ + Canceled: job.CanceledAt.Valid, + }, nil + } + s.Logger.Error(ctx, "failed to update logs length", slog.F("job_id", parsedID), slog.Error(err)) + return nil, xerrors.Errorf("update logs length: %w", err) } logs, err := s.Database.InsertProvisionerJobLogs(ctx, insertParams) @@ -932,6 +996,7 @@ func (s *server) UpdateJob(ctx context.Context, request *proto.UpdateJobRequest) s.Logger.Error(ctx, "failed to insert job logs", slog.F("job_id", parsedID), slog.Error(err)) return nil, xerrors.Errorf("insert job logs: %w", err) } + // Publish by the lowest log ID inserted so the log stream will fetch // everything from that point. lowestID := logs[0].ID diff --git a/coderd/provisionerdserver/provisionerdserver_test.go b/coderd/provisionerdserver/provisionerdserver_test.go index 6668483565..b6f9d82a59 100644 --- a/coderd/provisionerdserver/provisionerdserver_test.go +++ b/coderd/provisionerdserver/provisionerdserver_test.go @@ -928,6 +928,141 @@ func TestUpdateJob(t *testing.T) { require.Equal(t, workspaceTags[1].Key, "cat") require.Equal(t, workspaceTags[1].Value, "jinx") }) + + t.Run("LogSizeLimit", func(t *testing.T) { + t.Parallel() + srv, db, _, pd := setup(t, false, &overrides{}) + job := setupJob(t, db, pd.ID, pd.Tags) + + // Create a log message that exceeds the 1MB limit + largeOutput := strings.Repeat("a", 1048577) // 1MB + 1 byte + + _, err := srv.UpdateJob(ctx, &proto.UpdateJobRequest{ + JobId: job.String(), + Logs: []*proto.Log{{ + Source: proto.LogSource_PROVISIONER, + Level: sdkproto.LogLevel_INFO, + Output: largeOutput, + }}, + }) + require.NoError(t, err) // Should succeed but trigger overflow + + // Verify the overflow flag is set + jobResult, err := db.GetProvisionerJobByID(ctx, job) + require.NoError(t, err) + require.True(t, jobResult.LogsOverflowed) + }) + + t.Run("IncrementalLogSizeOverflow", func(t *testing.T) { + t.Parallel() + srv, db, _, pd := setup(t, false, &overrides{}) + job := setupJob(t, db, pd.ID, pd.Tags) + + // Send logs that together exceed the limit + mediumOutput := strings.Repeat("b", 524289) // Half a MB + 1 byte + + // First log - should succeed + _, err := srv.UpdateJob(ctx, &proto.UpdateJobRequest{ + JobId: job.String(), + Logs: []*proto.Log{{ + Source: proto.LogSource_PROVISIONER, + Level: sdkproto.LogLevel_INFO, + Output: mediumOutput, + }}, + }) + require.NoError(t, err) + + // Verify overflow flag not yet set + jobResult, err := db.GetProvisionerJobByID(ctx, job) + require.NoError(t, err) + require.False(t, jobResult.LogsOverflowed) + + // Second log - should trigger overflow + _, err = srv.UpdateJob(ctx, &proto.UpdateJobRequest{ + JobId: job.String(), + Logs: []*proto.Log{{ + Source: proto.LogSource_PROVISIONER, + Level: sdkproto.LogLevel_INFO, + Output: mediumOutput, + }}, + }) + require.NoError(t, err) + + // Verify overflow flag is set + jobResult, err = db.GetProvisionerJobByID(ctx, job) + require.NoError(t, err) + require.True(t, jobResult.LogsOverflowed) + }) + + t.Run("LogSizeTracking", func(t *testing.T) { + t.Parallel() + srv, db, _, pd := setup(t, false, &overrides{}) + job := setupJob(t, db, pd.ID, pd.Tags) + + logOutput := "test log message" + expectedSize := int32(len(logOutput)) // #nosec G115 - Log length is 16. + + _, err := srv.UpdateJob(ctx, &proto.UpdateJobRequest{ + JobId: job.String(), + Logs: []*proto.Log{{ + Source: proto.LogSource_PROVISIONER, + Level: sdkproto.LogLevel_INFO, + Output: logOutput, + }}, + }) + require.NoError(t, err) + + // Verify the logs_length is correctly tracked + jobResult, err := db.GetProvisionerJobByID(ctx, job) + require.NoError(t, err) + require.Equal(t, expectedSize, jobResult.LogsLength) + require.False(t, jobResult.LogsOverflowed) + }) + + t.Run("LogOverflowStopsProcessing", func(t *testing.T) { + t.Parallel() + srv, db, _, pd := setup(t, false, &overrides{}) + job := setupJob(t, db, pd.ID, pd.Tags) + + // First: trigger overflow + largeOutput := strings.Repeat("a", 1048577) // 1MB + 1 byte + _, err := srv.UpdateJob(ctx, &proto.UpdateJobRequest{ + JobId: job.String(), + Logs: []*proto.Log{{ + Source: proto.LogSource_PROVISIONER, + Level: sdkproto.LogLevel_INFO, + Output: largeOutput, + }}, + }) + require.NoError(t, err) + + // Get the initial log count + initialLogs, err := db.GetProvisionerLogsAfterID(ctx, database.GetProvisionerLogsAfterIDParams{ + JobID: job, + CreatedAfter: -1, + }) + require.NoError(t, err) + initialCount := len(initialLogs) + + // Second: try to send more logs - should be ignored + _, err = srv.UpdateJob(ctx, &proto.UpdateJobRequest{ + JobId: job.String(), + Logs: []*proto.Log{{ + Source: proto.LogSource_PROVISIONER, + Level: sdkproto.LogLevel_INFO, + Output: "this should be ignored", + }}, + }) + require.NoError(t, err) + + // Verify no new logs were added + finalLogs, err := db.GetProvisionerLogsAfterID(ctx, database.GetProvisionerLogsAfterIDParams{ + JobID: job, + CreatedAfter: -1, + }) + require.NoError(t, err) + require.Equal(t, initialCount, len(finalLogs)) + }) } func TestFailJob(t *testing.T) { diff --git a/coderd/provisionerjobs.go b/coderd/provisionerjobs.go index 800b2916ef..e9ab526098 100644 --- a/coderd/provisionerjobs.go +++ b/coderd/provisionerjobs.go @@ -363,6 +363,7 @@ func convertProvisionerJob(pj database.GetProvisionerJobsByIDsWithQueuePositionR Tags: provisionerJob.Tags, QueuePosition: int(pj.QueuePosition), QueueSize: int(pj.QueueSize), + LogsOverflowed: provisionerJob.LogsOverflowed, } // Applying values optional to the struct. if provisionerJob.StartedAt.Valid { diff --git a/coderd/templateversions.go b/coderd/templateversions.go index 2a6e09d949..2c02268bba 100644 --- a/coderd/templateversions.go +++ b/coderd/templateversions.go @@ -552,6 +552,7 @@ func (api *API) postTemplateVersionDryRun(rw http.ResponseWriter, r *http.Reques Valid: true, RawMessage: metadataRaw, }, + LogsOverflowed: false, }) if err != nil { httpapi.Write(ctx, rw, http.StatusInternalServerError, codersdk.Response{ @@ -1646,6 +1647,7 @@ func (api *API) postTemplateVersionsByOrganization(rw http.ResponseWriter, r *ht Valid: true, RawMessage: traceMetadataRaw, }, + LogsOverflowed: false, }) if err != nil { httpapi.Write(ctx, rw, http.StatusInternalServerError, codersdk.Response{ diff --git a/coderd/wsbuilder/wsbuilder.go b/coderd/wsbuilder/wsbuilder.go index 52567b463b..73e449ee5b 100644 --- a/coderd/wsbuilder/wsbuilder.go +++ b/coderd/wsbuilder/wsbuilder.go @@ -409,6 +409,7 @@ func (b *Builder) buildTx(authFunc func(action policy.Action, object rbac.Object Valid: true, RawMessage: traceMetadataRaw, }, + LogsOverflowed: false, }) if err != nil { return nil, nil, nil, BuildError{http.StatusInternalServerError, "insert provisioner job", err} diff --git a/codersdk/provisionerdaemons.go b/codersdk/provisionerdaemons.go index 5fbda371b8..e36f995f16 100644 --- a/codersdk/provisionerdaemons.go +++ b/codersdk/provisionerdaemons.go @@ -188,6 +188,7 @@ type ProvisionerJob struct { Type ProvisionerJobType `json:"type" table:"type"` AvailableWorkers []uuid.UUID `json:"available_workers,omitempty" format:"uuid" table:"available workers"` Metadata ProvisionerJobMetadata `json:"metadata" table:"metadata,recursive_inline"` + LogsOverflowed bool `json:"logs_overflowed" table:"logs overflowed"` } // ProvisionerJobLog represents the provisioner log entry annotated with source and level. diff --git a/docs/reference/api/builds.md b/docs/reference/api/builds.md index fb491405df..a465575bae 100644 --- a/docs/reference/api/builds.md +++ b/docs/reference/api/builds.md @@ -52,6 +52,7 @@ curl -X GET http://coder-server:8080/api/v2/users/{user}/workspace/{workspacenam "template_version_id": "0ba39c92-1f1b-4c32-aa3e-9925d7713eb1", "workspace_build_id": "badaf2eb-96c5-4050-9f1d-db2d39ca5478" }, + "logs_overflowed": true, "metadata": { "template_display_name": "string", "template_icon": "string", @@ -289,6 +290,7 @@ curl -X GET http://coder-server:8080/api/v2/workspacebuilds/{workspacebuild} \ "template_version_id": "0ba39c92-1f1b-4c32-aa3e-9925d7713eb1", "workspace_build_id": "badaf2eb-96c5-4050-9f1d-db2d39ca5478" }, + "logs_overflowed": true, "metadata": { "template_display_name": "string", "template_icon": "string", @@ -1015,6 +1017,7 @@ curl -X GET http://coder-server:8080/api/v2/workspacebuilds/{workspacebuild}/sta "template_version_id": "0ba39c92-1f1b-4c32-aa3e-9925d7713eb1", "workspace_build_id": "badaf2eb-96c5-4050-9f1d-db2d39ca5478" }, + "logs_overflowed": true, "metadata": { "template_display_name": "string", "template_icon": "string", @@ -1325,6 +1328,7 @@ curl -X GET http://coder-server:8080/api/v2/workspaces/{workspace}/builds \ "template_version_id": "0ba39c92-1f1b-4c32-aa3e-9925d7713eb1", "workspace_build_id": "badaf2eb-96c5-4050-9f1d-db2d39ca5478" }, + "logs_overflowed": true, "metadata": { "template_display_name": "string", "template_icon": "string", @@ -1540,6 +1544,7 @@ Status Code **200** | `»»» error` | string | false | | | | `»»» template_version_id` | string(uuid) | false | | | | `»»» workspace_build_id` | string(uuid) | false | | | +| `»» logs_overflowed` | boolean | false | | | | `»» metadata` | [codersdk.ProvisionerJobMetadata](schemas.md#codersdkprovisionerjobmetadata) | false | | | | `»»» template_display_name` | string | false | | | | `»»» template_icon` | string | false | | | @@ -1816,6 +1821,7 @@ curl -X POST http://coder-server:8080/api/v2/workspaces/{workspace}/builds \ "template_version_id": "0ba39c92-1f1b-4c32-aa3e-9925d7713eb1", "workspace_build_id": "badaf2eb-96c5-4050-9f1d-db2d39ca5478" }, + "logs_overflowed": true, "metadata": { "template_display_name": "string", "template_icon": "string", diff --git a/docs/reference/api/organizations.md b/docs/reference/api/organizations.md index 497e3f56d4..d418a1fcba 100644 --- a/docs/reference/api/organizations.md +++ b/docs/reference/api/organizations.md @@ -407,6 +407,7 @@ curl -X GET http://coder-server:8080/api/v2/organizations/{organization}/provisi "template_version_id": "0ba39c92-1f1b-4c32-aa3e-9925d7713eb1", "workspace_build_id": "badaf2eb-96c5-4050-9f1d-db2d39ca5478" }, + "logs_overflowed": true, "metadata": { "template_display_name": "string", "template_icon": "string", @@ -457,6 +458,7 @@ Status Code **200** | `»» error` | string | false | | | | `»» template_version_id` | string(uuid) | false | | | | `»» workspace_build_id` | string(uuid) | false | | | +| `» logs_overflowed` | boolean | false | | | | `» metadata` | [codersdk.ProvisionerJobMetadata](schemas.md#codersdkprovisionerjobmetadata) | false | | | | `»» template_display_name` | string | false | | | | `»» template_icon` | string | false | | | @@ -534,6 +536,7 @@ curl -X GET http://coder-server:8080/api/v2/organizations/{organization}/provisi "template_version_id": "0ba39c92-1f1b-4c32-aa3e-9925d7713eb1", "workspace_build_id": "badaf2eb-96c5-4050-9f1d-db2d39ca5478" }, + "logs_overflowed": true, "metadata": { "template_display_name": "string", "template_icon": "string", diff --git a/docs/reference/api/schemas.md b/docs/reference/api/schemas.md index 033ef6e196..dcbd00628e 100644 --- a/docs/reference/api/schemas.md +++ b/docs/reference/api/schemas.md @@ -5906,6 +5906,7 @@ Only certain features set these fields: - FeatureManagedAgentLimit| "template_version_id": "0ba39c92-1f1b-4c32-aa3e-9925d7713eb1", "workspace_build_id": "badaf2eb-96c5-4050-9f1d-db2d39ca5478" }, + "logs_overflowed": true, "metadata": { "template_display_name": "string", "template_icon": "string", @@ -5943,6 +5944,7 @@ Only certain features set these fields: - FeatureManagedAgentLimit| | `file_id` | string | false | | | | `id` | string | false | | | | `input` | [codersdk.ProvisionerJobInput](#codersdkprovisionerjobinput) | false | | | +| `logs_overflowed` | boolean | false | | | | `metadata` | [codersdk.ProvisionerJobMetadata](#codersdkprovisionerjobmetadata) | false | | | | `organization_id` | string | false | | | | `queue_position` | integer | false | | | @@ -7626,6 +7628,7 @@ Restarts will only happen on weekdays in this list on weeks which line up with W "template_version_id": "0ba39c92-1f1b-4c32-aa3e-9925d7713eb1", "workspace_build_id": "badaf2eb-96c5-4050-9f1d-db2d39ca5478" }, + "logs_overflowed": true, "metadata": { "template_display_name": "string", "template_icon": "string", @@ -8802,6 +8805,7 @@ If the schedule is empty, the user will be updated to use the default schedule.| "template_version_id": "0ba39c92-1f1b-4c32-aa3e-9925d7713eb1", "workspace_build_id": "badaf2eb-96c5-4050-9f1d-db2d39ca5478" }, + "logs_overflowed": true, "metadata": { "template_display_name": "string", "template_icon": "string", @@ -9911,6 +9915,7 @@ If the schedule is empty, the user will be updated to use the default schedule.| "template_version_id": "0ba39c92-1f1b-4c32-aa3e-9925d7713eb1", "workspace_build_id": "badaf2eb-96c5-4050-9f1d-db2d39ca5478" }, + "logs_overflowed": true, "metadata": { "template_display_name": "string", "template_icon": "string", @@ -10642,6 +10647,7 @@ If the schedule is empty, the user will be updated to use the default schedule.| "template_version_id": "0ba39c92-1f1b-4c32-aa3e-9925d7713eb1", "workspace_build_id": "badaf2eb-96c5-4050-9f1d-db2d39ca5478" }, + "logs_overflowed": true, "metadata": { "template_display_name": "string", "template_icon": "string", diff --git a/docs/reference/api/templates.md b/docs/reference/api/templates.md index 0db4ef8d04..ea2e2c50cc 100644 --- a/docs/reference/api/templates.md +++ b/docs/reference/api/templates.md @@ -479,6 +479,7 @@ curl -X GET http://coder-server:8080/api/v2/organizations/{organization}/templat "template_version_id": "0ba39c92-1f1b-4c32-aa3e-9925d7713eb1", "workspace_build_id": "badaf2eb-96c5-4050-9f1d-db2d39ca5478" }, + "logs_overflowed": true, "metadata": { "template_display_name": "string", "template_icon": "string", @@ -577,6 +578,7 @@ curl -X GET http://coder-server:8080/api/v2/organizations/{organization}/templat "template_version_id": "0ba39c92-1f1b-4c32-aa3e-9925d7713eb1", "workspace_build_id": "badaf2eb-96c5-4050-9f1d-db2d39ca5478" }, + "logs_overflowed": true, "metadata": { "template_display_name": "string", "template_icon": "string", @@ -699,6 +701,7 @@ curl -X POST http://coder-server:8080/api/v2/organizations/{organization}/templa "template_version_id": "0ba39c92-1f1b-4c32-aa3e-9925d7713eb1", "workspace_build_id": "badaf2eb-96c5-4050-9f1d-db2d39ca5478" }, + "logs_overflowed": true, "metadata": { "template_display_name": "string", "template_icon": "string", @@ -1264,6 +1267,7 @@ curl -X GET http://coder-server:8080/api/v2/templates/{template}/versions \ "template_version_id": "0ba39c92-1f1b-4c32-aa3e-9925d7713eb1", "workspace_build_id": "badaf2eb-96c5-4050-9f1d-db2d39ca5478" }, + "logs_overflowed": true, "metadata": { "template_display_name": "string", "template_icon": "string", @@ -1337,6 +1341,7 @@ Status Code **200** | `»»» error` | string | false | | | | `»»» template_version_id` | string(uuid) | false | | | | `»»» workspace_build_id` | string(uuid) | false | | | +| `»» logs_overflowed` | boolean | false | | | | `»» metadata` | [codersdk.ProvisionerJobMetadata](schemas.md#codersdkprovisionerjobmetadata) | false | | | | `»»» template_display_name` | string | false | | | | `»»» template_icon` | string | false | | | @@ -1543,6 +1548,7 @@ curl -X GET http://coder-server:8080/api/v2/templates/{template}/versions/{templ "template_version_id": "0ba39c92-1f1b-4c32-aa3e-9925d7713eb1", "workspace_build_id": "badaf2eb-96c5-4050-9f1d-db2d39ca5478" }, + "logs_overflowed": true, "metadata": { "template_display_name": "string", "template_icon": "string", @@ -1616,6 +1622,7 @@ Status Code **200** | `»»» error` | string | false | | | | `»»» template_version_id` | string(uuid) | false | | | | `»»» workspace_build_id` | string(uuid) | false | | | +| `»» logs_overflowed` | boolean | false | | | | `»» metadata` | [codersdk.ProvisionerJobMetadata](schemas.md#codersdkprovisionerjobmetadata) | false | | | | `»»» template_display_name` | string | false | | | | `»»» template_icon` | string | false | | | @@ -1712,6 +1719,7 @@ curl -X GET http://coder-server:8080/api/v2/templateversions/{templateversion} \ "template_version_id": "0ba39c92-1f1b-4c32-aa3e-9925d7713eb1", "workspace_build_id": "badaf2eb-96c5-4050-9f1d-db2d39ca5478" }, + "logs_overflowed": true, "metadata": { "template_display_name": "string", "template_icon": "string", @@ -1819,6 +1827,7 @@ curl -X PATCH http://coder-server:8080/api/v2/templateversions/{templateversion} "template_version_id": "0ba39c92-1f1b-4c32-aa3e-9925d7713eb1", "workspace_build_id": "badaf2eb-96c5-4050-9f1d-db2d39ca5478" }, + "logs_overflowed": true, "metadata": { "template_display_name": "string", "template_icon": "string", @@ -2016,6 +2025,7 @@ curl -X POST http://coder-server:8080/api/v2/templateversions/{templateversion}/ "template_version_id": "0ba39c92-1f1b-4c32-aa3e-9925d7713eb1", "workspace_build_id": "badaf2eb-96c5-4050-9f1d-db2d39ca5478" }, + "logs_overflowed": true, "metadata": { "template_display_name": "string", "template_icon": "string", @@ -2089,6 +2099,7 @@ curl -X GET http://coder-server:8080/api/v2/templateversions/{templateversion}/d "template_version_id": "0ba39c92-1f1b-4c32-aa3e-9925d7713eb1", "workspace_build_id": "badaf2eb-96c5-4050-9f1d-db2d39ca5478" }, + "logs_overflowed": true, "metadata": { "template_display_name": "string", "template_icon": "string", diff --git a/docs/reference/api/workspaces.md b/docs/reference/api/workspaces.md index debcb421e0..d7187259b5 100644 --- a/docs/reference/api/workspaces.md +++ b/docs/reference/api/workspaces.md @@ -107,6 +107,7 @@ of the template will be used. "template_version_id": "0ba39c92-1f1b-4c32-aa3e-9925d7713eb1", "workspace_build_id": "badaf2eb-96c5-4050-9f1d-db2d39ca5478" }, + "logs_overflowed": true, "metadata": { "template_display_name": "string", "template_icon": "string", @@ -394,6 +395,7 @@ curl -X GET http://coder-server:8080/api/v2/users/{user}/workspace/{workspacenam "template_version_id": "0ba39c92-1f1b-4c32-aa3e-9925d7713eb1", "workspace_build_id": "badaf2eb-96c5-4050-9f1d-db2d39ca5478" }, + "logs_overflowed": true, "metadata": { "template_display_name": "string", "template_icon": "string", @@ -706,6 +708,7 @@ of the template will be used. "template_version_id": "0ba39c92-1f1b-4c32-aa3e-9925d7713eb1", "workspace_build_id": "badaf2eb-96c5-4050-9f1d-db2d39ca5478" }, + "logs_overflowed": true, "metadata": { "template_display_name": "string", "template_icon": "string", @@ -996,6 +999,7 @@ curl -X GET http://coder-server:8080/api/v2/workspaces \ "template_version_id": "0ba39c92-1f1b-4c32-aa3e-9925d7713eb1", "workspace_build_id": "badaf2eb-96c5-4050-9f1d-db2d39ca5478" }, + "logs_overflowed": true, "metadata": { "template_display_name": "string", "template_icon": "string", @@ -1267,6 +1271,7 @@ curl -X GET http://coder-server:8080/api/v2/workspaces/{workspace} \ "template_version_id": "0ba39c92-1f1b-4c32-aa3e-9925d7713eb1", "workspace_build_id": "badaf2eb-96c5-4050-9f1d-db2d39ca5478" }, + "logs_overflowed": true, "metadata": { "template_display_name": "string", "template_icon": "string", @@ -1670,6 +1675,7 @@ curl -X PUT http://coder-server:8080/api/v2/workspaces/{workspace}/dormant \ "template_version_id": "0ba39c92-1f1b-4c32-aa3e-9925d7713eb1", "workspace_build_id": "badaf2eb-96c5-4050-9f1d-db2d39ca5478" }, + "logs_overflowed": true, "metadata": { "template_display_name": "string", "template_icon": "string", diff --git a/docs/reference/cli/provisioner_jobs_list.md b/docs/reference/cli/provisioner_jobs_list.md index 07ad02f419..a0bff8554d 100644 --- a/docs/reference/cli/provisioner_jobs_list.md +++ b/docs/reference/cli/provisioner_jobs_list.md @@ -45,10 +45,10 @@ Select which organization (uuid or name) to use. ### -c, --column -| | | -|---------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| Type | [id\|created at\|started at\|completed at\|canceled at\|error\|error code\|status\|worker id\|worker name\|file id\|tags\|queue position\|queue size\|organization id\|template version id\|workspace build id\|type\|available workers\|template version name\|template id\|template name\|template display name\|template icon\|workspace id\|workspace name\|organization\|queue] | -| Default | created at,id,type,template display name,status,queue,tags | +| | | +|---------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| Type | [id\|created at\|started at\|completed at\|canceled at\|error\|error code\|status\|worker id\|worker name\|file id\|tags\|queue position\|queue size\|organization id\|template version id\|workspace build id\|type\|available workers\|template version name\|template id\|template name\|template display name\|template icon\|workspace id\|workspace name\|logs overflowed\|organization\|queue] | +| Default | created at,id,type,template display name,status,queue,tags | Columns to display in table output. diff --git a/enterprise/cli/testdata/coder_provisioner_jobs_list_--help.golden b/enterprise/cli/testdata/coder_provisioner_jobs_list_--help.golden index f380a03348..8e22f78e97 100644 --- a/enterprise/cli/testdata/coder_provisioner_jobs_list_--help.golden +++ b/enterprise/cli/testdata/coder_provisioner_jobs_list_--help.golden @@ -11,7 +11,7 @@ OPTIONS: -O, --org string, $CODER_ORGANIZATION Select which organization (uuid or name) to use. - -c, --column [id|created at|started at|completed at|canceled at|error|error code|status|worker id|worker name|file id|tags|queue position|queue size|organization id|template version id|workspace build id|type|available workers|template version name|template id|template name|template display name|template icon|workspace id|workspace name|organization|queue] (default: created at,id,type,template display name,status,queue,tags) + -c, --column [id|created at|started at|completed at|canceled at|error|error code|status|worker id|worker name|file id|tags|queue position|queue size|organization id|template version id|workspace build id|type|available workers|template version name|template id|template name|template display name|template icon|workspace id|workspace name|logs overflowed|organization|queue] (default: created at,id,type,template display name,status,queue,tags) Columns to display in table output. -l, --limit int, $CODER_PROVISIONER_JOB_LIST_LIMIT (default: 50) diff --git a/site/src/api/typesGenerated.ts b/site/src/api/typesGenerated.ts index 6165198c6f..3bd870d601 100644 --- a/site/src/api/typesGenerated.ts +++ b/site/src/api/typesGenerated.ts @@ -2156,6 +2156,7 @@ export interface ProvisionerJob { readonly type: ProvisionerJobType; readonly available_workers?: readonly string[]; readonly metadata: ProvisionerJobMetadata; + readonly logs_overflowed: boolean; } // From codersdk/provisionerdaemons.go diff --git a/site/src/modules/resources/AgentRow.tsx b/site/src/modules/resources/AgentRow.tsx index 20c551fc73..ab0e5884c4 100644 --- a/site/src/modules/resources/AgentRow.tsx +++ b/site/src/modules/resources/AgentRow.tsx @@ -87,7 +87,8 @@ export const AgentRow: FC = ({ logs.push({ id: -1, level: "error", - output: "Startup logs exceeded the max size of 1MB!", + output: + "Startup logs exceeded the max size of 1MB, and will not continue to be written to the database! Logs will continue to be written to the /tmp/coder-startup-script.log file in the workspace.", created_at: new Date().toISOString(), source_id: "", }); diff --git a/site/src/modules/workspaces/WorkspaceBuildLogs/WorkspaceBuildLogs.tsx b/site/src/modules/workspaces/WorkspaceBuildLogs/WorkspaceBuildLogs.tsx index fcf6f0dbee..20c929406d 100644 --- a/site/src/modules/workspaces/WorkspaceBuildLogs/WorkspaceBuildLogs.tsx +++ b/site/src/modules/workspaces/WorkspaceBuildLogs/WorkspaceBuildLogs.tsx @@ -1,9 +1,9 @@ import { type Interpolation, type Theme, useTheme } from "@emotion/react"; -import type { ProvisionerJobLog } from "api/typesGenerated"; +import type { ProvisionerJobLog, WorkspaceBuild } from "api/typesGenerated"; import type { Line } from "components/Logs/LogLine"; import { DEFAULT_LOG_LINE_SIDE_PADDING, Logs } from "components/Logs/Logs"; import dayjs from "dayjs"; -import { type FC, Fragment, type HTMLAttributes } from "react"; +import { type FC, Fragment, type HTMLAttributes, useMemo } from "react"; import { BODY_FONT_FAMILY, MONOSPACE_FONT_FAMILY } from "theme/constants"; const Language = { @@ -42,15 +42,37 @@ interface WorkspaceBuildLogsProps extends HTMLAttributes { hideTimestamps?: boolean; sticky?: boolean; logs: ProvisionerJobLog[]; + build?: WorkspaceBuild; } export const WorkspaceBuildLogs: FC = ({ hideTimestamps, sticky, logs, + build, ...attrs }) => { const theme = useTheme(); + + const processedLogs = useMemo(() => { + const allLogs = logs || []; + + // Add synthetic overflow message if needed + if (build?.job?.logs_overflowed) { + allLogs.push({ + id: -1, + created_at: new Date().toISOString(), + log_level: "error", + log_source: "provisioner", + output: + "Provisioner logs exceeded the max size of 1MB. Will not continue to write provisioner logs for workspace build.", + stage: "overflow", + }); + } + + return allLogs; + }, [logs, build?.job?.logs_overflowed]); + const groupedLogsByStage = groupLogsByStage(logs); return ( diff --git a/site/src/pages/WorkspaceBuildPage/WorkspaceBuildPageView.tsx b/site/src/pages/WorkspaceBuildPage/WorkspaceBuildPageView.tsx index 6add701c8b..3a45653557 100644 --- a/site/src/pages/WorkspaceBuildPage/WorkspaceBuildPageView.tsx +++ b/site/src/pages/WorkspaceBuildPage/WorkspaceBuildPageView.tsx @@ -212,7 +212,24 @@ export const WorkspaceBuildPageView: FC = ({ )} - {tabState.value === "build" && } + {build?.job?.logs_overflowed && ( + + Provisioner logs exceeded the max size of 1MB. Will not continue + to write provisioner logs for workspace build. + + )} + + {tabState.value === "build" && ( + + )} {tabState.value !== "build" && selectedAgent && ( )} @@ -261,7 +278,10 @@ const ScrollArea: FC> = (props) => { ); }; -const BuildLogsContent: FC<{ logs?: ProvisionerJobLog[] }> = ({ logs }) => { +const BuildLogsContent: FC<{ + logs?: ProvisionerJobLog[]; + build?: WorkspaceBuild; +}> = ({ logs, build }) => { if (!logs) { return ; } @@ -278,6 +298,7 @@ const BuildLogsContent: FC<{ logs?: ProvisionerJobLog[] }> = ({ logs }) => { }, }} logs={sortLogsByCreatedAt(logs)} + build={build} /> ); }; diff --git a/site/src/testHelpers/entities.ts b/site/src/testHelpers/entities.ts index f91a29cb48..78dd9e4e86 100644 --- a/site/src/testHelpers/entities.ts +++ b/site/src/testHelpers/entities.ts @@ -689,6 +689,7 @@ export const MockProvisionerJob: TypesGen.ProvisionerJob = { template_version_name: "test-version", workspace_name: "test-workspace", }, + logs_overflowed: false, }; export const MockFailedProvisionerJob: TypesGen.ProvisionerJob = {