opensource/coder
3
0
Fork 0
mirror of https://github.com/coder/coder.git synced 2026-09-24 15:04:27 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat: Add buffering to provisioner job logs (#4918)

Browse Source 
* feat: Add bufferring to provisioner job logs

This should improve overall build performance, and especially under load.

It removes the old `id` column on the `provisioner_job_logs` table
and replaces it with an auto-incrementing big integer to preserve order.

Funny enough, we never had to care about order before because inserts
would at minimum be 1ms different. Now they aren't, so the order needs
to be preserved.

* Fix log bufferring

* Fix frontend log streaming

* Fix JS test
This commit is contained in:
Kyle Carberry
2022-11-06 20:50:34 -06:00
committed by GitHub
parent 531f7cd489
commit 30281852d6
34 changed files with 300 additions and 476 deletions
Download Patch File Download Diff File Expand all files Collapse all files
coderd/database/databasefake/databasefake.go
+8 -6
View File
@@ -2052,17 +2052,14 @@ func (q *fakeQuerier) GetProvisionerLogsByIDBetween(_ context.Context, arg datab
if jobLog.JobID != arg.JobID {
continue
}
if !arg.CreatedBefore.IsZero() && jobLog.CreatedAt.After(arg.CreatedBefore) {
if arg.CreatedBefore != 0 && jobLog.ID > arg.CreatedBefore {
continue
}
if !arg.CreatedAfter.IsZero() && jobLog.CreatedAt.Before(arg.CreatedAfter) {
if arg.CreatedAfter != 0 && jobLog.ID < arg.CreatedAfter {
continue
}
logs = append(logs, jobLog)
}
if len(logs) == 0 {
return nil, sql.ErrNoRows
}
return logs, nil
}
@@ -2212,10 +2209,15 @@ func (q *fakeQuerier) InsertProvisionerJobLogs(_ context.Context, arg database.I
defer q.mutex.Unlock()
logs := make([]database.ProvisionerJobLog, 0)
id := int64(1)
if len(q.provisionerJobLogs) > 0 {
id = q.provisionerJobLogs[len(q.provisionerJobLogs)-1].ID
}
for index, output := range arg.Output {
id++
logs = append(logs, database.ProvisionerJobLog{
ID: id,
JobID: arg.JobID,
ID: arg.ID[index],
CreatedAt: arg.CreatedAt[index],
Source: arg.Source[index],
Level: arg.Level[index],
coderd/database/dump.sql
Generated
+13 -2
View File
@@ -272,15 +272,24 @@ CREATE TABLE provisioner_daemons (
);
CREATE TABLE provisioner_job_logs (
id uuid NOT NULL,
job_id uuid NOT NULL,
created_at timestamp with time zone NOT NULL,
source log_source NOT NULL,
level log_level NOT NULL,
stage character varying(128) NOT NULL,
output character varying(1024) NOT NULL
output character varying(1024) NOT NULL,
id bigint NOT NULL
);
CREATE SEQUENCE provisioner_job_logs_id_seq
START WITH 1
INCREMENT BY 1
NO MINVALUE
NO MAXVALUE
CACHE 1;
ALTER SEQUENCE provisioner_job_logs_id_seq OWNED BY provisioner_job_logs.id;
CREATE TABLE provisioner_jobs (
id uuid NOT NULL,
created_at timestamp with time zone NOT NULL,
@@ -463,6 +472,8 @@ CREATE TABLE workspaces (
ALTER TABLE ONLY licenses ALTER COLUMN id SET DEFAULT nextval('licenses_id_seq'::regclass);
ALTER TABLE ONLY provisioner_job_logs ALTER COLUMN id SET DEFAULT nextval('provisioner_job_logs_id_seq'::regclass);
ALTER TABLE ONLY agent_stats
ADD CONSTRAINT agent_stats_pkey PRIMARY KEY (id);
coderd/database/migrations/000071_provisioner_log_lines.down.sql
+3
View File
@@ -0,0 +1,3 @@
ALTER TABLE provisioner_job_logs DROP COLUMN id;
ALTER TABLE provisioner_job_logs ADD COLUMN id uuid NOT NULL DEFAULT gen_random_uuid();
coderd/database/migrations/000071_provisioner_log_lines.up.sql
+3
View File
@@ -0,0 +1,3 @@
ALTER TABLE provisioner_job_logs DROP COLUMN id;
ALTER TABLE provisioner_job_logs ADD COLUMN id BIGSERIAL PRIMARY KEY;
coderd/database/models.go
+1 -1
View File
@@ -545,13 +545,13 @@ type ProvisionerJob struct {
}
type ProvisionerJobLog struct {
ID uuid.UUID `db:"id" json:"id"`
JobID uuid.UUID `db:"job_id" json:"job_id"`
CreatedAt time.Time `db:"created_at" json:"created_at"`
Source LogSource `db:"source" json:"source"`
Level LogLevel `db:"level" json:"level"`
Stage string `db:"stage" json:"stage"`
Output string `db:"output" json:"output"`
ID int64 `db:"id" json:"id"`
}
type Replica struct {
coderd/database/queries.sql.go
+14 -19
View File
@@ -2345,23 +2345,21 @@ func (q *sqlQuerier) UpdateProvisionerDaemonByID(ctx context.Context, arg Update
const getProvisionerLogsByIDBetween = `-- name: GetProvisionerLogsByIDBetween :many
SELECT
id, job_id, created_at, source, level, stage, output
job_id, created_at, source, level, stage, output, id
FROM
provisioner_job_logs
WHERE
job_id = $1
AND (
created_at >= $2
OR created_at <= $3
)
ORDER BY
created_at DESC
id > $2
OR id < $3
) ORDER BY id
`
type GetProvisionerLogsByIDBetweenParams struct {
JobID uuid.UUID `db:"job_id" json:"job_id"`
CreatedAfter time.Time `db:"created_after" json:"created_after"`
CreatedBefore time.Time `db:"created_before" json:"created_before"`
CreatedAfter int64 `db:"created_after" json:"created_after"`
CreatedBefore int64 `db:"created_before" json:"created_before"`
}
func (q *sqlQuerier) GetProvisionerLogsByIDBetween(ctx context.Context, arg GetProvisionerLogsByIDBetweenParams) ([]ProvisionerJobLog, error) {
@@ -2374,13 +2372,13 @@ func (q *sqlQuerier) GetProvisionerLogsByIDBetween(ctx context.Context, arg GetP
for rows.Next() {
var i ProvisionerJobLog
if err := rows.Scan(
&i.ID,
&i.JobID,
&i.CreatedAt,
&i.Source,
&i.Level,
&i.Stage,
&i.Output,
&i.ID,
); err != nil {
return nil, err
}
@@ -2399,17 +2397,15 @@ const insertProvisionerJobLogs = `-- name: InsertProvisionerJobLogs :many
INSERT INTO
provisioner_job_logs
SELECT
unnest($1 :: uuid [ ]) AS id,
$2 :: uuid AS job_id,
unnest($3 :: timestamptz [ ]) AS created_at,
unnest($4 :: log_source [ ]) AS source,
unnest($5 :: log_level [ ]) AS LEVEL,
unnest($6 :: VARCHAR(128) [ ]) AS stage,
unnest($7 :: VARCHAR(1024) [ ]) AS output RETURNING id, job_id, created_at, source, level, stage, output
$1 :: uuid AS job_id,
unnest($2 :: timestamptz [ ]) AS created_at,
unnest($3 :: log_source [ ]) AS source,
unnest($4 :: log_level [ ]) AS LEVEL,
unnest($5 :: VARCHAR(128) [ ]) AS stage,
unnest($6 :: VARCHAR(1024) [ ]) AS output RETURNING job_id, created_at, source, level, stage, output, id
`
type InsertProvisionerJobLogsParams struct {
ID []uuid.UUID `db:"id" json:"id"`
JobID uuid.UUID `db:"job_id" json:"job_id"`
CreatedAt []time.Time `db:"created_at" json:"created_at"`
Source []LogSource `db:"source" json:"source"`
@@ -2420,7 +2416,6 @@ type InsertProvisionerJobLogsParams struct {
func (q *sqlQuerier) InsertProvisionerJobLogs(ctx context.Context, arg InsertProvisionerJobLogsParams) ([]ProvisionerJobLog, error) {
rows, err := q.db.QueryContext(ctx, insertProvisionerJobLogs,
pq.Array(arg.ID),
arg.JobID,
pq.Array(arg.CreatedAt),
pq.Array(arg.Source),
@@ -2436,13 +2431,13 @@ func (q *sqlQuerier) InsertProvisionerJobLogs(ctx context.Context, arg InsertPro
for rows.Next() {
var i ProvisionerJobLog
if err := rows.Scan(
&i.ID,
&i.JobID,
&i.CreatedAt,
&i.Source,
&i.Level,
&i.Stage,
&i.Output,
&i.ID,
); err != nil {
return nil, err
}
coderd/database/queries/provisionerjoblogs.sql
+3 -6
View File
@@ -6,17 +6,14 @@ FROM
WHERE
job_id = @job_id
AND (
created_at >= @created_after
OR created_at <= @created_before
)
ORDER BY
created_at DESC;
id > @created_after
OR id < @created_before
) ORDER BY id;
-- name: InsertProvisionerJobLogs :many
INSERT INTO
provisioner_job_logs
SELECT
unnest(@id :: uuid [ ]) AS id,
@job_id :: uuid AS job_id,
unnest(@created_at :: timestamptz [ ]) AS created_at,
unnest(@source :: log_source [ ]) AS source,