refactor: Generalize log ownership to allow for scratch jobs (#182)

* refactor: Generalize log ownership to allow for scratch jobs Importing may fail when creating a project. We don't want to lose this output, but we don't want to allow users to create a failing project. This generalizes logs to soon enable one-off situations where a user can upload their archive, create a project, and watch the output parse to completion. * Improve file table schema by using hash * Fix racey test by allowing logs before * Add debug logging for PostgreSQL insert
2026-09-24 15:04:27 +08:00 · 2022-02-07 15:32:37 -06:00
parent bde732f2ef
commit ed705f6af2
26 changed files with 892 additions and 935 deletions
@@ -37,7 +37,7 @@ WHERE
      SKIP LOCKED
    LIMIT
      1
-  ) RETURNING id, created_at, updated_at, started_at, cancelled_at, completed_at, error, initiator_id, provisioner, type, project_id, input, worker_id
+  ) RETURNING id, created_at, updated_at, started_at, cancelled_at, completed_at, error, initiator_id, provisioner, type, input, worker_id
 `

 type AcquireProvisionerJobParams struct {
@@ -66,7 +66,6 @@ func (q *sqlQuerier) AcquireProvisionerJob(ctx context.Context, arg AcquireProvi
 		&i.InitiatorID,
 		&i.Provisioner,
 		&i.Type,
-		&i.ProjectID,
 		&i.Input,
 		&i.WorkerID,
 	)
@@ -107,6 +106,29 @@ func (q *sqlQuerier) GetAPIKeyByID(ctx context.Context, id string) (APIKey, erro
 	return i, err
 }

+const getFileByHash = `-- name: GetFileByHash :one
+SELECT
+  hash, created_at, mimetype, data
+FROM
+  file
+WHERE
+  hash = $1
+LIMIT
+  1
+`
+
+func (q *sqlQuerier) GetFileByHash(ctx context.Context, hash string) (File, error) {
+	row := q.db.QueryRowContext(ctx, getFileByHash, hash)
+	var i File
+	err := row.Scan(
+		&i.Hash,
+		&i.CreatedAt,
+		&i.Mimetype,
+		&i.Data,
+	)
+	return i, err
+}
+
 const getOrganizationByID = `-- name: GetOrganizationByID :one
 SELECT
  id, name, description, created_at, updated_at, "default", auto_off_threshold, cpu_provisioning_rate, memory_provisioning_rate, workspace_auto_off
@@ -500,57 +522,6 @@ func (q *sqlQuerier) GetProjectVersionByProjectIDAndName(ctx context.Context, ar
 	return i, err
 }

-const getProjectVersionLogsByIDBetween = `-- name: GetProjectVersionLogsByIDBetween :many
-SELECT
-  id, project_version_id, created_at, source, level, output
-FROM
-  project_version_log
-WHERE
-  project_version_id = $1
-  AND (
-    created_at >= $2
-    OR created_at <= $3
-  )
-ORDER BY
-  created_at
-`
-
-type GetProjectVersionLogsByIDBetweenParams struct {
-	ProjectVersionID uuid.UUID `db:"project_version_id" json:"project_version_id"`
-	CreatedAfter     time.Time `db:"created_after" json:"created_after"`
-	CreatedBefore    time.Time `db:"created_before" json:"created_before"`
-}
-
-func (q *sqlQuerier) GetProjectVersionLogsByIDBetween(ctx context.Context, arg GetProjectVersionLogsByIDBetweenParams) ([]ProjectVersionLog, error) {
-	rows, err := q.db.QueryContext(ctx, getProjectVersionLogsByIDBetween, arg.ProjectVersionID, arg.CreatedAfter, arg.CreatedBefore)
-	if err != nil {
-		return nil, err
-	}
-	defer rows.Close()
-	var items []ProjectVersionLog
-	for rows.Next() {
-		var i ProjectVersionLog
-		if err := rows.Scan(
-			&i.ID,
-			&i.ProjectVersionID,
-			&i.CreatedAt,
-			&i.Source,
-			&i.Level,
-			&i.Output,
-		); err != nil {
-			return nil, err
-		}
-		items = append(items, i)
-	}
-	if err := rows.Close(); err != nil {
-		return nil, err
-	}
-	if err := rows.Err(); err != nil {
-		return nil, err
-	}
-	return items, nil
-}
-
 const getProjectsByOrganizationIDs = `-- name: GetProjectsByOrganizationIDs :many
 SELECT
  id, created_at, updated_at, organization_id, name, provisioner, active_version_id
@@ -651,7 +622,7 @@ func (q *sqlQuerier) GetProvisionerDaemons(ctx context.Context) ([]ProvisionerDa

 const getProvisionerJobByID = `-- name: GetProvisionerJobByID :one
 SELECT
-  id, created_at, updated_at, started_at, cancelled_at, completed_at, error, initiator_id, provisioner, type, project_id, input, worker_id
+  id, created_at, updated_at, started_at, cancelled_at, completed_at, error, initiator_id, provisioner, type, input, worker_id
 FROM
  provisioner_job
 WHERE
@@ -672,13 +643,63 @@ func (q *sqlQuerier) GetProvisionerJobByID(ctx context.Context, id uuid.UUID) (P
 		&i.InitiatorID,
 		&i.Provisioner,
 		&i.Type,
-		&i.ProjectID,
 		&i.Input,
 		&i.WorkerID,
 	)
 	return i, err
 }

+const getProvisionerLogsByIDBetween = `-- name: GetProvisionerLogsByIDBetween :many
+SELECT
+  id, job_id, created_at, source, level, output
+FROM
+  provisioner_job_log
+WHERE
+  job_id = $1
+  AND (
+    created_at >= $2
+    OR created_at <= $3
+  )
+ORDER BY
+  created_at
+`
+
+type GetProvisionerLogsByIDBetweenParams struct {
+	JobID         uuid.UUID `db:"job_id" json:"job_id"`
+	CreatedAfter  time.Time `db:"created_after" json:"created_after"`
+	CreatedBefore time.Time `db:"created_before" json:"created_before"`
+}
+
+func (q *sqlQuerier) GetProvisionerLogsByIDBetween(ctx context.Context, arg GetProvisionerLogsByIDBetweenParams) ([]ProvisionerJobLog, error) {
+	rows, err := q.db.QueryContext(ctx, getProvisionerLogsByIDBetween, arg.JobID, arg.CreatedAfter, arg.CreatedBefore)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+	var items []ProvisionerJobLog
+	for rows.Next() {
+		var i ProvisionerJobLog
+		if err := rows.Scan(
+			&i.ID,
+			&i.JobID,
+			&i.CreatedAt,
+			&i.Source,
+			&i.Level,
+			&i.Output,
+		); err != nil {
+			return nil, err
+		}
+		items = append(items, i)
+	}
+	if err := rows.Close(); err != nil {
+		return nil, err
+	}
+	if err := rows.Err(); err != nil {
+		return nil, err
+	}
+	return items, nil
+}
+
 const getUserByEmailOrUsername = `-- name: GetUserByEmailOrUsername :one
 SELECT
  id, email, name, revoked, login_type, hashed_password, created_at, updated_at, temporary_password, avatar_hash, ssh_key_regenerated_at, username, dotfiles_git_uri, roles, status, relatime, gpg_key_regenerated_at, _decomissioned, shell
@@ -1011,57 +1032,6 @@ func (q *sqlQuerier) GetWorkspaceHistoryByWorkspaceIDWithoutAfter(ctx context.Co
 	return i, err
 }

-const getWorkspaceHistoryLogsByIDBetween = `-- name: GetWorkspaceHistoryLogsByIDBetween :many
-SELECT
-  id, workspace_history_id, created_at, source, level, output
-FROM
-  workspace_history_log
-WHERE
-  workspace_history_id = $1
-  AND (
-    created_at >= $2
-    OR created_at <= $3
-  )
-ORDER BY
-  created_at
-`
-
-type GetWorkspaceHistoryLogsByIDBetweenParams struct {
-	WorkspaceHistoryID uuid.UUID `db:"workspace_history_id" json:"workspace_history_id"`
-	CreatedAfter       time.Time `db:"created_after" json:"created_after"`
-	CreatedBefore      time.Time `db:"created_before" json:"created_before"`
-}
-
-func (q *sqlQuerier) GetWorkspaceHistoryLogsByIDBetween(ctx context.Context, arg GetWorkspaceHistoryLogsByIDBetweenParams) ([]WorkspaceHistoryLog, error) {
-	rows, err := q.db.QueryContext(ctx, getWorkspaceHistoryLogsByIDBetween, arg.WorkspaceHistoryID, arg.CreatedAfter, arg.CreatedBefore)
-	if err != nil {
-		return nil, err
-	}
-	defer rows.Close()
-	var items []WorkspaceHistoryLog
-	for rows.Next() {
-		var i WorkspaceHistoryLog
-		if err := rows.Scan(
-			&i.ID,
-			&i.WorkspaceHistoryID,
-			&i.CreatedAt,
-			&i.Source,
-			&i.Level,
-			&i.Output,
-		); err != nil {
-			return nil, err
-		}
-		items = append(items, i)
-	}
-	if err := rows.Close(); err != nil {
-		return nil, err
-	}
-	if err := rows.Err(); err != nil {
-		return nil, err
-	}
-	return items, nil
-}
-
 const getWorkspaceResourcesByHistoryID = `-- name: GetWorkspaceResourcesByHistoryID :many
 SELECT
  id, created_at, workspace_history_id, type, name, workspace_agent_token, workspace_agent_id
@@ -1282,6 +1252,37 @@ func (q *sqlQuerier) InsertAPIKey(ctx context.Context, arg InsertAPIKeyParams) (
 	return i, err
 }

+const insertFile = `-- name: InsertFile :one
+INSERT INTO
+  file (hash, created_at, mimetype, data)
+VALUES
+  ($1, $2, $3, $4) RETURNING hash, created_at, mimetype, data
+`
+
+type InsertFileParams struct {
+	Hash      string    `db:"hash" json:"hash"`
+	CreatedAt time.Time `db:"created_at" json:"created_at"`
+	Mimetype  string    `db:"mimetype" json:"mimetype"`
+	Data      []byte    `db:"data" json:"data"`
+}
+
+func (q *sqlQuerier) InsertFile(ctx context.Context, arg InsertFileParams) (File, error) {
+	row := q.db.QueryRowContext(ctx, insertFile,
+		arg.Hash,
+		arg.CreatedAt,
+		arg.Mimetype,
+		arg.Data,
+	)
+	var i File
+	err := row.Scan(
+		&i.Hash,
+		&i.CreatedAt,
+		&i.Mimetype,
+		&i.Data,
+	)
+	return i, err
+}
+
 const insertOrganization = `-- name: InsertOrganization :one
 INSERT INTO
  organizations (id, name, description, created_at, updated_at)
@@ -1628,64 +1629,6 @@ func (q *sqlQuerier) InsertProjectVersion(ctx context.Context, arg InsertProject
 	return i, err
 }

-const insertProjectVersionLogs = `-- name: InsertProjectVersionLogs :many
-INSERT INTO
-  project_version_log
-SELECT
-  $1 :: uuid AS project_version_id,
-  unnest($2 :: uuid [ ]) AS id,
-  unnest($3 :: timestamptz [ ]) AS created_at,
-  unnest($4 :: log_source [ ]) as source,
-  unnest($5 :: log_level [ ]) as level,
-  unnest($6 :: varchar(1024) [ ]) as output RETURNING id, project_version_id, created_at, source, level, output
-`
-
-type InsertProjectVersionLogsParams struct {
-	ProjectVersionID uuid.UUID   `db:"project_version_id" json:"project_version_id"`
-	ID               []uuid.UUID `db:"id" json:"id"`
-	CreatedAt        []time.Time `db:"created_at" json:"created_at"`
-	Source           []LogSource `db:"source" json:"source"`
-	Level            []LogLevel  `db:"level" json:"level"`
-	Output           []string    `db:"output" json:"output"`
-}
-
-func (q *sqlQuerier) InsertProjectVersionLogs(ctx context.Context, arg InsertProjectVersionLogsParams) ([]ProjectVersionLog, error) {
-	rows, err := q.db.QueryContext(ctx, insertProjectVersionLogs,
-		arg.ProjectVersionID,
-		pq.Array(arg.ID),
-		pq.Array(arg.CreatedAt),
-		pq.Array(arg.Source),
-		pq.Array(arg.Level),
-		pq.Array(arg.Output),
-	)
-	if err != nil {
-		return nil, err
-	}
-	defer rows.Close()
-	var items []ProjectVersionLog
-	for rows.Next() {
-		var i ProjectVersionLog
-		if err := rows.Scan(
-			&i.ID,
-			&i.ProjectVersionID,
-			&i.CreatedAt,
-			&i.Source,
-			&i.Level,
-			&i.Output,
-		); err != nil {
-			return nil, err
-		}
-		items = append(items, i)
-	}
-	if err := rows.Close(); err != nil {
-		return nil, err
-	}
-	if err := rows.Err(); err != nil {
-		return nil, err
-	}
-	return items, nil
-}
-
 const insertProvisionerDaemon = `-- name: InsertProvisionerDaemon :one
 INSERT INTO
  provisioner_daemon (id, created_at, name, provisioners)
@@ -1727,11 +1670,10 @@ INSERT INTO
    initiator_id,
    provisioner,
    type,
-    project_id,
    input
  )
 VALUES
-  ($1, $2, $3, $4, $5, $6, $7, $8) RETURNING id, created_at, updated_at, started_at, cancelled_at, completed_at, error, initiator_id, provisioner, type, project_id, input, worker_id
+  ($1, $2, $3, $4, $5, $6, $7) RETURNING id, created_at, updated_at, started_at, cancelled_at, completed_at, error, initiator_id, provisioner, type, input, worker_id
 `

 type InsertProvisionerJobParams struct {
@@ -1741,7 +1683,6 @@ type InsertProvisionerJobParams struct {
 	InitiatorID string             `db:"initiator_id" json:"initiator_id"`
 	Provisioner ProvisionerType    `db:"provisioner" json:"provisioner"`
 	Type        ProvisionerJobType `db:"type" json:"type"`
-	ProjectID   uuid.UUID          `db:"project_id" json:"project_id"`
 	Input       json.RawMessage    `db:"input" json:"input"`
 }

@@ -1753,7 +1694,6 @@ func (q *sqlQuerier) InsertProvisionerJob(ctx context.Context, arg InsertProvisi
 		arg.InitiatorID,
 		arg.Provisioner,
 		arg.Type,
-		arg.ProjectID,
 		arg.Input,
 	)
 	var i ProvisionerJob
@@ -1768,13 +1708,70 @@ func (q *sqlQuerier) InsertProvisionerJob(ctx context.Context, arg InsertProvisi
 		&i.InitiatorID,
 		&i.Provisioner,
 		&i.Type,
-		&i.ProjectID,
 		&i.Input,
 		&i.WorkerID,
 	)
 	return i, err
 }

+const insertProvisionerJobLogs = `-- name: InsertProvisionerJobLogs :many
+INSERT INTO
+  provisioner_job_log
+SELECT
+  unnest($1 :: uuid [ ]) AS id,
+  $2 :: uuid AS job_id,
+  unnest($3 :: timestamptz [ ]) AS created_at,
+  unnest($4 :: log_source [ ]) as source,
+  unnest($5 :: log_level [ ]) as level,
+  unnest($6 :: varchar(1024) [ ]) as output RETURNING id, job_id, created_at, source, level, output
+`
+
+type InsertProvisionerJobLogsParams struct {
+	ID        []uuid.UUID `db:"id" json:"id"`
+	JobID     uuid.UUID   `db:"job_id" json:"job_id"`
+	CreatedAt []time.Time `db:"created_at" json:"created_at"`
+	Source    []LogSource `db:"source" json:"source"`
+	Level     []LogLevel  `db:"level" json:"level"`
+	Output    []string    `db:"output" json:"output"`
+}
+
+func (q *sqlQuerier) InsertProvisionerJobLogs(ctx context.Context, arg InsertProvisionerJobLogsParams) ([]ProvisionerJobLog, error) {
+	rows, err := q.db.QueryContext(ctx, insertProvisionerJobLogs,
+		pq.Array(arg.ID),
+		arg.JobID,
+		pq.Array(arg.CreatedAt),
+		pq.Array(arg.Source),
+		pq.Array(arg.Level),
+		pq.Array(arg.Output),
+	)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+	var items []ProvisionerJobLog
+	for rows.Next() {
+		var i ProvisionerJobLog
+		if err := rows.Scan(
+			&i.ID,
+			&i.JobID,
+			&i.CreatedAt,
+			&i.Source,
+			&i.Level,
+			&i.Output,
+		); err != nil {
+			return nil, err
+		}
+		items = append(items, i)
+	}
+	if err := rows.Close(); err != nil {
+		return nil, err
+	}
+	if err := rows.Err(); err != nil {
+		return nil, err
+	}
+	return items, nil
+}
+
 const insertUser = `-- name: InsertUser :one
 INSERT INTO
  users (
@@ -1992,64 +1989,6 @@ func (q *sqlQuerier) InsertWorkspaceHistory(ctx context.Context, arg InsertWorks
 	return i, err
 }

-const insertWorkspaceHistoryLogs = `-- name: InsertWorkspaceHistoryLogs :many
-INSERT INTO
-  workspace_history_log
-SELECT
-  unnest($1 :: uuid [ ]) AS id,
-  $2 :: uuid AS workspace_history_id,
-  unnest($3 :: timestamptz [ ]) AS created_at,
-  unnest($4 :: log_source [ ]) as source,
-  unnest($5 :: log_level [ ]) as level,
-  unnest($6 :: varchar(1024) [ ]) as output RETURNING id, workspace_history_id, created_at, source, level, output
-`
-
-type InsertWorkspaceHistoryLogsParams struct {
-	ID                 []uuid.UUID `db:"id" json:"id"`
-	WorkspaceHistoryID uuid.UUID   `db:"workspace_history_id" json:"workspace_history_id"`
-	CreatedAt          []time.Time `db:"created_at" json:"created_at"`
-	Source             []LogSource `db:"source" json:"source"`
-	Level              []LogLevel  `db:"level" json:"level"`
-	Output             []string    `db:"output" json:"output"`
-}
-
-func (q *sqlQuerier) InsertWorkspaceHistoryLogs(ctx context.Context, arg InsertWorkspaceHistoryLogsParams) ([]WorkspaceHistoryLog, error) {
-	rows, err := q.db.QueryContext(ctx, insertWorkspaceHistoryLogs,
-		pq.Array(arg.ID),
-		arg.WorkspaceHistoryID,
-		pq.Array(arg.CreatedAt),
-		pq.Array(arg.Source),
-		pq.Array(arg.Level),
-		pq.Array(arg.Output),
-	)
-	if err != nil {
-		return nil, err
-	}
-	defer rows.Close()
-	var items []WorkspaceHistoryLog
-	for rows.Next() {
-		var i WorkspaceHistoryLog
-		if err := rows.Scan(
-			&i.ID,
-			&i.WorkspaceHistoryID,
-			&i.CreatedAt,
-			&i.Source,
-			&i.Level,
-			&i.Output,
-		); err != nil {
-			return nil, err
-		}
-		items = append(items, i)
-	}
-	if err := rows.Close(); err != nil {
-		return nil, err
-	}
-	if err := rows.Err(); err != nil {
-		return nil, err
-	}
-	return items, nil
-}
-
 const insertWorkspaceResource = `-- name: InsertWorkspaceResource :one
 INSERT INTO
  workspace_resource (