opensource/coder
3
0
Fork 0
mirror of https://github.com/coder/coder.git synced 2026-09-24 15:04:27 +08:00
Code Issues Packages Projects Releases Wiki Activity

refactor: Allow provisioner jobs to be disconnected from projects (#194)

Browse Source 
* Nest jobs under an organization

* Rename project parameter to parameter schema

* Update references when computing project parameters

* Add files endpoint

* Allow one-off project import jobs

* Allow variables to be injected that are not defined by the schema

* Update API to use jobs first

* Fix CLI tests

* Fix linting

* Fix hex length for files table

* Reduce memory allocation for windows
This commit is contained in:
Kyle Carberry
2022-02-08 12:00:44 -06:00
committed by GitHub
parent 4c5e443b63
commit 7364933e65
37 changed files with 1380 additions and 995 deletions
Download Patch File Download Diff File Expand all files Collapse all files
coderd/projectparameter/projectparameter.go
+67 -54
View File
@@ -15,11 +15,11 @@ import (
// Scope targets identifiers to pull parameters from.
type Scope struct {
OrganizationID string
ProjectID uuid.UUID
ProjectVersionID uuid.UUID
UserID sql.NullString
WorkspaceID uuid.NullUUID
ImportJobID uuid.UUID
OrganizationID string
ProjectID uuid.NullUUID
UserID sql.NullString
WorkspaceID uuid.NullUUID
}
// Value represents a computed parameter.
@@ -35,29 +35,27 @@ type Value struct {
// Compute accepts a scope in which parameter values are sourced.
// These sources are iterated in a hierarchical fashion to determine
// the runtime parameter values for a project.
func Compute(ctx context.Context, db database.Store, scope Scope) ([]Value, error) {
func Compute(ctx context.Context, db database.Store, scope Scope, additional ...database.ParameterValue) ([]Value, error) {
compute := &compute{
db: db,
computedParameterByName: map[string]Value{},
projectVersionParametersByName: map[string]database.ProjectVersionParameter{},
db: db,
computedParameterByName: map[string]Value{},
parameterSchemasByName: map[string]database.ParameterSchema{},
}
// All parameters for the project version!
projectVersionParameters, err := db.GetProjectVersionParametersByVersionID(ctx, scope.ProjectVersionID)
// All parameters for the import job ID!
parameterSchemas, err := db.GetParameterSchemasByJobID(ctx, scope.ImportJobID)
if errors.Is(err, sql.ErrNoRows) {
// This occurs when the project version has defined
// no parameters, so we have nothing to compute!
return []Value{}, nil
err = nil
}
if err != nil {
return nil, xerrors.Errorf("get project parameters: %w", err)
}
for _, projectVersionParameter := range projectVersionParameters {
compute.projectVersionParametersByName[projectVersionParameter.Name] = projectVersionParameter
for _, projectVersionParameter := range parameterSchemas {
compute.parameterSchemasByName[projectVersionParameter.Name] = projectVersionParameter
}
// Organization parameters come first!
err = compute.inject(ctx, database.GetParameterValuesByScopeParams{
err = compute.injectScope(ctx, database.GetParameterValuesByScopeParams{
Scope: database.ParameterScopeOrganization,
ScopeID: scope.OrganizationID,
})
@@ -66,7 +64,7 @@ func Compute(ctx context.Context, db database.Store, scope Scope) ([]Value, erro
}
// Default project parameter values come second!
for _, projectVersionParameter := range projectVersionParameters {
for _, projectVersionParameter := range parameterSchemas {
if !projectVersionParameter.DefaultSourceValue.Valid {
continue
}
@@ -89,25 +87,27 @@ func Compute(ctx context.Context, db database.Store, scope Scope) ([]Value, erro
},
DefaultValue: true,
Scope: database.ParameterScopeProject,
ScopeID: scope.ProjectID.String(),
ScopeID: scope.ProjectID.UUID.String(),
}
default:
return nil, xerrors.Errorf("unsupported source scheme for project version parameter %q: %q", projectVersionParameter.Name, string(projectVersionParameter.DefaultSourceScheme))
}
}
// Project parameters come third!
err = compute.inject(ctx, database.GetParameterValuesByScopeParams{
Scope: database.ParameterScopeProject,
ScopeID: scope.ProjectID.String(),
})
if err != nil {
return nil, err
if scope.ProjectID.Valid {
// Project parameters come third!
err = compute.injectScope(ctx, database.GetParameterValuesByScopeParams{
Scope: database.ParameterScopeProject,
ScopeID: scope.ProjectID.UUID.String(),
})
if err != nil {
return nil, err
}
}
if scope.UserID.Valid {
// User parameters come fourth!
err = compute.inject(ctx, database.GetParameterValuesByScopeParams{
err = compute.injectScope(ctx, database.GetParameterValuesByScopeParams{
Scope: database.ParameterScopeUser,
ScopeID: scope.UserID.String,
})
@@ -118,7 +118,7 @@ func Compute(ctx context.Context, db database.Store, scope Scope) ([]Value, erro
if scope.WorkspaceID.Valid {
// Workspace parameters come last!
err = compute.inject(ctx, database.GetParameterValuesByScopeParams{
err = compute.injectScope(ctx, database.GetParameterValuesByScopeParams{
Scope: database.ParameterScopeWorkspace,
ScopeID: scope.WorkspaceID.UUID.String(),
})
@@ -127,7 +127,14 @@ func Compute(ctx context.Context, db database.Store, scope Scope) ([]Value, erro
}
}
for _, projectVersionParameter := range compute.projectVersionParametersByName {
for _, parameterValue := range additional {
err = compute.injectSingle(parameterValue)
if err != nil {
return nil, xerrors.Errorf("inject %q: %w", parameterValue.Name, err)
}
}
for _, projectVersionParameter := range compute.parameterSchemasByName {
if _, ok := compute.computedParameterByName[projectVersionParameter.Name]; ok {
continue
}
@@ -145,13 +152,13 @@ func Compute(ctx context.Context, db database.Store, scope Scope) ([]Value, erro
}
type compute struct {
db database.Store
computedParameterByName map[string]Value
projectVersionParametersByName map[string]database.ProjectVersionParameter
db database.Store
computedParameterByName map[string]Value
parameterSchemasByName map[string]database.ParameterSchema
}
// Validates and computes the value for parameters; setting the value on "parameterByName".
func (c *compute) inject(ctx context.Context, scopeParams database.GetParameterValuesByScopeParams) error {
func (c *compute) injectScope(ctx context.Context, scopeParams database.GetParameterValuesByScopeParams) error {
scopedParameters, err := c.db.GetParameterValuesByScope(ctx, scopeParams)
if errors.Is(err, sql.ErrNoRows) {
err = nil
@@ -161,39 +168,45 @@ func (c *compute) inject(ctx context.Context, scopeParams database.GetParameterV
}
for _, scopedParameter := range scopedParameters {
projectVersionParameter, hasProjectVersionParameter := c.projectVersionParametersByName[scopedParameter.Name]
if !hasProjectVersionParameter {
// Don't inject parameters that aren't defined by the project.
continue
err = c.injectSingle(scopedParameter)
if err != nil {
return xerrors.Errorf("inject single %q: %w", scopedParameter.Name, err)
}
}
return nil
}
func (c *compute) injectSingle(scopedParameter database.ParameterValue) error {
parameterSchema, hasParameterSchema := c.parameterSchemasByName[scopedParameter.Name]
if hasParameterSchema {
// Don't inject parameters that aren't defined by the project.
_, hasExistingParameter := c.computedParameterByName[scopedParameter.Name]
if hasExistingParameter {
// If a parameter already exists, check if this variable can override it.
// Injection hierarchy is the responsibility of the caller. This check ensures
// project parameters cannot be overridden if already set.
if !projectVersionParameter.AllowOverrideSource && scopedParameter.Scope != database.ParameterScopeProject {
continue
if !parameterSchema.AllowOverrideSource && scopedParameter.Scope != database.ParameterScopeProject {
return nil
}
}
}
destinationScheme, err := convertDestinationScheme(scopedParameter.DestinationScheme)
if err != nil {
return xerrors.Errorf("convert destination scheme: %w", err)
}
destinationScheme, err := convertDestinationScheme(scopedParameter.DestinationScheme)
if err != nil {
return xerrors.Errorf("convert destination scheme: %w", err)
}
switch scopedParameter.SourceScheme {
case database.ParameterSourceSchemeData:
c.computedParameterByName[projectVersionParameter.Name] = Value{
Proto: &proto.ParameterValue{
DestinationScheme: destinationScheme,
Name: scopedParameter.SourceValue,
Value: scopedParameter.DestinationValue,
},
}
default:
return xerrors.Errorf("unsupported source scheme: %q", string(projectVersionParameter.DefaultSourceScheme))
switch scopedParameter.SourceScheme {
case database.ParameterSourceSchemeData:
c.computedParameterByName[scopedParameter.Name] = Value{
Proto: &proto.ParameterValue{
DestinationScheme: destinationScheme,
Name: scopedParameter.SourceValue,
Value: scopedParameter.DestinationValue,
},
}
default:
return xerrors.Errorf("unsupported source scheme: %q", string(parameterSchema.DefaultSourceScheme))
}
return nil
}
coderd/projectparameter/projectparameter_test.go
+54 -18
View File
@@ -19,9 +19,12 @@ func TestCompute(t *testing.T) {
t.Parallel()
generateScope := func() projectparameter.Scope {
return projectparameter.Scope{
OrganizationID: uuid.New().String(),
ProjectID: uuid.New(),
ProjectVersionID: uuid.New(),
ImportJobID: uuid.New(),
OrganizationID: uuid.NewString(),
ProjectID: uuid.NullUUID{
UUID: uuid.New(),
Valid: true,
},
WorkspaceID: uuid.NullUUID{
UUID: uuid.New(),
Valid: true,
@@ -36,9 +39,9 @@ func TestCompute(t *testing.T) {
AllowOverrideSource bool
AllowOverrideDestination bool
DefaultDestinationScheme database.ParameterDestinationScheme
ProjectVersionID uuid.UUID
ImportJobID uuid.UUID
}
generateProjectParameter := func(t *testing.T, db database.Store, opts projectParameterOptions) database.ProjectVersionParameter {
generateProjectParameter := func(t *testing.T, db database.Store, opts projectParameterOptions) database.ParameterSchema {
if opts.DefaultDestinationScheme == "" {
opts.DefaultDestinationScheme = database.ParameterDestinationSchemeEnvironmentVariable
}
@@ -48,10 +51,10 @@ func TestCompute(t *testing.T) {
require.NoError(t, err)
destinationValue, err := cryptorand.String(8)
require.NoError(t, err)
param, err := db.InsertProjectVersionParameter(context.Background(), database.InsertProjectVersionParameterParams{
param, err := db.InsertParameterSchema(context.Background(), database.InsertParameterSchemaParams{
ID: uuid.New(),
Name: name,
ProjectVersionID: opts.ProjectVersionID,
JobID: opts.ImportJobID,
DefaultSourceScheme: database.ParameterSourceSchemeData,
DefaultSourceValue: sql.NullString{
String: sourceValue,
@@ -73,10 +76,10 @@ func TestCompute(t *testing.T) {
t.Parallel()
db := databasefake.New()
scope := generateScope()
parameter, err := db.InsertProjectVersionParameter(context.Background(), database.InsertProjectVersionParameterParams{
ID: uuid.New(),
ProjectVersionID: scope.ProjectVersionID,
Name: "hey",
parameter, err := db.InsertParameterSchema(context.Background(), database.InsertParameterSchemaParams{
ID: uuid.New(),
JobID: scope.ImportJobID,
Name: "hey",
})
require.NoError(t, err)
@@ -92,7 +95,7 @@ func TestCompute(t *testing.T) {
db := databasefake.New()
scope := generateScope()
parameter := generateProjectParameter(t, db, projectParameterOptions{
ProjectVersionID: scope.ProjectVersionID,
ImportJobID: scope.ImportJobID,
DefaultDestinationScheme: database.ParameterDestinationSchemeProvisionerVariable,
})
values, err := projectparameter.Compute(context.Background(), db, scope)
@@ -101,7 +104,7 @@ func TestCompute(t *testing.T) {
value := values[0]
require.True(t, value.DefaultValue)
require.Equal(t, database.ParameterScopeProject, value.Scope)
require.Equal(t, scope.ProjectID.String(), value.ScopeID)
require.Equal(t, scope.ProjectID.UUID.String(), value.ScopeID)
require.Equal(t, value.Proto.Name, parameter.DefaultDestinationValue.String)
require.Equal(t, value.Proto.DestinationScheme, proto.ParameterDestination_PROVISIONER_VARIABLE)
require.Equal(t, value.Proto.Value, parameter.DefaultSourceValue.String)
@@ -112,7 +115,7 @@ func TestCompute(t *testing.T) {
db := databasefake.New()
scope := generateScope()
parameter := generateProjectParameter(t, db, projectParameterOptions{
ProjectVersionID: scope.ProjectVersionID,
ImportJobID: scope.ImportJobID,
})
_, err := db.InsertParameterValue(context.Background(), database.InsertParameterValueParams{
ID: uuid.New(),
@@ -138,13 +141,13 @@ func TestCompute(t *testing.T) {
db := databasefake.New()
scope := generateScope()
parameter := generateProjectParameter(t, db, projectParameterOptions{
ProjectVersionID: scope.ProjectVersionID,
ImportJobID: scope.ImportJobID,
})
value, err := db.InsertParameterValue(context.Background(), database.InsertParameterValueParams{
ID: uuid.New(),
Name: parameter.Name,
Scope: database.ParameterScopeProject,
ScopeID: scope.ProjectID.String(),
ScopeID: scope.ProjectID.UUID.String(),
SourceScheme: database.ParameterSourceSchemeData,
SourceValue: "nop",
DestinationScheme: database.ParameterDestinationSchemeEnvironmentVariable,
@@ -164,7 +167,7 @@ func TestCompute(t *testing.T) {
db := databasefake.New()
scope := generateScope()
parameter := generateProjectParameter(t, db, projectParameterOptions{
ProjectVersionID: scope.ProjectVersionID,
ImportJobID: scope.ImportJobID,
})
_, err := db.InsertParameterValue(context.Background(), database.InsertParameterValueParams{
ID: uuid.New(),
@@ -190,7 +193,7 @@ func TestCompute(t *testing.T) {
scope := generateScope()
parameter := generateProjectParameter(t, db, projectParameterOptions{
AllowOverrideSource: true,
ProjectVersionID: scope.ProjectVersionID,
ImportJobID: scope.ImportJobID,
})
_, err := db.InsertParameterValue(context.Background(), database.InsertParameterValueParams{
ID: uuid.New(),
@@ -209,4 +212,37 @@ func TestCompute(t *testing.T) {
require.Len(t, values, 1)
require.Equal(t, false, values[0].DefaultValue)
})
t.Run("AdditionalOverwriteWorkspace", func(t *testing.T) {
t.Parallel()
db := databasefake.New()
scope := generateScope()
parameter := generateProjectParameter(t, db, projectParameterOptions{
AllowOverrideSource: true,
ImportJobID: scope.ImportJobID,
})
_, err := db.InsertParameterValue(context.Background(), database.InsertParameterValueParams{
ID: uuid.New(),
Name: parameter.Name,
Scope: database.ParameterScopeWorkspace,
ScopeID: scope.WorkspaceID.UUID.String(),
SourceScheme: database.ParameterSourceSchemeData,
SourceValue: "nop",
DestinationScheme: database.ParameterDestinationSchemeEnvironmentVariable,
DestinationValue: "projectvalue",
})
require.NoError(t, err)
values, err := projectparameter.Compute(context.Background(), db, scope, database.ParameterValue{
Name: parameter.Name,
Scope: database.ParameterScopeUser,
SourceScheme: database.ParameterSourceSchemeData,
SourceValue: "nop",
DestinationScheme: database.ParameterDestinationSchemeEnvironmentVariable,
DestinationValue: "testing",
})
require.NoError(t, err)
require.Len(t, values, 1)
require.Equal(t, "testing", values[0].Proto.Value)
})
}