feat: Update generated schema for provisionerd (#121)

provisioner/terraform/parse.go

@@ -1,7 +1,6 @@
 package terraform
 
 import (
-	"context"
 	"encoding/json"
 	"os"
 
@@ -12,24 +11,32 @@ import (
 )
 
 // Parse extracts Terraform variables from source-code.
-func (*terraform) Parse(_ context.Context, request *proto.Parse_Request) (*proto.Parse_Response, error) {
+func (*terraform) Parse(request *proto.Parse_Request, stream proto.DRPCProvisioner_ParseStream) error {
+	defer func() {
+		_ = stream.CloseSend()
+	}()
+
 	module, diags := tfconfig.LoadModule(request.Directory)
 	if diags.HasErrors() {
-		return nil, xerrors.Errorf("load module: %w", diags.Err())
+		return xerrors.Errorf("load module: %w", diags.Err())
 	}
 	parameters := make([]*proto.ParameterSchema, 0, len(module.Variables))
 	for _, v := range module.Variables {
 		schema, err := convertVariableToParameter(v)
 		if err != nil {
-			return nil, xerrors.Errorf("convert variable %q: %w", v.Name, err)
+			return xerrors.Errorf("convert variable %q: %w", v.Name, err)
 		}
 
 		parameters = append(parameters, schema)
 	}
 
-	return &proto.Parse_Response{
-		ParameterSchemas: parameters,
-	}, nil
+	return stream.Send(&proto.Parse_Response{
+		Type: &proto.Parse_Response_Complete{
+			Complete: &proto.Parse_Complete{
+				ParameterSchemas: parameters,
+			},
+		},
+	})
 }
 
 // Converts a Terraform variable to a provisioner parameter.

provisioner/terraform/parse_test.go

@@ -49,10 +49,14 @@ func TestParse(t *testing.T) {
 			}`,
 		},
 		Response: &proto.Parse_Response{
-			ParameterSchemas: []*proto.ParameterSchema{{
-				Name:        "A",
-				Description: "Testing!",
-			}},
+			Type: &proto.Parse_Response_Complete{
+				Complete: &proto.Parse_Complete{
+					ParameterSchemas: []*proto.ParameterSchema{{
+						Name:        "A",
+						Description: "Testing!",
+					}},
+				},
+			},
 		},
 	}, {
 		Name: "default-variable-value",
@@ -62,17 +66,21 @@ func TestParse(t *testing.T) {
 			}`,
 		},
 		Response: &proto.Parse_Response{
-			ParameterSchemas: []*proto.ParameterSchema{{
-				Name: "A",
-				DefaultSource: &proto.ParameterSource{
-					Scheme: proto.ParameterSource_DATA,
-					Value:  "\"wow\"",
+			Type: &proto.Parse_Response_Complete{
+				Complete: &proto.Parse_Complete{
+					ParameterSchemas: []*proto.ParameterSchema{{
+						Name: "A",
+						DefaultSource: &proto.ParameterSource{
+							Scheme: proto.ParameterSource_DATA,
+							Value:  "\"wow\"",
+						},
+						DefaultDestination: &proto.ParameterDestination{
+							Scheme: proto.ParameterDestination_PROVISIONER_VARIABLE,
+							Value:  "A",
+						},
+					}},
 				},
-				DefaultDestination: &proto.ParameterDestination{
-					Scheme: proto.ParameterDestination_PROVISIONER_VARIABLE,
-					Value:  "A",
-				},
-			}},
+			},
 		},
 	}, {
 		Name: "variable-validation",
@@ -84,10 +92,15 @@ func TestParse(t *testing.T) {
 			}`,
 		},
 		Response: &proto.Parse_Response{
-			ParameterSchemas: []*proto.ParameterSchema{{
-				Name:                "A",
-				ValidationCondition: `var.A == "value"`,
-			}},
+			Type: &proto.Parse_Response_Complete{
+				Complete: &proto.Parse_Complete{
+					ParameterSchemas: []*proto.ParameterSchema{{
+						Name:                "A",
+						ValidationCondition: `var.A == "value"`,
+					},
+					},
+				},
+			},
 		},
 	}} {
 		testCase := testCase
@@ -106,13 +119,23 @@ func TestParse(t *testing.T) {
 			})
 			require.NoError(t, err)
 
-			// Ensure the want and got are equivalent!
-			want, err := json.Marshal(testCase.Response)
-			require.NoError(t, err)
-			got, err := json.Marshal(response)
-			require.NoError(t, err)
+			for {
+				msg, err := response.Recv()
+				require.NoError(t, err)
 
-			require.Equal(t, string(want), string(got))
+				if msg.GetComplete() == nil {
+					continue
+				}
+
+				// Ensure the want and got are equivalent!
+				want, err := json.Marshal(testCase.Response)
+				require.NoError(t, err)
+				got, err := json.Marshal(msg)
+				require.NoError(t, err)
+
+				require.Equal(t, string(want), string(got))
+				break
+			}
 		})
 	}
 }

provisioner/terraform/provision.go

@@ -1,10 +1,13 @@
 package terraform
 
 import (
-	"context"
+	"bufio"
+	"encoding/json"
 	"fmt"
+	"io"
 	"os"
 	"path/filepath"
+	"strings"
 
 	"github.com/hashicorp/terraform-exec/tfexec"
 	"golang.org/x/xerrors"
@@ -13,32 +16,52 @@ import (
 )
 
 // Provision executes `terraform apply`.
-func (t *terraform) Provision(ctx context.Context, request *proto.Provision_Request) (*proto.Provision_Response, error) {
+func (t *terraform) Provision(request *proto.Provision_Request, stream proto.DRPCProvisioner_ProvisionStream) error {
+	ctx := stream.Context()
 	statefilePath := filepath.Join(request.Directory, "terraform.tfstate")
-	err := os.WriteFile(statefilePath, request.State, 0600)
-	if err != nil {
-		return nil, xerrors.Errorf("write statefile %q: %w", statefilePath, err)
+	if len(request.State) > 0 {
+		err := os.WriteFile(statefilePath, request.State, 0600)
+		if err != nil {
+			return xerrors.Errorf("write statefile %q: %w", statefilePath, err)
+		}
 	}
 
 	terraform, err := tfexec.NewTerraform(request.Directory, t.binaryPath)
 	if err != nil {
-		return nil, xerrors.Errorf("create new terraform executor: %w", err)
+		return xerrors.Errorf("create new terraform executor: %w", err)
 	}
 	version, _, err := terraform.Version(ctx, false)
 	if err != nil {
-		return nil, xerrors.Errorf("get terraform version: %w", err)
+		return xerrors.Errorf("get terraform version: %w", err)
 	}
 	if !version.GreaterThanOrEqual(minimumTerraformVersion) {
-		return nil, xerrors.Errorf("terraform version %q is too old. required >= %q", version.String(), minimumTerraformVersion.String())
+		return xerrors.Errorf("terraform version %q is too old. required >= %q", version.String(), minimumTerraformVersion.String())
 	}
 
+	reader, writer := io.Pipe()
+	defer reader.Close()
+	defer writer.Close()
+	go func() {
+		scanner := bufio.NewScanner(reader)
+		for scanner.Scan() {
+			_ = stream.Send(&proto.Provision_Response{
+				Type: &proto.Provision_Response_Log{
+					Log: &proto.Log{
+						Level:  proto.LogLevel_INFO,
+						Output: scanner.Text(),
+					},
+				},
+			})
+		}
+	}()
+	terraform.SetStdout(writer)
 	err = terraform.Init(ctx)
 	if err != nil {
-		return nil, xerrors.Errorf("initialize terraform: %w", err)
+		return xerrors.Errorf("initialize terraform: %w", err)
 	}
 
 	env := map[string]string{}
-	options := make([]tfexec.ApplyOption, 0)
+	options := []tfexec.ApplyOption{tfexec.JSON(true)}
 	for _, param := range request.ParameterValues {
 		switch param.DestinationScheme {
 		case proto.ParameterDestination_ENVIRONMENT_VARIABLE:
@@ -46,26 +69,73 @@ func (t *terraform) Provision(ctx context.Context, request *proto.Provision_Requ
 		case proto.ParameterDestination_PROVISIONER_VARIABLE:
 			options = append(options, tfexec.Var(fmt.Sprintf("%s=%s", param.Name, param.Value)))
 		default:
-			return nil, xerrors.Errorf("unsupported parameter type %q for %q", param.DestinationScheme, param.Name)
+			return xerrors.Errorf("unsupported parameter type %q for %q", param.DestinationScheme, param.Name)
 		}
 	}
 	err = terraform.SetEnv(env)
 	if err != nil {
-		return nil, xerrors.Errorf("apply environment variables: %w", err)
+		return xerrors.Errorf("apply environment variables: %w", err)
 	}
 
+	reader, writer = io.Pipe()
+	defer reader.Close()
+	defer writer.Close()
+	go func() {
+		decoder := json.NewDecoder(reader)
+		for {
+			var log terraformProvisionLog
+			err := decoder.Decode(&log)
+			if err != nil {
+				return
+			}
+
+			logLevel, err := convertTerraformLogLevel(log.Level)
+			if err != nil {
+				// Not a big deal, but we should handle this at some point!
+				continue
+			}
+			_ = stream.Send(&proto.Provision_Response{
+				Type: &proto.Provision_Response_Log{
+					Log: &proto.Log{
+						Level:  logLevel,
+						Output: log.Message,
+					},
+				},
+			})
+
+			if log.Diagnostic == nil {
+				continue
+			}
+
+			// If the diagnostic is provided, let's provide a bit more info!
+			logLevel, err = convertTerraformLogLevel(log.Diagnostic.Severity)
+			if err != nil {
+				continue
+			}
+			_ = stream.Send(&proto.Provision_Response{
+				Type: &proto.Provision_Response_Log{
+					Log: &proto.Log{
+						Level:  logLevel,
+						Output: log.Diagnostic.Detail,
+					},
+				},
+			})
+		}
+	}()
+
+	terraform.SetStdout(writer)
 	err = terraform.Apply(ctx, options...)
 	if err != nil {
-		return nil, xerrors.Errorf("apply terraform: %w", err)
+		return xerrors.Errorf("apply terraform: %w", err)
 	}
 
 	statefileContent, err := os.ReadFile(statefilePath)
 	if err != nil {
-		return nil, xerrors.Errorf("read file %q: %w", statefilePath, err)
+		return xerrors.Errorf("read file %q: %w", statefilePath, err)
 	}
 	state, err := terraform.ShowStateFile(ctx, statefilePath)
 	if err != nil {
-		return nil, xerrors.Errorf("show state file %q: %w", statefilePath, err)
+		return xerrors.Errorf("show state file %q: %w", statefilePath, err)
 	}
 	resources := make([]*proto.Resource, 0)
 	if state.Values != nil {
@@ -77,8 +147,42 @@ func (t *terraform) Provision(ctx context.Context, request *proto.Provision_Requ
 		}
 	}
 
-	return &proto.Provision_Response{
-		Resources: resources,
-		State:     statefileContent,
-	}, nil
+	return stream.Send(&proto.Provision_Response{
+		Type: &proto.Provision_Response_Complete{
+			Complete: &proto.Provision_Complete{
+				State:     statefileContent,
+				Resources: resources,
+			},
+		},
+	})
+}
+
+type terraformProvisionLog struct {
+	Level   string `json:"@level"`
+	Message string `json:"@message"`
+
+	Diagnostic *terraformProvisionLogDiagnostic `json:"diagnostic"`
+}
+
+type terraformProvisionLogDiagnostic struct {
+	Severity string `json:"severity"`
+	Summary  string `json:"summary"`
+	Detail   string `json:"detail"`
+}
+
+func convertTerraformLogLevel(logLevel string) (proto.LogLevel, error) {
+	switch strings.ToLower(logLevel) {
+	case "trace":
+		return proto.LogLevel_TRACE, nil
+	case "debug":
+		return proto.LogLevel_DEBUG, nil
+	case "info":
+		return proto.LogLevel_INFO, nil
+	case "warn":
+		return proto.LogLevel_WARN, nil
+	case "error":
+		return proto.LogLevel_ERROR, nil
+	default:
+		return proto.LogLevel(0), xerrors.Errorf("invalid log level %q", logLevel)
+	}
 }

provisioner/terraform/provision_test.go

@@ -56,7 +56,11 @@ func TestProvision(t *testing.T) {
 				Value:             "example",
 			}},
 		},
-		Response: &proto.Provision_Response{},
+		Response: &proto.Provision_Response{
+			Type: &proto.Provision_Response_Complete{
+				Complete: &proto.Provision_Complete{},
+			},
+		},
 	}, {
 		Name: "missing-variable",
 		Files: map[string]string{
@@ -70,10 +74,14 @@ func TestProvision(t *testing.T) {
 			"main.tf": `resource "null_resource" "A" {}`,
 		},
 		Response: &proto.Provision_Response{
-			Resources: []*proto.Resource{{
-				Name: "A",
-				Type: "null_resource",
-			}},
+			Type: &proto.Provision_Response_Complete{
+				Complete: &proto.Provision_Complete{
+					Resources: []*proto.Resource{{
+						Name: "A",
+						Type: "null_resource",
+					}},
+				},
+			},
 		},
 	}, {
 		Name: "invalid-sourcecode",
@@ -100,20 +108,34 @@ func TestProvision(t *testing.T) {
 				request.State = testCase.Request.State
 			}
 			response, err := api.Provision(ctx, request)
-			if testCase.Error {
-				require.Error(t, err)
-				return
+			require.NoError(t, err)
+			for {
+				msg, err := response.Recv()
+				if msg != nil && msg.GetLog() != nil {
+					continue
+				}
+				if testCase.Error {
+					require.Error(t, err)
+					return
+				}
+				require.NoError(t, err)
+
+				if msg.GetComplete() == nil {
+					continue
+				}
+
+				require.NoError(t, err)
+				require.Greater(t, len(msg.GetComplete().State), 0)
+
+				resourcesGot, err := json.Marshal(msg.GetComplete().Resources)
+				require.NoError(t, err)
+
+				resourcesWant, err := json.Marshal(testCase.Response.GetComplete().Resources)
+				require.NoError(t, err)
+
+				require.Equal(t, string(resourcesWant), string(resourcesGot))
+				break
 			}
-			require.NoError(t, err)
-			require.Greater(t, len(response.State), 0)
-
-			resourcesGot, err := json.Marshal(response.Resources)
-			require.NoError(t, err)
-
-			resourcesWant, err := json.Marshal(testCase.Response.Resources)
-			require.NoError(t, err)
-
-			require.Equal(t, string(resourcesWant), string(resourcesGot))
 		})
 	}
 }