feat: Add backend API support for resource metadata (#3242)

* Initial support for metadata in provisioner API and Terraform provisioner

* add support for nullable metadata fields

* handle metadata fields in provisionerd and API

provisioner/terraform/resources.go

@@ -33,6 +33,19 @@ type agentAppAttributes struct {
 	RelativePath bool   `mapstructure:"relative_path"`
 }
 
+// A mapping of attributes on the "coder_metadata" resource.
+type metadataAttributes struct {
+	ResourceID string         `mapstructure:"resource_id"`
+	Items      []metadataItem `mapstructure:"item"`
+}
+
+type metadataItem struct {
+	Key       string `mapstructure:"key"`
+	Value     string `mapstructure:"value"`
+	Sensitive bool   `mapstructure:"sensitive"`
+	IsNull    bool   `mapstructure:"is_null"`
+}
+
 // ConvertResources consumes Terraform state and a GraphViz representation produced by
 // `terraform graph` to produce resources consumable by Coder.
 func ConvertResources(module *tfjson.StateModule, rawGraph string) ([]*proto.Resource, error) {
@@ -48,16 +61,30 @@ func ConvertResources(module *tfjson.StateModule, rawGraph string) ([]*proto.Res
 	resources := make([]*proto.Resource, 0)
 	resourceAgents := map[string][]*proto.Agent{}
 
-	// Indexes Terraform resources by it's label. The label
-	// is what "terraform graph" uses to reference nodes.
+	// Indexes Terraform resources by their label and ID.
+	// The label is what "terraform graph" uses to reference nodes, and the ID
+	// is used by "coder_metadata" resources to refer to their targets. (The ID
+	// field is only available when reading a state file, and not when reading a
+	// plan file.)
 	tfResourceByLabel := map[string]*tfjson.StateResource{}
+	resourceLabelByID := map[string]string{}
 	var findTerraformResources func(mod *tfjson.StateModule)
 	findTerraformResources = func(mod *tfjson.StateModule) {
 		for _, module := range mod.ChildModules {
 			findTerraformResources(module)
 		}
 		for _, resource := range mod.Resources {
-			tfResourceByLabel[convertAddressToLabel(resource.Address)] = resource
+			label := convertAddressToLabel(resource.Address)
+			// index by label
+			tfResourceByLabel[label] = resource
+			// index by ID, if it exists
+			id, ok := resource.AttributeValues["id"]
+			if ok {
+				idString, ok := id.(string)
+				if ok {
+					resourceLabelByID[idString] = label
+				}
+			}
 		}
 	}
 	findTerraformResources(module)
@@ -205,23 +232,59 @@ func ConvertResources(module *tfjson.StateModule, rawGraph string) ([]*proto.Res
 		}
 	}
 
+	// Associate metadata blocks with resources.
+	resourceMetadata := map[string][]*proto.Resource_Metadata{}
+	for label, resource := range tfResourceByLabel {
+		if resource.Type != "coder_metadata" {
+			continue
+		}
+		var attrs metadataAttributes
+		err = mapstructure.Decode(resource.AttributeValues, &attrs)
+		if err != nil {
+			return nil, xerrors.Errorf("decode metadata attributes: %w", err)
+		}
+
+		if attrs.ResourceID == "" {
+			// TODO: detect this as an error
+			// At plan time, change.after_unknown.resource_id should be "true".
+			// At provision time, values.resource_id should be set.
+			continue
+		}
+		targetLabel, ok := resourceLabelByID[attrs.ResourceID]
+		if !ok {
+			return nil, xerrors.Errorf("attribute %s.resource_id = %q does not refer to a valid resource", label, attrs.ResourceID)
+		}
+
+		for _, item := range attrs.Items {
+			resourceMetadata[targetLabel] = append(resourceMetadata[targetLabel],
+				&proto.Resource_Metadata{
+					Key:       item.Key,
+					Value:     item.Value,
+					Sensitive: item.Sensitive,
+					IsNull:    item.IsNull,
+				})
+		}
+	}
+
 	for _, resource := range tfResourceByLabel {
 		if resource.Mode == tfjson.DataResourceMode {
 			continue
 		}
-		if resource.Type == "coder_agent" || resource.Type == "coder_agent_instance" || resource.Type == "coder_app" {
+		if resource.Type == "coder_agent" || resource.Type == "coder_agent_instance" || resource.Type == "coder_app" || resource.Type == "coder_metadata" {
 			continue
 		}
+		label := convertAddressToLabel(resource.Address)
 
-		agents, exists := resourceAgents[convertAddressToLabel(resource.Address)]
+		agents, exists := resourceAgents[label]
 		if exists {
 			applyAutomaticInstanceID(resource, agents)
 		}
 
 		resources = append(resources, &proto.Resource{
-			Name:   resource.Name,
-			Type:   resource.Type,
-			Agents: agents,
+			Name:     resource.Name,
+			Type:     resource.Type,
+			Agents:   agents,
+			Metadata: resourceMetadata[label],
 		})
 	}
 

provisioner/terraform/resources_test.go

@@ -14,6 +14,8 @@ import (
 	"github.com/coder/coder/cryptorand"
 	"github.com/coder/coder/provisioner/terraform"
 	"github.com/coder/coder/provisionersdk/proto"
+
+	protobuf "github.com/golang/protobuf/proto"
 )
 
 func TestConvertResources(t *testing.T) {
@@ -114,6 +116,24 @@ func TestConvertResources(t *testing.T) {
 				Auth: &proto.Agent_Token{},
 			}},
 		}},
+		// Tests fetching metadata about workspace resources.
+		"resource-metadata": {{
+			Name: "about",
+			Type: "null_resource",
+			Metadata: []*proto.Resource_Metadata{{
+				Key:   "hello",
+				Value: "world",
+			}, {
+				Key:    "null",
+				IsNull: true,
+			}, {
+				Key: "empty",
+			}, {
+				Key:       "secret",
+				Value:     "squirrel",
+				Sensitive: true,
+			}},
+		}},
 	} {
 		folderName := folderName
 		expected := expected
@@ -134,7 +154,16 @@ func TestConvertResources(t *testing.T) {
 				resources, err := terraform.ConvertResources(tfPlan.PlannedValues.RootModule, string(tfPlanGraph))
 				require.NoError(t, err)
 				sortResources(resources)
-				resourcesWant, err := json.Marshal(expected)
+
+				// plan does not contain metadata, so clone expected and remove it
+				var expectedNoMetadata []*proto.Resource
+				for _, resource := range expected {
+					resourceCopy, _ := protobuf.Clone(resource).(*proto.Resource)
+					resourceCopy.Metadata = nil
+					expectedNoMetadata = append(expectedNoMetadata, resourceCopy)
+				}
+
+				resourcesWant, err := json.Marshal(expectedNoMetadata)
 				require.NoError(t, err)
 				resourcesGot, err := json.Marshal(resources)
 				require.NoError(t, err)

provisioner/terraform/testdata/resource-metadata/resource-metadata.tf

@@ -0,0 +1,35 @@
+terraform {
+  required_providers {
+    coder = {
+      source  = "coder/coder"
+      version = "0.4.4"
+    }
+  }
+}
+
+resource "coder_agent" "main" {
+  os   = "linux"
+  arch = "amd64"
+}
+
+resource "null_resource" "about" {}
+
+resource "coder_metadata" "about_info" {
+  resource_id = null_resource.about.id
+  item {
+    key = "hello"
+    value = "world"
+  }
+  item {
+    key = "null"
+  }
+  item {
+    key = "empty"
+    value = ""
+  }
+  item {
+    key = "secret"
+    value = "squirrel"
+    sensitive = true
+  }
+}

provisioner/terraform/testdata/resource-metadata/resource-metadata.tfplan.dot

@@ -0,0 +1,21 @@
+digraph {
+	compound = "true"
+	newrank = "true"
+	subgraph "root" {
+		"[root] coder_agent.main (expand)" [label = "coder_agent.main", shape = "box"]
+		"[root] coder_metadata.about_info (expand)" [label = "coder_metadata.about_info", shape = "box"]
+		"[root] null_resource.about (expand)" [label = "null_resource.about", shape = "box"]
+		"[root] provider[\"registry.terraform.io/coder/coder\"]" [label = "provider[\"registry.terraform.io/coder/coder\"]", shape = "diamond"]
+		"[root] provider[\"registry.terraform.io/hashicorp/null\"]" [label = "provider[\"registry.terraform.io/hashicorp/null\"]", shape = "diamond"]
+		"[root] coder_agent.main (expand)" -> "[root] provider[\"registry.terraform.io/coder/coder\"]"
+		"[root] coder_metadata.about_info (expand)" -> "[root] null_resource.about (expand)"
+		"[root] coder_metadata.about_info (expand)" -> "[root] provider[\"registry.terraform.io/coder/coder\"]"
+		"[root] null_resource.about (expand)" -> "[root] provider[\"registry.terraform.io/hashicorp/null\"]"
+		"[root] provider[\"registry.terraform.io/coder/coder\"] (close)" -> "[root] coder_agent.main (expand)"
+		"[root] provider[\"registry.terraform.io/coder/coder\"] (close)" -> "[root] coder_metadata.about_info (expand)"
+		"[root] provider[\"registry.terraform.io/hashicorp/null\"] (close)" -> "[root] null_resource.about (expand)"
+		"[root] root" -> "[root] provider[\"registry.terraform.io/coder/coder\"] (close)"
+		"[root] root" -> "[root] provider[\"registry.terraform.io/hashicorp/null\"] (close)"
+	}
+}
+

provisioner/terraform/testdata/resource-metadata/resource-metadata.tfplan.json

@@ -0,0 +1,293 @@
+{
+  "format_version": "1.1",
+  "terraform_version": "1.2.6",
+  "planned_values": {
+    "root_module": {
+      "resources": [
+        {
+          "address": "coder_agent.main",
+          "mode": "managed",
+          "type": "coder_agent",
+          "name": "main",
+          "provider_name": "registry.terraform.io/coder/coder",
+          "schema_version": 0,
+          "values": {
+            "arch": "amd64",
+            "auth": "token",
+            "dir": null,
+            "env": null,
+            "os": "linux",
+            "startup_script": null
+          },
+          "sensitive_values": {}
+        },
+        {
+          "address": "coder_metadata.about_info",
+          "mode": "managed",
+          "type": "coder_metadata",
+          "name": "about_info",
+          "provider_name": "registry.terraform.io/coder/coder",
+          "schema_version": 0,
+          "values": {
+            "item": [
+              {
+                "key": "hello",
+                "sensitive": false,
+                "value": "world"
+              },
+              {
+                "key": "null",
+                "sensitive": false,
+                "value": null
+              },
+              {
+                "key": "empty",
+                "sensitive": false,
+                "value": ""
+              },
+              {
+                "key": "secret",
+                "sensitive": true,
+                "value": "squirrel"
+              }
+            ]
+          },
+          "sensitive_values": {
+            "item": [
+              {},
+              {},
+              {},
+              {}
+            ]
+          }
+        },
+        {
+          "address": "null_resource.about",
+          "mode": "managed",
+          "type": "null_resource",
+          "name": "about",
+          "provider_name": "registry.terraform.io/hashicorp/null",
+          "schema_version": 0,
+          "values": {
+            "triggers": null
+          },
+          "sensitive_values": {}
+        }
+      ]
+    }
+  },
+  "resource_changes": [
+    {
+      "address": "coder_agent.main",
+      "mode": "managed",
+      "type": "coder_agent",
+      "name": "main",
+      "provider_name": "registry.terraform.io/coder/coder",
+      "change": {
+        "actions": [
+          "create"
+        ],
+        "before": null,
+        "after": {
+          "arch": "amd64",
+          "auth": "token",
+          "dir": null,
+          "env": null,
+          "os": "linux",
+          "startup_script": null
+        },
+        "after_unknown": {
+          "id": true,
+          "init_script": true,
+          "token": true
+        },
+        "before_sensitive": false,
+        "after_sensitive": {}
+      }
+    },
+    {
+      "address": "coder_metadata.about_info",
+      "mode": "managed",
+      "type": "coder_metadata",
+      "name": "about_info",
+      "provider_name": "registry.terraform.io/coder/coder",
+      "change": {
+        "actions": [
+          "create"
+        ],
+        "before": null,
+        "after": {
+          "item": [
+            {
+              "key": "hello",
+              "sensitive": false,
+              "value": "world"
+            },
+            {
+              "key": "null",
+              "sensitive": false,
+              "value": null
+            },
+            {
+              "key": "empty",
+              "sensitive": false,
+              "value": ""
+            },
+            {
+              "key": "secret",
+              "sensitive": true,
+              "value": "squirrel"
+            }
+          ]
+        },
+        "after_unknown": {
+          "id": true,
+          "item": [
+            {
+              "is_null": true
+            },
+            {
+              "is_null": true
+            },
+            {
+              "is_null": true
+            },
+            {
+              "is_null": true
+            }
+          ],
+          "resource_id": true
+        },
+        "before_sensitive": false,
+        "after_sensitive": {
+          "item": [
+            {},
+            {},
+            {},
+            {}
+          ]
+        }
+      }
+    },
+    {
+      "address": "null_resource.about",
+      "mode": "managed",
+      "type": "null_resource",
+      "name": "about",
+      "provider_name": "registry.terraform.io/hashicorp/null",
+      "change": {
+        "actions": [
+          "create"
+        ],
+        "before": null,
+        "after": {
+          "triggers": null
+        },
+        "after_unknown": {
+          "id": true
+        },
+        "before_sensitive": false,
+        "after_sensitive": {}
+      }
+    }
+  ],
+  "configuration": {
+    "provider_config": {
+      "coder": {
+        "name": "coder",
+        "full_name": "registry.terraform.io/coder/coder",
+        "version_constraint": "0.4.4"
+      },
+      "null": {
+        "name": "null",
+        "full_name": "registry.terraform.io/hashicorp/null"
+      }
+    },
+    "root_module": {
+      "resources": [
+        {
+          "address": "coder_agent.main",
+          "mode": "managed",
+          "type": "coder_agent",
+          "name": "main",
+          "provider_config_key": "coder",
+          "expressions": {
+            "arch": {
+              "constant_value": "amd64"
+            },
+            "os": {
+              "constant_value": "linux"
+            }
+          },
+          "schema_version": 0
+        },
+        {
+          "address": "coder_metadata.about_info",
+          "mode": "managed",
+          "type": "coder_metadata",
+          "name": "about_info",
+          "provider_config_key": "coder",
+          "expressions": {
+            "item": [
+              {
+                "key": {
+                  "constant_value": "hello"
+                },
+                "value": {
+                  "constant_value": "world"
+                }
+              },
+              {
+                "key": {
+                  "constant_value": "null"
+                }
+              },
+              {
+                "key": {
+                  "constant_value": "empty"
+                },
+                "value": {
+                  "constant_value": ""
+                }
+              },
+              {
+                "key": {
+                  "constant_value": "secret"
+                },
+                "sensitive": {
+                  "constant_value": true
+                },
+                "value": {
+                  "constant_value": "squirrel"
+                }
+              }
+            ],
+            "resource_id": {
+              "references": [
+                "null_resource.about.id",
+                "null_resource.about"
+              ]
+            }
+          },
+          "schema_version": 0
+        },
+        {
+          "address": "null_resource.about",
+          "mode": "managed",
+          "type": "null_resource",
+          "name": "about",
+          "provider_config_key": "null",
+          "schema_version": 0
+        }
+      ]
+    }
+  },
+  "relevant_attributes": [
+    {
+      "resource": "null_resource.about",
+      "attribute": [
+        "id"
+      ]
+    }
+  ]
+}

provisioner/terraform/testdata/resource-metadata/resource-metadata.tfstate.dot

@@ -0,0 +1,21 @@
+digraph {
+	compound = "true"
+	newrank = "true"
+	subgraph "root" {
+		"[root] coder_agent.main (expand)" [label = "coder_agent.main", shape = "box"]
+		"[root] coder_metadata.about_info (expand)" [label = "coder_metadata.about_info", shape = "box"]
+		"[root] null_resource.about (expand)" [label = "null_resource.about", shape = "box"]
+		"[root] provider[\"registry.terraform.io/coder/coder\"]" [label = "provider[\"registry.terraform.io/coder/coder\"]", shape = "diamond"]
+		"[root] provider[\"registry.terraform.io/hashicorp/null\"]" [label = "provider[\"registry.terraform.io/hashicorp/null\"]", shape = "diamond"]
+		"[root] coder_agent.main (expand)" -> "[root] provider[\"registry.terraform.io/coder/coder\"]"
+		"[root] coder_metadata.about_info (expand)" -> "[root] null_resource.about (expand)"
+		"[root] coder_metadata.about_info (expand)" -> "[root] provider[\"registry.terraform.io/coder/coder\"]"
+		"[root] null_resource.about (expand)" -> "[root] provider[\"registry.terraform.io/hashicorp/null\"]"
+		"[root] provider[\"registry.terraform.io/coder/coder\"] (close)" -> "[root] coder_agent.main (expand)"
+		"[root] provider[\"registry.terraform.io/coder/coder\"] (close)" -> "[root] coder_metadata.about_info (expand)"
+		"[root] provider[\"registry.terraform.io/hashicorp/null\"] (close)" -> "[root] null_resource.about (expand)"
+		"[root] root" -> "[root] provider[\"registry.terraform.io/coder/coder\"] (close)"
+		"[root] root" -> "[root] provider[\"registry.terraform.io/hashicorp/null\"] (close)"
+	}
+}
+

provisioner/terraform/testdata/resource-metadata/resource-metadata.tfstate.json

@@ -0,0 +1,92 @@
+{
+  "format_version": "1.0",
+  "terraform_version": "1.2.6",
+  "values": {
+    "root_module": {
+      "resources": [
+        {
+          "address": "coder_agent.main",
+          "mode": "managed",
+          "type": "coder_agent",
+          "name": "main",
+          "provider_name": "registry.terraform.io/coder/coder",
+          "schema_version": 0,
+          "values": {
+            "arch": "amd64",
+            "auth": "token",
+            "dir": null,
+            "env": null,
+            "id": "33d1fa51-2987-43da-8f07-53131af520f2",
+            "init_script": "",
+            "os": "linux",
+            "startup_script": null,
+            "token": "5acee40a-6f4f-4e38-92a8-98b057d4945f"
+          },
+          "sensitive_values": {}
+        },
+        {
+          "address": "coder_metadata.about_info",
+          "mode": "managed",
+          "type": "coder_metadata",
+          "name": "about_info",
+          "provider_name": "registry.terraform.io/coder/coder",
+          "schema_version": 0,
+          "values": {
+            "id": "5b828eb0-938f-4fba-8599-701dce975627",
+            "item": [
+              {
+                "is_null": false,
+                "key": "hello",
+                "sensitive": false,
+                "value": "world"
+              },
+              {
+                "is_null": true,
+                "key": "null",
+                "sensitive": false,
+                "value": ""
+              },
+              {
+                "is_null": false,
+                "key": "empty",
+                "sensitive": false,
+                "value": ""
+              },
+              {
+                "is_null": false,
+                "key": "secret",
+                "sensitive": true,
+                "value": "squirrel"
+              }
+            ],
+            "resource_id": "6511007151682544728"
+          },
+          "sensitive_values": {
+            "item": [
+              {},
+              {},
+              {},
+              {}
+            ]
+          },
+          "depends_on": [
+            "null_resource.about"
+          ]
+        },
+        {
+          "address": "null_resource.about",
+          "mode": "managed",
+          "type": "null_resource",
+          "name": "about",
+          "provider_name": "registry.terraform.io/hashicorp/null",
+          "schema_version": 0,
+          "values": {
+            "id": "6511007151682544728",
+            "triggers": null
+          },
+          "sensitive_values": {}
+        }
+      ]
+    }
+  }
+}