Add Ownership tags when creating AWS Resources for External Audit Storage (#44454)

* Add Ownership tags when creating AWS Resources

This PR adds the ownership tags to all aws resources created.
This will help users track everything that was created by the
integration/cluster.

* set tags on creation

* add comment to why we are using PutBucketTagging

* ensure bucket props even if bucket already exists
This commit is contained in:
Marco Dinis
2024-07-22 14:35:45 +00:00
committed by GitHub
parent fc2f0a478a
commit d20ff07556
19 changed files with 219 additions and 64 deletions
+4 -2
View File
@@ -26,6 +26,8 @@ import (
"github.com/aws/aws-sdk-go-v2/service/ec2"
ec2types "github.com/aws/aws-sdk-go-v2/service/ec2/types"
"github.com/gravitational/trace"
"github.com/gravitational/teleport/lib/integrations/awsoidc/tags"
)
// CreateEC2ICERequest contains the required fields to create an AWS EC2 Instance Connect Endpoint.
@@ -46,7 +48,7 @@ type CreateEC2ICERequest struct {
// - teleport.dev/cluster: <cluster>
// - teleport.dev/origin: aws-oidc-integration
// - teleport.dev/integration: <integrationName>
ResourceCreationTags AWSTags
ResourceCreationTags tags.AWSTags
}
// EC2ICEEndpoint contains the information for a single Endpoint to be created.
@@ -83,7 +85,7 @@ func (req *CreateEC2ICERequest) CheckAndSetDefaults() error {
}
if len(req.ResourceCreationTags) == 0 {
req.ResourceCreationTags = defaultResourceCreationTags(req.Cluster, req.IntegrationName)
req.ResourceCreationTags = tags.DefaultResourceCreationTags(req.Cluster, req.IntegrationName)
}
return nil
@@ -29,6 +29,8 @@ import (
"github.com/google/go-cmp/cmp"
"github.com/gravitational/trace"
"github.com/stretchr/testify/require"
"github.com/gravitational/teleport/lib/integrations/awsoidc/tags"
)
type mockCreateEC2ICEClient struct {
@@ -185,7 +187,7 @@ func TestCreateEC2ICERequest(t *testing.T) {
SubnetID: "subnet-123",
SecurityGroupIDs: []string{"sg-1", "sg-2"},
}},
ResourceCreationTags: AWSTags{
ResourceCreationTags: tags.AWSTags{
"teleport.dev/origin": "integration_awsoidc",
"teleport.dev/cluster": "teleport-cluster",
"teleport.dev/integration": "teleportdev",
@@ -27,6 +27,7 @@ import (
"github.com/gravitational/trace"
"github.com/gravitational/teleport"
"github.com/gravitational/teleport/lib/integrations/awsoidc/tags"
)
// DeployDatabaseServiceRequest contains the required fields to deploy multiple Teleport Databases Services.
@@ -58,7 +59,7 @@ type DeployDatabaseServiceRequest struct {
TeleportVersionTag string
// ResourceCreationTags is used to add tags when creating resources in AWS.
ResourceCreationTags AWSTags
ResourceCreationTags tags.AWSTags
// DeploymentJoinTokenName is the Teleport IAM Join Token name that the deployed service must use to join the cluster.
DeploymentJoinTokenName string
@@ -115,7 +116,7 @@ func (r *DeployDatabaseServiceRequest) CheckAndSetDefaults() error {
}
if r.ResourceCreationTags == nil {
r.ResourceCreationTags = defaultResourceCreationTags(r.TeleportClusterName, r.IntegrationName)
r.ResourceCreationTags = tags.DefaultResourceCreationTags(r.TeleportClusterName, r.IntegrationName)
}
r.ecsClusterName = normalizeECSClusterName(r.TeleportClusterName)
@@ -39,6 +39,7 @@ import (
"github.com/gravitational/teleport"
"github.com/gravitational/teleport/api/types"
"github.com/gravitational/teleport/lib/integrations/awsoidc/tags"
)
func TestDeployDatabaseServiceRequest_CheckAndSetDefaults(t *testing.T) {
@@ -157,7 +158,7 @@ func TestDeployDatabaseServiceRequest_CheckAndSetDefaults(t *testing.T) {
Region: "r",
TaskRoleARN: "arn",
IntegrationName: "teleportdev",
ResourceCreationTags: AWSTags{
ResourceCreationTags: tags.AWSTags{
"teleport.dev/origin": "integration_awsoidc",
"teleport.dev/cluster": "mycluster",
"teleport.dev/integration": "teleportdev",
@@ -202,7 +203,7 @@ type mockDeployServiceClient struct {
iamTokenMissing bool
iamAccessDeniedListServices bool
defaultTags AWSTags
defaultTags tags.AWSTags
}
// DescribeClusters lists ECS Clusters.
+6 -5
View File
@@ -35,6 +35,7 @@ import (
"github.com/gravitational/teleport"
"github.com/gravitational/teleport/api/types"
"github.com/gravitational/teleport/api/utils/retryutils"
"github.com/gravitational/teleport/lib/integrations/awsoidc/tags"
"github.com/gravitational/teleport/lib/modules"
)
@@ -141,7 +142,7 @@ type DeployServiceRequest struct {
IntegrationName string
// ResourceCreationTags is used to add tags when creating resources in AWS.
ResourceCreationTags AWSTags
ResourceCreationTags tags.AWSTags
// DeploymentMode is the identifier of a deployment mode - which Teleport Services to enable and their configuration.
DeploymentMode string
@@ -250,7 +251,7 @@ func (r *DeployServiceRequest) CheckAndSetDefaults() error {
}
if r.ResourceCreationTags == nil {
r.ResourceCreationTags = defaultResourceCreationTags(r.TeleportClusterName, r.IntegrationName)
r.ResourceCreationTags = tags.DefaultResourceCreationTags(r.TeleportClusterName, r.IntegrationName)
}
if r.TeleportConfigString == "" {
@@ -471,7 +472,7 @@ type upsertTaskRequest struct {
ClusterName string
ServiceName string
TeleportVersionTag string
ResourceCreationTags AWSTags
ResourceCreationTags tags.AWSTags
Region string
TeleportConfigB64 string
}
@@ -542,7 +543,7 @@ func upsertTask(ctx context.Context, clt DeployServiceClient, req upsertTaskRequ
// It will re-create if its status is INACTIVE.
// If the cluster status is not ACTIVE, an error is returned.
// The cluster is returned.
func upsertCluster(ctx context.Context, clt DeployServiceClient, clusterName string, resourceCreationTags AWSTags) (*ecsTypes.Cluster, error) {
func upsertCluster(ctx context.Context, clt DeployServiceClient, clusterName string, resourceCreationTags tags.AWSTags) (*ecsTypes.Cluster, error) {
describeClustersResponse, err := clt.DescribeClusters(ctx, &ecs.DescribeClustersInput{
Clusters: []string{clusterName},
Include: []ecsTypes.ClusterField{
@@ -687,7 +688,7 @@ func deployServiceNetworkConfiguration(subnetIDs, securityGroups []string) *ecsT
type upsertServiceRequest struct {
ServiceName string
ClusterName string
ResourceCreationTags AWSTags
ResourceCreationTags tags.AWSTags
SubnetIDs []string
SecurityGroups []string
}
@@ -30,6 +30,7 @@ import (
awsapiutils "github.com/gravitational/teleport/api/utils/aws"
awslib "github.com/gravitational/teleport/lib/cloud/aws"
"github.com/gravitational/teleport/lib/integrations/awsoidc/tags"
awslibutils "github.com/gravitational/teleport/lib/utils/aws"
)
@@ -68,7 +69,7 @@ type DeployServiceIAMConfigureRequest struct {
// - teleport.dev/cluster: <cluster>
// - teleport.dev/origin: aws-oidc-integration
// - teleport.dev/integration: <integrationName>
ResourceCreationTags AWSTags
ResourceCreationTags tags.AWSTags
// partitionID is the AWS Partition ID.
// Eg, aws, aws-cn, aws-us-gov
@@ -103,7 +104,7 @@ func (r *DeployServiceIAMConfigureRequest) CheckAndSetDefaults() error {
}
if len(r.ResourceCreationTags) == 0 {
r.ResourceCreationTags = defaultResourceCreationTags(r.Cluster, r.IntegrationName)
r.ResourceCreationTags = tags.DefaultResourceCreationTags(r.Cluster, r.IntegrationName)
}
r.partitionID = awsapiutils.GetPartitionFromRegion(r.Region)
@@ -29,6 +29,8 @@ import (
"github.com/aws/aws-sdk-go-v2/service/sts"
"github.com/gravitational/trace"
"github.com/stretchr/testify/require"
"github.com/gravitational/teleport/lib/integrations/awsoidc/tags"
)
var badParameterCheck = func(t require.TestingT, err error, msgAndArgs ...interface{}) {
@@ -72,7 +74,7 @@ func TestDeployServiceIAMConfigReqDefaults(t *testing.T) {
TaskRole: "taskrole",
partitionID: "aws",
IntegrationRoleDeployServicePolicy: "DeployService",
ResourceCreationTags: AWSTags{
ResourceCreationTags: tags.AWSTags{
"teleport.dev/cluster": "mycluster",
"teleport.dev/integration": "myintegration",
"teleport.dev/origin": "integration_awsoidc",
@@ -32,6 +32,7 @@ import (
"github.com/gravitational/teleport"
"github.com/gravitational/teleport/api/types"
"github.com/gravitational/teleport/lib/automaticupgrades"
"github.com/gravitational/teleport/lib/integrations/awsoidc/tags"
)
func TestDeployServiceRequest(t *testing.T) {
@@ -152,7 +153,7 @@ func TestDeployServiceRequest(t *testing.T) {
TaskName: stringPointer("mycluster-teleport-database-service"),
DeploymentJoinTokenName: "discover-aws-oidc-iam-token",
IntegrationName: "teleportdev",
ResourceCreationTags: AWSTags{
ResourceCreationTags: tags.AWSTags{
"teleport.dev/origin": "integration_awsoidc",
"teleport.dev/cluster": "mycluster",
"teleport.dev/integration": "teleportdev",
@@ -32,6 +32,7 @@ import (
"github.com/gravitational/teleport"
"github.com/gravitational/teleport/lib/automaticupgrades"
awslib "github.com/gravitational/teleport/lib/cloud/aws"
"github.com/gravitational/teleport/lib/integrations/awsoidc/tags"
)
// waitDuration specifies the amount of time to wait for a service to become healthy after an update.
@@ -44,7 +45,7 @@ type UpdateServiceRequest struct {
// TeleportVersionTag specifies the desired teleport version in the format "13.4.0"
TeleportVersionTag string
// OwnershipTags specifies ownership tags
OwnershipTags AWSTags
OwnershipTags tags.AWSTags
}
// CheckAndSetDefaults checks and sets default config values.
@@ -90,7 +91,7 @@ func UpdateDeployService(ctx context.Context, clt DeployServiceClient, log *slog
return nil
}
func updateServiceContainerImage(ctx context.Context, clt DeployServiceClient, log *slog.Logger, service *ecsTypes.Service, teleportImage string, ownershipTags AWSTags) error {
func updateServiceContainerImage(ctx context.Context, clt DeployServiceClient, log *slog.Logger, service *ecsTypes.Service, teleportImage string, ownershipTags tags.AWSTags) error {
taskDefinition, err := getManagedTaskDefinition(ctx, clt, aws.ToString(service.TaskDefinition), ownershipTags)
if err != nil {
return trace.Wrap(err)
@@ -166,7 +167,7 @@ func getAllServiceNamesForCluster(ctx context.Context, clt DeployServiceClient,
return ret, nil
}
func getManagedServices(ctx context.Context, clt DeployServiceClient, log *slog.Logger, teleportClusterName string, ownershipTags AWSTags) ([]ecsTypes.Service, error) {
func getManagedServices(ctx context.Context, clt DeployServiceClient, log *slog.Logger, teleportClusterName string, ownershipTags tags.AWSTags) ([]ecsTypes.Service, error) {
// The Cluster name is created using the Teleport Cluster Name.
// Check the DeployDatabaseServiceRequest.CheckAndSetDefaults
// and DeployServiceRequest.CheckAndSetDefaults.
@@ -224,7 +225,7 @@ func getManagedServices(ctx context.Context, clt DeployServiceClient, log *slog.
return ecsServices, nil
}
func getManagedTaskDefinition(ctx context.Context, clt DeployServiceClient, taskDefinitionName string, ownershipTags AWSTags) (*ecsTypes.TaskDefinition, error) {
func getManagedTaskDefinition(ctx context.Context, clt DeployServiceClient, taskDefinitionName string, ownershipTags tags.AWSTags) (*ecsTypes.TaskDefinition, error) {
describeTaskDefinitionOut, err := clt.DescribeTaskDefinition(ctx, &ecs.DescribeTaskDefinitionInput{
TaskDefinition: aws.String(taskDefinitionName),
Include: []ecsTypes.TaskDefinitionField{ecsTypes.TaskDefinitionFieldTags},
@@ -30,6 +30,7 @@ import (
"github.com/gravitational/teleport"
"github.com/gravitational/teleport/api/types"
"github.com/gravitational/teleport/lib/automaticupgrades"
"github.com/gravitational/teleport/lib/integrations/awsoidc/tags"
"github.com/gravitational/teleport/lib/utils"
)
@@ -160,7 +161,7 @@ func TestUpdateDeployServices(t *testing.T) {
clusterName := "my-cluster"
integrationName := "my-integration"
ownershipTags := defaultResourceCreationTags(clusterName, integrationName)
ownershipTags := tags.DefaultResourceCreationTags(clusterName, integrationName)
teleportVersion := teleport.Version
log := utils.NewSlogLoggerForTests().With("test", t.Name())
+3 -2
View File
@@ -41,6 +41,7 @@ import (
"github.com/gravitational/teleport/api/types"
awslib "github.com/gravitational/teleport/lib/cloud/aws"
"github.com/gravitational/teleport/lib/defaults"
"github.com/gravitational/teleport/lib/integrations/awsoidc/tags"
awsutil "github.com/gravitational/teleport/lib/utils/aws"
"github.com/gravitational/teleport/lib/utils/oidc"
)
@@ -98,7 +99,7 @@ type IdPIAMConfigureRequest struct {
// IntegrationRole is the Integration's AWS Role used to set up Teleport as an OIDC IdP.
IntegrationRole string
ownershipTags AWSTags
ownershipTags tags.AWSTags
}
// CheckAndSetDefaults ensures the required fields are present.
@@ -151,7 +152,7 @@ func (r *IdPIAMConfigureRequest) CheckAndSetDefaults() error {
}
}
r.ownershipTags = defaultResourceCreationTags(r.Cluster, r.IntegrationName)
r.ownershipTags = tags.DefaultResourceCreationTags(r.Cluster, r.IntegrationName)
return nil
}
@@ -38,6 +38,7 @@ import (
"github.com/stretchr/testify/require"
"github.com/gravitational/teleport/lib"
"github.com/gravitational/teleport/lib/integrations/awsoidc/tags"
)
func TestIdPIAMConfigReqDefaults(t *testing.T) {
@@ -79,7 +80,7 @@ func TestIdPIAMConfigReqDefaults(t *testing.T) {
ProxyPublicAddress: "https://proxy.example.com",
issuer: "proxy.example.com",
issuerURL: "https://proxy.example.com",
ownershipTags: AWSTags{
ownershipTags: tags.AWSTags{
"teleport.dev/cluster": "mycluster",
"teleport.dev/integration": "myintegration",
"teleport.dev/origin": "integration_awsoidc",
@@ -110,7 +111,7 @@ func TestIdPIAMConfigReqDefaults(t *testing.T) {
S3JWKSContentsB64: base64EncodedString,
issuer: "bucket-1.s3.amazonaws.com/prefix-2",
issuerURL: "https://bucket-1.s3.amazonaws.com/prefix-2",
ownershipTags: AWSTags{
ownershipTags: tags.AWSTags{
"teleport.dev/cluster": "mycluster",
"teleport.dev/integration": "myintegration",
"teleport.dev/origin": "integration_awsoidc",
@@ -16,15 +16,18 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package awsoidc
package tags
import (
"fmt"
"maps"
"strings"
athenatypes "github.com/aws/aws-sdk-go-v2/service/athena/types"
ec2Types "github.com/aws/aws-sdk-go-v2/service/ec2/types"
ecsTypes "github.com/aws/aws-sdk-go-v2/service/ecs/types"
iamTypes "github.com/aws/aws-sdk-go-v2/service/iam/types"
s3types "github.com/aws/aws-sdk-go-v2/service/s3/types"
"github.com/gravitational/teleport/api/types"
)
@@ -41,12 +44,12 @@ func (d AWSTags) String() string {
return strings.Join(tagsString, ", ")
}
// defaultResourceCreationTags returns the default tags that should be applied when creating new AWS resources.
// DefaultResourceCreationTags returns the default tags that should be applied when creating new AWS resources.
// The following tags are returned:
// - teleport.dev/cluster: <clusterName>
// - teleport.dev/origin: aws-oidc-integration
// - teleport.dev/integration: <integrationName>
func defaultResourceCreationTags(clusterName, integrationName string) AWSTags {
func DefaultResourceCreationTags(clusterName, integrationName string) AWSTags {
return AWSTags{
types.ClusterLabel: clusterName,
types.OriginLabel: types.OriginIntegrationAWSOIDC,
@@ -58,7 +61,6 @@ func defaultResourceCreationTags(clusterName, integrationName string) AWSTags {
func (d AWSTags) ToECSTags() []ecsTypes.Tag {
ecsTags := make([]ecsTypes.Tag, 0, len(d))
for k, v := range d {
k, v := k, v
ecsTags = append(ecsTags, ecsTypes.Tag{
Key: &k,
Value: &v,
@@ -71,7 +73,6 @@ func (d AWSTags) ToECSTags() []ecsTypes.Tag {
func (d AWSTags) ToEC2Tags() []ec2Types.Tag {
ec2Tags := make([]ec2Types.Tag, 0, len(d))
for k, v := range d {
k, v := k, v
ec2Tags = append(ec2Tags, ec2Types.Tag{
Key: &k,
Value: &v,
@@ -118,7 +119,6 @@ func (d AWSTags) MatchesIAMTags(resourceTags []iamTypes.Tag) bool {
func (d AWSTags) ToIAMTags() []iamTypes.Tag {
iamTags := make([]iamTypes.Tag, 0, len(d))
for k, v := range d {
k, v := k, v
iamTags = append(iamTags, iamTypes.Tag{
Key: &k,
Value: &v,
@@ -126,3 +126,33 @@ func (d AWSTags) ToIAMTags() []iamTypes.Tag {
}
return iamTags
}
// ToS3Tags returns the default tags using the expected type for S3 resources: [s3types.Tag]
func (d AWSTags) ToS3Tags() []s3types.Tag {
s3Tags := make([]s3types.Tag, 0, len(d))
for k, v := range d {
s3Tags = append(s3Tags, s3types.Tag{
Key: &k,
Value: &v,
})
}
return s3Tags
}
// ToAthenaTags returns the default tags using the expected type for Athena resources: [athenatypes.Tag]
func (d AWSTags) ToAthenaTags() []athenatypes.Tag {
athenaTags := make([]athenatypes.Tag, 0, len(d))
for k, v := range d {
athenaTags = append(athenaTags, athenatypes.Tag{
Key: &k,
Value: &v,
})
}
return athenaTags
}
// ToMap returns the default tags using the expected type for other aws resources.
// Eg Glue resources
func (d AWSTags) ToMap() map[string]string {
return maps.Clone((map[string]string)(d))
}
@@ -16,7 +16,7 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package awsoidc
package tags
import (
"testing"
@@ -31,7 +31,7 @@ import (
func TestDefaultTags(t *testing.T) {
clusterName := "mycluster"
integrationName := "myawsaccount"
d := defaultResourceCreationTags(clusterName, integrationName)
d := DefaultResourceCreationTags(clusterName, integrationName)
expectedTags := AWSTags{
"teleport.dev/cluster": "mycluster",
@@ -35,6 +35,7 @@ import (
"github.com/gravitational/trace"
eastypes "github.com/gravitational/teleport/api/types/externalauditstorage"
"github.com/gravitational/teleport/lib/integrations/awsoidc/tags"
awsutil "github.com/gravitational/teleport/lib/utils/aws"
)
@@ -53,6 +54,9 @@ type BootstrapInfraParams struct {
Spec *eastypes.ExternalAuditStorageSpec
Region string
ClusterName string
IntegrationName string
}
// BootstrapAthenaClient is a subset of [athena.Client] methods needed for athena bootstrap.
@@ -81,6 +85,8 @@ type BootstrapS3Client interface {
PutBucketVersioning(ctx context.Context, params *s3.PutBucketVersioningInput, optFns ...func(*s3.Options)) (*s3.PutBucketVersioningOutput, error)
// Creates a new lifecycle configuration for the bucket or replaces an existing lifecycle configuration.
PutBucketLifecycleConfiguration(ctx context.Context, params *s3.PutBucketLifecycleConfigurationInput, optFns ...func(*s3.Options)) (*s3.PutBucketLifecycleConfigurationOutput, error)
// Adds tags to a bucket.
PutBucketTagging(ctx context.Context, params *s3.PutBucketTaggingInput, optFns ...func(*s3.Options)) (*s3.PutBucketTaggingOutput, error)
}
// BootstrapInfra bootstraps External Audit Storage infrastructure.
@@ -98,6 +104,10 @@ func BootstrapInfra(ctx context.Context, params BootstrapInfraParams) error {
return trace.BadParameter("param S3 required")
case params.Region == "":
return trace.BadParameter("param Region required")
case params.ClusterName == "":
return trace.BadParameter("param Cluster Name required")
case params.IntegrationName == "":
return trace.BadParameter("param Integration Name required")
case params.Spec == nil:
return trace.BadParameter("param Spec required")
}
@@ -107,19 +117,23 @@ func BootstrapInfra(ctx context.Context, params BootstrapInfraParams) error {
return trace.Wrap(err)
}
if err := createLTSBucket(ctx, params.S3, ltsBucket, params.Region); err != nil {
ownershipTags := tags.DefaultResourceCreationTags(params.ClusterName, params.IntegrationName)
s3OwnershipTags := ownershipTags.ToS3Tags()
if err := createLTSBucket(ctx, params.S3, ltsBucket, params.Region, s3OwnershipTags); err != nil {
return trace.Wrap(err)
}
if err := createTransientBucket(ctx, params.S3, transientBucket, params.Region); err != nil {
if err := createTransientBucket(ctx, params.S3, transientBucket, params.Region, s3OwnershipTags); err != nil {
return trace.Wrap(err)
}
if err := createAthenaWorkgroup(ctx, params.Athena, params.Spec.AthenaWorkgroup); err != nil {
athenaOwnershipTags := ownershipTags.ToAthenaTags()
if err := createAthenaWorkgroup(ctx, params.Athena, params.Spec.AthenaWorkgroup, athenaOwnershipTags); err != nil {
return trace.Wrap(err)
}
if err := createGlueInfra(ctx, params.Glue, params.Spec.GlueTable, params.Spec.GlueDatabase, ltsBucket); err != nil {
if err := createGlueInfra(ctx, params.Glue, params.Region, params.Spec.GlueTable, params.Spec.GlueDatabase, ltsBucket, ownershipTags.ToMap()); err != nil {
return trace.Wrap(err)
}
@@ -132,11 +146,15 @@ func BootstrapInfra(ctx context.Context, params BootstrapInfraParams) error {
// * Object locking enabled with Governance mode and default retention of 4 years
// * Object ownership set to BucketOwnerEnforced
// * Default SSE-S3 encryption
func createLTSBucket(ctx context.Context, clt BootstrapS3Client, bucketName string, region string) error {
func createLTSBucket(ctx context.Context, clt BootstrapS3Client, bucketName string, region string, ownershipTags []s3types.Tag) error {
fmt.Printf("Creating long term storage S3 bucket %s\n", bucketName)
err := createBucket(ctx, clt, bucketName, region, true)
if err != nil && !trace.IsAlreadyExists(err) {
return trace.Wrap(err, "creating long term storage S3 bucket")
err := ensureBucket(ctx, clt, bucketName, region, true)
if err != nil {
return trace.Wrap(err, "creating transient S3 bucket")
}
if err := applyBucketProperties(ctx, clt, bucketName, ownershipTags); err != nil {
return trace.Wrap(err, "applying S3 bucket properties")
}
fmt.Printf("Applying object lock configuration to long term storage S3 bucket with default retention period of %d years\n", defaultObjectLockRetentionYears)
@@ -160,13 +178,17 @@ func createLTSBucket(ctx context.Context, clt BootstrapS3Client, bucketName stri
// policy is created that cleans up transient storage:
// * Query results expire after 1 day
// * DeleteMarkers, NonCurrentVersions and IncompleteMultipartUploads are also removed
func createTransientBucket(ctx context.Context, clt BootstrapS3Client, bucketName string, region string) error {
func createTransientBucket(ctx context.Context, clt BootstrapS3Client, bucketName string, region string, ownershipTags []s3types.Tag) error {
fmt.Printf("Creating transient storage S3 bucket %s\n", bucketName)
err := createBucket(ctx, clt, bucketName, region, false)
if err != nil && !trace.IsAlreadyExists(err) {
err := ensureBucket(ctx, clt, bucketName, region, false)
if err != nil {
return trace.Wrap(err, "creating transient S3 bucket")
}
if err := applyBucketProperties(ctx, clt, bucketName, ownershipTags); err != nil {
return trace.Wrap(err, "applying S3 bucket properties")
}
fmt.Println("Applying bucket lifecycle configuration to transient storage S3 bucket")
_, err = clt.PutBucketLifecycleConfiguration(ctx, &s3.PutBucketLifecycleConfigurationInput{
Bucket: &bucketName,
@@ -202,7 +224,7 @@ func createTransientBucket(ctx context.Context, clt BootstrapS3Client, bucketNam
return trace.Wrap(awsutil.ConvertS3Error(err), "setting lifecycle configuration on S3 bucket")
}
func createBucket(ctx context.Context, clt BootstrapS3Client, bucketName string, region string, objectLock bool) error {
func ensureBucket(ctx context.Context, clt BootstrapS3Client, bucketName string, region string, objectLock bool) error {
_, err := clt.CreateBucket(ctx, &s3.CreateBucketInput{
Bucket: &bucketName,
CreateBucketConfiguration: awsutil.CreateBucketConfiguration(region),
@@ -210,25 +232,48 @@ func createBucket(ctx context.Context, clt BootstrapS3Client, bucketName string,
ACL: s3types.BucketCannedACLPrivate,
ObjectOwnership: s3types.ObjectOwnershipBucketOwnerEnforced,
})
if err != nil {
convertedErr := awsutil.ConvertS3Error(err)
switch {
case convertedErr == nil:
return nil
case trace.IsAlreadyExists(convertedErr):
return nil
default:
return trace.Wrap(convertedErr)
}
}
func applyBucketProperties(ctx context.Context, clt BootstrapS3Client, bucketName string, ownershipTags []s3types.Tag) error {
// s3:CreateBucket doesn't support tags, so the best we can do is
// to tag the bucket shortly after creating it
fmt.Printf("Adding tags to S3 bucket %s\n", bucketName)
if _, err := clt.PutBucketTagging(ctx, &s3.PutBucketTaggingInput{
Bucket: &bucketName,
Tagging: &s3types.Tagging{TagSet: ownershipTags},
}); err != nil {
return trace.Wrap(awsutil.ConvertS3Error(err))
}
_, err = clt.PutBucketVersioning(ctx, &s3.PutBucketVersioningInput{
fmt.Printf("Enabling Bucket versioning to S3 bucket %s\n", bucketName)
if _, err := clt.PutBucketVersioning(ctx, &s3.PutBucketVersioningInput{
Bucket: &bucketName,
VersioningConfiguration: &s3types.VersioningConfiguration{
Status: s3types.BucketVersioningStatusEnabled,
},
})
return trace.Wrap(awsutil.ConvertS3Error(err), "setting versioning configuration on S3 bucket")
}); err != nil {
return trace.Wrap(awsutil.ConvertS3Error(err), "setting versioning configuration on S3 bucket")
}
return nil
}
// createAthenaWorkgroup creates an athena workgroup in which to run athena sql queries.
func createAthenaWorkgroup(ctx context.Context, clt BootstrapAthenaClient, workgroup string) error {
func createAthenaWorkgroup(ctx context.Context, clt BootstrapAthenaClient, workgroup string, ownershipTags []athenatypes.Tag) error {
fmt.Printf("Creating Athena workgroup %s\n", workgroup)
_, err := clt.CreateWorkGroup(ctx, &athena.CreateWorkGroupInput{
Name: &workgroup,
Configuration: &athenatypes.WorkGroupConfiguration{},
Tags: ownershipTags,
})
if err != nil && !strings.Contains(err.Error(), "is already created") {
return trace.Wrap(err, "creating Athena workgroup")
@@ -243,13 +288,14 @@ func createAthenaWorkgroup(ctx context.Context, clt BootstrapAthenaClient, workg
// * CreateDatabase
// * CreateTable
// * UpdateTable
func createGlueInfra(ctx context.Context, clt BootstrapGlueClient, table, database, eventBucket string) error {
func createGlueInfra(ctx context.Context, clt BootstrapGlueClient, region, table, database, eventBucket string, ownershipTags map[string]string) error {
fmt.Printf("Creating Glue database %s\n", database)
_, err := clt.CreateDatabase(ctx, &glue.CreateDatabaseInput{
DatabaseInput: &gluetypes.DatabaseInput{
Name: &database,
Description: aws.String(glueDatabaseDescription),
},
Tags: ownershipTags,
})
if err != nil {
var aee *gluetypes.AlreadyExistsException
@@ -20,6 +20,7 @@ package externalauditstorage_test
import (
"context"
"maps"
"net/url"
"testing"
@@ -138,11 +139,13 @@ func TestBootstrapInfra(t *testing.T) {
athenaClt := &mockBootstrapAthenaClient{}
glueClt := &mockBootstrapGlueClient{}
err := externalauditstorage.BootstrapInfra(testCtx, externalauditstorage.BootstrapInfraParams{
Athena: athenaClt,
Glue: glueClt,
S3: s3Clt,
Spec: tc.spec,
Region: tc.region,
Athena: athenaClt,
Glue: glueClt,
S3: s3Clt,
Spec: tc.spec,
Region: tc.region,
ClusterName: "my-cluster",
IntegrationName: "my-integration",
})
if tc.errWanted != "" {
require.ErrorContainsf(t, err, tc.errWanted, "the error returned did not contain: %s", tc.errWanted)
@@ -175,12 +178,45 @@ func TestBootstrapInfra(t *testing.T) {
// Re-run bootstrap
assert.NoError(t, externalauditstorage.BootstrapInfra(testCtx, externalauditstorage.BootstrapInfraParams{
Athena: athenaClt,
Glue: glueClt,
S3: s3Clt,
Spec: tc.spec,
Region: tc.region,
Athena: athenaClt,
Glue: glueClt,
S3: s3Clt,
Spec: tc.spec,
Region: tc.region,
ClusterName: "my-cluster",
IntegrationName: "my-integration",
}))
// Enrure ownership tags were set on all resources.
// S3 Buckets
for bucketName, bucket := range s3Clt.buckets {
require.ElementsMatch(t,
[]s3types.Tag{
{Key: aws.String("teleport.dev/cluster"), Value: aws.String("my-cluster")},
{Key: aws.String("teleport.dev/origin"), Value: aws.String("integration_awsoidc")},
{Key: aws.String("teleport.dev/integration"), Value: aws.String("my-integration")},
},
bucket.tags,
"tags in bucket=%s do not match the ownership tags", bucketName)
}
// Athena Workgroup
require.ElementsMatch(t,
[]athenatypes.Tag{
{Key: aws.String("teleport.dev/cluster"), Value: aws.String("my-cluster")},
{Key: aws.String("teleport.dev/origin"), Value: aws.String("integration_awsoidc")},
{Key: aws.String("teleport.dev/integration"), Value: aws.String("my-integration")},
},
athenaClt.tags,
)
// Glue Database
require.Equal(t,
map[string]string{
"teleport.dev/cluster": "my-cluster",
"teleport.dev/origin": "integration_awsoidc",
"teleport.dev/integration": "my-integration",
},
glueClt.databaseTags,
)
})
}
}
@@ -191,15 +227,18 @@ type mockBootstrapS3Client struct {
type bucket struct {
locationConstraint s3types.BucketLocationConstraint
tags []s3types.Tag
}
type mockBootstrapAthenaClient struct {
workgroup string
tags []athenatypes.Tag
}
type mockBootstrapGlueClient struct {
table string
database string
table string
database string
databaseTags map[string]string
}
func (c *mockBootstrapS3Client) CreateBucket(ctx context.Context, params *s3.CreateBucketInput, optFns ...func(*s3.Options)) (*s3.CreateBucketOutput, error) {
@@ -219,6 +258,19 @@ func (c *mockBootstrapS3Client) CreateBucket(ctx context.Context, params *s3.Cre
return &s3.CreateBucketOutput{}, nil
}
func (c *mockBootstrapS3Client) PutBucketTagging(ctx context.Context, params *s3.PutBucketTaggingInput, optFns ...func(*s3.Options)) (*s3.PutBucketTaggingOutput, error) {
bucket, ok := c.buckets[*params.Bucket]
if !ok {
// bucket doesn't exist return no such bucket error
return nil, &s3types.NoSuchBucket{Message: aws.String("The bucket doesn't exist")}
}
bucket.tags = params.Tagging.TagSet
c.buckets[*params.Bucket] = bucket
return &s3.PutBucketTaggingOutput{}, nil
}
func (c *mockBootstrapS3Client) PutObjectLockConfiguration(ctx context.Context, params *s3.PutObjectLockConfigurationInput, optFns ...func(*s3.Options)) (*s3.PutObjectLockConfigurationOutput, error) {
if _, ok := c.buckets[*params.Bucket]; !ok {
// bucket doesn't exist return no such bucket error
@@ -251,6 +303,7 @@ func (c *mockBootstrapAthenaClient) CreateWorkGroup(ctx context.Context, params
}
c.workgroup = *params.Name
c.tags = params.Tags
return &athena.CreateWorkGroupOutput{}, nil
}
@@ -280,6 +333,7 @@ func (c *mockBootstrapGlueClient) CreateDatabase(ctx context.Context, params *gl
}
c.database = *params.DatabaseInput.Name
c.databaseTags = maps.Clone(params.Tags)
return &glue.CreateDatabaseOutput{}, nil
}
@@ -25,6 +25,12 @@ type ExternalAuditStorageConfiguration struct {
Bootstrap bool
// Region is the AWS Region used.
Region string
// ClusterName is the Teleport cluster name.
// Used for resource tagging.
ClusterName string
// IntegrationName is the Teleport AWS OIDC Integration name.
// Used for resource tagging.
IntegrationName string
// Role is the AWS IAM Role associated with the OIDC integration.
Role string
// Policy is the name to use for the IAM policy.
@@ -191,7 +191,9 @@ func onIntegrationConfExternalAuditCmd(ctx context.Context, params easconfig.Ext
GlueDatabase: params.GlueDatabase,
GlueTable: params.GlueTable,
},
Region: params.Region,
Region: params.Region,
ClusterName: params.ClusterName,
IntegrationName: params.IntegrationName,
})
if err != nil {
return trace.Wrap(err)
+2
View File
@@ -518,6 +518,8 @@ func Run(options Options) (app *kingpin.Application, executedCommand string, con
integrationConfExternalAuditCmd := integrationConfigureCmd.Command("externalauditstorage", "Bootstraps required infrastructure and adds required IAM permissions for External Audit Storage logs.")
integrationConfExternalAuditCmd.Flag("bootstrap", "Bootstrap required infrastructure.").Default("false").BoolVar(&ccf.IntegrationConfExternalAuditStorageArguments.Bootstrap)
integrationConfExternalAuditCmd.Flag("aws-region", "AWS region.").Required().StringVar(&ccf.IntegrationConfExternalAuditStorageArguments.Region)
integrationConfExternalAuditCmd.Flag("cluster-name", "Teleport Cluster name.").Required().StringVar(&ccf.IntegrationConfExternalAuditStorageArguments.ClusterName)
integrationConfExternalAuditCmd.Flag("integration", "AWS OIDC Integration name.").Required().StringVar(&ccf.IntegrationConfExternalAuditStorageArguments.IntegrationName)
integrationConfExternalAuditCmd.Flag("role", "The IAM Role used by the AWS OIDC Integration.").Required().StringVar(&ccf.IntegrationConfExternalAuditStorageArguments.Role)
integrationConfExternalAuditCmd.Flag("policy", "The name for the Policy to attach to the IAM role.").Required().StringVar(&ccf.IntegrationConfExternalAuditStorageArguments.Policy)
integrationConfExternalAuditCmd.Flag("session-recordings", "The S3 URI where session recordings are stored.").Required().StringVar(&ccf.IntegrationConfExternalAuditStorageArguments.SessionRecordingsURI)