Generate AWS regions list from AWS SDK V2 partitions file (#53104)

* feat: generate aws regions from SDK v2

* refactor(aws): only sort regions once

* chore(aws): add license header to generated file

* refactor: move global regions to another variable

* refactor: code review suggestions

* refactor(make): update go-generate target

* chore(ci): add target to lint go-generate up-to-date

* refactor(aws): drop global regions

* refactor(workflows): update target and remove git command

* refactor(aws): wrap regions value in sync.OnceValue

* refactor: move aws regions into a sub package of utils aws

* chore(api): remove global regions matching

* fix(makefile): target name

* chore(gomod): go mod tidy

* fix(tctl): add missing awsutils package

* chore(gomod): move indirect aws-sdk-go into indirect deps block

* add a temporary alias to avoid break e

---------

Co-authored-by: STeve (Xin) Huang <huangin52@gmail.com>

.github/workflows/lint.yaml

@@ -156,6 +156,9 @@ jobs:
         # We have to add the current directory as a safe directory or else git commands will not work as expected.
         run: git config --global --add safe.directory $(realpath .) && make derive-up-to-date
 
+      - name: Check if go generated files are up to date
+        run: make go-generate-up-to-date
+
   lint-rust:
     name: Lint (Rust)
     runs-on: ubuntu-22.04

Makefile

@@ -1627,6 +1627,19 @@ terraform-resources-up-to-date: must-start-clean/host
 		exit 1; \
 	fi
 
+# go-generate will execute `go generate` and generate go code.
+.PHONY: go-generate
+go-generate:
+	go generate ./lib/...
+
+# go-generate-up-to-date checks if the generated code is up to date.
+.PHONY: go-generate-up-to-date
+go-generate-up-to-date: must-start-clean/host go-generate
+	@if ! git diff --quiet; then \
+		./build.assets/please-run.sh "go generate lib" "make go-generate"; \
+		exit 1; \
+	fi
+
 print/env:
 	env
 

api/utils/aws/identifiers.go

@@ -167,7 +167,7 @@ var (
 	//
 	// Reference:
 	// https://github.com/aws/aws-sdk-go-v2/blob/main/codegen/smithy-aws-go-codegen/src/main/resources/software/amazon/smithy/aws/go/codegen/endpoints.json
-	matchRegion = regexp.MustCompile(`^[a-z]{2}(-gov|-iso|-isob|-isoe)?-\w+-\d+$`)
+	matchRegion = regexp.MustCompile(`^[a-z]{2}(-gov|-iso|-isob|-isoe|-isof)?-\w+-\d+$`)
 
 	// https://docs.aws.amazon.com/athena/latest/APIReference/API_CreateWorkGroup.html
 	matchAthenaWorkgroupName = regexp.MustCompile(`^[a-zA-Z0-9._-]{1,128}$`).MatchString

go.mod

@@ -42,7 +42,6 @@ require (
 	github.com/andybalholm/brotli v1.1.1
 	github.com/aquasecurity/libbpfgo v0.5.1-libbpf-1.2
 	github.com/armon/go-radix v1.0.0
-	github.com/aws/aws-sdk-go v1.55.6
 	github.com/aws/aws-sdk-go-v2 v1.36.3
 	github.com/aws/aws-sdk-go-v2/config v1.29.12
 	github.com/aws/aws-sdk-go-v2/credentials v1.17.65
@@ -285,6 +284,7 @@ require (
 	github.com/apache/arrow-go/v18 v18.0.0 // indirect
 	github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect
 	github.com/atotto/clipboard v0.1.4 // indirect
+	github.com/aws/aws-sdk-go v1.55.6 // indirect
 	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.10 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.34 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.34 // indirect

integrations/event-handler/go.mod

@@ -62,7 +62,6 @@ require (
 	github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 // indirect
 	github.com/armon/go-radix v1.0.0 // indirect
 	github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect
-	github.com/aws/aws-sdk-go v1.55.6 // indirect
 	github.com/aws/aws-sdk-go-v2 v1.36.3 // indirect
 	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.10 // indirect
 	github.com/aws/aws-sdk-go-v2/config v1.29.12 // indirect

lib/config/configuration.go

@@ -67,7 +67,7 @@ import (
 	"github.com/gravitational/teleport/lib/services"
 	"github.com/gravitational/teleport/lib/tlsca"
 	"github.com/gravitational/teleport/lib/utils"
-	awsutils "github.com/gravitational/teleport/lib/utils/aws"
+	awsregion "github.com/gravitational/teleport/lib/utils/aws/region"
 	logutils "github.com/gravitational/teleport/lib/utils/log"
 )
 
@@ -1502,12 +1502,12 @@ func applyDiscoveryConfig(fc *FileConfig, cfg *servicecfg.Config) error {
 		}
 
 		for _, region := range matcher.Regions {
-			if !awsutils.IsKnownRegion(region) {
+			if !awsregion.IsKnownRegion(region) {
 				const message = "AWS matcher uses unknown region" +
 					"This is either a typo or a new AWS region that is unknown to the AWS SDK used to compile this binary. "
 				slog.WarnContext(context.Background(), message,
 					"region", region,
-					"known_regions", awsutils.GetKnownRegions(),
+					"known_regions", awsregion.GetKnownRegions(),
 				)
 			}
 		}

lib/srv/discovery/fetchers/aws-sync/s3.go

@@ -33,7 +33,7 @@ import (
 	"google.golang.org/protobuf/types/known/timestamppb"
 
 	accessgraphv1alpha "github.com/gravitational/teleport/gen/proto/go/accessgraph/v1alpha"
-	awsutil "github.com/gravitational/teleport/lib/utils/aws"
+	awsregion "github.com/gravitational/teleport/lib/utils/aws/region"
 )
 
 // s3Client defines a subset of the AWS S3 client API.
@@ -294,7 +294,7 @@ func isS3BucketNoTagSet(err error) bool {
 }
 
 func (a *Fetcher) listS3Buckets(ctx context.Context) ([]s3types.Bucket, func(*string) (string, error), error) {
-	region := awsutil.GetKnownRegions()[0]
+	region := awsregion.GetKnownRegions()[0]
 	if len(a.Regions) > 0 {
 		region = a.Regions[0]
 	}

lib/utils/aws/aws.go

@@ -38,6 +38,7 @@ import (
 	apievents "github.com/gravitational/teleport/api/types/events"
 	apiawsutils "github.com/gravitational/teleport/api/utils/aws"
 	"github.com/gravitational/teleport/lib/utils"
+	awsregion "github.com/gravitational/teleport/lib/utils/aws/region"
 )
 
 const (
@@ -521,3 +522,8 @@ func iamResourceARN(partition, accountID, resourceType, resourceName string) str
 		Resource:  fmt.Sprintf("%s/%s", resourceType, resourceName),
 	}.String()
 }
+
+// IsKnownRegion returns true if provided region is one of the "well-known" AWS
+// regions.
+// TODO(greedy52): DELETE once e is updated to use the package directly.
+var IsKnownRegion = awsregion.IsKnownRegion

lib/utils/aws/identitycenterutils/scim.go

@@ -24,7 +24,7 @@ import (
 
 	"github.com/gravitational/trace"
 
-	awsutils "github.com/gravitational/teleport/lib/utils/aws"
+	awsregion "github.com/gravitational/teleport/lib/utils/aws/region"
 )
 
 // matchAWSICEndpointIDField matches an alphanumeric value separated by a hyphen.
@@ -73,7 +73,7 @@ func EnsureSCIMEndpointURL(baseURL *url.URL) (*url.URL, error) {
 		return nil, trace.BadParameter("unrecognized SCIM endpoint")
 	}
 	region := domainParts[1]
-	if !awsutils.IsKnownRegion(region) {
+	if !awsregion.IsKnownRegion(region) {
 		return nil, trace.BadParameter("region %q is invalid", region)
 	}
 	if domainParts[2] != "amazonaws" || domainParts[3] != "com" {

lib/utils/aws/region/generate.go

@@ -0,0 +1,20 @@
+// Teleport
+// Copyright (C) 2025 Gravitational, Inc.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package aws
+
+//go:generate go run ./internal/codegen.go
+//go:generate gofmt -w -s .

lib/utils/aws/region/internal/codegen.go

@@ -0,0 +1,181 @@
+//go:build ignore
+// +build ignore
+
+// Teleport
+// Copyright (C) 2025 Gravitational, Inc.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+package main
+
+import (
+	"encoding/json"
+	"log"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"text/template"
+
+	"github.com/gravitational/trace"
+)
+
+var tmpl = template.Must(template.New("generate").
+	Funcs(map[string]any{
+		"quote": func(v string) string {
+			return strconv.Quote(v)
+		},
+	}).
+	Parse(`
+// Teleport
+// Copyright (C) 2025 Gravitational, Inc.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+// Code generated by internal/region/codegen.go. DO NOT EDIT.
+
+package aws
+
+import (
+	"sync"
+)
+
+// Region represents an AWS region.
+type Region struct {
+	// Name is the region name.
+	Name string
+	// Description is the human readable region name.
+	Description string
+}
+
+// regions is the set of available AWS regions based on the AWS SDK definition.
+var regions = sync.OnceValue(func() map[string]Region {
+	return map[string]Region{
+		{{- range $name, $desc := $.Regions }}
+		{{ quote $name }}: Region{
+			Name: {{ quote $name }},
+			Description: {{ quote $desc }},
+		},
+		{{- end }}
+	}
+})
+`))
+
+// TemplateData is the data passed to the template.
+type TemplateData struct {
+	Regions map[string]string
+}
+
+// AWSPartitions contains the struct representation of the AWS SDK patitions
+// JSON file.
+type AWSPartitions struct {
+	Partitions []struct {
+		Regions map[string]struct {
+			Description string `json:"description"`
+		} `json:"regions"`
+	} `json:"partitions"`
+}
+
+func main() {
+	partitionsContents, err := loadSDKPartitionsFile()
+	if err != nil {
+		log.Fatalf("failed to parse patitions: %s", trace.DebugReport(err))
+	}
+
+	data, err := convertToTemplateData(partitionsContents)
+	if err != nil {
+		log.Fatalf("failed to parse partitions file: %s", trace.DebugReport(err))
+	}
+
+	file, err := os.Create(outputFilename)
+	if err != nil {
+		log.Fatalf("failed to create output file: %v", err)
+	}
+	defer func() {
+		if err := file.Close(); err != nil {
+			log.Fatalf("failed to close output file: %v", err)
+		}
+	}()
+
+	if err := tmpl.Execute(file, data); err != nil {
+		log.Fatalf("failed to generate file: %v", err)
+	}
+}
+
+func convertToTemplateData(partitionsContents []byte) (*TemplateData, error) {
+	var awsPartitions AWSPartitions
+	if err := json.Unmarshal(partitionsContents, &awsPartitions); err != nil {
+		return nil, trace.Wrap(err)
+	}
+
+	if len(awsPartitions.Partitions) == 0 {
+		return nil, trace.BadParameter("generated an empty regions list. check if the partitions JSON file format has changed")
+	}
+
+	data := &TemplateData{
+		Regions: make(map[string]string),
+	}
+
+	for _, partition := range awsPartitions.Partitions {
+		for name, region := range partition.Regions {
+			// For now, we're not including the global regions.
+			if strings.HasSuffix(name, awsGlobalRegionSuffix) {
+				continue
+			}
+
+			data.Regions[name] = region.Description
+		}
+	}
+
+	return data, nil
+}
+
+// loadSDKPartitionsFile loads the partitions JSON file from AWS SDK.
+func loadSDKPartitionsFile() ([]byte, error) {
+	cmd := exec.Command("go", "list", "-m", "-f", "{{.Dir}}", awsPackageName)
+	awsPackageDir, err := cmd.Output()
+	if err != nil {
+		return nil, trace.Wrap(err)
+	}
+
+	contents, err := os.ReadFile(filepath.Join(strings.TrimSpace(string(awsPackageDir)), awsPartitionsFilePath))
+	if err != nil {
+		return nil, trace.Wrap(err)
+	}
+
+	return contents, nil
+}
+
+const (
+	// outputFilename is the generated filename.
+	outputFilename = "regions.go"
+	// awsPackageName is the AWS package used to find the paritions references.
+	awsPackageName = "github.com/aws/aws-sdk-go-v2"
+	// awsPartitionsFilePath is the patitions JSON file path inside the package.
+	awsPartitionsFilePath = "internal/endpoints/awsrulesfn/partitions.json"
+	// globalRegionSuffix is a strings suffix that indicates a region is a
+	// global region.
+	awsGlobalRegionSuffix = "-global"
+)

lib/utils/aws/region/region.go

@@ -22,31 +22,20 @@ import (
 	"maps"
 	"slices"
 	"sync"
-
-	"github.com/aws/aws-sdk-go/aws/endpoints"
 )
 
+var sortedRegions = sync.OnceValue(func() []string {
+	return slices.Sorted(maps.Keys(regions()))
+})
+
 // IsKnownRegion returns true if provided region is one of the "well-known"
 // AWS regions.
 func IsKnownRegion(region string) bool {
-	return slices.Contains(GetKnownRegions(), region)
+	_, ok := regions()[region]
+	return ok
 }
 
-// GetKnownRegions returns a list of "well-known" AWS regions generated from
-// AWS SDK.
+// GetKnownRegions returns a list of "well-known" AWS regions.
 func GetKnownRegions() []string {
-	knownRegionsOnce.Do(func() {
-		var regions []string
-		partitions := endpoints.DefaultPartitions()
-		for _, partition := range partitions {
-			regions = append(regions, slices.Collect(maps.Keys(partition.Regions()))...)
-		}
-		knownRegions = regions
-	})
-	return knownRegions
+	return sortedRegions()
 }
-
-var (
-	knownRegions     []string
-	knownRegionsOnce sync.Once
-)

lib/utils/aws/region/region_test.go

@@ -52,6 +52,7 @@ func TestGetKnownRegions(t *testing.T) {
 	})
 
 }
+
 func TestIsKnownRegion(t *testing.T) {
 	for _, region := range GetKnownRegions() {
 		require.True(t, IsKnownRegion(region))

lib/utils/aws/region/regions.go

@@ -0,0 +1,205 @@
+// Teleport
+// Copyright (C) 2025 Gravitational, Inc.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+// Code generated by internal/region/codegen.go. DO NOT EDIT.
+
+package aws
+
+import (
+	"sync"
+)
+
+// Region represents an AWS region.
+type Region struct {
+	// Name is the region name.
+	Name string
+	// Description is the human readable region name.
+	Description string
+}
+
+// regions is the set of available AWS regions based on the AWS SDK definition.
+var regions = sync.OnceValue(func() map[string]Region {
+	return map[string]Region{
+		"af-south-1": {
+			Name:        "af-south-1",
+			Description: "Africa (Cape Town)",
+		},
+		"ap-east-1": {
+			Name:        "ap-east-1",
+			Description: "Asia Pacific (Hong Kong)",
+		},
+		"ap-northeast-1": {
+			Name:        "ap-northeast-1",
+			Description: "Asia Pacific (Tokyo)",
+		},
+		"ap-northeast-2": {
+			Name:        "ap-northeast-2",
+			Description: "Asia Pacific (Seoul)",
+		},
+		"ap-northeast-3": {
+			Name:        "ap-northeast-3",
+			Description: "Asia Pacific (Osaka)",
+		},
+		"ap-south-1": {
+			Name:        "ap-south-1",
+			Description: "Asia Pacific (Mumbai)",
+		},
+		"ap-south-2": {
+			Name:        "ap-south-2",
+			Description: "Asia Pacific (Hyderabad)",
+		},
+		"ap-southeast-1": {
+			Name:        "ap-southeast-1",
+			Description: "Asia Pacific (Singapore)",
+		},
+		"ap-southeast-2": {
+			Name:        "ap-southeast-2",
+			Description: "Asia Pacific (Sydney)",
+		},
+		"ap-southeast-3": {
+			Name:        "ap-southeast-3",
+			Description: "Asia Pacific (Jakarta)",
+		},
+		"ap-southeast-4": {
+			Name:        "ap-southeast-4",
+			Description: "Asia Pacific (Melbourne)",
+		},
+		"ap-southeast-5": {
+			Name:        "ap-southeast-5",
+			Description: "Asia Pacific (Malaysia)",
+		},
+		"ap-southeast-7": {
+			Name:        "ap-southeast-7",
+			Description: "Asia Pacific (Thailand)",
+		},
+		"ca-central-1": {
+			Name:        "ca-central-1",
+			Description: "Canada (Central)",
+		},
+		"ca-west-1": {
+			Name:        "ca-west-1",
+			Description: "Canada West (Calgary)",
+		},
+		"cn-north-1": {
+			Name:        "cn-north-1",
+			Description: "China (Beijing)",
+		},
+		"cn-northwest-1": {
+			Name:        "cn-northwest-1",
+			Description: "China (Ningxia)",
+		},
+		"eu-central-1": {
+			Name:        "eu-central-1",
+			Description: "Europe (Frankfurt)",
+		},
+		"eu-central-2": {
+			Name:        "eu-central-2",
+			Description: "Europe (Zurich)",
+		},
+		"eu-isoe-west-1": {
+			Name:        "eu-isoe-west-1",
+			Description: "EU ISOE West",
+		},
+		"eu-north-1": {
+			Name:        "eu-north-1",
+			Description: "Europe (Stockholm)",
+		},
+		"eu-south-1": {
+			Name:        "eu-south-1",
+			Description: "Europe (Milan)",
+		},
+		"eu-south-2": {
+			Name:        "eu-south-2",
+			Description: "Europe (Spain)",
+		},
+		"eu-west-1": {
+			Name:        "eu-west-1",
+			Description: "Europe (Ireland)",
+		},
+		"eu-west-2": {
+			Name:        "eu-west-2",
+			Description: "Europe (London)",
+		},
+		"eu-west-3": {
+			Name:        "eu-west-3",
+			Description: "Europe (Paris)",
+		},
+		"il-central-1": {
+			Name:        "il-central-1",
+			Description: "Israel (Tel Aviv)",
+		},
+		"me-central-1": {
+			Name:        "me-central-1",
+			Description: "Middle East (UAE)",
+		},
+		"me-south-1": {
+			Name:        "me-south-1",
+			Description: "Middle East (Bahrain)",
+		},
+		"mx-central-1": {
+			Name:        "mx-central-1",
+			Description: "Mexico (Central)",
+		},
+		"sa-east-1": {
+			Name:        "sa-east-1",
+			Description: "South America (Sao Paulo)",
+		},
+		"us-east-1": {
+			Name:        "us-east-1",
+			Description: "US East (N. Virginia)",
+		},
+		"us-east-2": {
+			Name:        "us-east-2",
+			Description: "US East (Ohio)",
+		},
+		"us-gov-east-1": {
+			Name:        "us-gov-east-1",
+			Description: "AWS GovCloud (US-East)",
+		},
+		"us-gov-west-1": {
+			Name:        "us-gov-west-1",
+			Description: "AWS GovCloud (US-West)",
+		},
+		"us-iso-east-1": {
+			Name:        "us-iso-east-1",
+			Description: "US ISO East",
+		},
+		"us-iso-west-1": {
+			Name:        "us-iso-west-1",
+			Description: "US ISO WEST",
+		},
+		"us-isob-east-1": {
+			Name:        "us-isob-east-1",
+			Description: "US ISOB East (Ohio)",
+		},
+		"us-isof-east-1": {
+			Name:        "us-isof-east-1",
+			Description: "US ISOF EAST",
+		},
+		"us-isof-south-1": {
+			Name:        "us-isof-south-1",
+			Description: "US ISOF SOUTH",
+		},
+		"us-west-1": {
+			Name:        "us-west-1",
+			Description: "US West (N. California)",
+		},
+		"us-west-2": {
+			Name:        "us-west-2",
+			Description: "US West (Oregon)",
+		},
+	}
+})

tool/tctl/common/plugin/awsic.go

@@ -33,6 +33,7 @@ import (
 	"github.com/gravitational/teleport/lib/client"
 	awsutils "github.com/gravitational/teleport/lib/utils/aws"
 	icutils "github.com/gravitational/teleport/lib/utils/aws/identitycenterutils"
+	awsregion "github.com/gravitational/teleport/lib/utils/aws/region"
 )
 
 type awsICArgs struct {
@@ -53,7 +54,7 @@ type awsICArgs struct {
 }
 
 func (a *awsICArgs) validate(ctx context.Context, log *slog.Logger) error {
-	if !awsutils.IsKnownRegion(a.region) {
+	if !awsregion.IsKnownRegion(a.region) {
 		return trace.BadParameter("unknown AWS region: %s", a.region)
 	}