chore: support external types in typescript codegen (#9633)

* chore: support external types in typescript codegen * fix enums on external packages * Support clibase.struct * Add regexp * Make gen with updated generator
2026-09-22 05:05:20 +08:00 · 2023-09-12 18:21:09 -05:00
parent 641bf272ed
commit 18c34ee456
5 changed files with 255 additions and 84 deletions
@@ -15,6 +15,7 @@ import (
 	"text/template"

 	"github.com/fatih/structtag"
+	"golang.org/x/exp/slices"
 	"golang.org/x/text/cases"
 	"golang.org/x/text/language"
 	"golang.org/x/tools/go/packages"
@@ -26,17 +27,35 @@ import (
 )

 var (
+	// baseDirs are the directories to introspect for types to generate.
 	baseDirs = [...]string{"./codersdk", "./coderd/healthcheck", "./coderd/healthcheck/derphealth"}
-	indent   = "  "
+	// externalTypes are types that are not in the baseDirs, but we want to
+	// support. These are usually types that are used in the baseDirs.
+	// Do not include things like "Database", as that would break the idea
+	// of splitting db and api types.
+	// Only include dirs that are client facing packages.
+	externalTypeDirs = [...]string{"./cli/clibase"}
+	indent           = "  "
 )

 func main() {
 	ctx := context.Background()
 	log := slog.Make(sloghuman.Sink(os.Stderr))
+
+	external := []*Generator{}
+	for _, dir := range externalTypeDirs {
+		extGen, err := ParseDirectory(ctx, log, dir)
+		if err != nil {
+			log.Fatal(ctx, fmt.Sprintf("parse external directory %s: %s", dir, err.Error()))
+		}
+		extGen.onlyOptIn = true
+		external = append(external, extGen)
+	}
+
 	_, _ = fmt.Print("// Code generated by 'make site/src/api/typesGenerated.ts'. DO NOT EDIT.\n\n")
 	for _, baseDir := range baseDirs {
 		_, _ = fmt.Printf("// The code below is generated from %s.\n\n", strings.TrimPrefix(baseDir, "./"))
-		output, err := Generate(baseDir)
+		output, err := Generate(baseDir, external...)
 		if err != nil {
 			log.Fatal(ctx, err.Error())
 		}
@@ -44,18 +63,40 @@ func main() {
 		// Just cat the output to a file to capture it
 		_, _ = fmt.Print(output, "\n\n")
 	}
+
+	for i, ext := range external {
+		var ts *TypescriptTypes
+		for {
+			var err error
+			start := len(ext.allowList)
+			ts, err = ext.generateAll()
+			if err != nil {
+				log.Fatal(ctx, fmt.Sprintf("generate external: %s", err.Error()))
+			}
+			if len(ext.allowList) != start {
+				// This is so dumb, but basically the allowList can grow, and if
+				// it does, we need to regenerate.
+				continue
+			}
+			break
+		}
+
+		dir := externalTypeDirs[i]
+		_, _ = fmt.Printf("// The code below is generated from %s.\n\n", strings.TrimPrefix(dir, "./"))
+		_, _ = fmt.Print(ts.String(), "\n\n")
+	}
 }

-func Generate(directory string) (string, error) {
+func Generate(directory string, externals ...*Generator) (string, error) {
 	ctx := context.Background()
 	log := slog.Make(sloghuman.Sink(os.Stderr))
-	codeBlocks, err := GenerateFromDirectory(ctx, log, directory)
+	gen, err := GenerateFromDirectory(ctx, log, directory, externals...)
 	if err != nil {
 		return "", err
 	}

 	// Just cat the output to a file to capture it
-	return codeBlocks.String(), nil
+	return gen.cachedResult.String(), nil
 }

 // TypescriptTypes holds all the code blocks created.
@@ -109,23 +150,34 @@ func (t TypescriptTypes) String() string {
 	return strings.TrimRight(s.String(), "\n")
 }

-// GenerateFromDirectory will return all the typescript code blocks for a directory
-func GenerateFromDirectory(ctx context.Context, log slog.Logger, directory string) (*TypescriptTypes, error) {
-	g := Generator{
-		log:      log,
-		builtins: make(map[string]string),
+func ParseDirectory(ctx context.Context, log slog.Logger, directory string, externals ...*Generator) (*Generator, error) {
+	g := &Generator{
+		log:       log,
+		builtins:  make(map[string]string),
+		externals: externals,
 	}
 	err := g.parsePackage(ctx, directory)
 	if err != nil {
 		return nil, xerrors.Errorf("parse package %q: %w", directory, err)
 	}

-	codeBlocks, err := g.generateAll()
+	return g, nil
+}
+
+// GenerateFromDirectory will return all the typescript code blocks for a directory
+func GenerateFromDirectory(ctx context.Context, log slog.Logger, directory string, externals ...*Generator) (*Generator, error) {
+	g, err := ParseDirectory(ctx, log, directory, externals...)
 	if err != nil {
-		return nil, xerrors.Errorf("parse package %q: %w", directory, err)
+		return nil, err
 	}

-	return codeBlocks, nil
+	codeBlocks, err := g.generateAll()
+	if err != nil {
+		return nil, xerrors.Errorf("generate package %q: %w", directory, err)
+	}
+	g.cachedResult = codeBlocks
+
+	return g, nil
 }

 type Generator struct {
@@ -133,6 +185,16 @@ type Generator struct {
 	pkg *packages.Package
 	log slog.Logger

+	// allowList if set only generates types in the allow list.
+	// This is kinda a hack to get around the fact that external types
+	// only should generate referenced types, and multiple packages can
+	// reference the same external types.
+	onlyOptIn bool
+	allowList []string
+
+	// externals are other packages referenced. Optional
+	externals []*Generator
+
 	// builtins is kinda a hack to get around the fact that using builtin
 	// generic constraints is common. We want to support them even though
 	// they are external to our package.
@@ -141,6 +203,8 @@ type Generator struct {
 	// cannot be implemented in go. So they are a first class thing that we just
 	// have to make a static string for ¯\_(ツ)_/¯
 	builtins map[string]string
+
+	cachedResult *TypescriptTypes
 }

 // parsePackage takes a list of patterns such as a directory, and parses them.
@@ -180,6 +244,10 @@ func (g *Generator) generateAll() (*TypescriptTypes, error) {
 		AllowedTypes: make(map[string]struct{}),
 	}

+	for _, a := range g.allowList {
+		m.AllowedTypes[strings.TrimSpace(a)] = struct{}{}
+	}
+
 	// Look for comments that indicate to ignore a type for typescript generation.
 	ignoreRegex := regexp.MustCompile("@typescript-ignore[:]?(?P<ignored_types>.*)")
 	for _, file := range g.pkg.Syntax {
@@ -303,11 +371,16 @@ func (g *Generator) generateOne(m *Maps, obj types.Object) error {
 	}

 	// If we have allowed types, only allow those to be generated.
-	if _, ok := m.AllowedTypes[obj.Name()]; len(m.AllowedTypes) > 0 && !ok {
-		return nil
+	if _, ok := m.AllowedTypes[obj.Name()]; (len(m.AllowedTypes) > 0 || g.onlyOptIn) && !ok {
+		// Allow constants to pass through, they are only included if the enum
+		// is allowed.
+		_, ok := obj.(*types.Const)
+		if !ok {
+			return nil
+		}
 	}

-	objName := objName(obj)
+	objectName := objName(obj)

 	switch obj := obj.(type) {
 	// All named types are type declarations
@@ -322,13 +395,13 @@ func (g *Generator) generateOne(m *Maps, obj types.Object) error {
 			// Structs are obvious.
 			codeBlock, err := g.buildStruct(obj, underNamed)
 			if err != nil {
-				return xerrors.Errorf("generate %q: %w", objName, err)
+				return xerrors.Errorf("generate %q: %w", objectName, err)
 			}
-			m.Structs[objName] = codeBlock
+			m.Structs[objectName] = codeBlock
 		case *types.Basic:
 			// type <Name> string
 			// These are enums. Store to expand later.
-			m.Enums[objName] = obj
+			m.Enums[objectName] = obj
 		case *types.Map, *types.Array, *types.Slice:
 			// Declared maps that are not structs are still valid codersdk objects.
 			// Handle them custom by calling 'typescriptType' directly instead of
@@ -337,7 +410,7 @@ func (g *Generator) generateOne(m *Maps, obj types.Object) error {
 			// These are **NOT** enums, as a map in Go would never be used for an enum.
 			ts, err := g.typescriptType(obj.Type().Underlying())
 			if err != nil {
-				return xerrors.Errorf("(map) generate %q: %w", objName, err)
+				return xerrors.Errorf("(map) generate %q: %w", objectName, err)
 			}

 			var str strings.Builder
@@ -347,8 +420,8 @@ func (g *Generator) generateOne(m *Maps, obj types.Object) error {
 				_, _ = str.WriteRune('\n')
 			}
 			// Use similar output syntax to enums.
-			_, _ = str.WriteString(fmt.Sprintf("export type %s = %s\n", objName, ts.ValueType))
-			m.Structs[objName] = str.String()
+			_, _ = str.WriteString(fmt.Sprintf("export type %s = %s\n", objectName, ts.ValueType))
+			m.Structs[objectName] = str.String()
 		case *types.Interface:
 			// Interfaces are used as generics. Non-generic interfaces are
 			// not supported.
@@ -366,9 +439,9 @@ func (g *Generator) generateOne(m *Maps, obj types.Object) error {

 				block, err := g.buildUnion(obj, union)
 				if err != nil {
-					return xerrors.Errorf("generate union %q: %w", objName, err)
+					return xerrors.Errorf("generate union %q: %w", objectName, err)
 				}
-				m.Generics[objName] = block
+				m.Generics[objectName] = block
 			}
 		case *types.Signature:
 		// Ignore named functions.
@@ -383,13 +456,13 @@ func (g *Generator) generateOne(m *Maps, obj types.Object) error {
 	case *types.Const:
 		// We only care about named constant types, since they are enums
 		if named, ok := obj.Type().(*types.Named); ok {
-			name := named.Obj().Name()
-			m.EnumConsts[name] = append(m.EnumConsts[name], obj)
+			enumObjName := objName(named.Obj())
+			m.EnumConsts[enumObjName] = append(m.EnumConsts[enumObjName], obj)
 		}
 	case *types.Func:
 		// Noop
 	default:
-		_, _ = fmt.Println(objName)
+		_, _ = fmt.Println(objectName)
 	}
 	return nil
 }
@@ -751,9 +824,22 @@ func (g *Generator) typescriptType(ty types.Type) (TypescriptType, error) {
 		n := ty

 		// These are external named types that we handle uniquely.
+		// This is unfortunate, but our current code assumes all defined
+		// types are enums, but these are really just basic primitives.
+		// We would need to add more logic to determine this, but for now
+		// just hard code them.
 		switch n.String() {
+		case "github.com/coder/coder/v2/cli/clibase.Regexp":
+			return TypescriptType{ValueType: "string"}, nil
+		case "github.com/coder/coder/v2/cli/clibase.HostPort":
+			// Custom marshal json to be a string
+			return TypescriptType{ValueType: "string"}, nil
+		case "github.com/coder/coder/v2/cli/clibase.StringArray":
+			return TypescriptType{ValueType: "string[]"}, nil
 		case "github.com/coder/coder/v2/cli/clibase.String":
 			return TypescriptType{ValueType: "string"}, nil
+		case "github.com/coder/coder/v2/cli/clibase.YAMLConfigPath":
+			return TypescriptType{ValueType: "string"}, nil
 		case "github.com/coder/coder/v2/cli/clibase.Strings":
 			return TypescriptType{ValueType: "string[]"}, nil
 		case "github.com/coder/coder/v2/cli/clibase.Int64":
@@ -783,14 +869,42 @@ func (g *Generator) typescriptType(ty types.Type) (TypescriptType, error) {
 			return TypescriptType{ValueType: "string"}, nil
 		}

+		// Some hard codes are a bit trickier.
+		//nolint:gocritic,revive // I prefer the switch for extensibility later.
+		switch {
+		// Struct is a generic, so the type has generic constraints in the string.
+		case regexp.MustCompile(`github\.com/coder/coder/v2/cli/clibase.Struct\[.*\]`).MatchString(n.String()):
+			// The marshal json just marshals the underlying value.
+			str, ok := ty.Underlying().(*types.Struct)
+			if ok {
+				return g.typescriptType(str.Field(0).Type())
+			}
+		}
+
 		// Then see if the type is defined elsewhere. If it is, we can just
 		// put the objName as it will be defined in the typescript codeblock
 		// we generate.
 		objName := objName(n.Obj())
 		genericName := ""
 		genericTypes := make(map[string]string)
-		pkgName := n.Obj().Pkg().Name()
-		if obj := g.pkg.Types.Scope().Lookup(n.Obj().Name()); g.pkg.Name == pkgName && obj != nil {
+
+		obj, objGen, local := g.lookupNamedReference(n)
+		if obj != nil {
+			if g.onlyOptIn && !slices.Contains(g.allowList, n.Obj().Name()) {
+				// This is kludgy, but if we are an external package,
+				// we need to also include dependencies. There is no
+				// good way to return all extra types we need to include,
+				// so just add them to the allow list and hope the caller notices
+				// the slice grew...
+				g.allowList = append(g.allowList, n.Obj().Name())
+			}
+			if !local {
+				objGen.allowList = append(objGen.allowList, n.Obj().Name())
+				g.log.Debug(context.Background(), "found external type",
+					"name", objName,
+					"ext_pkg", objGen.pkg.String(),
+				)
+			}
 			// Sweet! Using other typescript types as fields. This could be an
 			// enum or another struct
 			if args := n.TypeArgs(); args != nil && args.Len() > 0 {
@@ -817,10 +931,13 @@ func (g *Generator) typescriptType(ty types.Type) (TypescriptType, error) {
 				genericName = objName + fmt.Sprintf("<%s>", strings.Join(genericNames, ", "))
 				objName += fmt.Sprintf("<%s>", strings.Join(genericConstraints, ", "))
 			}
+
+			cmt := ""
 			return TypescriptType{
-				GenericTypes: genericTypes,
-				GenericValue: genericName,
-				ValueType:    objName,
+				GenericTypes:  genericTypes,
+				GenericValue:  genericName,
+				ValueType:     objName,
+				AboveTypeLine: cmt,
 			}, nil
 		}

@@ -842,7 +959,10 @@ func (g *Generator) typescriptType(ty types.Type) (TypescriptType, error) {
 		if err != nil {
 			return TypescriptType{}, xerrors.Errorf("named underlying: %w", err)
 		}
-		ts.AboveTypeLine = indentedComment(fmt.Sprintf("This is likely an enum in an external package (%q)", n.String()))
+		if ts.AboveTypeLine == "" {
+			// If no comment exists explaining where this type comes from, add one.
+			ts.AboveTypeLine = indentedComment(fmt.Sprintf("This is likely an enum in an external package (%q)", n.String()))
+		}
 		return ts, nil
 	case *types.Pointer:
 		// Dereference pointers.
@@ -868,6 +988,20 @@ func (g *Generator) typescriptType(ty types.Type) (TypescriptType, error) {
 				),
 			}, nil
 		}
+
+		// Do support "Stringer" interfaces, they likely can get string
+		// marshalled.
+		for i := 0; i < intf.NumMethods(); i++ {
+			meth := intf.Method(i)
+			if meth.Name() == "String" {
+				return TypescriptType{
+					ValueType:     "string",
+					AboveTypeLine: indentedComment("actual value is an interface that implements 'String()'"),
+					Optional:      false,
+				}, nil
+			}
+		}
+
 		// All complex interfaces should be named. So if we get here, that means
 		// we are using anonymous interfaces. Which is just weird and not supported.
 		// Example:
@@ -928,6 +1062,22 @@ func (g *Generator) typescriptType(ty types.Type) (TypescriptType, error) {
 	return TypescriptType{}, xerrors.Errorf("unknown type: %s", ty.String())
 }

+func (g *Generator) lookupNamedReference(n *types.Named) (obj types.Object, generator *Generator, local bool) {
+	pkgName := n.Obj().Pkg().Name()
+
+	if obj := g.pkg.Types.Scope().Lookup(n.Obj().Name()); g.pkg.Name == pkgName && obj != nil {
+		return obj, g, true
+	}
+
+	for _, ext := range g.externals {
+		if obj := ext.pkg.Types.Scope().Lookup(n.Obj().Name()); ext.pkg.Name == pkgName && obj != nil {
+			return obj, ext, false
+		}
+	}
+
+	return nil, nil, false
+}
+
 // isBuiltIn returns the string for a builtin type that we want to support
 // if the name is a reserved builtin type. This is for types like 'comparable'.
 // These types are not implemented in golang, so we just have to hardcode it.