Embed webassets natively into teleport instead of attaching to the binary (#5935)

Switch to go1.16. Use embed package to embed webassets instead of ad-hoc attaching to binary

.drone.yml

@@ -10,7 +10,7 @@ type: kubernetes
 name: test
 environment:
   GID: "1000"
-  RUNTIME: go1.15.5
+  RUNTIME: go1.16.2
   UID: "1000"
 trigger:
   event:
@@ -333,7 +333,7 @@ type: kubernetes
 name: push-build-linux-amd64
 environment:
   GID: "1000"
-  RUNTIME: go1.15.5
+  RUNTIME: go1.16.2
   UID: "1000"
 trigger:
   event:
@@ -427,7 +427,7 @@ type: kubernetes
 name: push-build-linux-386
 environment:
   GID: "1000"
-  RUNTIME: go1.15.5
+  RUNTIME: go1.16.2
   UID: "1000"
 trigger:
   event:
@@ -521,7 +521,7 @@ type: kubernetes
 name: push-build-linux-amd64-fips
 environment:
   GID: "1000"
-  RUNTIME: go1.15.5
+  RUNTIME: go1.16.2
   UID: "1000"
 trigger:
   event:
@@ -619,7 +619,7 @@ type: kubernetes
 name: push-build-windows-amd64
 environment:
   GID: "1000"
-  RUNTIME: go1.15.5
+  RUNTIME: go1.16.2
   UID: "1000"
 trigger:
   event:
@@ -810,7 +810,7 @@ type: kubernetes
 name: push-build-linux-arm
 environment:
   GID: "1000"
-  RUNTIME: go1.15.5
+  RUNTIME: go1.16.2
   UID: "1000"
 trigger:
   event:
@@ -904,7 +904,7 @@ type: kubernetes
 name: push-build-linux-arm64
 environment:
   GID: "1000"
-  RUNTIME: go1.15.5
+  RUNTIME: go1.16.2
   UID: "1000"
 trigger:
   event:
@@ -998,7 +998,7 @@ type: kubernetes
 name: push-build-linux-arm64-fips
 environment:
   GID: "1000"
-  RUNTIME: go1.15.5
+  RUNTIME: go1.16.2
   UID: "1000"
 trigger:
   event:
@@ -1326,7 +1326,7 @@ kind: pipeline
 type: kubernetes
 name: build-linux-amd64
 environment:
-  RUNTIME: go1.15.5
+  RUNTIME: go1.16.2
 trigger:
   event:
     include:
@@ -1422,7 +1422,7 @@ kind: pipeline
 type: kubernetes
 name: build-linux-amd64-fips
 environment:
-  RUNTIME: go1.15.5
+  RUNTIME: go1.16.2
 trigger:
   event:
     include:
@@ -1518,7 +1518,7 @@ kind: pipeline
 type: kubernetes
 name: build-linux-amd64-centos6
 environment:
-  RUNTIME: go1.15.5
+  RUNTIME: go1.16.2
 trigger:
   event:
     include:
@@ -1617,7 +1617,7 @@ kind: pipeline
 type: kubernetes
 name: build-linux-amd64-centos6-fips
 environment:
-  RUNTIME: go1.15.5
+  RUNTIME: go1.16.2
 trigger:
   event:
     include:
@@ -2181,7 +2181,7 @@ kind: pipeline
 type: kubernetes
 name: build-linux-386
 environment:
-  RUNTIME: go1.15.5
+  RUNTIME: go1.16.2
 trigger:
   event:
     include:
@@ -2899,7 +2899,7 @@ kind: pipeline
 type: kubernetes
 name: build-linux-arm
 environment:
-  RUNTIME: go1.15.5
+  RUNTIME: go1.16.2
 trigger:
   event:
     include:
@@ -2995,7 +2995,7 @@ kind: pipeline
 type: kubernetes
 name: build-linux-arm64
 environment:
-  RUNTIME: go1.15.5
+  RUNTIME: go1.16.2
 trigger:
   event:
     include:
@@ -3091,7 +3091,7 @@ kind: pipeline
 type: kubernetes
 name: build-linux-arm64-fips
 environment:
-  RUNTIME: go1.15.5
+  RUNTIME: go1.16.2
 trigger:
   event:
     include:
@@ -3889,7 +3889,7 @@ kind: pipeline
 type: kubernetes
 name: build-windows-amd64
 environment:
-  RUNTIME: go1.15.5
+  RUNTIME: go1.16.2
 trigger:
   event:
     include:
@@ -4742,6 +4742,6 @@ volumes:
       name: drone-s3-debrepo-pvc
 ---
 kind: signature
-hmac: cffe37fc9bdcc947e927875a202940b47559076b0f67333295813e7bdb8ff422
+hmac: 1ca2684a9adf84751a36a3c891b01e82011050bc1b6d284ef92f33acddbef71d
 
 ...

.gitignore

@@ -5,6 +5,9 @@ debug
 default.etcd
 darwin
 
+# web asset directory
+/lib/web/build
+
 # usually release tarballs get in the way
 *.gz
 *.zip

Makefile

@@ -18,6 +18,7 @@ DOCKER_IMAGE_CI ?= quay.io/gravitational/teleport-ci
 
 # These are standard autotools variables, don't change them please
 BUILDDIR ?= build
+ASSETS_BUILDDIR ?= lib/web/build
 BINDIR ?= /usr/local/bin
 DATADIR ?= /usr/local/share/teleport
 ADDFLAGS ?=
@@ -120,7 +121,7 @@ $(BUILDDIR)/tctl:
 
 .PHONY: $(BUILDDIR)/teleport
 $(BUILDDIR)/teleport: ensure-webassets
-	GOOS=$(OS) GOARCH=$(ARCH) $(CGOFLAG) go build -tags "$(PAM_TAG) $(FIPS_TAG) $(BPF_TAG)" -o $(BUILDDIR)/teleport $(BUILDFLAGS) ./tool/teleport
+	GOOS=$(OS) GOARCH=$(ARCH) $(CGOFLAG) go build -tags "$(PAM_TAG) $(FIPS_TAG) $(BPF_TAG) $(WEBASSETS_TAG)" -o $(BUILDDIR)/teleport $(BUILDFLAGS) ./tool/teleport
 
 .PHONY: $(BUILDDIR)/tsh
 $(BUILDDIR)/tsh:
@@ -132,12 +133,9 @@ $(BUILDDIR)/tsh:
 # only tsh is built.
 #
 .PHONY:full
-full: all $(BUILDDIR)/webassets.zip
+full: $(ASSETS_BUILDDIR)/webassets.zip
 ifneq ("$(OS)", "windows")
-	@echo "---> Attaching OSS web assets."
-	cat $(BUILDDIR)/webassets.zip >> $(BUILDDIR)/teleport
-	rm -fr $(BUILDDIR)/webassets.zip
-	zip -q -A $(BUILDDIR)/teleport
+	$(MAKE) all WEBASSETS_TAG="webassets_embed"
 endif
 
 #
@@ -211,7 +209,10 @@ release-unix: clean full
 	tar -czf $(RELEASE).tar.gz teleport
 	rm -rf teleport
 	@echo "---> Created $(RELEASE).tar.gz."
-	@if [ -f e/Makefile ]; then $(MAKE) -C e release; fi
+	@if [ -f e/Makefile ]; then \
+		rm -fr $(WEBASSETS_BUILDDIR)/webassets.zip; \
+		$(MAKE) -C e release; \
+	fi
 
 #
 # make release-windows - Produces a binary release tarball containing teleport,
@@ -372,12 +373,15 @@ tag:
 
 # build/webassets.zip archive contains the web assets (UI) which gets
 # appended to teleport binary
-$(BUILDDIR)/webassets.zip:
+$(ASSETS_BUILDDIR)/webassets.zip: | $(ASSETS_BUILDDIR)
 ifneq ("$(OS)", "windows")
 	@echo "---> Building OSS web assets."
-	cd webassets/teleport/ ; zip -qr ../../$(BUILDDIR)/webassets.zip .
+	cd webassets/teleport/ ; zip -qr ../../$@ .
 endif
 
+$(ASSETS_BUILDDIR):
+	mkdir -p $@
+
 .PHONY: test-package
 test-package: remove-temp-files
 	go test -v ./$(p)

build.assets/Dockerfile

@@ -97,7 +97,7 @@ ENV GOGOPROTO_ROOT ${GOPATH}/src/github.com/gogo/protobuf
 RUN (curl -L -o /tmp/${PROTOC_TARBALL} https://github.com/google/protobuf/releases/download/v${PROTOC_VER}/${PROTOC_TARBALL} && \
      cd /tmp && unzip /tmp/${PROTOC_TARBALL} -d /usr/local && \
      rm /tmp/${PROTOC_TARBALL})
-RUN (go get -u github.com/gogo/protobuf/proto github.com/gogo/protobuf/protoc-gen-gogo github.com/gogo/protobuf/gogoproto golang.org/x/tools/cmd/goimports && \
+RUN (git clone https://github.com/gogo/protobuf.git ${GOPATH}/src/github.com/gogo/protobuf && go install golang.org/x/tools/cmd/goimports@latest && \
      cd ${GOPATH}/src/github.com/gogo/protobuf && \
      git reset --hard ${GOGO_PROTO_TAG} && \
      make install)

build.assets/Makefile

@@ -13,7 +13,7 @@ TEST_KUBE ?=
 
 OS ?= linux
 ARCH ?= amd64
-RUNTIME ?= go1.15.5
+RUNTIME ?= go1.16.2
 
 UID := $$(id -u)
 GID := $$(id -g)

dronegen/common.go

@@ -55,7 +55,7 @@ var (
 	// TODO(gus): Set this from `make -C build.assets print-runtime-version` or similar rather
 	// than hardcoding it. Also remove the usage of RUNTIME as a pipeline-level environment variable
 	// (as support for these varies among Drone runners) and only set it for steps that need it.
-	goRuntime = value{raw: "go1.15.5"}
+	goRuntime = value{raw: "go1.16.2"}
 )
 
 type buildType struct {

go.mod

@@ -1,6 +1,6 @@
 module github.com/gravitational/teleport
 
-go 1.15
+go 1.16
 
 require (
 	cloud.google.com/go v0.60.0

lib/service/service.go

@@ -3409,8 +3409,8 @@ func findPublicAddr(authClient auth.AccessPoint, a App) (string, error) {
 // It uses external configuration to make the decision
 func newHTTPFileSystem() (http.FileSystem, error) {
 	if !isDebugMode() {
-		fs, err := web.NewStaticFileSystem()
-		if err != nil {
+		fs, err := web.NewStaticFileSystem() //nolint:staticcheck
+		if err != nil {                      //nolint:staticcheck
 			return nil, trace.Wrap(err)
 		}
 		return fs, nil

lib/web/assets.go

@@ -0,0 +1,169 @@
+/*
+Copyright 2021 Gravitational, Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package web
+
+import (
+	"archive/zip"
+	"io"
+	"net/http"
+	"os"
+	"path"
+	"path/filepath"
+	"strings"
+
+	"github.com/gravitational/trace"
+
+	"github.com/kardianos/osext"
+)
+
+// NewDebugFileSystem returns the HTTP file system implementation rooted
+// at the specified assetsPath.
+func NewDebugFileSystem(assetsPath string) (http.FileSystem, error) {
+	assetsToCheck := []string{"index.html", "/app"}
+	if assetsPath == "" {
+		exePath, err := osext.ExecutableFolder()
+		if err != nil {
+			return nil, trace.Wrap(err)
+		}
+
+		_, err = os.Stat(path.Join(exePath, "../../e"))
+		isEnterprise := !os.IsNotExist(err)
+
+		if isEnterprise {
+			// enterprise web assets
+			assetsPath = path.Join(exePath, "../../webassets/e/teleport")
+		} else {
+			// community web assets
+			assetsPath = path.Join(exePath, "../webassets/teleport")
+		}
+	}
+
+	for _, af := range assetsToCheck {
+		_, err := os.Stat(filepath.Join(assetsPath, af))
+		if err != nil {
+			return nil, trace.Wrap(err)
+		}
+	}
+	log.Infof("Using filesystem for serving web assets: %s.", assetsPath)
+	return http.Dir(assetsPath), nil
+}
+
+const (
+	webAssetsMissingError = "the teleport binary was built without web assets, try building with `make release`"
+	webAssetsReadError    = "failure reading web assets from the binary"
+)
+
+func readZipArchive(r io.ReaderAt, size int64) (ResourceMap, error) {
+	zreader, err := zip.NewReader(r, size)
+	if err != nil {
+		// this often happens when teleport is launched without the web assets
+		// zip file attached to the binary. for launching it in such mode
+		// set DEBUG environment variable to 1
+		if err == zip.ErrFormat {
+			return nil, trace.NotFound(webAssetsMissingError)
+		}
+		return nil, trace.NotFound("%s %v", webAssetsReadError, err)
+	}
+	entries := make(ResourceMap)
+	for _, file := range zreader.File {
+		if file.FileInfo().IsDir() {
+			continue
+		}
+		entries[file.Name] = file
+	}
+	// no entries found?
+	if len(entries) == 0 {
+		return nil, trace.Wrap(os.ErrInvalid)
+	}
+	return entries, nil
+}
+
+// resource struct implements http.File interface on top of zip.File object
+type resource struct {
+	reader io.ReadCloser
+	file   *zip.File
+	pos    int64
+}
+
+func (rsc *resource) Read(p []byte) (n int, err error) {
+	n, err = rsc.reader.Read(p)
+	rsc.pos += int64(n)
+	return n, err
+}
+
+func (rsc *resource) Seek(offset int64, whence int) (int64, error) {
+	var (
+		pos int64
+		err error
+	)
+	// zip.File does not support seeking. To implement Seek on top of it,
+	// we close the existing reader, re-open it, and read 'offset' bytes from
+	// the beginning
+	if err = rsc.reader.Close(); err != nil {
+		return 0, err
+	}
+	if rsc.reader, err = rsc.file.Open(); err != nil {
+		return 0, err
+	}
+	switch whence {
+	case io.SeekStart:
+		pos = offset
+	case io.SeekCurrent:
+		pos = rsc.pos + offset
+	case io.SeekEnd:
+		pos = int64(rsc.file.UncompressedSize64) + offset
+	}
+	if pos > 0 {
+		b := make([]byte, pos)
+		if _, err = rsc.reader.Read(b); err != nil {
+			return 0, err
+		}
+	}
+	rsc.pos = pos
+	return pos, nil
+}
+
+func (rsc *resource) Readdir(count int) ([]os.FileInfo, error) {
+	return nil, trace.Wrap(os.ErrPermission)
+}
+
+func (rsc *resource) Stat() (os.FileInfo, error) {
+	return rsc.file.FileInfo(), nil
+}
+
+func (rsc *resource) Close() (err error) {
+	log.Debugf("zip::Close(%s).", rsc.file.FileInfo().Name())
+	return rsc.reader.Close()
+}
+
+type ResourceMap map[string]*zip.File
+
+func (rm ResourceMap) Open(name string) (http.File, error) {
+	log.Debugf("GET zip:%s.", name)
+	f, ok := rm[strings.Trim(name, "/")]
+	if !ok {
+		return nil, trace.Wrap(os.ErrNotExist)
+	}
+	reader, err := f.Open()
+	if err != nil {
+		return nil, trace.Wrap(err)
+	}
+	return &resource{
+		reader: reader,
+		file:   f,
+	}, nil
+}

lib/web/static.go

@@ -1,5 +1,7 @@
+// +build !webassets_embed
+
 /*
-Copyright 2015 Gravitational, Inc.
+Copyright 2021 Gravitational, Inc.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -12,192 +14,17 @@ distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
-
 */
 
 package web
 
 import (
-	"archive/zip"
-	"io"
 	"net/http"
-	"os"
-	"path"
-	"path/filepath"
-	"strings"
 
 	"github.com/gravitational/trace"
-
-	"github.com/kardianos/osext"
 )
 
-const (
-	webAssetsMissingError = "the teleport binary was built without web assets, try building with `make release`"
-	webAssetsReadError    = "failure reading web assets from the binary"
-)
-
-// NewStaticFileSystem returns the initialized implementation of http.FileSystem
-// interface which can be used to serve Teleport Proxy Web UI
-func NewStaticFileSystem() (http.FileSystem, error) {
-	// Use the zip archive attached to the executable:
-	return loadZippedExeAssets()
-}
-
-// NewDebugFileSystem returns the HTTP file system implementation rooted
-// at the specified assetsPath.
-func NewDebugFileSystem(assetsPath string) (http.FileSystem, error) {
-	assetsToCheck := []string{"index.html", "/app"}
-	if assetsPath == "" {
-		exePath, err := osext.ExecutableFolder()
-		if err != nil {
-			return nil, trace.Wrap(err)
-		}
-
-		_, err = os.Stat(path.Join(exePath, "../../e"))
-		isEnterprise := !os.IsNotExist(err)
-
-		if isEnterprise {
-			// enterprise web assets
-			assetsPath = path.Join(exePath, "../../webassets/e/teleport")
-		} else {
-			// community web assets
-			assetsPath = path.Join(exePath, "../webassets/teleport")
-		}
-	}
-
-	for _, af := range assetsToCheck {
-		_, err := os.Stat(filepath.Join(assetsPath, af))
-		if err != nil {
-			return nil, trace.Wrap(err)
-		}
-	}
-	log.Infof("Using filesystem for serving web assets: %s.", assetsPath)
-	return http.Dir(assetsPath), nil
-}
-
-// LoadWebResources returns a filesystem implementation compatible
-// with http.Serve.
-//
-// The "filesystem" is served from a zip file attached at the end of
-// the executable
-//
-func loadZippedExeAssets() (ResourceMap, error) {
-	// open ourselves (teleport binary) for reading:
-	// NOTE: the file stays open to serve future Read() requests
-	myExe, err := osext.Executable()
-	if err != nil {
-		return nil, trace.Wrap(err)
-	}
-	return readZipArchive(myExe)
-}
-
-func readZipArchive(archivePath string) (ResourceMap, error) {
-	file, err := os.Open(archivePath)
-	if err != nil {
-		return nil, trace.Wrap(err)
-	}
-	// feed the binary into the zip reader and enumerate all files
-	// found in the attached zip file:
-	info, err := file.Stat()
-	if err != nil {
-		return nil, trace.Wrap(err)
-	}
-	zreader, err := zip.NewReader(file, info.Size())
-	if err != nil {
-		// this often happens when teleport is launched without the web assets
-		// zip file attached to the binary. for launching it in such mode
-		// set DEBUG environment variable to 1
-		if err == zip.ErrFormat {
-			return nil, trace.NotFound(webAssetsMissingError)
-		}
-		return nil, trace.NotFound("%s %v", webAssetsReadError, err)
-	}
-	entries := make(ResourceMap)
-	for _, file := range zreader.File {
-		if file.FileInfo().IsDir() {
-			continue
-		}
-		entries[file.Name] = file
-	}
-	// no entries found?
-	if len(entries) == 0 {
-		return nil, trace.Wrap(os.ErrInvalid)
-	}
-	return entries, nil
-}
-
-// resource struct implements http.File interface on top of zip.File object
-type resource struct {
-	reader io.ReadCloser
-	file   *zip.File
-	pos    int64
-}
-
-func (rsc *resource) Read(p []byte) (n int, err error) {
-	n, err = rsc.reader.Read(p)
-	rsc.pos += int64(n)
-	return n, err
-}
-
-func (rsc *resource) Seek(offset int64, whence int) (int64, error) {
-	var (
-		pos int64
-		err error
-	)
-	// zip.File does not support seeking. To implement Seek on top of it,
-	// we close the existing reader, re-open it, and read 'offset' bytes from
-	// the beginning
-	if err = rsc.reader.Close(); err != nil {
-		return 0, err
-	}
-	if rsc.reader, err = rsc.file.Open(); err != nil {
-		return 0, err
-	}
-	switch whence {
-	case io.SeekStart:
-		pos = offset
-	case io.SeekCurrent:
-		pos = rsc.pos + offset
-	case io.SeekEnd:
-		pos = int64(rsc.file.UncompressedSize64) + offset
-	}
-	if pos > 0 {
-		b := make([]byte, pos)
-		if _, err = rsc.reader.Read(b); err != nil {
-			return 0, err
-		}
-	}
-	rsc.pos = pos
-	return pos, nil
-}
-
-func (rsc *resource) Readdir(count int) ([]os.FileInfo, error) {
-	return nil, trace.Wrap(os.ErrPermission)
-}
-
-func (rsc *resource) Stat() (os.FileInfo, error) {
-	return rsc.file.FileInfo(), nil
-}
-
-func (rsc *resource) Close() (err error) {
-	log.Debugf("zip::Close(%s).", rsc.file.FileInfo().Name())
-	return rsc.reader.Close()
-}
-
-type ResourceMap map[string]*zip.File
-
-func (rm ResourceMap) Open(name string) (http.File, error) {
-	log.Debugf("GET zip:%s.", name)
-	f, ok := rm[strings.Trim(name, "/")]
-	if !ok {
-		return nil, trace.Wrap(os.ErrNotExist)
-	}
-	reader, err := f.Open()
-	if err != nil {
-		return nil, trace.Wrap(err)
-	}
-	return &resource{
-		reader: reader,
-		file:   f,
-	}, nil
+// NewStaticFileSystem is a no-op in this build mode.
+func NewStaticFileSystem() (http.FileSystem, error) { //nolint:staticcheck
+	return nil, trace.NotFound(webAssetsMissingError)
 }

lib/web/static_embed.go

@@ -0,0 +1,34 @@
+// +build webassets_embed
+
+/*
+Copyright 2021 Gravitational, Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package web
+
+import (
+	"bytes"
+	_ "embed"
+	"net/http"
+)
+
+//go:embed build/webassets.zip
+var webassetsZip []byte
+
+// NewStaticFileSystem returns the initialized implementation of http.FileSystem
+// interface which can be used to serve Teleport Proxy Web UI
+func NewStaticFileSystem() (http.FileSystem, error) {
+	return readZipArchive(bytes.NewReader(webassetsZip), int64(len(webassetsZip)))
+}

lib/web/static_test.go

@@ -19,8 +19,10 @@ package web
 import (
 	"io"
 	"io/ioutil"
+	"os"
 	"strings"
 
+	"github.com/gravitational/trace"
 	"gopkg.in/check.v1"
 )
 
@@ -46,7 +48,7 @@ func (s *StaticSuite) TestLocalFS(c *check.C) {
 }
 
 func (s *StaticSuite) TestZipFS(c *check.C) {
-	fs, err := readZipArchive("../../fixtures/assets.zip")
+	fs, err := readZipArchiveAt("../../fixtures/assets.zip")
 	c.Assert(err, check.IsNil)
 	c.Assert(fs, check.NotNil)
 
@@ -87,3 +89,19 @@ func (s *StaticSuite) TestZipFS(c *check.C) {
 	c.Assert(err, check.IsNil)
 	c.Assert(len(bytes), check.Equals, 100)
 }
+
+func readZipArchiveAt(path string) (ResourceMap, error) {
+	file, err := os.Open(path)
+	if err != nil {
+		return nil, trace.Wrap(err)
+	}
+	// file needs to stay open for http.FileSystem reads to work
+	//
+	// feed the binary into the zip reader and enumerate all files
+	// found in the attached zip file:
+	info, err := file.Stat()
+	if err != nil {
+		return nil, trace.Wrap(err)
+	}
+	return readZipArchive(file, info.Size())
+}