diff --git a/.drone.yml b/.drone.yml index d06838fcd0a..cee9a30d34f 100644 --- a/.drone.yml +++ b/.drone.yml @@ -10,7 +10,7 @@ type: kubernetes name: test environment: GID: "1000" - RUNTIME: go1.15.5 + RUNTIME: go1.16.2 UID: "1000" trigger: event: @@ -333,7 +333,7 @@ type: kubernetes name: push-build-linux-amd64 environment: GID: "1000" - RUNTIME: go1.15.5 + RUNTIME: go1.16.2 UID: "1000" trigger: event: @@ -427,7 +427,7 @@ type: kubernetes name: push-build-linux-386 environment: GID: "1000" - RUNTIME: go1.15.5 + RUNTIME: go1.16.2 UID: "1000" trigger: event: @@ -521,7 +521,7 @@ type: kubernetes name: push-build-linux-amd64-fips environment: GID: "1000" - RUNTIME: go1.15.5 + RUNTIME: go1.16.2 UID: "1000" trigger: event: @@ -619,7 +619,7 @@ type: kubernetes name: push-build-windows-amd64 environment: GID: "1000" - RUNTIME: go1.15.5 + RUNTIME: go1.16.2 UID: "1000" trigger: event: @@ -810,7 +810,7 @@ type: kubernetes name: push-build-linux-arm environment: GID: "1000" - RUNTIME: go1.15.5 + RUNTIME: go1.16.2 UID: "1000" trigger: event: @@ -904,7 +904,7 @@ type: kubernetes name: push-build-linux-arm64 environment: GID: "1000" - RUNTIME: go1.15.5 + RUNTIME: go1.16.2 UID: "1000" trigger: event: @@ -998,7 +998,7 @@ type: kubernetes name: push-build-linux-arm64-fips environment: GID: "1000" - RUNTIME: go1.15.5 + RUNTIME: go1.16.2 UID: "1000" trigger: event: @@ -1326,7 +1326,7 @@ kind: pipeline type: kubernetes name: build-linux-amd64 environment: - RUNTIME: go1.15.5 + RUNTIME: go1.16.2 trigger: event: include: @@ -1422,7 +1422,7 @@ kind: pipeline type: kubernetes name: build-linux-amd64-fips environment: - RUNTIME: go1.15.5 + RUNTIME: go1.16.2 trigger: event: include: @@ -1518,7 +1518,7 @@ kind: pipeline type: kubernetes name: build-linux-amd64-centos6 environment: - RUNTIME: go1.15.5 + RUNTIME: go1.16.2 trigger: event: include: @@ -1617,7 +1617,7 @@ kind: pipeline type: kubernetes name: build-linux-amd64-centos6-fips environment: - RUNTIME: go1.15.5 + RUNTIME: go1.16.2 trigger: event: include: @@ -2181,7 +2181,7 @@ kind: pipeline type: kubernetes name: build-linux-386 environment: - RUNTIME: go1.15.5 + RUNTIME: go1.16.2 trigger: event: include: @@ -2899,7 +2899,7 @@ kind: pipeline type: kubernetes name: build-linux-arm environment: - RUNTIME: go1.15.5 + RUNTIME: go1.16.2 trigger: event: include: @@ -2995,7 +2995,7 @@ kind: pipeline type: kubernetes name: build-linux-arm64 environment: - RUNTIME: go1.15.5 + RUNTIME: go1.16.2 trigger: event: include: @@ -3091,7 +3091,7 @@ kind: pipeline type: kubernetes name: build-linux-arm64-fips environment: - RUNTIME: go1.15.5 + RUNTIME: go1.16.2 trigger: event: include: @@ -3889,7 +3889,7 @@ kind: pipeline type: kubernetes name: build-windows-amd64 environment: - RUNTIME: go1.15.5 + RUNTIME: go1.16.2 trigger: event: include: @@ -4742,6 +4742,6 @@ volumes: name: drone-s3-debrepo-pvc --- kind: signature -hmac: cffe37fc9bdcc947e927875a202940b47559076b0f67333295813e7bdb8ff422 +hmac: 1ca2684a9adf84751a36a3c891b01e82011050bc1b6d284ef92f33acddbef71d ... diff --git a/.gitignore b/.gitignore index d4de8b44d9b..d77f26ca6e5 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,9 @@ debug default.etcd darwin +# web asset directory +/lib/web/build + # usually release tarballs get in the way *.gz *.zip diff --git a/Makefile b/Makefile index 903d4757f44..790f68039ca 100644 --- a/Makefile +++ b/Makefile @@ -18,6 +18,7 @@ DOCKER_IMAGE_CI ?= quay.io/gravitational/teleport-ci # These are standard autotools variables, don't change them please BUILDDIR ?= build +ASSETS_BUILDDIR ?= lib/web/build BINDIR ?= /usr/local/bin DATADIR ?= /usr/local/share/teleport ADDFLAGS ?= @@ -120,7 +121,7 @@ $(BUILDDIR)/tctl: .PHONY: $(BUILDDIR)/teleport $(BUILDDIR)/teleport: ensure-webassets - GOOS=$(OS) GOARCH=$(ARCH) $(CGOFLAG) go build -tags "$(PAM_TAG) $(FIPS_TAG) $(BPF_TAG)" -o $(BUILDDIR)/teleport $(BUILDFLAGS) ./tool/teleport + GOOS=$(OS) GOARCH=$(ARCH) $(CGOFLAG) go build -tags "$(PAM_TAG) $(FIPS_TAG) $(BPF_TAG) $(WEBASSETS_TAG)" -o $(BUILDDIR)/teleport $(BUILDFLAGS) ./tool/teleport .PHONY: $(BUILDDIR)/tsh $(BUILDDIR)/tsh: @@ -132,12 +133,9 @@ $(BUILDDIR)/tsh: # only tsh is built. # .PHONY:full -full: all $(BUILDDIR)/webassets.zip +full: $(ASSETS_BUILDDIR)/webassets.zip ifneq ("$(OS)", "windows") - @echo "---> Attaching OSS web assets." - cat $(BUILDDIR)/webassets.zip >> $(BUILDDIR)/teleport - rm -fr $(BUILDDIR)/webassets.zip - zip -q -A $(BUILDDIR)/teleport + $(MAKE) all WEBASSETS_TAG="webassets_embed" endif # @@ -211,7 +209,10 @@ release-unix: clean full tar -czf $(RELEASE).tar.gz teleport rm -rf teleport @echo "---> Created $(RELEASE).tar.gz." - @if [ -f e/Makefile ]; then $(MAKE) -C e release; fi + @if [ -f e/Makefile ]; then \ + rm -fr $(WEBASSETS_BUILDDIR)/webassets.zip; \ + $(MAKE) -C e release; \ + fi # # make release-windows - Produces a binary release tarball containing teleport, @@ -372,12 +373,15 @@ tag: # build/webassets.zip archive contains the web assets (UI) which gets # appended to teleport binary -$(BUILDDIR)/webassets.zip: +$(ASSETS_BUILDDIR)/webassets.zip: | $(ASSETS_BUILDDIR) ifneq ("$(OS)", "windows") @echo "---> Building OSS web assets." - cd webassets/teleport/ ; zip -qr ../../$(BUILDDIR)/webassets.zip . + cd webassets/teleport/ ; zip -qr ../../$@ . endif +$(ASSETS_BUILDDIR): + mkdir -p $@ + .PHONY: test-package test-package: remove-temp-files go test -v ./$(p) diff --git a/build.assets/Dockerfile b/build.assets/Dockerfile index 6f3e6d8d121..d416ff89446 100644 --- a/build.assets/Dockerfile +++ b/build.assets/Dockerfile @@ -97,7 +97,7 @@ ENV GOGOPROTO_ROOT ${GOPATH}/src/github.com/gogo/protobuf RUN (curl -L -o /tmp/${PROTOC_TARBALL} https://github.com/google/protobuf/releases/download/v${PROTOC_VER}/${PROTOC_TARBALL} && \ cd /tmp && unzip /tmp/${PROTOC_TARBALL} -d /usr/local && \ rm /tmp/${PROTOC_TARBALL}) -RUN (go get -u github.com/gogo/protobuf/proto github.com/gogo/protobuf/protoc-gen-gogo github.com/gogo/protobuf/gogoproto golang.org/x/tools/cmd/goimports && \ +RUN (git clone https://github.com/gogo/protobuf.git ${GOPATH}/src/github.com/gogo/protobuf && go install golang.org/x/tools/cmd/goimports@latest && \ cd ${GOPATH}/src/github.com/gogo/protobuf && \ git reset --hard ${GOGO_PROTO_TAG} && \ make install) diff --git a/build.assets/Makefile b/build.assets/Makefile index 31b7383d352..9f74ccd0810 100644 --- a/build.assets/Makefile +++ b/build.assets/Makefile @@ -13,7 +13,7 @@ TEST_KUBE ?= OS ?= linux ARCH ?= amd64 -RUNTIME ?= go1.15.5 +RUNTIME ?= go1.16.2 UID := $$(id -u) GID := $$(id -g) diff --git a/dronegen/common.go b/dronegen/common.go index b10488eee86..d16b7bf5a1a 100644 --- a/dronegen/common.go +++ b/dronegen/common.go @@ -55,7 +55,7 @@ var ( // TODO(gus): Set this from `make -C build.assets print-runtime-version` or similar rather // than hardcoding it. Also remove the usage of RUNTIME as a pipeline-level environment variable // (as support for these varies among Drone runners) and only set it for steps that need it. - goRuntime = value{raw: "go1.15.5"} + goRuntime = value{raw: "go1.16.2"} ) type buildType struct { diff --git a/e b/e index 248c7900d79..03805e26e5c 160000 --- a/e +++ b/e @@ -1 +1 @@ -Subproject commit 248c7900d79241db91786a13cda5706816bbe18b +Subproject commit 03805e26e5c54eeb146a688bc09f8957a216fb81 diff --git a/go.mod b/go.mod index b583eb1f02e..d32ef26a14b 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/gravitational/teleport -go 1.15 +go 1.16 require ( cloud.google.com/go v0.60.0 diff --git a/lib/service/service.go b/lib/service/service.go index 960ea489a4f..c896ed5ccc8 100644 --- a/lib/service/service.go +++ b/lib/service/service.go @@ -3409,8 +3409,8 @@ func findPublicAddr(authClient auth.AccessPoint, a App) (string, error) { // It uses external configuration to make the decision func newHTTPFileSystem() (http.FileSystem, error) { if !isDebugMode() { - fs, err := web.NewStaticFileSystem() - if err != nil { + fs, err := web.NewStaticFileSystem() //nolint:staticcheck + if err != nil { //nolint:staticcheck return nil, trace.Wrap(err) } return fs, nil diff --git a/lib/web/assets.go b/lib/web/assets.go new file mode 100644 index 00000000000..a0206249860 --- /dev/null +++ b/lib/web/assets.go @@ -0,0 +1,169 @@ +/* +Copyright 2021 Gravitational, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package web + +import ( + "archive/zip" + "io" + "net/http" + "os" + "path" + "path/filepath" + "strings" + + "github.com/gravitational/trace" + + "github.com/kardianos/osext" +) + +// NewDebugFileSystem returns the HTTP file system implementation rooted +// at the specified assetsPath. +func NewDebugFileSystem(assetsPath string) (http.FileSystem, error) { + assetsToCheck := []string{"index.html", "/app"} + if assetsPath == "" { + exePath, err := osext.ExecutableFolder() + if err != nil { + return nil, trace.Wrap(err) + } + + _, err = os.Stat(path.Join(exePath, "../../e")) + isEnterprise := !os.IsNotExist(err) + + if isEnterprise { + // enterprise web assets + assetsPath = path.Join(exePath, "../../webassets/e/teleport") + } else { + // community web assets + assetsPath = path.Join(exePath, "../webassets/teleport") + } + } + + for _, af := range assetsToCheck { + _, err := os.Stat(filepath.Join(assetsPath, af)) + if err != nil { + return nil, trace.Wrap(err) + } + } + log.Infof("Using filesystem for serving web assets: %s.", assetsPath) + return http.Dir(assetsPath), nil +} + +const ( + webAssetsMissingError = "the teleport binary was built without web assets, try building with `make release`" + webAssetsReadError = "failure reading web assets from the binary" +) + +func readZipArchive(r io.ReaderAt, size int64) (ResourceMap, error) { + zreader, err := zip.NewReader(r, size) + if err != nil { + // this often happens when teleport is launched without the web assets + // zip file attached to the binary. for launching it in such mode + // set DEBUG environment variable to 1 + if err == zip.ErrFormat { + return nil, trace.NotFound(webAssetsMissingError) + } + return nil, trace.NotFound("%s %v", webAssetsReadError, err) + } + entries := make(ResourceMap) + for _, file := range zreader.File { + if file.FileInfo().IsDir() { + continue + } + entries[file.Name] = file + } + // no entries found? + if len(entries) == 0 { + return nil, trace.Wrap(os.ErrInvalid) + } + return entries, nil +} + +// resource struct implements http.File interface on top of zip.File object +type resource struct { + reader io.ReadCloser + file *zip.File + pos int64 +} + +func (rsc *resource) Read(p []byte) (n int, err error) { + n, err = rsc.reader.Read(p) + rsc.pos += int64(n) + return n, err +} + +func (rsc *resource) Seek(offset int64, whence int) (int64, error) { + var ( + pos int64 + err error + ) + // zip.File does not support seeking. To implement Seek on top of it, + // we close the existing reader, re-open it, and read 'offset' bytes from + // the beginning + if err = rsc.reader.Close(); err != nil { + return 0, err + } + if rsc.reader, err = rsc.file.Open(); err != nil { + return 0, err + } + switch whence { + case io.SeekStart: + pos = offset + case io.SeekCurrent: + pos = rsc.pos + offset + case io.SeekEnd: + pos = int64(rsc.file.UncompressedSize64) + offset + } + if pos > 0 { + b := make([]byte, pos) + if _, err = rsc.reader.Read(b); err != nil { + return 0, err + } + } + rsc.pos = pos + return pos, nil +} + +func (rsc *resource) Readdir(count int) ([]os.FileInfo, error) { + return nil, trace.Wrap(os.ErrPermission) +} + +func (rsc *resource) Stat() (os.FileInfo, error) { + return rsc.file.FileInfo(), nil +} + +func (rsc *resource) Close() (err error) { + log.Debugf("zip::Close(%s).", rsc.file.FileInfo().Name()) + return rsc.reader.Close() +} + +type ResourceMap map[string]*zip.File + +func (rm ResourceMap) Open(name string) (http.File, error) { + log.Debugf("GET zip:%s.", name) + f, ok := rm[strings.Trim(name, "/")] + if !ok { + return nil, trace.Wrap(os.ErrNotExist) + } + reader, err := f.Open() + if err != nil { + return nil, trace.Wrap(err) + } + return &resource{ + reader: reader, + file: f, + }, nil +} diff --git a/lib/web/static.go b/lib/web/static.go index cdcd83f3fb9..4e9cd052bc1 100644 --- a/lib/web/static.go +++ b/lib/web/static.go @@ -1,5 +1,7 @@ +// +build !webassets_embed + /* -Copyright 2015 Gravitational, Inc. +Copyright 2021 Gravitational, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,192 +14,17 @@ distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - */ package web import ( - "archive/zip" - "io" "net/http" - "os" - "path" - "path/filepath" - "strings" "github.com/gravitational/trace" - - "github.com/kardianos/osext" ) -const ( - webAssetsMissingError = "the teleport binary was built without web assets, try building with `make release`" - webAssetsReadError = "failure reading web assets from the binary" -) - -// NewStaticFileSystem returns the initialized implementation of http.FileSystem -// interface which can be used to serve Teleport Proxy Web UI -func NewStaticFileSystem() (http.FileSystem, error) { - // Use the zip archive attached to the executable: - return loadZippedExeAssets() -} - -// NewDebugFileSystem returns the HTTP file system implementation rooted -// at the specified assetsPath. -func NewDebugFileSystem(assetsPath string) (http.FileSystem, error) { - assetsToCheck := []string{"index.html", "/app"} - if assetsPath == "" { - exePath, err := osext.ExecutableFolder() - if err != nil { - return nil, trace.Wrap(err) - } - - _, err = os.Stat(path.Join(exePath, "../../e")) - isEnterprise := !os.IsNotExist(err) - - if isEnterprise { - // enterprise web assets - assetsPath = path.Join(exePath, "../../webassets/e/teleport") - } else { - // community web assets - assetsPath = path.Join(exePath, "../webassets/teleport") - } - } - - for _, af := range assetsToCheck { - _, err := os.Stat(filepath.Join(assetsPath, af)) - if err != nil { - return nil, trace.Wrap(err) - } - } - log.Infof("Using filesystem for serving web assets: %s.", assetsPath) - return http.Dir(assetsPath), nil -} - -// LoadWebResources returns a filesystem implementation compatible -// with http.Serve. -// -// The "filesystem" is served from a zip file attached at the end of -// the executable -// -func loadZippedExeAssets() (ResourceMap, error) { - // open ourselves (teleport binary) for reading: - // NOTE: the file stays open to serve future Read() requests - myExe, err := osext.Executable() - if err != nil { - return nil, trace.Wrap(err) - } - return readZipArchive(myExe) -} - -func readZipArchive(archivePath string) (ResourceMap, error) { - file, err := os.Open(archivePath) - if err != nil { - return nil, trace.Wrap(err) - } - // feed the binary into the zip reader and enumerate all files - // found in the attached zip file: - info, err := file.Stat() - if err != nil { - return nil, trace.Wrap(err) - } - zreader, err := zip.NewReader(file, info.Size()) - if err != nil { - // this often happens when teleport is launched without the web assets - // zip file attached to the binary. for launching it in such mode - // set DEBUG environment variable to 1 - if err == zip.ErrFormat { - return nil, trace.NotFound(webAssetsMissingError) - } - return nil, trace.NotFound("%s %v", webAssetsReadError, err) - } - entries := make(ResourceMap) - for _, file := range zreader.File { - if file.FileInfo().IsDir() { - continue - } - entries[file.Name] = file - } - // no entries found? - if len(entries) == 0 { - return nil, trace.Wrap(os.ErrInvalid) - } - return entries, nil -} - -// resource struct implements http.File interface on top of zip.File object -type resource struct { - reader io.ReadCloser - file *zip.File - pos int64 -} - -func (rsc *resource) Read(p []byte) (n int, err error) { - n, err = rsc.reader.Read(p) - rsc.pos += int64(n) - return n, err -} - -func (rsc *resource) Seek(offset int64, whence int) (int64, error) { - var ( - pos int64 - err error - ) - // zip.File does not support seeking. To implement Seek on top of it, - // we close the existing reader, re-open it, and read 'offset' bytes from - // the beginning - if err = rsc.reader.Close(); err != nil { - return 0, err - } - if rsc.reader, err = rsc.file.Open(); err != nil { - return 0, err - } - switch whence { - case io.SeekStart: - pos = offset - case io.SeekCurrent: - pos = rsc.pos + offset - case io.SeekEnd: - pos = int64(rsc.file.UncompressedSize64) + offset - } - if pos > 0 { - b := make([]byte, pos) - if _, err = rsc.reader.Read(b); err != nil { - return 0, err - } - } - rsc.pos = pos - return pos, nil -} - -func (rsc *resource) Readdir(count int) ([]os.FileInfo, error) { - return nil, trace.Wrap(os.ErrPermission) -} - -func (rsc *resource) Stat() (os.FileInfo, error) { - return rsc.file.FileInfo(), nil -} - -func (rsc *resource) Close() (err error) { - log.Debugf("zip::Close(%s).", rsc.file.FileInfo().Name()) - return rsc.reader.Close() -} - -type ResourceMap map[string]*zip.File - -func (rm ResourceMap) Open(name string) (http.File, error) { - log.Debugf("GET zip:%s.", name) - f, ok := rm[strings.Trim(name, "/")] - if !ok { - return nil, trace.Wrap(os.ErrNotExist) - } - reader, err := f.Open() - if err != nil { - return nil, trace.Wrap(err) - } - return &resource{ - reader: reader, - file: f, - }, nil +// NewStaticFileSystem is a no-op in this build mode. +func NewStaticFileSystem() (http.FileSystem, error) { //nolint:staticcheck + return nil, trace.NotFound(webAssetsMissingError) } diff --git a/lib/web/static_embed.go b/lib/web/static_embed.go new file mode 100644 index 00000000000..bfa28a7608f --- /dev/null +++ b/lib/web/static_embed.go @@ -0,0 +1,34 @@ +// +build webassets_embed + +/* +Copyright 2021 Gravitational, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package web + +import ( + "bytes" + _ "embed" + "net/http" +) + +//go:embed build/webassets.zip +var webassetsZip []byte + +// NewStaticFileSystem returns the initialized implementation of http.FileSystem +// interface which can be used to serve Teleport Proxy Web UI +func NewStaticFileSystem() (http.FileSystem, error) { + return readZipArchive(bytes.NewReader(webassetsZip), int64(len(webassetsZip))) +} diff --git a/lib/web/static_test.go b/lib/web/static_test.go index fa96a4f2452..79f47fbbc7c 100644 --- a/lib/web/static_test.go +++ b/lib/web/static_test.go @@ -19,8 +19,10 @@ package web import ( "io" "io/ioutil" + "os" "strings" + "github.com/gravitational/trace" "gopkg.in/check.v1" ) @@ -46,7 +48,7 @@ func (s *StaticSuite) TestLocalFS(c *check.C) { } func (s *StaticSuite) TestZipFS(c *check.C) { - fs, err := readZipArchive("../../fixtures/assets.zip") + fs, err := readZipArchiveAt("../../fixtures/assets.zip") c.Assert(err, check.IsNil) c.Assert(fs, check.NotNil) @@ -87,3 +89,19 @@ func (s *StaticSuite) TestZipFS(c *check.C) { c.Assert(err, check.IsNil) c.Assert(len(bytes), check.Equals, 100) } + +func readZipArchiveAt(path string) (ResourceMap, error) { + file, err := os.Open(path) + if err != nil { + return nil, trace.Wrap(err) + } + // file needs to stay open for http.FileSystem reads to work + // + // feed the binary into the zip reader and enumerate all files + // found in the attached zip file: + info, err := file.Stat() + if err != nil { + return nil, trace.Wrap(err) + } + return readZipArchive(file, info.Size()) +}