mirror of
https://github.com/coder/coder.git
synced 2026-09-24 15:04:27 +08:00
feat: Check decompressed coder-slim binaries via SHA1 (#2556)
This commit is contained in:
Mathias Fredriksson
@@ -58,7 +58,7 @@ done
|
||||
# Check dependencies
|
||||
dependencies go
|
||||
if [[ $compress != 0 ]]; then
|
||||
dependencies tar zstd
|
||||
dependencies shasum tar zstd
|
||||
|
||||
if [[ $compress != [0-9]* ]] || [[ $compress -gt 22 ]] || [[ $compress -lt 1 ]]; then
|
||||
error "Invalid value for compress, must in in the range of [1, 22]"
|
||||
@@ -110,10 +110,24 @@ for f in ./coder-slim_*; do
|
||||
done
|
||||
|
||||
if [[ $compress != 0 ]]; then
|
||||
log "--- Compressing coder-slim binaries using zstd level $compress ($dest_dir/coder.tar.zst)"
|
||||
pushd "$dest_dir"
|
||||
tar cf coder.tar coder-*
|
||||
sha_file=coder.sha1
|
||||
sha_dest="$dest_dir/$sha_file"
|
||||
log "--- Generating SHA1 for coder-slim binaries ($sha_dest)"
|
||||
shasum -b -a 1 coder-* | tee $sha_file
|
||||
echo "$sha_dest"
|
||||
log
|
||||
log
|
||||
|
||||
tar_name=coder.tar.zst
|
||||
tar_dest="$dest_dir/$tar_name"
|
||||
log "--- Compressing coder-slim binaries using zstd level $compress ($tar_dest)"
|
||||
tar cf coder.tar $sha_file coder-*
|
||||
rm coder-*
|
||||
zstd --force --ultra --long -"${compress}" --rm --no-progress coder.tar -o coder.tar.zst
|
||||
zstd --force --ultra --long -"${compress}" --rm --no-progress coder.tar -o $tar_name
|
||||
echo "$tar_dest"
|
||||
log
|
||||
log
|
||||
|
||||
popd
|
||||
fi
|
||||
|
||||
+105
-4
@@ -4,6 +4,8 @@ import (
|
||||
"archive/tar"
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha1" //#nosec // Not used for cryptography.
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
@@ -20,6 +22,7 @@ import (
|
||||
"github.com/klauspost/compress/zstd"
|
||||
"github.com/unrolled/secure"
|
||||
"golang.org/x/exp/slices"
|
||||
"golang.org/x/sync/errgroup"
|
||||
"golang.org/x/xerrors"
|
||||
)
|
||||
|
||||
@@ -439,12 +442,18 @@ func ExtractOrReadBinFS(dest string, siteFS fs.FS) (http.FileSystem, error) {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
n, err := extractBin(dest, archive)
|
||||
ok, err := verifyBinSha1IsCurrent(dest, siteFS)
|
||||
if err != nil {
|
||||
return nil, xerrors.Errorf("extract coder binaries failed: %w", err)
|
||||
return nil, xerrors.Errorf("verify coder binaries sha1 failed: %w", err)
|
||||
}
|
||||
if n == 0 {
|
||||
return nil, xerrors.New("no files were extracted from coder binaries archive")
|
||||
if !ok {
|
||||
n, err := extractBin(dest, archive)
|
||||
if err != nil {
|
||||
return nil, xerrors.Errorf("extract coder binaries failed: %w", err)
|
||||
}
|
||||
if n == 0 {
|
||||
return nil, xerrors.New("no files were extracted from coder binaries archive")
|
||||
}
|
||||
}
|
||||
|
||||
return dir, nil
|
||||
@@ -461,6 +470,98 @@ func filterFiles(files []fs.DirEntry, names ...string) []fs.DirEntry {
|
||||
return filtered
|
||||
}
|
||||
|
||||
// errHashMismatch is a sentinel error used in verifyBinSha1IsCurrent.
|
||||
var errHashMismatch = xerrors.New("hash mismatch")
|
||||
|
||||
func verifyBinSha1IsCurrent(dest string, siteFS fs.FS) (ok bool, err error) {
|
||||
b1, err := fs.ReadFile(siteFS, "bin/coder.sha1")
|
||||
if err != nil {
|
||||
return false, xerrors.Errorf("read coder sha1 from embedded fs failed: %w", err)
|
||||
}
|
||||
// Parse sha1 file.
|
||||
shaFiles := make(map[string][]byte)
|
||||
for _, line := range bytes.Split(bytes.TrimSpace(b1), []byte{'\n'}) {
|
||||
parts := bytes.Split(line, []byte{' ', '*'})
|
||||
if len(parts) != 2 {
|
||||
return false, xerrors.Errorf("malformed sha1 file: %w", err)
|
||||
}
|
||||
shaFiles[string(parts[1])] = parts[0]
|
||||
}
|
||||
if len(shaFiles) == 0 {
|
||||
return false, xerrors.Errorf("empty sha1 file: %w", err)
|
||||
}
|
||||
|
||||
b2, err := os.ReadFile(filepath.Join(dest, "coder.sha1"))
|
||||
if err != nil {
|
||||
if xerrors.Is(err, fs.ErrNotExist) {
|
||||
return false, nil
|
||||
}
|
||||
return false, xerrors.Errorf("read coder sha1 failed: %w", err)
|
||||
}
|
||||
|
||||
// Check shasum files for equality for early-exit.
|
||||
if !bytes.Equal(b1, b2) {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
var eg errgroup.Group
|
||||
// Speed up startup by verifying files concurrently. Concurrency
|
||||
// is limited to save resources / early-exit. Early-exit speed
|
||||
// could be improved by using a context aware io.Reader and
|
||||
// passing the context from errgroup.WithContext.
|
||||
eg.SetLimit(3)
|
||||
|
||||
// Verify the hash of each on-disk binary.
|
||||
for file, hash1 := range shaFiles {
|
||||
file := file
|
||||
hash1 := hash1
|
||||
eg.Go(func() error {
|
||||
hash2, err := sha1HashFile(filepath.Join(dest, file))
|
||||
if err != nil {
|
||||
if xerrors.Is(err, fs.ErrNotExist) {
|
||||
return errHashMismatch
|
||||
}
|
||||
return xerrors.Errorf("hash file failed: %w", err)
|
||||
}
|
||||
if !bytes.Equal(hash1, hash2) {
|
||||
return errHashMismatch
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
err = eg.Wait()
|
||||
if err != nil {
|
||||
if xerrors.Is(err, errHashMismatch) {
|
||||
return false, nil
|
||||
}
|
||||
return false, err
|
||||
}
|
||||
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// sha1HashFile computes a SHA1 hash of the file, returning the hex
|
||||
// representation.
|
||||
func sha1HashFile(name string) ([]byte, error) {
|
||||
//#nosec // Not used for cryptography.
|
||||
hash := sha1.New()
|
||||
f, err := os.Open(name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
_, err = io.Copy(hash, f)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
b := make([]byte, hash.Size())
|
||||
hash.Sum(b[:0])
|
||||
|
||||
return []byte(hex.EncodeToString(b)), nil
|
||||
}
|
||||
|
||||
func extractBin(dest string, r io.Reader) (numExtraced int, err error) {
|
||||
opts := []zstd.DOption{
|
||||
// Concurrency doesn't help us when decoding the tar and
|
||||
|
||||
+138
-34
@@ -1,12 +1,16 @@
|
||||
package site_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/fs"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"testing/fstest"
|
||||
@@ -175,6 +179,41 @@ func TestShouldCacheFile(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
const (
|
||||
binCoderSha1 = "bin/coder.sha1"
|
||||
binCoderTarZstd = "bin/coder.tar.zst"
|
||||
)
|
||||
|
||||
func sampleBinFS() fstest.MapFS {
|
||||
return fstest.MapFS{
|
||||
binCoderSha1: &fstest.MapFile{
|
||||
Data: []byte("55641d5d56bbb8ccf5850fe923bd971b86364604 *coder-linux-amd64\n"),
|
||||
},
|
||||
binCoderTarZstd: &fstest.MapFile{
|
||||
// echo -n compressed >coder-linux-amd64
|
||||
// shasum -b -a 1 coder-linux-amd64 | tee coder.sha1
|
||||
// tar cf coder.tar coder.sha1 coder-linux-amd64
|
||||
// zstd --long --ultra -22 coder.tar
|
||||
Data: []byte{
|
||||
0x28, 0xb5, 0x2f, 0xfd, 0x64, 0x00, 0x27, 0xb5, 0x04, 0x00, 0x12, 0x08,
|
||||
0x1a, 0x1a, 0x90, 0xa7, 0x0e, 0x00, 0x0c, 0x19, 0x7c, 0xfb, 0xa0, 0xa1,
|
||||
0x5d, 0x21, 0xee, 0xae, 0xa8, 0x35, 0x65, 0x26, 0x57, 0x6e, 0x9a, 0xee,
|
||||
0xaf, 0x77, 0x94, 0x01, 0xf8, 0xec, 0x3d, 0x86, 0x1c, 0xdc, 0xb1, 0x76,
|
||||
0x8d, 0x31, 0x8a, 0x00, 0xf6, 0x77, 0xa9, 0x48, 0x24, 0x06, 0x42, 0xa1,
|
||||
0x08, 0x14, 0x4e, 0x67, 0x5f, 0x47, 0x4a, 0x8f, 0xf1, 0x6a, 0x8d, 0xc1,
|
||||
0x5a, 0x36, 0xea, 0xb6, 0x16, 0x52, 0x4a, 0x79, 0x7f, 0xbf, 0xb2, 0x77,
|
||||
0x63, 0x4b, 0x0e, 0x4b, 0x41, 0x12, 0xe2, 0x25, 0x98, 0x05, 0x73, 0x53,
|
||||
0x35, 0x71, 0xf5, 0x68, 0x37, 0xb7, 0x61, 0x45, 0x3e, 0xd9, 0x47, 0x99,
|
||||
0x3d, 0x51, 0xd3, 0xe0, 0x09, 0x10, 0xf6, 0xc7, 0x0a, 0x10, 0x20, 0x50,
|
||||
0x2b, 0x2e, 0x6d, 0x03, 0xf2, 0x21, 0xef, 0xc7, 0xa8, 0xc4, 0x3b, 0x8c,
|
||||
0x03, 0x64, 0x1a, 0xd9, 0x9d, 0x01, 0x60, 0xac, 0x94, 0x5a, 0x08, 0x05,
|
||||
0x4d, 0xb2, 0xd1, 0x0a, 0x99, 0x14, 0x48, 0xe3, 0xd9, 0x01, 0x99, 0x1d,
|
||||
0xe0, 0xda, 0xd4, 0xbd, 0xd4, 0xc6, 0x51, 0x0d,
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func TestServingBin(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
@@ -194,6 +233,12 @@ func TestServingBin(t *testing.T) {
|
||||
},
|
||||
}
|
||||
|
||||
sampleBinFSCorrupted := sampleBinFS()
|
||||
copy(sampleBinFSCorrupted[binCoderTarZstd].Data[10:], bytes.Repeat([]byte{0}, 10)) // Zero portion of archive.
|
||||
|
||||
sampleBinFSMissingSha256 := sampleBinFS()
|
||||
delete(sampleBinFSMissingSha256, binCoderSha1)
|
||||
|
||||
type req struct {
|
||||
url string
|
||||
wantStatus int
|
||||
@@ -207,52 +252,35 @@ func TestServingBin(t *testing.T) {
|
||||
}{
|
||||
{
|
||||
name: "Extract and serve bin",
|
||||
fs: fstest.MapFS{
|
||||
"bin/coder.tar.zst": &fstest.MapFile{
|
||||
// echo iamcoder >coder-linux-amd64
|
||||
// tar cf coder.tar coder-linux-amd64
|
||||
// zstd --long --ultra -22 coder.tar
|
||||
Data: []byte{
|
||||
0x28, 0xb5, 0x2f, 0xfd, 0x64, 0x00, 0x27, 0xf5, 0x02, 0x00, 0x12, 0xc4,
|
||||
0x0e, 0x16, 0xa0, 0xb5, 0x39, 0x00, 0xe8, 0x67, 0x59, 0xaf, 0xe3, 0xdd,
|
||||
0x8d, 0xfe, 0x47, 0xe8, 0x9d, 0x9c, 0x44, 0x0b, 0x75, 0x70, 0x61, 0x52,
|
||||
0x0d, 0x56, 0xaa, 0x16, 0xb9, 0x5a, 0x0a, 0x4b, 0x40, 0xd2, 0x7a, 0x05,
|
||||
0xd1, 0xd7, 0xe3, 0xf9, 0xf9, 0x07, 0xef, 0xda, 0x77, 0x04, 0xff, 0xe8,
|
||||
0x7a, 0x94, 0x56, 0x9a, 0x40, 0x3b, 0x94, 0x61, 0x18, 0x91, 0x90, 0x21,
|
||||
0x0c, 0x00, 0xf3, 0xc5, 0xe5, 0xd8, 0x80, 0x10, 0x06, 0x0a, 0x08, 0x86,
|
||||
0xb2, 0x00, 0x60, 0x12, 0x70, 0xd3, 0x51, 0x05, 0x04, 0x20, 0x16, 0x2c,
|
||||
0x79, 0xad, 0x01, 0xc0, 0xf5, 0x28, 0x08, 0x03, 0x1c, 0x4c, 0x84, 0xf4,
|
||||
},
|
||||
},
|
||||
},
|
||||
fs: sampleBinFS(),
|
||||
reqs: []req{
|
||||
{url: "/bin/coder-linux-amd64", wantStatus: http.StatusOK, wantBody: []byte("iamcoder\n")},
|
||||
{url: "/bin/coder-linux-amd64", wantStatus: http.StatusOK, wantBody: []byte("compressed")},
|
||||
{url: "/bin/GITKEEP", wantStatus: http.StatusNotFound},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "Error on invalid archive",
|
||||
name: "Extract and serve bin fails due to missing sha256",
|
||||
fs: sampleBinFSMissingSha256,
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "Error on invalid archive",
|
||||
fs: sampleBinFSCorrupted,
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "Error on malformed sha256 file",
|
||||
fs: fstest.MapFS{
|
||||
"bin/coder.tar.zst": &fstest.MapFile{
|
||||
Data: []byte{
|
||||
0x28, 0xb5, 0x2f, 0xfd, 0x64, 0x00, 0x27, 0xf5, 0x02, 0x00, 0x12, 0xc4,
|
||||
0x0e, 0x16, 0xa0, 0xb5, 0x39, 0x00, 0xe8, 0x67, 0x59, 0xaf, 0xe3, 0xdd,
|
||||
0x8d, 0xfe, 0x47, 0xe8, 0x9d, 0x9c, 0x44, 0x0b, 0x75, 0x70, 0x61, 0x52,
|
||||
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // Zeroed from above test.
|
||||
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // Zeroed from above test.
|
||||
0x7a, 0x94, 0x56, 0x9a, 0x40, 0x3b, 0x94, 0x61, 0x18, 0x91, 0x90, 0x21,
|
||||
0x0c, 0x00, 0xf3, 0xc5, 0xe5, 0xd8, 0x80, 0x10, 0x06, 0x0a, 0x08, 0x86,
|
||||
0xb2, 0x00, 0x60, 0x12, 0x70, 0xd3, 0x51, 0x05, 0x04, 0x20, 0x16, 0x2c,
|
||||
0x79, 0xad, 0x01, 0xc0, 0xf5, 0x28, 0x08, 0x03, 0x1c, 0x4c, 0x84, 0xf4,
|
||||
},
|
||||
},
|
||||
binCoderSha1: &fstest.MapFile{Data: []byte("byebye")},
|
||||
binCoderTarZstd: sampleBinFS()[binCoderTarZstd],
|
||||
},
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "Error on empty archive",
|
||||
fs: fstest.MapFS{
|
||||
"bin/coder.tar.zst": &fstest.MapFile{Data: []byte{}},
|
||||
binCoderSha1: &fstest.MapFile{Data: []byte{}},
|
||||
binCoderTarZstd: &fstest.MapFile{Data: []byte{}},
|
||||
},
|
||||
wantErr: true,
|
||||
},
|
||||
@@ -336,6 +364,82 @@ func TestServingBin(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractOrReadBinFS(t *testing.T) {
|
||||
t.Parallel()
|
||||
t.Run("DoubleExtractDoesNotModifyFiles", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
siteFS := sampleBinFS()
|
||||
dest := t.TempDir()
|
||||
_, err := site.ExtractOrReadBinFS(dest, siteFS)
|
||||
require.NoError(t, err)
|
||||
|
||||
checkModtime := func() map[string]time.Time {
|
||||
m := make(map[string]time.Time)
|
||||
|
||||
err = filepath.WalkDir(dest, func(path string, d fs.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if d.IsDir() {
|
||||
return nil // Only check the files.
|
||||
}
|
||||
stat, err := d.Info()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
m[path] = stat.ModTime()
|
||||
return nil
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
return m
|
||||
}
|
||||
|
||||
firstModtimes := checkModtime()
|
||||
|
||||
_, err = site.ExtractOrReadBinFS(dest, siteFS)
|
||||
require.NoError(t, err)
|
||||
|
||||
secondModtimes := checkModtime()
|
||||
|
||||
assert.Equal(t, firstModtimes, secondModtimes, "second extract should not modify files")
|
||||
})
|
||||
t.Run("SHA256MismatchCausesReExtract", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
siteFS := sampleBinFS()
|
||||
dest := t.TempDir()
|
||||
_, err := site.ExtractOrReadBinFS(dest, siteFS)
|
||||
require.NoError(t, err)
|
||||
|
||||
bin := filepath.Join(dest, "bin", "coder-linux-amd64")
|
||||
f, err := os.OpenFile(bin, os.O_WRONLY, 0o600)
|
||||
require.NoError(t, err)
|
||||
|
||||
dontWant := []byte("hello")
|
||||
_, err = f.WriteAt(dontWant, 0) // Overwrite the start of file.
|
||||
assert.NoError(t, err) // Assert to allow f.Close.
|
||||
|
||||
err = f.Close()
|
||||
require.NoError(t, err)
|
||||
|
||||
_, err = site.ExtractOrReadBinFS(dest, siteFS)
|
||||
require.NoError(t, err)
|
||||
|
||||
f, err = os.Open(bin)
|
||||
require.NoError(t, err)
|
||||
defer f.Close()
|
||||
|
||||
got := make([]byte, 5) // hello
|
||||
_, err = f.Read(got)
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.NotEqual(t, dontWant, got, "file should be overwritten on hash mismatch")
|
||||
})
|
||||
}
|
||||
|
||||
func TestServeAPIResponse(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
|
||||
Reference in New Issue
Block a user