chore(coderd): extract fileszip to package archive for reuse (#15229)

Related to https://github.com/coder/coder/issues/15087
As part of sniffing the workspace tags from an uploaded file, we need to
be able to handle both zip and tar files. Extracting the functions to
a separate `archive` package will be helpful here.

coderd/files.go

@@ -16,6 +16,7 @@ import (
 	"github.com/google/uuid"
 
 	"cdr.dev/slog"
+	"github.com/coder/coder/v2/archive"
 	"github.com/coder/coder/v2/coderd/database"
 	"github.com/coder/coder/v2/coderd/database/dbtime"
 	"github.com/coder/coder/v2/coderd/httpapi"
@@ -27,7 +28,7 @@ const (
 	tarMimeType = "application/x-tar"
 	zipMimeType = "application/zip"
 
-	httpFileMaxBytes = 10 * (10 << 20)
+	HTTPFileMaxBytes = 10 * (10 << 20)
 )
 
 // @Summary Upload file
@@ -55,7 +56,7 @@ func (api *API) postFile(rw http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	r.Body = http.MaxBytesReader(rw, r.Body, httpFileMaxBytes)
+	r.Body = http.MaxBytesReader(rw, r.Body, HTTPFileMaxBytes)
 	data, err := io.ReadAll(r.Body)
 	if err != nil {
 		httpapi.Write(ctx, rw, http.StatusBadRequest, codersdk.Response{
@@ -75,7 +76,7 @@ func (api *API) postFile(rw http.ResponseWriter, r *http.Request) {
 			return
 		}
 
-		data, err = CreateTarFromZip(zipReader)
+		data, err = archive.CreateTarFromZip(zipReader, HTTPFileMaxBytes)
 		if err != nil {
 			httpapi.Write(ctx, rw, http.StatusInternalServerError, codersdk.Response{
 				Message: "Internal error processing .zip archive.",
@@ -181,7 +182,7 @@ func (api *API) fileByID(rw http.ResponseWriter, r *http.Request) {
 
 		rw.Header().Set("Content-Type", codersdk.ContentTypeZip)
 		rw.WriteHeader(http.StatusOK)
-		err = WriteZipArchive(rw, tar.NewReader(bytes.NewReader(file.Data)))
+		err = archive.WriteZip(rw, tar.NewReader(bytes.NewReader(file.Data)), HTTPFileMaxBytes)
 		if err != nil {
 			api.Logger.Error(ctx, "invalid .zip archive", slog.F("file_id", fileID), slog.F("mimetype", file.Mimetype), slog.Error(err))
 		}

coderd/files_test.go

@@ -5,14 +5,13 @@ import (
 	"bytes"
 	"context"
 	"net/http"
-	"os"
-	"path/filepath"
 	"testing"
 
 	"github.com/google/uuid"
 	"github.com/stretchr/testify/require"
 
-	"github.com/coder/coder/v2/coderd"
+	"github.com/coder/coder/v2/archive"
+	"github.com/coder/coder/v2/archive/archivetest"
 	"github.com/coder/coder/v2/coderd/coderdtest"
 	"github.com/coder/coder/v2/codersdk"
 	"github.com/coder/coder/v2/testutil"
@@ -84,8 +83,8 @@ func TestDownload(t *testing.T) {
 		// given
 		ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitLong)
 		defer cancel()
-		tarball, err := os.ReadFile(filepath.Join("testdata", "test.tar"))
-		require.NoError(t, err)
+
+		tarball := archivetest.TestTarFileBytes()
 
 		// when
 		resp, err := client.Upload(ctx, codersdk.ContentTypeTar, bytes.NewReader(tarball))
@@ -97,7 +96,7 @@ func TestDownload(t *testing.T) {
 		require.Len(t, data, len(tarball))
 		require.Equal(t, codersdk.ContentTypeTar, contentType)
 		require.Equal(t, tarball, data)
-		assertSampleTarFile(t, data)
+		archivetest.AssertSampleTarFile(t, data)
 	})
 
 	t.Run("InsertZip_DownloadTar", func(t *testing.T) {
@@ -106,8 +105,7 @@ func TestDownload(t *testing.T) {
 		_ = coderdtest.CreateFirstUser(t, client)
 
 		// given
-		zipContent, err := os.ReadFile(filepath.Join("testdata", "test.zip"))
-		require.NoError(t, err)
+		zipContent := archivetest.TestZipFileBytes()
 
 		ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitLong)
 		defer cancel()
@@ -123,7 +121,7 @@ func TestDownload(t *testing.T) {
 
 		// Note: creating a zip from a tar will result in some loss of information
 		// as zip files do not store UNIX user:group data.
-		assertSampleTarFile(t, data)
+		archivetest.AssertSampleTarFile(t, data)
 	})
 
 	t.Run("InsertTar_DownloadZip", func(t *testing.T) {
@@ -132,11 +130,10 @@ func TestDownload(t *testing.T) {
 		_ = coderdtest.CreateFirstUser(t, client)
 
 		// given
-		tarball, err := os.ReadFile(filepath.Join("testdata", "test.tar"))
-		require.NoError(t, err)
+		tarball := archivetest.TestTarFileBytes()
 
 		tarReader := tar.NewReader(bytes.NewReader(tarball))
-		expectedZip, err := coderd.CreateZipFromTar(tarReader)
+		expectedZip, err := archive.CreateZipFromTar(tarReader, 10240)
 		require.NoError(t, err)
 
 		ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitLong)
@@ -151,6 +148,6 @@ func TestDownload(t *testing.T) {
 		// then
 		require.Equal(t, codersdk.ContentTypeZip, contentType)
 		require.Equal(t, expectedZip, data)
-		assertSampleZipFile(t, data)
+		archivetest.AssertSampleZipFile(t, data)
 	})
 }

coderd/fileszip.go

@@ -1,112 +0,0 @@
-package coderd
-
-import (
-	"archive/tar"
-	"archive/zip"
-	"bytes"
-	"errors"
-	"io"
-	"log"
-	"strings"
-)
-
-func CreateTarFromZip(zipReader *zip.Reader) ([]byte, error) {
-	var tarBuffer bytes.Buffer
-	err := writeTarArchive(&tarBuffer, zipReader)
-	if err != nil {
-		return nil, err
-	}
-	return tarBuffer.Bytes(), nil
-}
-
-func writeTarArchive(w io.Writer, zipReader *zip.Reader) error {
-	tarWriter := tar.NewWriter(w)
-	defer tarWriter.Close()
-
-	for _, file := range zipReader.File {
-		err := processFileInZipArchive(file, tarWriter)
-		if err != nil {
-			return err
-		}
-	}
-	return nil
-}
-
-func processFileInZipArchive(file *zip.File, tarWriter *tar.Writer) error {
-	fileReader, err := file.Open()
-	if err != nil {
-		return err
-	}
-	defer fileReader.Close()
-
-	err = tarWriter.WriteHeader(&tar.Header{
-		Name:    file.Name,
-		Size:    file.FileInfo().Size(),
-		Mode:    int64(file.Mode()),
-		ModTime: file.Modified,
-		// Note: Zip archives do not store ownership information.
-		Uid: 1000,
-		Gid: 1000,
-	})
-	if err != nil {
-		return err
-	}
-
-	n, err := io.CopyN(tarWriter, fileReader, httpFileMaxBytes)
-	log.Println(file.Name, n, err)
-	if errors.Is(err, io.EOF) {
-		err = nil
-	}
-	return err
-}
-
-func CreateZipFromTar(tarReader *tar.Reader) ([]byte, error) {
-	var zipBuffer bytes.Buffer
-	err := WriteZipArchive(&zipBuffer, tarReader)
-	if err != nil {
-		return nil, err
-	}
-	return zipBuffer.Bytes(), nil
-}
-
-func WriteZipArchive(w io.Writer, tarReader *tar.Reader) error {
-	zipWriter := zip.NewWriter(w)
-	defer zipWriter.Close()
-
-	for {
-		tarHeader, err := tarReader.Next()
-		if errors.Is(err, io.EOF) {
-			break
-		}
-
-		if err != nil {
-			return err
-		}
-
-		zipHeader, err := zip.FileInfoHeader(tarHeader.FileInfo())
-		if err != nil {
-			return err
-		}
-		zipHeader.Name = tarHeader.Name
-		// Some versions of unzip do not check the mode on a file entry and
-		// simply assume that entries with a trailing path separator (/) are
-		// directories, and that everything else is a file. Give them a hint.
-		if tarHeader.FileInfo().IsDir() && !strings.HasSuffix(tarHeader.Name, "/") {
-			zipHeader.Name += "/"
-		}
-
-		zipEntry, err := zipWriter.CreateHeader(zipHeader)
-		if err != nil {
-			return err
-		}
-
-		_, err = io.CopyN(zipEntry, tarReader, httpFileMaxBytes)
-		if errors.Is(err, io.EOF) {
-			err = nil
-		}
-		if err != nil {
-			return err
-		}
-	}
-	return nil // don't need to flush as we call `writer.Close()`
-}

coderd/fileszip_test.go

@@ -1,251 +0,0 @@
-package coderd_test
-
-import (
-	"archive/tar"
-	"archive/zip"
-	"bytes"
-	"io"
-	"io/fs"
-	"os"
-	"os/exec"
-	"path/filepath"
-	"runtime"
-	"strings"
-	"testing"
-	"time"
-
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-	"golang.org/x/xerrors"
-
-	"github.com/coder/coder/v2/coderd"
-	"github.com/coder/coder/v2/testutil"
-)
-
-func TestCreateTarFromZip(t *testing.T) {
-	t.Parallel()
-	if runtime.GOOS != "linux" {
-		t.Skip("skipping this test on non-Linux platform")
-	}
-
-	// Read a zip file we prepared earlier
-	ctx := testutil.Context(t, testutil.WaitShort)
-	zipBytes, err := os.ReadFile(filepath.Join("testdata", "test.zip"))
-	require.NoError(t, err, "failed to read sample zip file")
-	// Assert invariant
-	assertSampleZipFile(t, zipBytes)
-
-	zr, err := zip.NewReader(bytes.NewReader(zipBytes), int64(len(zipBytes)))
-	require.NoError(t, err, "failed to parse sample zip file")
-
-	tarBytes, err := coderd.CreateTarFromZip(zr)
-	require.NoError(t, err, "failed to convert zip to tar")
-
-	assertSampleTarFile(t, tarBytes)
-
-	tempDir := t.TempDir()
-	tempFilePath := filepath.Join(tempDir, "test.tar")
-	err = os.WriteFile(tempFilePath, tarBytes, 0o600)
-	require.NoError(t, err, "failed to write converted tar file")
-
-	cmd := exec.CommandContext(ctx, "tar", "--extract", "--verbose", "--file", tempFilePath, "--directory", tempDir)
-	require.NoError(t, cmd.Run(), "failed to extract converted tar file")
-	assertExtractedFiles(t, tempDir, true)
-}
-
-func TestCreateZipFromTar(t *testing.T) {
-	t.Parallel()
-	if runtime.GOOS != "linux" {
-		t.Skip("skipping this test on non-Linux platform")
-	}
-	t.Run("OK", func(t *testing.T) {
-		t.Parallel()
-		tarBytes, err := os.ReadFile(filepath.Join(".", "testdata", "test.tar"))
-		require.NoError(t, err, "failed to read sample tar file")
-
-		tr := tar.NewReader(bytes.NewReader(tarBytes))
-		zipBytes, err := coderd.CreateZipFromTar(tr)
-		require.NoError(t, err)
-
-		assertSampleZipFile(t, zipBytes)
-
-		tempDir := t.TempDir()
-		tempFilePath := filepath.Join(tempDir, "test.zip")
-		err = os.WriteFile(tempFilePath, zipBytes, 0o600)
-		require.NoError(t, err, "failed to write converted zip file")
-
-		ctx := testutil.Context(t, testutil.WaitShort)
-		cmd := exec.CommandContext(ctx, "unzip", tempFilePath, "-d", tempDir)
-		require.NoError(t, cmd.Run(), "failed to extract converted zip file")
-
-		assertExtractedFiles(t, tempDir, false)
-	})
-
-	t.Run("MissingSlashInDirectoryHeader", func(t *testing.T) {
-		t.Parallel()
-
-		// Given: a tar archive containing a directory entry that has the directory
-		// mode bit set but the name is missing a trailing slash
-
-		var tarBytes bytes.Buffer
-		tw := tar.NewWriter(&tarBytes)
-		tw.WriteHeader(&tar.Header{
-			Name:     "dir",
-			Typeflag: tar.TypeDir,
-			Size:     0,
-		})
-		require.NoError(t, tw.Flush())
-		require.NoError(t, tw.Close())
-
-		// When: we convert this to a zip
-		tr := tar.NewReader(&tarBytes)
-		zipBytes, err := coderd.CreateZipFromTar(tr)
-		require.NoError(t, err)
-
-		// Then: the resulting zip should contain a corresponding directory
-		zr, err := zip.NewReader(bytes.NewReader(zipBytes), int64(len(zipBytes)))
-		require.NoError(t, err)
-		for _, zf := range zr.File {
-			switch zf.Name {
-			case "dir":
-				require.Fail(t, "missing trailing slash in directory name")
-			case "dir/":
-				require.True(t, zf.Mode().IsDir(), "should be a directory")
-			default:
-				require.Fail(t, "unexpected file in archive")
-			}
-		}
-	})
-}
-
-// nolint:revive // this is a control flag but it's in a unit test
-func assertExtractedFiles(t *testing.T, dir string, checkModePerm bool) {
-	t.Helper()
-
-	_ = filepath.Walk(dir, func(path string, info fs.FileInfo, err error) error {
-		relPath := strings.TrimPrefix(path, dir)
-		switch relPath {
-		case "", "/test.zip", "/test.tar": // ignore
-		case "/test":
-			stat, err := os.Stat(path)
-			assert.NoError(t, err, "failed to stat path %q", path)
-			assert.True(t, stat.IsDir(), "expected path %q to be a directory")
-			if checkModePerm {
-				assert.Equal(t, fs.ModePerm&0o755, stat.Mode().Perm(), "expected mode 0755 on directory")
-			}
-			assert.Equal(t, archiveRefTime(t).UTC(), stat.ModTime().UTC(), "unexpected modtime of %q", path)
-		case "/test/hello.txt":
-			stat, err := os.Stat(path)
-			assert.NoError(t, err, "failed to stat path %q", path)
-			assert.False(t, stat.IsDir(), "expected path %q to be a file")
-			if checkModePerm {
-				assert.Equal(t, fs.ModePerm&0o644, stat.Mode().Perm(), "expected mode 0644 on file")
-			}
-			bs, err := os.ReadFile(path)
-			assert.NoError(t, err, "failed to read file %q", path)
-			assert.Equal(t, "hello", string(bs), "unexpected content in file %q", path)
-		case "/test/dir":
-			stat, err := os.Stat(path)
-			assert.NoError(t, err, "failed to stat path %q", path)
-			assert.True(t, stat.IsDir(), "expected path %q to be a directory")
-			if checkModePerm {
-				assert.Equal(t, fs.ModePerm&0o755, stat.Mode().Perm(), "expected mode 0755 on directory")
-			}
-		case "/test/dir/world.txt":
-			stat, err := os.Stat(path)
-			assert.NoError(t, err, "failed to stat path %q", path)
-			assert.False(t, stat.IsDir(), "expected path %q to be a file")
-			if checkModePerm {
-				assert.Equal(t, fs.ModePerm&0o644, stat.Mode().Perm(), "expected mode 0644 on file")
-			}
-			bs, err := os.ReadFile(path)
-			assert.NoError(t, err, "failed to read file %q", path)
-			assert.Equal(t, "world", string(bs), "unexpected content in file %q", path)
-		default:
-			assert.Fail(t, "unexpected path", relPath)
-		}
-
-		return nil
-	})
-}
-
-func assertSampleTarFile(t *testing.T, tarBytes []byte) {
-	t.Helper()
-
-	tr := tar.NewReader(bytes.NewReader(tarBytes))
-	for {
-		hdr, err := tr.Next()
-		if err != nil {
-			if err == io.EOF {
-				return
-			}
-			require.NoError(t, err)
-		}
-
-		// Note: ignoring timezones here.
-		require.Equal(t, archiveRefTime(t).UTC(), hdr.ModTime.UTC())
-
-		switch hdr.Name {
-		case "test/":
-			require.Equal(t, hdr.Typeflag, byte(tar.TypeDir))
-		case "test/hello.txt":
-			require.Equal(t, hdr.Typeflag, byte(tar.TypeReg))
-			bs, err := io.ReadAll(tr)
-			if err != nil && !xerrors.Is(err, io.EOF) {
-				require.NoError(t, err)
-			}
-			require.Equal(t, "hello", string(bs))
-		case "test/dir/":
-			require.Equal(t, hdr.Typeflag, byte(tar.TypeDir))
-		case "test/dir/world.txt":
-			require.Equal(t, hdr.Typeflag, byte(tar.TypeReg))
-			bs, err := io.ReadAll(tr)
-			if err != nil && !xerrors.Is(err, io.EOF) {
-				require.NoError(t, err)
-			}
-			require.Equal(t, "world", string(bs))
-		default:
-			require.Failf(t, "unexpected file in tar", hdr.Name)
-		}
-	}
-}
-
-func assertSampleZipFile(t *testing.T, zipBytes []byte) {
-	t.Helper()
-
-	zr, err := zip.NewReader(bytes.NewReader(zipBytes), int64(len(zipBytes)))
-	require.NoError(t, err)
-
-	for _, f := range zr.File {
-		// Note: ignoring timezones here.
-		require.Equal(t, archiveRefTime(t).UTC(), f.Modified.UTC())
-		switch f.Name {
-		case "test/", "test/dir/":
-			// directory
-		case "test/hello.txt":
-			rc, err := f.Open()
-			require.NoError(t, err)
-			bs, err := io.ReadAll(rc)
-			_ = rc.Close()
-			require.NoError(t, err)
-			require.Equal(t, "hello", string(bs))
-		case "test/dir/world.txt":
-			rc, err := f.Open()
-			require.NoError(t, err)
-			bs, err := io.ReadAll(rc)
-			_ = rc.Close()
-			require.NoError(t, err)
-			require.Equal(t, "world", string(bs))
-		default:
-			require.Failf(t, "unexpected file in zip", f.Name)
-		}
-	}
-}
-
-// archiveRefTime is the Go reference time. The contents of the sample tar and zip files
-// in testdata/ all have their modtimes set to the below in some timezone.
-func archiveRefTime(t *testing.T) time.Time {
-	locMST, err := time.LoadLocation("MST")
-	require.NoError(t, err, "failed to load MST timezone")
-	return time.Date(2006, 1, 2, 3, 4, 5, 0, locMST)
-}