opensource/coder
3
0
Fork 0
mirror of https://github.com/coder/coder.git synced 2026-09-24 15:04:27 +08:00
Code Issues Packages Projects Releases Wiki Activity

Stream template upload (#6035)

Browse Source 
Resolves #5718
This commit is contained in:
Ammar Bandukwala
2023-02-04 20:07:09 +00:00
committed by GitHub
parent 77fd34be94
commit a422cc00e8
12 changed files with 69 additions and 46 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cli/templateinit.go
+2 -1
View File
@@ -1,6 +1,7 @@
package cli
import (
"bytes"
"fmt"
"os"
"path/filepath"
@@ -70,7 +71,7 @@ func templateInit() *cobra.Command {
if err != nil {
return err
}
err = provisionersdk.Untar(directory, archive)
err = provisionersdk.Untar(directory, bytes.NewReader(archive))
if err != nil {
return err
}
cli/templatepush.go
+21 -8
View File
@@ -1,6 +1,7 @@
package cli
import (
"bufio"
"fmt"
"io"
"os"
@@ -34,14 +35,17 @@ func (pf *templateUploadFlags) stdin() bool {
func (pf *templateUploadFlags) upload(cmd *cobra.Command, client *codersdk.Client) (*codersdk.UploadResponse, error) {
var (
content []byte
err error
content io.Reader
pipeErrCh = make(chan error, 1)
)
if pf.stdin() {
content, err = io.ReadAll(cmd.InOrStdin())
content = cmd.InOrStdin()
// No piping if reading from stdin.
pipeErrCh <- nil
close(pipeErrCh)
} else {
prettyDir := prettyDirectoryPath(pf.directory)
_, err = cliui.Prompt(cmd, cliui.PromptOptions{
_, err := cliui.Prompt(cmd, cliui.PromptOptions{
Text: fmt.Sprintf("Upload %q?", prettyDir),
IsConfirm: true,
Default: cliui.ConfirmYes,
@@ -50,10 +54,16 @@ func (pf *templateUploadFlags) upload(cmd *cobra.Command, client *codersdk.Clien
return nil, err
}
content, err = provisionersdk.Tar(pf.directory, provisionersdk.TemplateArchiveLimit)
}
if err != nil {
return nil, xerrors.Errorf("read tar: %w", err)
pipeReader, pipeWriter := io.Pipe()
go func() {
defer pipeWriter.Close()
defer close(pipeErrCh)
bufWr := bufio.NewWriter(pipeWriter)
defer bufWr.Flush()
pipeErrCh <- provisionersdk.Tar(bufWr, pf.directory, provisionersdk.TemplateArchiveLimit)
}()
defer pipeReader.Close()
content = pipeReader
}
spin := spinner.New(spinner.CharSets[5], 100*time.Millisecond)
@@ -66,6 +76,9 @@ func (pf *templateUploadFlags) upload(cmd *cobra.Command, client *codersdk.Clien
if err != nil {
return nil, xerrors.Errorf("upload: %w", err)
}
if err = <-pipeErrCh; err != nil {
return nil, xerrors.Errorf("pipe: %w", err)
}
return &resp, nil
}
coderd/coderdtest/authorize.go
+2 -1
View File
@@ -1,6 +1,7 @@
package coderdtest
import (
"bytes"
"context"
"fmt"
"io"
@@ -384,7 +385,7 @@ func NewAuthTester(ctx context.Context, t *testing.T, client *codersdk.Client, a
template := CreateTemplate(t, client, admin.OrganizationID, version.ID)
workspace := CreateWorkspace(t, client, admin.OrganizationID, template.ID)
AwaitWorkspaceBuildJob(t, client, workspace.LatestBuild.ID)
file, err := client.Upload(ctx, codersdk.ContentTypeTar, make([]byte, 1024))
file, err := client.Upload(ctx, codersdk.ContentTypeTar, bytes.NewReader(make([]byte, 1024)))
require.NoError(t, err, "upload file")
workspace, err = client.Workspace(ctx, workspace.ID)
require.NoError(t, err, "workspace resources")
coderd/coderdtest/coderdtest.go
+2 -2
View File
@@ -525,7 +525,7 @@ func CreateTemplateVersion(t *testing.T, client *codersdk.Client, organizationID
t.Helper()
data, err := echo.Tar(res)
require.NoError(t, err)
file, err := client.Upload(context.Background(), codersdk.ContentTypeTar, data)
file, err := client.Upload(context.Background(), codersdk.ContentTypeTar, bytes.NewReader(data))
require.NoError(t, err)
templateVersion, err := client.CreateTemplateVersion(context.Background(), organizationID, codersdk.CreateTemplateVersionRequest{
FileID: file.ID,
@@ -572,7 +572,7 @@ func CreateTemplate(t *testing.T, client *codersdk.Client, organization uuid.UUI
func UpdateTemplateVersion(t *testing.T, client *codersdk.Client, organizationID uuid.UUID, res *echo.Responses, templateID uuid.UUID) codersdk.TemplateVersion {
data, err := echo.Tar(res)
require.NoError(t, err)
file, err := client.Upload(context.Background(), codersdk.ContentTypeTar, data)
file, err := client.Upload(context.Background(), codersdk.ContentTypeTar, bytes.NewReader(data))
require.NoError(t, err)
templateVersion, err := client.CreateTemplateVersion(context.Background(), organizationID, codersdk.CreateTemplateVersionRequest{
TemplateID: templateID,
coderd/files_test.go
+6 -5
View File
@@ -1,6 +1,7 @@
package coderd_test
import (
"bytes"
"context"
"net/http"
"testing"
@@ -23,7 +24,7 @@ func TestPostFiles(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitLong)
defer cancel()
_, err := client.Upload(ctx, "bad", []byte{'a'})
_, err := client.Upload(ctx, "bad", bytes.NewReader([]byte{'a'}))
require.Error(t, err)
})
@@ -35,7 +36,7 @@ func TestPostFiles(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitLong)
defer cancel()
_, err := client.Upload(ctx, codersdk.ContentTypeTar, make([]byte, 1024))
_, err := client.Upload(ctx, codersdk.ContentTypeTar, bytes.NewReader(make([]byte, 1024)))
require.NoError(t, err)
})
@@ -48,9 +49,9 @@ func TestPostFiles(t *testing.T) {
defer cancel()
data := make([]byte, 1024)
_, err := client.Upload(ctx, codersdk.ContentTypeTar, data)
_, err := client.Upload(ctx, codersdk.ContentTypeTar, bytes.NewReader(data))
require.NoError(t, err)
_, err = client.Upload(ctx, codersdk.ContentTypeTar, data)
_, err = client.Upload(ctx, codersdk.ContentTypeTar, bytes.NewReader(data))
require.NoError(t, err)
})
}
@@ -79,7 +80,7 @@ func TestDownload(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitLong)
defer cancel()
resp, err := client.Upload(ctx, codersdk.ContentTypeTar, make([]byte, 1024))
resp, err := client.Upload(ctx, codersdk.ContentTypeTar, bytes.NewReader(make([]byte, 1024)))
require.NoError(t, err)
data, contentType, err := client.Download(ctx, resp.ID)
require.NoError(t, err)
coderd/templateversions_test.go
+3 -2
View File
@@ -1,6 +1,7 @@
package coderd_test
import (
"bytes"
"context"
"net/http"
"testing"
@@ -108,7 +109,7 @@ func TestPostTemplateVersionsByOrganization(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitLong)
defer cancel()
file, err := client.Upload(ctx, codersdk.ContentTypeTar, data)
file, err := client.Upload(ctx, codersdk.ContentTypeTar, bytes.NewReader(data))
require.NoError(t, err)
version, err := client.CreateTemplateVersion(ctx, user.OrganizationID, codersdk.CreateTemplateVersionRequest{
Name: "bananas",
@@ -895,7 +896,7 @@ func TestPaginatedTemplateVersions(t *testing.T) {
templateVersionIDs := make([]uuid.UUID, total)
data, err := echo.Tar(nil)
require.NoError(t, err)
file, err := client.Upload(egCtx, codersdk.ContentTypeTar, data)
file, err := client.Upload(egCtx, codersdk.ContentTypeTar, bytes.NewReader(data))
require.NoError(t, err)
for i := 0; i < total; i++ {
i := i
codersdk/client.go
+6 -4
View File
@@ -109,10 +109,13 @@ func (c *Client) Request(ctx context.Context, method, path string, body interfac
var r io.Reader
if body != nil {
if data, ok := body.([]byte); ok {
switch data := body.(type) {
case io.Reader:
r = data
case []byte:
r = bytes.NewReader(data)
} else {
// Assume JSON if not bytes.
default:
// Assume JSON in all other cases.
buf := bytes.NewBuffer(nil)
enc := json.NewEncoder(buf)
enc.SetEscapeHTML(false)
@@ -120,7 +123,6 @@ func (c *Client) Request(ctx context.Context, method, path string, body interfac
if err != nil {
return nil, xerrors.Errorf("encode body: %w", err)
}
r = buf
}
}
codersdk/files.go
+2 -2
View File
@@ -21,8 +21,8 @@ type UploadResponse struct {
// Upload uploads an arbitrary file with the content type provided.
// This is used to upload a source-code archive.
func (c *Client) Upload(ctx context.Context, contentType string, content []byte) (UploadResponse, error) {
res, err := c.Request(ctx, http.MethodPost, "/api/v2/files", content, func(r *http.Request) {
func (c *Client) Upload(ctx context.Context, contentType string, rd io.Reader) (UploadResponse, error) {
res, err := c.Request(ctx, http.MethodPost, "/api/v2/files", rd, func(r *http.Request) {
r.Header.Set("Content-Type", contentType)
})
if err != nil {
enterprise/coderd/provisionerdaemons_test.go
+2 -1
View File
@@ -1,6 +1,7 @@
package coderd_test
import (
"bytes"
"context"
"net/http"
"testing"
@@ -119,7 +120,7 @@ func TestProvisionerDaemonServe(t *testing.T) {
}},
})
require.NoError(t, err)
file, err := client.Upload(context.Background(), codersdk.ContentTypeTar, data)
file, err := client.Upload(context.Background(), codersdk.ContentTypeTar, bytes.NewReader(data))
require.NoError(t, err)
version, err := client.CreateTemplateVersion(context.Background(), user.OrganizationID, codersdk.CreateTemplateVersionRequest{
enterprise/coderd/templates_test.go
+2 -1
View File
@@ -1,6 +1,7 @@
package coderd_test
import (
"bytes"
"context"
"net/http"
"testing"
@@ -300,7 +301,7 @@ func TestTemplateACL(t *testing.T) {
data, err := echo.Tar(nil)
require.NoError(t, err)
file, err := client1.Upload(context.Background(), codersdk.ContentTypeTar, data)
file, err := client1.Upload(context.Background(), codersdk.ContentTypeTar, bytes.NewReader(data))
require.NoError(t, err)
_, err = client1.CreateTemplateVersion(ctx, user.OrganizationID, codersdk.CreateTemplateVersionRequest{
provisionersdk/archive.go
+12 -14
View File
@@ -2,7 +2,6 @@ package provisionersdk
import (
"archive/tar"
"bytes"
"io"
"os"
"path/filepath"
@@ -32,25 +31,24 @@ func dirHasExt(dir string, ext string) (bool, error) {
}
// Tar archives a Terraform directory.
func Tar(directory string, limit int64) ([]byte, error) {
var buffer bytes.Buffer
tarWriter := tar.NewWriter(&buffer)
func Tar(w io.Writer, directory string, limit int64) error {
tarWriter := tar.NewWriter(w)
totalSize := int64(0)
const tfExt = ".tf"
hasTf, err := dirHasExt(directory, tfExt)
if err != nil {
return nil, err
return err
}
if !hasTf {
absPath, err := filepath.Abs(directory)
if err != nil {
return nil, err
return err
}
// Show absolute path to aid in debugging. E.g. showing "." is
// useless.
return nil, xerrors.Errorf(
return xerrors.Errorf(
"%s is not a valid template since it has no %s files",
absPath, tfExt,
)
@@ -111,20 +109,20 @@ func Tar(directory string, limit int64) ([]byte, error) {
return data.Close()
})
if err != nil {
return nil, err
return err
}
err = tarWriter.Flush()
if err != nil {
return nil, err
return err
}
return buffer.Bytes(), nil
return nil
}
// Untar extracts the archive to a provided directory.
func Untar(directory string, archive []byte) error {
reader := tar.NewReader(bytes.NewReader(archive))
func Untar(directory string, r io.Reader) error {
tarReader := tar.NewReader(r)
for {
header, err := reader.Next()
header, err := tarReader.Next()
if xerrors.Is(err, io.EOF) {
return nil
}
@@ -149,7 +147,7 @@ func Untar(directory string, archive []byte) error {
return err
}
// Max file size of 10MB.
_, err = io.CopyN(file, reader, (1<<20)*10)
_, err = io.CopyN(file, tarReader, (1<<20)*10)
if xerrors.Is(err, io.EOF) {
err = nil
}
provisionersdk/archive_test.go
+9 -5
View File
@@ -1,6 +1,8 @@
package provisionersdk_test
import (
"bytes"
"io"
"os"
"path/filepath"
"testing"
@@ -18,7 +20,7 @@ func TestTar(t *testing.T) {
file, err := os.CreateTemp(dir, "")
require.NoError(t, err)
_ = file.Close()
_, err = provisionersdk.Tar(dir, 1024)
err = provisionersdk.Tar(io.Discard, dir, 1024)
require.Error(t, err)
})
t.Run("Valid", func(t *testing.T) {
@@ -27,7 +29,7 @@ func TestTar(t *testing.T) {
file, err := os.CreateTemp(dir, "*.tf")
require.NoError(t, err)
_ = file.Close()
_, err = provisionersdk.Tar(dir, 1024)
err = provisionersdk.Tar(io.Discard, dir, 1024)
require.NoError(t, err)
})
t.Run("HiddenFiles", func(t *testing.T) {
@@ -71,10 +73,11 @@ func TestTar(t *testing.T) {
file.Name, err = filepath.Rel(dir, tmpFile.Name())
require.NoError(t, err)
}
content, err := provisionersdk.Tar(dir, 1024)
archive := new(bytes.Buffer)
err := provisionersdk.Tar(archive, dir, 1024)
require.NoError(t, err)
dir = t.TempDir()
err = provisionersdk.Untar(dir, content)
err = provisionersdk.Untar(dir, archive)
require.NoError(t, err)
for _, file := range files {
_, err = os.Stat(filepath.Join(dir, file.Name))
@@ -94,7 +97,8 @@ func TestUntar(t *testing.T) {
file, err := os.CreateTemp(dir, "*.tf")
require.NoError(t, err)
_ = file.Close()
archive, err := provisionersdk.Tar(dir, 1024)
archive := new(bytes.Buffer)
err = provisionersdk.Tar(archive, dir, 1024)
require.NoError(t, err)
dir = t.TempDir()
err = provisionersdk.Untar(dir, archive)