mirror of
https://github.com/coder/coder.git
synced 2026-09-24 15:04:27 +08:00
feat: collect workspace logs in support bundles (#26694)
Add workspace-side file collection to `coder support bundle` via repeatable --workspace-file flags. The agent resolves the requested paths or globs inside the remote workspace and streams back a tar with a manifest and the collected files; nothing is read from the machine running the command. - Add POST /api/v0/bundle-files to the agent's agentfiles package. - Expand env vars in the agent's environment; paths must then be absolute or start with ~/ (the agent user's home directory). - Support ** globs and tail oversized files. - Record requested patterns, per-path errors, truncation, and the applied limits in a manifest. - Unpack the archive into the bundle under agent/workspace_files/, recording dropped entries in collection_errors.txt. - Write a manifest-only archive marking collection as unsupported for agents that predate the endpoint. - Bound collection: 64 KB request body, 10000 files, 10 MiB per file, 100 MiB total including archive overhead, 110 MiB client-side read cap, 5 minute timeout. Closes #26020
This commit is contained in:
+1
-1
@@ -475,7 +475,7 @@ func (a *agent) init() {
|
||||
a.containerAPI = agentcontainers.NewAPI(a.logger.Named("containers"), containerAPIOpts...)
|
||||
|
||||
pathStore := agentgit.NewPathStore()
|
||||
a.filesAPI = agentfiles.NewAPI(a.logger.Named("files"), a.filesystem, pathStore)
|
||||
a.filesAPI = agentfiles.NewAPI(a.logger.Named("files"), a.filesystem, pathStore, agentfiles.WithEnvInfo(a.envInfo))
|
||||
a.processAPI = agentproc.NewAPI(a.logger.Named("processes"), a.execer, a.filesystem, pathStore, a.envInfo, a.updateCommandEnv, func() string {
|
||||
if m := a.manifest.Load(); m != nil {
|
||||
return m.Directory
|
||||
|
||||
@@ -9,8 +9,16 @@ import (
|
||||
)
|
||||
|
||||
// lexicalPath returns raw as a cleaned, absolute path with ~
|
||||
// expanded and symlinks left unresolved.
|
||||
// expanded against the current user's home and symlinks left
|
||||
// unresolved.
|
||||
func lexicalPath(raw string) (string, error) {
|
||||
return lexicalPathIn(os.UserHomeDir, raw)
|
||||
}
|
||||
|
||||
// lexicalPathIn is lexicalPath with home injected. home is called
|
||||
// only when a ~ prefix needs expanding, so an absolute path resolves
|
||||
// even when home is unavailable.
|
||||
func lexicalPathIn(home func() (string, error), raw string) (string, error) {
|
||||
raw = strings.TrimSpace(raw)
|
||||
if raw == "" {
|
||||
return "", xerrors.New("path is empty")
|
||||
@@ -18,14 +26,14 @@ func lexicalPath(raw string) (string, error) {
|
||||
|
||||
// ~user forms are intentionally unsupported.
|
||||
if raw == "~" || strings.HasPrefix(raw, "~/") {
|
||||
home, err := os.UserHomeDir()
|
||||
h, err := home()
|
||||
if err != nil {
|
||||
return "", xerrors.Errorf("expand home dir: %w", err)
|
||||
}
|
||||
if raw == "~" {
|
||||
raw = home
|
||||
raw = h
|
||||
} else {
|
||||
raw = filepath.Join(home, raw[2:])
|
||||
raw = filepath.Join(h, raw[2:])
|
||||
}
|
||||
}
|
||||
|
||||
@@ -40,7 +48,19 @@ func lexicalPath(raw string) (string, error) {
|
||||
// CanonicalizePath returns lexicalPath with symlinks resolved
|
||||
// when the target exists.
|
||||
func CanonicalizePath(raw string) (string, error) {
|
||||
cleaned, err := lexicalPath(raw)
|
||||
return resolveCanonicalPath(os.UserHomeDir, raw)
|
||||
}
|
||||
|
||||
// CanonicalizePathIn is CanonicalizePath with ~ expanded against
|
||||
// the given home directory instead of the current user's.
|
||||
func CanonicalizePathIn(home string, raw string) (string, error) {
|
||||
return resolveCanonicalPath(func() (string, error) { return home, nil }, raw)
|
||||
}
|
||||
|
||||
// resolveCanonicalPath implements both CanonicalizePath and
|
||||
// CanonicalizePathIn.
|
||||
func resolveCanonicalPath(home func() (string, error), raw string) (string, error) {
|
||||
cleaned, err := lexicalPathIn(home, raw)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
@@ -66,6 +66,17 @@ func TestCanonicalizePath_BareTildeExpandsToHome(t *testing.T) {
|
||||
require.Equal(t, want, got)
|
||||
}
|
||||
|
||||
func TestCanonicalizePathIn_ExpandsAgainstGivenHome(t *testing.T) {
|
||||
t.Parallel()
|
||||
home := testutil.TempDirResolved(t)
|
||||
got, err := agentcontext.CanonicalizePathIn(home, "~/.coder")
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, filepath.Join(home, ".coder"), got)
|
||||
|
||||
_, err = agentcontext.CanonicalizePathIn(home, "relative/path")
|
||||
require.Error(t, err)
|
||||
}
|
||||
|
||||
func TestCanonicalizePath_FollowsSymlinks(t *testing.T) {
|
||||
t.Parallel()
|
||||
if runtime.GOOS == "windows" {
|
||||
|
||||
+34
-7
@@ -8,20 +8,46 @@ import (
|
||||
|
||||
"cdr.dev/slog/v3"
|
||||
"github.com/coder/coder/v2/agent/agentgit"
|
||||
"github.com/coder/coder/v2/agent/usershell"
|
||||
"github.com/coder/coder/v2/codersdk/workspacesdk"
|
||||
)
|
||||
|
||||
// API exposes file-related operations performed through the agent.
|
||||
type API struct {
|
||||
logger slog.Logger
|
||||
filesystem afero.Fs
|
||||
pathStore *agentgit.PathStore
|
||||
logger slog.Logger
|
||||
filesystem afero.Fs
|
||||
pathStore *agentgit.PathStore
|
||||
envInfo usershell.EnvInfoer
|
||||
bundleFilesLimits workspacesdk.BundleFilesLimits
|
||||
}
|
||||
|
||||
func NewAPI(logger slog.Logger, filesystem afero.Fs, pathStore *agentgit.PathStore) *API {
|
||||
// Option configures the API.
|
||||
type Option func(*API)
|
||||
|
||||
// WithBundleFilesLimits overrides the bundle files collection limits.
|
||||
func WithBundleFilesLimits(limits workspacesdk.BundleFilesLimits) Option {
|
||||
return func(api *API) {
|
||||
api.bundleFilesLimits = limits
|
||||
}
|
||||
}
|
||||
|
||||
// WithEnvInfo overrides how the agent user's home directory is resolved.
|
||||
func WithEnvInfo(envInfo usershell.EnvInfoer) Option {
|
||||
return func(api *API) {
|
||||
api.envInfo = envInfo
|
||||
}
|
||||
}
|
||||
|
||||
func NewAPI(logger slog.Logger, filesystem afero.Fs, pathStore *agentgit.PathStore, opts ...Option) *API {
|
||||
api := &API{
|
||||
logger: logger,
|
||||
filesystem: filesystem,
|
||||
pathStore: pathStore,
|
||||
logger: logger,
|
||||
filesystem: filesystem,
|
||||
pathStore: pathStore,
|
||||
envInfo: usershell.SystemEnvInfo{},
|
||||
bundleFilesLimits: defaultBundleFilesLimits,
|
||||
}
|
||||
for _, opt := range opts {
|
||||
opt(api)
|
||||
}
|
||||
return api
|
||||
}
|
||||
@@ -36,6 +62,7 @@ func (api *API) Routes() http.Handler {
|
||||
r.Get("/read-file-lines", api.HandleReadFileLines)
|
||||
r.Post("/write-file", api.HandleWriteFile)
|
||||
r.Post("/edit-files", api.HandleEditFiles)
|
||||
r.Post("/bundle-files", api.HandleBundleFiles)
|
||||
|
||||
return r
|
||||
}
|
||||
|
||||
@@ -0,0 +1,394 @@
|
||||
package agentfiles
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"io/fs"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/bmatcuk/doublestar/v4"
|
||||
"golang.org/x/xerrors"
|
||||
|
||||
"cdr.dev/slog/v3"
|
||||
"github.com/coder/coder/v2/agent/agentcontext"
|
||||
"github.com/coder/coder/v2/coderd/httpapi"
|
||||
"github.com/coder/coder/v2/codersdk/workspacesdk"
|
||||
)
|
||||
|
||||
const (
|
||||
bundleFilesRequestMaxBytes = 64 * 1024
|
||||
// bundleFilesWriteTimeout gives slow links well over the server's 20s
|
||||
// WriteTimeout to stream the archive.
|
||||
bundleFilesWriteTimeout = 5 * time.Minute
|
||||
|
||||
tarBlockSize = 512
|
||||
)
|
||||
|
||||
// defaultBundleFilesLimits caps a single collection. Tar headers, block
|
||||
// padding, and manifest file entries are charged against MaxTotalBytes,
|
||||
// so it approximately bounds the response size.
|
||||
var defaultBundleFilesLimits = workspacesdk.BundleFilesLimits{
|
||||
MaxFiles: 10000,
|
||||
MaxBytesPerFile: 10 * 1024 * 1024,
|
||||
MaxTotalBytes: 100 * 1024 * 1024,
|
||||
}
|
||||
|
||||
var errBundleFilesFileLimit = xerrors.New("bundle files file count limit reached")
|
||||
|
||||
// HandleBundleFiles streams a tar archive of the requested workspace
|
||||
// files. Environment variables in paths are expanded in the agent's
|
||||
// environment; paths must then be absolute or start with ~/, which
|
||||
// resolves against the agent user's home directory.
|
||||
func (api *API) HandleBundleFiles(w http.ResponseWriter, r *http.Request) {
|
||||
var req workspacesdk.BundleFilesRequest
|
||||
r.Body = http.MaxBytesReader(w, r.Body, bundleFilesRequestMaxBytes)
|
||||
if !httpapi.Read(r.Context(), w, r, &req) {
|
||||
return
|
||||
}
|
||||
|
||||
home, err := api.envInfo.HomeDir()
|
||||
if err != nil {
|
||||
api.logger.Error(r.Context(), "get user home dir", slog.Error(err))
|
||||
httpapi.InternalServerError(w, xerrors.Errorf("get user home dir: %w", err))
|
||||
return
|
||||
}
|
||||
|
||||
if err := http.NewResponseController(w).SetWriteDeadline(time.Now().Add(bundleFilesWriteTimeout)); err != nil {
|
||||
api.logger.Warn(r.Context(), "extend bundle files write deadline", slog.Error(err))
|
||||
}
|
||||
|
||||
clientCtx := r.Context()
|
||||
ctx, cancel := context.WithTimeout(clientCtx, bundleFilesWriteTimeout)
|
||||
defer cancel()
|
||||
|
||||
w.Header().Set("Content-Type", "application/x-tar")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
if err := collectBundleFiles(ctx, clientCtx, home, req, w, api.bundleFilesLimits); err != nil {
|
||||
api.logger.Error(clientCtx, "collect bundle files", slog.Error(err))
|
||||
}
|
||||
}
|
||||
|
||||
// collectBundleFiles streams a tar with the requested files under files/
|
||||
// and a manifest.json describing the collection. Per-path problems are
|
||||
// recorded in the manifest, not fatal. ctx bounds the collection;
|
||||
// clientCtx is the request context.
|
||||
func collectBundleFiles(ctx, clientCtx context.Context, home string, req workspacesdk.BundleFilesRequest, w io.Writer, limits workspacesdk.BundleFilesLimits) error {
|
||||
manifest := workspacesdk.BundleFilesManifest{Requested: req.Paths, Limits: limits}
|
||||
paths := req.Paths
|
||||
|
||||
home, err := filepath.Abs(home)
|
||||
if err != nil {
|
||||
// Collect nothing; the archive still carries the manifest.
|
||||
appendManifestError(&manifest, "", "", "resolve home directory: "+err.Error())
|
||||
paths = nil
|
||||
}
|
||||
|
||||
tw := tar.NewWriter(w)
|
||||
c := &bundleFilesCollector{
|
||||
tw: tw,
|
||||
clientCtx: clientCtx,
|
||||
home: home,
|
||||
limits: limits,
|
||||
manifest: &manifest,
|
||||
seenPaths: map[string]struct{}{},
|
||||
remainingBytes: limits.MaxTotalBytes,
|
||||
}
|
||||
for _, requested := range paths {
|
||||
if !c.collectPattern(ctx, requested) {
|
||||
break
|
||||
}
|
||||
}
|
||||
if clientCtx.Err() != nil {
|
||||
// The client is gone; there is nobody to receive the manifest.
|
||||
return xerrors.Errorf("client disconnected: %w", clientCtx.Err())
|
||||
}
|
||||
|
||||
manifestJSON, err := json.MarshalIndent(manifest, "", " ")
|
||||
if err != nil {
|
||||
return xerrors.Errorf("marshal manifest: %w", err)
|
||||
}
|
||||
err = tw.WriteHeader(&tar.Header{
|
||||
Name: "manifest.json",
|
||||
Mode: 0o644,
|
||||
Size: int64(len(manifestJSON)),
|
||||
ModTime: time.Now(),
|
||||
})
|
||||
if err != nil {
|
||||
return xerrors.Errorf("create manifest in archive: %w", err)
|
||||
}
|
||||
if _, err := tw.Write(manifestJSON); err != nil {
|
||||
return xerrors.Errorf("write manifest: %w", err)
|
||||
}
|
||||
if err := tw.Close(); err != nil {
|
||||
return xerrors.Errorf("close archive: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// bundleFilesCollector streams matched files into an archive while
|
||||
// enforcing limits and recording per-path problems in the manifest.
|
||||
// Collect methods return false once a global limit ends collection.
|
||||
type bundleFilesCollector struct {
|
||||
tw *tar.Writer
|
||||
clientCtx context.Context
|
||||
home string
|
||||
limits workspacesdk.BundleFilesLimits
|
||||
manifest *workspacesdk.BundleFilesManifest
|
||||
seenPaths map[string]struct{}
|
||||
remainingBytes int64
|
||||
filesWritten int
|
||||
}
|
||||
|
||||
func (c *bundleFilesCollector) collectPattern(ctx context.Context, requested string) bool {
|
||||
if ctx.Err() != nil {
|
||||
return c.stopCanceled(requested, "")
|
||||
}
|
||||
if c.filesWritten >= c.limits.MaxFiles {
|
||||
return c.stop(requested, "", "file count limit reached")
|
||||
}
|
||||
|
||||
matches, matchesTruncated, err := bundleFileMatches(ctx, c.home, requested, c.limits.MaxFiles-c.filesWritten)
|
||||
if err != nil {
|
||||
if ctx.Err() != nil {
|
||||
return c.stopCanceled(requested, "")
|
||||
}
|
||||
appendManifestError(c.manifest, requested, "", err.Error())
|
||||
return true
|
||||
}
|
||||
if len(matches) == 0 {
|
||||
appendManifestError(c.manifest, requested, "", "no matches")
|
||||
return true
|
||||
}
|
||||
if matchesTruncated {
|
||||
c.manifest.Truncated = true
|
||||
appendManifestError(c.manifest, requested, "", "file count limit reached")
|
||||
}
|
||||
|
||||
for _, abs := range matches {
|
||||
if !c.collectFile(ctx, requested, abs) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func (c *bundleFilesCollector) collectFile(ctx context.Context, requested string, abs string) bool {
|
||||
if ctx.Err() != nil {
|
||||
return c.stopCanceled(requested, abs)
|
||||
}
|
||||
if c.filesWritten >= c.limits.MaxFiles {
|
||||
return c.stop(requested, abs, "file count limit reached")
|
||||
}
|
||||
// Each entry costs a tar header block before any data fits.
|
||||
if c.remainingBytes <= tarBlockSize {
|
||||
return c.stop(requested, abs, "total byte limit reached")
|
||||
}
|
||||
if _, ok := c.seenPaths[abs]; ok {
|
||||
return true
|
||||
}
|
||||
c.seenPaths[abs] = struct{}{}
|
||||
|
||||
// Stat before open: opening a FIFO would block. Stat follows symlinks,
|
||||
// so a directly requested symlink collects its target.
|
||||
info, err := os.Stat(abs)
|
||||
if err != nil {
|
||||
reason := "stat path: " + err.Error()
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
reason = "does not exist"
|
||||
}
|
||||
appendManifestError(c.manifest, requested, abs, reason)
|
||||
return true
|
||||
}
|
||||
if !info.Mode().IsRegular() {
|
||||
appendManifestError(c.manifest, requested, abs, "not a regular file: "+fileModeTypeName(info.Mode()))
|
||||
return true
|
||||
}
|
||||
|
||||
bytesToWrite := min(info.Size(), c.limits.MaxBytesPerFile, c.remainingBytes-tarBlockSize)
|
||||
entry := workspacesdk.BundleFilesManifestEntry{
|
||||
Requested: requested,
|
||||
Path: abs,
|
||||
ArchivePath: BundleFilesArchivePath(abs),
|
||||
Size: info.Size(),
|
||||
ModTime: info.ModTime(),
|
||||
BytesWritten: bytesToWrite,
|
||||
Truncated: bytesToWrite < info.Size(),
|
||||
}
|
||||
c.manifest.Truncated = c.manifest.Truncated || entry.Truncated
|
||||
if err := writeBundleFileEntry(c.tw, abs, entry); err != nil {
|
||||
appendManifestError(c.manifest, requested, abs, err.Error())
|
||||
return true
|
||||
}
|
||||
c.manifest.Files = append(c.manifest.Files, entry)
|
||||
// The last file may overshoot the budget by under a block; the bound
|
||||
// is approximate, not exact.
|
||||
entryJSON, _ := json.Marshal(entry)
|
||||
c.remainingBytes -= tarEntrySize(bytesToWrite) + int64(len(entryJSON))
|
||||
c.filesWritten++
|
||||
return true
|
||||
}
|
||||
|
||||
// stop marks the manifest truncated, records the reason, and halts
|
||||
// collection.
|
||||
func (c *bundleFilesCollector) stop(requested string, filePath string, reason string) bool {
|
||||
c.manifest.Truncated = true
|
||||
appendManifestError(c.manifest, requested, filePath, reason)
|
||||
return false
|
||||
}
|
||||
|
||||
// stopCanceled halts collection after the collection context ended: a
|
||||
// timeout is recorded in the manifest and the archive is finished, while a
|
||||
// client disconnect makes the caller abort without a manifest.
|
||||
func (c *bundleFilesCollector) stopCanceled(requested string, filePath string) bool {
|
||||
if c.clientCtx.Err() != nil {
|
||||
return false
|
||||
}
|
||||
return c.stop(requested, filePath, "exceeded maximum collection time")
|
||||
}
|
||||
|
||||
// bundleFileMatches expands requested against home and returns matching
|
||||
// cleaned absolute paths. Non-glob paths return a single candidate without
|
||||
// checking existence; the caller reports missing files on stat.
|
||||
func bundleFileMatches(ctx context.Context, home string, requested string, maxMatches int) ([]string, bool, error) {
|
||||
// Env vars expand from the agent environment and ~ resolves against
|
||||
// the agent home, matching the agent's expandPathToAbs. Glob patterns
|
||||
// never exist on disk, so canonicalization keeps them lexical.
|
||||
abs, err := agentcontext.CanonicalizePathIn(home, os.ExpandEnv(requested))
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
if !strings.ContainsAny(abs, "*?{[") {
|
||||
return []string{abs}, false, nil
|
||||
}
|
||||
|
||||
base, pattern := doublestar.SplitPattern(filepath.ToSlash(abs))
|
||||
matches := make([]string, 0, min(maxMatches, 64))
|
||||
// WithNoFollow avoids symlink cycles. Checking the limit before the
|
||||
// append keeps matches from growing past maxMatches.
|
||||
err = doublestar.GlobWalk(bundleFilesFS{ctx: ctx, fsys: os.DirFS(base)}, pattern, func(match string, _ fs.DirEntry) error {
|
||||
if len(matches) >= maxMatches {
|
||||
return errBundleFilesFileLimit
|
||||
}
|
||||
matches = append(matches, filepath.Join(base, filepath.FromSlash(match)))
|
||||
return nil
|
||||
}, doublestar.WithFilesOnly(), doublestar.WithNoFollow())
|
||||
matchesTruncated := errors.Is(err, errBundleFilesFileLimit)
|
||||
if err != nil && !matchesTruncated {
|
||||
return nil, false, xerrors.Errorf("glob pattern: %w", err)
|
||||
}
|
||||
// doublestar does not guarantee ordering, so sort for a deterministic
|
||||
// archive.
|
||||
slices.Sort(matches)
|
||||
return matches, matchesTruncated, nil
|
||||
}
|
||||
|
||||
// bundleFilesFS cancels a glob walk once the request context ends. Only
|
||||
// Open is implemented; the fs.ReadDir and fs.Stat helpers fall back to it,
|
||||
// so every filesystem operation of the walk passes the context check.
|
||||
type bundleFilesFS struct {
|
||||
ctx context.Context
|
||||
fsys fs.FS
|
||||
}
|
||||
|
||||
func (f bundleFilesFS) Open(name string) (fs.File, error) {
|
||||
if err := f.ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return f.fsys.Open(name)
|
||||
}
|
||||
|
||||
// BundleFilesArchivePath maps a cleaned absolute path to its archive entry
|
||||
// name: files/ plus the path with the leading separator trimmed and any
|
||||
// Windows drive colon dropped, keeping the name fs.ValidPath-safe.
|
||||
func BundleFilesArchivePath(abs string) string {
|
||||
p := strings.TrimPrefix(filepath.ToSlash(abs), "/")
|
||||
if len(p) >= 2 && p[1] == ':' {
|
||||
p = p[:1] + p[2:]
|
||||
}
|
||||
return "files/" + p
|
||||
}
|
||||
|
||||
// fileModeTypeName names the type of a non-regular file.
|
||||
func fileModeTypeName(mode fs.FileMode) string {
|
||||
switch {
|
||||
case mode.IsDir():
|
||||
return "directory"
|
||||
case mode&fs.ModeSymlink != 0:
|
||||
return "symlink"
|
||||
case mode&fs.ModeNamedPipe != 0:
|
||||
return "named pipe"
|
||||
case mode&fs.ModeSocket != 0:
|
||||
return "socket"
|
||||
case mode&fs.ModeDevice != 0, mode&fs.ModeCharDevice != 0:
|
||||
return "device"
|
||||
default:
|
||||
return "irregular file"
|
||||
}
|
||||
}
|
||||
|
||||
// writeBundleFileEntry writes the last entry.BytesWritten bytes of the
|
||||
// file at abs to the archive at entry.ArchivePath. A file that shrinks
|
||||
// after stat is zero-padded to the declared size, since a short entry
|
||||
// would corrupt every entry after it; the short read is still an error.
|
||||
func writeBundleFileEntry(tw *tar.Writer, abs string, entry workspacesdk.BundleFilesManifestEntry) error {
|
||||
f, err := os.Open(abs)
|
||||
if err != nil {
|
||||
return xerrors.Errorf("open file: %w", err)
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
if entry.BytesWritten < entry.Size {
|
||||
if _, err := f.Seek(entry.Size-entry.BytesWritten, io.SeekStart); err != nil {
|
||||
return xerrors.Errorf("seek tail: %w", err)
|
||||
}
|
||||
}
|
||||
err = tw.WriteHeader(&tar.Header{
|
||||
Name: entry.ArchivePath,
|
||||
Mode: 0o644,
|
||||
Size: entry.BytesWritten,
|
||||
ModTime: entry.ModTime,
|
||||
})
|
||||
if err != nil {
|
||||
return xerrors.Errorf("create archive entry: %w", err)
|
||||
}
|
||||
n, err := io.Copy(tw, io.LimitReader(f, entry.BytesWritten))
|
||||
if err == nil && n < entry.BytesWritten {
|
||||
err = io.ErrUnexpectedEOF
|
||||
}
|
||||
if err != nil {
|
||||
if _, padErr := io.CopyN(tw, zeroReader{}, entry.BytesWritten-n); padErr != nil {
|
||||
return xerrors.Errorf("pad short entry: %w", padErr)
|
||||
}
|
||||
return xerrors.Errorf("copy file: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// tarEntrySize returns the archive bytes a file entry consumes: one
|
||||
// header block plus the data rounded up to whole blocks.
|
||||
func tarEntrySize(dataBytes int64) int64 {
|
||||
return tarBlockSize + (dataBytes+tarBlockSize-1)/tarBlockSize*tarBlockSize
|
||||
}
|
||||
|
||||
type zeroReader struct{}
|
||||
|
||||
func (zeroReader) Read(p []byte) (int, error) {
|
||||
clear(p)
|
||||
return len(p), nil
|
||||
}
|
||||
|
||||
func appendManifestError(m *workspacesdk.BundleFilesManifest, requested string, filePath string, reason string) {
|
||||
m.Errors = append(m.Errors, workspacesdk.BundleFilesManifestError{
|
||||
Requested: requested,
|
||||
Path: filePath,
|
||||
Reason: reason,
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,246 @@
|
||||
package agentfiles_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/spf13/afero"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"cdr.dev/slog/v3"
|
||||
"cdr.dev/slog/v3/sloggers/slogtest"
|
||||
"github.com/coder/coder/v2/agent/agentfiles"
|
||||
"github.com/coder/coder/v2/agent/usershell"
|
||||
"github.com/coder/coder/v2/codersdk/workspacesdk"
|
||||
"github.com/coder/coder/v2/testutil"
|
||||
)
|
||||
|
||||
func TestBundleFilesCollectsExpandedPathsAndGlobs(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
home := testutil.TempDirResolved(t)
|
||||
writeBundleSourceFile(t, home, ".vscode-server/data/logs/20260706T101112/remoteagent.log", "remote agent")
|
||||
writeBundleSourceFile(t, home, ".vscode-server/data/logs/20260706T101112/exthost1/exthost.log", "exthost")
|
||||
writeBundleSourceFile(t, home, ".vscode-server/data/logs/20260706T101112/exthost1/output.txt", "skip")
|
||||
writeBundleSourceFile(t, home, ".local/share/code-server/coder-logs/app.log", "code server log")
|
||||
writeBundleSourceFile(t, home, ".cache/JetBrains/RemoteDev/dist/241.15989.150/log/idea.log", "idea log")
|
||||
writeBundleSourceFile(t, home, "brace/one.log", "one")
|
||||
writeBundleSourceFile(t, home, "brace/two.txt", "two")
|
||||
writeBundleSourceFile(t, home, "brace/skip.json", "skip")
|
||||
|
||||
entries := readBundleFilesArchive(t, requestBundleFiles(t, newBundleFilesHandler(t, home), []string{
|
||||
filepath.Join(home, ".vscode-server/data/logs/20260706T101112/remoteagent.log"),
|
||||
"~/.vscode-server/data/logs/**/*.log",
|
||||
"~/.local/share/code-server/coder-logs/app.log",
|
||||
"~/.cache/JetBrains/RemoteDev/dist/*/log/idea.log",
|
||||
"~/brace/*.{log,txt}",
|
||||
}))
|
||||
|
||||
requireBundleEntry(t, entries, home, ".vscode-server/data/logs/20260706T101112/remoteagent.log", "remote agent")
|
||||
requireBundleEntry(t, entries, home, ".vscode-server/data/logs/20260706T101112/exthost1/exthost.log", "exthost")
|
||||
requireBundleEntry(t, entries, home, ".local/share/code-server/coder-logs/app.log", "code server log")
|
||||
requireBundleEntry(t, entries, home, ".cache/JetBrains/RemoteDev/dist/241.15989.150/log/idea.log", "idea log")
|
||||
requireBundleEntry(t, entries, home, "brace/one.log", "one")
|
||||
requireBundleEntry(t, entries, home, "brace/two.txt", "two")
|
||||
require.NotContains(t, entries.files, bundleArchivePath(t, home, ".vscode-server/data/logs/20260706T101112/exthost1/output.txt"))
|
||||
require.NotContains(t, entries.files, bundleArchivePath(t, home, "brace/skip.json"))
|
||||
require.Empty(t, entries.manifest.Errors)
|
||||
// remoteagent.log matches both the absolute path and the ** glob; it
|
||||
// must be archived once.
|
||||
require.Len(t, entries.manifest.Files, 6)
|
||||
}
|
||||
|
||||
func TestBundleFilesCollectsAbsolutePathsOutsideHome(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
home := testutil.TempDirResolved(t)
|
||||
outside := testutil.TempDirResolved(t)
|
||||
writeBundleSourceFile(t, outside, "service.log", "outside log")
|
||||
writeBundleSourceFile(t, outside, "glob/a.log", "glob a")
|
||||
|
||||
entries := readBundleFilesArchive(t, requestBundleFiles(t, newBundleFilesHandler(t, home), []string{
|
||||
filepath.Join(outside, "service.log"),
|
||||
filepath.Join(outside, "glob", "*.log"),
|
||||
}))
|
||||
|
||||
requireBundleEntry(t, entries, outside, "service.log", "outside log")
|
||||
requireBundleEntry(t, entries, outside, "glob/a.log", "glob a")
|
||||
require.Empty(t, entries.manifest.Errors)
|
||||
require.Len(t, entries.manifest.Files, 2)
|
||||
}
|
||||
|
||||
func TestBundleFilesRejectedPathsAreNonFatal(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
home := testutil.TempDirResolved(t)
|
||||
writeBundleSourceFile(t, home, "kept.log", "kept")
|
||||
require.NoError(t, os.MkdirAll(filepath.Join(home, "somedir"), 0o700))
|
||||
|
||||
entries := readBundleFilesArchive(t, requestBundleFiles(t, newBundleFilesHandler(t, home), []string{
|
||||
"~/kept.log",
|
||||
"relative.log",
|
||||
"~/missing.log",
|
||||
"~/somedir",
|
||||
"~/no-matches/**/*.log",
|
||||
}))
|
||||
|
||||
requireBundleEntry(t, entries, home, "kept.log", "kept")
|
||||
require.Len(t, entries.manifest.Files, 1)
|
||||
requireBundleFilesManifestErrors(t, entries.manifest.Errors,
|
||||
"is not absolute",
|
||||
"does not exist",
|
||||
"not a regular file: directory",
|
||||
"no matches",
|
||||
)
|
||||
}
|
||||
|
||||
func TestBundleFilesTailBytesTruncation(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
home := testutil.TempDirResolved(t)
|
||||
writeBundleSourceFile(t, home, "large.log", "0123456789")
|
||||
|
||||
entries := readBundleFilesArchive(t, requestBundleFiles(t, newBundleFilesHandler(t, home, agentfiles.WithBundleFilesLimits(workspacesdk.BundleFilesLimits{
|
||||
MaxFiles: 10,
|
||||
MaxBytesPerFile: 4,
|
||||
MaxTotalBytes: 100 * 1024,
|
||||
})), []string{"~/large.log"}))
|
||||
|
||||
requireBundleEntry(t, entries, home, "large.log", "6789")
|
||||
require.Len(t, entries.manifest.Files, 1)
|
||||
require.True(t, entries.manifest.Files[0].Truncated)
|
||||
require.Equal(t, int64(10), entries.manifest.Files[0].Size)
|
||||
require.Equal(t, int64(4), entries.manifest.Files[0].BytesWritten)
|
||||
}
|
||||
|
||||
func TestBundleFilesFileAndByteLimits(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
home := testutil.TempDirResolved(t)
|
||||
writeBundleSourceFile(t, home, "one.log", "1111")
|
||||
writeBundleSourceFile(t, home, "two.log", "2222")
|
||||
writeBundleSourceFile(t, home, "three.log", "3333")
|
||||
|
||||
entries := readBundleFilesArchive(t, requestBundleFiles(t, newBundleFilesHandler(t, home, agentfiles.WithBundleFilesLimits(workspacesdk.BundleFilesLimits{
|
||||
MaxFiles: 1,
|
||||
MaxBytesPerFile: 100,
|
||||
// One 512-byte tar header plus 3 data bytes: the first file is
|
||||
// truncated to 3 bytes by the total budget.
|
||||
MaxTotalBytes: 515,
|
||||
})), []string{"~/*.log"}))
|
||||
|
||||
require.Len(t, entries.files, 1)
|
||||
require.True(t, entries.manifest.Truncated)
|
||||
require.Equal(t, int64(3), entries.manifest.Files[0].BytesWritten)
|
||||
// The glob walk itself stops at the file cap.
|
||||
requireBundleFilesManifestErrors(t, entries.manifest.Errors, "file count limit reached")
|
||||
}
|
||||
|
||||
func TestBundleFilesDedupeByCleanedPath(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
home := testutil.TempDirResolved(t)
|
||||
writeBundleSourceFile(t, home, "dup.log", "one")
|
||||
writeBundleSourceFile(t, home, "other.log", "two")
|
||||
|
||||
entries := readBundleFilesArchive(t, requestBundleFiles(t, newBundleFilesHandler(t, home), []string{
|
||||
"~/dup.log",
|
||||
"~/./dup.log",
|
||||
filepath.Join(home, "somedir", "..", "dup.log"),
|
||||
"~/other.log",
|
||||
}))
|
||||
|
||||
requireBundleEntry(t, entries, home, "dup.log", "one")
|
||||
requireBundleEntry(t, entries, home, "other.log", "two")
|
||||
require.Len(t, entries.manifest.Files, 2)
|
||||
}
|
||||
|
||||
// fakeBundleEnvInfo overrides the home directory so tests can point path
|
||||
// expansion at a temp dir.
|
||||
type fakeBundleEnvInfo struct {
|
||||
usershell.SystemEnvInfo
|
||||
home string
|
||||
}
|
||||
|
||||
func (e fakeBundleEnvInfo) HomeDir() (string, error) {
|
||||
return e.home, nil
|
||||
}
|
||||
|
||||
func newBundleFilesHandler(t *testing.T, home string, opts ...agentfiles.Option) http.Handler {
|
||||
t.Helper()
|
||||
|
||||
logger := slogtest.Make(t, &slogtest.Options{IgnoreErrors: true}).Leveled(slog.LevelDebug)
|
||||
opts = append([]agentfiles.Option{agentfiles.WithEnvInfo(fakeBundleEnvInfo{home: home})}, opts...)
|
||||
return agentfiles.NewAPI(logger, afero.NewOsFs(), nil, opts...).Routes()
|
||||
}
|
||||
|
||||
func requestBundleFiles(t *testing.T, handler http.Handler, paths []string) []byte {
|
||||
t.Helper()
|
||||
|
||||
body, err := json.Marshal(workspacesdk.BundleFilesRequest{Paths: paths})
|
||||
require.NoError(t, err)
|
||||
req := httptest.NewRequest(http.MethodPost, "/bundle-files", bytes.NewReader(body))
|
||||
res := httptest.NewRecorder()
|
||||
handler.ServeHTTP(res, req)
|
||||
|
||||
require.Equal(t, http.StatusOK, res.Code)
|
||||
require.Equal(t, "application/x-tar", res.Header().Get("Content-Type"))
|
||||
return res.Body.Bytes()
|
||||
}
|
||||
|
||||
func writeBundleSourceFile(t *testing.T, dir string, rel string, content string) {
|
||||
t.Helper()
|
||||
|
||||
path := filepath.Join(dir, filepath.FromSlash(rel))
|
||||
require.NoError(t, os.MkdirAll(filepath.Dir(path), 0o700))
|
||||
require.NoError(t, os.WriteFile(path, []byte(content), 0o600))
|
||||
}
|
||||
|
||||
// bundleArchivePath returns the expected archive entry name for the file
|
||||
// at dir/rel.
|
||||
func bundleArchivePath(t *testing.T, dir string, rel string) string {
|
||||
t.Helper()
|
||||
|
||||
return agentfiles.BundleFilesArchivePath(filepath.Join(dir, filepath.FromSlash(rel)))
|
||||
}
|
||||
|
||||
func requireBundleEntry(t *testing.T, entries bundleFilesArchive, dir string, rel string, content string) {
|
||||
t.Helper()
|
||||
|
||||
require.Equal(t, content, string(entries.files[bundleArchivePath(t, dir, rel)]))
|
||||
}
|
||||
|
||||
type bundleFilesArchive struct {
|
||||
manifest workspacesdk.BundleFilesManifest
|
||||
files map[string][]byte
|
||||
}
|
||||
|
||||
func readBundleFilesArchive(t *testing.T, data []byte) bundleFilesArchive {
|
||||
t.Helper()
|
||||
|
||||
entries := bundleFilesArchive{files: testutil.ReadTar(t, data)}
|
||||
manifestJSON, ok := entries.files["manifest.json"]
|
||||
require.True(t, ok, "archive should contain manifest.json")
|
||||
delete(entries.files, "manifest.json")
|
||||
require.NoError(t, json.Unmarshal(manifestJSON, &entries.manifest))
|
||||
require.NotEmpty(t, entries.manifest.Requested)
|
||||
return entries
|
||||
}
|
||||
|
||||
func requireBundleFilesManifestErrors(t *testing.T, errs []workspacesdk.BundleFilesManifestError, contains ...string) {
|
||||
t.Helper()
|
||||
|
||||
for _, want := range contains {
|
||||
found := slices.ContainsFunc(errs, func(e workspacesdk.BundleFilesManifestError) bool {
|
||||
return strings.Contains(e.Reason, want)
|
||||
})
|
||||
require.Truef(t, found, "expected manifest error containing %q in %#v", want, errs)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user