feat: collect workspace logs in support bundles (#26694)

Add workspace-side file collection to `coder support bundle` via
repeatable --workspace-file flags. The agent resolves the requested
paths or globs inside the remote workspace and streams back a tar with
a manifest and the collected files; nothing is read from the machine
running the command.

- Add POST /api/v0/bundle-files to the agent's agentfiles package.
- Expand env vars in the agent's environment; paths must then be
  absolute or start with ~/ (the agent user's home directory).
- Support ** globs and tail oversized files.
- Record requested patterns, per-path errors, truncation, and the
  applied limits in a manifest.
- Unpack the archive into the bundle under agent/workspace_files/,
  recording dropped entries in collection_errors.txt.
- Write a manifest-only archive marking collection as unsupported for
  agents that predate the endpoint.
- Bound collection: 64 KB request body, 10000 files, 10 MiB per file,
  100 MiB total including archive overhead, 110 MiB client-side read
  cap, 5 minute timeout.

Closes #26020
This commit is contained in:
Ehab Younes
2026-07-16 13:00:32 +03:00
committed by GitHub
parent f997afa220
commit 35ade9e3d2
19 changed files with 1390 additions and 71 deletions
+1 -1
View File
@@ -475,7 +475,7 @@ func (a *agent) init() {
a.containerAPI = agentcontainers.NewAPI(a.logger.Named("containers"), containerAPIOpts...)
pathStore := agentgit.NewPathStore()
a.filesAPI = agentfiles.NewAPI(a.logger.Named("files"), a.filesystem, pathStore)
a.filesAPI = agentfiles.NewAPI(a.logger.Named("files"), a.filesystem, pathStore, agentfiles.WithEnvInfo(a.envInfo))
a.processAPI = agentproc.NewAPI(a.logger.Named("processes"), a.execer, a.filesystem, pathStore, a.envInfo, a.updateCommandEnv, func() string {
if m := a.manifest.Load(); m != nil {
return m.Directory
+25 -5
View File
@@ -9,8 +9,16 @@ import (
)
// lexicalPath returns raw as a cleaned, absolute path with ~
// expanded and symlinks left unresolved.
// expanded against the current user's home and symlinks left
// unresolved.
func lexicalPath(raw string) (string, error) {
return lexicalPathIn(os.UserHomeDir, raw)
}
// lexicalPathIn is lexicalPath with home injected. home is called
// only when a ~ prefix needs expanding, so an absolute path resolves
// even when home is unavailable.
func lexicalPathIn(home func() (string, error), raw string) (string, error) {
raw = strings.TrimSpace(raw)
if raw == "" {
return "", xerrors.New("path is empty")
@@ -18,14 +26,14 @@ func lexicalPath(raw string) (string, error) {
// ~user forms are intentionally unsupported.
if raw == "~" || strings.HasPrefix(raw, "~/") {
home, err := os.UserHomeDir()
h, err := home()
if err != nil {
return "", xerrors.Errorf("expand home dir: %w", err)
}
if raw == "~" {
raw = home
raw = h
} else {
raw = filepath.Join(home, raw[2:])
raw = filepath.Join(h, raw[2:])
}
}
@@ -40,7 +48,19 @@ func lexicalPath(raw string) (string, error) {
// CanonicalizePath returns lexicalPath with symlinks resolved
// when the target exists.
func CanonicalizePath(raw string) (string, error) {
cleaned, err := lexicalPath(raw)
return resolveCanonicalPath(os.UserHomeDir, raw)
}
// CanonicalizePathIn is CanonicalizePath with ~ expanded against
// the given home directory instead of the current user's.
func CanonicalizePathIn(home string, raw string) (string, error) {
return resolveCanonicalPath(func() (string, error) { return home, nil }, raw)
}
// resolveCanonicalPath implements both CanonicalizePath and
// CanonicalizePathIn.
func resolveCanonicalPath(home func() (string, error), raw string) (string, error) {
cleaned, err := lexicalPathIn(home, raw)
if err != nil {
return "", err
}
+11
View File
@@ -66,6 +66,17 @@ func TestCanonicalizePath_BareTildeExpandsToHome(t *testing.T) {
require.Equal(t, want, got)
}
func TestCanonicalizePathIn_ExpandsAgainstGivenHome(t *testing.T) {
t.Parallel()
home := testutil.TempDirResolved(t)
got, err := agentcontext.CanonicalizePathIn(home, "~/.coder")
require.NoError(t, err)
require.Equal(t, filepath.Join(home, ".coder"), got)
_, err = agentcontext.CanonicalizePathIn(home, "relative/path")
require.Error(t, err)
}
func TestCanonicalizePath_FollowsSymlinks(t *testing.T) {
t.Parallel()
if runtime.GOOS == "windows" {
+34 -7
View File
@@ -8,20 +8,46 @@ import (
"cdr.dev/slog/v3"
"github.com/coder/coder/v2/agent/agentgit"
"github.com/coder/coder/v2/agent/usershell"
"github.com/coder/coder/v2/codersdk/workspacesdk"
)
// API exposes file-related operations performed through the agent.
type API struct {
logger slog.Logger
filesystem afero.Fs
pathStore *agentgit.PathStore
logger slog.Logger
filesystem afero.Fs
pathStore *agentgit.PathStore
envInfo usershell.EnvInfoer
bundleFilesLimits workspacesdk.BundleFilesLimits
}
func NewAPI(logger slog.Logger, filesystem afero.Fs, pathStore *agentgit.PathStore) *API {
// Option configures the API.
type Option func(*API)
// WithBundleFilesLimits overrides the bundle files collection limits.
func WithBundleFilesLimits(limits workspacesdk.BundleFilesLimits) Option {
return func(api *API) {
api.bundleFilesLimits = limits
}
}
// WithEnvInfo overrides how the agent user's home directory is resolved.
func WithEnvInfo(envInfo usershell.EnvInfoer) Option {
return func(api *API) {
api.envInfo = envInfo
}
}
func NewAPI(logger slog.Logger, filesystem afero.Fs, pathStore *agentgit.PathStore, opts ...Option) *API {
api := &API{
logger: logger,
filesystem: filesystem,
pathStore: pathStore,
logger: logger,
filesystem: filesystem,
pathStore: pathStore,
envInfo: usershell.SystemEnvInfo{},
bundleFilesLimits: defaultBundleFilesLimits,
}
for _, opt := range opts {
opt(api)
}
return api
}
@@ -36,6 +62,7 @@ func (api *API) Routes() http.Handler {
r.Get("/read-file-lines", api.HandleReadFileLines)
r.Post("/write-file", api.HandleWriteFile)
r.Post("/edit-files", api.HandleEditFiles)
r.Post("/bundle-files", api.HandleBundleFiles)
return r
}
+394
View File
@@ -0,0 +1,394 @@
package agentfiles
import (
"archive/tar"
"context"
"encoding/json"
"errors"
"io"
"io/fs"
"net/http"
"os"
"path/filepath"
"slices"
"strings"
"time"
"github.com/bmatcuk/doublestar/v4"
"golang.org/x/xerrors"
"cdr.dev/slog/v3"
"github.com/coder/coder/v2/agent/agentcontext"
"github.com/coder/coder/v2/coderd/httpapi"
"github.com/coder/coder/v2/codersdk/workspacesdk"
)
const (
bundleFilesRequestMaxBytes = 64 * 1024
// bundleFilesWriteTimeout gives slow links well over the server's 20s
// WriteTimeout to stream the archive.
bundleFilesWriteTimeout = 5 * time.Minute
tarBlockSize = 512
)
// defaultBundleFilesLimits caps a single collection. Tar headers, block
// padding, and manifest file entries are charged against MaxTotalBytes,
// so it approximately bounds the response size.
var defaultBundleFilesLimits = workspacesdk.BundleFilesLimits{
MaxFiles: 10000,
MaxBytesPerFile: 10 * 1024 * 1024,
MaxTotalBytes: 100 * 1024 * 1024,
}
var errBundleFilesFileLimit = xerrors.New("bundle files file count limit reached")
// HandleBundleFiles streams a tar archive of the requested workspace
// files. Environment variables in paths are expanded in the agent's
// environment; paths must then be absolute or start with ~/, which
// resolves against the agent user's home directory.
func (api *API) HandleBundleFiles(w http.ResponseWriter, r *http.Request) {
var req workspacesdk.BundleFilesRequest
r.Body = http.MaxBytesReader(w, r.Body, bundleFilesRequestMaxBytes)
if !httpapi.Read(r.Context(), w, r, &req) {
return
}
home, err := api.envInfo.HomeDir()
if err != nil {
api.logger.Error(r.Context(), "get user home dir", slog.Error(err))
httpapi.InternalServerError(w, xerrors.Errorf("get user home dir: %w", err))
return
}
if err := http.NewResponseController(w).SetWriteDeadline(time.Now().Add(bundleFilesWriteTimeout)); err != nil {
api.logger.Warn(r.Context(), "extend bundle files write deadline", slog.Error(err))
}
clientCtx := r.Context()
ctx, cancel := context.WithTimeout(clientCtx, bundleFilesWriteTimeout)
defer cancel()
w.Header().Set("Content-Type", "application/x-tar")
w.WriteHeader(http.StatusOK)
if err := collectBundleFiles(ctx, clientCtx, home, req, w, api.bundleFilesLimits); err != nil {
api.logger.Error(clientCtx, "collect bundle files", slog.Error(err))
}
}
// collectBundleFiles streams a tar with the requested files under files/
// and a manifest.json describing the collection. Per-path problems are
// recorded in the manifest, not fatal. ctx bounds the collection;
// clientCtx is the request context.
func collectBundleFiles(ctx, clientCtx context.Context, home string, req workspacesdk.BundleFilesRequest, w io.Writer, limits workspacesdk.BundleFilesLimits) error {
manifest := workspacesdk.BundleFilesManifest{Requested: req.Paths, Limits: limits}
paths := req.Paths
home, err := filepath.Abs(home)
if err != nil {
// Collect nothing; the archive still carries the manifest.
appendManifestError(&manifest, "", "", "resolve home directory: "+err.Error())
paths = nil
}
tw := tar.NewWriter(w)
c := &bundleFilesCollector{
tw: tw,
clientCtx: clientCtx,
home: home,
limits: limits,
manifest: &manifest,
seenPaths: map[string]struct{}{},
remainingBytes: limits.MaxTotalBytes,
}
for _, requested := range paths {
if !c.collectPattern(ctx, requested) {
break
}
}
if clientCtx.Err() != nil {
// The client is gone; there is nobody to receive the manifest.
return xerrors.Errorf("client disconnected: %w", clientCtx.Err())
}
manifestJSON, err := json.MarshalIndent(manifest, "", " ")
if err != nil {
return xerrors.Errorf("marshal manifest: %w", err)
}
err = tw.WriteHeader(&tar.Header{
Name: "manifest.json",
Mode: 0o644,
Size: int64(len(manifestJSON)),
ModTime: time.Now(),
})
if err != nil {
return xerrors.Errorf("create manifest in archive: %w", err)
}
if _, err := tw.Write(manifestJSON); err != nil {
return xerrors.Errorf("write manifest: %w", err)
}
if err := tw.Close(); err != nil {
return xerrors.Errorf("close archive: %w", err)
}
return nil
}
// bundleFilesCollector streams matched files into an archive while
// enforcing limits and recording per-path problems in the manifest.
// Collect methods return false once a global limit ends collection.
type bundleFilesCollector struct {
tw *tar.Writer
clientCtx context.Context
home string
limits workspacesdk.BundleFilesLimits
manifest *workspacesdk.BundleFilesManifest
seenPaths map[string]struct{}
remainingBytes int64
filesWritten int
}
func (c *bundleFilesCollector) collectPattern(ctx context.Context, requested string) bool {
if ctx.Err() != nil {
return c.stopCanceled(requested, "")
}
if c.filesWritten >= c.limits.MaxFiles {
return c.stop(requested, "", "file count limit reached")
}
matches, matchesTruncated, err := bundleFileMatches(ctx, c.home, requested, c.limits.MaxFiles-c.filesWritten)
if err != nil {
if ctx.Err() != nil {
return c.stopCanceled(requested, "")
}
appendManifestError(c.manifest, requested, "", err.Error())
return true
}
if len(matches) == 0 {
appendManifestError(c.manifest, requested, "", "no matches")
return true
}
if matchesTruncated {
c.manifest.Truncated = true
appendManifestError(c.manifest, requested, "", "file count limit reached")
}
for _, abs := range matches {
if !c.collectFile(ctx, requested, abs) {
return false
}
}
return true
}
func (c *bundleFilesCollector) collectFile(ctx context.Context, requested string, abs string) bool {
if ctx.Err() != nil {
return c.stopCanceled(requested, abs)
}
if c.filesWritten >= c.limits.MaxFiles {
return c.stop(requested, abs, "file count limit reached")
}
// Each entry costs a tar header block before any data fits.
if c.remainingBytes <= tarBlockSize {
return c.stop(requested, abs, "total byte limit reached")
}
if _, ok := c.seenPaths[abs]; ok {
return true
}
c.seenPaths[abs] = struct{}{}
// Stat before open: opening a FIFO would block. Stat follows symlinks,
// so a directly requested symlink collects its target.
info, err := os.Stat(abs)
if err != nil {
reason := "stat path: " + err.Error()
if errors.Is(err, fs.ErrNotExist) {
reason = "does not exist"
}
appendManifestError(c.manifest, requested, abs, reason)
return true
}
if !info.Mode().IsRegular() {
appendManifestError(c.manifest, requested, abs, "not a regular file: "+fileModeTypeName(info.Mode()))
return true
}
bytesToWrite := min(info.Size(), c.limits.MaxBytesPerFile, c.remainingBytes-tarBlockSize)
entry := workspacesdk.BundleFilesManifestEntry{
Requested: requested,
Path: abs,
ArchivePath: BundleFilesArchivePath(abs),
Size: info.Size(),
ModTime: info.ModTime(),
BytesWritten: bytesToWrite,
Truncated: bytesToWrite < info.Size(),
}
c.manifest.Truncated = c.manifest.Truncated || entry.Truncated
if err := writeBundleFileEntry(c.tw, abs, entry); err != nil {
appendManifestError(c.manifest, requested, abs, err.Error())
return true
}
c.manifest.Files = append(c.manifest.Files, entry)
// The last file may overshoot the budget by under a block; the bound
// is approximate, not exact.
entryJSON, _ := json.Marshal(entry)
c.remainingBytes -= tarEntrySize(bytesToWrite) + int64(len(entryJSON))
c.filesWritten++
return true
}
// stop marks the manifest truncated, records the reason, and halts
// collection.
func (c *bundleFilesCollector) stop(requested string, filePath string, reason string) bool {
c.manifest.Truncated = true
appendManifestError(c.manifest, requested, filePath, reason)
return false
}
// stopCanceled halts collection after the collection context ended: a
// timeout is recorded in the manifest and the archive is finished, while a
// client disconnect makes the caller abort without a manifest.
func (c *bundleFilesCollector) stopCanceled(requested string, filePath string) bool {
if c.clientCtx.Err() != nil {
return false
}
return c.stop(requested, filePath, "exceeded maximum collection time")
}
// bundleFileMatches expands requested against home and returns matching
// cleaned absolute paths. Non-glob paths return a single candidate without
// checking existence; the caller reports missing files on stat.
func bundleFileMatches(ctx context.Context, home string, requested string, maxMatches int) ([]string, bool, error) {
// Env vars expand from the agent environment and ~ resolves against
// the agent home, matching the agent's expandPathToAbs. Glob patterns
// never exist on disk, so canonicalization keeps them lexical.
abs, err := agentcontext.CanonicalizePathIn(home, os.ExpandEnv(requested))
if err != nil {
return nil, false, err
}
if !strings.ContainsAny(abs, "*?{[") {
return []string{abs}, false, nil
}
base, pattern := doublestar.SplitPattern(filepath.ToSlash(abs))
matches := make([]string, 0, min(maxMatches, 64))
// WithNoFollow avoids symlink cycles. Checking the limit before the
// append keeps matches from growing past maxMatches.
err = doublestar.GlobWalk(bundleFilesFS{ctx: ctx, fsys: os.DirFS(base)}, pattern, func(match string, _ fs.DirEntry) error {
if len(matches) >= maxMatches {
return errBundleFilesFileLimit
}
matches = append(matches, filepath.Join(base, filepath.FromSlash(match)))
return nil
}, doublestar.WithFilesOnly(), doublestar.WithNoFollow())
matchesTruncated := errors.Is(err, errBundleFilesFileLimit)
if err != nil && !matchesTruncated {
return nil, false, xerrors.Errorf("glob pattern: %w", err)
}
// doublestar does not guarantee ordering, so sort for a deterministic
// archive.
slices.Sort(matches)
return matches, matchesTruncated, nil
}
// bundleFilesFS cancels a glob walk once the request context ends. Only
// Open is implemented; the fs.ReadDir and fs.Stat helpers fall back to it,
// so every filesystem operation of the walk passes the context check.
type bundleFilesFS struct {
ctx context.Context
fsys fs.FS
}
func (f bundleFilesFS) Open(name string) (fs.File, error) {
if err := f.ctx.Err(); err != nil {
return nil, err
}
return f.fsys.Open(name)
}
// BundleFilesArchivePath maps a cleaned absolute path to its archive entry
// name: files/ plus the path with the leading separator trimmed and any
// Windows drive colon dropped, keeping the name fs.ValidPath-safe.
func BundleFilesArchivePath(abs string) string {
p := strings.TrimPrefix(filepath.ToSlash(abs), "/")
if len(p) >= 2 && p[1] == ':' {
p = p[:1] + p[2:]
}
return "files/" + p
}
// fileModeTypeName names the type of a non-regular file.
func fileModeTypeName(mode fs.FileMode) string {
switch {
case mode.IsDir():
return "directory"
case mode&fs.ModeSymlink != 0:
return "symlink"
case mode&fs.ModeNamedPipe != 0:
return "named pipe"
case mode&fs.ModeSocket != 0:
return "socket"
case mode&fs.ModeDevice != 0, mode&fs.ModeCharDevice != 0:
return "device"
default:
return "irregular file"
}
}
// writeBundleFileEntry writes the last entry.BytesWritten bytes of the
// file at abs to the archive at entry.ArchivePath. A file that shrinks
// after stat is zero-padded to the declared size, since a short entry
// would corrupt every entry after it; the short read is still an error.
func writeBundleFileEntry(tw *tar.Writer, abs string, entry workspacesdk.BundleFilesManifestEntry) error {
f, err := os.Open(abs)
if err != nil {
return xerrors.Errorf("open file: %w", err)
}
defer f.Close()
if entry.BytesWritten < entry.Size {
if _, err := f.Seek(entry.Size-entry.BytesWritten, io.SeekStart); err != nil {
return xerrors.Errorf("seek tail: %w", err)
}
}
err = tw.WriteHeader(&tar.Header{
Name: entry.ArchivePath,
Mode: 0o644,
Size: entry.BytesWritten,
ModTime: entry.ModTime,
})
if err != nil {
return xerrors.Errorf("create archive entry: %w", err)
}
n, err := io.Copy(tw, io.LimitReader(f, entry.BytesWritten))
if err == nil && n < entry.BytesWritten {
err = io.ErrUnexpectedEOF
}
if err != nil {
if _, padErr := io.CopyN(tw, zeroReader{}, entry.BytesWritten-n); padErr != nil {
return xerrors.Errorf("pad short entry: %w", padErr)
}
return xerrors.Errorf("copy file: %w", err)
}
return nil
}
// tarEntrySize returns the archive bytes a file entry consumes: one
// header block plus the data rounded up to whole blocks.
func tarEntrySize(dataBytes int64) int64 {
return tarBlockSize + (dataBytes+tarBlockSize-1)/tarBlockSize*tarBlockSize
}
type zeroReader struct{}
func (zeroReader) Read(p []byte) (int, error) {
clear(p)
return len(p), nil
}
func appendManifestError(m *workspacesdk.BundleFilesManifest, requested string, filePath string, reason string) {
m.Errors = append(m.Errors, workspacesdk.BundleFilesManifestError{
Requested: requested,
Path: filePath,
Reason: reason,
})
}
+246
View File
@@ -0,0 +1,246 @@
package agentfiles_test
import (
"bytes"
"encoding/json"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"slices"
"strings"
"testing"
"github.com/spf13/afero"
"github.com/stretchr/testify/require"
"cdr.dev/slog/v3"
"cdr.dev/slog/v3/sloggers/slogtest"
"github.com/coder/coder/v2/agent/agentfiles"
"github.com/coder/coder/v2/agent/usershell"
"github.com/coder/coder/v2/codersdk/workspacesdk"
"github.com/coder/coder/v2/testutil"
)
func TestBundleFilesCollectsExpandedPathsAndGlobs(t *testing.T) {
t.Parallel()
home := testutil.TempDirResolved(t)
writeBundleSourceFile(t, home, ".vscode-server/data/logs/20260706T101112/remoteagent.log", "remote agent")
writeBundleSourceFile(t, home, ".vscode-server/data/logs/20260706T101112/exthost1/exthost.log", "exthost")
writeBundleSourceFile(t, home, ".vscode-server/data/logs/20260706T101112/exthost1/output.txt", "skip")
writeBundleSourceFile(t, home, ".local/share/code-server/coder-logs/app.log", "code server log")
writeBundleSourceFile(t, home, ".cache/JetBrains/RemoteDev/dist/241.15989.150/log/idea.log", "idea log")
writeBundleSourceFile(t, home, "brace/one.log", "one")
writeBundleSourceFile(t, home, "brace/two.txt", "two")
writeBundleSourceFile(t, home, "brace/skip.json", "skip")
entries := readBundleFilesArchive(t, requestBundleFiles(t, newBundleFilesHandler(t, home), []string{
filepath.Join(home, ".vscode-server/data/logs/20260706T101112/remoteagent.log"),
"~/.vscode-server/data/logs/**/*.log",
"~/.local/share/code-server/coder-logs/app.log",
"~/.cache/JetBrains/RemoteDev/dist/*/log/idea.log",
"~/brace/*.{log,txt}",
}))
requireBundleEntry(t, entries, home, ".vscode-server/data/logs/20260706T101112/remoteagent.log", "remote agent")
requireBundleEntry(t, entries, home, ".vscode-server/data/logs/20260706T101112/exthost1/exthost.log", "exthost")
requireBundleEntry(t, entries, home, ".local/share/code-server/coder-logs/app.log", "code server log")
requireBundleEntry(t, entries, home, ".cache/JetBrains/RemoteDev/dist/241.15989.150/log/idea.log", "idea log")
requireBundleEntry(t, entries, home, "brace/one.log", "one")
requireBundleEntry(t, entries, home, "brace/two.txt", "two")
require.NotContains(t, entries.files, bundleArchivePath(t, home, ".vscode-server/data/logs/20260706T101112/exthost1/output.txt"))
require.NotContains(t, entries.files, bundleArchivePath(t, home, "brace/skip.json"))
require.Empty(t, entries.manifest.Errors)
// remoteagent.log matches both the absolute path and the ** glob; it
// must be archived once.
require.Len(t, entries.manifest.Files, 6)
}
func TestBundleFilesCollectsAbsolutePathsOutsideHome(t *testing.T) {
t.Parallel()
home := testutil.TempDirResolved(t)
outside := testutil.TempDirResolved(t)
writeBundleSourceFile(t, outside, "service.log", "outside log")
writeBundleSourceFile(t, outside, "glob/a.log", "glob a")
entries := readBundleFilesArchive(t, requestBundleFiles(t, newBundleFilesHandler(t, home), []string{
filepath.Join(outside, "service.log"),
filepath.Join(outside, "glob", "*.log"),
}))
requireBundleEntry(t, entries, outside, "service.log", "outside log")
requireBundleEntry(t, entries, outside, "glob/a.log", "glob a")
require.Empty(t, entries.manifest.Errors)
require.Len(t, entries.manifest.Files, 2)
}
func TestBundleFilesRejectedPathsAreNonFatal(t *testing.T) {
t.Parallel()
home := testutil.TempDirResolved(t)
writeBundleSourceFile(t, home, "kept.log", "kept")
require.NoError(t, os.MkdirAll(filepath.Join(home, "somedir"), 0o700))
entries := readBundleFilesArchive(t, requestBundleFiles(t, newBundleFilesHandler(t, home), []string{
"~/kept.log",
"relative.log",
"~/missing.log",
"~/somedir",
"~/no-matches/**/*.log",
}))
requireBundleEntry(t, entries, home, "kept.log", "kept")
require.Len(t, entries.manifest.Files, 1)
requireBundleFilesManifestErrors(t, entries.manifest.Errors,
"is not absolute",
"does not exist",
"not a regular file: directory",
"no matches",
)
}
func TestBundleFilesTailBytesTruncation(t *testing.T) {
t.Parallel()
home := testutil.TempDirResolved(t)
writeBundleSourceFile(t, home, "large.log", "0123456789")
entries := readBundleFilesArchive(t, requestBundleFiles(t, newBundleFilesHandler(t, home, agentfiles.WithBundleFilesLimits(workspacesdk.BundleFilesLimits{
MaxFiles: 10,
MaxBytesPerFile: 4,
MaxTotalBytes: 100 * 1024,
})), []string{"~/large.log"}))
requireBundleEntry(t, entries, home, "large.log", "6789")
require.Len(t, entries.manifest.Files, 1)
require.True(t, entries.manifest.Files[0].Truncated)
require.Equal(t, int64(10), entries.manifest.Files[0].Size)
require.Equal(t, int64(4), entries.manifest.Files[0].BytesWritten)
}
func TestBundleFilesFileAndByteLimits(t *testing.T) {
t.Parallel()
home := testutil.TempDirResolved(t)
writeBundleSourceFile(t, home, "one.log", "1111")
writeBundleSourceFile(t, home, "two.log", "2222")
writeBundleSourceFile(t, home, "three.log", "3333")
entries := readBundleFilesArchive(t, requestBundleFiles(t, newBundleFilesHandler(t, home, agentfiles.WithBundleFilesLimits(workspacesdk.BundleFilesLimits{
MaxFiles: 1,
MaxBytesPerFile: 100,
// One 512-byte tar header plus 3 data bytes: the first file is
// truncated to 3 bytes by the total budget.
MaxTotalBytes: 515,
})), []string{"~/*.log"}))
require.Len(t, entries.files, 1)
require.True(t, entries.manifest.Truncated)
require.Equal(t, int64(3), entries.manifest.Files[0].BytesWritten)
// The glob walk itself stops at the file cap.
requireBundleFilesManifestErrors(t, entries.manifest.Errors, "file count limit reached")
}
func TestBundleFilesDedupeByCleanedPath(t *testing.T) {
t.Parallel()
home := testutil.TempDirResolved(t)
writeBundleSourceFile(t, home, "dup.log", "one")
writeBundleSourceFile(t, home, "other.log", "two")
entries := readBundleFilesArchive(t, requestBundleFiles(t, newBundleFilesHandler(t, home), []string{
"~/dup.log",
"~/./dup.log",
filepath.Join(home, "somedir", "..", "dup.log"),
"~/other.log",
}))
requireBundleEntry(t, entries, home, "dup.log", "one")
requireBundleEntry(t, entries, home, "other.log", "two")
require.Len(t, entries.manifest.Files, 2)
}
// fakeBundleEnvInfo overrides the home directory so tests can point path
// expansion at a temp dir.
type fakeBundleEnvInfo struct {
usershell.SystemEnvInfo
home string
}
func (e fakeBundleEnvInfo) HomeDir() (string, error) {
return e.home, nil
}
func newBundleFilesHandler(t *testing.T, home string, opts ...agentfiles.Option) http.Handler {
t.Helper()
logger := slogtest.Make(t, &slogtest.Options{IgnoreErrors: true}).Leveled(slog.LevelDebug)
opts = append([]agentfiles.Option{agentfiles.WithEnvInfo(fakeBundleEnvInfo{home: home})}, opts...)
return agentfiles.NewAPI(logger, afero.NewOsFs(), nil, opts...).Routes()
}
func requestBundleFiles(t *testing.T, handler http.Handler, paths []string) []byte {
t.Helper()
body, err := json.Marshal(workspacesdk.BundleFilesRequest{Paths: paths})
require.NoError(t, err)
req := httptest.NewRequest(http.MethodPost, "/bundle-files", bytes.NewReader(body))
res := httptest.NewRecorder()
handler.ServeHTTP(res, req)
require.Equal(t, http.StatusOK, res.Code)
require.Equal(t, "application/x-tar", res.Header().Get("Content-Type"))
return res.Body.Bytes()
}
func writeBundleSourceFile(t *testing.T, dir string, rel string, content string) {
t.Helper()
path := filepath.Join(dir, filepath.FromSlash(rel))
require.NoError(t, os.MkdirAll(filepath.Dir(path), 0o700))
require.NoError(t, os.WriteFile(path, []byte(content), 0o600))
}
// bundleArchivePath returns the expected archive entry name for the file
// at dir/rel.
func bundleArchivePath(t *testing.T, dir string, rel string) string {
t.Helper()
return agentfiles.BundleFilesArchivePath(filepath.Join(dir, filepath.FromSlash(rel)))
}
func requireBundleEntry(t *testing.T, entries bundleFilesArchive, dir string, rel string, content string) {
t.Helper()
require.Equal(t, content, string(entries.files[bundleArchivePath(t, dir, rel)]))
}
type bundleFilesArchive struct {
manifest workspacesdk.BundleFilesManifest
files map[string][]byte
}
func readBundleFilesArchive(t *testing.T, data []byte) bundleFilesArchive {
t.Helper()
entries := bundleFilesArchive{files: testutil.ReadTar(t, data)}
manifestJSON, ok := entries.files["manifest.json"]
require.True(t, ok, "archive should contain manifest.json")
delete(entries.files, "manifest.json")
require.NoError(t, json.Unmarshal(manifestJSON, &entries.manifest))
require.NotEmpty(t, entries.manifest.Requested)
return entries
}
func requireBundleFilesManifestErrors(t *testing.T, errs []workspacesdk.BundleFilesManifestError, contains ...string) {
t.Helper()
for _, want := range contains {
found := slices.ContainsFunc(errs, func(e workspacesdk.BundleFilesManifestError) bool {
return strings.Contains(e.Reason, want)
})
require.Truef(t, found, "expected manifest error containing %q in %#v", want, errs)
}
}