feat: collect workspace logs in support bundles (#26694)

Add workspace-side file collection to `coder support bundle` via
repeatable --workspace-file flags. The agent resolves the requested
paths or globs inside the remote workspace and streams back a tar with
a manifest and the collected files; nothing is read from the machine
running the command.

- Add POST /api/v0/bundle-files to the agent's agentfiles package.
- Expand env vars in the agent's environment; paths must then be
  absolute or start with ~/ (the agent user's home directory).
- Support ** globs and tail oversized files.
- Record requested patterns, per-path errors, truncation, and the
  applied limits in a manifest.
- Unpack the archive into the bundle under agent/workspace_files/,
  recording dropped entries in collection_errors.txt.
- Write a manifest-only archive marking collection as unsupported for
  agents that predate the endpoint.
- Bound collection: 64 KB request body, 10000 files, 10 MiB per file,
  100 MiB total including archive overhead, 110 MiB client-side read
  cap, 5 minute timeout.

Closes #26020

cli/support.go

@@ -1,15 +1,20 @@
 package cli
 
 import (
+	"archive/tar"
 	"archive/zip"
 	"bytes"
 	"context"
 	"encoding/base64"
 	"encoding/json"
+	"errors"
 	"fmt"
+	"io"
+	"io/fs"
 	"net/http"
 	"net/url"
 	"os"
+	"path"
 	"path/filepath"
 	"strings"
 	"text/tabwriter"
@@ -41,8 +46,9 @@ func (r *RootCmd) support() *serpent.Command {
 	return supportCmd
 }
 
-var supportBundleBlurb = cliui.Bold("This will collect the following information:\n") +
-	`  - Coder deployment version
+func supportBundleBlurb(workspaceFilePatterns []string) string {
+	blurb := cliui.Bold("This will collect the following information:\n") +
+		`  - Coder deployment version
   - Coder deployment Configuration (sanitized), including enabled experiments
   - Coder deployment health snapshot
   - Coder deployment stats (aggregated workspace/session metrics)
@@ -55,20 +61,29 @@ var supportBundleBlurb = cliui.Bold("This will collect the following information
   - Agent details (with environment variable sanitized)
   - Agent network diagnostics
   - Agent logs
-  - License status
+`
+	if len(workspaceFilePatterns) > 0 {
+		blurb += "  - Workspace files matching:\n"
+		for _, pattern := range workspaceFilePatterns {
+			blurb += "    - " + pattern + "\n"
+		}
+	}
+	return blurb + `  - License status
   - pprof profiling data (if --pprof is enabled)
 ` + cliui.Bold("Note: ") +
-	cliui.Wrap("While we try to sanitize sensitive data from support bundles, we cannot guarantee that they do not contain information that you or your organization may consider sensitive.\n") +
-	cliui.Bold("Please confirm that you will:\n") +
-	"  - Review the support bundle before distribution\n" +
-	"  - Only distribute it via trusted channels\n" +
-	cliui.Bold("Continue? ")
+		cliui.Wrap("While we try to sanitize sensitive data from support bundles, we cannot guarantee that they do not contain information that you or your organization may consider sensitive.\n") +
+		cliui.Bold("Please confirm that you will:\n") +
+		"  - Review the support bundle before distribution\n" +
+		"  - Only distribute it via trusted channels\n" +
+		cliui.Bold("Continue? ")
+}
 
 func (r *RootCmd) supportBundle() *serpent.Command {
 	var outputPath string
 	var coderURLOverride string
 	var workspacesTotalCap64 int64 = 10
 	var templateName string
+	var workspaceFilePatterns []string
 	var pprof bool
 	cmd := &serpent.Command{
 		Use:   "bundle [<workspace>] [<agent>]",
@@ -89,7 +104,7 @@ func (r *RootCmd) supportBundle() *serpent.Command {
 				cliLog = cliLog.AppendSinks(sloghuman.Sink(inv.Stderr))
 			}
 			ans, err := cliui.Prompt(inv, cliui.PromptOptions{
-				Text:      supportBundleBlurb,
+				Text:      supportBundleBlurb(workspaceFilePatterns),
 				Secret:    false,
 				IsConfirm: true,
 			})
@@ -249,12 +264,13 @@ func (r *RootCmd) supportBundle() *serpent.Command {
 			deps := support.Deps{
 				Client: client,
 				// Support adds a sink so we don't need to supply one ourselves.
-				Log:                clientLog,
-				WorkspaceID:        wsID,
-				AgentID:            agtID,
-				WorkspacesTotalCap: int(workspacesTotalCap64),
-				TemplateID:         templateID,
-				CollectPprof:       pprof,
+				Log:                   clientLog,
+				WorkspaceID:           wsID,
+				AgentID:               agtID,
+				WorkspacesTotalCap:    int(workspacesTotalCap64),
+				TemplateID:            templateID,
+				WorkspaceFilePatterns: workspaceFilePatterns,
+				CollectPprof:          pprof,
 			}
 
 			bun, err := support.Run(inv.Context(), &deps)
@@ -302,6 +318,12 @@ func (r *RootCmd) supportBundle() *serpent.Command {
 			Description: "Template name to include in the support bundle. Use org_name/template_name if template name is reused across multiple organizations.",
 			Value:       serpent.StringOf(&templateName),
 		},
+		{
+			Flag:        "workspace-file",
+			Env:         "CODER_SUPPORT_BUNDLE_WORKSPACE_FILE",
+			Description: "File path or glob to collect from inside the remote workspace. Environment variables are expanded in the workspace; paths must then be absolute or start with ~/, which resolves against the agent user's home directory. Files local to the machine running this command are not collected. Can be specified multiple times.",
+			Value:       serpent.StringArrayOf(&workspaceFilePatterns),
+		},
 		{
 			Flag:        "pprof",
 			Env:         "CODER_SUPPORT_BUNDLE_PPROF",
@@ -549,6 +571,10 @@ func writeBundle(src *support.Bundle, dest *zip.Writer) error {
 		}
 	}
 
+	if err := writeWorkspaceFilesArchive(src.Agent.WorkspaceFilesArchive, dest, supportBundleWorkspaceFilesMaxBytes); err != nil {
+		return xerrors.Errorf("write workspace files: %w", err)
+	}
+
 	// Write pprof binary data
 	if err := writePprofData(src.Pprof, dest); err != nil {
 		return xerrors.Errorf("write pprof data: %w", err)
@@ -560,6 +586,91 @@ func writeBundle(src *support.Bundle, dest *zip.Writer) error {
 	return nil
 }
 
+// supportBundleWorkspaceFilesMaxBytes guards against a misbehaving agent;
+// the agent itself caps collection at 100 MiB.
+const supportBundleWorkspaceFilesMaxBytes int64 = 110 * 1024 * 1024
+
+// writeWorkspaceFilesArchive unpacks the agent's tar into the bundle under
+// agent/workspace_files/; dropped entries are recorded in collection_errors.txt.
+func writeWorkspaceFilesArchive(src []byte, dest *zip.Writer, maxBytes int64) error {
+	if len(src) == 0 {
+		return nil
+	}
+	tr := tar.NewReader(bytes.NewReader(src))
+	remaining := maxBytes
+	var skipped []string
+	for {
+		hdr, err := tr.Next()
+		if errors.Is(err, io.EOF) {
+			break
+		}
+		if err != nil {
+			// A malformed archive shouldn't sink the rest of the bundle.
+			skipped = append(skipped, fmt.Sprintf("read workspace files archive: %s", err))
+			break
+		}
+		name, ok := safeWorkspaceFilesArchiveEntryName(hdr.Name)
+		if !ok || hdr.Typeflag != tar.TypeReg {
+			skipped = append(skipped, fmt.Sprintf("%s: unexpected entry", hdr.Name))
+			continue
+		}
+		if hdr.Size > remaining {
+			// Only a misbehaving agent exceeds the budget; stop trusting
+			// the rest of the archive.
+			skipped = append(skipped, fmt.Sprintf("%s: %d bytes exceeds remaining %d byte budget, aborting", name, hdr.Size, remaining))
+			break
+		}
+		// A failed create means the output zip itself is broken.
+		f, err := dest.Create(path.Join("agent/workspace_files", name))
+		if err != nil {
+			return xerrors.Errorf("create workspace files entry %q: %w", name, err)
+		}
+		// io.CopyN bounds the copy at hdr.Size so a header lying about
+		// size cannot make us read past the entry; copy failures are
+		// recorded, not fatal.
+		n, err := io.CopyN(f, tr, hdr.Size)
+		remaining -= n
+		if errors.Is(err, io.EOF) {
+			err = nil
+		}
+		if err != nil {
+			skipped = append(skipped, fmt.Sprintf("%s: copy: %s (entry may be truncated)", name, err))
+		}
+	}
+	return writeWorkspaceFilesCollectionErrors(dest, skipped)
+}
+
+// writeWorkspaceFilesCollectionErrors records dropped workspace file entries in the
+// bundle instead of failing it.
+func writeWorkspaceFilesCollectionErrors(dest *zip.Writer, skipped []string) error {
+	if len(skipped) == 0 {
+		return nil
+	}
+	f, err := dest.Create("agent/workspace_files/collection_errors.txt")
+	if err != nil {
+		return xerrors.Errorf("create workspace files errors: %w", err)
+	}
+	body := "# workspace file entries dropped while assembling the support bundle\n" +
+		strings.Join(skipped, "\n") + "\n"
+	if _, err := f.Write([]byte(body)); err != nil {
+		return xerrors.Errorf("write workspace files errors: %w", err)
+	}
+	return nil
+}
+
+// safeWorkspaceFilesArchiveEntryName returns name when it is safe to embed in
+// the bundle: a valid slash path within the expected layout. Backslashes
+// are rejected; some Windows extractors treat them as separators.
+func safeWorkspaceFilesArchiveEntryName(name string) (string, bool) {
+	if strings.Contains(name, `\`) || !fs.ValidPath(name) {
+		return "", false
+	}
+	if name != "manifest.json" && !strings.HasPrefix(name, "files/") {
+		return "", false
+	}
+	return name, true
+}
+
 func writePprofData(pprof support.Pprof, dest *zip.Writer) error {
 	// Write server pprof data directly to pprof directory
 	if pprof.Server != nil {

cli/support_internal_test.go

@@ -0,0 +1,124 @@
+package cli
+
+import (
+	"archive/tar"
+	"archive/zip"
+	"bytes"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/coder/coder/v2/testutil"
+)
+
+func TestSafeWorkspaceFilesArchiveEntryName(t *testing.T) {
+	t.Parallel()
+
+	for _, tt := range []struct {
+		name string
+		ok   bool
+	}{
+		{name: "manifest.json", ok: true},
+		{name: "files/server.log", ok: true},
+		{name: "./files/server.log", ok: false},
+		{name: "../manifest.json", ok: false},
+		{name: "/manifest.json", ok: false},
+		{name: "files/nested/../server.log", ok: false},
+		{name: "files/../../manifest.json", ok: false},
+		{name: "files\\nested\\server.log", ok: false},
+		{name: `files/nested\..\server.log`, ok: false},
+		{name: "other/server.log", ok: false},
+	} {
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+
+			got, ok := safeWorkspaceFilesArchiveEntryName(tt.name)
+			require.Equal(t, tt.ok, ok)
+			if tt.ok {
+				require.Equal(t, tt.name, got)
+			}
+		})
+	}
+}
+
+func TestWriteWorkspaceFilesArchive(t *testing.T) {
+	t.Parallel()
+
+	t.Run("UnpacksManifestAndFiles", func(t *testing.T) {
+		t.Parallel()
+
+		agentArchive := makeWorkspaceFilesArchive(t,
+			"files/server.log", "server log",
+			"manifest.json", `{"files":[{"archive_path":"files/server.log"}]}`,
+			"../escape.log", "should be dropped and recorded",
+		)
+
+		var bundle bytes.Buffer
+		bundleZip := zip.NewWriter(&bundle)
+		require.NoError(t, writeWorkspaceFilesArchive(agentArchive, bundleZip, supportBundleWorkspaceFilesMaxBytes))
+		require.NoError(t, bundleZip.Close())
+
+		entries := testutil.ReadZip(t, bundle.Bytes())
+		require.Equal(t, "server log", string(entries["agent/workspace_files/files/server.log"]))
+		require.Contains(t, entries, "agent/workspace_files/manifest.json")
+		require.Contains(t, string(entries["agent/workspace_files/collection_errors.txt"]), "../escape.log")
+		require.Len(t, entries, 3)
+	})
+
+	t.Run("AbortsOnEntryBeyondBudget", func(t *testing.T) {
+		t.Parallel()
+
+		agentArchive := makeWorkspaceFilesArchive(t,
+			"files/ok.log", "ok",
+			"files/big.log", "this entry is too big",
+			"files/after.log", "never reached",
+		)
+
+		var bundle bytes.Buffer
+		bundleZip := zip.NewWriter(&bundle)
+		// A 4 byte budget fits ok.log; big.log exceeds it and aborts the
+		// rest.
+		require.NoError(t, writeWorkspaceFilesArchive(agentArchive, bundleZip, 4))
+		require.NoError(t, bundleZip.Close())
+
+		entries := testutil.ReadZip(t, bundle.Bytes())
+		require.Equal(t, "ok", string(entries["agent/workspace_files/files/ok.log"]))
+		require.NotContains(t, entries, "agent/workspace_files/files/big.log")
+		require.NotContains(t, entries, "agent/workspace_files/files/after.log")
+		errs := string(entries["agent/workspace_files/collection_errors.txt"])
+		require.Contains(t, errs, "files/big.log")
+		require.Contains(t, errs, "budget")
+	})
+
+	t.Run("MalformedArchiveDoesNotFail", func(t *testing.T) {
+		t.Parallel()
+
+		var bundle bytes.Buffer
+		bundleZip := zip.NewWriter(&bundle)
+		require.NoError(t, writeWorkspaceFilesArchive([]byte("not a tar"), bundleZip, supportBundleWorkspaceFilesMaxBytes))
+		require.NoError(t, bundleZip.Close())
+
+		entries := testutil.ReadZip(t, bundle.Bytes())
+		require.Contains(t, string(entries["agent/workspace_files/collection_errors.txt"]), "read workspace files archive")
+	})
+}
+
+// makeWorkspaceFilesArchive tars alternating name/content pairs in order.
+func makeWorkspaceFilesArchive(t *testing.T, pairs ...string) []byte {
+	t.Helper()
+
+	require.Zero(t, len(pairs)%2)
+	var buf bytes.Buffer
+	tw := tar.NewWriter(&buf)
+	for i := 0; i < len(pairs); i += 2 {
+		require.NoError(t, tw.WriteHeader(&tar.Header{
+			Name: pairs[i],
+			Mode: 0o644,
+			Size: int64(len(pairs[i+1])),
+		}))
+		_, err := tw.Write([]byte(pairs[i+1]))
+		require.NoError(t, err)
+	}
+	require.NoError(t, tw.Close())
+	return buf.Bytes()
+}

cli/support_test.go

@@ -12,6 +12,7 @@ import (
 	"os"
 	"path/filepath"
 	"runtime"
+	"strings"
 	"testing"
 	"time"
 
@@ -21,6 +22,7 @@ import (
 	"tailscale.com/ipn/ipnstate"
 
 	"github.com/coder/coder/v2/agent"
+	"github.com/coder/coder/v2/agent/agentfiles"
 	"github.com/coder/coder/v2/agent/agenttest"
 	"github.com/coder/coder/v2/cli/clitest"
 	"github.com/coder/coder/v2/coderd/coderdtest"
@@ -306,6 +308,80 @@ func TestSupportBundle(t *testing.T) {
 	})
 }
 
+func TestSupportBundleCollectsWorkspaceFiles(t *testing.T) {
+	if runtime.GOOS == "windows" {
+		t.Skip("for some reason, windows fails to remove tempdirs sometimes")
+	}
+
+	var dc codersdk.DeploymentConfig
+	dc.Values = coderdtest.DeploymentValues(t)
+	dc.Values.Prometheus.Enable = true
+	secretValue := uuid.NewString()
+	seedSecretDeploymentOptions(t, &dc, secretValue)
+	client, closer, api := coderdtest.NewWithAPI(t, &coderdtest.Options{
+		DeploymentValues: dc.Values,
+		HealthcheckFunc: func(_ context.Context, _ string, _ *healthcheck.Progress) *healthsdk.HealthcheckReport {
+			return &healthsdk.HealthcheckReport{
+				Time:     time.Now(),
+				Healthy:  true,
+				Severity: health.SeverityOK,
+			}
+		},
+	})
+	t.Cleanup(func() { closer.Close() })
+	owner := coderdtest.CreateFirstUser(t, client)
+	workspaceWithAgent := setupSupportBundleTestFixture(testutil.Context(t, testutil.WaitLong), t, api.Database, owner.OrganizationID, owner.UserID, func(agents []*proto.Agent) []*proto.Agent {
+		agents[0].Env["SECRET_VALUE"] = secretValue
+		return agents
+	})
+
+	// The agent resolves requested paths against $HOME (USERPROFILE on
+	// Windows). The agent log dir is separate so collection does not race
+	// live agent logs. The resolved dir matches the agent's canonicalized
+	// manifest paths (the macOS temp dir is a symlink).
+	home := testutil.TempDirResolved(t)
+	t.Setenv("HOME", home)
+	t.Setenv("USERPROFILE", home)
+	require.NoError(t, os.MkdirAll(filepath.Join(home, "testlogs", "nested"), 0o700))
+	require.NoError(t, os.WriteFile(filepath.Join(home, "testlogs", "server.log"), []byte("server log"), 0o600))
+	require.NoError(t, os.WriteFile(filepath.Join(home, "testlogs", "nested", "nested.log"), []byte("nested log"), 0o600))
+
+	logDir := t.TempDir()
+	require.NoError(t, os.WriteFile(filepath.Join(logDir, "coder-agent.log"), []byte("hello from the agent"), 0o600))
+	agt := agenttest.New(t, client.URL, workspaceWithAgent.AgentToken, func(o *agent.Options) {
+		o.LogDir = logDir
+	})
+	defer agt.Close()
+	coderdtest.NewWorkspaceAgentWaiter(t, client, workspaceWithAgent.Workspace.ID).Wait()
+
+	d := t.TempDir()
+	bundlePath := filepath.Join(d, "bundle.zip")
+	// The exact path and the glob both match server.log to cover
+	// deduplication end to end.
+	inv, root := clitest.New(t,
+		"support", "bundle", workspaceWithAgent.Workspace.Name,
+		"--workspace-file", "$HOME/testlogs/server.log",
+		"--workspace-file", "$HOME/testlogs/**/*.log",
+		"--output-file", bundlePath,
+		"--yes",
+	)
+	// nolint: gocritic // requires owner privilege
+	clitest.SetupConfig(t, client, root)
+	err := inv.WithContext(testutil.Context(t, testutil.WaitLong)).Run()
+	require.NoError(t, err)
+
+	assertBundleContents(t, bundlePath, true, true, []string{secretValue})
+	entries := readZipEntries(t, bundlePath)
+	serverLogEntry := "agent/workspace_files/" + agentfiles.BundleFilesArchivePath(filepath.Join(home, "testlogs", "server.log"))
+	nestedLogEntry := "agent/workspace_files/" + agentfiles.BundleFilesArchivePath(filepath.Join(home, "testlogs", "nested", "nested.log"))
+	require.Equal(t, "server log", string(entries[serverLogEntry]))
+	require.Equal(t, "nested log", string(entries[nestedLogEntry]))
+	var manifest workspacesdk.BundleFilesManifest
+	require.NoError(t, json.Unmarshal(entries["agent/workspace_files/manifest.json"], &manifest))
+	require.Equal(t, []string{"$HOME/testlogs/server.log", "$HOME/testlogs/**/*.log"}, manifest.Requested)
+	require.Len(t, manifest.Files, 2, "server.log should be deduplicated across the exact path and the glob")
+}
+
 // nolint:revive // It's a control flag, but this is just a test.
 func assertBundleContents(t *testing.T, path string, wantWorkspace bool, wantAgent bool, badValues []string) {
 	t.Helper()
@@ -314,6 +390,11 @@ func assertBundleContents(t *testing.T, path string, wantWorkspace bool, wantAge
 	defer r.Close()
 	for _, f := range r.File {
 		assertDoesNotContain(t, f, badValues...)
+		if strings.HasPrefix(f.Name, "agent/workspace_files/files/") {
+			bs := readBytesFromZip(t, f)
+			require.NotEmpty(t, bs, "workspace log file should not be empty")
+			continue
+		}
 		switch f.Name {
 		case "deployment/buildinfo.json":
 			var v codersdk.BuildInfoResponse
@@ -490,6 +571,10 @@ func assertBundleContents(t *testing.T, path string, wantWorkspace bool, wantAge
 				continue
 			}
 			require.Contains(t, string(bs), "started up")
+		case "agent/workspace_files/manifest.json":
+			var v workspacesdk.BundleFilesManifest
+			decodeJSONFromZip(t, f, &v)
+			require.NotEmpty(t, v.Requested, "workspace log file manifest should include requested paths")
 		case "logs.txt":
 			bs := readBytesFromZip(t, f)
 			require.NotEmpty(t, bs, "logs should not be empty")
@@ -517,11 +602,21 @@ func readBytesFromZip(t *testing.T, f *zip.File) []byte {
 	t.Helper()
 	rc, err := f.Open()
 	require.NoError(t, err, "open file from zip")
+	defer rc.Close()
 	bs, err := io.ReadAll(rc)
 	require.NoError(t, err, "read bytes from zip")
 	return bs
 }
 
+// readZipEntries reads every entry of the zip at zipPath into memory.
+func readZipEntries(t *testing.T, zipPath string) map[string][]byte {
+	t.Helper()
+
+	data, err := os.ReadFile(zipPath)
+	require.NoError(t, err, "read zip file")
+	return testutil.ReadZip(t, data)
+}
+
 func assertDoesNotContain(t *testing.T, f *zip.File, vals ...string) {
 	t.Helper()
 	bs := readBytesFromZip(t, f)

cli/testdata/coder_support_bundle_--help.golden

@@ -28,6 +28,13 @@ OPTIONS:
           Override the URL to your Coder deployment. This may be useful, for
           example, if you need to troubleshoot a specific Coder replica.
 
+      --workspace-file string-array, $CODER_SUPPORT_BUNDLE_WORKSPACE_FILE
+          File path or glob to collect from inside the remote workspace.
+          Environment variables are expanded in the workspace; paths must then
+          be absolute or start with ~/, which resolves against the agent user's
+          home directory. Files local to the machine running this command are
+          not collected. Can be specified multiple times.
+
       --workspaces-total-cap int, $CODER_SUPPORT_BUNDLE_WORKSPACES_TOTAL_CAP
           Maximum number of workspaces to include in the support bundle. Set to
           0 or negative value to disable the cap. Defaults to 10.