fix(dogfood/contents): fix resource metadata under nix (#16349)

Co-authored-by: Thomas Kosiewski <tk@coder.com>

.github/workflows/dogfood.yaml

@@ -37,9 +37,6 @@ jobs:
       - name: Setup Nix
         uses: DeterminateSystems/nix-installer-action@e50d5f73bfe71c2dd0aa4218de8f4afa59f8f81d # v16
 
-      - name: Setup GHA Nix cache
-        uses: DeterminateSystems/magic-nix-cache-action@6221693898146dc97e38ad0e013488a16477a4c4 # v9
-
       - name: Get branch name
         id: branch-name
         uses: tj-actions/branch-names@6871f53176ad61624f978536bbf089c574dc19a2 # v8.0.1

dogfood/contents/main.tf

@@ -249,7 +249,7 @@ resource "coder_agent" "dev" {
     key          = "swap_usage_host"
     order        = 4
     script       = <<EOT
-      #!/bin/bash
+      #!/usr/bin/env bash
       echo "$(free -b | awk '/^Swap/ { printf("%.1f/%.1f", $3/1024.0/1024.0/1024.0, $2/1024.0/1024.0/1024.0) }') GiB"
     EOT
     interval     = 10
@@ -262,7 +262,7 @@ resource "coder_agent" "dev" {
     order        = 5
     # get load avg scaled by number of cores
     script   = <<EOT
-      #!/bin/bash
+      #!/usr/bin/env bash
       echo "`cat /proc/loadavg | awk '{ print $1 }'` `nproc`" | awk '{ printf "%0.2f", $1/$2 }'
     EOT
     interval = 60
@@ -283,7 +283,7 @@ resource "coder_agent" "dev" {
     key          = "word"
     order        = 7
     script       = <<EOT
-      #!/bin/bash
+      #!/usr/bin/env bash
       curl -o - --silent https://www.merriam-webster.com/word-of-the-day 2>&1 | awk ' $0 ~ "Word of the Day: [A-z]+" { print $5; exit }'
     EOT
     interval     = 86400
@@ -291,6 +291,7 @@ resource "coder_agent" "dev" {
   }
 
   startup_script = <<-EOT
+    #!/usr/bin/env bash
     set -eux -o pipefail
 
     # Allow synchronization between scripts.

flake.nix

@@ -71,70 +71,91 @@
           vendorHash = null;
         };
 
+        # Packages required to build the frontend
+        frontendPackages =
+          with pkgs;
+          [
+            cairo
+            pango
+            pixman
+            libpng
+            libjpeg
+            giflib
+            librsvg
+            python312Packages.setuptools # Needed for node-gyp
+          ]
+          ++ (lib.optionals stdenv.targetPlatform.isDarwin [
+            darwin.apple_sdk.frameworks.Foundation
+            xcbuild
+          ]);
+
         # The minimal set of packages to build Coder.
-        devShellPackages = with pkgs; [
+        devShellPackages =
-          # google-chrome is not available on aarch64 linux
+          with pkgs;
-          (lib.optionalDrvAttr (!stdenv.isLinux || !stdenv.isAarch64) google-chrome)
+          [
-          # strace is not available on OSX
+            # google-chrome is not available on aarch64 linux
-          (lib.optionalDrvAttr (!pkgs.stdenv.isDarwin) strace)
+            (lib.optionalDrvAttr (!stdenv.isLinux || !stdenv.isAarch64) google-chrome)
-          bat
+            # strace is not available on OSX
-          cairo
+            (lib.optionalDrvAttr (!pkgs.stdenv.isDarwin) strace)
-          curl
+            bat
-          delve
+            cairo
-          dive
+            curl
-          drpc.defaultPackage.${system}
+            delve
-          formatter
+            dive
-          fzf
+            drpc.defaultPackage.${system}
-          gcc13
+            formatter
-          gdk
+            fzf
-          getopt
+            gcc13
-          gh
+            gdk
-          git
+            getopt
-          (lib.optionalDrvAttr stdenv.isLinux glibcLocales)
+            gh
-          gnumake
+            git
-          gnused
+            (lib.optionalDrvAttr stdenv.isLinux glibcLocales)
-          go_1_22
+            gnumake
-          go-migrate
+            gnused
-          (pinnedPkgs.golangci-lint)
+            go_1_22
-          gopls
+            go-migrate
-          gotestsum
+            (pinnedPkgs.golangci-lint)
-          jq
+            gopls
-          kubectl
+            gotestsum
-          kubectx
+            jq
-          kubernetes-helm
+            kubectl
-          lazygit
+            kubectx
-          less
+            kubernetes-helm
-          mockgen
+            lazygit
-          moreutils
+            less
-          neovim
+            mockgen
-          nfpm
+            moreutils
-          nix-prefetch-git
+            neovim
-          nodejs
+            nfpm
-          openssh
+            nix-prefetch-git
-          openssl
+            nodejs
-          pango
+            openssh
-          pixman
+            openssl
-          pkg-config
+            pango
-          playwright-driver.browsers
+            pixman
-          pnpm
+            pkg-config
-          postgresql_16
+            playwright-driver.browsers
-          proto_gen_go_1_30
+            pnpm
-          protobuf_23
+            postgresql_16
-          ripgrep
+            proto_gen_go_1_30
-          shellcheck
+            protobuf_23
-          (pinnedPkgs.shfmt)
+            ripgrep
-          sqlc
+            shellcheck
-          terraform
+            (pinnedPkgs.shfmt)
-          typos
+            sqlc
-          # Needed for many LD system libs!
+            terraform
-          (lib.optional stdenv.isLinux util-linux)
+            typos
-          vim
+            # Needed for many LD system libs!
-          wget
+            (lib.optional stdenv.isLinux util-linux)
-          yq-go
+            vim
-          zip
+            wget
-          zsh
+            yq-go
-          zstd
+            zip
-        ];
+            zsh
+            zstd
+          ]
+          ++ frontendPackages;
 
         docker = pkgs.callPackage ./nix/docker.nix { };
 
@@ -144,22 +165,7 @@
 
           src = ./site/.;
           # Required for the `canvas` package!
-          extraBuildInputs =
+          extraBuildInputs = frontendPackages;
-            with pkgs;
-            [
-              cairo
-              pango
-              pixman
-              libpng
-              libjpeg
-              giflib
-              librsvg
-              python312Packages.setuptools
-            ]
-            ++ (lib.optionals stdenv.targetPlatform.isDarwin [
-              darwin.apple_sdk.frameworks.Foundation
-              xcbuild
-            ]);
           installInPlace = true;
           distDir = "out";
         };
@@ -219,6 +225,9 @@
             LOCALE_ARCHIVE =
               with pkgs;
               lib.optionalDrvAttr stdenv.isLinux "${glibcLocales}/lib/locale/locale-archive";
+
+            NODE_OPTIONS = "--max-old-space-size=8192";
+            GOPRIVATE = "coder.com,cdr.dev,go.coder.com,github.com/cdr,github.com/coder";
           };
         };
 
@@ -252,13 +261,20 @@
               drv = devShells.default.overrideAttrs (oldAttrs: {
                 buildInputs =
                   (with pkgs; [
-                    busybox
                     coreutils
                     nix
                     curl.bin # Ensure the actual curl binary is included in the PATH
                     glibc.bin # Ensure the glibc binaries are included in the PATH
+                    jq.bin
                     binutils # ld and strings
                     filebrowser # Ensure that we're not redownloading filebrowser on each launch
+                    systemd.out
+                    service-wrapper
+                    docker_26
+                    shadow.out
+                    su
+                    ncurses # clear
+                    unzip
                   ])
                   ++ oldAttrs.buildInputs;
               });

nix/docker.nix

@@ -13,12 +13,14 @@
   runCommand,
   writeShellScriptBin,
   writeText,
+  writeTextFile,
   cacert,
   storeDir ? builtins.storeDir,
   pigz,
   zstd,
   stdenv,
   glibc,
+  sudo,
 }:
 let
   inherit (lib)
@@ -31,10 +33,18 @@ let
 
   inherit (dockerTools)
     streamLayeredImage
-    binSh
     usrBinEnv
+    caCertificates
     ;
 
+  # This provides /bin/sh, pointing to bashInteractive.
+  # The use of bashInteractive here is intentional to support cases like `docker run -it <image_name>`, so keep these use cases in mind if making any changes to how this works.
+  binSh = runCommand "bin-sh" { } ''
+    mkdir -p $out/bin
+    ln -s ${bashInteractive}/bin/bash $out/bin/sh
+    ln -s ${bashInteractive}/bin/bash $out/bin/bash
+  '';
+
   compressors = {
     none = {
       ext = "";
@@ -88,10 +98,11 @@ let
 
       staticPath = "${dirOf shell}:${
         lib.makeBinPath (
-          lib.flatten [
+          (lib.flatten [
             builder
             drv.buildInputs
-          ]
+          ])
+          ++ [ "/usr" ]
         )
       }";
 
@@ -123,11 +134,78 @@ let
         experimental-features = nix-command flakes
       '';
 
-      etcNixConf = runCommand "etcd-nix-conf" { } ''
+      etcNixConf = runCommand "etc-nix-conf" { } ''
         mkdir -p $out/etc/nix/
         ln -s ${nixConfFile} $out/etc/nix/nix.conf
       '';
 
+      sudoersFile = writeText "sudoers" ''
+        root ALL=(ALL) ALL
+        ${toString uname} ALL=(ALL) NOPASSWD:ALL
+      '';
+
+      etcSudoers = runCommand "etc-sudoers" { } ''
+        mkdir -p $out/etc/
+        cp ${sudoersFile} $out/etc/sudoers
+        chmod 440 $out/etc/sudoers
+      '';
+
+      pamSudoFile = writeText "pam-sudo" ''
+        auth       sufficient   pam_rootok.so
+        auth       required     pam_permit.so
+        account    required     pam_permit.so
+        session    required     pam_permit.so
+        session    optional     pam_xauth.so
+      '';
+
+      etcPamSudo = runCommand "etc-pam-sudo" { } ''
+        mkdir -p $out/etc/pam.d/
+        cp ${pamSudoFile} $out/etc/pam.d/sudo
+
+        # We can’t chown in a sandbox, but that’s okay for Nix store.
+        chmod 644 $out/etc/pam.d/sudo
+      '';
+
+      # Add our Docker init script
+      dockerInit = writeTextFile {
+        name = "initd-docker";
+        destination = "/etc/init.d/docker";
+        executable = true;
+
+        text = ''
+          #!/usr/bin/env sh
+          ### BEGIN INIT INFO
+          # Provides:          docker
+          # Required-Start:    $remote_fs $syslog
+          # Required-Stop:     $remote_fs $syslog
+          # Default-Start:     2 3 4 5
+          # Default-Stop:      0 1 6
+          # Short-Description: Start and stop Docker daemon
+          # Description:       This script starts and stops the Docker daemon.
+          ### END INIT INFO
+
+          case "$1" in
+            start)
+                echo "Starting dockerd"
+                SSL_CERT_FILE="${cacert}/etc/ssl/certs/ca-bundle.crt" dockerd --group=${toString gid} &
+                ;;
+            stop)
+                echo "Stopping dockerd"
+                killall dockerd
+                ;;
+            restart)
+                $0 stop
+                $0 start
+                ;;
+            *)
+                echo "Usage: $0 {start|stop|restart}"
+                exit 1
+                ;;
+          esac
+          exit 0
+        '';
+      };
+
       # https://github.com/NixOS/nix/blob/2.8.0/src/libstore/globals.hh#L464-L465
       sandboxBuildDir = "/build";
 
@@ -165,16 +243,15 @@ let
           LD_LIBRARY_PATH = lib.makeLibraryPath [ stdenv.cc.cc ];
         }
         // drvEnv
-        // {
+        // rec {
-
           # https://github.com/NixOS/nix/blob/2.8.0/src/libstore/build/local-derivation-goal.cc#L1008-L1010
           NIX_BUILD_TOP = sandboxBuildDir;
 
           # https://github.com/NixOS/nix/blob/2.8.0/src/libstore/build/local-derivation-goal.cc#L1012-L1013
-          TMPDIR = sandboxBuildDir;
+          TMPDIR = TMP;
-          TEMPDIR = sandboxBuildDir;
+          TEMPDIR = TMP;
-          TMP = sandboxBuildDir;
+          TMP = "/tmp";
-          TEMP = "/tmp";
+          TEMP = TMP;
 
           # https://github.com/NixOS/nix/blob/2.8.0/src/libstore/build/local-derivation-goal.cc#L1015-L1019
           PWD = homeDirectory;
@@ -193,7 +270,10 @@ let
       contents = [
         binSh
         usrBinEnv
+        caCertificates
         etcNixConf
+        etcSudoers
+        etcPamSudo
         (fakeNss.override {
           # Allows programs to look up the build user's home directory
           # https://github.com/NixOS/nix/blob/ffe155abd36366a870482625543f9bf924a58281/src/libstore/build/local-derivation-goal.cc#L906-L910
@@ -204,8 +284,10 @@ let
           ];
           extraGroupLines = [
             "${toString uname}:!:${toString gid}:"
+            "docker:!:${toString (builtins.sub gid 1)}:${toString uname}"
           ];
         })
+        dockerInit
       ];
 
       fakeRootCommands = ''
@@ -241,6 +323,22 @@ let
           mkdir -p ./lib64
           ln -s "${glibc}/lib64/ld-linux-x86-64.so.2" ./lib64/ld-linux-x86-64.so.2
         fi
+
+        # Copy sudo from the Nix store to a "normal" path in the container
+        mkdir -p ./usr/bin
+        cp ${sudo}/bin/sudo ./usr/bin/sudo
+
+        # Ensure root owns it & set setuid bit
+        chown 0:0 ./usr/bin/sudo
+        chmod 4755 ./usr/bin/sudo
+
+        chown root:root ./etc/pam.d/sudo
+        chown root:root ./etc/sudoers
+
+        # Create /var/run and chown it so docker command
+        # doesnt encounter permission issues.
+        mkdir -p ./var/run/
+        chown -R ${toString uid}:${toString gid} ./var/run/
       '';
 
       # Run this image as the given uid/gid