diff --git a/Dockerfile b/Dockerfile
index b28d92b3cb..34b398093d 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -15,12 +15,16 @@ LABEL \
 	org.opencontainers.image.version="$CODER_VERSION" \
 	org.opencontainers.image.licenses="AGPL-3.0"
 
-# Create coder group and user. We cannot use `addgroup` and `adduser` because
-# they won't work if we're building the image for a different architecture.
-COPY --chown=root:root --chmod=644 group passwd /etc/
-
 # The coder binary is injected by scripts/build_docker.sh.
 COPY --chown=coder:coder --chmod=755 coder /opt/coder
 
+# Create coder group and user. We cannot use `addgroup` and `adduser` because
+# they won't work if we're building the image for a different architecture.
+COPY --chown=root:root --chmod=644 group passwd /etc/
+COPY --chown=coder:coder --chmod=700 empty-dir /home/coder
+
 USER coder:coder
+ENV HOME=/home/coder
+WORKDIR /home/coder
+
 ENTRYPOINT [ "/opt/coder", "server" ]
diff --git a/scripts/build_docker.sh b/scripts/build_docker.sh
index de5c3c0dbd..c84913e658 100755
--- a/scripts/build_docker.sh
+++ b/scripts/build_docker.sh
@@ -110,6 +110,7 @@ docker rm "$temp_container_id" 1>&2
 
 echo "coder:x:1000:coder" >>./group
 echo "coder:x:1000:1000::/:/bin/sh" >>./passwd
+mkdir ./empty-dir
 
 docker buildx build \
 	--platform "$arch" \