diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index a65c411ceb..1464c029a3 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,6 +1,6 @@ { "name": "Development environments on your infrastructure", - "image": "codercom/oss-dogfood:pre-nix", + "image": "codercom/oss-dogfood:latest", "features": { // See all possible options here https://github.com/devcontainers/features/tree/main/src/docker-in-docker diff --git a/.github/workflows/dogfood.yaml b/.github/workflows/dogfood.yaml index f90531e031..99b49033bc 100644 --- a/.github/workflows/dogfood.yaml +++ b/.github/workflows/dogfood.yaml @@ -5,15 +5,11 @@ on: branches: - main paths: - - "flake.nix" - - "flake.lock" - "dogfood/**" - ".github/workflows/dogfood.yaml" # Uncomment these lines when testing with CI. # pull_request: # paths: - # - "flake.nix" - # - "flake.lock" # - "dogfood/**" # - ".github/workflows/dogfood.yaml" workflow_dispatch: @@ -37,13 +33,8 @@ jobs: tag=${tag//\//--} echo "tag=${tag}" >> $GITHUB_OUTPUT - - name: Install Nix - uses: DeterminateSystems/nix-installer-action@v8 - - - name: Run the Magic Nix Cache - uses: DeterminateSystems/magic-nix-cache-action@v2 - - - run: nix build .#devEnvImage && ./result | docker load + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 - name: Login to DockerHub uses: docker/login-action@v3 @@ -51,10 +42,15 @@ jobs: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_PASSWORD }} - - name: Tag and Push - run: | - docker tag codercom/oss-dogfood:latest codercom/oss-dogfood:${{ steps.docker-tag-name.outputs.tag }} - docker push codercom/oss-dogfood -a + - name: Build and push + uses: docker/build-push-action@v4 + with: + context: "{{defaultContext}}:dogfood" + pull: true + push: true + tags: "codercom/oss-dogfood:${{ steps.docker-tag-name.outputs.tag }},codercom/oss-dogfood:latest" + cache-from: type=registry,ref=codercom/oss-dogfood:latest + cache-to: type=inline deploy_template: needs: deploy_image diff --git a/dogfood/Dockerfile b/dogfood/Dockerfile new file mode 100644 index 0000000000..17774d53da --- /dev/null +++ b/dogfood/Dockerfile @@ -0,0 +1,348 @@ +FROM rust:slim AS rust-utils +# Install rust helper programs +# ENV CARGO_NET_GIT_FETCH_WITH_CLI=true +ENV CARGO_INSTALL_ROOT=/tmp/ +RUN cargo install exa bat ripgrep typos-cli watchexec-cli + +FROM ubuntu:jammy AS go + +RUN apt-get update && apt-get install --yes curl gcc +# Install Go manually, so that we can control the version +ARG GO_VERSION=1.21.4 +RUN mkdir --parents /usr/local/go + +# Boring Go is needed to build FIPS-compliant binaries. +RUN curl --silent --show-error --location \ + "https://go.dev/dl/go${GO_VERSION}.linux-amd64.tar.gz" \ + -o /usr/local/go.tar.gz + +RUN tar --extract --gzip --directory=/usr/local/go --file=/usr/local/go.tar.gz --strip-components=1 + +ENV PATH=$PATH:/usr/local/go/bin + +# Install Go utilities. +ARG GOPATH="/tmp/" +RUN mkdir --parents "$GOPATH" && \ + # moq for Go tests. + go install github.com/matryer/moq@v0.2.3 && \ + # swag for Swagger doc generation + go install github.com/swaggo/swag/cmd/swag@v1.7.4 && \ + # go-swagger tool to generate the go coder api client + go install github.com/go-swagger/go-swagger/cmd/swagger@v0.28.0 && \ + # goimports for updating imports + go install golang.org/x/tools/cmd/goimports@v0.1.7 && \ + # protoc-gen-go is needed to build sysbox from source + go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.30 && \ + # drpc support for v2 + go install storj.io/drpc/cmd/protoc-gen-go-drpc@v0.0.33 && \ + # migrate for migration support for v2 + go install github.com/golang-migrate/migrate/v4/cmd/migrate@v4.15.1 && \ + # goreleaser for compiling v2 binaries + go install github.com/goreleaser/goreleaser@v1.6.1 && \ + # Install the latest version of gopls for editors that support + # the language server protocol + go install golang.org/x/tools/gopls@latest && \ + # gotestsum makes test output more readable + go install gotest.tools/gotestsum@v1.9.0 && \ + # goveralls collects code coverage metrics from tests + # and sends to Coveralls + go install github.com/mattn/goveralls@v0.0.11 && \ + # kind for running Kubernetes-in-Docker, needed for tests + go install sigs.k8s.io/kind@v0.10.0 && \ + # helm-docs generates our Helm README based on a template and the + # charts and values files + go install github.com/norwoodj/helm-docs/cmd/helm-docs@v1.5.0 && \ + # sqlc for Go code generation + go install github.com/sqlc-dev/sqlc/cmd/sqlc@v1.20.0 && \ + # gcr-cleaner-cli used by CI to prune unused images + go install github.com/sethvargo/gcr-cleaner/cmd/gcr-cleaner-cli@v0.5.1 && \ + # ruleguard for checking custom rules, without needing to run all of + # golangci-lint. Check the go.mod in the release of golangci-lint that + # we're using for the version of go-critic that it embeds, then check + # the version of ruleguard in go-critic for that tag. + go install github.com/quasilyte/go-ruleguard/cmd/ruleguard@v0.3.13 && \ + # go-fuzz for fuzzy testing. they don't publish releases so we rely on latest. + go install github.com/dvyukov/go-fuzz/go-fuzz@latest && \ + go install github.com/dvyukov/go-fuzz/go-fuzz-build@latest && \ + # go-releaser for building 'fat binaries' that work cross-platform + go install github.com/goreleaser/goreleaser@v1.6.1 && \ + go install mvdan.cc/sh/v3/cmd/shfmt@latest && \ + # nfpm is used with `make build` to make release packages + go install github.com/goreleaser/nfpm/v2/cmd/nfpm@v2.16.0 && \ + # yq v4 is used to process yaml files in coder v2. Conflicts with + # yq v3 used in v1. + go install github.com/mikefarah/yq/v4@v4.30.6 && \ + mv /tmp/bin/yq /tmp/bin/yq4 && \ + go install github.com/golang/mock/mockgen@v1.6.0 + +FROM gcr.io/coder-dev-1/alpine:3.18 as proto +WORKDIR /tmp +RUN apk add curl unzip +RUN curl -L -o protoc.zip https://github.com/protocolbuffers/protobuf/releases/download/v23.3/protoc-23.3-linux-x86_64.zip +RUN unzip protoc.zip + +FROM ubuntu:jammy + +SHELL ["/bin/bash", "-c"] + +# Updated certificates are necessary to use the teraswitch mirror. +# This must be ran before copying in configuration since the config replaces +# the default mirror with teraswitch. +RUN apt-get update && apt-get install --yes ca-certificates + +COPY files / + +# Install packages from apt repositories +ARG DEBIAN_FRONTEND="noninteractive" + +RUN apt-get update --quiet && apt-get install --yes \ + apt-transport-https \ + apt-utils \ + bash \ + bash-completion \ + bats \ + bind9-dnsutils \ + build-essential \ + ca-certificates \ + cmake \ + crypto-policies \ + curl \ + fd-find \ + file \ + git \ + gnupg \ + graphviz \ + htop \ + httpie \ + inetutils-tools \ + iproute2 \ + iputils-ping \ + iputils-tracepath \ + jq \ + language-pack-en \ + less \ + lsb-release \ + man \ + meld \ + net-tools \ + openjdk-11-jdk-headless \ + openssh-server \ + openssl \ + libssl-dev \ + pkg-config \ + python3 \ + python3-pip \ + rsync \ + shellcheck \ + strace \ + sudo \ + tcptraceroute \ + termshark \ + traceroute \ + vim \ + wget \ + xauth \ + zip \ + ncdu \ + cargo \ + asciinema \ + zsh \ + ansible \ + neovim \ + google-cloud-sdk \ + google-cloud-sdk-datastore-emulator \ + kubectl \ + postgresql-13 \ + containerd.io \ + docker-ce \ + docker-ce-cli \ + docker-compose-plugin \ + packer \ + terraform \ + fish \ + unzip \ + zstd \ + screen \ + gettext-base && \ + # Delete package cache to avoid consuming space in layer + apt-get clean && \ + # Configure FIPS-compliant policies + update-crypto-policies --set FIPS + +# Install the docker buildx component. +RUN DOCKER_BUILDX_VERSION=$(curl -s "https://api.github.com/repos/docker/buildx/releases/latest" | grep '"tag_name":' | sed -E 's/.*"(v[^"]+)".*/\1/') && \ + mkdir -p /usr/local/lib/docker/cli-plugins && \ + curl -Lo /usr/local/lib/docker/cli-plugins/docker-buildx "https://github.com/docker/buildx/releases/download/${DOCKER_BUILDX_VERSION}/buildx-${DOCKER_BUILDX_VERSION}.linux-amd64" && \ + chmod a+x /usr/local/lib/docker/cli-plugins/docker-buildx + +# See https://github.com/cli/cli/issues/6175#issuecomment-1235984381 for proof +# the apt repository is unreliable +RUN GH_CLI_VERSION=$(curl -s "https://api.github.com/repos/cli/cli/releases/latest" | grep '"tag_name":' | sed -E 's/.*"v([^"]+)".*/\1/') && \ + curl -L https://github.com/cli/cli/releases/download/v${GH_CLI_VERSION}/gh_${GH_CLI_VERSION}_linux_amd64.deb -o gh.deb && \ + dpkg -i gh.deb && \ + rm gh.deb + +# Install Lazygit +# See https://github.com/jesseduffield/lazygit#ubuntu +RUN LAZYGIT_VERSION=$(curl -s "https://api.github.com/repos/jesseduffield/lazygit/releases/latest" | grep '"tag_name":' | sed -E 's/.*"v*([^"]+)".*/\1/') && \ + curl -Lo lazygit.tar.gz "https://github.com/jesseduffield/lazygit/releases/latest/download/lazygit_${LAZYGIT_VERSION}_Linux_x86_64.tar.gz" && \ + tar xf lazygit.tar.gz -C /usr/local/bin lazygit + +# Install frontend utilities +RUN apt-get update && \ + # Node.js (from nodesource) and Yarn (from yarnpkg) + curl -fsSL https://deb.nodesource.com/setup_18.x | sudo -E bash - &&\ + apt-get install --yes --quiet \ + nodejs yarn \ + # Install browsers for e2e testing + google-chrome-stable microsoft-edge-beta && \ + # Pre-install system dependencies that Playwright needs. npx doesn't work here + # for some reason. See https://github.com/microsoft/playwright-cli/issues/136 + npm i -g playwright@1.36.2 pnpm@^8 && playwright install-deps && \ + npm cache clean --force + +# Ensure PostgreSQL binaries are in the users $PATH. +RUN update-alternatives --install /usr/local/bin/initdb initdb /usr/lib/postgresql/13/bin/initdb 100 && \ + update-alternatives --install /usr/local/bin/postgres postgres /usr/lib/postgresql/13/bin/postgres 100 + +# Create links for injected dependencies +RUN ln --symbolic /var/tmp/coder/coder-cli/coder /usr/local/bin/coder && \ + ln --symbolic /var/tmp/coder/code-server/bin/code-server /usr/local/bin/code-server + +# Disable the PostgreSQL systemd service. +# Coder uses a custom timescale container to test the database instead. +RUN systemctl disable \ + postgresql + +# Configure systemd services for CVMs +RUN systemctl enable \ + docker \ + ssh + +# Install tools with published releases, where that is the +# preferred/recommended installation method. +ARG CLOUD_SQL_PROXY_VERSION=2.2.0 \ + DIVE_VERSION=0.10.0 \ + DOCKER_GCR_VERSION=2.1.8 \ + GOLANGCI_LINT_VERSION=1.52.2 \ + GRYPE_VERSION=0.61.1 \ + HELM_VERSION=3.12.0 \ + KUBE_LINTER_VERSION=0.6.3 \ + KUBECTX_VERSION=0.9.4 \ + STRIPE_VERSION=1.14.5 \ + TERRAGRUNT_VERSION=0.45.11 \ + TRIVY_VERSION=0.41.0 + +# cloud_sql_proxy, for connecting to cloudsql instances +# the upstream go.mod prevents this from being installed with go install +RUN curl --silent --show-error --location --output /usr/local/bin/cloud_sql_proxy "https://storage.googleapis.com/cloud-sql-connectors/cloud-sql-proxy/v${CLOUD_SQL_PROXY_VERSION}/cloud-sql-proxy.linux.amd64" && \ + chmod a=rx /usr/local/bin/cloud_sql_proxy && \ + # dive for scanning image layer utilization metrics in CI + curl --silent --show-error --location "https://github.com/wagoodman/dive/releases/download/v${DIVE_VERSION}/dive_${DIVE_VERSION}_linux_amd64.tar.gz" | \ + tar --extract --gzip --directory=/usr/local/bin --file=- dive && \ + # docker-credential-gcr is a Docker credential helper for pushing/pulling + # images from Google Container Registry and Artifact Registry + curl --silent --show-error --location "https://github.com/GoogleCloudPlatform/docker-credential-gcr/releases/download/v${DOCKER_GCR_VERSION}/docker-credential-gcr_linux_amd64-${DOCKER_GCR_VERSION}.tar.gz" | \ + tar --extract --gzip --directory=/usr/local/bin --file=- docker-credential-gcr && \ + # golangci-lint performs static code analysis for our Go code + curl --silent --show-error --location "https://github.com/golangci/golangci-lint/releases/download/v${GOLANGCI_LINT_VERSION}/golangci-lint-${GOLANGCI_LINT_VERSION}-linux-amd64.tar.gz" | \ + tar --extract --gzip --directory=/usr/local/bin --file=- --strip-components=1 "golangci-lint-${GOLANGCI_LINT_VERSION}-linux-amd64/golangci-lint" && \ + # Anchore Grype for scanning container images for security issues + curl --silent --show-error --location "https://github.com/anchore/grype/releases/download/v${GRYPE_VERSION}/grype_${GRYPE_VERSION}_linux_amd64.tar.gz" | \ + tar --extract --gzip --directory=/usr/local/bin --file=- grype && \ + # Helm is necessary for deploying Coder + curl --silent --show-error --location "https://get.helm.sh/helm-v${HELM_VERSION}-linux-amd64.tar.gz" | \ + tar --extract --gzip --directory=/usr/local/bin --file=- --strip-components=1 linux-amd64/helm && \ + # kube-linter for linting Kubernetes objects, including those + # that Helm generates from our charts + curl --silent --show-error --location "https://github.com/stackrox/kube-linter/releases/download/${KUBE_LINTER_VERSION}/kube-linter-linux" --output /usr/local/bin/kube-linter && \ + # kubens and kubectx for managing Kubernetes namespaces and contexts + curl --silent --show-error --location "https://github.com/ahmetb/kubectx/releases/download/v${KUBECTX_VERSION}/kubectx_v${KUBECTX_VERSION}_linux_x86_64.tar.gz" | \ + tar --extract --gzip --directory=/usr/local/bin --file=- kubectx && \ + curl --silent --show-error --location "https://github.com/ahmetb/kubectx/releases/download/v${KUBECTX_VERSION}/kubens_v${KUBECTX_VERSION}_linux_x86_64.tar.gz" | \ + tar --extract --gzip --directory=/usr/local/bin --file=- kubens && \ + # stripe for coder.com billing API + curl --silent --show-error --location "https://github.com/stripe/stripe-cli/releases/download/v${STRIPE_VERSION}/stripe_${STRIPE_VERSION}_linux_x86_64.tar.gz" | \ + tar --extract --gzip --directory=/usr/local/bin --file=- stripe && \ + # terragrunt for running Terraform and Terragrunt files + curl --silent --show-error --location --output /usr/local/bin/terragrunt "https://github.com/gruntwork-io/terragrunt/releases/download/v${TERRAGRUNT_VERSION}/terragrunt_linux_amd64" && \ + chmod a=rx /usr/local/bin/terragrunt && \ + # AquaSec Trivy for scanning container images for security issues + curl --silent --show-error --location "https://github.com/aquasecurity/trivy/releases/download/v${TRIVY_VERSION}/trivy_${TRIVY_VERSION}_Linux-64bit.tar.gz" | \ + tar --extract --gzip --directory=/usr/local/bin --file=- trivy + +# Add Vercel globally. We can't install it in packages.json, because it +# includes Go files which make golangci-lint unhappy. +RUN yarn global add --prefix=/usr/local \ + vercel \ + typescript \ + typescript-language-server \ + prettier && \ + yarn cache clean + +# We use yq during "make deploy" to manually substitute out fields in +# our helm values.yaml file. See https://github.com/helm/helm/issues/3141 +# +# TODO: update to 4.x, we can't do this now because it included breaking +# changes (yq w doesn't work anymore) +# RUN curl --silent --show-error --location "https://github.com/mikefarah/yq/releases/download/v4.9.0/yq_linux_amd64.tar.gz" | \ +# tar --extract --gzip --directory=/usr/local/bin --file=- ./yq_linux_amd64 && \ +# mv /usr/local/bin/yq_linux_amd64 /usr/local/bin/yq + +RUN curl --silent --show-error --location --output /usr/local/bin/yq "https://github.com/mikefarah/yq/releases/download/3.3.0/yq_linux_amd64" && \ + chmod a=rx /usr/local/bin/yq + +# Install GoLand. +RUN mkdir --parents /usr/local/goland && \ + curl --silent --show-error --location "https://download.jetbrains.com/go/goland-2021.2.tar.gz" | \ + tar --extract --gzip --directory=/usr/local/goland --file=- --strip-components=1 && \ + ln --symbolic /usr/local/goland/bin/goland.sh /usr/local/bin/goland + +# Install Antlrv4, needed to generate paramlang lexer/parser +RUN curl --silent --show-error --location --output /usr/local/lib/antlr-4.9.2-complete.jar "https://www.antlr.org/download/antlr-4.9.2-complete.jar" +ENV CLASSPATH="/usr/local/lib/antlr-4.9.2-complete.jar:${PATH}" + +# Add coder user and allow use of docker/sudo +RUN useradd coder \ + --create-home \ + --shell=/bin/bash \ + --groups=docker \ + --uid=1000 \ + --user-group + +# Adjust OpenSSH config +RUN echo "PermitUserEnvironment yes" >>/etc/ssh/sshd_config && \ + echo "X11Forwarding yes" >>/etc/ssh/sshd_config && \ + echo "X11UseLocalhost no" >>/etc/ssh/sshd_config + +# We avoid copying the extracted directory since COPY slows to minutes when there +# are a lot of small files. +COPY --from=go /usr/local/go.tar.gz /usr/local/go.tar.gz +RUN mkdir /usr/local/go && \ + tar --extract --gzip --directory=/usr/local/go --file=/usr/local/go.tar.gz --strip-components=1 + +ENV PATH=$PATH:/usr/local/go/bin + +RUN update-alternatives --install /usr/local/bin/gofmt gofmt /usr/local/go/bin/gofmt 100 + +COPY --from=go /tmp/bin /usr/local/bin +COPY --from=rust-utils /tmp/bin /usr/local/bin +COPY --from=proto /tmp/bin /usr/local/bin +COPY --from=proto /tmp/include /usr/local/bin/include + +USER coder + +# Ensure go bins are in the 'coder' user's path. Note that no go bins are +# installed in this docker file, as they'd be mounted over by the persistent +# home volume. +ENV PATH="/home/coder/go/bin:${PATH}" + +# This setting prevents Go from using the public checksum database for +# our module path prefixes. It is required because these are in private +# repositories that require authentication. +# +# For details, see: https://golang.org/ref/mod#private-modules +ENV GOPRIVATE="coder.com,cdr.dev,go.coder.com,github.com/cdr,github.com/coder" + +# Increase memory allocation to NodeJS +ENV NODE_OPTIONS="--max-old-space-size=8192" diff --git a/dogfood/Makefile b/dogfood/Makefile new file mode 100644 index 0000000000..061530f50d --- /dev/null +++ b/dogfood/Makefile @@ -0,0 +1,10 @@ +.PHONY: docker-build docker-push + +branch=$(shell git rev-parse --abbrev-ref HEAD) +build_tag=codercom/oss-dogfood:${branch} + +build: + DOCKER_BUILDKIT=1 docker build . -t ${build_tag} + +push: build + docker push ${build_tag} diff --git a/dogfood/files/etc/apt/apt.conf.d/80-no-recommends b/dogfood/files/etc/apt/apt.conf.d/80-no-recommends new file mode 100644 index 0000000000..8cb79c9638 --- /dev/null +++ b/dogfood/files/etc/apt/apt.conf.d/80-no-recommends @@ -0,0 +1,6 @@ +// Do not install recommended packages by default +APT::Install-Recommends "0"; + +// Do not install suggested packages by default (this is already +// the Ubuntu default) +APT::Install-Suggests "0"; diff --git a/dogfood/files/etc/apt/apt.conf.d/80-retries b/dogfood/files/etc/apt/apt.conf.d/80-retries new file mode 100644 index 0000000000..d7ee518525 --- /dev/null +++ b/dogfood/files/etc/apt/apt.conf.d/80-retries @@ -0,0 +1 @@ +APT::Acquire::Retries "3"; diff --git a/dogfood/files/etc/apt/preferences.d/docker b/dogfood/files/etc/apt/preferences.d/docker new file mode 100644 index 0000000000..a92c0abb03 --- /dev/null +++ b/dogfood/files/etc/apt/preferences.d/docker @@ -0,0 +1,19 @@ +# Ignore all packages from this repository by default +Package: * +Pin: origin download.docker.com +Pin-Priority: 1 + +# Docker Community Edition +Package: docker-ce +Pin: origin download.docker.com +Pin-Priority: 500 + +# Docker command-line tool +Package: docker-ce-cli +Pin: origin download.docker.com +Pin-Priority: 500 + +# containerd runtime +Package: containerd.io +Pin: origin download.docker.com +Pin-Priority: 500 diff --git a/dogfood/files/etc/apt/preferences.d/github-cli b/dogfood/files/etc/apt/preferences.d/github-cli new file mode 100644 index 0000000000..d2dce9f5f3 --- /dev/null +++ b/dogfood/files/etc/apt/preferences.d/github-cli @@ -0,0 +1,8 @@ +# Ignore all packages from this repository by default +Package: * +Pin: origin cli.github.com +Pin-Priority: 1 + +Package: gh +Pin: origin cli.github.com +Pin-Priority: 500 diff --git a/dogfood/files/etc/apt/preferences.d/google-chrome b/dogfood/files/etc/apt/preferences.d/google-chrome new file mode 100644 index 0000000000..4551ec390f --- /dev/null +++ b/dogfood/files/etc/apt/preferences.d/google-chrome @@ -0,0 +1,16 @@ +# Ignore all packages from this repository by default +Package: * +Pin: origin dl.google.com +Pin-Priority: 1 + +Package: google-chrome-stable +Pin: origin dl.google.com +Pin-Priority: 500 + +Package: google-chrome-beta +Pin: origin dl.google.com +Pin-Priority: 500 + +Package: google-chrome-unstable +Pin: origin dl.google.com +Pin-Priority: 500 diff --git a/dogfood/files/etc/apt/preferences.d/google-cloud b/dogfood/files/etc/apt/preferences.d/google-cloud new file mode 100644 index 0000000000..637b0e9bb3 --- /dev/null +++ b/dogfood/files/etc/apt/preferences.d/google-cloud @@ -0,0 +1,19 @@ +# Ignore all packages from this repository by default +Package: * +Pin: origin packages.cloud.google.com +Pin-Priority: 1 + +# Google Cloud SDK for gcloud and gsutil CLI tools +Package: google-cloud-sdk +Pin: origin packages.cloud.google.com +Pin-Priority: 500 + +# Datastore emulator for working with the licensor +Package: google-cloud-sdk-datastore-emulator +Pin: origin packages.cloud.google.com +Pin-Priority: 500 + +# Kubectl for working with Kubernetes (GKE) +Package: kubectl +Pin: origin packages.cloud.google.com +Pin-Priority: 500 diff --git a/dogfood/files/etc/apt/preferences.d/hashicorp b/dogfood/files/etc/apt/preferences.d/hashicorp new file mode 100644 index 0000000000..4323f331cc --- /dev/null +++ b/dogfood/files/etc/apt/preferences.d/hashicorp @@ -0,0 +1,14 @@ +# Ignore all packages from this repository by default +Package: * +Pin: origin apt.releases.hashicorp.com +Pin-Priority: 1 + +# Packer for creating virtual machine disk images +Package: packer +Pin: origin apt.releases.hashicorp.com +Pin-Priority: 500 + +# Terraform for managing infrastructure +Package: terraform +Pin: origin apt.releases.hashicorp.com +Pin-Priority: 500 diff --git a/dogfood/files/etc/apt/preferences.d/microsoft-edge b/dogfood/files/etc/apt/preferences.d/microsoft-edge new file mode 100644 index 0000000000..2441961ada --- /dev/null +++ b/dogfood/files/etc/apt/preferences.d/microsoft-edge @@ -0,0 +1,12 @@ +# Ignore all packages from this repository by default +Package: * +Pin: origin packages.microsoft.com +Pin-Priority: 1 + +Package: microsoft-edge-beta +Pin: origin packages.microsoft.com +Pin-Priority: 500 + +Package: microsoft-edge-dev +Pin: origin packages.microsoft.com +Pin-Priority: 500 diff --git a/dogfood/files/etc/apt/preferences.d/nodesource b/dogfood/files/etc/apt/preferences.d/nodesource new file mode 100644 index 0000000000..de55d55534 --- /dev/null +++ b/dogfood/files/etc/apt/preferences.d/nodesource @@ -0,0 +1,9 @@ +# Ignore all packages from this repository by default +Package: * +Pin: origin deb.nodesource.com +Pin-Priority: 1 + +# Node.js for building the frontend +Package: nodejs +Pin: origin deb.nodesource.com +Pin-Priority: 500 diff --git a/dogfood/files/etc/apt/preferences.d/ppa b/dogfood/files/etc/apt/preferences.d/ppa new file mode 100644 index 0000000000..1dc9da8f9f --- /dev/null +++ b/dogfood/files/etc/apt/preferences.d/ppa @@ -0,0 +1,19 @@ +# Ignore all packages from this repository by default +Package: * +Pin: origin ppa.launchpad.net +Pin-Priority: 1 + +# Ansible +Package: ansible-base +Pin: origin ppa.launchpad.net +Pin-Priority: 500 + +# Neovim +Package: neovim +Pin: origin ppa.launchpad.net +Pin-Priority: 500 + +# Neovim Runtime +Package: neovim-runtime +Pin: origin ppa.launchpad.net +Pin-Priority: 500 diff --git a/dogfood/files/etc/apt/preferences.d/yarnpkg b/dogfood/files/etc/apt/preferences.d/yarnpkg new file mode 100644 index 0000000000..7237fcad5c --- /dev/null +++ b/dogfood/files/etc/apt/preferences.d/yarnpkg @@ -0,0 +1,9 @@ +# Ignore all packages from this repository by default +Package: * +Pin: origin dl.yarnpkg.com +Pin-Priority: 1 + +# Yarn for managing Node.js packages +Package: yarn +Pin: origin dl.yarnpkg.com +Pin-Priority: 500 diff --git a/dogfood/files/etc/apt/sources.list b/dogfood/files/etc/apt/sources.list new file mode 100644 index 0000000000..745bcefcf2 --- /dev/null +++ b/dogfood/files/etc/apt/sources.list @@ -0,0 +1,3 @@ +deb https://mirror.pit.teraswitch.com/ubuntu/ jammy main restricted universe +deb https://mirror.pit.teraswitch.com/ubuntu/ jammy-updates main restricted universe +deb https://mirror.pit.teraswitch.com/ubuntu/ jammy-backports main restricted universe diff --git a/dogfood/files/etc/apt/sources.list.d/docker.list b/dogfood/files/etc/apt/sources.list.d/docker.list new file mode 100644 index 0000000000..f00cada1ad --- /dev/null +++ b/dogfood/files/etc/apt/sources.list.d/docker.list @@ -0,0 +1 @@ +deb [signed-by=/usr/share/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu jammy stable diff --git a/dogfood/files/etc/apt/sources.list.d/google-chrome.list b/dogfood/files/etc/apt/sources.list.d/google-chrome.list new file mode 100644 index 0000000000..8dd71926f2 --- /dev/null +++ b/dogfood/files/etc/apt/sources.list.d/google-chrome.list @@ -0,0 +1 @@ +deb [signed-by=/usr/share/keyrings/google-chrome.gpg] https://dl.google.com/linux/chrome/deb/ stable main diff --git a/dogfood/files/etc/apt/sources.list.d/google-cloud.list b/dogfood/files/etc/apt/sources.list.d/google-cloud.list new file mode 100644 index 0000000000..24df98effe --- /dev/null +++ b/dogfood/files/etc/apt/sources.list.d/google-cloud.list @@ -0,0 +1 @@ +deb [signed-by=/usr/share/keyrings/google-cloud.gpg] https://packages.cloud.google.com/apt cloud-sdk main diff --git a/dogfood/files/etc/apt/sources.list.d/hashicorp.list b/dogfood/files/etc/apt/sources.list.d/hashicorp.list new file mode 100644 index 0000000000..6e60053905 --- /dev/null +++ b/dogfood/files/etc/apt/sources.list.d/hashicorp.list @@ -0,0 +1 @@ +deb [signed-by=/usr/share/keyrings/hashicorp.gpg] https://apt.releases.hashicorp.com jammy main diff --git a/dogfood/files/etc/apt/sources.list.d/microsoft-edge.list b/dogfood/files/etc/apt/sources.list.d/microsoft-edge.list new file mode 100644 index 0000000000..f0c036f79a --- /dev/null +++ b/dogfood/files/etc/apt/sources.list.d/microsoft-edge.list @@ -0,0 +1 @@ +deb [signed-by=/usr/share/keyrings/microsoft.gpg] https://packages.microsoft.com/repos/edge stable main diff --git a/dogfood/files/etc/apt/sources.list.d/nodesource.list b/dogfood/files/etc/apt/sources.list.d/nodesource.list new file mode 100644 index 0000000000..a328c2c3c4 --- /dev/null +++ b/dogfood/files/etc/apt/sources.list.d/nodesource.list @@ -0,0 +1 @@ +deb [signed-by=/usr/share/keyrings/nodesource.gpg] https://deb.nodesource.com/node_16.x jammy main diff --git a/dogfood/files/etc/apt/sources.list.d/postgresql.list b/dogfood/files/etc/apt/sources.list.d/postgresql.list new file mode 100644 index 0000000000..10262f3e64 --- /dev/null +++ b/dogfood/files/etc/apt/sources.list.d/postgresql.list @@ -0,0 +1 @@ +deb [signed-by=/usr/share/keyrings/postgresql.gpg] https://apt.postgresql.org/pub/repos/apt jammy-pgdg main diff --git a/dogfood/files/etc/apt/sources.list.d/ppa.list b/dogfood/files/etc/apt/sources.list.d/ppa.list new file mode 100644 index 0000000000..e817c20915 --- /dev/null +++ b/dogfood/files/etc/apt/sources.list.d/ppa.list @@ -0,0 +1,2 @@ +deb [signed-by=/usr/share/keyrings/ansible.gpg] https://ppa.launchpadcontent.net/ansible/ansible/ubuntu focal main +deb [signed-by=/usr/share/keyrings/neovim.gpg] https://ppa.launchpadcontent.net/neovim-ppa/stable/ubuntu focal main diff --git a/dogfood/files/etc/apt/sources.list.d/security.list b/dogfood/files/etc/apt/sources.list.d/security.list new file mode 100644 index 0000000000..1f3dae8d09 --- /dev/null +++ b/dogfood/files/etc/apt/sources.list.d/security.list @@ -0,0 +1 @@ +deb http://security.ubuntu.com/ubuntu/ jammy-security main restricted universe diff --git a/dogfood/files/etc/apt/sources.list.d/yarnpkg.list b/dogfood/files/etc/apt/sources.list.d/yarnpkg.list new file mode 100644 index 0000000000..ada8a06f7b --- /dev/null +++ b/dogfood/files/etc/apt/sources.list.d/yarnpkg.list @@ -0,0 +1 @@ +deb [signed-by=/usr/share/keyrings/yarnpkg.gpg] https://dl.yarnpkg.com/debian/ stable main diff --git a/dogfood/files/etc/default/google-chrome b/dogfood/files/etc/default/google-chrome new file mode 100644 index 0000000000..8620a60543 --- /dev/null +++ b/dogfood/files/etc/default/google-chrome @@ -0,0 +1,4 @@ +# These settings are required to prevent the postinst script +# from modifying /etc/apt/sources.list.d +repo_add_once="false" +repo_reenable_on_distupgrade="false" diff --git a/dogfood/files/etc/default/microsoft-edge-beta b/dogfood/files/etc/default/microsoft-edge-beta new file mode 100644 index 0000000000..8620a60543 --- /dev/null +++ b/dogfood/files/etc/default/microsoft-edge-beta @@ -0,0 +1,4 @@ +# These settings are required to prevent the postinst script +# from modifying /etc/apt/sources.list.d +repo_add_once="false" +repo_reenable_on_distupgrade="false" diff --git a/dogfood/files/etc/docker/daemon.json b/dogfood/files/etc/docker/daemon.json new file mode 100644 index 0000000000..8e19eeeec1 --- /dev/null +++ b/dogfood/files/etc/docker/daemon.json @@ -0,0 +1,3 @@ +{ + "registry-mirrors": ["https://mirror.gcr.io"] +} diff --git a/dogfood/files/etc/sudoers.d/nopasswd b/dogfood/files/etc/sudoers.d/nopasswd new file mode 100644 index 0000000000..3283f44556 --- /dev/null +++ b/dogfood/files/etc/sudoers.d/nopasswd @@ -0,0 +1 @@ +coder ALL=(ALL) NOPASSWD:ALL diff --git a/dogfood/files/usr/share/keyrings/ansible.gpg b/dogfood/files/usr/share/keyrings/ansible.gpg new file mode 100644 index 0000000000..1731dd2b2f Binary files /dev/null and b/dogfood/files/usr/share/keyrings/ansible.gpg differ diff --git a/dogfood/files/usr/share/keyrings/docker.gpg b/dogfood/files/usr/share/keyrings/docker.gpg new file mode 100644 index 0000000000..e5dc8cfda8 Binary files /dev/null and b/dogfood/files/usr/share/keyrings/docker.gpg differ diff --git a/dogfood/files/usr/share/keyrings/github-cli.gpg b/dogfood/files/usr/share/keyrings/github-cli.gpg new file mode 100644 index 0000000000..eddea90bd7 Binary files /dev/null and b/dogfood/files/usr/share/keyrings/github-cli.gpg differ diff --git a/dogfood/files/usr/share/keyrings/google-chrome.gpg b/dogfood/files/usr/share/keyrings/google-chrome.gpg new file mode 100644 index 0000000000..cee005a738 Binary files /dev/null and b/dogfood/files/usr/share/keyrings/google-chrome.gpg differ diff --git a/dogfood/files/usr/share/keyrings/google-cloud.gpg b/dogfood/files/usr/share/keyrings/google-cloud.gpg new file mode 100644 index 0000000000..0f478144f1 Binary files /dev/null and b/dogfood/files/usr/share/keyrings/google-cloud.gpg differ diff --git a/dogfood/files/usr/share/keyrings/hashicorp.gpg b/dogfood/files/usr/share/keyrings/hashicorp.gpg new file mode 100644 index 0000000000..674dd40c42 Binary files /dev/null and b/dogfood/files/usr/share/keyrings/hashicorp.gpg differ diff --git a/dogfood/files/usr/share/keyrings/microsoft.gpg b/dogfood/files/usr/share/keyrings/microsoft.gpg new file mode 100644 index 0000000000..0cffae08d0 Binary files /dev/null and b/dogfood/files/usr/share/keyrings/microsoft.gpg differ diff --git a/dogfood/files/usr/share/keyrings/neovim.gpg b/dogfood/files/usr/share/keyrings/neovim.gpg new file mode 100644 index 0000000000..b88f69c53b Binary files /dev/null and b/dogfood/files/usr/share/keyrings/neovim.gpg differ diff --git a/dogfood/files/usr/share/keyrings/nodesource.gpg b/dogfood/files/usr/share/keyrings/nodesource.gpg new file mode 100644 index 0000000000..4f3ec4ed79 Binary files /dev/null and b/dogfood/files/usr/share/keyrings/nodesource.gpg differ diff --git a/dogfood/files/usr/share/keyrings/postgresql.gpg b/dogfood/files/usr/share/keyrings/postgresql.gpg new file mode 100644 index 0000000000..afa15cb108 Binary files /dev/null and b/dogfood/files/usr/share/keyrings/postgresql.gpg differ diff --git a/dogfood/files/usr/share/keyrings/yarnpkg.gpg b/dogfood/files/usr/share/keyrings/yarnpkg.gpg new file mode 100644 index 0000000000..32a096756e Binary files /dev/null and b/dogfood/files/usr/share/keyrings/yarnpkg.gpg differ diff --git a/dogfood/main.tf b/dogfood/main.tf index d5439434f8..034be9006b 100644 --- a/dogfood/main.tf +++ b/dogfood/main.tf @@ -255,15 +255,15 @@ locals { registry_name = "codercom/oss-dogfood" } data "docker_registry_image" "dogfood" { - // This is temporarily pinned to a pre-nix version of the image at commit - // 6cdf1c73c until the Nix kinks are worked out. - name = "${local.registry_name}:pre-nix" + name = "${local.registry_name}:latest" } resource "docker_image" "dogfood" { name = "${local.registry_name}@${data.docker_registry_image.dogfood.sha256_digest}" pull_triggers = [ - data.docker_registry_image.dogfood.sha256_digest + data.docker_registry_image.dogfood.sha256_digest, + sha1(join("", [for f in fileset(path.module, "files/*") : filesha1(f)])), + filesha1("Dockerfile"), ] keep_locally = true } diff --git a/dogfood/update-keys.sh b/dogfood/update-keys.sh new file mode 100755 index 0000000000..9ebaf77bb5 --- /dev/null +++ b/dogfood/update-keys.sh @@ -0,0 +1,62 @@ +#!/usr/bin/env bash + +set -euo pipefail + +PROJECT_ROOT="$(git rev-parse --show-toplevel)" + +curl_flags=( + --silent + --show-error + --location +) + +gpg_flags=( + --dearmor + --yes +) + +pushd "$PROJECT_ROOT/dogfood/files/usr/share/keyrings" +# Upstream Docker signing key +curl "${curl_flags[@]}" "https://download.docker.com/linux/ubuntu/gpg" | + gpg "${gpg_flags[@]}" --output="docker.gpg" + +# Google Cloud signing key +curl "${curl_flags[@]}" "https://packages.cloud.google.com/apt/doc/apt-key.gpg" | + gpg "${gpg_flags[@]}" --output="google-cloud.gpg" + +# Google Linux Software repository signing key (Chrome) +curl "${curl_flags[@]}" "https://dl.google.com/linux/linux_signing_key.pub" | + gpg "${gpg_flags[@]}" --output="google-chrome.gpg" + +# Microsoft repository signing key (Edge) +curl "${curl_flags[@]}" "https://packages.microsoft.com/keys/microsoft.asc" | + gpg "${gpg_flags[@]}" --output="microsoft.gpg" + +# Upstream PostgreSQL signing key +curl "${curl_flags[@]}" "https://www.postgresql.org/media/keys/ACCC4CF8.asc" | + gpg "${gpg_flags[@]}" --output="postgresql.gpg" + +# NodeSource signing key +curl "${curl_flags[@]}" "https://deb.nodesource.com/gpgkey/nodesource.gpg.key" | + gpg "${gpg_flags[@]}" --output="nodesource.gpg" + +# Yarnpkg signing key +curl "${curl_flags[@]}" "https://dl.yarnpkg.com/debian/pubkey.gpg" | + gpg "${gpg_flags[@]}" --output="yarnpkg.gpg" + +# Ansible PPA signing key +curl "${curl_flags[@]}" "https://keyserver.ubuntu.com/pks/lookup?op=get&search=0x6125e2a8c77f2818fb7bd15b93c4a3fd7bb9c367" | + gpg "${gpg_flags[@]}" --output="ansible.gpg" + +# Neovim signing key +curl "${curl_flags[@]}" "https://keyserver.ubuntu.com/pks/lookup?op=get&search=0x9dbb0be9366964f134855e2255f96fcf8231b6dd" | + gpg "${gpg_flags[@]}" --output="neovim.gpg" + +# Hashicorp signing key +curl "${curl_flags[@]}" "https://apt.releases.hashicorp.com/gpg" | + gpg "${gpg_flags[@]}" --output="hashicorp.gpg" + +# GitHub CLI signing key +curl "${curl_flags[@]}" "https://cli.github.com/packages/githubcli-archive-keyring.gpg" | + gpg "${gpg_flags[@]}" --output="github-cli.gpg" +popd diff --git a/flake.nix b/flake.nix index 172f91d271..82e633fcd2 100644 --- a/flake.nix +++ b/flake.nix @@ -75,167 +75,8 @@ zsh zstd ]; - # We separate these to reduce the size of the dev shell for packages that we only - # want in the image. - devImagePackages = with pkgs; [ - docker - exa - freetype - glib - harfbuzz - nix - nixpkgs-fmt - screen - ]; - - # This is the base image for our Docker container used for development. - # Use `nix-prefetch-docker ubuntu --arch amd64 --image-tag lunar` to get this. - baseDevEnvImage = pkgs.dockerTools.pullImage { - imageName = "ubuntu"; - imageDigest = "sha256:7a520eeb6c18bc6d32a21bb7edcf673a7830813c169645d51c949cecb62387d0"; - sha256 = "ajZzFSG/q7F5wAXfBOPpYBT+aVy8lqAXtBzkmAe2SeE="; - finalImageName = "ubuntu"; - finalImageTag = "lunar"; - }; - # This is an intermediate stage that adds sudo with the setuid bit set. - # Nix doesn't allow setuid binaries in the store, so we have to do this - # in a separate stage. - intermediateDevEnvImage = pkgs.dockerTools.buildImage { - name = "intermediate"; - fromImage = baseDevEnvImage; - runAsRoot = '' - #!${pkgs.runtimeShell} - ${pkgs.dockerTools.shadowSetup} - userdel ubuntu - groupadd docker - useradd coder \ - --create-home \ - --shell=/bin/bash \ - --uid=1000 \ - --user-group \ - --groups docker - cp ${pkgs.sudo}/bin/sudo usr/bin/sudo - chmod 4755 usr/bin/sudo - mkdir -p /etc/init.d - ''; - }; - allPackages = devShellPackages ++ devImagePackages; - # Environment variables that live in `/etc/environment` in the container. - # These will also be applied to the container config. - devEnvVars = [ - "PATH=${pkgs.lib.makeBinPath (allPackages)}:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/coder/go/bin" - "LD_LIBRARY_PATH=${pkgs.lib.makeLibraryPath allPackages}" - # This setting prevents Go from using the public checksum database for - # our module path prefixes. It is required because these are in private - # repositories that require authentication. - # - # For details, see: https://golang.org/ref/mod#private-modules - "GOPRIVATE=coder.com,cdr.dev,go.coder.com,github.com/cdr,github.com/coder" - # Increase memory allocation to NodeJS - "NODE_OPTIONS=--max_old_space_size=8192" - "TERM=xterm-256color" - "LANG=en_US.UTF-8" - "LOCALE_ARCHIVE=/usr/lib/locale/locale-archive" - ]; - # Builds our development environment image with all the tools included. - # Using Nix instead of Docker is **significantly** faster. This _build_ - # doesn't really build anything, it just copies pre-built binaries into - # a container and adds them to the $PATH. - # - # To test changes and iterate on this, you can run: - # > nix build .#devEnvImage && ./result | docker load - # This will import the image into your local Docker daemon. - devEnvImage = pkgs.dockerTools.streamLayeredImage { - name = "codercom/oss-dogfood"; - tag = "latest"; - fromImage = intermediateDevEnvImage; - maxLayers = 64; - contents = [ - # Required for `sudo` to persist the proper `PATH`. - ( - pkgs.writeTextDir "etc/environment" (pkgs.lib.strings.concatLines devEnvVars) - ) - # Allows `coder` to use `sudo` without a password. - ( - pkgs.writeTextDir "etc/sudoers" '' - coder ALL=(ALL) NOPASSWD:ALL - '' - ) - # Also allows `coder` to use `sudo` without a password. - ( - pkgs.writeTextDir "etc/pam.d/other" '' - account sufficient pam_unix.so - auth sufficient pam_rootok.so - password requisite pam_unix.so nullok yescrypt - session required pam_unix.so - '' - ) - # This allows users to chsh. - ( - pkgs.writeTextDir "etc/pam.d/chsh" '' - auth sufficient pam_rootok.so - '' - ) - # The default Nix config! - ( - pkgs.writeTextDir "etc/nix/nix.conf" '' - experimental-features = nix-command flakes - '' - ) - # Allow people to change shells! - ( - pkgs.writeTextDir "etc/shells" '' - /bin/bash - ${pkgs.zsh}/bin/zsh - '' - ) - # This is the debian script for managing Docker with `sudo service docker ...`. - ( - pkgs.writeTextFile { - name = "docker"; - destination = "/etc/init.d/docker"; - executable = true; - text = (builtins.readFile ( - pkgs.fetchFromGitHub - { - owner = "moby"; - repo = "moby"; - rev = "ae737656f9817fbd5afab96aa083754cfb81aab0"; - sha256 = "sha256-oS3WplsxhKHCuHwL4/ytsCNJ1N/SZhlUZmzZTf81AoE="; - } + "/contrib/init/sysvinit-debian/docker" - )); - } - ) - # The Docker script above looks here for the daemon binary location. - # Because we're injecting it with Nix, it's not in the default spot. - ( - pkgs.writeTextDir "etc/default/docker" '' - DOCKERD=${pkgs.docker}/bin/dockerd - '' - ) - # The same as `sudo apt install ca-certificates -y'. - ( - pkgs.writeTextDir "etc/ssl/certs/ca-certificates.crt" - (builtins.readFile "${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt") - ) - ]; - # Required for the UTF-8 locale to exist! - extraCommands = '' - mkdir -p usr/lib/locale - cp -a ${pkgs.glibcLocales}/lib/locale/locale-archive usr/lib/locale/locale-archive - ''; - - config = { - Env = devEnvVars; - Entrypoint = [ "/bin/bash" ]; - User = "coder"; - }; - }; in { - packages = { - devEnvImage = devEnvImage; - }; defaultPackage = formatter; # or replace it with your desired default package. devShell = pkgs.mkShell { buildInputs = devShellPackages; }; }