diff --git a/.github/workflows/coder.yaml b/.github/workflows/coder.yaml index 669f5989c8..ff7054279f 100644 --- a/.github/workflows/coder.yaml +++ b/.github/workflows/coder.yaml @@ -500,35 +500,21 @@ jobs: - name: Install zstd run: sudo apt-get install -y zstd - - name: Build site - run: make -B site/out/index.html - - name: Build Release run: | set -euo pipefail go mod download - mkdir -p ./dist - # build slim binaries - ./scripts/build_go_slim.sh \ - --output ./dist/ \ - --compress 22 \ - linux:amd64,armv7,arm64 \ - windows:amd64,arm64 \ - darwin:amd64,arm64 - - # build linux amd64 packages - ./scripts/build_go_matrix.sh \ - --output ./dist/ \ - --package-linux \ - linux:amd64 \ - windows:amd64 + version="$(./scripts/version.sh)" + make -j \ + build/coder_"$version"_windows_amd64.zip \ + build/coder_"$version"_linux_amd64.{tar.gz,deb} - name: Install Release run: | gcloud config set project coder-dogfood gcloud config set compute/zone us-central1-a - gcloud compute scp ./dist/coder_*_linux_amd64.deb coder:/tmp/coder.deb + gcloud compute scp ./build/coder_*_linux_amd64.deb coder:/tmp/coder.deb gcloud compute ssh coder -- sudo dpkg -i --force-confdef /tmp/coder.deb gcloud compute ssh coder -- sudo systemctl daemon-reload @@ -539,12 +525,9 @@ jobs: with: name: coder path: | - ./dist/*.zip - ./dist/*.exe - ./dist/*.tar.gz - ./dist/*.apk - ./dist/*.deb - ./dist/*.rpm + ./build/*.zip + ./build/*.tar.gz + ./build/*.deb retention-days: 7 test-js: diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 1940367808..9dbc9ffd93 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -73,50 +73,28 @@ jobs: - name: Install nfpm run: go install github.com/goreleaser/nfpm/v2/cmd/nfpm@v2.16.0 - - name: Install zstd run: sudo apt-get install -y zstd - - name: Build Site - run: make site/out/index.html - - name: Build Linux and Windows Binaries run: | set -euo pipefail go mod download - mkdir -p ./dist - # build slim binaries - ./scripts/build_go_slim.sh \ - --output ./dist/ \ - --compress 22 \ - linux:amd64,armv7,arm64 \ - windows:amd64,arm64 \ - darwin:amd64,arm64 - - # build linux and windows binaries - ./scripts/build_go_matrix.sh \ - --output ./dist/ \ - --archive \ - --package-linux \ - linux:amd64,armv7,arm64 \ - windows:amd64,arm64 + version="$(./scripts/version.sh)" + make gen/mark-fresh + make -j \ + -W coderd/database/querier.go \ + build/coder_"$version"_linux_{amd64,arm64,armv7}.{tar.gz,apk,deb,rpm} \ + build/coder_"$version"_windows_{amd64,arm64}.zip \ - name: Build Linux Docker images run: | set -euxo pipefail - # build and (maybe) push Docker images for each architecture - images=() - for arch in amd64 armv7 arm64; do - img="$( - ./scripts/build_docker.sh \ - ${{ (!github.event.inputs.dry_run && !github.event.inputs.snapshot) && '--push' || '' }} \ - --arch "$arch" \ - ./dist/coder_*_linux_"$arch" - )" - images+=("$img") - done + # build Docker images for each architecture + version="$(./scripts/version.sh)" + make -j build/coder_"$version"_linux_{amd64,arm64,armv7}.tag # we can't build multi-arch if the images aren't pushed, so quit now # if dry-running @@ -125,19 +103,18 @@ jobs: exit 0 fi - # build and push multi-arch manifest - ./scripts/build_docker_multiarch.sh \ - --push \ - "${images[@]}" + # build and push multi-arch manifest, this depends on the other images + # being pushed so will automatically push them. + make -j push/build/coder_"$version"_linux.tag # if the current version is equal to the highest (according to semver) # version in the repo, also create a multi-arch image as ":latest" and # push it if [[ "$(git tag | grep '^v' | grep -vE '(rc|dev|-|\+|\/)' | sort -r --version-sort | head -n1)" == "v$(./scripts/version.sh)" ]]; then ./scripts/build_docker_multiarch.sh \ - --push \ --target "$(./scripts/image_tag.sh --version latest)" \ - "${images[@]}" + --push \ + $(cat build/coder_"$version"_linux_{amd64,arm64,armv7}.tag) fi - name: Upload binary artifacts @@ -145,11 +122,11 @@ jobs: with: name: linux path: | - dist/*.zip - dist/*.tar.gz - dist/*.apk - dist/*.deb - dist/*.rpm + ./build/*.zip + ./build/*.tar.gz + ./build/*.apk + ./build/*.deb + ./build/*.rpm # The mac binaries get built on mac runners because they need to be signed, # and the signing tool only runs on mac. This darwin job only builds the Mac @@ -211,30 +188,17 @@ jobs: # Used for compressing embedded slim binaries brew install zstd - - name: Build Site - run: make site/out/index.html - - name: Build darwin Binaries (with signatures) run: | set -euo pipefail go mod download - mkdir -p ./dist - # build slim binaries - ./scripts/build_go_slim.sh \ - --output ./dist/ \ - --compress 22 \ - linux:amd64,armv7,arm64 \ - windows:amd64,arm64 \ - darwin:amd64,arm64 - - # build darwin binaries - ./scripts/build_go_matrix.sh \ - --output ./dist/ \ - --archive \ - --sign-darwin \ - darwin:amd64,arm64 + version="$(./scripts/version.sh)" + make gen/mark-fresh + make -j \ + build/coder_"$version"_darwin_{amd64,arm64}.zip env: + CODER_SIGN_DARWIN: "1" AC_USERNAME: ${{ secrets.AC_USERNAME }} AC_PASSWORD: ${{ secrets.AC_PASSWORD }} AC_APPLICATION_IDENTITY: BDB050EB749EDD6A80C6F119BF1382ECA119CCCC @@ -243,7 +207,7 @@ jobs: uses: actions/upload-artifact@v3 with: name: darwin - path: ./dist/coder_*.zip + path: ./build/*.zip publish: runs-on: ubuntu-latest @@ -284,8 +248,11 @@ jobs: - name: Publish Helm run: | set -euxo pipefail - ./scripts/helm.sh --push - mv ./dist/*.tgz ./artifacts/ + + version="$(./scripts/version.sh)" + make -j \ + build/coder_helm_"$version".tgz + mv ./build/*.tgz ./artifacts/ - name: Publish Release run: | diff --git a/.gitignore b/.gitignore index ae2c40f0a5..b1a821d914 100644 --- a/.gitignore +++ b/.gitignore @@ -30,6 +30,7 @@ site/**/*.typegen.ts site/build-storybook.log # Build +build/ dist/ site/out/ diff --git a/Makefile b/Makefile index 3c1a1d74aa..84165051e0 100644 --- a/Makefile +++ b/Makefile @@ -1,96 +1,344 @@ -.DEFAULT_GOAL := build +# This is the Coder Makefile. The build directory for most tasks is `build/`. +# +# These are the targets you're probably looking for: +# - clean +# - build-fat: builds all "fat" binaries for all architectures +# - build-slim: builds all "slim" binaries (no frontend or slim binaries +# embedded) for all architectures +# - release: simulate a release (mostly, does not push images) +# - build/coder(-slim)?_${os}_${arch}(.exe)?: build a single fat binary +# - build/coder_${os}_${arch}.(zip|tar.gz): build a release archive +# - build/coder_linux_${arch}.(apk|deb|rpm): build a release Linux package +# - build/coder_${version}_linux_${arch}.tag: build a release Linux Docker image +# - build/coder_helm.tgz: build a release Helm chart + +.DEFAULT_GOAL := build-fat # Use a single bash shell for each job, and immediately exit on failure SHELL := bash -.SHELLFLAGS = -ceu +.SHELLFLAGS := -ceu .ONESHELL: # This doesn't work on directories. # See https://stackoverflow.com/questions/25752543/make-delete-on-error-for-directory-targets .DELETE_ON_ERROR: -INSTALL_DIR=$(shell go env GOPATH)/bin -GOOS=$(shell go env GOOS) -GOARCH=$(shell go env GOARCH) -VERSION=$(shell ./scripts/version.sh) +# Don't print the commands in the file unless you specify VERBOSE. This is +# essentially the same as putting "@" at the start of each line. +ifndef VERBOSE +.SILENT: +endif -bin: $(shell find . -not -path './vendor/*' -type f -name '*.go') go.mod go.sum $(shell find ./examples/templates) - @echo "== This builds slim binaries for command-line usage." - @echo "== Use \"make build\" to embed the site." +# Create the output directories if they do not exist. +$(shell mkdir -p build site/out/bin) - mkdir -p ./dist - rm -rf ./dist/coder-slim_* - rm -f ./site/out/bin/coder* - ./scripts/build_go_slim.sh \ - --compress 6 \ +GOOS := $(shell go env GOOS) +GOARCH := $(shell go env GOARCH) +GOOS_BIN_EXT := $(if $(filter windows, $(GOOS)),.exe,) +VERSION := $(shell ./scripts/version.sh) + +# All ${OS}_${ARCH} combos we build for. Windows binaries have the .exe suffix. +OS_ARCHES := \ + linux_amd64 linux_arm64 linux_armv7 \ + darwin_amd64 darwin_arm64 \ + windows_amd64.exe windows_arm64.exe + +# Archive formats and their corresponding ${OS}_${ARCH} combos. +ARCHIVE_TAR_GZ := linux_amd64 linux_arm64 linux_armv7 +ARCHIVE_ZIP := \ + darwin_amd64 darwin_arm64 \ + windows_amd64 windows_arm64 + +# All package formats we build and the ${OS}_${ARCH} combos we build them for. +PACKAGE_FORMATS := apk deb rpm +PACKAGE_OS_ARCHES := linux_amd64 linux_armv7 linux_arm64 + +# All architectures we build Docker images for (Linux only). +DOCKER_ARCHES := amd64 arm64 armv7 + +# Computed variables based on the above. +CODER_SLIM_BINARIES := $(addprefix build/coder-slim_$(VERSION)_,$(OS_ARCHES)) +CODER_FAT_BINARIES := $(addprefix build/coder_$(VERSION)_,$(OS_ARCHES)) +CODER_ALL_BINARIES := $(CODER_SLIM_BINARIES) $(CODER_FAT_BINARIES) +CODER_TAR_GZ_ARCHIVES := $(foreach os_arch, $(ARCHIVE_TAR_GZ), build/coder_$(VERSION)_$(os_arch).tar.gz) +CODER_ZIP_ARCHIVES := $(foreach os_arch, $(ARCHIVE_ZIP), build/coder_$(VERSION)_$(os_arch).zip) +CODER_ALL_ARCHIVES := $(CODER_TAR_GZ_ARCHIVES) $(CODER_ZIP_ARCHIVES) +CODER_ALL_PACKAGES := $(foreach os_arch, $(PACKAGE_OS_ARCHES), $(addprefix build/coder_$(VERSION)_$(os_arch).,$(PACKAGE_FORMATS))) +CODER_ARCH_IMAGES := $(foreach arch, $(DOCKER_ARCHES), build/coder_$(VERSION)_linux_$(arch).tag) +CODER_ARCH_IMAGES_PUSHED := $(addprefix push/, $(CODER_ARCH_IMAGES)) +CODER_MAIN_IMAGE := build/coder_$(VERSION)_linux.tag + +CODER_SLIM_NOVERSION_BINARIES := $(addprefix build/coder-slim_,$(OS_ARCHES)) +CODER_FAT_NOVERSION_BINARIES := $(addprefix build/coder_,$(OS_ARCHES)) +CODER_ALL_NOVERSION_IMAGES := $(foreach arch, $(DOCKER_ARCHES), build/coder_linux_$(arch).tag) build/coder_linux.tag +CODER_ALL_NOVERSION_IMAGES_PUSHED := $(addprefix push/, $(CODER_ALL_NOVERSION_IMAGES)) + + +clean: + rm -rf build site/out + mkdir -p build site/out/bin + git restore site/out +.PHONY: clean + +build-slim: $(CODER_SLIM_BINARIES) +.PHONY: build-slim + +build-fat build-full build: $(CODER_FAT_BINARIES) +.PHONY: build-fat build-full build + +release: $(CODER_FAT_BINARIES) $(CODER_ALL_ARCHIVES) $(CODER_ALL_PACKAGES) $(CODER_ARCH_IMAGES) build/coder_helm_$(VERSION).tgz +.PHONY: release + +build/coder-slim_$(VERSION)_checksums.sha1 site/out/bin/coder.sha1: $(CODER_SLIM_BINARIES) + pushd ./site/out/bin + openssl dgst -r -sha1 coder-* | tee coder.sha1 + popd + + cp "site/out/bin/coder.sha1" "build/coder-slim_$(VERSION)_checksums.sha1" + +build/coder-slim_$(VERSION).tar: build/coder-slim_$(VERSION)_checksums.sha1 $(CODER_SLIM_BINARIES) + pushd ./site/out/bin + tar cf "../../../build/$(@F)" coder-* + popd + +build/coder-slim_$(VERSION).tar.zst site/out/bin/coder.tar.zst: build/coder-slim_$(VERSION).tar + zstd -6 \ + --force \ + --ultra \ + --long \ + --no-progress \ + -o "build/coder-slim_$(VERSION).tar.zst" \ + "build/coder-slim_$(VERSION).tar" + + cp "build/coder-slim_$(VERSION).tar.zst" "site/out/bin/coder.tar.zst" + # delete the uncompressed binaries from the embedded dir + rm site/out/bin/coder-* + +# Redirect from version-less targets to the versioned ones. There is a similar +# target for slim binaries below. +# +# Called like this: +# make build/coder_linux_amd64 +# make build/coder_windows_amd64.exe +$(CODER_FAT_NOVERSION_BINARIES): build/coder_%: build/coder_$(VERSION)_% + rm -f "$@" + ln "$<" "$@" + +# Same as above, but for slim binaries. +# +# Called like this: +# make build/coder-slim_linux_amd64 +# make build/coder-slim_windows_amd64.exe +$(CODER_SLIM_NOVERSION_BINARIES): build/coder-slim_%: build/coder-slim_$(VERSION)_% + rm -f "$@" + ln "$<" "$@" + +# "fat" binaries always depend on the site and the compressed slim binaries. +$(CODER_FAT_BINARIES): \ + site/out/index.html \ + site/out/bin/coder.sha1 \ + site/out/bin/coder.tar.zst + +# This is a handy block that parses the target to determine whether it's "slim" +# or "fat", which OS was specified and which architecture was specified. +# +# It populates the following variables: mode, os, arch_ext, arch, ext (without +# dot). +define get-mode-os-arch-ext = + mode="$$([[ "$@" = build/coder-slim* ]] && echo "slim" || echo "fat")" + os="$$(echo $@ | cut -d_ -f3)" + arch_ext="$$(echo $@ | cut -d_ -f4)" + if [[ "$$arch_ext" == *.* ]]; then + arch="$$(echo $$arch_ext | cut -d. -f1)" + ext="$${arch_ext#*.}" + else + arch="$$arch_ext" + ext="" + fi +endef + +# This task handles all builds, for both "fat" and "slim" binaries. It parses +# the target name to get the metadata for the build, so it must be specified in +# this format: +# build/coder(-slim)?_${version}_${os}_${arch}(.exe)? +# +# You should probably use the non-version targets above instead if you're +# calling this manually. +$(CODER_ALL_BINARIES): go.mod go.sum \ + $(shell find . -not -path './vendor/*' -type f -name '*.go') \ + $(shell find ./examples/templates) + + $(get-mode-os-arch-ext) + if [[ "$$os" != "windows" ]] && [[ "$$ext" != "" ]]; then + echo "ERROR: Invalid build binary extension" 1>&2 + exit 1 + fi + if [[ "$$os" == "windows" ]] && [[ "$$ext" != exe ]]; then + echo "ERROR: Windows binaries must have an .exe extension." 1>&2 + exit 1 + fi + + build_args=( \ + --os "$$os" \ + --arch "$$arch" \ --version "$(VERSION)" \ - --output ./dist/ \ - linux:amd64,armv7,arm64 \ - windows:amd64,arm64 \ - darwin:amd64,arm64 -.PHONY: bin + --output "$@" \ + ) + if [ "$$mode" == "slim" ]; then + build_args+=(--slim) + fi -GO_FILES=$(shell find . -not -path './vendor/*' -type f -name '*.go') go.mod go.sum $(shell find ./examples/templates) + ./scripts/build_go.sh "$${build_args[@]}" -build: site/out/index.html $(GO_FILES) - rm -rf ./dist - mkdir -p ./dist - rm -f ./site/out/bin/coder* + if [[ "$$mode" == "slim" ]]; then + dot_ext="" + if [[ "$$ext" != "" ]]; then + dot_ext=".$$ext" + fi - # build slim artifacts and copy them to the site output directory - ./scripts/build_go_slim.sh \ + cp "$@" "./site/out/bin/coder-$$os-$$arch$$dot_ext" + fi + +# This task builds all archives. It parses the target name to get the metadata +# for the build, so it must be specified in this format: +# build/coder_${version}_${os}_${arch}.${format} +# +# The following OS/arch/format combinations are supported: +# .tar.gz: linux_amd64, linux_arm64, linux_armv7 +# .zip: darwin_amd64, darwin_arm64, windows_amd64, windows_arm64 +# +# This depends on all fat binaries because it's difficult to do dynamic +# dependencies due to the .exe requirement on Windows. These targets are +# typically only used during release anyways. +$(CODER_ALL_ARCHIVES): $(CODER_FAT_BINARIES) + $(get-mode-os-arch-ext) + bin_ext="" + if [[ "$$os" == "windows" ]]; then + bin_ext=".exe" + fi + + ./scripts/archive.sh \ + --format "$$ext" \ + --os "$$os" \ + --output "$@" \ + "build/coder_$(VERSION)_$${os}_$${arch}$${bin_ext}" + +# This task builds all packages. It parses the target name to get the metadata +# for the build, so it must be specified in this format: +# build/coder_${version}_linux_${arch}.${format} +# +# Supports apk, deb, rpm for all linux targets. +# +# This depends on all Linux fat binaries and archives because it's difficult to +# do dynamic dependencies due to the extensions in the filenames. These targets +# are typically only used during release anyways. +# +# Packages need to run after the archives are built, otherwise they cause tar +# errors like "file changed as we read it". +CODER_PACKAGE_DEPS := $(foreach os_arch, $(PACKAGE_OS_ARCHES), build/coder_$(VERSION)_$(os_arch) build/coder_$(VERSION)_$(os_arch).tar.gz) +$(CODER_ALL_PACKAGES): $(CODER_PACKAGE_DEPS) + $(get-mode-os-arch-ext) + + ./scripts/package.sh \ + --arch "$$arch" \ + --format "$$ext" \ --version "$(VERSION)" \ - --compress 6 \ - --output ./dist/ \ - linux:amd64,armv7,arm64 \ - windows:amd64,arm64 \ - darwin:amd64,arm64 + --output "$@" \ + "build/coder_$(VERSION)_$${os}_$${arch}" - # build not-so-slim artifacts with the default name format - ./scripts/build_go_matrix.sh \ +# Redirect from version-less Docker image targets to the versioned ones. +# +# Called like this: +# make build/coder_linux_amd64.tag +$(CODER_ALL_NOVERSION_IMAGES): build/coder_%: build/coder_$(VERSION)_% +.PHONY: $(CODER_ALL_NOVERSION_IMAGES) + +# Redirect from version-less push Docker image targets to the versioned ones. +# +# Called like this: +# make push/build/coder_linux_amd64.tag +$(CODER_ALL_NOVERSION_IMAGES_PUSHED): push/build/coder_%: push/build/coder_$(VERSION)_% +.PHONY: $(CODER_ALL_NOVERSION_IMAGES_PUSHED) + +# This task builds all Docker images. It parses the target name to get the +# metadata for the build, so it must be specified in this format: +# build/coder_${version}_${os}_${arch}.tag +# +# Supports linux_amd64, linux_arm64, linux_armv7. +# +# Images need to run after the archives and packages are built, otherwise they +# cause errors like "file changed as we read it". +$(CODER_ARCH_IMAGES): build/coder_$(VERSION)_%.tag: \ + build/coder_$(VERSION)_% \ + build/coder_$(VERSION)_%.apk \ + build/coder_$(VERSION)_%.deb \ + build/coder_$(VERSION)_%.rpm \ + build/coder_$(VERSION)_%.tar.gz + + $(get-mode-os-arch-ext) + + image_tag="$$(./scripts/image_tag.sh --arch "$$arch" --version "$(VERSION)")" + ./scripts/build_docker.sh \ + --arch "$$arch" \ + --target "$$image_tag" \ --version "$(VERSION)" \ - --output ./dist/ \ - --archive \ - --package-linux \ - linux:amd64,armv7,arm64 \ - windows:amd64,arm64 \ - darwin:amd64,arm64 -.PHONY: build + "build/coder_$(VERSION)_$${os}_$${arch}" -# Builds a test binary for just Linux -build-linux-test: site/out/index.html $(GO_FILES) - rm -rf ./dist - mkdir -p ./dist - rm -f ./site/out/bin/coder* + echo "$$image_tag" > "$@" - # build slim artifacts and copy them to the site output directory - ./scripts/build_go_slim.sh \ +# Multi-arch Docker image. This requires all architecture-specific images to be +# built AND pushed. +$(CODER_MAIN_IMAGE): $(CODER_ARCH_IMAGES_PUSHED) + image_tag="$$(./scripts/image_tag.sh --version "$(VERSION)")" + ./scripts/build_docker_multiarch.sh \ + --target "$$image_tag" \ --version "$(VERSION)" \ - --compress 6 \ - --output ./dist/ \ - linux:amd64,armv7,arm64 \ - windows:amd64,arm64 \ - darwin:amd64,arm64 + $(foreach img, $^, "$$(cat "$(img:push/%=%)")") - # build not-so-slim artifacts with the default name format - ./scripts/build_go_matrix.sh \ + echo "$$image_tag" > "$@" + +# Push a Docker image. +$(CODER_ARCH_IMAGES_PUSHED): push/%: % + image_tag="$$(cat "$<")" + docker push "$$image_tag" +.PHONY: $(CODER_ARCH_IMAGES_PUSHED) + +# Push the multi-arch Docker manifest. +push/$(CODER_MAIN_IMAGE): $(CODER_MAIN_IMAGE) + image_tag="$$(cat "$<")" + docker manifest push "$$image_tag" +.PHONY: push/$(CODER_MAIN_IMAGE) + +# Shortcut for Helm chart package. +build/coder_helm.tgz: build/coder_helm_$(VERSION).tgz + rm -f "$@" + ln "$<" "$@" + +# Helm chart package. +build/coder_helm_$(VERSION).tgz: + ./scripts/helm.sh \ --version "$(VERSION)" \ - --output ./dist/ \ - --archive \ - --package-linux \ - linux:amd64 -.PHONY: build-linux-test + --output "$@" -# Runs migrations to output a dump of the database. -coderd/database/dump.sql: coderd/database/gen/dump/main.go $(wildcard coderd/database/migrations/*.sql) - go run coderd/database/gen/dump/main.go +site/out/index.html: $(shell find ./site -not -path './site/node_modules/*' -type f -name '*.tsx') $(shell find ./site -not -path './site/node_modules/*' -type f -name '*.ts') site/package.json + ./scripts/yarn_install.sh + cd site + yarn typegen + yarn build -# Generates Go code for querying the database. -coderd/database/querier.go: coderd/database/sqlc.yaml coderd/database/dump.sql $(wildcard coderd/database/queries/*.sql) coderd/database/gen/enum/main.go - coderd/database/generate.sh +install: build/coder_$(VERSION)_$(GOOS)_$(GOARCH)$(GOOS_BIN_EXT) + install_dir="$$(go env GOPATH)/bin" + output_file="$${install_dir}/coder$(GOOS_BIN_EXT)" + + mkdir -p "$$install_dir" + cp "$<" "$$output_file" +.PHONY: install + +fmt: fmt/prettier fmt/terraform fmt/shfmt +.PHONY: fmt fmt/prettier: - @echo "--- prettier" + echo "--- prettier" cd site # Avoid writing files in CI to reduce file write activity ifdef CI @@ -105,7 +353,7 @@ fmt/terraform: $(wildcard *.tf) .PHONY: fmt/terraform fmt/shfmt: $(shell shfmt -f .) - @echo "--- shfmt" + echo "--- shfmt" # Only do diff check in CI, errors on diff. ifdef CI shfmt -d $(shell shfmt -f .) @@ -114,30 +362,6 @@ else endif .PHONY: fmt/shfmt -fmt: fmt/prettier fmt/terraform fmt/shfmt -.PHONY: fmt - -gen: coderd/database/querier.go peerbroker/proto/peerbroker.pb.go provisionersdk/proto/provisioner.pb.go provisionerd/proto/provisionerd.pb.go site/src/api/typesGenerated.ts -.PHONY: gen - -install: site/out/index.html $(shell find . -not -path './vendor/*' -type f -name '*.go') go.mod go.sum $(shell find ./examples/templates) - @output_file="$(INSTALL_DIR)/coder" - - @if [[ "$(GOOS)" == "windows" ]]; then - @output_file="$${output_file}.exe" - @fi - - @echo "-- Building CLI for $(GOOS) $(GOARCH) at $$output_file" - - ./scripts/build_go.sh \ - --version "$(VERSION)" \ - --output "$$output_file" \ - --os "$(GOOS)" \ - --arch "$(GOARCH)" - - @echo -.PHONY: install - lint: lint/shellcheck lint/go .PHONY: lint @@ -148,10 +372,45 @@ lint/go: # Use shfmt to determine the shell files, takes editorconfig into consideration. lint/shellcheck: $(shell shfmt -f .) - @echo "--- shellcheck" + echo "--- shellcheck" shellcheck --external-sources $(shell shfmt -f .) .PHONY: lint/shellcheck +# all gen targets should be added here and to gen/mark-fresh +gen: \ + coderd/database/dump.sql \ + coderd/database/querier.go \ + peerbroker/proto/peerbroker.pb.go \ + provisionersdk/proto/provisioner.pb.go \ + provisionerd/proto/provisionerd.pb.go \ + site/src/api/typesGenerated.ts +.PHONY: gen + +# Mark all generated files as fresh so make thinks they're up-to-date. This is +# used during releases so we don't run generation scripts. +gen/mark-fresh: + files="coderd/database/dump.sql coderd/database/querier.go peerbroker/proto/peerbroker.pb.go provisionersdk/proto/provisioner.pb.go provisionerd/proto/provisionerd.pb.go site/src/api/typesGenerated.ts" + for file in $$files; do + echo "$$file" + if [ ! -f "$$file" ]; then + echo "File '$$file' does not exist" + exit 1 + fi + + # touch sets the mtime of the file to the current time + touch $$file + done +.PHONY: gen/mark-fresh + +# Runs migrations to output a dump of the database schema after migrations are +# applied. +coderd/database/dump.sql: coderd/database/gen/dump/main.go $(wildcard coderd/database/migrations/*.sql) + go run coderd/database/gen/dump/main.go + +# Generates Go code for querying the database. +coderd/database/querier.go: coderd/database/sqlc.yaml coderd/database/dump.sql $(wildcard coderd/database/queries/*.sql) coderd/database/gen/enum/main.go + ./coderd/database/generate.sh + peerbroker/proto/peerbroker.pb.go: peerbroker/proto/peerbroker.proto protoc \ --go_out=. \ @@ -160,14 +419,6 @@ peerbroker/proto/peerbroker.pb.go: peerbroker/proto/peerbroker.proto --go-drpc_opt=paths=source_relative \ ./peerbroker/proto/peerbroker.proto -provisionerd/proto/provisionerd.pb.go: provisionerd/proto/provisionerd.proto - protoc \ - --go_out=. \ - --go_opt=paths=source_relative \ - --go-drpc_out=. \ - --go-drpc_opt=paths=source_relative \ - ./provisionerd/proto/provisionerd.proto - provisionersdk/proto/provisioner.pb.go: provisionersdk/proto/provisioner.proto protoc \ --go_out=. \ @@ -176,13 +427,13 @@ provisionersdk/proto/provisioner.pb.go: provisionersdk/proto/provisioner.proto --go-drpc_opt=paths=source_relative \ ./provisionersdk/proto/provisioner.proto -site/out/index.html: $(shell find ./site -not -path './site/node_modules/*' -type f -name '*.tsx') $(shell find ./site -not -path './site/node_modules/*' -type f -name '*.ts') site/package.json - ./scripts/yarn_install.sh - cd site - yarn typegen - yarn build - # Restores GITKEEP files! - git checkout HEAD out +provisionerd/proto/provisionerd.pb.go: provisionerd/proto/provisionerd.proto + protoc \ + --go_out=. \ + --go_opt=paths=source_relative \ + --go-drpc_out=. \ + --go-drpc_opt=paths=source_relative \ + ./provisionerd/proto/provisionerd.proto site/src/api/typesGenerated.ts: scripts/apitypings/main.go $(shell find codersdk -type f -name '*.go') go run scripts/apitypings/main.go > site/src/api/typesGenerated.ts diff --git a/scripts/archive.sh b/scripts/archive.sh index 4f75e20c4a..647ef9a714 100755 --- a/scripts/archive.sh +++ b/scripts/archive.sh @@ -3,7 +3,7 @@ # This script creates an archive containing the given binary renamed to # `coder(.exe)?`, as well as the README.md and LICENSE files from the repo root. # -# Usage: ./archive.sh --format tar.gz [--output path/to/output.tar.gz] [--sign-darwin] [--agpl] path/to/binary +# Usage: ./archive.sh --format tar.gz --os linux/darwin/windows [--output path/to/output.tar.gz] [--sign-darwin] [--agpl] path/to/binary # # The --format parameter must be set, and must either be "zip" or "tar.gz". # @@ -14,9 +14,10 @@ # utility and then notarized using the `gon` utility, which may take a while. # $AC_APPLICATION_IDENTITY must be set and the signing certificate must be # imported for this to work. Also, the input binary must already be signed with -# the `codesign` tool.= +# the `codesign` tool. # -# If the --agpl parameter is specified, only includes AGPL license. +# If the --agpl parameter is specified, only the AGPL license is included in the +# outputted archive. # # The absolute output path is printed on success. @@ -26,10 +27,11 @@ source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" format="" output_path="" -sign_darwin=0 +sign_darwin="${CODER_SIGN_DARWIN:-0}" +os="" agpl="${CODER_BUILD_AGPL:-0}" -args="$(getopt -o "" -l format:,output:,sign-darwin,agpl -- "$@")" +args="$(getopt -o "" -l format:,output:,sign-darwin,os:,agpl -- "$@")" eval set -- "$args" while true; do case "$1" in @@ -46,10 +48,11 @@ while true; do output_path="$(realpath "$2")" shift 2 ;; + --os) + os="$2" + shift 2 + ;; --sign-darwin) - if [[ "${AC_APPLICATION_IDENTITY:-}" == "" ]]; then - error "AC_APPLICATION_IDENTITY must be set when --sign-darwin is supplied" - fi sign_darwin=1 shift ;; @@ -79,6 +82,11 @@ if [[ ! -f "$1" ]]; then fi input_file="$(realpath "$1")" +sign_darwin="$([[ "$sign_darwin" == 1 ]] && [[ "$os" == "darwin" ]] && echo 1 || echo 0)" +if [[ "$sign_darwin" == 1 ]] && [[ "${AC_APPLICATION_IDENTITY:-}" == "" ]]; then + error "AC_APPLICATION_IDENTITY must be set when --sign-darwin or CODER_SIGN_DARWIN=1 is supplied" +fi + # Check dependencies if [[ "$format" == "zip" ]]; then dependencies zip diff --git a/scripts/build_docker.sh b/scripts/build_docker.sh index 1c23668e02..06b1b2fbe6 100755 --- a/scripts/build_docker.sh +++ b/scripts/build_docker.sh @@ -3,14 +3,14 @@ # This script builds a Docker image of Coder containing the given binary, for # the given architecture. Only linux binaries are supported at this time. # -# Usage: ./build_docker.sh --arch amd64 [--version 1.2.3] [--push] path/to/coder +# Usage: ./build_docker.sh --arch amd64 [--version 1.2.3] [--target image_tag] [--push] path/to/coder # # The --arch parameter is required and accepts a Golang arch specification. It # will be automatically mapped to a suitable architecture that Docker accepts # before being passed to `docker buildx build`. # # The image will be built and tagged against the image tag returned by -# ./image_tag.sh. +# ./image_tag.sh unless a --target parameter is supplied. # # If no version is specified, defaults to the version from ./version.sh. # @@ -23,10 +23,11 @@ set -euo pipefail source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" arch="" +image_tag="" version="" push=0 -args="$(getopt -o "" -l arch:,version:,push -- "$@")" +args="$(getopt -o "" -l arch:,target:,version:,push -- "$@")" eval set -- "$args" while true; do case "$1" in @@ -34,6 +35,10 @@ while true; do arch="$2" shift 2 ;; + --target) + image_tag="$2" + shift 2 + ;; --version) version="$2" shift 2 @@ -65,7 +70,9 @@ if [[ "$version" == "" ]]; then version="$(execrelative ./version.sh)" fi -image_tag="$(execrelative ./image_tag.sh --arch "$arch" --version="$version")" +if [[ "$image_tag" == "" ]]; then + image_tag="$(execrelative ./image_tag.sh --arch "$arch" --version="$version")" +fi if [[ "$#" != 1 ]]; then error "Exactly one argument must be provided to this script, $# were supplied" diff --git a/scripts/build_docker_multiarch.sh b/scripts/build_docker_multiarch.sh index 375815d119..f2ce0c16f5 100755 --- a/scripts/build_docker_multiarch.sh +++ b/scripts/build_docker_multiarch.sh @@ -75,7 +75,7 @@ for image_tag in "$@"; do create_args+=(--amend "$image_tag") done -# Sadly, multi-arch images don't seem to support labels. +# Sadly, manifests don't seem to support labels. log "--- Creating multi-arch Docker image ($target)" docker manifest create \ "$target" \ diff --git a/scripts/build_go.sh b/scripts/build_go.sh index fc5825e827..3057c3e8cb 100755 --- a/scripts/build_go.sh +++ b/scripts/build_go.sh @@ -12,7 +12,7 @@ # with "vX" (e.g. "v7", "v8"). # # Unless overridden via --output, the built binary will be dropped in -# "$repo_root/dist/coder_$version_$os_$arch" (with a ".exe" suffix for windows +# "$repo_root/build/coder_$version_$os_$arch" (with a ".exe" suffix for windows # builds) and the absolute path to the binary will be printed to stdout on # completion. # @@ -26,13 +26,12 @@ set -euo pipefail # shellcheck source=scripts/lib.sh source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" -cdroot version="" os="${GOOS:-linux}" arch="${GOARCH:-amd64}" slim="${CODER_SLIM_BUILD:-0}" -sign_darwin=0 +sign_darwin="${CODER_SIGN_DARWIN:-0}" output_path="" agpl="${CODER_BUILD_AGPL:-0}" @@ -53,6 +52,7 @@ while true; do shift 2 ;; --output) + mkdir -p "$(dirname "$2")" output_path="$(realpath "$2")" shift 2 ;; @@ -81,6 +81,8 @@ while true; do esac done +cdroot + # Remove the "v" prefix. version="${version#v}" if [[ "$version" == "" ]]; then @@ -102,9 +104,8 @@ fi # Compute default output path. if [[ "$output_path" == "" ]]; then - dist_dir="dist" - mkdir -p "$dist_dir" - output_path="${dist_dir}/coder_${version}_${os}_${arch}" + mkdir -p "build" + output_path="build/coder_${version}_${os}_${arch}" if [[ "$os" == "windows" ]]; then output_path+=".exe" fi @@ -132,7 +133,13 @@ CGO_ENABLED=0 GOOS="$os" GOARCH="$arch" GOARM="$arm_version" go build \ "$cmd_path" 1>&2 if [[ "$sign_darwin" == 1 ]] && [[ "$os" == "darwin" ]]; then - codesign -s "$AC_APPLICATION_IDENTITY" -f -v --timestamp --options runtime "$output_path" + codesign \ + -f -v \ + -s "$AC_APPLICATION_IDENTITY" \ + --timestamp \ + --options runtime \ + "$output_path" \ + 1>&2 fi echo "$output_path" diff --git a/scripts/build_go_matrix.sh b/scripts/build_go_matrix.sh deleted file mode 100755 index cb6b2bedec..0000000000 --- a/scripts/build_go_matrix.sh +++ /dev/null @@ -1,243 +0,0 @@ -#!/usr/bin/env bash - -# This script builds multiple Go binaries for Coder with the given OS and -# architecture combinations. -# -# Usage: ./build_go_matrix.sh [--version 1.2.3-devel+abcdef] [--output dist/] [--slim] [--sign-darwin] [--archive] [--package-linux] [--agpl] os1:arch1,arch2 os2:arch1 os1:arch3 -# -# If no OS:arch combinations are provided, nothing will happen and no error will -# be returned. Slim builds are disabled by default. If no version is specified, -# defaults to the version from ./version.sh -# -# The --output parameter must be a directory with a trailing slash where all -# files will be dropped with the default name scheme -# `coder_$version_$os_$arch(.exe)?`, or must contain the `{os}` and `{arch}` -# template variables. You may also use `{version}`. Note that for windows builds -# the `.exe` suffix will be appended automatically. -# -# Unless overridden via --output, the built binary will be dropped in -# "$repo_root/dist/coder_$version_$os_$arch" (with a ".exe" suffix for windows -# builds). -# -# If the --sign-darwin parameter is specified, all darwin binaries will be -# signed using the `codesign` utility. $AC_APPLICATION_IDENTITY must be set and -# the signing certificate must be imported for this to work. -# -# If the --archive parameter is specified, all binaries will be archived using -# ./archive.sh. The --sign-darwin parameter will be carried through, and all -# archive files will be dropped in the output directory with the same name as -# the binary and the .zip (for windows and darwin) or .tar.gz extension. -# -# If the --package-linux parameter is specified, all linux binaries will be -# packaged using ./package.sh. Requires the nfpm binary. -# -# If the --agpl parameter is specified, builds only the AGPL-licensed code (no -# Coder enterprise features). - -set -euo pipefail -# shellcheck source=scripts/lib.sh -source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" - -version="" -output_path="" -slim=0 -sign_darwin=0 -archive=0 -package_linux=0 -agpl=0 - -args="$(getopt -o "" -l version:,output:,slim,sign-darwin,archive,package-linux,agpl -- "$@")" -eval set -- "$args" -while true; do - case "$1" in - --version) - version="$2" - shift 2 - ;; - --output) - output_path="$2" - shift 2 - ;; - --slim) - slim=1 - shift - ;; - --sign-darwin) - if [[ "${AC_APPLICATION_IDENTITY:-}" == "" ]]; then - error "AC_APPLICATION_IDENTITY must be set when --sign-darwin is supplied" - fi - sign_darwin=1 - shift - ;; - --archive) - archive=1 - shift - ;; - --package-linux) - package_linux=1 - shift - ;; - --agpl) - agpl=1 - shift - ;; - --) - shift - break - ;; - *) - error "Unrecognized option: $1" - ;; - esac -done - -# Verify the output path template. -if [[ "$output_path" == "" ]]; then - # Input paths are relative, so we don't cdroot at the top, but for this case - # we want it to be relative to the root. - cdroot - mkdir -p dist - output_path="$(realpath "dist/coder_{version}_{os}_{arch}")" -elif [[ "$output_path" == */ ]]; then - output_path="${output_path}coder_{version}_{os}_{arch}" -elif [[ "$output_path" != *"{os}"* ]] || [[ "$output_path" != *"{arch}"* ]]; then - # If the output path isn't a directory (ends with /) then it must have - # template variables. - error "Templated output path '$output_path' must contain {os} and {arch}" -fi - -mkdir -p "$(dirname "$output_path")" -output_path="$(realpath "$output_path")" - -# Remove the "v" prefix. -version="${version#v}" -if [[ "$version" == "" ]]; then - version="$(execrelative ./version.sh)" -fi - -# Parse the os:arch specs into an array. -specs=() -may_zip=0 -may_tar=0 -for spec in "$@"; do - spec_os="$(echo "$spec" | cut -d ":" -f 1)" - if [[ "$spec_os" == "" ]] || [[ "$spec_os" == *" "* ]]; then - error "Could not parse matrix build spec '$spec': invalid OS '$spec_os'" - fi - - # Determine which dependencies we need. - if [[ "$spec_os" == "windows" ]] || [[ "$spec_os" == "darwin" ]]; then - may_zip=1 - else - may_tar=1 - fi - - # No quoting is important here. - for spec_arch in $(echo "$spec" | cut -d ":" -f 2 | tr "," "\n"); do - if [[ "$spec_arch" == "" ]] || [[ "$spec_os" == *" "* ]]; then - error "Could not parse matrix build spec '$spec': invalid architecture '$spec_arch'" - fi - - specs+=("$spec_os:$spec_arch") - done -done - -# Remove duplicate specs while maintaining the same order. -specs_str="${specs[*]}" -specs=() -for s in $(echo "$specs_str" | tr " " "\n" | awk '!a[$0]++'); do - specs+=("$s") -done - -# Check dependencies -dependencies go -if [[ "$sign_darwin" == 1 ]]; then - dependencies jq codesign gon -fi -if [[ "$archive" == 1 ]]; then - if [[ "$may_zip" == 1 ]]; then - dependencies zip - fi - if [[ "$may_tar" == 1 ]]; then - dependencies tar - fi -fi -if [[ "$package_linux" == 1 ]]; then - dependencies nfpm -fi - -bin_name="coder" -build_args=() -if [[ "$slim" == 1 ]]; then - bin_name+="-slim" - build_args+=(--slim) -fi -if [[ "$sign_darwin" == 1 ]]; then - build_args+=(--sign-darwin) -fi -if [[ "$agpl" == 1 ]]; then - build_args+=(--agpl) -fi - -# Build each spec. -for spec in "${specs[@]}"; do - spec_os="$(echo "$spec" | cut -d ":" -f 1)" - spec_arch="$(echo "$spec" | cut -d ":" -f 2)" - - # Craft output path from the template. - spec_output="$output_path" - spec_output="${spec_output//\{os\}/"$spec_os"}" - spec_output="${spec_output//\{arch\}/"$spec_arch"}" - spec_output="${spec_output//\{version\}/"$version"}" - - spec_output_binary="$spec_output" - if [[ "$spec_os" == "windows" ]]; then - spec_output_binary+=".exe" - fi - - # Ensure parent dir. - mkdir -p "$(dirname "$spec_output")" - - log "--- Building $bin_name for $spec_os $spec_arch ($spec_output_binary)" - execrelative ./build_go.sh \ - --version "$version" \ - --os "$spec_os" \ - --arch "$spec_arch" \ - --output "$spec_output_binary" \ - "${build_args[@]}" - log - log - - if [[ "$archive" == 1 ]]; then - spec_archive_format="tar.gz" - if [[ "$spec_os" == "windows" ]] || [[ "$spec_os" == "darwin" ]]; then - spec_archive_format="zip" - fi - spec_output_archive="$spec_output.$spec_archive_format" - - archive_args=() - if [[ "$sign_darwin" == 1 ]] && [[ "$spec_os" == "darwin" ]]; then - archive_args+=(--sign-darwin) - fi - if [[ "$agpl" == 1 ]]; then - archive_args+=(--agpl) - fi - - log "--- Creating archive for $spec_os $spec_arch ($spec_output_archive)" - execrelative ./archive.sh \ - --format "$spec_archive_format" \ - --output "$spec_output_archive" \ - "${archive_args[@]}" \ - "$spec_output_binary" - log - log - fi - - if [[ "$package_linux" == 1 ]] && [[ "$spec_os" == "linux" ]]; then - execrelative ./package.sh \ - --arch "$spec_arch" \ - --version "$version" \ - "$spec_output_binary" - log - fi -done diff --git a/scripts/build_go_slim.sh b/scripts/build_go_slim.sh deleted file mode 100755 index 69320a8a3c..0000000000 --- a/scripts/build_go_slim.sh +++ /dev/null @@ -1,146 +0,0 @@ -#!/usr/bin/env bash - -# This script builds multiple "slim" Go binaries for Coder with the given OS and -# architecture combinations. This wraps ./build_go_matrix.sh. -# -# Usage: ./build_go_slim.sh [--version 1.2.3-devel+abcdef] [--output dist/] [--compress 22] [--agpl] os1:arch1,arch2 os2:arch1 os1:arch3 -# -# If no OS:arch combinations are provided, nothing will happen and no error will -# be returned. If no version is specified, defaults to the version from -# ./version.sh -# -# The --output parameter differs from ./build_go_matrix.sh, in that it does not -# accept variables such as `{os}` and `{arch}` and only accepts a directory -# ending with `/`. -# -# The built binaries are additionally copied to the site output directory so -# they can be packaged into non-slim binaries correctly. -# -# When the --compress parameter is provided, the binaries in site/bin -# will be compressed using zstd into site/bin/coder.tar.zst, this helps reduce -# final binary size significantly. -# -# If the --agpl parameter is specified, builds only the AGPL-licensed code (no -# Coder enterprise features). - -set -euo pipefail -shopt -s nullglob -# shellcheck source=scripts/lib.sh -source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" - -version="" -output_path="" -compress=0 -agpl=0 - -args="$(getopt -o "" -l version:,output:,compress:,agpl -- "$@")" -eval set -- "$args" -while true; do - case "$1" in - --version) - version="$2" - shift 2 - ;; - --output) - output_path="$2" - shift 2 - ;; - --compress) - compress="$2" - shift 2 - ;; - --agpl) - agpl=1 - shift - ;; - --) - shift - break - ;; - *) - error "Unrecognized option: $1" - ;; - esac -done - -# Check dependencies -dependencies go -if [[ $compress != 0 ]]; then - dependencies openssl tar zstd - - if [[ $compress != [0-9]* ]] || [[ $compress -gt 22 ]] || [[ $compress -lt 1 ]]; then - error "Invalid value for compress, must in in the range of [1, 22]" - fi -fi - -# Remove the "v" prefix. -version="${version#v}" -if [[ "$version" == "" ]]; then - version="$(execrelative ./version.sh)" -fi - -# Verify the output path. -if [[ "$output_path" == "" ]]; then - # Input paths are relative, so we don't cdroot at the top, but for this case - # we want it to be relative to the root. - cdroot - mkdir -p dist - output_path="$(realpath "dist/coder-slim_{version}_{os}_{arch}")" -elif [[ "$output_path" != */ ]] || [[ "$output_path" == *"{"* ]]; then - error "The output path '$output_path' cannot contain variables and must end with a slash" -else - mkdir -p "$output_path" - output_path="$(realpath "${output_path}coder-slim_{version}_{os}_{arch}")" -fi - -build_args=(--slim) -if [[ "$agpl" == 1 ]]; then - build_args+=(--agpl) -fi - -./scripts/build_go_matrix.sh \ - --version "$version" \ - --output "$output_path" \ - "${build_args[@]}" \ - "$@" - -cdroot -dest_dir="./site/out/bin" -mkdir -p "$dest_dir" -dest_dir="$(realpath "$dest_dir")" - -# Copy the binaries to the site directory. -cd "$(dirname "$output_path")" -for f in ./coder-slim_*; do - # Remove ./ prefix - name="${f#./}" - # Remove "-slim_$version" - truncated="${name//-slim_$version/}" - # Replace underscores with hyphens - hyphenated="${truncated//_/-}" - dest="$dest_dir/$hyphenated" - cp "$f" "$dest" -done - -if [[ $compress != 0 ]]; then - pushd "$dest_dir" - sha_file=coder.sha1 - sha_dest="$dest_dir/$sha_file" - log "--- Generating SHA1 for coder-slim binaries ($sha_dest)" - openssl dgst -r -sha1 coder-* | tee $sha_file - echo "$sha_dest" - log - log - - tar_name=coder.tar.zst - tar_dest="$dest_dir/$tar_name" - log "--- Compressing coder-slim binaries using zstd level $compress ($tar_dest)" - tar cf coder.tar $sha_file coder-* - rm coder-* - zstd --force --ultra --long -"${compress}" --rm --no-progress coder.tar -o $tar_name - echo "$tar_dest" - log - log - - popd -fi diff --git a/scripts/check_unstaged.sh b/scripts/check_unstaged.sh index 47bdd265ce..a6de5f0204 100755 --- a/scripts/check_unstaged.sh +++ b/scripts/check_unstaged.sh @@ -1,30 +1,28 @@ #!/bin/bash set -euo pipefail +# shellcheck source=scripts/lib.sh +source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" +cdroot -SCRIPT_DIR=$(dirname "${BASH_SOURCE[0]}") -PROJECT_ROOT=$(cd "$SCRIPT_DIR" && git rev-parse --show-toplevel) +FILES="$(git ls-files --other --modified --exclude-standard)" +if [[ "$FILES" != "" ]]; then + mapfile -t files <<<"$FILES" -( - cd "${PROJECT_ROOT}" + log + log "The following files contain unstaged changes:" + log + for file in "${files[@]}"; do + log " - $file" + done - FILES="$(git ls-files --other --modified --exclude-standard)" - if [[ "$FILES" != "" ]]; then - mapfile -t files <<<"$FILES" + log + log "These are the changes:" + log + for file in "${files[@]}"; do + git --no-pager diff "$file" 1>&2 + done - echo "The following files contain unstaged changes:" - echo - for file in "${files[@]}"; do - echo " - $file" - done - echo - - echo "These are the changes:" - echo - for file in "${files[@]}"; do - git --no-pager diff "$file" - done - exit 1 - fi -) -exit 0 + log + error "Unstaged changes, see above for details." +fi diff --git a/scripts/coder-dev.sh b/scripts/coder-dev.sh index f824b4b5a8..5f9b2d02fb 100755 --- a/scripts/coder-dev.sh +++ b/scripts/coder-dev.sh @@ -7,17 +7,18 @@ set -euo pipefail SCRIPT_DIR=$(dirname "${BASH_SOURCE[0]}") # shellcheck disable=SC1091,SC1090 source "${SCRIPT_DIR}/lib.sh" -PROJECT_ROOT=$(cd "$SCRIPT_DIR" && git rev-parse --show-toplevel) -CODER_DEV_DIR="$PROJECT_ROOT/.coderv2/" -CODER_DEV_BIN="${CODER_DEV_DIR}/coder" -if [[ ! -d "${CODER_DEV_DIR}" ]]; then - mkdir -p "${CODER_DEV_DIR}" -fi +GOOS="$(go env GOOS)" +GOARCH="$(go env GOARCH)" +CODER_DEV_BIN="build/coder_${GOOS}_${GOARCH}" + +cdroot +mkdir -p ./.coderv2 +CODER_DEV_DIR="$(realpath ./.coderv2)" if [[ ! -x "${CODER_DEV_BIN}" ]]; then echo "Run this command first:" - echo "go build -o ${CODER_DEV_BIN} ${PROJECT_ROOT}/enterprise/cmd/coder" + echo " make $CODER_DEV_BIN" exit 1 fi diff --git a/scripts/develop.sh b/scripts/develop.sh index df0cafd2ba..0020b724b0 100755 --- a/scripts/develop.sh +++ b/scripts/develop.sh @@ -5,19 +5,28 @@ # If the --agpl parameter is specified, builds only the AGPL-licensed code (no # Coder enterprise features). +SCRIPT_DIR=$(dirname "${BASH_SOURCE[0]}") +# shellcheck disable=SC1091,SC1090 +source "${SCRIPT_DIR}/lib.sh" + # Allow toggling verbose output -[[ -n ${VERBOSE:-""} ]] && set -x +[[ -n ${VERBOSE:-} ]] && set -x set -euo pipefail -agpl="${CODER_BUILD_AGPL:-0}" -args="$(getopt -o "" -l agpl -- "$@")" +password="${CODER_DEV_ADMIN_PASSWORD:-password}" + +args="$(getopt -o "" -l agpl,password: -- "$@")" eval set -- "$args" while true; do case "$1" in --agpl) - agpl=1 + export CODER_BUILD_AGPL=1 shift ;; + --password) + password="$2" + shift 2 + ;; --) shift break @@ -28,43 +37,20 @@ while true; do esac done -SCRIPT_DIR=$(dirname "${BASH_SOURCE[0]}") -# shellcheck disable=SC1091,SC1090 -source "${SCRIPT_DIR}/lib.sh" -PROJECT_ROOT=$(cd "$SCRIPT_DIR" && git rev-parse --show-toplevel) -CODER_DEV_BIN="${PROJECT_ROOT}/.coderv2/coder" -set +u -CODER_DEV_ADMIN_PASSWORD="${CODER_DEV_ADMIN_PASSWORD:-password}" -set -u - # Preflight checks: ensure we have our required dependencies, and make sure nothing is listening on port 3000 or 8080 dependencies curl git go make yarn curl --fail http://127.0.0.1:3000 >/dev/null 2>&1 && echo '== ERROR: something is listening on port 3000. Kill it and re-run this script.' && exit 1 curl --fail http://127.0.0.1:8080 >/dev/null 2>&1 && echo '== ERROR: something is listening on port 8080. Kill it and re-run this script.' && exit 1 -if [[ ! -e ./site/out/bin/coder.sha1 && ! -e ./site/out/bin/coder.tar.zst ]]; then - log - log "=======================================================================" - log "== Run 'make bin' before running this command to build binaries. ==" - log "== Without these binaries, workspaces will fail to start! ==" - log "=======================================================================" - log - exit 1 -fi +# Compile the CLI binary. This should also compile the frontend and refresh +# node_modules if necessary. +GOOS="$(go env GOOS)" +GOARCH="$(go env GOARCH)" +make "build/coder_${GOOS}_${GOARCH}" -cmd_path="enterprise/cmd/coder" -if [[ "$agpl" == 1 ]]; then - cmd_path="cmd/coder" -fi - -# Compile the CLI binary once just so we don't waste time compiling things multiple times -go build -tags embed -o "${CODER_DEV_BIN}" "${PROJECT_ROOT}/${cmd_path}" # Use the coder dev shim so we don't overwrite the user's existing Coder config. CODER_DEV_SHIM="${PROJECT_ROOT}/scripts/coder-dev.sh" -# Run yarn install, to make sure node_modules are ready to go -"$PROJECT_ROOT/scripts/yarn_install.sh" - # This is a way to run multiple processes in parallel, and have Ctrl-C work correctly # to kill both at the same time. For more details, see: # https://stackoverflow.com/questions/3004811/how-do-you-run-multiple-programs-in-parallel-from-a-bash-script @@ -73,51 +59,49 @@ CODER_DEV_SHIM="${PROJECT_ROOT}/scripts/coder-dev.sh" # rather than leaving things in an inconsistent state. trap 'kill -TERM -$$' ERR cdroot - "${CODER_DEV_SHIM}" server --address 127.0.0.1:3000 --in-memory --tunnel || kill -INT -$$ & + "${CODER_DEV_SHIM}" server --address 127.0.0.1:3000 --tunnel || kill -INT -$$ & echo '== Waiting for Coder to become ready' timeout 60s bash -c 'until curl -s --fail http://localhost:3000 > /dev/null 2>&1; do sleep 0.5; done' - # create the first user, the admin - "${CODER_DEV_SHIM}" login http://127.0.0.1:3000 --username=admin --email=admin@coder.com --password="${CODER_DEV_ADMIN_PASSWORD}" || + # Try to create the initial admin user. + "${CODER_DEV_SHIM}" login http://127.0.0.1:3000 --username=admin --email=admin@coder.com --password="${password}" || echo 'Failed to create admin user. To troubleshoot, try running this command manually.' - # || true to always exit code 0. If this fails, whelp. - "${CODER_DEV_SHIM}" users create --email=member@coder.com --username=member --password="${CODER_DEV_ADMIN_PASSWORD}" || + # Try to create a regular user. + "${CODER_DEV_SHIM}" users create --email=member@coder.com --username=member --password="${password}" || echo 'Failed to create regular user. To troubleshoot, try running this command manually.' - # If we have docker available, then let's try to create a template! - template_name="" - if docker info >/dev/null 2>&1; then - temp_template_dir=$(mktemp -d) - echo code-server | "${CODER_DEV_SHIM}" templates init "${temp_template_dir}" - # shellcheck disable=SC1090 - source <(go env | grep GOARCH) - DOCKER_HOST=$(docker context inspect --format '{{.Endpoints.docker.Host}}') - printf 'docker_arch: "%s"\ndocker_host: "%s"\n' "${GOARCH}" "${DOCKER_HOST}" | tee "${temp_template_dir}/params.yaml" - template_name="docker-${GOARCH}" + # If we have docker available and the "docker" template doesn't already + # exist, then let's try to create a template! + example_template="code-server" + template_name="docker" + if docker info >/dev/null 2>&1 && ! "${CODER_DEV_SHIM}" templates versions list "${template_name}"; then + temp_template_dir="$(mktemp -d)" + echo "${example_template}" | "${CODER_DEV_SHIM}" templates init "${temp_template_dir}" + + DOCKER_HOST="$(docker context inspect --format '{{ .Endpoints.docker.Host }}')" + printf 'docker_arch: "%s"\ndocker_host: "%s"\n' "${GOARCH}" "${DOCKER_HOST}" >"${temp_template_dir}/params.yaml" ( - "${CODER_DEV_SHIM}" templates create "${template_name}" --directory "${temp_template_dir}" --parameter-file "${temp_template_dir}/params.yaml" --yes && - rm -rfv "${temp_template_dir}" # Only delete template dir if template creation succeeds + "${CODER_DEV_SHIM}" templates create "${template_name}" --directory "${temp_template_dir}" --parameter-file "${temp_template_dir}/params.yaml" --yes + rm -rfv "${temp_template_dir}" # Only delete template dir if template creation succeeds ) || echo "Failed to create a template. The template files are in ${temp_template_dir}" fi # Start the frontend once we have a template up and running CODER_HOST=http://127.0.0.1:3000 INSPECT_XSTATE=true yarn --cwd=./site dev || kill -INT -$$ & + log - log "=======================================================================" - log "== ==" - log "== Coder is now running in development mode. ==" - log "== API : http://localhost:3000 ==" - log "== Web UI: http://localhost:8080 ==" - if [[ -n "${template_name}" ]]; then - log "== ==" - log "== Docker template ${template_name} is ready to use! ==" - log "== Use ./scripts/coder-dev.sh to talk to this instance! ==" - log "== ==" - fi - log "=======================================================================" + log "====================================================================" + log "== ==" + log "== Coder is now running in development mode. ==" + log "== API: http://localhost:3000 ==" + log "== Web UI: http://localhost:8080 ==" + log "== ==" + log "== Use ./scripts/coder-dev.sh to talk to this instance! ==" + log "====================================================================" log + # Wait for both frontend and backend to exit. wait ) diff --git a/scripts/helm.sh b/scripts/helm.sh index 5978a5f373..33b556f010 100755 --- a/scripts/helm.sh +++ b/scripts/helm.sh @@ -9,7 +9,7 @@ # If no version is specified, defaults to the version from ./version.sh. # # If no output path is specified, defaults to -# "$repo_root/dist/coder_helm_$version.tgz". +# "$repo_root/build/coder_helm_$version.tgz". # # If the --push parameter is specified, the resulting artifact will be published # to the Coder OSS repo. This requires `gsutil` to be installed and configured. @@ -56,8 +56,8 @@ fi if [[ "$output_path" == "" ]]; then cdroot - mkdir -p dist - output_path="$(realpath "dist/coder_helm_$version.tgz")" + mkdir -p build + output_path="$(realpath "build/coder_helm_$version.tgz")" fi # Check dependencies diff --git a/scripts/package.sh b/scripts/package.sh index 1b07e8fd56..50db188aba 100755 --- a/scripts/package.sh +++ b/scripts/package.sh @@ -1,22 +1,21 @@ #!/usr/bin/env bash -# This script creates Linux packages for the given binary. It will output a -# .rpm, .deb and .apk file in the same directory as the input file with the same -# filename (except the package format suffix). +# This script creates a Linux package for the given binary. # -# ./package.sh --arch amd64 [--version 1.2.3] path/to/coder +# ./package.sh --arch amd64 --format "(apk|deb|rpm)" --output "path/to/coder.apk" [--version 1.2.3] path/to/coder # -# The --arch parameter is required. If no version is specified, defaults to the -# version from ./version.sh. +# If no version is specified, defaults to the version from ./version.sh. set -euo pipefail # shellcheck source=scripts/lib.sh source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" -version="" arch="" +format="" +output_path="" +version="" -args="$(getopt -o "" -l arch:,version: -- "$@")" +args="$(getopt -o "" -l arch:,format:,output:,version: -- "$@")" eval set -- "$args" while true; do case "$1" in @@ -24,6 +23,15 @@ while true; do arch="$2" shift 2 ;; + --format) + format="$2" + shift 2 + ;; + --output) + mkdir -p "$(dirname "$2")" + output_path="$(realpath "$2")" + shift 2 + ;; --version) version="$2" shift 2 @@ -41,6 +49,12 @@ done if [[ "$arch" == "" ]]; then error "--arch is a required parameter" fi +if [[ "$format" != "apk" ]] && [[ "$format" != "deb" ]] && [[ "$format" != "rpm" ]]; then + error "--format is a required parameter and must be one of 'apk', 'deb', or 'rpm'" +fi +if [[ "$output_path" == "" ]]; then + error "--output is a required parameter" +fi if [[ "$#" != 1 ]]; then error "Exactly one argument must be provided to this script, $# were supplied" @@ -74,18 +88,12 @@ ln "$(realpath coder.service)" "$temp_dir/" ln "$(realpath preinstall.sh)" "$temp_dir/" ln "$(realpath scripts/nfpm.yaml)" "$temp_dir/" -cd "$temp_dir" +pushd "$temp_dir" +GOARCH="$arch" CODER_VERSION="$version" nfpm package \ + -f nfpm.yaml \ + -p "$format" \ + -t "$output_path" \ + 1>&2 +popd -formats=(apk deb rpm) -for format in "${formats[@]}"; do - output_path="$input_file.$format" - log "--- Building $format package ($output_path)" - - GOARCH="$arch" CODER_VERSION="$version" nfpm package \ - -f nfpm.yaml \ - -p "$format" \ - -t "$output_path" -done - -cdroot rm -rf "$temp_dir" diff --git a/scripts/publish_release.sh b/scripts/publish_release.sh index 338ffbb99f..80725cca9a 100755 --- a/scripts/publish_release.sh +++ b/scripts/publish_release.sh @@ -3,6 +3,10 @@ # This script generates release notes and publishes all of the given assets to # GitHub releases. Depends on GitHub CLI. # +# THIS IS NOT INTENDED TO BE CALLED BY DEVELOPERS! This is called by the release +# pipeline to do the final publish step. If you want to create a release use: +# git tag -a -m "$ver" "$ver" && git push origin "$ver" +# # Usage: ./publish_release.sh [--version 1.2.3] [--dry-run] path/to/asset1 path/to/asset2 ... # # The supplied images must already be pushed to the registry or this will fail. @@ -25,6 +29,10 @@ set -euo pipefail # shellcheck source=scripts/lib.sh source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" +if [[ "${CI:-}" == "" ]]; then + error "This script must be run in CI" +fi + version="" dry_run=0 diff --git a/scripts/yarn_install.sh b/scripts/yarn_install.sh index 45a7dbf7e5..ed235ba3da 100755 --- a/scripts/yarn_install.sh +++ b/scripts/yarn_install.sh @@ -5,39 +5,37 @@ # # Usage: yarn_install.sh [optional extra flags] -set -eo pipefail +set -euo pipefail +# shellcheck source=scripts/lib.sh +source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" -SCRIPT_DIR=$(dirname "${BASH_SOURCE[0]}") -PROJECT_ROOT=$(cd "$SCRIPT_DIR" && git rev-parse --show-toplevel) +cdroot +cd site -( - cd "$PROJECT_ROOT/site" +yarn_flags=( + # Do not execute install scripts + # TODO: check if build works properly with this enabled + # --ignore-scripts - yarn_flags=( - # Do not execute install scripts - # TODO: check if build works properly with this enabled - # --ignore-scripts - - # Check if existing node_modules are valid - # TODO: determine if this is necessary - # --check-files - ) - - if [[ -n ${CI:-} ]]; then - yarn_flags+=( - # Install dependencies from lockfile, ensuring builds are fully - # reproducible - --frozen-lockfile - # Suppress progress information - --silent - # Disable interactive prompts for build - --non-interactive - ) - fi - - # Append whatever is specified on the command line - yarn_flags+=("$@") - - echo "+ yarn install ${yarn_flags[*]}" - yarn install "${yarn_flags[@]}" + # Check if existing node_modules are valid + # TODO: determine if this is necessary + # --check-files ) + +if [[ -n ${CI:-} ]]; then + yarn_flags+=( + # Install dependencies from lockfile, ensuring builds are fully + # reproducible + --frozen-lockfile + # Suppress progress information + --silent + # Disable interactive prompts for build + --non-interactive + ) +fi + +# Append whatever is specified on the command line +yarn_flags+=("$@") + +echo "+ yarn install ${yarn_flags[*]}" +yarn install "${yarn_flags[@]}" diff --git a/site/webpack.common.ts b/site/webpack.common.ts index e482c4e1a5..ff82412c8f 100644 --- a/site/webpack.common.ts +++ b/site/webpack.common.ts @@ -60,6 +60,7 @@ export const createCommonWebpackConfig = (options?: { skipTypecheck: boolean }): // REMARK: It's important to use [contenthash] here to invalidate caches. filename: "bundle.[contenthash].js", path: path.resolve(__dirname, "out"), + clean: false, }, // modules specify how different modules are loaded diff --git a/site/webpack.prod.ts b/site/webpack.prod.ts index 8b4a4e88dc..92ae6c6d47 100644 --- a/site/webpack.prod.ts +++ b/site/webpack.prod.ts @@ -47,8 +47,11 @@ export const config: Configuration = { output: { ...commonWebpackConfig.output, - // regenerate the entire dist/ directory when producing production builds - clean: true, + // Regenerate the entire out/ directory (except GITKEEP and out/bin/) when + // producing production builds + clean: { + keep: /(GITKEEP|bin\/)/, + }, }, plugins: [