diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 9dbc9ffd93..dae0b3d3b3 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -1,10 +1,4 @@ # GitHub release workflow. -# -# This workflow is a bit complicated because we have to build darwin binaries on -# a mac runner, but the mac runners are extremely slow. So instead of running -# the entire release on a mac (which will take an hour to run), we run only the -# mac build on a mac, and the rest on a linux runner. The final release is then -# published using a final linux runner. name: release on: push: @@ -31,7 +25,7 @@ env: CODER_RELEASE: ${{ github.event.inputs.snapshot && 'false' || 'true' }} jobs: - linux-windows: + release: runs-on: ubuntu-latest env: # Necessary for Docker manifest @@ -72,11 +66,38 @@ jobs: js-${{ runner.os }}- - name: Install nfpm - run: go install github.com/goreleaser/nfpm/v2/cmd/nfpm@v2.16.0 + run: | + set -euo pipefail + wget -O /tmp/nfpm.deb https://github.com/goreleaser/nfpm/releases/download/v2.18.1/nfpm_amd64.deb + sudo dpkg -i /tmp/nfpm.deb - name: Install zstd run: sudo apt-get install -y zstd - - name: Build Linux and Windows Binaries + - name: Install rcodesign + run: | + set -euo pipefail + + # Install a prebuilt binary of rcodesign for linux amd64. Once the + # following PR is merged and released upstream, we can download + # directly from GitHub releases instead: + # https://github.com/indygreg/PyOxidizer/pull/635 + wget -O /tmp/rcodesign https://cdn.discordapp.com/attachments/283356472258199552/1016767245717872700/rcodesign + sudo install --mode 755 /tmp/rcodesign /usr/local/bin/rcodesign + + - name: Setup Apple Developer certificate and API key + run: | + set -euo pipefail + touch /tmp/{apple_cert.p12,apple_cert_password.txt,apple_apikey.p8} + chmod 600 /tmp/{apple_cert.p12,apple_cert_password.txt,apple_apikey.p8} + echo "$AC_CERTIFICATE_P12_BASE64" | base64 -d > /tmp/apple_cert.p12 + echo "$AC_CERTIFICATE_PASSWORD" > /tmp/apple_cert_password.txt + echo "$AC_APIKEY_P8_BASE64" | base64 -d > /tmp/apple_apikey.p8 + env: + AC_CERTIFICATE_P12_BASE64: ${{ secrets.AC_CERTIFICATE_P12_BASE64 }} + AC_CERTIFICATE_PASSWORD: ${{ secrets.AC_CERTIFICATE_PASSWORD }} + AC_APIKEY_P8_BASE64: ${{ secrets.AC_APIKEY_P8_BASE64 }} + + - name: Build binaries run: | set -euo pipefail go mod download @@ -84,9 +105,19 @@ jobs: version="$(./scripts/version.sh)" make gen/mark-fresh make -j \ - -W coderd/database/querier.go \ - build/coder_"$version"_linux_{amd64,arm64,armv7}.{tar.gz,apk,deb,rpm} \ - build/coder_"$version"_windows_{amd64,arm64}.zip \ + build/coder_"$version"_linux_{amd64,armv7,arm64}.{tar.gz,apk,deb,rpm} \ + build/coder_"$version"_{darwin,windows}_{amd64,arm64}.zip \ + build/coder_helm_"$version".tgz + env: + CODER_SIGN_DARWIN: "1" + AC_CERTIFICATE_FILE: /tmp/apple_cert.p12 + AC_CERTIFICATE_PASSWORD_FILE: /tmp/apple_cert_password.txt + AC_APIKEY_ISSUER_ID: ${{ secrets.AC_APIKEY_ISSUER_ID }} + AC_APIKEY_ID: ${{ secrets.AC_APIKEY_ID }} + AC_APIKEY_FILE: /tmp/apple_apikey.p8 + + - name: Delete Apple Developer certificate and API key + run: rm -f /tmp/{apple_cert.p12,apple_cert_password.txt,apple_apikey.p8} - name: Build Linux Docker images run: | @@ -112,157 +143,37 @@ jobs: # push it if [[ "$(git tag | grep '^v' | grep -vE '(rc|dev|-|\+|\/)' | sort -r --version-sort | head -n1)" == "v$(./scripts/version.sh)" ]]; then ./scripts/build_docker_multiarch.sh \ - --target "$(./scripts/image_tag.sh --version latest)" \ --push \ + --target "$(./scripts/image_tag.sh --version latest)" \ $(cat build/coder_"$version"_linux_{amd64,arm64,armv7}.tag) fi - - name: Upload binary artifacts - uses: actions/upload-artifact@v3 - with: - name: linux - path: | - ./build/*.zip - ./build/*.tar.gz - ./build/*.apk - ./build/*.deb - ./build/*.rpm + - name: ls build + run: ls -lh build - # The mac binaries get built on mac runners because they need to be signed, - # and the signing tool only runs on mac. This darwin job only builds the Mac - # binaries and uploads them as job artifacts used by the publish step. - darwin: - runs-on: macos-latest - steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - # If the event that triggered the build was an annotated tag (which our - # tags are supposed to be), actions/checkout has a bug where the tag in - # question is only a lightweight tag and not a full annotated tag. This - # command seems to fix it. - # https://github.com/actions/checkout/issues/290 - - name: Fetch git tags - run: git fetch --tags --force - - - uses: actions/setup-go@v3 - with: - go-version: "~1.19" - - - name: Import Signing Certificates - uses: Apple-Actions/import-codesign-certs@v1 - with: - p12-file-base64: ${{ secrets.AC_CERTIFICATE_P12_BASE64 }} - p12-password: ${{ secrets.AC_CERTIFICATE_PASSWORD }} - - - name: Cache Node - id: cache-node - uses: actions/cache@v3 - with: - path: | - **/node_modules - .eslintcache - key: js-${{ runner.os }}-test-${{ hashFiles('**/yarn.lock') }} - restore-keys: | - js-${{ runner.os }}- - - - name: Install dependencies - run: | - set -euo pipefail - # The version of bash that macOS ships with is too old - brew install bash - - # The version of make that macOS ships with is too old - brew install make - echo "$(brew --prefix)/opt/make/libexec/gnubin" >> $GITHUB_PATH - - # BSD getopt is incompatible with the build scripts - brew install gnu-getopt - echo "$(brew --prefix)/opt/gnu-getopt/bin" >> $GITHUB_PATH - - # Used for notarizing the binaries - brew tap mitchellh/gon - brew install mitchellh/gon/gon - - # Used for compressing embedded slim binaries - brew install zstd - - - name: Build darwin Binaries (with signatures) - run: | - set -euo pipefail - go mod download - - version="$(./scripts/version.sh)" - make gen/mark-fresh - make -j \ - build/coder_"$version"_darwin_{amd64,arm64}.zip - env: - CODER_SIGN_DARWIN: "1" - AC_USERNAME: ${{ secrets.AC_USERNAME }} - AC_PASSWORD: ${{ secrets.AC_PASSWORD }} - AC_APPLICATION_IDENTITY: BDB050EB749EDD6A80C6F119BF1382ECA119CCCC - - - name: Upload Binary Artifacts - uses: actions/upload-artifact@v3 - with: - name: darwin - path: ./build/*.zip - - publish: - runs-on: ubuntu-latest - needs: - - linux-windows - - darwin - steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - # If the event that triggered the build was an annotated tag (which our - # tags are supposed to be), actions/checkout has a bug where the tag in - # question is only a lightweight tag and not a full annotated tag. This - # command seems to fix it. - # https://github.com/actions/checkout/issues/290 - - name: Fetch git tags - run: git fetch --tags --force - - - name: mkdir artifacts - run: mkdir artifacts - - - name: Download darwin Artifacts - uses: actions/download-artifact@v3 - with: - name: darwin - path: artifacts - - - name: Download Linux and Windows Artifacts - uses: actions/download-artifact@v3 - with: - name: linux - path: artifacts - - - name: ls artifacts - run: ls artifacts - - - name: Publish Helm - run: | - set -euxo pipefail - - version="$(./scripts/version.sh)" - make -j \ - build/coder_helm_"$version".tgz - mv ./build/*.tgz ./artifacts/ - - - name: Publish Release + - name: Publish release run: | ./scripts/publish_release.sh \ ${{ (github.event.inputs.dry_run || github.event.inputs.snapshot) && '--dry-run' }} \ - ./artifacts/*.zip \ - ./artifacts/*.tar.gz \ - ./artifacts/*.tgz \ - ./artifacts/*.apk \ - ./artifacts/*.deb \ - ./artifacts/*.rpm + ./build/*.zip \ + ./build/*.tar.gz \ + ./build/*.tgz \ + ./build/*.apk \ + ./build/*.deb \ + ./build/*.rpm env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Upload artifacts to actions (if dry-run or snapshot) + if: ${{ github.event.inputs.dry_run || github.event.inputs.snapshot }} + uses: actions/upload-artifact@v2 + with: + name: release-artifacts + path: | + ./build/*.zip + ./build/*.tar.gz + ./build/*.tgz + ./build/*.apk + ./build/*.deb + ./build/*.rpm + retention-days: 7 diff --git a/scripts/archive.sh b/scripts/archive.sh index 647ef9a714..fd861a86d6 100755 --- a/scripts/archive.sh +++ b/scripts/archive.sh @@ -10,11 +10,9 @@ # If the --output parameter is not set, the default output path is the binary # path (minus any .exe suffix) plus the format extension ".zip" or ".tar.gz". # -# If --sign-darwin is specified, the zip file is signed with the `codesign` -# utility and then notarized using the `gon` utility, which may take a while. -# $AC_APPLICATION_IDENTITY must be set and the signing certificate must be -# imported for this to work. Also, the input binary must already be signed with -# the `codesign` tool. +# If --sign-darwin is specified, the zip file will be notarized using +# ./notarize_darwin.sh, which may take a while. Read that file for more details +# on the requirements. # # If the --agpl parameter is specified, only the AGPL license is included in the # outputted archive. @@ -82,11 +80,6 @@ if [[ ! -f "$1" ]]; then fi input_file="$(realpath "$1")" -sign_darwin="$([[ "$sign_darwin" == 1 ]] && [[ "$os" == "darwin" ]] && echo 1 || echo 0)" -if [[ "$sign_darwin" == 1 ]] && [[ "${AC_APPLICATION_IDENTITY:-}" == "" ]]; then - error "AC_APPLICATION_IDENTITY must be set when --sign-darwin or CODER_SIGN_DARWIN=1 is supplied" -fi - # Check dependencies if [[ "$format" == "zip" ]]; then dependencies zip @@ -94,8 +87,11 @@ fi if [[ "$format" == "tar.gz" ]]; then dependencies tar fi + +sign_darwin="$([[ "$sign_darwin" == 1 ]] && [[ "$os" == "darwin" ]] && echo 1 || echo 0)" if [[ "$sign_darwin" == 1 ]]; then - dependencies jq codesign gon + dependencies rcodesign + requiredenvs AC_APIKEY_ISSUER_ID AC_APIKEY_ID AC_APIKEY_FILE fi # Determine default output path. @@ -139,7 +135,7 @@ rm -rf "$temp_dir" if [[ "$sign_darwin" == 1 ]]; then log "Notarizing archive..." - execrelative ./sign_darwin.sh "$output_path" + execrelative ./notarize_darwin.sh "$output_path" fi echo "$output_path" diff --git a/scripts/build_go.sh b/scripts/build_go.sh index 3057c3e8cb..3a86498901 100755 --- a/scripts/build_go.sh +++ b/scripts/build_go.sh @@ -16,9 +16,9 @@ # builds) and the absolute path to the binary will be printed to stdout on # completion. # -# If the --sign-darwin parameter is specified and the OS is darwin, binaries -# will be signed using the `codesign` utility. $AC_APPLICATION_IDENTITY must be -# set and the signing certificate must be imported for this to work. +# If the --sign-darwin parameter is specified and the OS is darwin, the output +# binary will be signed using ./sign_darwin.sh. Read that file for more details +# on the requirements. # # If the --agpl parameter is specified, builds only the AGPL-licensed code (no # Coder enterprise features). @@ -65,9 +65,6 @@ while true; do shift ;; --sign-darwin) - if [[ "${AC_APPLICATION_IDENTITY:-}" == "" ]]; then - error "AC_APPLICATION_IDENTITY must be set when --sign-darwin is supplied" - fi sign_darwin=1 shift ;; @@ -92,7 +89,8 @@ fi # Check dependencies dependencies go if [[ "$sign_darwin" == 1 ]]; then - dependencies codesign + dependencies rcodesign + requiredenvs AC_CERTIFICATE_FILE AC_CERTIFICATE_PASSWORD_FILE fi build_args=( @@ -133,13 +131,7 @@ CGO_ENABLED=0 GOOS="$os" GOARCH="$arch" GOARM="$arm_version" go build \ "$cmd_path" 1>&2 if [[ "$sign_darwin" == 1 ]] && [[ "$os" == "darwin" ]]; then - codesign \ - -f -v \ - -s "$AC_APPLICATION_IDENTITY" \ - --timestamp \ - --options runtime \ - "$output_path" \ - 1>&2 + execrelative ./sign_darwin.sh "$output_path" 1>&2 fi echo "$output_path" diff --git a/scripts/lib.sh b/scripts/lib.sh index 02b8f234ed..67922db3dd 100644 --- a/scripts/lib.sh +++ b/scripts/lib.sh @@ -81,6 +81,21 @@ dependencies() { fi } +requiredenvs() { + local fail=0 + for env in "$@"; do + if [[ "${!env:-}" == "" ]]; then + log "ERROR: The '$env' environment variable is required, but is not set." + fail=1 + fi + done + + if [[ "$fail" == 1 ]]; then + log + error "One or more required environment variables are not set, check above log output for more details." + fi +} + # maybedryrun prints the given program and flags, and then, if the first # argument is 0, executes it. The reason the first argument should be 0 is that # it is expected that you have a dry_run variable in your script that is set to diff --git a/scripts/notarize_darwin.sh b/scripts/notarize_darwin.sh new file mode 100755 index 0000000000..fa02d8d614 --- /dev/null +++ b/scripts/notarize_darwin.sh @@ -0,0 +1,72 @@ +#!/usr/bin/env bash + +# This script notarizes the provided zip file using an Apple Developer account. +# +# Usage: ./notarize_darwin.sh path/to/zipfile.zip +# +# The provided zip file must contain a coder binary that has already been signed +# using ./sign_darwin.sh. +# +# On success, all of the contained binaries inside the input zip file will +# notarized. This does not make any changes to the zip or contained files +# itself, but GateKeeper checks will pass for the binaries inside the zip file +# as long as the device is connected to the internet to download the +# notarization ticket from Apple. +# +# You can check if a binary is notarized by running the following command on a +# Mac: +# spctl --assess -vvv -t install path/to/binary +# +# Depends on the rcodesign utility. Requires the following environment variables +# to be set: +# - $AC_APIKEY_ISSUER_ID: The issuer UUID of the Apple App Store Connect API +# key. +# - $AC_APIKEY_ID: The key ID of the Apple App Store Connect API key. +# - $AC_APIKEY_FILE: The path to the private key P8 file of the Apple App Store +# Connect API key. + +set -euo pipefail +# shellcheck source=scripts/lib.sh +source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" + +# Check dependencies +dependencies rcodesign +requiredenvs AC_APIKEY_ISSUER_ID AC_APIKEY_ID AC_APIKEY_FILE + +# Encode the notarization key components into a JSON file for easily calling +# `rcodesign notary-submit`. +key_file="$(mktemp)" +chmod 600 "$key_file" +trap 'rm -f "$key_file"' EXIT +rcodesign encode-app-store-connect-api-key \ + "$AC_APIKEY_ISSUER_ID" \ + "$AC_APIKEY_ID" \ + "$AC_APIKEY_FILE" \ + >"$key_file" + +# The notarization process is very fragile and heavily dependent on Apple's +# notarization server not returning server errors, so we retry this step twice +# with a delay of 30 seconds between attempts. +rc=0 +for i in $(seq 1 2); do + # -v is quite verbose, the default output is pretty good on it's own. Adding + # -v makes it dump the credentials used for uploading to Apple's S3 bucket. + rcodesign notary-submit \ + --api-key-path "$key_file" \ + --wait \ + "$@" \ + 1>&2 && rc=0 && break || rc=$? + + log "rcodesign exit code: $rc" + if [[ $i -lt 5 ]]; then + log + log "Retrying notarization in 30 seconds" + log + sleep 30 + else + log + log "Giving up :(" + fi +done + +exit $rc diff --git a/scripts/sign_darwin.sh b/scripts/sign_darwin.sh index 9511b41302..c168825215 100755 --- a/scripts/sign_darwin.sh +++ b/scripts/sign_darwin.sh @@ -1,62 +1,39 @@ #!/usr/bin/env bash -# This script notarizes the provided zip file. +# This script signs the provided darwin binary with an Apple Developer +# certificate. # -# Usage: ./publish_release.sh [--version 1.2.3] [--dry-run] path/to/asset1 path/to/asset2 ... +# Usage: ./sign_darwin.sh path/to/binary # -# The provided zip file must contain a coder binary that has already been signed -# using the codesign tool. +# On success, the input file will be signed using the Apple Developer +# certificate. # -# On success, the input file will be successfully signed and notarized. +# You can check if a binary is signed by running the following command on a Mac: +# codesign -dvv path/to/binary # -# Depends on codesign and gon utilities. Requires the $AC_APPLICATION_IDENTITY -# environment variable to be set. +# You can also run the following command to verify the signature on other +# systems, but it may be less accurate: +# rcodesign verify path/to/binary +# +# Depends on the rcodesign utility. Requires the following environment variables +# to be set: +# - $AC_CERTIFICATE_FILE: The path to the Apple Developer P12 certificate file. +# - $AC_CERTIFICATE_PASSWORD_FILE: The path to the file containing the password +# for the Apple Developer certificate. set -euo pipefail # shellcheck source=scripts/lib.sh source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" -if [[ "${AC_APPLICATION_IDENTITY:-}" == "" ]]; then - error "AC_APPLICATION_IDENTITY must be set for ./sign_darwin.sh" -fi - # Check dependencies -dependencies jq codesign gon +dependencies rcodesign +requiredenvs AC_CERTIFICATE_FILE AC_CERTIFICATE_PASSWORD_FILE -output_path="$1" - -# Create the gon config. -config="$(mktemp -d)/gon.json" -jq -r --null-input --arg path "$output_path" '{ - "notarize": [ - { - "path": $path, - "bundle_id": "com.coder.cli" - } - ] -}' >"$config" - -# Sign the zip file with our certificate. -codesign -s "$AC_APPLICATION_IDENTITY" -f -v --timestamp --options runtime "$output_path" - -# Notarize the signed zip file. -# -# The notarization process is very fragile and heavily dependent on Apple's -# notarization server not returning server errors, so we retry this step twice -# with a delay of a minute between attempts. -rc=0 -for i in $(seq 1 2); do - gon "$config" && rc=0 && break || rc=$? - log "gon exit code: $rc" - if [[ $i -lt 5 ]]; then - log - log "Retrying notarization in 60 seconds" - log - sleep 60 - else - log - log "Giving up :(" - fi -done - -exit $rc +# -v is quite verbose, the default output is pretty good on it's own. +rcodesign sign \ + --binary-identifier "com.coder.cli" \ + --p12-file "$AC_CERTIFICATE_FILE" \ + --p12-password-file "$AC_CERTIFICATE_PASSWORD_FILE" \ + --code-signature-flags runtime \ + "$@" \ + 1>&2 diff --git a/site/webpack.common.ts b/site/webpack.common.ts index ff82412c8f..6c6e662493 100644 --- a/site/webpack.common.ts +++ b/site/webpack.common.ts @@ -60,6 +60,9 @@ export const createCommonWebpackConfig = (options?: { skipTypecheck: boolean }): // REMARK: It's important to use [contenthash] here to invalidate caches. filename: "bundle.[contenthash].js", path: path.resolve(__dirname, "out"), + // Don't clean output directory on rebuilds to save time. This is the + // default behavior in webpack. We override this for production in + // webpack.prod.ts. clean: false, }, diff --git a/site/webpack.prod.ts b/site/webpack.prod.ts index 92ae6c6d47..b96af66534 100644 --- a/site/webpack.prod.ts +++ b/site/webpack.prod.ts @@ -48,7 +48,8 @@ export const config: Configuration = { ...commonWebpackConfig.output, // Regenerate the entire out/ directory (except GITKEEP and out/bin/) when - // producing production builds + // producing production builds. This is important to ensure that old files + // don't get left behind and embedded in the release binaries. clean: { keep: /(GITKEEP|bin\/)/, },