selfhosted
/
coder
mirror of https://github.com/coder/coder.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
coder/.github/workflows/ci.yaml

739 lines
24 KiB
YAML
Raw Blame History

name: ci
on:
push:
branches:
- main
pull_request:
workflow_dispatch:
permissions:
actions: none
checks: none
contents: read
deployments: none
issues: none
packages: write
pull-requests: none
repository-projects: none
security-events: none
statuses: none
# Cancel in-progress runs for pull requests when developers push
# additional changes
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}
jobs:
changes:
runs-on: ubuntu-latest
outputs:
docs-only: ${{ steps.filter.outputs.docs_count == steps.filter.outputs.all_count }}
go: ${{ steps.filter.outputs.go }}
ts: ${{ steps.filter.outputs.ts }}
k8s: ${{ steps.filter.outputs.k8s }}
ci: ${{ steps.filter.outputs.ci }}
offlinedocs-only: ${{ steps.filter.outputs.offlinedocs_count == steps.filter.outputs.all_count }}
offlinedocs: ${{ steps.filter.outputs.offlinedocs }}
steps:
- name: Checkout
uses: actions/checkout@v3
with:
fetch-depth: 1
# For pull requests it's not necessary to checkout the code
- name: check changed files
uses: dorny/paths-filter@v2
id: filter
with:
filters: |
all:
- "**"
docs:
- "docs/**"
- "README.md"
- "examples/templates/**"
- "examples/web-server/**"
- "examples/monitoring/**"
- "examples/lima/**"
go:
- "**.sql"
- "**.go"
- "**.golden"
- "go.mod"
- "go.sum"
# Other non-Go files that may affect Go code:
- "**.rego"
- "**.sh"
- "**.tpl"
- "**.gotmpl"
- "**.gotpl"
- "Makefile"
- "site/static/error.html"
# Main repo directories for completeness in case other files are
# touched:
- "agent/**"
- "cli/**"
- "cmd/**"
- "coderd/**"
- "enterprise/**"
- "examples/*"
- "provisioner/**"
- "provisionerd/**"
- "provisionersdk/**"
- "pty/**"
- "scaletest/**"
- "tailnet/**"
- "testutil/**"
ts:
- "site/**"
- "Makefile"
k8s:
- "helm/**"
- "scripts/Dockerfile"
- "scripts/Dockerfile.base"
- "scripts/helm.sh"
ci:
- ".github/actions/**"
- ".github/workflows/ci.yaml"
offlinedocs:
- "offlinedocs/**"
- id: debug
run: |
echo "${{ toJSON(steps.filter )}}"
lint:
needs: changes
if: needs.changes.outputs.offlinedocs-only == 'false' || needs.changes.outputs.ci == 'true' || github.ref == 'refs/heads/main'
runs-on: ${{ github.repository_owner == 'coder' && 'buildjet-8vcpu-ubuntu-2204' || 'ubuntu-latest' }}
steps:
- name: Checkout
uses: actions/checkout@v3
with:
fetch-depth: 1
- name: Setup Node
uses: ./.github/actions/setup-node
- name: Setup Go
uses: ./.github/actions/setup-go
- name: Get golangci-lint cache dir
run: |
go install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.53.2
dir=$(golangci-lint cache status | awk '/Dir/ { print $2 }')
echo "LINT_CACHE_DIR=$dir" >> $GITHUB_ENV
- name: golangci-lint cache
uses: buildjet/cache@v3
with:
path: |
${{ env.LINT_CACHE_DIR }}
key: golangci-lint-${{ runner.os }}-${{ hashFiles('**/*.go') }}
restore-keys: |
golangci-lint-${{ runner.os }}-
# Check for any typos
- name: Check for typos
uses: crate-ci/typos@v1.16.4
with:
config: .github/workflows/typos.toml
- name: Fix the typos
if: ${{ failure() }}
run: |
echo "::notice:: you can automatically fix typos from your CLI:
cargo install typos-cli
typos -c .github/workflows/typos.toml -w"
# Needed for helm chart linting
- name: Install helm
uses: azure/setup-helm@v3
with:
version: v3.9.2
- name: make lint
run: |
make --output-sync=line -j lint
gen:
timeout-minutes: 8
runs-on: ${{ github.repository_owner == 'coder' && 'buildjet-8vcpu-ubuntu-2204' || 'ubuntu-latest' }}
needs: changes
if: needs.changes.outputs.docs-only == 'false' || needs.changes.outputs.ci == 'true' || github.ref == 'refs/heads/main'
steps:
- name: Checkout
uses: actions/checkout@v3
with:
fetch-depth: 1
- name: Install Nix
uses: DeterminateSystems/nix-installer-action@v4
- name: Run the Magic Nix Cache
uses: DeterminateSystems/magic-nix-cache-action@v2
- name: make gen
run: "nix-shell --command 'make --output-sync -j -B gen'"
- name: Check for unstaged files
run: ./scripts/check_unstaged.sh
fmt:
needs: changes
if: needs.changes.outputs.offlinedocs-only == 'false' || needs.changes.outputs.ci == 'true' || github.ref == 'refs/heads/main'
runs-on: ${{ github.repository_owner == 'coder' && 'buildjet-8vcpu-ubuntu-2204' || 'ubuntu-latest' }}
timeout-minutes: 7
steps:
- name: Checkout
uses: actions/checkout@v3
with:
fetch-depth: 1
- name: Setup Node
uses: ./.github/actions/setup-node
- name: Setup Go
uses: buildjet/setup-go@v4
with:
# This doesn't need caching. It's super fast anyways!
cache: false
go-version: 1.20.7
- name: Install shfmt
run: go install mvdan.cc/sh/v3/cmd/shfmt@v3.5.0
- name: make fmt
run: |
export PATH=${PATH}:$(go env GOPATH)/bin
make --output-sync -j -B fmt
- name: Check for unstaged files
run: ./scripts/check_unstaged.sh
test-go:
runs-on: ${{ matrix.os == 'ubuntu-latest' && github.repository_owner == 'coder' && 'buildjet-4vcpu-ubuntu-2204' || matrix.os == 'macos-latest' && github.repository_owner == 'coder' && 'macos-latest-xl' || matrix.os == 'windows-2019' && github.repository_owner == 'coder' && 'windows-latest-8-cores' || matrix.os }}
needs: changes
if: needs.changes.outputs.go == 'true' || needs.changes.outputs.ci == 'true' || github.ref == 'refs/heads/main'
timeout-minutes: 20
strategy:
fail-fast: false
matrix:
os:
- ubuntu-latest
- macos-latest
- windows-2019
steps:
- name: Checkout
uses: actions/checkout@v3
with:
fetch-depth: 1
- name: Setup Go
uses: ./.github/actions/setup-go
- name: Setup Terraform
uses: ./.github/actions/setup-tf
- name: Test with Mock Database
id: test
shell: bash
run: |
# Code coverage is more computationally expensive and also
# prevents test caching, so we disable it on alternate operating
# systems.
if [ "${{ matrix.os }}" == "ubuntu-latest" ]; then
echo "cover=true" >> $GITHUB_OUTPUT
export COVERAGE_FLAGS='-covermode=atomic -coverprofile="gotests.coverage" -coverpkg=./...'
else
echo "cover=false" >> $GITHUB_OUTPUT
fi
# By default Go will use the number of logical CPUs, which
# is a fine default.
PARALLEL_FLAG=""
export TS_DEBUG_DISCO=true
gotestsum --junitfile="gotests.xml" --jsonfile="gotests.json" \
--packages="./..." -- $PARALLEL_FLAG -short -failfast $COVERAGE_FLAGS
- name: Print test stats
if: success() || failure()
run: |
# Artifacts are not available after rerunning a job,
# so we need to print the test stats to the log.
go run ./scripts/ci-report/main.go gotests.json | tee gotests_stats.json
- name: Upload test stats to Datadog
uses: ./.github/actions/upload-datadog
if: success() || failure()
with:
api-key: ${{ secrets.DATADOG_API_KEY }}
- name: Check code coverage
uses: codecov/codecov-action@v3
# This action has a tendency to error out unexpectedly, it has
# the `fail_ci_if_error` option that defaults to `false`, but
# that is no guarantee, see:
# https://github.com/codecov/codecov-action/issues/788
continue-on-error: true
if: steps.test.outputs.cover && github.actor != 'dependabot[bot]' && !github.event.pull_request.head.repo.fork
with:
token: ${{ secrets.CODECOV_TOKEN }}
files: ./gotests.coverage
flags: unittest-go-${{ matrix.os }}
test-go-pg:
runs-on: ${{ github.repository_owner == 'coder' && 'buildjet-8vcpu-ubuntu-2204' || 'ubuntu-latest' }}
needs: changes
if: needs.changes.outputs.go == 'true' || needs.changes.outputs.ci == 'true' || github.ref == 'refs/heads/main'
# This timeout must be greater than the timeout set by `go test` in
# `make test-postgres` to ensure we receive a trace of running
# goroutines. Setting this to the timeout +5m should work quite well
# even if some of the preceding steps are slow.
timeout-minutes: 25
steps:
- name: Checkout
uses: actions/checkout@v3
with:
fetch-depth: 1
- name: Setup Go
uses: ./.github/actions/setup-go
- name: Setup Terraform
uses: ./.github/actions/setup-tf
- name: Test with PostgreSQL Database
run: |
export TS_DEBUG_DISCO=true
make test-postgres
- name: Print test stats
if: success() || failure()
run: |
# Artifacts are not available after rerunning a job,
# so we need to print the test stats to the log.
go run ./scripts/ci-report/main.go gotests.json | tee gotests_stats.json
- name: Upload test stats to Datadog
uses: ./.github/actions/upload-datadog
if: success() || failure()
with:
api-key: ${{ secrets.DATADOG_API_KEY }}
- name: Check code coverage
uses: codecov/codecov-action@v3
# This action has a tendency to error out unexpectedly, it has
# the `fail_ci_if_error` option that defaults to `false`, but
# that is no guarantee, see:
# https://github.com/codecov/codecov-action/issues/788
continue-on-error: true
if: github.actor != 'dependabot[bot]' && !github.event.pull_request.head.repo.fork
with:
token: ${{ secrets.CODECOV_TOKEN }}
files: ./gotests.coverage
flags: unittest-go-postgres-linux
test-go-race:
runs-on: ${{ github.repository_owner == 'coder' && 'buildjet-8vcpu-ubuntu-2204' || 'ubuntu-latest' }}
needs: changes
if: needs.changes.outputs.go == 'true' || needs.changes.outputs.ci == 'true' || github.ref == 'refs/heads/main'
timeout-minutes: 25
steps:
- name: Checkout
uses: actions/checkout@v3
with:
fetch-depth: 1
- name: Setup Go
uses: ./.github/actions/setup-go
- name: Setup Terraform
uses: ./.github/actions/setup-tf
- name: Run Tests
run: |
gotestsum --junitfile="gotests.xml" -- -race ./...
- name: Upload test stats to Datadog
uses: ./.github/actions/upload-datadog
if: always()
with:
api-key: ${{ secrets.DATADOG_API_KEY }}
deploy:
name: "deploy"
runs-on: ${{ github.repository_owner == 'coder' && 'buildjet-8vcpu-ubuntu-2204' || 'ubuntu-latest' }}
timeout-minutes: 30
needs: changes
if: |
github.ref == 'refs/heads/main' && !github.event.pull_request.head.repo.fork
&& needs.changes.outputs.docs-only == 'false'
permissions:
contents: read
id-token: write
steps:
- name: Checkout
uses: actions/checkout@v3
with:
fetch-depth: 0
- name: Authenticate to Google Cloud
uses: google-github-actions/auth@v1
with:
workload_identity_provider: projects/573722524737/locations/global/workloadIdentityPools/github/providers/github
service_account: coder-ci@coder-dogfood.iam.gserviceaccount.com
- name: Set up Google Cloud SDK
uses: google-github-actions/setup-gcloud@v1
- name: Setup Node
uses: ./.github/actions/setup-node
- name: Setup Go
uses: ./.github/actions/setup-go
- name: Install goimports
run: go install golang.org/x/tools/cmd/goimports@latest
- name: Install nfpm
run: go install github.com/goreleaser/nfpm/v2/cmd/nfpm@v2.16.0
- name: Install zstd
run: sudo apt-get install -y zstd
- name: Build Release
run: |
set -euo pipefail
go mod download
version="$(./scripts/version.sh)"
make gen/mark-fresh
make -j \
build/coder_"$version"_windows_amd64.zip \
build/coder_"$version"_linux_amd64.{tar.gz,deb}
- name: Install Release
run: |
set -euo pipefail
regions=(
# gcp-region-id instance-name systemd-service-name
"us-central1-a coder coder"
"australia-southeast1-b coder-sydney coder-workspace-proxy"
"europe-west3-c coder-europe coder-workspace-proxy"
"southamerica-east1-b coder-brazil coder-workspace-proxy"
)
deb_pkg="./build/coder_$(./scripts/version.sh)_linux_amd64.deb"
if [ ! -f "$deb_pkg" ]; then
echo "deb package not found: $deb_pkg"
ls -l ./build
exit 1
fi
gcloud config set project coder-dogfood
for region in "${regions[@]}"; do
echo "::group::$region"
set -- $region
set -x
gcloud config set compute/zone "$1"
gcloud compute scp "$deb_pkg" "${2}:/tmp/coder.deb"
gcloud compute ssh "$2" -- /bin/sh -c "set -eux; sudo dpkg -i --force-confdef /tmp/coder.deb; sudo systemctl daemon-reload; sudo service '$3' restart"
set +x
echo "::endgroup::"
done
- name: Upload build artifacts
uses: actions/upload-artifact@v3
with:
name: coder
path: |
./build/*.zip
./build/*.tar.gz
./build/*.deb
retention-days: 7
test-js:
runs-on: ${{ github.repository_owner == 'coder' && 'buildjet-8vcpu-ubuntu-2204' || 'ubuntu-latest' }}
needs: changes
if: needs.changes.outputs.ts == 'true' || needs.changes.outputs.ci == 'true' || github.ref == 'refs/heads/main'
timeout-minutes: 20
steps:
- name: Checkout
uses: actions/checkout@v3
with:
fetch-depth: 1
- name: Setup Node
uses: ./.github/actions/setup-node
- run: pnpm test:ci --max-workers $(nproc)
working-directory: site
- name: Check code coverage
uses: codecov/codecov-action@v3
# This action has a tendency to error out unexpectedly, it has
# the `fail_ci_if_error` option that defaults to `false`, but
# that is no guarantee, see:
# https://github.com/codecov/codecov-action/issues/788
continue-on-error: true
if: github.actor != 'dependabot[bot]' && !github.event.pull_request.head.repo.fork
with:
token: ${{ secrets.CODECOV_TOKEN }}
files: ./site/coverage/lcov.info
flags: unittest-js
test-e2e:
runs-on: ${{ github.repository_owner == 'coder' && 'buildjet-8vcpu-ubuntu-2204' || 'ubuntu-latest' }}
needs: changes
if: needs.changes.outputs.go == 'true' || needs.changes.outputs.ts == 'true' || needs.changes.outputs.ci == 'true' || github.ref == 'refs/heads/main'
timeout-minutes: 20
steps:
- name: Checkout
uses: actions/checkout@v3
with:
fetch-depth: 1
- name: Setup Node
uses: ./.github/actions/setup-node
- name: Setup Go
uses: ./.github/actions/setup-go
- name: Setup Terraform
uses: ./.github/actions/setup-tf
- name: Install Nix
uses: DeterminateSystems/nix-installer-action@v4
- name: Run the Magic Nix Cache
uses: DeterminateSystems/magic-nix-cache-action@v2
- name: Build
run: |
nix-shell --command 'make -B site/out/index.html'
- run: pnpm playwright:install
working-directory: site
- run: pnpm playwright:test
env:
DEBUG: pw:api
working-directory: site
- name: Upload Playwright Failed Tests
if: always() && github.actor != 'dependabot[bot]' && runner.os == 'Linux' && !github.event.pull_request.head.repo.fork
uses: actions/upload-artifact@v3
with:
name: failed-test-videos
path: ./site/test-results/**/*.webm
retention-days: 7
chromatic:
# REMARK: this is only used to build storybook and deploy it to Chromatic.
runs-on: ubuntu-latest
needs: changes
if: needs.changes.outputs.ts == 'true' || needs.changes.outputs.ci == 'true'
steps:
- name: Checkout
uses: actions/checkout@v3
with:
# Required by Chromatic for build-over-build history, otherwise we
# only get 1 commit on shallow checkout.
fetch-depth: 0
- name: Setup Node
uses: ./.github/actions/setup-node
# This step is not meant for mainline because any detected changes to
# storybook snapshots will require manual approval/review in order for
# the check to pass. This is desired in PRs, but not in mainline.
- name: Publish to Chromatic (non-mainline)
if: github.ref != 'refs/heads/main' && github.repository_owner == 'coder'
uses: chromaui/action@v1
env:
NODE_OPTIONS: "--max_old_space_size=4096"
STORYBOOK: true
with:
buildScriptName: "storybook:build"
exitOnceUploaded: true
# This will prevent CI from failing when Chromatic detects visual changes
exitZeroOnChanges: true
# Chromatic states its fine to make this token public. See:
# https://www.chromatic.com/docs/github-actions#forked-repositories
projectToken: 695c25b6cb65
workingDir: "./site"
# Prevent excessive build runs on minor version changes
skip: "@(renovate/**|dependabot/**)"
# Run TurboSnap to trace file dependencies to related stories
# and tell chromatic to only take snapshots of relevent stories
onlyChanged: true
# This is a separate step for mainline only that auto accepts and changes
# instead of holding CI up. Since we squash/merge, this is defensive to
# avoid the same changeset from requiring review once squashed into
# main. Chromatic is supposed to be able to detect that we use squash
# commits, but it's good to be defensive in case, otherwise CI remains
# infinitely "in progress" in mainline unless we re-review each build.
- name: Publish to Chromatic (mainline)
if: github.ref == 'refs/heads/main' && github.repository_owner == 'coder'
uses: chromaui/action@v1
env:
NODE_OPTIONS: "--max_old_space_size=4096"
STORYBOOK: true
with:
autoAcceptChanges: true
# This will prevent CI from failing when Chromatic detects visual changes
exitZeroOnChanges: true
buildScriptName: "storybook:build"
projectToken: 695c25b6cb65
workingDir: "./site"
# Run TurboSnap to trace file dependencies to related stories
# and tell chromatic to only take snapshots of relevent stories
onlyChanged: true
offlinedocs:
name: offlinedocs
needs: changes
runs-on: ${{ github.repository_owner == 'coder' && 'buildjet-8vcpu-ubuntu-2204' || 'ubuntu-latest' }}
if: needs.changes.outputs.offlinedocs == 'true' || needs.changes.outputs.ci == 'true'
steps:
- name: Checkout
uses: actions/checkout@v3
with:
# 0 is required here for version.sh to work.
fetch-depth: 0
- name: Setup Node
uses: ./.github/actions/setup-node
with:
directory: offlinedocs
- name: Setup Go
uses: ./.github/actions/setup-go
- name: Install go tools
run: |
go install github.com/golang/mock/mockgen@v1.6.0
- name: Setup sqlc
uses: ./.github/actions/setup-sqlc
- name: Format
run: |
cd offlinedocs
pnpm format:check
- name: Lint
run: |
cd offlinedocs
pnpm lint
- name: Build
run: |
make -j build/coder_docs_"$(./scripts/version.sh)".tgz
required:
runs-on: ubuntu-latest
needs:
- fmt
- lint
- gen
- test-go
- test-go-pg
- test-go-race
- test-js
- test-e2e
- offlinedocs
# Allow this job to run even if the needed jobs fail, are skipped or
# cancelled.
if: always()
steps:
- name: Ensure required checks
run: |
echo "Checking required checks"
echo "- fmt: ${{ needs.fmt.result }}"
echo "- lint: ${{ needs.lint.result }}"
echo "- gen: ${{ needs.gen.result }}"
echo "- test-go: ${{ needs.test-go.result }}"
echo "- test-go-pg: ${{ needs.test-go-pg.result }}"
echo "- test-go-race: ${{ needs.test-go-race.result }}"
echo "- test-js: ${{ needs.test-js.result }}"
echo
# We allow skipped jobs to pass, but not failed or cancelled jobs.
if [[ "${{ contains(needs.*.result, 'failure') }}" == "true" || "${{ contains(needs.*.result, 'cancelled') }}" == "true" ]]; then
echo "One of the required checks has failed or has been cancelled"
exit 1
fi
echo "Required checks have passed"
build-main-image:
# This build and publihes ghcr.io/coder/coder-preview:main for each merge commit to main branch.
# We are only building this for amd64 plateform. (>95% pulls are for amd64)
needs: changes
if: github.ref == 'refs/heads/main' && needs.changes.outputs.docs-only == 'false'
runs-on: ${{ github.repository_owner == 'coder' && 'buildjet-8vcpu-ubuntu-2204' || 'ubuntu-latest' }}
env:
DOCKER_CLI_EXPERIMENTAL: "enabled"
steps:
- name: Checkout
uses: actions/checkout@v3
with:
fetch-depth: 0
- name: Setup Node
uses: ./.github/actions/setup-node
- name: Setup Go
uses: ./.github/actions/setup-go
- name: Setup sqlc
uses: ./.github/actions/setup-sqlc
- name: GHCR Login
uses: docker/login-action@v2
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Build and push Linux amd64 Docker image
id: build_and_push
run: |
set -euxo pipefail
go mod download
make gen/mark-fresh
export DOCKER_IMAGE_NO_PREREQUISITES=true
version="$(./scripts/version.sh)"
export CODER_IMAGE_BUILD_BASE_TAG="$(CODER_IMAGE_BASE=coder-base ./scripts/image_tag.sh --version "$version")"
make -j build/coder_linux_amd64
./scripts/build_docker.sh \
--arch amd64 \
--target ghcr.io/coder/coder-preview:main \
--version $version \
--push \
build/coder_linux_amd64
# Tag image with new package tag and push
tag=$(echo "$version" | sed 's/+/-/g')
docker tag ghcr.io/coder/coder-preview:main ghcr.io/coder/coder-preview:main-$tag
docker push ghcr.io/coder/coder-preview:main-$tag
- name: Prune old images
uses: vlaurin/action-ghcr-prune@v0.5.0
with:
token: ${{ secrets.GITHUB_TOKEN }}
organization: coder
container: coder-preview
keep-younger-than: 7 # days
keep-tags-regexes: ^pr
prune-tags-regexes: ^main-
prune-untagged: true
Reference in New Issue View Git Blame Copy Permalink