chore: parallelize image build (#55)
* Add flag for image build scripts to filter by distribution (currently: arch, centos, ubuntu) * Build images sequentially in separate jobs that run in parallel according to base distribution * Cancel in-flight pull request builds
This commit is contained in:
Jonathan Yu
GitHub
parent
a370353a28
commit
6fa1567c96
|
|
@ -0,0 +1,68 @@
|
|||
name: build
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
|
||||
pull_request:
|
||||
branches:
|
||||
- main
|
||||
|
||||
jobs:
|
||||
# Quick checks, running linters, checking formatting, etc
|
||||
quick:
|
||||
runs-on: ubuntu-20.04
|
||||
steps:
|
||||
- name: Cancel previous runs
|
||||
if: ${{ github.event_name == 'pull_request' }}
|
||||
uses: styfle/cancel-workflow-action@0.8.0
|
||||
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v2
|
||||
|
||||
- name: Install Node.js
|
||||
uses: actions/setup-node@v2
|
||||
with:
|
||||
node-version: 12.x
|
||||
|
||||
- name: Install Node.js modules
|
||||
run: yarn install
|
||||
|
||||
- name: Check formatting
|
||||
run: yarn format:check
|
||||
|
||||
images:
|
||||
runs-on: ubuntu-20.04
|
||||
strategy:
|
||||
matrix:
|
||||
job:
|
||||
- arch
|
||||
- centos
|
||||
- ubuntu
|
||||
name: images/${{ matrix.job}}
|
||||
steps:
|
||||
- name: Cancel previous runs
|
||||
if: ${{ github.event_name == 'pull_request' }}
|
||||
uses: styfle/cancel-workflow-action@0.8.0
|
||||
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v2
|
||||
|
||||
- name: Build ${{ matrix.job }} images
|
||||
run: |
|
||||
${{ github.workspace }}/scripts/build_images.sh \
|
||||
--tag=${{ matrix.job }}
|
||||
|
||||
- name: Authenticate to Docker Hub
|
||||
if: ${{ github.event_name != 'pull_request' }}
|
||||
uses: docker/login-action@v1
|
||||
with:
|
||||
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
|
||||
- name: Push images to Docker Hub
|
||||
if: ${{ github.event_name != 'pull_request' }}
|
||||
run: |
|
||||
${{ github.workspace}}/scripts/push_images.sh \
|
||||
--tag=${{ matrix.job}}
|
||||
|
|
@ -1,29 +0,0 @@
|
|||
# File generated by bin/generate-actions-yaml.sh
|
||||
name: ci
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- master
|
||||
pull_request:
|
||||
branches:
|
||||
- master
|
||||
|
||||
jobs:
|
||||
build-images:
|
||||
runs-on: ubuntu-20.04
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v2
|
||||
- name: Check Format
|
||||
run: yarn && yarn fmt:check
|
||||
- name: Login to DockerHub
|
||||
uses: docker/login-action@v1
|
||||
with:
|
||||
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
- name: Build images
|
||||
run: ./bin/build-images.sh
|
||||
- name: Push images
|
||||
if: ${{ github.event_name != 'pull_request' }}
|
||||
run: ./bin/push-images.sh
|
||||
|
|
@ -1,38 +0,0 @@
|
|||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
# Some images need to build before others, everything else can go in any order
|
||||
ORDERED_IMAGES=("base" "multieditor")
|
||||
|
||||
function main() {
|
||||
# Move to top level
|
||||
pushd $(git rev-parse --show-toplevel)
|
||||
|
||||
# Run on each image, but use an order that has ORDERED_IMAGES run first
|
||||
image_dirs=($(ls images))
|
||||
all_images=("${ORDERED_IMAGES[@]}" "${image_dirs[@]}")
|
||||
seen_images=()
|
||||
|
||||
for image in ${all_images[@]}; do
|
||||
# Skip image if we've already seen it, duplicates come from ORDERED_IMAGES
|
||||
if [[ "${seen_images[@]}" =~ "${image}" ]]; then
|
||||
continue
|
||||
fi
|
||||
seen_images+=("$image")
|
||||
|
||||
# Build each Dockerfile image
|
||||
dockerfiles=$(ls images/$image/Dockerfile*)
|
||||
for dockerfile in ${dockerfiles}; do
|
||||
tag=${dockerfile##*.}
|
||||
echo "Building codercom/enterprise-$image:$tag..."
|
||||
docker build ./images/$image --file "./images/$image/Dockerfile.$tag" --tag "codercom/enterprise-$image:$tag" --quiet
|
||||
done
|
||||
done
|
||||
|
||||
# Pop back to original dir
|
||||
popd
|
||||
|
||||
echo "Successfully built all images"
|
||||
}
|
||||
|
||||
main
|
||||
|
|
@ -1,26 +0,0 @@
|
|||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
function main() {
|
||||
# Move to top level
|
||||
pushd $(git rev-parse --show-toplevel)
|
||||
|
||||
# Add actions for building each image, maintaining desired image order
|
||||
image_dirs=($(ls images))
|
||||
|
||||
for image in ${image_dirs[@]}; do
|
||||
# Push each Dockerfile images
|
||||
dockerfiles=$(ls images/$image/Dockerfile*)
|
||||
for dockerfile in ${dockerfiles}; do
|
||||
tag=${dockerfile##*.}
|
||||
docker push "codercom/enterprise-$image:$tag"
|
||||
done
|
||||
done
|
||||
|
||||
# Pop back to original dir
|
||||
popd
|
||||
|
||||
echo "Successfully pushed all images"
|
||||
}
|
||||
|
||||
main
|
||||
|
|
@ -1,8 +1,8 @@
|
|||
{
|
||||
"private": true,
|
||||
"scripts": {
|
||||
"fmt:check": "prettier --check '**/*.{md,yaml}'",
|
||||
"fmt": "prettier --write '**/*.{md,yaml}'"
|
||||
"format:check": "prettier --check '**/*.{css,html,js,json,jsx,ts,tsx,yaml,yml}'",
|
||||
"format:write": "prettier --write '**/*.{css,html,js,json,jsx,ts,tsx,yaml,yml}'"
|
||||
},
|
||||
"devDependencies": {
|
||||
"husky": "^4.3.8",
|
||||
|
|
@ -15,6 +15,8 @@
|
|||
}
|
||||
},
|
||||
"lint-staged": {
|
||||
"*.{md,yaml}": "prettier --write"
|
||||
"*.{css,html,js,json,jsx,ts,tsx,yaml,yml}": [
|
||||
"prettier --write"
|
||||
]
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,108 @@
|
|||
#!/usr/bin/env bash
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
cd "$(dirname "$0")"
|
||||
source "./lib.sh"
|
||||
|
||||
check_dependencies \
|
||||
docker
|
||||
|
||||
source "./images.sh"
|
||||
|
||||
PROJECT_ROOT="$(git rev-parse --show-toplevel)"
|
||||
TAG="ubuntu"
|
||||
DRY_RUN=false
|
||||
QUIET=false
|
||||
|
||||
function usage() {
|
||||
echo "Usage: $(basename "$0") [options]"
|
||||
echo
|
||||
echo "This script builds Coder's container images."
|
||||
echo
|
||||
echo "Options:"
|
||||
echo " -h, --help Show this help text and exit"
|
||||
echo " --dry-run Show commands that would run, but"
|
||||
echo " do not run them"
|
||||
echo " --tag=<tag> Select an image tag group to build,"
|
||||
echo " one of: arch, centos, ubuntu)"
|
||||
echo " --quiet Suppress container build output"
|
||||
exit 1
|
||||
}
|
||||
|
||||
# Allow a failing exit status, as user input can cause this
|
||||
set +o errexit
|
||||
options=$(getopt \
|
||||
--name="$(basename "$0")" \
|
||||
--longoptions=" \
|
||||
help, \
|
||||
dry-run, \
|
||||
tag:, \
|
||||
quiet" \
|
||||
--options="h" \
|
||||
-- "$@")
|
||||
# allow checking the exit code separately here, because we need both
|
||||
# the response data and the exit code
|
||||
# shellcheck disable=SC2181
|
||||
if [ $? -ne 0 ]; then
|
||||
usage
|
||||
fi
|
||||
set -o errexit
|
||||
|
||||
eval set -- "$options"
|
||||
while true; do
|
||||
case "${1:-}" in
|
||||
--dry-run)
|
||||
DRY_RUN=true
|
||||
;;
|
||||
--tag)
|
||||
shift
|
||||
TAG="$1"
|
||||
;;
|
||||
--quiet)
|
||||
QUIET=true
|
||||
;;
|
||||
-h|--help)
|
||||
usage
|
||||
;;
|
||||
--)
|
||||
shift
|
||||
break
|
||||
;;
|
||||
*)
|
||||
# Default case, print an error and quit. This code shouldn't be
|
||||
# reachable, because getopt should return an error exit code.
|
||||
echo "Unknown option: $1"
|
||||
usage
|
||||
;;
|
||||
esac
|
||||
shift
|
||||
done
|
||||
|
||||
docker_flags=()
|
||||
|
||||
if [ $QUIET = true ]; then
|
||||
docker_flags+=(
|
||||
--quiet
|
||||
)
|
||||
fi
|
||||
|
||||
for image in "${IMAGES[@]}"; do
|
||||
image_dir="$PROJECT_ROOT/images/$image"
|
||||
image_file="Dockerfile.$TAG"
|
||||
image_ref="codercom/enterprise-$image:$TAG"
|
||||
image_path="$image_dir/$image_file"
|
||||
|
||||
if [ ! -f "$image_path" ]; then
|
||||
if [ $QUIET = false ]; then
|
||||
echo "Path '$image_path' does not exist; skipping" >&2
|
||||
fi
|
||||
continue
|
||||
fi
|
||||
|
||||
run_trace $DRY_RUN docker build \
|
||||
"${docker_flags[@]}" \
|
||||
"$image_dir" \
|
||||
--file="$image_path" \
|
||||
--tag="$image_ref" \| indent
|
||||
done
|
||||
|
|
@ -0,0 +1,22 @@
|
|||
#!/usr/bin/env bash
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
IMAGES=(
|
||||
# These must be built first because others build FROM these
|
||||
"base"
|
||||
"multieditor"
|
||||
|
||||
# We can build these images in any order
|
||||
"android"
|
||||
"configure"
|
||||
"goland"
|
||||
"golang"
|
||||
"intellij"
|
||||
"java"
|
||||
"jupyter"
|
||||
"node"
|
||||
"pycharm"
|
||||
"ruby"
|
||||
"webstorm"
|
||||
)
|
||||
|
|
@ -0,0 +1,94 @@
|
|||
#!/usr/bin/env bash
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# Emit a message to stderr and exit.
|
||||
#
|
||||
# This prints the arguments to stderr before exiting.
|
||||
#
|
||||
# Example:
|
||||
# error "Missing flag abc"
|
||||
# program-failure-info | error
|
||||
function error() {
|
||||
set +x
|
||||
echo
|
||||
echo "$@" "$(cat)" >&2
|
||||
echo
|
||||
exit 1
|
||||
}
|
||||
|
||||
# Check if dependencies are available.
|
||||
#
|
||||
# If any dependencies are missing, an error message will be printed to
|
||||
# stderr and the program will exit, running traps on EXIT beforehand.
|
||||
#
|
||||
# Example:
|
||||
# check_dependencies git bash node
|
||||
function check_dependencies() {
|
||||
local missing=false
|
||||
for command in "$@"; do
|
||||
if ! command -v "$command" &> /dev/null; then
|
||||
echo "$0: script requires '$command', but it is not in your PATH" >&2
|
||||
missing=true
|
||||
fi
|
||||
done
|
||||
|
||||
if [ "$missing" = true ]; then
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
# Indent output by (indent) levels
|
||||
#
|
||||
# Example:
|
||||
# echo "example" | indent 2
|
||||
# cat file.txt | indent
|
||||
function indent() {
|
||||
local indentSize=2
|
||||
local indent=1
|
||||
if [ -n "${1:-}" ]; then
|
||||
indent="$1"
|
||||
fi
|
||||
pr --omit-header --indent=$((indent * indentSize))
|
||||
}
|
||||
|
||||
# Run a command, with tracing.
|
||||
#
|
||||
# This prints a command to stderr for tracing, in a format similar to
|
||||
# the bash xtrace option (i.e. set -x, set -o xtrace), then runs it using
|
||||
# eval if the first argument (dry run) is false.
|
||||
#
|
||||
# If dry run is true, the command and arguments will be printed, but
|
||||
# not executed.
|
||||
#
|
||||
# Example:
|
||||
# run_trace $DRY_RUN rm -rf /
|
||||
# run_trace false echo "abc" \| indent
|
||||
function run_trace() {
|
||||
local args=("$@")
|
||||
local dry_run="${args[0]}"
|
||||
args=("${args[@]:1}")
|
||||
|
||||
# If we're running in GitHub Actions, use the special syntax
|
||||
# to start/end a group for each traced command
|
||||
if [[ -n "${GITHUB_ACTIONS:-}" && "$dry_run" = false ]]; then
|
||||
echo "::group::Run ${args[*]}" >&2
|
||||
else
|
||||
echo "+ ${args[*]}" >&2
|
||||
fi
|
||||
|
||||
local exit_status=0
|
||||
if [ "$dry_run" = false ]; then
|
||||
eval "${args[@]}"
|
||||
|
||||
# Save the exit status code from eval, otherwise it will be
|
||||
# obscured by future commands.
|
||||
exit_status="$?"
|
||||
fi
|
||||
|
||||
if [[ -n "${GITHUB_ACTIONS:-}" && "$dry_run" = false ]]; then
|
||||
echo "::endgroup::" >&2
|
||||
fi
|
||||
|
||||
return "$exit_status"
|
||||
}
|
||||
|
|
@ -0,0 +1,106 @@
|
|||
#!/usr/bin/env bash
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
cd "$(dirname "$0")"
|
||||
source "./lib.sh"
|
||||
|
||||
check_dependencies \
|
||||
docker
|
||||
|
||||
source "./images.sh"
|
||||
|
||||
PROJECT_ROOT="$(git rev-parse --show-toplevel)"
|
||||
TAG="ubuntu"
|
||||
DRY_RUN=false
|
||||
QUIET=false
|
||||
|
||||
function usage() {
|
||||
echo "Usage: $(basename "$0") [options]"
|
||||
echo
|
||||
echo "This script pushes Coder's container images to Docker Hub."
|
||||
echo
|
||||
echo "Options:"
|
||||
echo " -h, --help Show this help text and exit"
|
||||
echo " --dry-run Show commands that would run, but"
|
||||
echo " do not run them"
|
||||
echo " --tag=<tag> Select an image tag group to build,"
|
||||
echo " one of: arch, centos, ubuntu)"
|
||||
echo " --quiet Suppress container build output"
|
||||
exit 1
|
||||
}
|
||||
|
||||
# Allow a failing exit status, as user input can cause this
|
||||
set +o errexit
|
||||
options=$(getopt \
|
||||
--name="$(basename "$0")" \
|
||||
--longoptions=" \
|
||||
help, \
|
||||
dry-run, \
|
||||
tag:, \
|
||||
quiet" \
|
||||
--options="h" \
|
||||
-- "$@")
|
||||
# allow checking the exit code separately here, because we need both
|
||||
# the response data and the exit code
|
||||
# shellcheck disable=SC2181
|
||||
if [ $? -ne 0 ]; then
|
||||
usage
|
||||
fi
|
||||
set -o errexit
|
||||
|
||||
eval set -- "$options"
|
||||
while true; do
|
||||
case "${1:-}" in
|
||||
--dry-run)
|
||||
DRY_RUN=true
|
||||
;;
|
||||
--tag)
|
||||
shift
|
||||
TAG="$1"
|
||||
;;
|
||||
--quiet)
|
||||
QUIET=true
|
||||
;;
|
||||
-h|--help)
|
||||
usage
|
||||
;;
|
||||
--)
|
||||
shift
|
||||
break
|
||||
;;
|
||||
*)
|
||||
# Default case, print an error and quit. This code shouldn't be
|
||||
# reachable, because getopt should return an error exit code.
|
||||
echo "Unknown option: $1"
|
||||
usage
|
||||
;;
|
||||
esac
|
||||
shift
|
||||
done
|
||||
|
||||
docker_flags=()
|
||||
|
||||
if [ $QUIET = true ]; then
|
||||
docker_flags+=(
|
||||
--quiet
|
||||
)
|
||||
fi
|
||||
|
||||
for image in "${IMAGES[@]}"; do
|
||||
image_dir="$PROJECT_ROOT/images/$image"
|
||||
image_file="Dockerfile.$TAG"
|
||||
image_ref="codercom/enterprise-$image:$TAG"
|
||||
image_path="$image_dir/$image_file"
|
||||
|
||||
if [ ! -f "$image_path" ]; then
|
||||
if [ $QUIET = false ]; then
|
||||
echo "Path '$image_path' does not exist; skipping" >&2
|
||||
fi
|
||||
continue
|
||||
fi
|
||||
|
||||
run_trace $DRY_RUN docker push \
|
||||
"${docker_flags[@]}" \
|
||||
"$image_ref" \| indent
|
||||
done
|
||||
Loading…
Reference in New Issue