omnibus-gitlab/gitlab-ci-config/gitlab-com.yml

# This config lists the jobs that will be run on omnibus-gitlab project in
# gitlab.com.

#############
# Templates #
#############
.com-docs-except-triggers: &com-docs-except-triggers
  except:
    refs:
      - /./@gitlab/omnibus-gitlab
      - triggers
      - /^deps/
      - /./@gitlab-org/build/omnibus-gitlab-mirror
    variables:
      - $DEPS_PIPELINE || $CACHE_UPDATE || $DEPENDENCY_SCANNING
      - $CI_COMMIT_REF_NAME =~ /^[0-9]+-[0-9]+-auto-deploy-[0-9]+$/

.com-except-triggers: &com-except-triggers
  except:
    refs:
      - /./@gitlab/omnibus-gitlab
      - triggers
      - /^deps/
      - /(^docs[\/-].*|.*-docs$)/
      - /./@gitlab-org/build/omnibus-gitlab-mirror
    variables:
      - $DEPS_PIPELINE || $CACHE_UPDATE || $DEPENDENCY_SCANNING
      - $CI_COMMIT_REF_NAME =~ /^[0-9]+-[0-9]+-auto-deploy-[0-9]+$/

.com-trigger-only: &com-trigger-only
  only:
    - triggers@gitlab-org/build/omnibus-gitlab-mirror
    - pipelines@gitlab-org/build/omnibus-gitlab-mirror

.com-trigger-and-nightly-only: &com-trigger-and-nightly-only
  rules:
    - if: '$DEPS_PIPELINE || $DEPENDENCY_SCANNING'
      when: never
    - if: '$CI_PIPELINE_SOURCE == "trigger" && $CI_PROJECT_PATH == "gitlab-org/build/omnibus-gitlab-mirror"'
    - if: '$CI_PIPELINE_SOURCE == "pipeline" && $CI_PROJECT_PATH == "gitlab-org/build/omnibus-gitlab-mirror"'
    - if: '$CI_PIPELINE_SOURCE == "schedule" && $NIGHTLY == "true" && $CI_PROJECT_PATH == "gitlab-org/build/omnibus-gitlab-mirror"'
    - when: never

.depscan-rules: &depscan-rules
  only:
    refs:
      - schedules@gitlab-org/omnibus-gitlab
    variables:
      - $DEPENDENCY_SCANNING

.knapsack-artifacts: &knapsack-artifacts
  expire_in: 31d
  paths:
  - knapsack/

.knapsack-state:
  services: []
  cache:
    key: "knapsack${CACHE_KEY_SUFFIX}"
    paths:
      - knapsack/
  artifacts: *knapsack-artifacts

.knapsack: &prepare_knapsack
  extends: .knapsack-state
  stage: prepare
  before_script: []
  script:
    - JOB_NAME=( $CI_JOB_NAME )
    - export DISTRO_NAME=${JOB_NAME[0]}
    - export DISTRO_VERSION=${JOB_NAME[1]}
    - mkdir -p knapsack/
    - '[[ -f knapsack/${DISTRO_NAME}_${DISTRO_VERSION}_main_rspec_report.json ]] || echo "{}" > knapsack/${DISTRO_NAME}_${DISTRO_VERSION}_main_rspec_report.json'
  <<: *com-except-triggers
  retry: 1
  needs:
    - rubocop

.gems-cache:
  cache:
    key: "gems-cache-${BUILDER_IMAGE_REVISION}${CACHE_KEY_SUFFIX}"
    paths:
      - gems
    policy: pull

.trigger-package-cache:
  cache:
    key: "Ubuntu-20.04-branch-${BUILDER_IMAGE_REVISION}-${CACHE_EDITION}${CACHE_KEY_SUFFIX}"
    paths:
      - cache
      - gems
      - assets_cache
      - node_modules
    policy: pull

.install-gems: &install-gems
  - bundle config set --local path 'gems'
  - bundle install -j $(nproc)
  - bundle binstubs --all

.build-package: &build-package
  - bundle exec rake cache:populate
  - bundle exec rake cache:restore
  - bundle exec rake build:project
  - bundle exec rake cache:bundle
  - bundle exec rake build:component_shas

.spec_template:
  extends: .gems-cache
  stage: tests
  before_script:
    # These jobs will not be run on dev, so we set ALTERNATIVE_SOURCES to true
    # so tests run fine on forks
    - export ALTERNATIVE_SOURCES="true";
    - *install-gems
  script:
    - JOB_NAME=( $CI_JOB_NAME )
    - export DISTRO_NAME=${JOB_NAME[0]}
    - export DISTRO_VERSION=${JOB_NAME[1]}
    - export KNAPSACK_REPORT_PATH=knapsack/${DISTRO_NAME}_${DISTRO_VERSION}_rspec_node_${CI_NODE_INDEX}_${CI_NODE_TOTAL}_report.json
    - export KNAPSACK_GENERATE_REPORT=true
    - export USE_KNAPSACK=true
    # To prevent current OS providing empty/old reports of other OSs as an
    # artifact. If not, they may overwrite the valid/new reports from those
    # corresponding OSs. So, removing everything except current OS's report.
    - cp knapsack/${DISTRO_NAME}_${DISTRO_VERSION}_main_rspec_report.json ${KNAPSACK_REPORT_PATH}.bak
    - rm -f knapsack/*.json
    - mv ${KNAPSACK_REPORT_PATH}.bak ${KNAPSACK_REPORT_PATH}
    - bundle exec rake "knapsack:rspec[--color --format RspecJunitFormatter --out junit_rspec.xml --format documentation]"
  <<: *com-except-triggers
  artifacts:
    <<: *knapsack-artifacts
    reports:
      junit: junit_rspec.xml
  retry: 1

.trigger-job:
  extends: .gems-cache
  image: "${RUBY_IMAGE}"
  stage: trigger-package
  when: manual
  script:
    - support/wait_for_sha
    - bundle exec rake build:trigger
  only:
    - branches@gitlab-org/omnibus-gitlab
    - branches@gitlab-org/security/omnibus-gitlab
  except:
    refs:
      - triggers
      - /^deps/
      - /(^docs[\/-].*|.*-docs$)/
    variables:
      - $DEPS_PIPELINE || $CACHE_UPDATE || $DEPENDENCY_SCANNING
      - $CI_COMMIT_REF_NAME =~ /^[0-9]+-[0-9]+-auto-deploy-[0-9]+$/
  needs: []
  timeout: 3h

.review-docs:
  image: "${RUBY_IMAGE}-alpine"
  stage: post-test
  cache: {}
  needs: []
  before_script:
    - gem install gitlab --no-doc
    # We need to download the script rather than clone the repo since the
    # review-docs-cleanup job will not be able to run when the branch gets
    # deleted (when merging the MR).
    - apk add --update openssl
    - wget https://gitlab.com/gitlab-org/gitlab/-/raw/master/scripts/trigger-build
    - chmod 755 trigger-build
  variables:
    GIT_STRATEGY: none
    DOCS_REVIEW_APPS_DOMAIN: 178.62.207.141.nip.io
  when: manual
  only:
    refs:
      - branches@gitlab-org/omnibus-gitlab
  except:
    variables:
      - $CI_COMMIT_REF_NAME =~ /^[0-9]+-[0-9]+-auto-deploy-[0-9]+$/
      - $DEPS_PIPELINE || $CACHE_UPDATE || $DEPENDENCY_SCANNING

#####################
# Cache update jobs #
#####################
update-gems-cache:
  extends: .gems-cache
  stage: update-cache
  image: "${RUBY_IMAGE}"
  before_script:
    - *install-gems
  script:
    - echo "Cache is up to date!"
  cache:
    policy: push  # We want to rebuild the cache from scratch to ensure stale dependencies are cleaned up.
  only:
    refs:
      - schedules  # Jobs that take advantage of this cache are in the canonical project and the mirror one
    variables:
      - $CACHE_UPDATE

# We need to populate the cache for jobs with the `gitlab-org-docker` tag. Ideally, we wouldn't need this if
# we'd use Kaniko to build the Docker images, allowing to use the `gitlab-org` tag instead of the `gitlab-org-docker` tag.
update-gems-cache-for-docker-jobs:
  extends:
    - update-gems-cache
    - .docker_job

update-trigger-package-cache:
  extends: .trigger-package-cache
  stage: update-cache
  image: "${BUILDER_IMAGE_REGISTRY}/ubuntu_20.04:${BUILDER_IMAGE_REVISION}"
  script:
    - *build-package
    - echo "Cache is up to date!"
  cache:
    policy: push  # We want to rebuild the cache from scratch to ensure stale dependencies are cleaned up.
  tags:
    - triggered-packages
  only:
    refs:
      - schedules@gitlab-org/build/omnibus-gitlab-mirror  # Jobs that take advantage of this cache are exclusively in the mirror project
      - pipelines@gitlab-org/build/omnibus-gitlab-mirror
    variables:
      - $CACHE_UPDATE

###########################
#     Branch pipeline     #
###########################

Trigger:ce-package:
  extends: .trigger-job

Trigger:ee-package:
  extends: .trigger-job
  variables:
    ee: "true"

rubocop:
  extends: .gems-cache
  stage: check
  image: "${RUBY_IMAGE}"
  before_script:
    - *install-gems
  script:
    - bundle exec rubocop --parallel
  <<: *com-except-triggers
  needs: []

# Perform documentation linting on Markdown files
docs-lint markdown:
  image: registry.gitlab.com/gitlab-org/gitlab-docs/lint-markdown:alpine-3.14-vale-2.10.4-markdownlint-0.28.1
  stage: check
  cache: {}
  needs: []
  before_script: []
  script:
    # Lint prose
    - vale --minAlertLevel error doc
    # Lint Markdown
    - markdownlint --config .markdownlint.yml 'doc/**/*.md'
  <<: *com-docs-except-triggers

# Perform link checks on published HTML files
docs-lint links:
  image: registry.gitlab.com/gitlab-org/gitlab-docs/lint-html:alpine-3.14-ruby-2.7.4-db71f027
  stage: check
  cache: {}
  needs: []
  before_script: []
  script:
    # Prepare docs for build
    - mv doc/ /tmp/gitlab-docs/content/omnibus
    - cd /tmp/gitlab-docs
    # Build HTML from Markdown
    - bundle exec nanoc
    # Check the internal links
    - bundle exec nanoc check internal_links
    # Check the internal anchor links
    - bundle exec nanoc check internal_anchors
  <<: *com-docs-except-triggers

yard:
  extends: .gems-cache
  image: "${RUBY_IMAGE}"
  stage: check
  needs: []
  before_script:
    # These jobs will not be run on dev, so we set ALTERNATIVE_SOURCES to true
    # so tests run fine on forks
    - export ALTERNATIVE_SOURCES="true";
    - *install-gems
  script:
    - bundle exec yardoc
  <<: *com-docs-except-triggers
  artifacts:
    expire_in: 1 week
    paths:
      - yard/*


# Trigger a docs build in gitlab-docs
# Useful to preview the docs changes live
# https://docs.gitlab.com/ee/development/documentation/index.html#previewing-the-changes-live
review-docs-deploy:
  extends:
    - .review-docs
  environment:
    name: review-docs/branch-${CI_COMMIT_REF_SLUG}
    # DOCS_REVIEW_APPS_DOMAIN and DOCS_GITLAB_REPO_SUFFIX are CI variables
    # Discussion: https://gitlab.com/gitlab-org/gitlab-foss/merge_requests/14236/diffs#note_40140693
    auto_stop_in: 2 weeks
    url: http://${DOCS_BRANCH}-${DOCS_GITLAB_REPO_SUFFIX}-${CI_COMMIT_REF_SLUG}.${DOCS_REVIEW_APPS_DOMAIN}/${DOCS_GITLAB_REPO_SUFFIX}
    on_stop: review-docs-cleanup
  script:
    - ./trigger-build docs deploy

# Cleanup remote environment of gitlab-docs
review-docs-cleanup:
  extends:
    - .review-docs
  environment:
    name: review-docs/branch-${CI_COMMIT_REF_SLUG}
    action: stop
  script:
    - ./trigger-build docs cleanup

danger-review:
  image: registry.gitlab.com/gitlab-org/gitlab-build-images:danger
  stage: check
  cache: {}
  needs: []
  before_script: []
  only:
    refs:
      - branches@gitlab-org/omnibus-gitlab
  except:
    refs:
      - schedules@gitlab-org/omnibus-gitlab
      - /^[\d-]+-stable(-patch-[\d]+)?$/
      - master
    variables:
      - $DEPS_PIPELINE || $CACHE_UPDATE || $DEPENDENCY_SCANNING
      - $CI_COMMIT_REF_NAME =~ /^[0-9]+-[0-9]+-auto-deploy-[0-9]+$/
  script:
    - danger --fail-on-errors=true

Centos 7 knapsack: *prepare_knapsack
Centos 8 knapsack: *prepare_knapsack
Debian 9 knapsack: *prepare_knapsack
Debian 10 knapsack: *prepare_knapsack
OpenSUSE 15.2 knapsack: *prepare_knapsack
Ubuntu 16.04 knapsack: *prepare_knapsack
Ubuntu 18.04 knapsack: *prepare_knapsack
Ubuntu 20.04 knapsack: *prepare_knapsack

Ubuntu 16.04 specs:
  image: "registry.gitlab.com/gitlab-org/gitlab-build-images:omnibus-gitlab-xenial"
  extends: .spec_template
  parallel: 6
  needs:
  - Ubuntu 16.04 knapsack
Ubuntu 18.04 specs :
  image: "registry.gitlab.com/gitlab-org/gitlab-build-images:omnibus-gitlab-bionic"
  extends: .spec_template
  parallel: 6
  needs:
  - Ubuntu 18.04 knapsack
Ubuntu 20.04 specs :
  image: "registry.gitlab.com/gitlab-org/gitlab-build-images:omnibus-gitlab-focal"
  extends: .spec_template
  parallel: 6
  needs:
  - Ubuntu 20.04 knapsack
Debian 9 specs :
  image: "registry.gitlab.com/gitlab-org/gitlab-build-images:omnibus-gitlab-stretch"
  extends: .spec_template
  parallel: 6
  needs:
  - Debian 9 knapsack
Debian 10 specs :
  image: "registry.gitlab.com/gitlab-org/gitlab-build-images:omnibus-gitlab-buster"
  extends: .spec_template
  parallel: 6
  needs:
  - Debian 10 knapsack
Centos 7 specs :
  image: "registry.gitlab.com/gitlab-org/gitlab-build-images:omnibus-gitlab-centos7"
  extends: .spec_template
  parallel: 6
  needs:
  - Centos 7 knapsack
Centos 8 specs :
  image: "registry.gitlab.com/gitlab-org/gitlab-build-images:omnibus-gitlab-centos8"
  extends: .spec_template
  parallel: 6
  needs:
  - Centos 8 knapsack
OpenSUSE 15.2 specs :
  image: "registry.gitlab.com/gitlab-org/gitlab-build-images:omnibus-gitlab-opensuse15.2"
  extends: .spec_template
  parallel: 6
  needs:
  - OpenSUSE 15.2 knapsack

update-knapsack:
  extends: .knapsack-state
  image: "${RUBY_IMAGE}"
  stage: post-test
  before_script: []
  script:
    - support/merge-reports knapsack
    - rm -f knapsack/*node*
  <<: *com-except-triggers
  retry: 1

############################
#     Trigger Pipeline     #
############################

Trigger:package:
  extends: .trigger-package-cache
  image: "${BUILDER_IMAGE_REGISTRY}/ubuntu_20.04:${BUILDER_IMAGE_REVISION}"
  stage: trigger-package
  script:
    - if [ -n "$TRIGGERED_USER" ] && [ -n "$TRIGGER_SOURCE" ]; then echo "Pipeline triggered by $TRIGGERED_USER at $TRIGGER_SOURCE"; fi
    - *build-package
    # Renaming so we can easily generate the artifact URL
    - mv pkg/ubuntu-focal/*.deb pkg/ubuntu-focal/gitlab.deb
    - mv pkg/ubuntu-focal/*.deb.size pkg/ubuntu-focal/gitlab.deb.size
  <<: *com-trigger-and-nightly-only
  artifacts:
    expire_in: 3 days
    when: always
    paths:
      - pkg/
  tags:
    - triggered-packages
  needs:
    - job: fetch-assets
      optional: true
    - job: generate-facts
      optional: true
      artifacts: true

Trigger:package_size_check:
  extends: .trigger-package-cache
  image: "${BUILDER_IMAGE_REGISTRY}/ubuntu_20.04:${BUILDER_IMAGE_REVISION}"
  stage: trigger-qa
  script:
    - bundle exec rake build:package:generate_sizefile
    - bundle exec rake check:package_size
  <<: *com-trigger-only
  needs:
    - job: Trigger:package
      artifacts: false

Trigger:gitlab-docker:
  extends:
    - .docker_job
    - .gems-cache
  stage: trigger-docker
  script:
    - if [ -n "$TRIGGERED_USER" ] && [ -n "$TRIGGER_SOURCE" ]; then echo "Pipeline triggered by $TRIGGERED_USER at $TRIGGER_SOURCE"; fi
    - bundle exec rake docker:build:image
    - bundle exec rake docker:push:triggered
  <<: *com-trigger-only
  needs:
    - job: Trigger:package
      artifacts: false
    - job: generate-facts
      optional: true
      artifacts: true

Trigger:qa-docker:
  extends:
    - .docker_job
    - .gems-cache
  stage: trigger-docker
  script:
    - if [ -n "$TRIGGERED_USER" ] && [ -n "$TRIGGER_SOURCE" ]; then echo "Pipeline triggered by $TRIGGERED_USER at $TRIGGER_SOURCE"; fi
    - bundle exec rake qa:build
    - bundle exec rake qa:push:triggered
  <<: *com-trigger-only
  except:
    variables:
      - $SKIP_QA_DOCKER == "true"
  needs:
    - Trigger:package
    - job: generate-facts
      optional: true
      artifacts: true

Trigger:qa-test:
  stage: trigger-qa
  image: "${PUBLIC_BUILDER_IMAGE_REGISTRY}/ruby_docker:${BUILDER_IMAGE_REVISION}"
  script:
    - if [ -n "$TRIGGERED_USER" ] && [ -n "$TRIGGER_SOURCE" ]; then echo "Pipeline triggered by $TRIGGERED_USER at $TRIGGER_SOURCE"; fi
    - bundle exec rake qa:test
  <<: *com-trigger-only
  needs:
    - job: Trigger:package
      artifacts: false
    - job: Trigger:gitlab-docker
      artifacts: false
    - job: Trigger:qa-docker
      artifacts: false
      optional: true

Trigger:letsencrypt-test:
  extends: .docker_job
  stage: trigger-qa
  script:
    - if [ -n "$TRIGGERED_USER" ] && [ -n "$TRIGGER_SOURCE" ]; then echo "Pipeline triggered by $TRIGGERED_USER at $TRIGGER_SOURCE"; fi
    - echo "${CI_REGISTRY_PASSWORD}" | docker login -u $CI_REGISTRY_USER --password-stdin $CI_REGISTRY
    - curl -L "https://github.com/docker/compose/releases/download/1.29.1/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose
    - chmod +x /usr/local/bin/docker-compose
    - bundle exec rake qa:test_letsencrypt
  <<: *com-trigger-only
  needs:
    - job: Trigger:gitlab-docker
      artifacts: false

Trigger:RAT:
  stage: trigger-qa
  image: "${RUBY_IMAGE}"
  when: manual
  script:
    - bundle exec rake qa:rat:trigger
  <<: *com-trigger-only
  needs:
    - job: Trigger:package
      artifacts: false
    - job: Trigger:qa-docker
      artifacts: false
      optional: true

create_omnibus_manifest:
  extends: .trigger-package-cache
  image: "${BUILDER_IMAGE_REGISTRY}/debian_10:${BUILDER_IMAGE_REVISION}"
  stage: prepare-assets
  variables:
    TERM: xterm-256color
  script:
    - bundle exec omnibus manifest gitlab -l nothing 2> /dev/null > version-manifest.json
  <<: *depscan-rules
  artifacts:
    expire_in: 7 days
    paths:
      - version-manifest.json

dependency_scanning:
  image: "registry.gitlab.com/gitlab-org/security-products/gitlab-depscan:2.1.0"
  stage: package
  variables:
    REPORT_PATH: ./
    NVD_DB_UPDATE: "true"
  before_script: []
  script:
    - echo ${CVEIGNORE} | sed -r "s/, */\n/g" >> .cveignore
    - /gitlab-depscan.sh version-manifest.json
  <<: *depscan-rules
  needs:
    - create_omnibus_manifest
  artifacts:
    expire_in: 7 days
    when: on_failure
    reports:
      dependency_scanning: gl-dependency-scanning-report.json
    paths:
      - dependency_report.txt
      - .cveignore

dependency_update:
  image: "${BUILDER_IMAGE_REGISTRY}/ruby_docker:${BUILDER_IMAGE_REVISION}"
  stage: prepare
  before_script: []
  script:
    - curl https://deps.app/install.sh | bash -s -- -b $HOME/bin
    - $HOME/bin/deps ci
  only:
    variables:
      - $DEPS_PIPELINE

dependencies_io_check:
  variables:
    ee: "true"
  image: "${RUBY_IMAGE}"
  stage: trigger-package
  script:
    - support/wait_for_sha
    - bundle exec rake build:trigger
  only:
    refs:
      - /^deps.*/@gitlab-org/omnibus-gitlab
  except:
    refs:
      - triggers
      - branches@gitlab/omnibus-gitlab
  needs: []

validate_packer_changes:
  before_script: []
  image: "${PUBLIC_BUILDER_IMAGE_REGISTRY}/debian_packer:${BUILDER_IMAGE_REVISION}"
  stage: check
  script:
    - cd "${CI_PROJECT_DIR}/support/packer" && packer validate ce.json
    - cd "${CI_PROJECT_DIR}/support/packer" && packer validate ee.json
  only:
    changes:
      - support/packer/*
  <<: *com-except-triggers

##############################
#     Scheduled pipeline     #
##############################

pages:
  image: "${PUBLIC_BUILDER_IMAGE_REGISTRY}/ubuntu_20.04:${BUILDER_IMAGE_REVISION}"
  stage: prepare
  needs:
    - yard
  script:
    - bundle exec rake license:generate_pages
    - mv ${LICENSE_S3_BUCKET} public
    - cp support/webpages/* public
    - cp yard/* public
  artifacts:
    paths:
      - public
  only:
    - schedules@gitlab-org/omnibus-gitlab
  except:
    variables:
      - $DEPS_PIPELINE || $CACHE_UPDATE || $DEPENDENCY_SCANNING