omnibus-gitlab/gitlab-ci-config/gitlab-com.yml

# This config lists the jobs that will be run on omnibus-gitlab project in
# gitlab.com.

#############
# Templates #
#############

.com-except-triggers: &com-except-triggers
  except:
    refs:
      - /./@gitlab/omnibus-gitlab
      - triggers
      - /^deps/
      - /(^docs[\/-].*|.*-docs$)/
      - /./@gitlab-org/build/omnibus-gitlab-mirror
    variables:
      - $DEPS_PIPELINE

.com-trigger-only: &com-trigger-only
  only:
    - triggers@gitlab-org/build/omnibus-gitlab-mirror
    - pipelines@gitlab-org/build/omnibus-gitlab-mirror

.dedicated-runner: &dedicated-runner
  tags:
    - gitlab-org

.knapsack-artifacts: &knapsack-artifacts
  expire_in: 31d
  paths:
  - knapsack/

.knapsack-state: &knapsack-state
  services: []
  cache:
    key: "knapsack"
    paths:
    - knapsack/
  artifacts: *knapsack-artifacts

.knapsack: &prepare_knapsack
  <<: *knapsack-state
  <<: *dedicated-runner
  stage: prepare
  before_script: []
  script:
    - JOB_NAME=( $CI_JOB_NAME )
    - export DISTRO_NAME=${JOB_NAME[0]}
    - export DISTRO_VERSION=${JOB_NAME[1]}
    - mkdir -p knapsack/
    - '[[ -f knapsack/${DISTRO_NAME}_${DISTRO_VERSION}_main_rspec_report.json ]] || echo "{}" > knapsack/${DISTRO_NAME}_${DISTRO_VERSION}_main_rspec_report.json'
  <<: *com-except-triggers
  retry: 1

.spec_template:
  stage: tests
  <<: *dedicated-runner
  before_script:
    # These jobs will not be run on dev, so we set ALTERNATIVE_SOURCES to true
    # so tests run fine on forks
    - export ALTERNATIVE_SOURCES="true";
    - bundle install -j $(nproc) --binstubs --path gems
  script:
    - JOB_NAME=( $CI_JOB_NAME )
    - export DISTRO_NAME=${JOB_NAME[0]}
    - export DISTRO_VERSION=${JOB_NAME[1]}
    - export KNAPSACK_REPORT_PATH=knapsack/${DISTRO_NAME}_${DISTRO_VERSION}_rspec_node_${CI_NODE_INDEX}_${CI_NODE_TOTAL}_report.json
    - export KNAPSACK_GENERATE_REPORT=true
    - export USE_KNAPSACK=true
    # To prevent current OS providing empty/old reports of other OSs as an
    # artifact. If not, they may overwrite the valid/new reports from those
    # corresponding OSs. So, removing everything except current OS's report.
    - cp knapsack/${DISTRO_NAME}_${DISTRO_VERSION}_main_rspec_report.json ${KNAPSACK_REPORT_PATH}.bak
    - rm -f knapsack/*.json
    - mv ${KNAPSACK_REPORT_PATH}.bak ${KNAPSACK_REPORT_PATH}
    - bundle exec rake "knapsack:rspec[--color --format RspecJunitFormatter --out junit_rspec.xml --format documentation]"
  <<: *com-except-triggers
  cache:
    key: "${CI_JOB_NAME}-${BUILDER_IMAGE_REVISION}"
    paths:
    - gems
  artifacts:
    <<: *knapsack-artifacts
    reports:
      junit: junit_rspec.xml
  retry: 1

.trigger-job: &trigger-job
  image: "ruby:2.4"
  stage: trigger-package
  <<: *dedicated-runner
  when: manual
  script:
    - bundle exec rake build:trigger
  only:
    - branches@gitlab-org/omnibus-gitlab
  except:
    refs:
      - triggers
      - /^deps/
      - /(^docs[\/-].*|.*-docs$)/
    variables:
      - $DEPS_PIPELINE

###########################
#     Branch pipeline     #
###########################

Trigger:ce-package: *trigger-job

Trigger:ee-package:
  <<: *trigger-job
  variables:
    ee: "true"

rubocop:
  stage: check
  <<: *dedicated-runner
  image: ruby:2.6
  before_script:
    - bundle install -j $(nproc) --binstubs --path gems --without default test
  script:
    - bundle exec rubocop --parallel
  <<: *com-except-triggers
  cache:
    key: "${CI_JOB_NAME}-${BUILDER_IMAGE_REVISION}"
    paths:
    - gems

docs-lint:
  image: "registry.gitlab.com/gitlab-org/gitlab-docs:lint"
  stage: check
  cache: {}
  dependencies: []
  before_script: []
  <<: *dedicated-runner
  script:
    # Lint prose
    - vale --minAlertLevel error doc
    # Lint Markdown
    - markdownlint --config .markdownlint.json 'doc/**/*.md'
    # Prepare docs for build
    - mv doc/ /tmp/gitlab-docs/content/omnibus
    - cd /tmp/gitlab-docs
    # Build HTML from Markdown
    - bundle exec nanoc
    # Check the internal links
    - bundle exec nanoc check internal_links
    # Check the internal anchor links
    - bundle exec nanoc check internal_anchors
  except:
    refs:
      - /./@gitlab/omnibus-gitlab
      - triggers
      - /^deps/
      - /./@gitlab-org/build/omnibus-gitlab-mirror
    variables:
      - $DEPS_PIPELINE

danger-review:
  image: registry.gitlab.com/gitlab-org/gitlab-build-images:danger
  stage: check
  cache: {}
  dependencies: []
  before_script: []
  only:
    refs:
      - branches@gitlab-org/omnibus-gitlab
  except:
    refs:
      - schedules@gitlab-org/omnibus-gitlab
      - /^[\d-]+-stable(-patch-[\d]+)?$/
    variables:
      - $DEPS_PIPELINE
  script:
    - danger --fail-on-errors=true

Centos 6 knapsack: *prepare_knapsack
Centos 7 knapsack: *prepare_knapsack
Centos 8 knapsack: *prepare_knapsack
Debian 8 knapsack: *prepare_knapsack
Debian 9 knapsack: *prepare_knapsack
Debian 10 knapsack: *prepare_knapsack
OpenSUSE 15.1 knapsack: *prepare_knapsack
Ubuntu 16.04 knapsack: *prepare_knapsack
Ubuntu 18.04 knapsack: *prepare_knapsack

Ubuntu 16.04 specs:
  image: "registry.gitlab.com/gitlab-org/gitlab-build-images:omnibus-gitlab-xenial"
  extends: .spec_template
  parallel: 6
  dependencies:
  - Ubuntu 16.04 knapsack
Ubuntu 18.04 specs :
  image: "registry.gitlab.com/gitlab-org/gitlab-build-images:omnibus-gitlab-bionic"
  extends: .spec_template
  parallel: 6
  dependencies:
  - Ubuntu 18.04 knapsack
Debian 8 specs :
  image: "registry.gitlab.com/gitlab-org/gitlab-build-images:omnibus-gitlab-jessie"
  extends: .spec_template
  parallel: 6
  dependencies:
  - Debian 8 knapsack
Debian 9 specs :
  image: "registry.gitlab.com/gitlab-org/gitlab-build-images:omnibus-gitlab-stretch"
  extends: .spec_template
  parallel: 6
  dependencies:
  - Debian 9 knapsack
Debian 10 specs :
  image: "registry.gitlab.com/gitlab-org/gitlab-build-images:omnibus-gitlab-buster"
  extends: .spec_template
  parallel: 6
  dependencies:
  - Debian 10 knapsack
Centos 6 specs :
  image: "registry.gitlab.com/gitlab-org/gitlab-build-images:omnibus-gitlab-centos6"
  extends: .spec_template
  parallel: 6
  dependencies:
  - Centos 6 knapsack
Centos 7 specs :
  image: "registry.gitlab.com/gitlab-org/gitlab-build-images:omnibus-gitlab-centos7"
  extends: .spec_template
  parallel: 6
  dependencies:
  - Centos 7 knapsack
Centos 8 specs :
  image: "registry.gitlab.com/gitlab-org/gitlab-build-images:omnibus-gitlab-centos8"
  extends: .spec_template
  parallel: 6
  dependencies:
  - Centos 8 knapsack
OpenSUSE 15.1 specs :
  image: "registry.gitlab.com/gitlab-org/gitlab-build-images:omnibus-gitlab-opensuse15.1"
  extends: .spec_template
  parallel: 6
  dependencies:
  - OpenSUSE 15.1 knapsack

update-knapsack:
  <<: *knapsack-state
  <<: *dedicated-runner
  image: "${RUBY_IMAGE}"
  stage: post-test
  before_script: []
  script:
    - support/merge-reports knapsack
    - rm -f knapsack/*node*
  <<: *com-except-triggers
  retry: 1

############################
#     Trigger Pipeline     #
############################

Trigger:package:
  image: "${BUILDER_IMAGE_REGISTRY}/ubuntu_16.04:${BUILDER_IMAGE_REVISION}"
  stage: trigger-package
  script:
    - if [ -n "$TRIGGERED_USER" ] && [ -n "$TRIGGER_SOURCE" ]; then echo "Pipeline triggered by $TRIGGERED_USER at $TRIGGER_SOURCE"; fi
    - bundle exec rake cache:populate
    - bundle exec rake cache:restore
    - bundle exec rake build:project
    - bundle exec rake cache:bundle
    # Renaming so we can easily generate the artifact URL
    - mv pkg/ubuntu-xenial/*.deb pkg/ubuntu-xenial/gitlab.deb
    - bundle exec rspec --color --format RspecJunitFormatter --out junit_ci_rspec.xml --format documentation ci_build_specs
    - bundle exec rake build:component_shas
  <<: *com-trigger-only
  artifacts:
    expire_in: 3 days
    when: always
    paths:
      - pkg/
    reports:
      junit: junit_ci_rspec.xml
  cache:
    key: "Ubuntu-16.04-branch-${BUILDER_IMAGE_REVISION}"
    paths:
    - cache
    - gems
    - assets_cache
    - .yarn-cache
  tags:
    - triggered-packages

Trigger:gitlab-docker:
  extends: .docker_job
  stage: trigger-docker
  script:
    - if [ -n "$TRIGGERED_USER" ] && [ -n "$TRIGGER_SOURCE" ]; then echo "Pipeline triggered by $TRIGGERED_USER at $TRIGGER_SOURCE"; fi
    # While triggering from omnibus repo in .com, we explicitly pass IMAGE_TAG
    # variable, which will be used to tag the final Docker image.
    # So, if IMAGE_TAG variable is empty, it means the trigger happened from
    # either CE or EE repository. In that case, we can use the GITLAB_VERSION
    # variable as IMAGE_TAG.
    - if [ -z "${IMAGE_TAG}" ]; then export IMAGE_TAG=${GITLAB_VERSION}; fi
    - bundle exec rake docker:build:image
    - bundle exec rake docker:push:triggered
  <<: *com-trigger-only

Trigger:measure-memory:
  extends: .measure_memory_script_artifacts
  dependencies: []
  stage: metrics
  when: manual
  <<: *com-trigger-only

Trigger:qa-docker:
  extends: .docker_job
  stage: trigger-docker
  script:
    - if [ -n "$TRIGGERED_USER" ] && [ -n "$TRIGGER_SOURCE" ]; then echo "Pipeline triggered by $TRIGGERED_USER at $TRIGGER_SOURCE"; fi
    # For builds triggered from omnibus project, we are explicitly specifying IMAGE_TAG
    - if [ -z "$IMAGE_TAG" ]; then export IMAGE_TAG=$GITLAB_VERSION; fi
    - bundle exec rake qa:build
    - bundle exec rake qa:push:triggered
  <<: *com-trigger-only
  dependencies:
    - Trigger:package

Trigger:qa-test:
  stage: trigger-qa
  image: "${PUBLIC_BUILDER_IMAGE_REGISTRY}/ruby_docker:${BUILDER_IMAGE_REVISION}"
  script:
    - if [ -n "$TRIGGERED_USER" ] && [ -n "$TRIGGER_SOURCE" ]; then echo "Pipeline triggered by $TRIGGERED_USER at $TRIGGER_SOURCE"; fi
    # For builds triggered from omnibus project, we are explicitly specifying IMAGE_TAG
    - if [ -z "$IMAGE_TAG" ]; then export IMAGE_TAG=$GITLAB_VERSION; fi
    - bundle exec rake qa:test
  <<: *com-trigger-only
  dependencies: []

Trigger:ha-validate:
  stage: trigger-ha-validate
  image: "ruby:2.4"
  when: manual
  script:
      - bundle exec rake qa:ha:validate
  <<: *com-trigger-only
  dependencies:
    - Trigger:ee-package

dependency_update:
  image: "${BUILDER_IMAGE_REGISTRY}/ruby_docker:${BUILDER_IMAGE_REVISION}"
  stage: prepare
  before_script: []
  script:
    - curl https://deps.app/install.sh | bash -s -- -b $HOME/bin
    - $HOME/bin/deps ci
  only:
    variables:
      - $DEPS_PIPELINE

dependencies_io_check:
  image: "${RUBY_IMAGE}"
  stage: trigger-package
  <<: *dedicated-runner
  script:
    - bundle exec rake build:trigger
  only:
    refs:
      - /^deps.*/@gitlab-org/omnibus-gitlab
  except:
    refs:
      - triggers
      - branches@gitlab/omnibus-gitlab

validate_packer_changes:
  before_script: []
  image: "${PUBLIC_BUILDER_IMAGE_REGISTRY}/debian_packer:${BUILDER_IMAGE_REVISION}"
  stage: check
  script:
    - cd "${CI_PROJECT_DIR}/support/packer" && packer validate ce.json
    - cd "${CI_PROJECT_DIR}/support/packer" && packer validate ee.json
  only:
    changes:
      - support/packer/*
  <<: *com-except-triggers

##############################
#     Scheduled pipeline     #
##############################

pages:
  image: "${PUBLIC_BUILDER_IMAGE_REGISTRY}/ubuntu_16.04:${BUILDER_IMAGE_REVISION}"
  stage: prepare
  script:
    - bundle exec rake license:generate_pages
    - mv ${LICENSE_S3_BUCKET} public
    - cp support/webpages/* public
  artifacts:
    paths:
      - public
  only:
    - schedules@gitlab-org/omnibus-gitlab
  except:
    variables:
      - $DEPS_PIPELINE