gitlab
/
omnibus-gitlab
mirror of https://gitlab.com/gitlab-org/omnibus-gitlab.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Revert "Merge branch 'revert-cache-changes' into 'master'" 

This reverts merge request !7521
This commit is contained in:
Reuben Pereira 2024-04-04 04:09:05 +00:00
parent e8d0c32383
commit 5719d82182
8 changed files with 51 additions and 115 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitlab-ci.yml
View File

@ -60,6 +60,7 @@ variables:
DOCS_GITLAB_REPO_SUFFIX: "omnibus"
CACHE_KEY_SUFFIX: '-v3'
CACHE_EDITION: "CE"
CACHE_POLICY: 'pull-push'
ISSUE_BOT_LABELS_EXTRA: "group::distribution"
BUNDLER_VERSION: "2.5.6"
# NOTE: When `NEXT_RUBY_VERSION` is updated, flip

24
gitlab-ci-config/dev-gitlab-org.yml
View File

@ -34,11 +34,13 @@
.branch-cache: &branch-cache
key: "${CI_JOB_NAME}-${BUILDER_IMAGE_REVISION}-${CACHE_EDITION}${CACHE_KEY_SUFFIX}"
paths: !reference [.cache-paths]
policy: $CACHE_POLICY
.tag-cache: &tag-cache
# Use the same cache key as branch builds, by manually appending `-branch` substring.
key: "${CI_JOB_NAME}-branch-${BUILDER_IMAGE_REVISION}-${CACHE_EDITION}${CACHE_KEY_SUFFIX}"
paths: !reference [.cache-paths]
policy: $CACHE_POLICY
.branch_template: &branch_build
stage: package
@ -282,6 +284,11 @@ Ubuntu-20.04-branch:
Ubuntu-22.04-branch:
image: "${BUILDER_IMAGE_REGISTRY}/ubuntu_22.04:${BUILDER_IMAGE_REVISION}"
extends: .branch_template
rules:
- !reference [.default_rules, rules]
- if: '$PIPELINE_TYPE =~ /_(NIGHTLY|BRANCH)_BUILD_PIPELINE$/'
- if: '$PIPELINE_TYPE =~ /TRIGGERED_(CE|EE)_PIPELINE/'
- if: '$PIPELINE_TYPE == "TRIGGER_CACHE_UPDATE_PIPELINE"'
Debian-10-branch:
image: "${BUILDER_IMAGE_REGISTRY}/debian_10:${BUILDER_IMAGE_REVISION}"
@ -307,6 +314,12 @@ Ubuntu-20.04-arm64-branch:
Ubuntu-22.04-arm64-branch:
image: "${BUILDER_IMAGE_REGISTRY}/ubuntu_22.04_arm64:${BUILDER_IMAGE_REVISION}"
extends: .arm64_branch_template
rules:
- !reference [.default_rules, rules]
- if: '$PIPELINE_TYPE =~ /_(NIGHTLY|BRANCH)_BUILD_PIPELINE$/'
- if: '$PIPELINE_TYPE =~ /TRIGGERED_(CE|EE)_PIPELINE/'
when: manual
allow_failure: true
CentOS-7-branch:
image: "${BUILDER_IMAGE_REGISTRY}/centos_7:${BUILDER_IMAGE_REVISION}"
extends: .branch_template
@ -368,6 +381,15 @@ Ubuntu-18.04-fips-branch:
Ubuntu-20.04-fips-branch:
image: "${BUILDER_IMAGE_REGISTRY}/ubuntu_20.04_fips:${BUILDER_IMAGE_REVISION}"
extends: .fips_branch_template
rules:
- !reference [.default_rules, rules]
- if: '$PIPELINE_TYPE == "EE_NIGHTLY_BUILD_PIPELINE"'
- if: '$PIPELINE_TYPE == "EE_BRANCH_BUILD_PIPELINE"'
when: manual
allow_failure: true
- if: '$PIPELINE_TYPE =~ /TRIGGERED_(CE|EE)_PIPELINE/'
when: manual
allow_failure: true
Docker-branch:
extends: .docker_job
@ -388,6 +410,8 @@ Docker-branch:
rules:
- !reference [.default_rules, rules]
- if: '$PIPELINE_TYPE =~ /_(NIGHTLY|BRANCH)_BUILD_PIPELINE$/'
- if: '$PIPELINE_TYPE =~ /TRIGGERED_(CE|EE)_PIPELINE/'
- if: '$PIPELINE_TYPE == "TRIGGER_CACHE_UPDATE_PIPELINE"'
QA-branch:
extends: .docker_job

96
gitlab-ci-config/gitlab-com.yml
View File

@ -42,7 +42,7 @@
- gems
- assets_cache
- node_modules
policy: pull
policy: $CACHE_POLICY
.trigger-fips-package-cache:
cache:
@ -52,7 +52,7 @@
- gems
- assets_cache
- node_modules
policy: pull
policy: $CACHE_POLICY
.install-gems: &install-gems
- gem install bundler:${BUNDLER_VERSION}
@ -133,6 +133,8 @@
junit: junit_rspec.xml
.base-trigger-job-variables:
# In trigger jobs, we don't want to pollute cache
CACHE_POLICY: 'pull'
ALTERNATIVE_SOURCES: 'true'
SECURITY_SOURCES: ${SECURITY_SOURCES}
BUILDER_IMAGE_REVISION: ${BUILDER_IMAGE_REVISION}
@ -178,17 +180,6 @@
when: manual
variables: !reference [.ce-trigger-job-variables]
.trigger-pipeline:
stage: trigger-qa
image: "${RUBY_IMAGE}"
allow_failure: true
rules:
- if: '$PIPELINE_TYPE =~ /TRIGGERED_(CE|EE)_PIPELINE/'
when: manual
needs:
- job: Trigger:package
artifacts: false
.review-docs:
image: "${RUBY_IMAGE}-alpine"
stage: post-test
@ -259,22 +250,6 @@ update-gems-cache-for-docker-jobs:
- update-gems-cache
- .docker_job
update-trigger-package-cache:
extends: .trigger-package-cache
stage: update-cache
image: "${BUILDER_IMAGE_REGISTRY}/ubuntu_22.04:${BUILDER_IMAGE_REVISION}"
script:
- !reference [.build-package]
- echo "Cache is up to date!"
cache:
policy: push # We want to rebuild the cache from scratch to ensure stale dependencies are cleaned up.
tags: !reference [.distribution-amd64-tags]
rules:
- if: '$PIPELINE_TYPE == "TRIGGER_CACHE_UPDATE_PIPELINE"'
needs:
- job: fetch-assets
optional: true
###########################
# Branch pipeline #
###########################
@ -526,25 +501,6 @@ update-knapsack:
# Trigger Pipeline #
############################
Trigger:package:
extends: .trigger-package-common
variables:
BASE_OS: "ubuntu_22.04"
PACKAGE_DIRECTORY: "pkg/ubuntu-jammy"
Trigger:package:fips:
extends:
- .trigger-package-common
- .trigger-fips-package-cache
variables:
USE_SYSTEM_SSL: "true"
BASE_OS: "ubuntu_20.04_fips"
PACKAGE_DIRECTORY: "pkg/ubuntu-focal_fips"
allow_failure: true
rules:
- if: '$PIPELINE_TYPE == "TRIGGERED_EE_PIPELINE"'
when: manual
package_size_check:
extends: .trigger-package-cache
image: "${BUILDER_IMAGE_REGISTRY}/ubuntu_22.04:${BUILDER_IMAGE_REVISION}"
@ -553,29 +509,11 @@ package_size_check:
- bundle exec rake build:package:generate_sizefile
- bundle exec rake check:package_size
needs:
- job: Trigger:package
- job: Ubuntu-22.04-branch
artifacts: false
rules:
- if: '$PIPELINE_TYPE =~ /TRIGGERED_(CE|EE)_PIPELINE/'
Trigger:gitlab-docker:
extends:
- .docker_job
- .gems-cache
stage: trigger-docker
script:
- bundle exec rake docker:build:image
- bundle exec rake docker:push:triggered
rules:
- if: '$PIPELINE_TYPE =~ /TRIGGERED_(CE|EE)_PIPELINE/'
- if: '$PIPELINE_TYPE == "TRIGGER_CACHE_UPDATE_PIPELINE"'
needs:
- job: Trigger:package
artifacts: false
- job: generate-facts
optional: true
artifacts: true
qa-subset-test:
extends:
- .qa-template
@ -602,9 +540,10 @@ qa-subset-test:
needs:
- job: generate-facts
artifacts: true
- job: Trigger:package
- job: Ubuntu-22.04-branch
artifacts: false
- job: Trigger:gitlab-docker
- job: Docker-branch
optional: true
artifacts: false
qa-remaining-test-manual:
@ -629,9 +568,10 @@ qa-remaining-test-manual:
needs:
- job: generate-facts
artifacts: true
- job: Trigger:package
- job: Ubuntu-22.04-branch
artifacts: false
- job: Trigger:gitlab-docker
- job: Docker-branch
optional: true
artifacts: false
letsencrypt-test:
@ -647,7 +587,7 @@ letsencrypt-test:
when: never
- if: '$PIPELINE_TYPE =~ /TRIGGERED_(CE|EE)_PIPELINE/'
needs:
- job: Trigger:gitlab-docker
- job: Docker-branch
artifacts: false
RAT:
@ -672,10 +612,6 @@ RAT:
needs:
- job: Ubuntu-22.04-branch
artifacts: false
optional: true
- job: Trigger:package
artifacts: false
optional: true
- job: generate-facts
artifacts: true
@ -687,10 +623,6 @@ RAT:FIPS:
needs:
- job: Ubuntu-20.04-fips-branch
artifacts: false
optional: true
- job: Trigger:package:fips
artifacts: false
optional: true
- job: generate-facts
artifacts: true
@ -712,7 +644,7 @@ GET:Geo:
when: manual
allow_failure: true
needs:
- job: Trigger:package
- job: Ubuntu-22.04-branch
artifacts: false
- job: generate-facts
artifacts: true
@ -793,7 +725,7 @@ pages:
.build-package-on-all-os-vars:
extends: .base-trigger-job-variables
SKIP_JOB_REGEX: '/Ubuntu-22.04|Docker|QA/'
SKIP_JOB_REGEX: '/Ubuntu-22.04|Ubuntu-20.04-fips|Docker|QA/'
PIPELINE_TYPE: "${EDITION}_BRANCH_BUILD_PIPELINE"
CACHE_EDITION: ${EDITION}
ee: ${ee}

2
gitlab-ci-config/workflow-rules.yml
View File

@ -28,6 +28,8 @@ workflow:
variables:
PIPELINE_TYPE: TRIGGER_CACHE_UPDATE_PIPELINE
CACHE_EDITION: "GITLAB"
# We want to rebuild the cache from scratch to ensure stale dependencies are cleaned up.
CACHE_POLICY: 'push'
- if: '$PIPELINE_TYPE == null && $CI_PROJECT_PATH == $QA_PROJECT_PATH && $CACHE_UPDATE == "true"'
variables:
PIPELINE_TYPE: TRIGGER_CACHE_UPDATE_PIPELINE

4
lib/gitlab/build/facts.rb
View File

@ -91,8 +91,8 @@ module Build
GITLAB_SEMVER_VERSION=#{Build::Info::Git.latest_stable_tag.tr('+', '-')}
RAT_REFERENCE_ARCHITECTURE=#{Gitlab::Util.get_env('RAT_REFERENCE_ARCHITECTURE') || 'omnibus-gitlab-mrs'}
RAT_FIPS_REFERENCE_ARCHITECTURE=#{Gitlab::Util.get_env('RAT_FIPS_REFERENCE_ARCHITECTURE') || 'omnibus-gitlab-mrs-fips-ubuntu'}
RAT_PACKAGE_URL=#{Build::Info::CI.package_download_url}
RAT_FIPS_PACKAGE_URL=#{Build::Info::CI.fips_package_download_url}
RAT_PACKAGE_URL=#{Build::Info::CI.package_download_url(fips: false)}
RAT_FIPS_PACKAGE_URL=#{Build::Info::CI.package_download_url(job_name: 'Ubuntu-20.04', fips: true)}
]
end

33
lib/gitlab/build/info/ci.rb
View File

@ -38,26 +38,6 @@ module Build
Gitlab::Util.get_env('CI_COMMIT_REF_SLUG')
end
def package_download_url
return Gitlab::Util.get_env('PACKAGE_URL') if Gitlab::Util.get_env('PACKAGE_URL')
# For builds running in dev.gitlab.org, or nightly pipelines in .com,
# use the artifact from `Ubuntu-22.04-branch` job. For the other build,
# which is essentially in triggered pipeline, use artifact from
# `Trigger:package` job.
if /dev.gitlab.org/.match?(Build::Info::CI.api_v4_url) || Build::Check.is_nightly?
Build::Info::CI.branch_build_package_download_url
else
Build::Info::CI.triggered_package_download_url(fips: false)
end
end
def fips_package_download_url
return Gitlab::Util.get_env('FIPS_PACKAGE_URL') if Gitlab::Util.get_env('FIPS_PACKAGE_URL')
Build::Info::CI.triggered_package_download_url(fips: false)
end
def artifact_url(job_name, file_path)
client = Gitlab::APIClient.new
target_job_id = client.get_job_id(job_name)
@ -67,15 +47,10 @@ module Build
URI("#{api_v4_url}/projects/#{project_id}/jobs/#{target_job_id}/artifacts/#{file_path}")
end
def triggered_package_download_url(fips: Build::Check.use_system_ssl?)
folder = fips ? 'ubuntu-focal_fips' : 'ubuntu-jammy'
job_name = fips ? 'Trigger:package:fips' : 'Trigger:package'
package_path = "pkg/#{folder}/gitlab.deb"
def package_download_url(job_name: "Ubuntu-22.04", arch: 'amd64', fips: Build::Check.use_system_ssl?)
return Gitlab::Util.get_env('FIPS_PACKAGE_URL') if fips && Gitlab::Util.get_env('FIPS_PACKAGE_URL')
return Gitlab::Util.get_env('PACKAGE_URL') if !fips && Gitlab::Util.get_env('PACKAGE_URL')
artifact_url(job_name, package_path)
end
def branch_build_package_download_url(job_name: "Ubuntu-22.04", arch: 'amd64')
case job_name
when /AlmaLinux-8/
# In EL world, amd64 is called x86_64
@ -95,6 +70,8 @@ module Build
folder = "#{folder}_aarch64"
end
job_name = "#{job_name}-fips" if fips
job_name = "#{job_name}-branch" unless Build::Info::CI.tag_name
package_path = "pkg/#{folder}/#{package_file_name}"

2
lib/gitlab/package_size.rb
View File

@ -13,7 +13,7 @@ class PackageSizeCheck
class << self
def fetch_sizefile
sizefile_url = Build::Info::CI.artifact_url("Trigger:package", "pkg/ubuntu-jammy/gitlab.deb.size")
sizefile_url = URI("#{Build::Info::CI.package_download_url}.size")
# We have to use net/http here because `gitlab` gem's `download_job_artifact_file`
# method doesn't support plain text files. It has to be either binary or valid JSON.

4
spec/lib/gitlab/build/image_spec.rb
View File

@ -159,7 +159,7 @@ RSpec.describe Build::Image do
"PACKAGECLOUD_REPO=download-package",
"RELEASE_PACKAGE=gitlab-ce",
"RELEASE_VERSION=12.121.12-ce.0",
"DOWNLOAD_URL=https://gitlab.com/api/v4/projects/20699/jobs/999999/artifacts/pkg/ubuntu-jammy/gitlab.deb",
"DOWNLOAD_URL=https://gitlab.com/api/v4/projects/20699/jobs/999999/artifacts/pkg/ubuntu-jammy/gitlab-ce_12.121.12-ce.0_amd64.deb",
"CI_JOB_TOKEN=NOT-CI-JOB-TOKEN\n"
].join("\n")
end
@ -180,7 +180,7 @@ RSpec.describe Build::Image do
"PACKAGECLOUD_REPO=download-package",
"RELEASE_PACKAGE=gitlab-ee",
"RELEASE_VERSION=12.121.12-ee.0",
"DOWNLOAD_URL=https://gitlab.com/api/v4/projects/20699/jobs/999999/artifacts/pkg/ubuntu-jammy/gitlab.deb",
"DOWNLOAD_URL=https://gitlab.com/api/v4/projects/20699/jobs/999999/artifacts/pkg/ubuntu-jammy/gitlab-ee_12.121.12-ee.0_amd64.deb",
"CI_JOB_TOKEN=NOT-CI-JOB-TOKEN\n"
].join("\n")
end