gitlab
/
omnibus-gitlab
mirror of https://gitlab.com/gitlab-org/omnibus-gitlab.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Revert "Remove `only:` condition for Ubuntu-16.04-branch" 

This reverts commit dacdfba996.
This commit is contained in:
John Jarvis 2019-05-01 20:53:34 +00:00 committed by DJ Mountney
parent 50a23a8e4d
commit 0dfea635bd
13 changed files with 269 additions and 89 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
.gitlab-ci.yml
View File

@ -32,6 +32,11 @@ variables:
ASSET_PATH: "assets-${CI_COMMIT_REF_SLUG}"
COMPILE_ASSETS: "false"
RUBY_IMAGE: "ruby:2.5"
# Format of the auto-deploy tag for auto-deploy builds.
# https://gitlab.com/gitlab-org/release/docs/blob/master/general/deploy/auto-deploy.md#auto-deploy-tagging
AUTO_DEPLOY_TAG_REGEX: '^\d+\.\d+\.\d+\+[^ ]{7,}\.[^ ]{7,}$'
# Default environment for auto-deploy
AUTO_DEPLOY_ENVIRONMENT: 'pre'
MAX_PACKAGE_SIZE_MB: "700"
### For services that need a docker daemon
@ -102,6 +107,8 @@ fetch-assets:
except:
variables:
- $COMPILE_ASSETS == "true"
# Format of the auto-deploy tag
- $CI_COMMIT_TAG =~ /^\d+\.\d+\.\d+\+[^ ]{7,}\.[^ ]{7,}$/
include:
- '/gitlab-ci-config/gitlab-com.yml'

3
Gemfile
View File

@ -34,7 +34,7 @@ gem 'knapsack'
gem 'docker-api'
gem 'aws-sdk'
gem 'google_drive'
gem 'gitlab'
gem 'http'
group :rubocop do
gem 'rubocop'
@ -47,4 +47,5 @@ group :test do
gem 'omnibus-ctl', '0.3.6'
gem 'fantaskspec'
gem 'rspec_junit_formatter'
gem 'pry'
end

24
Gemfile.lock
View File

@ -779,6 +779,7 @@ GEM
rspec (~> 3.0)
citrus (3.0.2)
cleanroom (1.0.0)
coderay (1.1.2)
declarative (0.0.10)
declarative-option (0.1.0)
diff-lcs (1.3)
@ -800,9 +801,6 @@ GEM
ffi-yajl (2.3.1)
libyajl2 (~> 1.2)
fuzzyurl (0.9.0)
gitlab (4.7.0)
httparty (>= 0.14.0)
terminal-table (>= 1.5.1)
google-api-client (0.10.3)
addressable (~> 2.3)
googleauth (~> 0.5)
@ -826,11 +824,15 @@ GEM
signet (~> 0.7)
hashie (3.5.7)
highline (1.6.20)
http (4.1.1)
addressable (~> 2.3)
http-cookie (~> 1.0)
http-form_data (~> 2.0)
http_parser.rb (~> 0.6.0)
http-cookie (1.0.3)
domain_name (~> 0.5)
httparty (0.16.3)
mime-types (~> 3.0)
multi_xml (>= 0.5.2)
http-form_data (2.1.1)
http_parser.rb (0.6.0)
httpclient (2.8.3)
hurley (0.2)
iniparse (1.4.4)
@ -853,6 +855,7 @@ GEM
little-plugger (~> 1.1)
multi_json (~> 1.10)
memoist (0.16.0)
method_source (0.9.2)
mime-types (3.1)
mime-types-data (~> 3.2015)
mime-types-data (3.2016.0521)
@ -866,7 +869,6 @@ GEM
mixlib-shellout (2.3.2)
mixlib-versioning (1.2.7)
multi_json (1.13.1)
multi_xml (0.6.0)
multipart-post (2.0.0)
net-scp (1.2.1)
net-ssh (>= 2.6.5)
@ -915,6 +917,9 @@ GEM
powerpack (0.1.1)
progressbar (1.10.0)
proxifier (1.0.3)
pry (0.12.2)
coderay (~> 1.1.0)
method_source (~> 0.9.0)
public_suffix (3.0.2)
rack (2.0.4)
rainbow (2.2.2)
@ -976,8 +981,6 @@ GEM
sfl
syslog-logger (1.6.8)
systemu (2.6.5)
terminal-table (1.8.0)
unicode-display_width (~> 1.1, >= 1.1.1)
thor (0.18.1)
toml-rb (1.1.2)
citrus (~> 3.0, > 3.0)
@ -1000,14 +1003,15 @@ DEPENDENCIES
chefspec
docker-api
fantaskspec
gitlab
google_drive
http
json
knapsack
ohai
omnibus!
omnibus-ctl (= 0.3.6)
package_cloud
pry
rainbow (~> 2.2)
rake
rspec

55
gitlab-ci-config/dev-gitlab-org.yml
View File

@ -5,9 +5,17 @@
# Templates #
#############
.except-auto-deploy-tag: &except-auto-deploy-tag
except:
variables:
# Format of the auto-deploy tag for auto-deploy builds.
# https://gitlab.com/gitlab-org/release/docs/blob/master/general/deploy/auto-deploy.md#auto-deploy-tagging
- $CI_COMMIT_TAG =~ /^\d+\.\d+\.\d+\+[^ ]{7,}\.[^ ]{7,}$/
.dev-tag-only: &dev-tag-only
only:
- tags@gitlab/omnibus-gitlab
<<: *except-auto-deploy-tag
.dev-except-branches: &dev-except-branches
only:
@ -84,6 +92,15 @@
.tag_template: &tag_build
stage: package-and-image
script:
# Always compile assets for auto-deploy builds,
# this is done for auto-deploy builds
# so that we do not have to wait for the compile assets job
# in the gitlab-ee pipeline.
- |
if echo $CI_COMMIT_TAG | grep -P -q "$AUTO_DEPLOY_TAG_REGEX"; then
echo "Setting COMPILE_ASSETS env variable for auto-deploy"
export COMPILE_ASSETS='true';
fi
- echo "$DEV_GITLAB_SSH_KEY" > ~/.ssh/id_rsa
- cp support/known_hosts ~/.ssh/known_hosts
- chmod -R 0600 ~/.ssh/
@ -116,6 +133,9 @@
- /.*ee\.\d/
variables:
- $ee == "true"
# Format of the auto-deploy tag for auto-deploy builds.
# https://gitlab.com/gitlab-org/release/docs/blob/master/general/deploy/auto-deploy.md#auto-deploy-tagging
- $CI_COMMIT_TAG =~ /^\d+\.\d+\.\d+\+[^ ]{7,}\.[^ ]{7,}$/
### This stage is used for uploading packages
### 1. To a staging package repository
@ -130,8 +150,7 @@
retry: 2
tags:
- promotion
only:
- tags@gitlab/omnibus-gitlab
<<: *dev-tag-only
### This stage is ran only on tag:
### 1. It is a manual action
@ -161,6 +180,10 @@
- /.*ee\.\d/
variables:
- $ee == "true"
# Format of the auto-deploy tag for auto-deploy builds.
# https://gitlab.com/gitlab-org/release/docs/blob/master/general/deploy/auto-deploy.md#auto-deploy-tagging
- $CI_COMMIT_TAG =~ /^\d+\.\d+\.\d+\+[^ ]{7,}\.[^ ]{7,}$/
###########################
# Branch pipeline #
@ -268,6 +291,9 @@ Ubuntu-16.04:
image: "${BUILDER_IMAGE_REGISTRY}/ubuntu_16.04:${BUILDER_IMAGE_REVISION}"
stage: gitlab_com:package
extends: .tag_template
# Also build for auto-deploy tags
except:
variables: []
## Build the docker image for QA
## This is necessary for the deployment
@ -288,21 +314,27 @@ Docker-QA:
dependencies: []
retry: 1
## Upload and deploy to staging for GitLab.com
## The version of the deployment is set by the
## VERSION file. This job runs also in nightly branch builds.
Ubuntu-16.04-staging:
## Upload and deploy to staging or preprod for GitLab.com
## Normal tagged releases are deployed to staging
## Auto-deploy tagged releases are deployed to preprod
Ubuntu-16.04-upload-deploy:
stage: gitlab_com:upload_deploy
extends: .staging_upload_template
dependencies:
- Ubuntu-16.04
- Ubuntu-16.04-branch
variables:
TAKEOFF_ENVIRONMENT: 'gstg'
script:
- |
if echo $CI_COMMIT_TAG | grep -P -q "$AUTO_DEPLOY_TAG_REGEX"; then
export DEPLOYER_ENVIRONMENT="$AUTO_DEPLOY_ENVIRONMENT";
else
export DEPLOYER_ENVIRONMENT='gstg';
fi
- bundle exec rake repository:upload:staging[$STAGING_REPO]
- if [ -n "$TAKEOFF_TRIGGER_TOKEN" ]; then bundle exec rake gitlab_com:takeoff; fi
- if [ -n "$DEPLOYER_TRIGGER_TOKEN" ]; then bundle exec rake gitlab_com:deployer; fi
<<: *dev-except-branches
except:
variables: []
### Package builds
Ubuntu-18.04:
@ -342,6 +374,7 @@ dependency_scanning:
- manifest=pkg/ubuntu-xenial/*.version-manifest.json
- curl -sSL https://gitlab.com/gitlab-org/security-products/gitlab-depscan/raw/master/gitlab-depscan.sh|bash -s $manifest
<<: *dev-except-branches
<<: *except-auto-deploy-tag
dependencies:
- Ubuntu-16.04
- Ubuntu-16.04-branch
@ -542,6 +575,7 @@ Raspberry-Pi-2-Stretch-release:
dependencies:
- Raspberry-Pi-2-Stretch
<<: *raspbian_release
<<: *except-auto-deploy-tag
AWS-Marketplace-release:
image: "${BUILDER_IMAGE_REGISTRY}/ruby_docker:${BUILDER_IMAGE_REVISION}"
@ -556,5 +590,8 @@ AWS-Marketplace-release:
except:
variables:
- $CI_COMMIT_TAG =~ /^.*\+rc.*/
# Format of the auto-deploy tag for auto-deploy builds.
# https://gitlab.com/gitlab-org/release/docs/blob/master/general/deploy/auto-deploy.md#auto-deploy-tagging
- $CI_COMMIT_TAG =~ /^\d+\.\d+\.\d+\+[^ ]{7,}\.[^ ]{7,}$/
dependencies:
- AWS

14
lib/gitlab/build/check.rb
View File

@ -3,24 +3,34 @@ require_relative "../util.rb"
module Build
class Check
AUTO_DEPLOY_TAG_REGEX = /^\d+\.\d+\.\d+\+[^ ]{7,}\.[^ ]{7,}$/
class << self
def is_ee?
Gitlab::Util.get_env('ee') == 'true' || \
Gitlab::Util.get_env('GITLAB_VERSION')&.end_with?('-ee') || \
File.read('VERSION').strip.end_with?('-ee')
File.read('VERSION').strip.end_with?('-ee') || \
is_auto_deploy?
end
def match_tag?(tag)
system(*%W[git describe --exact-match --match #{tag}])
end
def is_auto_deploy?
AUTO_DEPLOY_TAG_REGEX.match?(Build::Info.current_git_tag)
end
def is_patch_release?
# Major and minor releases have patch component as zero
Info.semver_version.split(".")[-1] != "0"
end
def is_rc_tag?
`git describe --exact-match`.include?("+rc")
Build::Info.current_git_tag.include?("+rc")
end
def ci_commit_tag?
Gitlab::Util.get_env('CI_COMMIT_TAG')
end
def is_latest_stable_tag?

9
lib/gitlab/build/info.rb
View File

@ -16,6 +16,11 @@ module Build
"gitlab-ce"
end
# For auto-deploy builds, we set the semver to the following which is
# derived directly from the auto-deploy tag:
# MAJOR.MINOR.PIPELINE_ID+<ee ref>-<omnibus ref>
# See https://gitlab.com/gitlab-org/release/docs/blob/master/general/deploy/auto-deploy.md#auto-deploy-tagging
#
# For nightly builds we fetch all GitLab components from master branch
# If there was no change inside of the omnibus-gitlab repository, the
# package version will remain the same but contents of the package will be
@ -170,6 +175,10 @@ module Build
contents << "TRIGGER_PRIVATE_TOKEN=#{token.chomp}\n" if token && !token.empty?
contents.join
end
def current_git_tag
`git describe --exact-match 2>/dev/null`
end
end
end
end

69
lib/gitlab/deployer_helper.rb Normal file
View File

@ -0,0 +1,69 @@
require "http"
require "json"
require_relative "util.rb"
class PipelineTriggerFailure < StandardError; end
class DeployerHelper
TRIGGER_RETRY_INTERVAL = 5
TRIGGER_RETRIES = 3
def initialize(trigger_token, deploy_env, trigger_ref)
@deploy_env = deploy_env
@trigger_token = trigger_token
@trigger_ref = trigger_ref
@endpoint = "https://#{trigger_host}/api/v4"
end
def trigger_deploy
begin
retries ||= 0
resp = HTTP.post(pipeline_trigger_url, form: form_data_for_trigger)
raise PipelineTriggerFailure unless resp.status == 201
rescue PipelineTriggerFailure
if (retries += 1) < TRIGGER_RETRIES
sleep TRIGGER_RETRY_INTERVAL
puts "Retrying pipeline trigger ##{retries} due to invalid status: #{resp.status}"
retry
end
raise "Unable to trigger pipeline after #{retries} retries"
end
web_url_from_trigger(resp)
end
def web_url_from_trigger(resp)
JSON.parse(resp.body.to_s)['web_url']
rescue JSON::ParserError
raise "Unable to parse response from pipeline trigger, got #{resp.body}"
end
def pipeline_trigger_url
"#{@endpoint}/projects/#{trigger_project}/trigger/pipeline"
end
def trigger_host
@trigger_host ||= Gitlab::Util.get_env('DEPLOYER_TRIGGER_HOST') || 'ops.gitlab.net'
end
def trigger_project
# Project ID 135 is the project ID of deployer on ops.gitlab.net
@trigger_project ||= Gitlab::Util.get_env('DEPLOYER_TRIGGER_PROJECT') || '135'
end
def release
@release ||= Build::Info.docker_tag
end
def form_data_for_trigger
{
'token' => @trigger_token,
'ref' => @trigger_ref,
}.merge(deployer_env_vars.map { |k, v| ["variables[#{k}]", v] }.to_h)
end
def deployer_env_vars
{
'DEPLOY_ENVIRONMENT' => @deploy_env,
'DEPLOY_VERSION' => Gitlab::Util.get_env('DEPLOYER_VERSION') || release,
'DEPLOY_USER' => Gitlab::Util.get_env('DEPLOYER_DEPLOY_USER') || 'deployer'
}
end
end

47
lib/gitlab/takeoff_helper.rb
View File

@ -1,47 +0,0 @@
require 'gitlab'
require_relative "util.rb"
class TakeoffHelper
def initialize(trigger_token, deploy_env)
@deploy_env = deploy_env
@trigger_token = trigger_token
Gitlab.endpoint = "https://#{trigger_host}/api/v4"
end
def client
@client ||= Gitlab::Client.new(token: nil, host: trigger_host)
end
def trigger_deploy
# For triggers we don't need an API token, so we explicitly set it to nil
response = client.run_trigger(
trigger_project,
@trigger_token,
:master,
takeoff_env_vars
)
response.web_url
end
def trigger_host
Gitlab::Util.get_env('TAKEOFF_TRIGGER_HOST') || 'ops.gitlab.net'
end
def trigger_project
# Project ID 135 is the project ID of takeoff on ops.gitlab.net
Gitlab::Util.get_env('TAKEOFF_TRIGGER_PROJECT') || '135'
end
def release
@release ||= Build::Info.docker_tag
end
def takeoff_env_vars
{
'DEPLOY_ENVIRONMENT': @deploy_env,
'DEPLOY_VERSION': Gitlab::Util.get_env('TAKEOFF_VERSION') || release,
'DEPLOY_REPO': Gitlab::Util.get_env('TAKEOFF_DEPLOY_REPO') || 'gitlab/pre-release',
'DEPLOY_USER': Gitlab::Util.get_env('TAKEOFF_DEPLOY_USER') || 'takeoff'
}
end
end

36
lib/gitlab/tasks/gitlab_com.rake
View File

@ -1,34 +1,34 @@
require 'gitlab'
require_relative '../takeoff_helper.rb'
require_relative '../deployer_helper.rb'
require_relative "../util.rb"
namespace :gitlab_com do
desc 'Tasks related to gitlab.com.'
task :takeoff do
%w[TAKEOFF_TRIGGER_TOKEN TAKEOFF_ENVIRONMENT].each do |env_var|
task :deployer do
%w[DEPLOYER_TRIGGER_TOKEN DEPLOYER_ENVIRONMENT].each do |env_var|
abort "This task requires #{env_var} to be set" unless ENV[env_var]
end
latest_tag = Build::Info.latest_tag
unless Build::Check.match_tag?(latest_tag)
puts "#{latest_tag} is not the latest tag, not doing anything."
exit
end
unless Build::Info.package == "gitlab-ee"
puts "#{Build::Info.package} is not an ee package, not doing anything."
exit
end
trigger_token = Gitlab::Util.get_env('TAKEOFF_TRIGGER_TOKEN')
deploy_env = Gitlab::Util.get_env('TAKEOFF_ENVIRONMENT')
if !Build::Check.is_auto_deploy? && !Build::Check.is_latest_tag?
latest_tag = Build::Info.latest_tag
puts "#{latest_tag} is not the latest tag, not doing anything."
exit
end
# there is only support for staging deployments
# from ci at this time, error here for safety.
raise NotImplementedError, "Environment #{deploy_env} not supported" unless deploy_env == "gstg"
trigger_token = Gitlab::Util.get_env('DEPLOYER_TRIGGER_TOKEN')
trigger_ref = Build::Check.is_auto_deploy? && Build::Check.ci_commit_tag? ? Gitlab::Util.get_env('CI_COMMIT_TAG') : :master
deploy_env = Gitlab::Util.get_env('DEPLOYER_ENVIRONMENT')
takeoff_helper = TakeoffHelper.new(trigger_token, deploy_env)
url = takeoff_helper.trigger_deploy
puts "Takeoff build triggered at #{url}"
# We do not support auto-deployments or triggered deployments
# to production from the omnibus pipeline, this check is here
# for safety
raise NotImplementedError, "Environment #{deploy_env} is not supported" if deploy_env.include?('gprd')
deployer_helper = DeployerHelper.new(trigger_token, deploy_env, trigger_ref)
url = deployer_helper.trigger_deploy
puts "Deployer build triggered at #{url} on #{trigger_ref} for the #{deploy_env} environment"
end
end

33
spec/gitlab/build/check_spec.rb
View File

@ -117,4 +117,37 @@ describe Build::Check do
expect(described_class.is_patch_release?).to be_falsey
end
end
describe 'is_rc_tag?' do
it 'returns true if it looks like an rc tag' do
# It will be the case if the tag is 9.3.0+rc6.ce.0
allow(Build::Info).to receive(:current_git_tag).and_return('9.3.0+rc6.ce.0')
expect(described_class.is_rc_tag?).to be_truthy
end
it 'returns false if it does not look like an rc tag' do
# This not be the case if tag is eg. 9.3.0+ce.0
allow(Build::Info).to receive(:current_git_tag).and_return('9.3.0+ce.0')
expect(described_class.is_rc_tag?).to be_falsey
end
end
describe 'is_auto_deploy?' do
it 'returns true if it looks like an auto-deploy tag' do
# This is the case if the tag is 11.10.12345+5159f2949cb.59c9fa631
allow(Build::Info).to receive(:current_git_tag).and_return('11.10.12345+5159f2949cb.59c9fa631')
expect(described_class.is_auto_deploy?).to be_truthy
end
it 'returns false if it does not look like an auto-deploy tag' do
# This not be the case if ag is eg. 9.3.0+ce.0
allow(Build::Info).to receive(:current_git_tag).and_return('9.3.0+ce.0')
expect(described_class.is_auto_deploy?).to be_falsey
end
end
describe 'ci_commit_tag?' do
it 'checks for the CI_COMMIT_TAG' do
allow(Gitlab::Util).to receive(:get_env).with('CI_COMMIT_TAG').and_return('11.10.12345+5159f2949cb.59c9fa631')
expect(described_class.ci_commit_tag?).to be_truthy
end
end
end

3
spec/gitlab/build/info_spec.rb
View File

@ -74,6 +74,7 @@ describe Build::Info do
describe 'for CE' do
before do
stub_is_ee(false)
allow(described_class).to receive(:`).with("git describe --exact-match 2>/dev/null").and_return('12.121.12+rc7.ce.0')
allow(described_class).to receive(:`).with("git -c versionsort.prereleaseSuffix=rc tag -l '*[+.]ce.*' --sort=-v:refname | head -1").and_return('12.121.12+rc7.ce.0')
end
@ -98,6 +99,7 @@ describe Build::Info do
describe 'for CE' do
before do
stub_is_ee(nil)
allow(described_class).to receive(:`).with("git describe --exact-match 2>/dev/null").and_return('12.121.12+ce.0')
allow(described_class).to receive(:`).with("git -c versionsort.prereleaseSuffix=rc tag -l '*[+.]ce.*' --sort=-v:refname | awk '!/rc/' | head -1").and_return('12.121.12+ce.0')
end
@ -136,6 +138,7 @@ describe Build::Info do
describe '.previous_version' do
it 'detects previous version correctly' do
allow(described_class).to receive(:`).with("git describe --exact-match 2>/dev/null").and_return('10.4.0+ee.0')
allow(Build::Info).to receive(:`).with(/git -c versionsort/).and_return("10.4.0+ee.0\n10.3.5+ee.0")
expect(described_class.previous_version).to eq("10.3.5-ee.0")

46
spec/gitlab/deployer_helper_spec.rb Normal file
View File

@ -0,0 +1,46 @@
require 'spec_helper'
require 'gitlab/build'
require 'gitlab/deployer_helper'
describe DeployerHelper do
subject(:service) { described_class.new('some-token', 'some-env', 'some-branch') }
describe '#trigger_deploy' do
it 'triggers an auto deploy' do
response = instance_double('response', body: JSON.dump(web_url: 'http://example.com'), status: 201)
allow(Build::Info).to receive(:docker_tag).and_return('some-version')
expect(HTTP)
.to receive(:post)
.with(
"https://ops.gitlab.net/api/v4/projects/135/trigger/pipeline",
form: {
"token" => "some-token",
"ref" => "some-branch",
"variables[DEPLOY_ENVIRONMENT]" => "some-env",
"variables[DEPLOY_VERSION]" => "some-version",
"variables[DEPLOY_USER]" => "deployer"
}
).and_return(response)
expect(service.trigger_deploy).to eq('http://example.com')
end
it 'triggers an auto deploy with retries' do
# Set this to zero so there we don't have delays during tests
stub_const('DeployerHelper::TRIGGER_RETRY_INTERVAL', 0)
response = instance_double('response', body: JSON.dump(web_url: 'http://example.com'), status: 401)
allow(Build::Info).to receive(:docker_tag).and_return('some-version')
expect(HTTP)
.to receive(:post)
.with(
"https://ops.gitlab.net/api/v4/projects/135/trigger/pipeline",
form: {
"token" => "some-token",
"ref" => "some-branch",
"variables[DEPLOY_ENVIRONMENT]" => "some-env",
"variables[DEPLOY_VERSION]" => "some-version",
"variables[DEPLOY_USER]" => "deployer"
}
).and_return(response).exactly(3).times
expect { service.trigger_deploy }.to raise_error(RuntimeError, "Unable to trigger pipeline after 3 retries")
end
end
end

12
support/is_gitlab_ee.sh
View File

@ -8,8 +8,16 @@ function ee_branch_var(){
[[ -n "${GITLAB_VERSION}" ]] && [[ "${GITLAB_VERSION}" == *-ee ]]
}
function ee_branch_name(){
function ee_branch_name(){
grep -q -E "\-ee" VERSION
}
ee_env_var || ee_branch_var || ee_branch_name
function is_auto_deploy_tag(){
echo "$CI_COMMIT_TAG" | grep -q -P -- '^\d+\.\d+\.\d+\+[^ ]{7,}\.[^ ]{7,}$'
}
function is_auto_deploy_branch(){
echo "$CI_COMMIT_REF_NAME" | grep -q -- '-auto-deploy-'
}
ee_env_var || ee_branch_var || ee_branch_name || is_auto_deploy_tag || is_auto_deploy_branch