gitlab
/
omnibus-gitlab
mirror of https://gitlab.com/gitlab-org/omnibus-gitlab.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'gitaly_shard_connections' into 'master' 

Gitaly shard connections

Closes #2153

See merge request !1462
This commit is contained in:
Marin Jankovski 2017-04-10 11:07:48 +00:00
parent 7110015f20 73b0f0dce3
commit 2096928144
12 changed files with 119 additions and 67 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
CHANGELOG.md
View File

@ -7,6 +7,7 @@ omnibus-gitlab repository.
- Remove deprecated satellites configuration
- Add configuration file for Gitaly
- Add support for Gitaly address per shard
9.0.4

5
files/gitlab-config-template/gitlab.rb.template
View File

@ -254,7 +254,10 @@ external_url 'GENERATED_EXTERNAL_URL'
###! Docs: https://docs.gitlab.com/omnibus/settings/configuration.html#storing-git-data-in-an-alternative-directory
###! **If you want to use a single non-default directory to store git data use a
###! path that doesn't contain symlinks.**
# git_data_dirs({ "default" => { "path" => "/var/opt/gitlab/git-data" } })
# git_data_dirs({ "default" => { "path" => "/var/opt/gitlab/git-data", 'gitaly_address' => 'unix:/var/opt/gitlab/gitaly/gitaly.socket' } })
### Gitaly settings
# gitlab_rails['gitaly_enabled'] = true
### For storing GitLab application uploads, eg. LFS objects, build artifacts
###! Docs: https://docs.gitlab.com/ce/development/shared_files.html

12
files/gitlab-cookbooks/gitlab/attributes/default.rb
View File

@ -139,6 +139,8 @@ default['gitlab']['gitlab-rails']['registry_api_url'] = nil
default['gitlab']['gitlab-rails']['registry_key_path'] = nil
default['gitlab']['gitlab-rails']['registry_path'] = nil
default['gitlab']['gitlab-rails']['registry_issuer'] = "omnibus-gitlab-issuer"
# Defaults set in libraries/gitlab_rails.rb
default['gitlab']['gitlab-rails']['repositories_storages'] = {}
####
# These LDAP settings are deprecated in favor of the new syntax. They are kept here for backwards compatibility.
@ -319,16 +321,14 @@ default['gitlab']['gitlab-shell']['dir'] = "/var/opt/gitlab/gitlab-shell"
default['gitlab']['gitlab-shell']['log_directory'] = "/var/log/gitlab/gitlab-shell/"
default['gitlab']['gitlab-shell']['log_level'] = nil
default['gitlab']['gitlab-shell']['audit_usernames'] = nil
default['gitlab']['gitlab-shell']['git_data_directories'] = {
"default" => { "path" => "/var/opt/gitlab/git-data" }
}
default['gitlab']['gitlab-rails']['repositories_storages'] = {
"default" => { "path" => "/var/opt/gitlab/git-data/repositories" }
}
default['gitlab']['gitlab-shell']['http_settings'] = nil
default['gitlab']['gitlab-shell']['auth_file'] = nil
default['gitlab']['gitlab-shell']['git_trace_log_file'] = nil
default['gitlab']['gitlab-shell']['custom_hooks_dir'] = nil
# DEPRECATED! Not used by gitlab-shell
default['gitlab']['gitlab-shell']['git_data_directories'] = {
"default" => { "path" => "/var/opt/gitlab/git-data" }
}
###
# PostgreSQL

15
files/gitlab-cookbooks/gitlab/libraries/gitaly.rb
View File

@ -38,5 +38,20 @@ module Gitaly
Gitlab['gitaly']['env'] = gitaly_env
end
def gitaly_address
socket_path = user_config['socket_path'] || package_default['socket_path']
"unix:#{socket_path}"
end
private
def user_config
Gitlab['gitaly']
end
def package_default
Gitlab['node']['gitlab']['gitaly'].to_hash
end
end
end

4
files/gitlab-cookbooks/gitlab/libraries/gitlab.rb
View File

@ -239,6 +239,9 @@ module Gitlab
def generate_config(node_name)
generate_secrets(node_name)
GitlabWorkhorse.parse_variables
# Parse Gitaly before gitlab-shell and gitlab-rails
# because we need details for repositories_storages
Gitaly.parse_variables
GitlabShell.parse_variables
GitlabRails.parse_variables
Logging.parse_variables
@ -249,7 +252,6 @@ module Gitlab
GitlabMattermost.parse_variables
GitlabPages.parse_variables
Registry.parse_variables
Gitaly.parse_variables
Prometheus.parse_variables
# Parse nginx variables last because we want all external_url to be
# parsed first

24
files/gitlab-cookbooks/gitlab/libraries/gitlab_rails.rb
View File

@ -16,6 +16,7 @@
#
require_relative 'nginx.rb'
require_relative 'gitaly.rb'
module GitlabRails
class << self
@ -26,6 +27,7 @@ module GitlabRails
parse_directories
parse_gitlab_trusted_proxies
parse_rack_attack_protected_paths
parse_gitaly_variables
end
def parse_directories
@ -33,6 +35,7 @@ module GitlabRails
parse_artifacts_dir
parse_lfs_objects_dir
parse_pages_dir
parse_repository_storage
end
def parse_external_url
@ -89,6 +92,18 @@ module GitlabRails
Gitlab['gitlab_rails']['pages_path'] ||= File.join(Gitlab['gitlab_rails']['shared_path'], 'pages')
end
def parse_repository_storage
return if Gitlab['gitlab_rails']['repositories_storages']
gitaly_address = Gitaly.gitaly_address
Gitlab['gitlab_rails']['repositories_storages'] ||= {
"default" => {
"path" => "/var/opt/gitlab/git-data/repositories",
"gitaly_address" => gitaly_address
}
}
end
def parse_gitlab_trusted_proxies
Gitlab['nginx']['real_ip_trusted_addresses'] ||= Gitlab['node']['gitlab']['nginx']['real_ip_trusted_addresses']
Gitlab['gitlab_rails']['trusted_proxies'] ||= Gitlab['nginx']['real_ip_trusted_addresses']
@ -136,6 +151,15 @@ module GitlabRails
"#{Gitlab['node']['package']['install-dir']}/embedded/service/gitlab-rails/public"
end
def parse_gitaly_variables
return unless Gitlab['gitlab_rails']['gitaly_enabled'].nil?
gitaly_enabled = Gitlab['gitaly']['enable']
gitaly_enabled = Gitlab['node']['gitlab']['gitaly']['enable'] if gitaly_enabled.nil?
Gitlab['gitlab_rails']['gitaly_enabled'] = gitaly_enabled
end
private
def any_service_role_defined?

4
files/gitlab-cookbooks/gitlab/libraries/gitlab_shell.rb
View File

@ -14,6 +14,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
require_relative 'gitaly.rb'
module GitlabShell
class << self
@ -26,6 +27,7 @@ module GitlabShell
git_data_dirs = Gitlab['git_data_dirs']
git_data_dir = Gitlab['git_data_dir']
return unless git_data_dirs.any? || git_data_dir
gitaly_address = Gitaly.gitaly_address
Gitlab['gitlab_shell']['git_data_directories'] ||=
if git_data_dirs.any?
@ -43,7 +45,7 @@ module GitlabShell
Gitlab['gitlab_rails']['repositories_storages'] ||=
Hash[Gitlab['gitlab_shell']['git_data_directories'].map do |name, data_directory|
[name, { 'path' => File.join(data_directory['path'], 'repositories') }]
[name, { 'path' => File.join(data_directory['path'], 'repositories'), 'gitaly_address' => gitaly_address }]
end]
end

5
files/gitlab-cookbooks/gitlab/recipes/gitlab-rails.rb
View File

@ -227,8 +227,6 @@ templatesymlink "Create a relative_url.rb and create a symlink to Rails root" do
end
end
gitaly_socket = node['gitlab']['gitaly']['socket_path'] if node['gitlab']['gitaly']['enable']
templatesymlink "Create a gitlab.yml and create a symlink to Rails root" do
link_from File.join(gitlab_rails_source_dir, "config/gitlab.yml")
link_to File.join(gitlab_rails_etc_dir, "gitlab.yml")
@ -247,8 +245,7 @@ templatesymlink "Create a gitlab.yml and create a symlink to Rails root" do
pages_external_http: node['gitlab']['gitlab-pages']['external_http'],
pages_external_https: node['gitlab']['gitlab-pages']['external_https'],
mattermost_host: mattermost_host,
mattermost_enabled: node['gitlab']['mattermost']['enable'] || !mattermost_host.nil?,
gitaly_socket: gitaly_socket
mattermost_enabled: node['gitlab']['mattermost']['enable'] || !mattermost_host.nil?
)
)
restarts dependent_services

15
files/gitlab-cookbooks/gitlab/templates/default/gitlab.yml.erb
View File

@ -349,18 +349,13 @@ production: &base
shared:
path: <%= @shared_path %>
<% if @gitaly_socket %>
# Gitaly settings
# This setting controls whether GitLab uses Gitaly
# Eventually Gitaly use will become mandatory and
# this option will disappear.
gitaly:
socket_path: <%= @gitaly_socket %>
# The socket_path setting is optional and obsolete. When this is set
# GitLab assumes it can reach a Gitaly services via a Unix socket at
# this path. When this is commented out GitLab will not use Gitaly.
#
# This setting is obsolete because we expect it to be moved under
# repositories/storages in GitLab 9.1.
#
<% end %>
enabled: <%= @gitaly_enabled %>
#
# 4. Advanced settings

26
spec/chef/gitlab_rails_spec.rb
View File

@ -1,26 +0,0 @@
require 'chef_helper'
require 'base64'
describe 'gitlab_rails' do
let(:chef_run) { ChefSpec::SoloRunner.new.converge('gitlab::default') }
context 'when there is a legacy GitLab Rails stuck_ci_builds_worker_cron key' do
before do
allow(Gitlab).to receive(:[]).and_call_original
stub_gitlab_rb(gitlab_rails: { stuck_ci_builds_worker_cron: '0 1 2 * *' })
end
it 'warns that this value is deprecated' do
allow(Chef::Log).to receive(:warn).and_call_original
expect(Chef::Log).to receive(:warn).with(/gitlab_rails\['stuck_ci_builds_worker_cron'\]/)
chef_run
end
it 'copies legacy value from legacy key to new one' do
chef_run
expect(Gitlab['gitlab_rails']['stuck_ci_jobs_worker_cron']).to eq('0 1 2 * *')
end
end
end

65
spec/chef/recipes/gitlab-rails_spec.rb
View File

@ -201,28 +201,67 @@ describe 'gitlab::gitlab-rails' do
context 'Gitaly settings' do
context 'by default' do
it 'sets the path to socket' do
it 'is enabled' do
expect(chef_run).to render_file(gitlab_yml_path)
.with_content(%r{gitaly:\s+socket_path:\s+/var/opt/gitlab/gitaly/gitaly.socket})
end
context 'when socket path is changed' do
it 'sets the path to socket' do
stub_gitlab_rb(gitaly: { socket_path: '/tmp/socket' })
expect(chef_run).to render_file(gitlab_yml_path)
.with_content(%r{gitaly:\s+socket_path:\s+/tmp/socket})
end
.with_content(%r{gitaly:\s+enabled: true})
end
end
context 'when gitaly is disabled' do
it 'sets the mattermost host' do
it 'disables gitaly in config' do
stub_gitlab_rb(gitaly: { enable: false })
expect(chef_run).not_to render_file(gitlab_yml_path)
.with_content(%r{gitaly:\s+socket_path:\s+/var/opt/gitlab/gitaly/gitaly.socket})
expect(chef_run).to render_file(gitlab_yml_path)
.with_content(%r{gitaly:\s+enabled: false})
end
end
context 'when gitaly service is connecting from a different node' do
before do
stub_gitlab_rb(
gitaly: { enable: false },
gitlab_rails: { gitaly_enabled: true }
)
end
it 'enables gitaly in config' do
expect(chef_run).to render_file(gitlab_yml_path)
.with_content(%r{gitaly:\s+enabled: true})
end
end
context 'when gitaly service is running on a different node' do
before do
stub_gitlab_rb(
gitlab_rails: { gitaly_enabled: false }
)
it 'disables gitaly in config' do
expect(chef_run).to render_file(gitlab_yml_path)
.with_content(%r{gitaly:\s+enabled: false})
end
end
end
end
context 'when there is a legacy GitLab Rails stuck_ci_builds_worker_cron key' do
before do
allow(Gitlab).to receive(:[]).and_call_original
stub_gitlab_rb(gitlab_rails: { stuck_ci_builds_worker_cron: '0 1 2 * *' })
end
it 'warns that this value is deprecated' do
allow(Chef::Log).to receive(:warn).and_call_original
expect(Chef::Log).to receive(:warn).with(/gitlab_rails\['stuck_ci_builds_worker_cron'\]/)
chef_run
end
it 'copies legacy value from legacy key to new one' do
chef_run
expect(Gitlab['gitlab_rails']['stuck_ci_jobs_worker_cron']).to eq('0 1 2 * *')
end
end
end

10
spec/chef/recipes/gitlab-shell_spec.rb
View File

@ -236,7 +236,7 @@ describe 'gitlab_shell::git_data_dir' do
it 'correctly sets the repository storage directories' do
expect(chef_run.node['gitlab']['gitlab-rails']['repositories_storages'])
.to eql('default' => { 'path' => '/tmp/user/git-data/repositories' })
.to eql('default' => { 'path' => '/tmp/user/git-data/repositories', 'gitaly_address' => 'unix:/var/opt/gitlab/gitaly/gitaly.socket' })
end
end
@ -259,8 +259,8 @@ describe 'gitlab_shell::git_data_dir' do
it 'correctly sets the repository storage directories' do
expect(chef_run.node['gitlab']['gitlab-rails']['repositories_storages']).to eql({
'default' => { 'path' => '/tmp/default/git-data/repositories' },
'overflow' => { 'path' => '/tmp/other/git-overflow-data/repositories' }
'default' => { 'path' => '/tmp/default/git-data/repositories', 'gitaly_address' => 'unix:/var/opt/gitlab/gitaly/gitaly.socket' },
'overflow' => { 'path' => '/tmp/other/git-overflow-data/repositories', 'gitaly_address' => 'unix:/var/opt/gitlab/gitaly/gitaly.socket' }
})
end
end
@ -287,8 +287,8 @@ describe 'gitlab_shell::git_data_dir' do
it 'correctly sets the repository storage directories' do
expect(chef_run.node['gitlab']['gitlab-rails']['repositories_storages']).to eql({
'default' => { 'path' => '/tmp/default/git-data/repositories' },
'overflow' => { 'path' => '/tmp/other/git-overflow-data/repositories' }
'default' => { 'path' => '/tmp/default/git-data/repositories', 'gitaly_address' => 'unix:/var/opt/gitlab/gitaly/gitaly.socket' },
'overflow' => { 'path' => '/tmp/other/git-overflow-data/repositories', 'gitaly_address' => 'unix:/var/opt/gitlab/gitaly/gitaly.socket' }
})
end
end