gitlab
/
omnibus-gitlab
mirror of https://gitlab.com/gitlab-org/omnibus-gitlab.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Move git_data_dirs related operation to gitaly recipe and library

This commit is contained in:
Balasankar "Balu" C 2018-01-16 14:01:39 +05:30
parent b69c23b1d0
commit 2003bc5d5f
8 changed files with 204 additions and 206 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

80
files/gitlab-cookbooks/gitaly/libraries/gitaly.rb
View File

@ -17,6 +17,12 @@
module Gitaly
class << self
def parse_variables
parse_git_data_dirs
parse_gitaly_storages
detect_deprecated_settings
end
def gitaly_address
socket_path = user_config['socket_path'] || package_default['socket_path']
listen_addr = user_config['listen_addr'] || package_default['listen_addr']
@ -29,6 +35,80 @@ module Gitaly
end
end
def detect_deprecated_settings
git_data_dirs = Gitlab['git_data_dirs']
deprecated_key_used = 'git_data_dir' if Gitlab['git_data_dir']
if git_data_dirs.any?
git_data_dirs.map do |name, data_directory|
if data_directory.is_a?(String)
deprecated_key_used = 'git_data_dirs'
break
end
end
end
if deprecated_key_used # rubocop:disable Style/GuardClause
warn_message = <<~EOS
Your #{deprecated_key_used} settings are deprecated.
Please update it to the following:
git_data_dirs(#{Chef::JSONCompat.to_json_pretty(converted_git_data_dirs)})
Please refer to https://docs.gitlab.com/omnibus/settings/configuration.html#storing-git-data-in-an-alternative-directory for updated documentation.
EOS
LoggingHelper.deprecation warn_message
end
end
def converted_git_data_dirs
# Converting values in old formats to the correct, new one.
# Before version 8.10 we used git_data_dir configuration, which had a string representing a path as value.
# From version 8.10 till version 8.17.8, we used git_data_dirs configuration, which had a { <name> => <path> } hash as value.
# Now, since version 9.0, git_data_dirs has a { <name> => {"path" => <path> } } hash as value.
# So we convert all the old formats to the new one, until we remove the support of them.
git_data_dirs = Gitlab['git_data_dirs']
git_data_dir = Gitlab['git_data_dir']
return { "default" => { "path" => "/var/opt/gitlab/git-data" } } unless git_data_dirs.any? || git_data_dir
if git_data_dirs.any?
Hash[git_data_dirs.map do |name, data_directory|
if data_directory.is_a?(String)
[name, { 'path' => data_directory }]
else
[name, data_directory]
end
end]
else
{ 'default' => { 'path' => git_data_dir } }
end
end
def parse_git_data_dirs
Gitlab['gitlab_rails']['repositories_storages'] =
Hash[converted_git_data_dirs.map do |name, data_directory|
shard_gitaly_address = data_directory['gitaly_address'] || gitaly_address
defaults = { 'path' => File.join(data_directory['path'], 'repositories'), 'gitaly_address' => shard_gitaly_address }
params = data_directory.merge(defaults)
[name, params]
end]
end
def parse_gitaly_storages
return unless Gitlab['gitaly']['storage'].nil?
storages = []
Gitlab['gitlab_rails']['repositories_storages'].each do |key, value|
storages << {
'name' => key,
'path' => value['path']
}
end
Gitlab['gitaly']['storage'] = storages
end
private
def user_config

20
files/gitlab-cookbooks/gitaly/recipes/enable.rb
View File

@ -15,12 +15,32 @@
# limitations under the License.
#
account_helper = AccountHelper.new(node)
git_user = account_helper.gitlab_user
working_dir = node['gitaly']['dir']
log_directory = node['gitaly']['log_directory']
env_directory = node['gitaly']['env_directory']
config_path = File.join(working_dir, "config.toml")
# Holds git-data, by default one shard at /var/opt/gitlab/git-data
# Can be changed by user using git_data_dirs option
Gitaly.converted_git_data_dirs.each do |_name, git_data_directory|
storage_directory git_data_directory['path'] do
owner git_user
mode "0700"
end
end
# Holds git repositories, by default at /var/opt/gitlab/git-data/repositories
# Should not be changed by user. Different permissions to git_data_dir set.
repositories_storages = node['gitlab']['gitlab-rails']['repositories_storages']
repositories_storages.each do |_name, repositories_storage|
storage_directory repositories_storage['path'] do
owner git_user
mode "2770"
end
end
directory working_dir do
owner account_helper.gitlab_user
mode '0700'

20
files/gitlab-cookbooks/gitlab/libraries/gitlab_rails.rb
View File

@ -26,7 +26,6 @@ module GitlabRails
parse_directories
parse_gitlab_trusted_proxies
parse_rack_attack_protected_paths
parse_gitaly_variables
end
def parse_directories
@ -197,24 +196,5 @@ module GitlabRails
"#{Gitlab['node']['package']['install-dir']}/embedded/service/gitlab-rails/public"
end
def parse_gitaly_variables
parse_gitaly_storages
end
# This method cannot be inside of libraries/gitaly.rb for now
# because storage gets parsed in libraries/gitlab_shell.rb
# and libraries/gitlab_rails.rb
def parse_gitaly_storages
return unless Gitlab['gitaly']['storage'].nil?
storages = []
Gitlab['gitlab_rails']['repositories_storages'].each do |key, value|
storages << {
'name' => key,
'path' => value['path']
}
end
Gitlab['gitaly']['storage'] = storages
end
end
end unless defined?(GitlabRails) # Prevent reloading during converge, so we can test

45
files/gitlab-cookbooks/gitlab/libraries/gitlab_shell.rb
View File

@ -19,7 +19,6 @@ require_relative '../../gitaly/libraries/gitaly.rb'
module GitlabShell
class << self
def parse_variables
parse_git_data_dirs
parse_auth_file
end
@ -27,50 +26,6 @@ module GitlabShell
Gitlab['gitlab_shell']['secret_token'] ||= SecretsHelper.generate_hex(64)
end
def parse_git_data_dirs
git_data_dirs = Gitlab['git_data_dirs']
git_data_dir = Gitlab['git_data_dir']
return unless git_data_dirs.any? || git_data_dir
gitaly_address = Gitaly.gitaly_address
deprecated_key_used = 'git_data_dir' if git_data_dir
Gitlab['gitlab_shell']['git_data_directories'] ||=
if git_data_dirs.any?
Hash[git_data_dirs.map do |name, data_directory|
if data_directory.is_a?(String)
deprecated_key_used = 'git_data_dirs'
[name, { 'path' => data_directory }]
else
[name, data_directory]
end
end]
else
{ 'default' => { 'path' => git_data_dir } }
end
if deprecated_key_used
warn_message = <<~EOS
Your #{deprecated_key_used} settings are deprecated.
Please update it to the following:
git_data_dirs(#{Chef::JSONCompat.to_json_pretty(Gitlab['gitlab_shell']['git_data_directories'])})
Please refer to https://docs.gitlab.com/omnibus/settings/configuration.html#storing-git-data-in-an-alternative-directory for updated documentation.
EOS
LoggingHelper.deprecation warn_message
end
Gitlab['gitlab_rails']['repositories_storages'] ||=
Hash[Gitlab['gitlab_shell']['git_data_directories'].map do |name, data_directory|
shard_gitaly_address = data_directory['gitaly_address'] || gitaly_address
defaults = { 'path' => File.join(data_directory['path'], 'repositories'), 'gitaly_address' => shard_gitaly_address }
params = data_directory.merge(defaults)
[name, params]
end]
end
def parse_auth_file
Gitlab['user']['home'] ||= Gitlab['node']['gitlab']['user']['home']
Gitlab['gitlab_shell']['auth_file'] ||= File.join(Gitlab['user']['home'], '.ssh', 'authorized_keys')

21
files/gitlab-cookbooks/gitlab/recipes/gitlab-shell.rb
View File

@ -21,32 +21,11 @@ git_user = account_helper.gitlab_user
git_group = account_helper.gitlab_group
gitlab_shell_dir = "/opt/gitlab/embedded/service/gitlab-shell"
gitlab_shell_var_dir = node['gitlab']['gitlab-shell']['dir']
git_data_directories = node['gitlab']['gitlab-shell']['git_data_directories']
repositories_storages = node['gitlab']['gitlab-rails']['repositories_storages']
ssh_dir = File.join(node['gitlab']['user']['home'], ".ssh")
authorized_keys = node['gitlab']['gitlab-shell']['auth_file']
log_directory = node['gitlab']['gitlab-shell']['log_directory']
hooks_directory = node['gitlab']['gitlab-rails']['gitlab_shell_hooks_path']
gitlab_shell_keys_check = File.join(gitlab_shell_dir, 'bin/gitlab-keys')
# Holds git-data, by default one shard at /var/opt/gitlab/git-data
# Can be changed by user using git_data_dirs option
git_data_directories.each do |_name, git_data_directory|
storage_directory git_data_directory['path'] do
owner git_user
mode "0700"
end
end
# Holds git repositories, by default at /var/opt/gitlab/git-data/repositories
# Should not be changed by user. Different permissions to git_data_dir set.
repositories_storages.each do |_name, repositories_storage|
storage_directory repositories_storage['path'] do
owner git_user
mode "2770"
end
end
# Creates `.ssh` directory to hold authorized_keys
[
ssh_dir,

2
files/gitlab-cookbooks/package/libraries/config/gitlab.rb
View File

@ -43,7 +43,7 @@ module Gitlab
attribute('repmgr')
attribute('repmgrd')
attribute('consul')
attribute('gitaly')
attribute('gitaly').use { Gitaly }
attribute('mattermost', priority: 30).use { GitlabMattermost } # Mattermost checks if GitLab is enabled on the same box
## Attributes under node['gitlab']

97
spec/chef/recipes/gitaly_spec.rb
View File

@ -140,10 +140,10 @@ describe 'gitaly' do
stub_gitlab_rb(
{
git_data_dirs:
{
'default' => { 'path' => '/tmp/default/git-data' },
'nfs1' => { 'path' => '/mnt/nfs1' }
}
{
'default' => { 'path' => '/tmp/default/git-data' },
'nfs1' => { 'path' => '/mnt/nfs1' }
}
}
)
end
@ -255,3 +255,92 @@ describe 'gitaly' do
it_behaves_like "enabled gitaly env", "HOME", '/my/random/path'
end
end
describe 'gitaly::git_data_dirs' do
let(:chef_run) { ChefSpec::SoloRunner.new(step_into: %w(templatesymlink storage_directory)).converge('gitlab::default') }
before do
allow(Gitlab).to receive(:[]).and_call_original
end
context 'when user has not specified git_data_dir' do
it 'defaults to correct path' do
expect(chef_run.node['gitlab']['gitlab-rails']['repositories_storages'])
.to eql('default' => { 'path' => '/var/opt/gitlab/git-data/repositories', 'gitaly_address' => 'unix:/var/opt/gitlab/gitaly/gitaly.socket' })
end
end
context 'when git_data_dir is set as a single directory' do
before { stub_gitlab_rb(git_data_dir: '/tmp/user/git-data') }
it 'correctly sets the repository storage directories' do
allow(Chef::Log).to receive(:warn)
expect(Chef::Log).to receive(:warn).with(/Your git_data_dir settings are deprecated/)
expect(chef_run.node['gitlab']['gitlab-rails']['repositories_storages'])
.to eql('default' => { 'path' => '/tmp/user/git-data/repositories', 'gitaly_address' => 'unix:/var/opt/gitlab/gitaly/gitaly.socket' })
end
end
context 'when gitaly is set to use a listen_addr instead of a socket' do
before { stub_gitlab_rb(git_data_dirs: { 'default' => { 'path' => '/tmp/user/git-data' } }, gitaly: { socket_path: '', listen_addr: 'localhost:8123' }) }
it 'correctly sets the repository storage directories' do
expect(chef_run.node['gitlab']['gitlab-rails']['repositories_storages'])
.to eql('default' => { 'path' => '/tmp/user/git-data/repositories', 'gitaly_address' => 'tcp://localhost:8123' })
end
end
context 'when git_data_dirs is set to multiple directories' do
before do
stub_gitlab_rb({
git_data_dirs: {
'default' => { 'path' => '/tmp/default/git-data' },
'overflow' => { 'path' => '/tmp/other/git-overflow-data' }
}
})
end
it 'correctly sets the repository storage directories' do
expect(chef_run.node['gitlab']['gitlab-rails']['repositories_storages']).to eql({
'default' => { 'path' => '/tmp/default/git-data/repositories', 'gitaly_address' => 'unix:/var/opt/gitlab/gitaly/gitaly.socket' },
'overflow' => { 'path' => '/tmp/other/git-overflow-data/repositories', 'gitaly_address' => 'unix:/var/opt/gitlab/gitaly/gitaly.socket' }
})
end
end
context 'when git_data_dirs is set to multiple directories with different gitaly addresses' do
before do
stub_gitlab_rb({
git_data_dirs: {
'default' => { 'path' => '/tmp/default/git-data' },
'overflow' => { 'path' => '/tmp/other/git-overflow-data', 'gitaly_address' => 'tcp://localhost:8123', 'gitaly_token' => '123secret456gitaly' }
}
})
end
it 'correctly sets the repository storage directories' do
expect(chef_run.node['gitlab']['gitlab-rails']['repositories_storages']).to eql({
'default' => { 'path' => '/tmp/default/git-data/repositories', 'gitaly_address' => 'unix:/var/opt/gitlab/gitaly/gitaly.socket' },
'overflow' => { 'path' => '/tmp/other/git-overflow-data/repositories', 'gitaly_address' => 'tcp://localhost:8123', 'gitaly_token' => '123secret456gitaly' }
})
end
end
context 'when git_data_dirs is set with deprecated settings structure' do
before do
stub_gitlab_rb({
git_data_dirs: {
'default' => '/tmp/default/git-data',
'overflow' => '/tmp/other/git-overflow-data'
}
})
end
it 'correctly sets the repository storage directories' do
expect(chef_run.node['gitlab']['gitlab-rails']['repositories_storages']).to eql({
'default' => { 'path' => '/tmp/default/git-data/repositories', 'gitaly_address' => 'unix:/var/opt/gitlab/gitaly/gitaly.socket' },
'overflow' => { 'path' => '/tmp/other/git-overflow-data/repositories', 'gitaly_address' => 'unix:/var/opt/gitlab/gitaly/gitaly.socket' }
})
end
end
end

125
spec/chef/recipes/gitlab-shell_spec.rb
View File

@ -11,6 +11,16 @@ describe 'gitlab::gitlab-shell' do
expect(chef_run).to run_execute('/opt/gitlab/embedded/service/gitlab-shell/bin/gitlab-keys check-permissions')
end
it 'defaults the auth_file to be within the user\'s home directory' do
stub_gitlab_rb(user: { home: '/tmp/user' })
expect(chef_run.node['gitlab']['gitlab-shell']['auth_file']).to eq('/tmp/user/.ssh/authorized_keys')
end
it 'uses custom auth_files set in gitlab.rb' do
stub_gitlab_rb(user: { home: '/tmp/user' }, gitlab_shell: { auth_file: '/tmp/authorized_keys' })
expect(chef_run.node['gitlab']['gitlab-shell']['auth_file']).to eq('/tmp/authorized_keys')
end
context 'when NOT running on selinux' do
before { stub_command('id -Z').and_return(false) }
@ -191,118 +201,3 @@ describe 'gitlab::gitlab-shell' do
end
end
end
describe 'gitlab_shell::git_data_dirs' do
let(:chef_run) { ChefSpec::SoloRunner.new(step_into: %w(templatesymlink storage_directory)).converge('gitlab::default') }
before do
allow(Gitlab).to receive(:[]).and_call_original
end
context 'when git_data_dir is set as a single directory' do
before { stub_gitlab_rb(git_data_dir: '/tmp/user/git-data') }
it 'correctly sets the shell git data directories' do
# Allow warn to be called for other messages without failing the test
allow(Chef::Log).to receive(:warn)
expect(Chef::Log).to receive(:warn).with(/Your git_data_dir settings are deprecated/)
expect(chef_run.node['gitlab']['gitlab-shell']['git_data_directories'])
.to eql('default' => { 'path' => '/tmp/user/git-data' })
end
it 'correctly sets the repository storage directories' do
expect(chef_run.node['gitlab']['gitlab-rails']['repositories_storages'])
.to eql('default' => { 'path' => '/tmp/user/git-data/repositories', 'gitaly_address' => 'unix:/var/opt/gitlab/gitaly/gitaly.socket' })
end
end
context 'when gitaly is set to use a listen_addr instead of a socket' do
before { stub_gitlab_rb(git_data_dirs: { 'default' => { 'path' => '/tmp/user/git-data' } }, gitaly: { socket_path: '', listen_addr: 'localhost:8123' }) }
it 'correctly sets the repository storage directories' do
expect(chef_run.node['gitlab']['gitlab-rails']['repositories_storages'])
.to eql('default' => { 'path' => '/tmp/user/git-data/repositories', 'gitaly_address' => 'tcp://localhost:8123' })
end
end
context 'when git_data_dirs is set to multiple directories' do
before do
stub_gitlab_rb({
git_data_dirs: {
'default' => { 'path' => '/tmp/default/git-data' },
'overflow' => { 'path' => '/tmp/other/git-overflow-data' }
}
})
end
it 'correctly sets the shell git data directories' do
expect(chef_run.node['gitlab']['gitlab-shell']['git_data_directories']).to eql({
'default' => { 'path' => '/tmp/default/git-data' },
'overflow' => { 'path' => '/tmp/other/git-overflow-data' }
})
end
it 'correctly sets the repository storage directories' do
expect(chef_run.node['gitlab']['gitlab-rails']['repositories_storages']).to eql({
'default' => { 'path' => '/tmp/default/git-data/repositories', 'gitaly_address' => 'unix:/var/opt/gitlab/gitaly/gitaly.socket' },
'overflow' => { 'path' => '/tmp/other/git-overflow-data/repositories', 'gitaly_address' => 'unix:/var/opt/gitlab/gitaly/gitaly.socket' }
})
end
end
context 'when git_data_dirs is set to multiple directories with different gitaly addresses' do
before do
stub_gitlab_rb({
git_data_dirs: {
'default' => { 'path' => '/tmp/default/git-data' },
'overflow' => { 'path' => '/tmp/other/git-overflow-data', 'gitaly_address' => 'tcp://localhost:8123', 'gitaly_token' => '123secret456gitaly' }
}
})
end
it 'correctly sets the repository storage directories' do
expect(chef_run.node['gitlab']['gitlab-rails']['repositories_storages']).to eql({
'default' => { 'path' => '/tmp/default/git-data/repositories', 'gitaly_address' => 'unix:/var/opt/gitlab/gitaly/gitaly.socket' },
'overflow' => { 'path' => '/tmp/other/git-overflow-data/repositories', 'gitaly_address' => 'tcp://localhost:8123', 'gitaly_token' => '123secret456gitaly' }
})
end
end
context 'when git_data_dirs is set with deprecated settings structure' do
before do
stub_gitlab_rb({
git_data_dirs: {
'default' => '/tmp/default/git-data',
'overflow' => '/tmp/other/git-overflow-data'
}
})
end
it 'correctly sets the shell git data directories' do
# Allow warn to be called for other messages without failing the test
allow(Chef::Log).to receive(:warn)
expect(Chef::Log).to receive(:warn).with(/Your git_data_dirs settings are deprecated/)
expect(chef_run.node['gitlab']['gitlab-shell']['git_data_directories']).to eql({
'default' => { 'path' => '/tmp/default/git-data' },
'overflow' => { 'path' => '/tmp/other/git-overflow-data' }
})
end
it 'correctly sets the repository storage directories' do
expect(chef_run.node['gitlab']['gitlab-rails']['repositories_storages']).to eql({
'default' => { 'path' => '/tmp/default/git-data/repositories', 'gitaly_address' => 'unix:/var/opt/gitlab/gitaly/gitaly.socket' },
'overflow' => { 'path' => '/tmp/other/git-overflow-data/repositories', 'gitaly_address' => 'unix:/var/opt/gitlab/gitaly/gitaly.socket' }
})
end
end
it 'defaults the auth_file to be within the user\'s home directory' do
stub_gitlab_rb(user: { home: '/tmp/user' })
expect(chef_run.node['gitlab']['gitlab-shell']['auth_file']).to eq('/tmp/user/.ssh/authorized_keys')
end
it 'uses custom auth_files set in gitlab.rb' do
stub_gitlab_rb(user: { home: '/tmp/user' }, gitlab_shell: { auth_file: '/tmp/authorized_keys' })
expect(chef_run.node['gitlab']['gitlab-shell']['auth_file']).to eq('/tmp/authorized_keys')
end
end