Merge branch 'manage_storage_directories' into 'master'

Manage storage directories

Fixes #998 

Supersedes !678 !568 

See merge request !689

doc/README.md

@@ -24,6 +24,8 @@ Omnibus is a way to package different services and tools required to run GitLab,
 - [Changing the name of the git user group](settings/configuration.md#changing-the-name-of-the-git-user-group)
 - [Specify numeric user and group identifiers](settings/configuration.md#specify-numeric-user-and-group-identifiers)
 - [Only start omnibus-gitlab services after a given filesystem is mounted](settings/configuration.md#only-start-omnibus-gitlab-services-after-a-given-filesystem-is-mounted)
+- [Disable user and group account management](settings/configuration.html#disable-user-and-group-account-management)
+- [Disable storage directory management](settings/configuration.html#disable-storage-directories-management)
 - [SMTP](settings/smtp.md)
 - [NGINX](settings/nginx.md)
 - [LDAP](settings/ldap.md)

doc/settings/configuration.md

@@ -286,6 +286,44 @@ redis['home'] = "/var/opt/redis-gitlab"
 # And so on for users/groups for GitLab CI GitLab Mattermost
 ```
 
+## Disable storage directories management
+
+The omnibus-gitlab package takes care of creating all the necessary directories
+with the correct ownership and permissions, as well as keeping this updated.
+
+Some of these directories will hold large amount of data so in certain setups,
+these directories will most likely be mounted on a NFS (or some other) share.
+
+Some types of mounts won't allow automatic creation of directories by root user
+ (default user for initial setup), eg. NFS with `no_root_squash` enabled on the
+share.
+
+In order to disable management of these directories,
+in `/etc/gitlab/gitlab.rb` set:
+
+```ruby
+manage_storage_directories['enable'] = false
+```
+
+**Warning** The omnibus-gitlab package still expects these directories to exist
+on the filesystem. It is up to the administrator to create and set correct
+permissions if this setting is set.
+
+Enabling this setting will prevent the creation of the following directories:
+
+| Default location | Permissions | Ownership | Purpose |
+| ---------------- | ----------- | --------- | ------- |
+| `/var/opt/gitlab/git-data`   | 0700 | git:root | Holds repositories directory |
+| `/var/opt/gitlab/git-data/repositories` | 2770 | git:git | Holds git repositories |
+| `/var/opt/gitlab/gitlab-rails/shared` | 0751 | git:gitlab-www | Holds large object directories |
+| `/var/opt/gitlab/gitlab-rails/shared/artifacts` | 0700 | git:root | Holds CI artifacts |
+| `/var/opt/gitlab/gitlab-rails/shared/lfs` | 0700 | git:root | Holds LFS objects |
+| `/var/opt/gitlab/gitlab-rails/uploads` | 0700 | git:root | Holds user attachments |
+| `/var/opt/gitlab/gitlab-pages` | 0750 | git:gitlab-www | Holds user pages |
+| `/var/opt/gitlab/gitlab-ci/builds` | 0700 | git:root | Holds CI build logs |
+
+
+
 ## Only start Omnibus-GitLab services after a given filesystem is mounted
 
 If you want to prevent omnibus-gitlab services (NGINX, Redis, Unicorn etc.)

files/gitlab-config-template/gitlab.rb.template

@@ -547,11 +547,18 @@ external_url 'GENERATED_EXTERNAL_URL'
 #############################
 # Users and groups accounts #
 #############################
-## Should omnibus-gitlab package manage users and groups accounts.
-## Only set if creating accounts manually
-##
+## Disable management of users and groups accounts.
+## Set only if creating accounts manually
+## See: http://doc.gitlab.com/omnibus/settings/configuration.html#disable-user-and-group-account-management
+# manage_accounts['enable'] = false
 
-# manage_accounts['enable'] = true
+#######################
+# Storage directories #
+#######################
+## Disable managing storage directories
+## Set only if the select directories are created manually
+## See: http://doc.gitlab.com/omnibus/settings/configuration.html#disable-storage-directories-management
+# manage_storage_directories['enable'] = false
 
 #######
 # Git #

files/gitlab-cookbooks/gitlab/attributes/default.rb

@@ -27,6 +27,9 @@ default['gitlab']['omnibus-gitconfig']['system'] = {
 # Create users and groups needed for the package
 default['gitlab']['manage-accounts']['enable'] = true
 
+# Create directories with correct permissions and ownership required by the pkg
+default['gitlab']['manage-storage-directories']['enable'] = true
+
 ####
 # The Git User that services run as
 ####

files/gitlab-cookbooks/gitlab/libraries/gitlab.rb

@@ -37,6 +37,7 @@ module Gitlab
   bootstrap Mash.new
   omnibus_gitconfig Mash.new
   manage_accounts Mash.new
+  manage_storage_directories Mash.new
   user Mash.new
   postgresql Mash.new
   redis Mash.new
@@ -486,6 +487,7 @@ module Gitlab
         "bootstrap",
         "omnibus_gitconfig",
         "manage_accounts",
+        "manage_storage_directories",
         "user",
         "redis",
         "ci_redis",

files/gitlab-cookbooks/gitlab/recipes/gitlab-rails.rb

@@ -45,13 +45,35 @@ directory File.dirname(gitlab_rails_log_dir) do
   recursive true
 end
 
-# We create shared_path with 711 allowing other users to enter into the directories
-# It's needed, because by default the shared_path is used to store pages which are served by gitlab-www:gitlab-www
-directory node['gitlab']['gitlab-rails']['shared_path'] do
-  owner gitlab_user
-  group account_helper.web_server_group
-  mode '0751'
-  recursive true
+if node['gitlab']['manage-storage-directories']['enable']
+  # We create shared_path with 751 allowing other users to enter into the directories
+  # It's needed, because by default the shared_path is used to store pages which are served by gitlab-www:gitlab-www
+  directory node['gitlab']['gitlab-rails']['shared_path'] do
+    owner gitlab_user
+    group account_helper.web_server_group
+    mode '0751'
+    recursive true
+  end
+
+  [
+    node['gitlab']['gitlab-rails']['artifacts_path'],
+    node['gitlab']['gitlab-rails']['lfs_storage_path'],
+    gitlab_rails_public_uploads_dir,
+    gitlab_ci_builds_dir
+  ].compact.each do |dir_name|
+    directory dir_name do
+      owner gitlab_user
+      mode '0700'
+      recursive true
+    end
+  end
+
+  directory node['gitlab']['gitlab-rails']['pages_path'] do
+    owner gitlab_user
+    group account_helper.web_server_group
+    mode '0750'
+    recursive true
+  end
 end
 
 [
@@ -59,11 +81,7 @@ end
   gitlab_rails_static_etc_dir,
   gitlab_rails_working_dir,
   gitlab_rails_tmp_dir,
-  gitlab_ci_builds_dir,
-  gitlab_rails_public_uploads_dir,
   node['gitlab']['gitlab-rails']['gitlab_repository_downloads_path'],
-  node['gitlab']['gitlab-rails']['artifacts_path'],
-  node['gitlab']['gitlab-rails']['lfs_storage_path'],
   gitlab_rails_log_dir
 ].compact.each do |dir_name|
   directory dir_name do
@@ -92,13 +110,6 @@ directory gitlab_ci_dir do
   recursive true
 end
 
-directory node['gitlab']['gitlab-rails']['pages_path'] do
-  owner gitlab_user
-  group account_helper.web_server_group
-  mode '0750'
-  recursive true
-end
-
 template File.join(gitlab_rails_static_etc_dir, "gitlab-rails-rc")
 
 dependent_services = []

files/gitlab-cookbooks/gitlab/recipes/gitlab-shell.rb

@@ -22,17 +22,26 @@ git_group = account_helper.gitlab_group
 gitlab_shell_dir = "/opt/gitlab/embedded/service/gitlab-shell"
 gitlab_shell_var_dir = "/var/opt/gitlab/gitlab-shell"
 repositories_path = node['gitlab']['gitlab-rails']['gitlab_shell_repos_path']
+git_data_directory = node['gitlab']['gitlab-shell']['git_data_directory']
 ssh_dir = File.join(node['gitlab']['user']['home'], ".ssh")
 authorized_keys = File.join(ssh_dir, "authorized_keys")
 log_directory = node['gitlab']['gitlab-shell']['log_directory']
 hooks_directory = node['gitlab']['gitlab-rails']['gitlab_shell_hooks_path']
 
-# Create directories because the git_user does not own its home directory
-directory repositories_path do
-  owner git_user
-  group git_group
-  mode "2770"
-  recursive true
+if node['gitlab']['manage-storage-directories']['enable']
+  # Create directories because the git_user does not own its home directory
+  directory repositories_path do
+    owner git_user
+    group git_group
+    mode "2770"
+    recursive true
+  end
+
+  directory git_data_directory do
+    owner git_user
+    mode "0700"
+    recursive true
+  end
 end
 
 directory ssh_dir do
@@ -68,8 +77,7 @@ end
 
 [
   log_directory,
-  gitlab_shell_var_dir,
-  node['gitlab']['gitlab-shell']['git_data_directory']
+  gitlab_shell_var_dir
 ].each do |dir|
   directory dir do
     owner git_user