Merge branch 'active_roles'

doc/README.md

@@ -42,6 +42,7 @@ Omnibus is a way to package different services and tools required to run GitLab,
 * [GitLab Mattermost](gitlab-mattermost/README.md) Set up the Mattermost messaging app that ships with Omnibus GitLab package.
 * [GitLab Prometheus](https://docs.gitlab.com/ce/administration/monitoring/performance/prometheus.html) Set up the Prometheus
   monitoring included in the Omnibus GitLab package.
+* [GitLab High Availability Roles](roles/README.md)
 
 #### Using docker image
 

doc/roles/README.md

@@ -0,0 +1,76 @@
+# Omnibus GitLab High Availability Roles
+
+>**Notes:**
+>- Introduced in GitLab EE 10.1.0
+>- The majority of these roles will only work on an [Enterprise Edition](https://about.gitlab.com/products/) installation of GitLab.
+
+Omnibus GitLab includes various software components/services to support running GitLab in
+a high availability configuration. By default, some of these supporting services
+are disabled, and Omnibus GitLab is configured to run as single node installation.
+Each service can be enabled or disabled using configuration settings in `/etc/gitlab/gitlab.rb`,
+but the introduction of `roles` allows you to easily enable a group of services,
+and provides better default configuration based on the high availability roles you
+have enabled.
+
+## Not specifying any Roles (the default configuration)
+
+When you don't configure GitLab with any roles, GitLab enables the default services for
+a single node install. These include things like PostgreSQL, Redis, Unicorn, Sidekiq,
+Gitaly, GitLab Workhorse, Nginx, etc.
+
+These can still be individually enable/disabled by the settings in your `/etc/gitlab/gitlab.rb`.
+
+## Specifying Roles
+
+Roles are passed as an array in `/etc/gitlab/gitlab.rb`
+
+Example specifying multiple roles:
+
+```ruby
+roles ['redis_sentinel_role', 'redis_master_role']
+```
+
+Example specifying a single role:
+```ruby
+roles ['geo_primary_role']
+```
+
+## Roles
+
+### Redis Server Roles
+
+Documentation on the use of the Redis Roles can be found in [Configuring Redis HA](https://docs.gitlab.com/ee/administration/high_availability/redis.html#configuring-redis-ha)
+
+- **redis_sentinel_role**
+
+  Enables the sentinel service on the machine,
+
+  *By default, enables no other services.*
+- **redis_master_role**
+
+  Enables the redis service and monitoring, and allows configuring the master password
+
+  *By default, enables no other services.*
+- **redis_slave_role**
+
+  Enables the redis service and monitoring
+
+  *By default, enables no other services.*
+
+### GitLab Geo Roles
+
+The GitLab Geo roles are used when setting up the database replication for GitLab
+Geo. See the [Geo Database Documentation](https://docs.gitlab.com/ee/gitlab-geo/database.html)
+for configuration steps.
+
+- **geo_primary_role**
+
+  Prepares the database for replication and configures the application as a Geo Primary.
+
+  *By default, enables all of GitLab's standard single node services. (Nginx, Unicorn, Redis, Sidekiq, etc)*
+- **geo_secondary_role**
+
+  Configures the secondary database for incoming replication and flags the
+  application as a Geo Secondary
+
+  *By default, enables all of GitLab's default single node services. (Nginx, Unicorn, Redis, Sidekiq, etc)*

files/gitlab-config-template/gitlab.rb.template

@@ -12,6 +12,15 @@
 ##! https://docs.gitlab.com/omnibus/settings/configuration.html#configuring-the-external-url-for-gitlab
 external_url 'GENERATED_EXTERNAL_URL'
 
+## Roles for multi-instance GitLab
+##! The default is to have no roles enabled, which results in GitLab running as an all-in-one instance.
+##! Options:
+##!   redis_sentinel_role redis_master_role redis_slave_role geo_primary_role geo_secondary_role
+##! For more deatils on each role, see:
+##! https://docs.gitlab.com/omnibus/roles/README.html#roles
+##!
+# roles ['redis_sentinel_role', 'redis_master_role']
+
 ## Legend
 ##! The following notations at the beginning of each line may be used to
 ##! differentiate between components of this file and to easily select them using

files/gitlab-cookbooks/package/libraries/config/gitlab.rb

@@ -23,6 +23,7 @@ module Gitlab
 
   ## Attributes that don't get passed to the node
   node nil
+  roles nil
   edition :ce
   git_data_dirs ConfigMash.new
 

files/gitlab-cookbooks/package/libraries/config/roles/default.rb

@@ -27,7 +27,7 @@ module DefaultRole # rubocop:disable Style/MultilineIfModifier (disabled so we c
     end
 
     def no_service_roles_enabled?
-      Gitlab.roles.select { |key, role| role[:manage_services] && Gitlab["#{key}_role"]['enable'] }.count.zero?
+      Gitlab.available_roles.select { |key, role| role[:manage_services] && Gitlab["#{key}_role"]['enable'] }.count.zero?
     end
   end
 end unless defined?(DefaultRole) # Prevent reloading during converge, so we can test

files/gitlab-cookbooks/package/libraries/config/services.rb

@@ -26,9 +26,9 @@ module Services
     service 'sidekiq',            groups: [DEFAULT_GROUP, 'rails', 'sidekiq']
     service 'gitlab_monitor',     groups: [DEFAULT_GROUP, 'rails', 'prometheus']
     service 'gitlab_workhorse',   groups: [DEFAULT_GROUP, 'rails']
+    service 'gitaly',             groups: [DEFAULT_GROUP, 'rails']
     service 'redis',              groups: [DEFAULT_GROUP, 'redis', 'redis_node']
     service 'redis_exporter',     groups: [DEFAULT_GROUP, 'redis', 'redis_node', 'prometheus']
-    service 'gitaly',             groups: [DEFAULT_GROUP]
     service 'postgresql',         groups: [DEFAULT_GROUP, 'postgres']
     service 'nginx',              groups: [DEFAULT_GROUP]
     service 'prometheus',         groups: [DEFAULT_GROUP, 'prometheus']

files/gitlab-cookbooks/package/libraries/helpers/roles_helper.rb

@@ -0,0 +1,41 @@
+#
+# Copyright:: Copyright (c) 2017 GitLab Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+module RolesHelper
+  class << self
+    def parse_enabled
+      return unless Gitlab['roles']
+
+      # convert hyphens to underscores to avoid user errors
+      # split or space or comma (allow both to avoid user errors)
+      active        = [Gitlab['roles']].flatten.map { |role| role.tr('-', '_') }
+      valid_roles   = Gitlab.available_roles.keys.map { |key| "#{key}_role" }
+      invalid_roles = active - valid_roles
+
+      # Ensure all active roles exist as valid role names
+      unless invalid_roles.empty?
+        raise "The following invalid roles have been set in 'roles': #{invalid_roles.join(', ')}"
+      end
+
+      active.each { |role_name| Gitlab[role_name]['enable'] = true }
+    end
+
+    def disable_all
+      Gitlab.available_roles.each { |name, _value| Gitlab["#{name}_role"]['enable'] = false }
+    end
+  end
+end

files/gitlab-cookbooks/package/libraries/helpers/settings_helper.rb

@@ -27,11 +27,11 @@ module SettingsHelper
   def self.extended(base)
     # Setup getter/setters for roles and settings
     class << base
-      attr_accessor :roles
+      attr_accessor :available_roles
       attr_accessor :settings
     end
 
-    base.roles = {}
+    base.available_roles = {}
     base.settings = {}
   end
 
@@ -62,11 +62,11 @@ module SettingsHelper
   #     will result in Gitlab['some_specific_role']['enable'] = true
   #     and node['roles']['some-specific']['enable'] = true
   def role(name, **config)
-    @roles[name] = HandledHash.new.merge!(
+    @available_roles[name] = HandledHash.new.merge!(
       { manage_services: true }
     ).merge(config)
     send("#{name}_role", Gitlab::ConfigMash.new)
-    @roles[name]
+    @available_roles[name]
   end
 
   # Create a new attribute with the given 'name' and config
@@ -129,7 +129,7 @@ module SettingsHelper
     end
 
     # Add the roles the the results
-    @roles.each do |key, value|
+    @available_roles.each do |key, value|
       rkey = key.tr('_', '-')
       results['roles'][rkey] = Gitlab["#{key}_role"]
     end
@@ -140,10 +140,11 @@ module SettingsHelper
   def load_roles
     # System services are enabled by default
     Services.enable_group(Services::SYSTEM_GROUP)
+    RolesHelper.parse_enabled
 
     # Load our roles
     DefaultRole.load_role
-    @roles.each do |key, value|
+    @available_roles.each do |key, value|
       handler = value.handler
       handler.load_role if handler && handler.respond_to?(:load_role)
     end

spec/libraries/roles_spec.rb

@@ -6,6 +6,43 @@ describe 'GitLabRoles' do
     allow(Services).to receive(:enable_group).and_call_original
   end
 
+  after do
+    RolesHelper.disable_all
+  end
+
+  describe 'roles config array' do
+    it 'enables roles listed in the roles array' do
+      stub_gitlab_rb(roles: %w(application_role geo_primary_role))
+
+      Gitlab.load_roles
+
+      expect(Gitlab['application_role']['enable']).to be true
+      expect(Gitlab['geo_primary_role']['enable']).to be true
+    end
+
+    it 'supports providing a single role as a string' do
+      stub_gitlab_rb(roles: 'geo_secondary_role')
+
+      Gitlab.load_roles
+
+      expect(Gitlab['geo_secondary_role']['enable']).to be true
+    end
+
+    it 'handles users specifying hyphens instead of underscores' do
+      stub_gitlab_rb(roles: ['geo-primary-role'])
+
+      Gitlab.load_roles
+
+      expect(Gitlab['geo_primary_role']['enable']).to be true
+    end
+
+    it 'throws errors when an invalid role is used' do
+      stub_gitlab_rb(roles: ['some_invalid_role'])
+
+      expect { Gitlab.load_roles }.to raise_error(RuntimeError, /invalid roles have been set/)
+    end
+  end
+
   describe 'DefaultRole' do
     before do
       allow(DefaultRole).to receive(:load_role).and_call_original