Merge branch 'master' into 'master'

feat: account for potential consul port override to determine patroni leader

See merge request https://gitlab.com/gitlab-org/omnibus-gitlab/-/merge_requests/7576

Merged-by: Yu Shao Pang (SQPC) <yushao.pang@squarepoint-capital.com>

doc/settings/memory_constrained_envs.md

@@ -145,14 +145,14 @@ We observed 100-400MB of memory usage reduction configuring Puma this way.
 ## Optimize Sidekiq
 
 Sidekiq is a background processing daemon. When configured with GitLab by default
-it runs with a high concurrency mode of `50`. This does impact how much memory it can
+it runs with a concurrency mode of `20`. This does impact how much memory it can
 allocate at a given time. It is advised to configure it to use a significantly
 smaller value of `5` or `10` (preferred).
 
 In `/etc/gitlab/gitlab.rb`:
 
 ```ruby
-sidekiq['max_concurrency'] = 10
+sidekiq['concurrency'] = 10
 ```
 
 ## Optimize Gitaly
@@ -249,7 +249,7 @@ and disable the Prometheus Metrics feature:
    ```ruby
    puma['worker_processes'] = 0
 
-   sidekiq['max_concurrency'] = 10
+   sidekiq['concurrency'] = 10
 
    prometheus_monitoring['enable'] = false
 

doc/settings/rpi.md

@@ -55,7 +55,7 @@ some running processes:
    ```ruby
    # Reduce the number of running workers to the minimum in order to reduce memory usage
    puma['worker_processes'] = 2
-   sidekiq['max_concurrency'] = 9
+   sidekiq['concurrency'] = 9
    # Turn off monitoring to reduce idle cpu and disk usage
    prometheus_monitoring['enable'] = false
    ```

files/gitlab-config-template/gitlab.rb.template

@@ -1280,9 +1280,7 @@ external_url 'GENERATED_EXTERNAL_URL'
 # sidekiq['log_format'] = "json"
 # sidekiq['shutdown_timeout'] = 4
 # sidekiq['interval'] = nil
-# sidekiq['concurrency'] = nil
-# sidekiq['max_concurrency'] = 20
-# sidekiq['min_concurrency'] = nil
+# sidekiq['concurrency'] = 20
 
 ##! GitLab allows route a job to a particular queue determined by an array of ##! routing rules.
 ##! Each routing rule is a tuple of queue selector query and corresponding queue. By default,

files/gitlab-cookbooks/gitlab/attributes/default.rb

@@ -728,9 +728,7 @@ default['gitlab']['sidekiq']['health_checks_listen_port'] = 8092
 
 # Cluster specific settings
 default['gitlab']['sidekiq']['interval'] = nil
-default['gitlab']['sidekiq']['concurrency'] = nil
-default['gitlab']['sidekiq']['max_concurrency'] = 20
-default['gitlab']['sidekiq']['min_concurrency'] = nil
+default['gitlab']['sidekiq']['concurrency'] = 20
 default['gitlab']['sidekiq']['queue_groups'] = ['*']
 default['gitlab']['sidekiq']['consul_service_name'] = 'sidekiq'
 default['gitlab']['sidekiq']['consul_service_meta'] = nil

files/gitlab-cookbooks/gitlab/templates/default/sv-sidekiq-run.erb

@@ -22,12 +22,6 @@ exec chpst -e /opt/gitlab/etc/gitlab-rails/env -P \
       <% if node['gitlab']['sidekiq']['concurrency'] %>
       -c <%= node['gitlab']['sidekiq']['concurrency'] %> \
       <% end %>
-      <% if node['gitlab']['sidekiq']['max_concurrency'] %>
-      -m <%= node['gitlab']['sidekiq']['max_concurrency'] %> \
-      <% end %>
-      <% if node['gitlab']['sidekiq']['min_concurrency'] %>
-      --min-concurrency <%= node['gitlab']['sidekiq']['min_concurrency'] %> \
-      <% end %>
       <% if node['gitlab']['sidekiq']['shutdown_timeout'] %>
       --timeout <%= node['gitlab']['sidekiq']['shutdown_timeout'] %> \
       <% end %>

files/gitlab-cookbooks/package/libraries/deprecations.rb

@@ -346,14 +346,14 @@ module Gitlab
           },
           {
             config_keys: %w(gitlab sidekiq min_concurrency),
-            deprecation: '16.9',
-            removal: '17.0',
+            deprecation: '16.9', # Remove message issue: https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/8491
+            removal: '17.0', # Removal issue: https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/3422
             note: "Starting with GitLab 17.0, `sidekiq['min_concurrency']` will be removed. Please follow https://docs.gitlab.com/ee/administration/sidekiq/extra_sidekiq_processes.html#manage-thread-counts-explicitly to use `sidekiq['concurrency']` instead."
           },
           {
             config_keys: %w(gitlab sidekiq max_concurrency),
-            deprecation: '16.9',
-            removal: '17.0',
+            deprecation: '16.9', # Remove message issue: https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/8491
+            removal: '17.0', # Removal issue: https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/3422
             note: "Starting with GitLab 17.0, `sidekiq['max_concurrency']` will be removed. Please follow https://docs.gitlab.com/ee/administration/sidekiq/extra_sidekiq_processes.html#manage-thread-counts-explicitly to use `sidekiq['concurrency']` instead."
           },
           {

files/gitlab-ctl-commands-ee/lib/postgresql/ee.rb

@@ -7,6 +7,7 @@ module GitlabCtl
         def get_primary
           node_attributes = GitlabCtl::Util.get_node_attributes
           consul_enable = node_attributes.dig('consul', 'enable')
+          consul_dns_port = node_attributes.dig('consul', 'configuration', 'ports', 'dns') || 8600
           postgresql_service_name = node_attributes.dig('patroni', 'scope')
 
           raise 'Consul agent is not enabled on this node' unless consul_enable
@@ -14,7 +15,7 @@ module GitlabCtl
           raise 'PostgreSQL service name is not defined' if postgresql_service_name.nil? || postgresql_service_name.empty?
 
           result = []
-          Resolv::DNS.open(nameserver_port: [['127.0.0.1', 8600]]) do |dns|
+          Resolv::DNS.open(nameserver_port: [['127.0.0.1', consul_dns_port]]) do |dns|
             ['master', 'standby-leader'].each do |postgresql_primary_service_name|
               result = dns.getresources("#{postgresql_primary_service_name}.#{postgresql_service_name}.service.consul", Resolv::DNS::Resource::IN::SRV).map do |srv|
                 "#{dns.getaddress(srv.target)}:#{srv.port}"

spec/chef/cookbooks/gitlab/recipes/sidekiq_spec.rb

@@ -26,7 +26,7 @@ RSpec.describe 'gitlab::sidekiq' do
           expect(content).to match(/rubyopt=\"-W:no-experimental\"/)
           expect(content).to include(%(RUBYOPT="${rubyopt}"))
           expect(content).to match(%r{bin/sidekiq-cluster})
-          expect(content).to match(/-m 20/) # max_concurrency
+          expect(content).to match(/-c 20/) # concurrency
           expect(content).to match(/--timeout 25/) # shutdown timeout
           expect(content).to match(/\*/) # all queues
         }
@@ -64,11 +64,19 @@ RSpec.describe 'gitlab::sidekiq' do
       before do
         stub_gitlab_rb(
           sidekiq: {
-            log_group: 'fugee'
+            log_group: 'fugee',
+            concurrency: 42
           }
         )
       end
       it_behaves_like 'enabled logged service', 'sidekiq', true, { log_directory_owner: 'git', log_group: 'fugee' }
+
+      it 'correctly renders out the sidekiq service file' do
+        expect(chef_run).to render_file("/opt/gitlab/sv/sidekiq/run")
+          .with_content { |content|
+            expect(content).to match(/-c 42/) # concurrency
+          }
+      end
     end
   end
 

spec/chef/gitlab-ctl-commands-ee/lib/postgresql/ee_spec.rb

@@ -37,6 +37,7 @@ RSpec.describe GitlabCtl::PostgreSQL::EE do
     end
 
     context 'when required settings are available' do
+      let(:dns_double) { instance_double(Resolv::DNS) }
       before do
         allow(GitlabCtl::Util).to receive(:get_node_attributes)
           .and_return(
@@ -49,17 +50,59 @@ RSpec.describe GitlabCtl::PostgreSQL::EE do
               }
             }
           )
-        allow_any_instance_of(Resolv::DNS).to receive(:getresources)
+        allow(Resolv::DNS).to receive(:open).and_return(dns_double)
+        allow(dns_double).to receive(:getresources)
           .with('master.fake.service.consul', Resolv::DNS::Resource::IN::SRV)
           .and_return([Struct.new(:target, :port).new('fake.address', 6432)])
-        allow_any_instance_of(Resolv::DNS).to receive(:getaddress)
+        allow(dns_double).to receive(:getaddress)
           .with('fake.address')
           .and_return('1.2.3.4')
       end
 
+      it 'initializes Resolv::DNS with port 8600' do
+        expect(Resolv::DNS).to receive(:open).with(nameserver_port: [['127.0.0.1', 8600]])
+      end
+
       it 'should get the list of PostgreSQL endpoints' do
         expect(described_class.get_primary).to eq ['1.2.3.4:6432']
       end
     end
   end
+
+  context 'when consul has a dns port override' do
+    let(:dns_double) { instance_double(Resolv::DNS) }
+    before do
+      allow(GitlabCtl::Util).to receive(:get_node_attributes)
+        .and_return(
+          {
+            'consul' => {
+              'enable' => true,
+              'configuration' => {
+                "ports" => {
+                  "dns" => 18600
+                }
+              }
+            },
+            'patroni' => {
+              'scope' => 'fake'
+            }
+          }
+        )
+      allow(Resolv::DNS).to receive(:open).and_return(dns_double)
+      allow(dns_double).to receive(:getresources)
+        .with('master.fake.service.consul', Resolv::DNS::Resource::IN::SRV)
+        .and_return([Struct.new(:target, :port).new('fake.address', 6432)])
+      allow(dns_double).to receive(:getaddress)
+        .with('fake.address')
+        .and_return('1.2.3.4')
+    end
+
+    it 'initializes Resolv::DNS with port 18600' do
+      expect(Resolv::DNS).to receive(:open).with(nameserver_port: [['127.0.0.1', 18600]])
+    end
+
+    it 'should get the list of PostgreSQL endpoints' do
+      expect(described_class.get_primary).to eq ['1.2.3.4:6432']
+    end
+  end
 end