Merge branch '8341-remove-pg-13' into 'master'

Remove PostgreSQL 13

Closes #8341

See merge request https://gitlab.com/gitlab-org/omnibus-gitlab/-/merge_requests/7546

Merged-by: Clemens Beck <cbeck@gitlab.com>
Approved-by: Ryan Egesdahl <regesdahl@gitlab.com>
Reviewed-by: Balasankar 'Balu' C <balasankar@gitlab.com>
Reviewed-by: Ryan Egesdahl <regesdahl@gitlab.com>

config/patches/postgresql/no_docs.patch

@@ -5,18 +5,18 @@ index 2352fc1..0191427 100644
 @@ -13,7 +13,7 @@ $(call recurse,all install,src config)
  docs:
  	$(MAKE) -C doc all
-
+ 
 -$(call recurse,world,doc src config contrib,all)
 +$(call recurse,world,src config contrib,all)
-
+ 
  # build src/ before contrib/
  world-contrib-recurse: world-src-recurse
 @@ -29,7 +29,7 @@ html man:
  install-docs:
  	$(MAKE) -C doc install
-
+ 
 -$(call recurse,install-world,doc src config contrib,install)
 +$(call recurse,install-world,src config contrib,install)
-
+ 
  # build src/ before contrib/
  install-world-contrib-recurse: install-world-src-recurse

config/patches/postgresql_new/no_docs.patch

@@ -1,22 +0,0 @@
-diff --git a/GNUmakefile.in b/GNUmakefile.in
-index 2352fc1..0191427 100644
---- a/GNUmakefile.in
-+++ b/GNUmakefile.in
-@@ -13,7 +13,7 @@ $(call recurse,all install,src config)
- docs:
- 	$(MAKE) -C doc all
- 
--$(call recurse,world,doc src config contrib,all)
-+$(call recurse,world,src config contrib,all)
- 
- # build src/ before contrib/
- world-contrib-recurse: world-src-recurse
-@@ -29,7 +29,7 @@ html man:
- install-docs:
- 	$(MAKE) -C doc install
- 
--$(call recurse,install-world,doc src config contrib,install)
-+$(call recurse,install-world,src config contrib,install)
- 
- # build src/ before contrib/
- install-world-contrib-recurse: install-world-src-recurse

config/patches/postgresql_old/no_docs.patch

@@ -1,22 +0,0 @@
-diff --git a/GNUmakefile.in b/GNUmakefile.in
-index 2352fc1..0191427 100644
---- a/GNUmakefile.in
-+++ b/GNUmakefile.in
-@@ -13,7 +13,7 @@ $(call recurse,all install,src config)
- docs:
- 	$(MAKE) -C doc all
-
--$(call recurse,world,doc src config contrib,all)
-+$(call recurse,world,src config contrib,all)
-
- # build src/ before contrib/
- world-contrib-recurse: world-src-recurse
-@@ -29,7 +29,7 @@ html man:
- install-docs:
- 	$(MAKE) -C doc install
-
--$(call recurse,install-world,doc src config contrib,install)
-+$(call recurse,install-world,src config contrib,install)
-
- # build src/ before contrib/
- install-world-contrib-recurse: install-world-src-recurse

config/software/gitlab-exporter.rb

@@ -23,7 +23,7 @@ skip_transitive_dependency_licensing true
 
 dependency 'ruby'
 dependency 'rubygems'
-dependency 'postgresql_new'
+dependency 'postgresql'
 
 build do
   patch source: 'add-license-file.patch'

config/software/gitlab-rails.rb

@@ -48,7 +48,6 @@ dependency 'curl'
 dependency 'rsync'
 dependency 'libicu'
 dependency 'postgresql'
-dependency 'postgresql_new'
 dependency 'python-docutils'
 dependency 'krb5'
 dependency 'registry'

config/software/postgresql.rb

@@ -16,7 +16,7 @@
 #
 
 name 'postgresql'
-default_version '13.14'
+default_version '14.11'
 
 license 'PostgreSQL'
 license_file 'COPYRIGHT'
@@ -30,11 +30,11 @@ dependency 'ncurses'
 dependency 'libossp-uuid'
 dependency 'config_guess'
 
-version '13.14' do
-  source sha256: 'b8df078551898960bd500dc5d38a177e9905376df81fe7f2b660a1407fa6a5ed'
+version '14.11' do
+  source sha256: 'a670bd7dce22dcad4297b261136b3b1d4a09a6f541719562aa14ca63bf2968a8'
 end
 
-major_version = '13'
+major_version = '14'
 
 source url: "https://ftp.postgresql.org/pub/source/v#{version}/postgresql-#{version}.tar.bz2"
 
@@ -58,6 +58,19 @@ build do
 
   make "world -j #{workers}", env: env
   make 'install-world', env: env
+
+  libpq = 'libpq.so.5'
+  link "#{prefix}/lib/#{libpq}", "#{install_dir}/embedded/lib/#{libpq}"
+
+  # NOTE: There are several dependencies which require these files in these
+  # locations and have dependency on `postgresql_new`. So when this block is
+  # changed to be in the `postgresql` software definition for default PG
+  # version changes, change those dependencies to `postgresql`.
+  block 'link bin files' do
+    Dir.glob("#{prefix}/bin/*").each do |bin_file|
+      link bin_file, "#{install_dir}/embedded/bin/#{File.basename(bin_file)}"
+    end
+  end
 end
 
 # exclude headers and static libraries from package

config/software/postgresql_new.rb

@@ -1,80 +0,0 @@
-#
-# Copyright 2012-2014 Chef Software, Inc.
-# Copyright:: Copyright (c) 2016 GitLab Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-name 'postgresql_new'
-default_version '14.11'
-
-license 'PostgreSQL'
-license_file 'COPYRIGHT'
-
-skip_transitive_dependency_licensing true
-
-dependency 'zlib'
-dependency 'openssl' unless Build::Check.use_system_ssl?
-dependency 'libedit'
-dependency 'ncurses'
-dependency 'libossp-uuid'
-dependency 'config_guess'
-
-version '14.11' do
-  source sha256: 'a670bd7dce22dcad4297b261136b3b1d4a09a6f541719562aa14ca63bf2968a8'
-end
-
-major_version = '14'
-libpq = 'libpq.so.5'
-
-source url: "https://ftp.postgresql.org/pub/source/v#{version}/postgresql-#{version}.tar.bz2"
-
-relative_path "postgresql-#{version}"
-
-build do
-  env = with_standard_compiler_flags(with_embedded_path)
-
-  env['CFLAGS'] << ' -fno-omit-frame-pointer'
-
-  prefix = "#{install_dir}/embedded/postgresql/#{major_version}"
-  update_config_guess(target: 'config')
-
-  patch source: 'no_docs.patch', target: 'GNUmakefile.in'
-
-  command './configure' \
-          " --prefix=#{prefix}" \
-          ' --with-libedit-preferred' \
-          ' --with-openssl' \
-          ' --with-uuid=ossp', env: env
-
-  make "world -j #{workers}", env: env
-  make 'install-world', env: env
-
-  link "#{prefix}/lib/#{libpq}", "#{install_dir}/embedded/lib/#{libpq}"
-
-  # NOTE: There are several dependencies which require these files in these
-  # locations and have dependency on `postgresql_new`. So when this block is
-  # changed to be in the `postgresql` software definition for default PG
-  # version changes, change those dependencies to `postgresql`.
-  block 'link bin files' do
-    Dir.glob("#{prefix}/bin/*").each do |bin_file|
-      link bin_file, "#{install_dir}/embedded/bin/#{File.basename(bin_file)}"
-    end
-  end
-end
-
-# exclude headers and static libraries from package
-project.exclude "embedded/postgresql/#{major_version}/include"
-project.exclude "embedded/postgresql/#{major_version}/lib/*.a"
-project.exclude "embedded/postgresql/#{major_version}/lib/pgxs"
-project.exclude "embedded/postgresql/#{major_version}/lib/pkgconfig"

config/software/postgresql_old.rb

@@ -1,67 +0,0 @@
-#
-# Copyright 2012-2014 Chef Software, Inc.
-# Copyright:: Copyright (c) 2016 GitLab Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-name 'postgresql_old'
-default_version '12.14'
-
-license 'PostgreSQL'
-license_file 'COPYRIGHT'
-
-skip_transitive_dependency_licensing true
-
-dependency 'zlib'
-dependency 'openssl' unless Build::Check.use_system_ssl?
-dependency 'libedit'
-dependency 'ncurses'
-dependency 'libossp-uuid'
-dependency 'config_guess'
-
-version '12.14' do
-  source sha256: '785610237d382c842d356e347138e58c06ffeae240e6cc0b52ac5ebcc30d043e'
-end
-
-major_version = '12'
-
-source url: "https://ftp.postgresql.org/pub/source/v#{version}/postgresql-#{version}.tar.bz2"
-
-relative_path "postgresql-#{version}"
-
-build do
-  env = with_standard_compiler_flags(with_embedded_path)
-
-  env['CFLAGS'] << ' -fno-omit-frame-pointer'
-
-  prefix = "#{install_dir}/embedded/postgresql/#{major_version}"
-  update_config_guess(target: 'config')
-
-  patch source: 'no_docs.patch', target: 'GNUmakefile.in'
-
-  command './configure' \
-          " --prefix=#{prefix}" \
-          ' --with-libedit-preferred' \
-          ' --with-openssl' \
-          ' --with-uuid=ossp', env: env
-
-  make "world -j #{workers}", env: env
-  make 'install-world', env: env
-end
-
-# exclude headers and static libraries from package
-project.exclude "embedded/postgresql/#{major_version}/include"
-project.exclude "embedded/postgresql/#{major_version}/lib/*.a"
-project.exclude "embedded/postgresql/#{major_version}/lib/pgxs"
-project.exclude "embedded/postgresql/#{major_version}/lib/pkgconfig"

config/software/psycopg2.rb

@@ -27,7 +27,7 @@ skip_transitive_dependency_licensing true
 source git: version.remote
 
 dependency 'python3'
-dependency 'postgresql_new'
+dependency 'postgresql'
 
 pg_major_version = '14'
 

config/templates/package-scripts/preinst.erb

@@ -60,7 +60,7 @@ upgrade_check() {
 }
 
 pg_check() {
-  PG_MIN_VERSION=13
+  PG_MIN_VERSION=14
 
   # Fetch the currently running version from the database
   # If this doesn't work, assume this isn't a database node

doc/settings/memory_constrained_envs.md

@@ -145,14 +145,14 @@ We observed 100-400MB of memory usage reduction configuring Puma this way.
 ## Optimize Sidekiq
 
 Sidekiq is a background processing daemon. When configured with GitLab by default
-it runs with a high concurrency mode of `50`. This does impact how much memory it can
+it runs with a concurrency mode of `20`. This does impact how much memory it can
 allocate at a given time. It is advised to configure it to use a significantly
 smaller value of `5` or `10` (preferred).
 
 In `/etc/gitlab/gitlab.rb`:
 
 ```ruby
-sidekiq['max_concurrency'] = 10
+sidekiq['concurrency'] = 10
 ```
 
 ## Optimize Gitaly
@@ -249,7 +249,7 @@ and disable the Prometheus Metrics feature:
    ```ruby
    puma['worker_processes'] = 0
 
-   sidekiq['max_concurrency'] = 10
+   sidekiq['concurrency'] = 10
 
    prometheus_monitoring['enable'] = false
 

doc/settings/rpi.md

@@ -55,7 +55,7 @@ some running processes:
    ```ruby
    # Reduce the number of running workers to the minimum in order to reduce memory usage
    puma['worker_processes'] = 2
-   sidekiq['max_concurrency'] = 9
+   sidekiq['concurrency'] = 9
    # Turn off monitoring to reduce idle cpu and disk usage
    prometheus_monitoring['enable'] = false
    ```

docker/assets/wrapper

@@ -152,7 +152,7 @@ gitlab-ctl reconfigure
 
 # This must be false when the opt-in PostgreSQL version is the default for pg-upgrade,
 # otherwise it must be true.
-ATTEMPT_AUTO_PG_UPGRADE='true'
+ATTEMPT_AUTO_PG_UPGRADE='false'
 
 # Make sure PostgreSQL is at the latest version.
 # If it fails, print a message with a workaround and exit

files/gitlab-config-template/gitlab.rb.template

@@ -1280,9 +1280,7 @@ external_url 'GENERATED_EXTERNAL_URL'
 # sidekiq['log_format'] = "json"
 # sidekiq['shutdown_timeout'] = 4
 # sidekiq['interval'] = nil
-# sidekiq['concurrency'] = nil
-# sidekiq['max_concurrency'] = 20
-# sidekiq['min_concurrency'] = nil
+# sidekiq['concurrency'] = 20
 
 ##! GitLab allows route a job to a particular queue determined by an array of ##! routing rules.
 ##! Each routing rule is a tuple of queue selector query and corresponding queue. By default,

files/gitlab-cookbooks/gitlab/attributes/default.rb

@@ -728,9 +728,7 @@ default['gitlab']['sidekiq']['health_checks_listen_port'] = 8092
 
 # Cluster specific settings
 default['gitlab']['sidekiq']['interval'] = nil
-default['gitlab']['sidekiq']['concurrency'] = nil
-default['gitlab']['sidekiq']['max_concurrency'] = 20
-default['gitlab']['sidekiq']['min_concurrency'] = nil
+default['gitlab']['sidekiq']['concurrency'] = 20
 default['gitlab']['sidekiq']['queue_groups'] = ['*']
 default['gitlab']['sidekiq']['consul_service_name'] = 'sidekiq'
 default['gitlab']['sidekiq']['consul_service_meta'] = nil

files/gitlab-cookbooks/gitlab/recipes/database_migrations.rb

@@ -35,10 +35,10 @@ env_variables['GITLAB_SHARED_RUNNERS_REGISTRATION_TOKEN'] = initial_runner_token
 ruby_block "check remote PG version" do
   block do
     remote_db_version = GitlabRailsEnvHelper.db_version
-    if remote_db_version && remote_db_version.to_f < 13
+    if remote_db_version && remote_db_version.to_f < 14
       LoggingHelper.warning(%q(
-        Note that PostgreSQL 13 is the minimum required PostgreSQL version in GitLab 16.0.
-        Support for PostgreSQL 12 has been removed.
+        Note that PostgreSQL 14 is the minimum required PostgreSQL version in GitLab 17.0.
+        Support for PostgreSQL 13 has been removed.
         To upgrade, please see: https://docs.gitlab.com/omnibus/settings/database.html#upgrade-packaged-postgresql-server
       ))
     end

files/gitlab-cookbooks/gitlab/templates/default/sv-sidekiq-run.erb

@@ -22,12 +22,6 @@ exec chpst -e /opt/gitlab/etc/gitlab-rails/env -P \
       <% if node['gitlab']['sidekiq']['concurrency'] %>
       -c <%= node['gitlab']['sidekiq']['concurrency'] %> \
       <% end %>
-      <% if node['gitlab']['sidekiq']['max_concurrency'] %>
-      -m <%= node['gitlab']['sidekiq']['max_concurrency'] %> \
-      <% end %>
-      <% if node['gitlab']['sidekiq']['min_concurrency'] %>
-      --min-concurrency <%= node['gitlab']['sidekiq']['min_concurrency'] %> \
-      <% end %>
       <% if node['gitlab']['sidekiq']['shutdown_timeout'] %>
       --timeout <%= node['gitlab']['sidekiq']['shutdown_timeout'] %> \
       <% end %>

files/gitlab-cookbooks/package/libraries/deprecations.rb

@@ -346,14 +346,14 @@ module Gitlab
           },
           {
             config_keys: %w(gitlab sidekiq min_concurrency),
-            deprecation: '16.9',
-            removal: '17.0',
+            deprecation: '16.9', # Remove message issue: https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/8491
+            removal: '17.0', # Removal issue: https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/3422
             note: "Starting with GitLab 17.0, `sidekiq['min_concurrency']` will be removed. Please follow https://docs.gitlab.com/ee/administration/sidekiq/extra_sidekiq_processes.html#manage-thread-counts-explicitly to use `sidekiq['concurrency']` instead."
           },
           {
             config_keys: %w(gitlab sidekiq max_concurrency),
-            deprecation: '16.9',
-            removal: '17.0',
+            deprecation: '16.9', # Remove message issue: https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/8491
+            removal: '17.0', # Removal issue: https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/3422
             note: "Starting with GitLab 17.0, `sidekiq['max_concurrency']` will be removed. Please follow https://docs.gitlab.com/ee/administration/sidekiq/extra_sidekiq_processes.html#manage-thread-counts-explicitly to use `sidekiq['concurrency']` instead."
           },
           {

files/gitlab-cookbooks/postgresql/recipes/bin.rb

@@ -40,7 +40,7 @@ end
 ruby_block 'check_postgresql_version_is_deprecated' do
   block do
     LoggingHelper.warning(%q(
-      Note that PostgreSQL 13 is the minimum required PostgreSQL version in GitLab 16.0.
+      Note that PostgreSQL 14 is the minimum required PostgreSQL version in GitLab 17.0.
       To upgrade, please see: https://docs.gitlab.com/omnibus/settings/database.html#upgrade-packaged-postgresql-server
     ))
   end

files/gitlab-ctl-commands/pg-upgrade.rb

@@ -747,7 +747,7 @@ def old_version
 end
 
 def default_version
-  PGVersion.parse(version_from_manifest('postgresql_new')) || PGVersion.parse(version_from_manifest('postgresql'))
+  PGVersion.parse(version_from_manifest('postgresql'))
 end
 
 def new_version

files/gitlab-ctl-commands/upgrade.rb

@@ -344,7 +344,7 @@ end
 def attempt_auto_pg_upgrade?
   # This must return false when the opt-in PostgreSQL version is the default for pg-upgrade,
   # otherwise it must be true.
-  true
+  false
 end
 
 def recommend_pg_upgrade?

spec/chef/cookbooks/gitlab/recipes/database_migrations_spec.rb

@@ -37,14 +37,14 @@ RSpec.describe 'gitlab::database-migrations' do
       before { stub_gitlab_rb(postgresql: { enable: false }) }
 
       it 'warns about outdated databases' do
-        allow(GitlabRailsEnvHelper).to receive(:db_version).and_return(12)
+        allow(GitlabRailsEnvHelper).to receive(:db_version).and_return(13)
 
         expect(migration_block).to notify('ruby_block[check remote PG version]').to(:run)
         expect(chef_run.ruby_block('check remote PG version').should_skip?(:run)).to be_falsey
 
         chef_run.ruby_block('check remote PG version').block.call
 
-        expect_logged_warning(/Support for PostgreSQL 12 has been removed/)
+        expect_logged_warning(/Support for PostgreSQL 13 has been removed/)
       end
     end
 

spec/chef/cookbooks/gitlab/recipes/sidekiq_spec.rb

@@ -26,7 +26,7 @@ RSpec.describe 'gitlab::sidekiq' do
           expect(content).to match(/rubyopt=\"-W:no-experimental\"/)
           expect(content).to include(%(RUBYOPT="${rubyopt}"))
           expect(content).to match(%r{bin/sidekiq-cluster})
-          expect(content).to match(/-m 20/) # max_concurrency
+          expect(content).to match(/-c 20/) # concurrency
           expect(content).to match(/--timeout 25/) # shutdown timeout
           expect(content).to match(/\*/) # all queues
         }
@@ -64,11 +64,19 @@ RSpec.describe 'gitlab::sidekiq' do
       before do
         stub_gitlab_rb(
           sidekiq: {
-            log_group: 'fugee'
+            log_group: 'fugee',
+            concurrency: 42
           }
         )
       end
       it_behaves_like 'enabled logged service', 'sidekiq', true, { log_directory_owner: 'git', log_group: 'fugee' }
+
+      it 'correctly renders out the sidekiq service file' do
+        expect(chef_run).to render_file("/opt/gitlab/sv/sidekiq/run")
+          .with_content { |content|
+            expect(content).to match(/-c 42/) # concurrency
+          }
+      end
     end
   end