Merge branch 'prometheus-1-6' into 'master'

Re-introduce the prometheus version update to 1.5

Closes #2243

See merge request !1524

config/software/prometheus.rb

@@ -19,7 +19,7 @@
 require "#{Omnibus::Config.project_root}/lib/gitlab/version"
 
 name 'prometheus'
-version = Gitlab::Version.new('prometheus', '1.5.2')
+version = Gitlab::Version.new('prometheus', '1.6.1')
 default_version version.print
 
 license 'APACHE-2.0'

files/gitlab-config-template/gitlab.rb.template

@@ -1268,25 +1268,21 @@ external_url 'GENERATED_EXTERNAL_URL'
 #
 ### Prometheus Memory Management
 #
-# Prometheus needs to be configured for how much memory is used for buffering metric data.
-# * Memory chunks are 1kb in size.
-# * The minimum memory needs to be 2x as many chunks as there are unique metrics time-series.
+# Prometheus needs to be configured for how much memory is used.
+# * This sets the target heap size.
+# * This value accounts for approximately 2/3 of the memory used by the server.
+# * The recommended memory is 4kb per unique metrics time-series.
 # See: https://prometheus.io/docs/operating/storage/#memory-usage
 #
-# prometheus['memory_chunks'] = (
-#   # Use 10k + 2% of total memory for chunk buffers.
-#   10_000 + (node['memory']['total'].to_i * 0.02)
-# ).to_i
-# prometheus['max_chunks_to_persist'] = (
-#   # Use 5k + 1% of total memory for max dirty chunks. (half of total chunks)
-#   5_000 + (node['memory']['total'].to_i * 0.01)
+# prometheus['target_heap_size'] = (
+#   # Use 25mb + 2% of total memory for Prometheus memory.
+#   26_214_400 + (node['memory']['total'].to_i * 1024 * 0.02 )
 # ).to_i
 #
 # prometheus['flags'] = {
 #   'storage.local.path' => "#{node['gitlab']['prometheus']['home']}/data",
 #   'storage.local.chunk-encoding-version' => user_config['chunk-encoding-version'],
-#   'storage.local.memory-chunks' => node['gitlab']['prometheus']['memory-chunks'],
-#   'storage.local.max-chunks-to-persist' => node['gitlab']['prometheus']['max-chunks-to-persist'],
+#   'storage.local.target-heap-size' => node['gitlab']['prometheus']['target-heap-size'],
 #   'config.file' => "#{node['gitlab']['prometheus']['home']}/prometheus.yml"
 # }
 

files/gitlab-cookbooks/gitlab/attributes/default.rb

@@ -888,13 +888,9 @@ default['gitlab']['prometheus']['scrape_interval'] = 15
 default['gitlab']['prometheus']['scrape_timeout'] = 15
 default['gitlab']['prometheus']['listen_address'] = 'localhost:9090'
 default['gitlab']['prometheus']['chunk_encoding_version'] = 2
-default['gitlab']['prometheus']['memory_chunks'] = (
-  # Use 10k + 2% of total memory for chunk buffers.
-  10_000 + (node['memory']['total'].to_i * 0.02)
-).to_i
-default['gitlab']['prometheus']['max_chunks_to_persist'] = (
-  # Use 5k + 1% of total memory for max dirty chunks. (half of total chunks)
-  5_000 + (node['memory']['total'].to_i * 0.01)
+default['gitlab']['prometheus']['target_heap_size'] = (
+  # Use 25mb + 2% of total memory for Prometheus memory.
+  26_214_400 + (node['memory']['total'].to_i * 1024 * 0.02)
 ).to_i
 
 ####

files/gitlab-cookbooks/gitlab/libraries/prometheus.rb

@@ -55,14 +55,12 @@ module Prometheus
       home_directory = user_config['home'] || default_config['home']
       listen_address = user_config['listen_address'] || default_config['listen_address']
       chunk_encoding_version = user_config['chunk_encoding_version'] || default_config['chunk_encoding_version']
-      memory_chunks = user_config['memory_chunks'] || default_config['memory_chunks']
-      max_chunks_to_persist = user_config['max_chunks_to_persist'] || default_config['max_chunks_to_persist']
+      target_heap_size = user_config['target_heap_size'] || default_config['target_heap_size']
       default_config['flags'] = {
         'web.listen-address' => listen_address,
         'storage.local.path' => File.join(home_directory, 'data'),
         'storage.local.chunk-encoding-version' => chunk_encoding_version.to_s,
-        'storage.local.memory-chunks' => memory_chunks.to_s,
-        'storage.local.max-chunks-to-persist' => max_chunks_to_persist.to_s,
+        'storage.local.target-heap-size' => target_heap_size.to_s,
         'config.file' => File.join(home_directory, 'prometheus.yml')
       }
 

spec/chef/recipes/prometheus_spec.rb

@@ -91,7 +91,7 @@ describe 'gitlab::prometheus' do
 
     it 'keeps the defaults that the user did not override' do
       expect(chef_run).to render_file('/opt/gitlab/sv/prometheus/run')
-        .with_content(/storage.local.memory-chunks=30971/)
+        .with_content(/storage.local.target-heap-size=47689236/)
       expect(chef_run).to render_file('/opt/gitlab/sv/prometheus/run')
         .with_content(/storage.local.path=foo/)
     end

spec/libraries/prometheus_helper_spec.rb

@@ -26,7 +26,7 @@ describe PrometheusHelper do
       it 'returns the correct default config string' do
         chef_run.converge('gitlab::default')
         expect(subject.flags('prometheus')).to eq(
-          '-web.listen-address=localhost:9090 -storage.local.path=/var/opt/gitlab/prometheus/data -storage.local.chunk-encoding-version=2 -storage.local.memory-chunks=30971 -storage.local.max-chunks-to-persist=15485 -config.file=/var/opt/gitlab/prometheus/prometheus.yml')
+          '-web.listen-address=localhost:9090 -storage.local.path=/var/opt/gitlab/prometheus/data -storage.local.chunk-encoding-version=2 -storage.local.target-heap-size=47689236 -config.file=/var/opt/gitlab/prometheus/prometheus.yml')
       end
     end
 
@@ -38,7 +38,7 @@ describe PrometheusHelper do
         chef_run.converge('gitlab::default')
 
         expect(subject.flags('prometheus')).to eq(
-          '-web.listen-address=localhost:9090 -storage.local.path=/fake/dir/data -storage.local.chunk-encoding-version=2 -storage.local.memory-chunks=30971 -storage.local.max-chunks-to-persist=15485 -config.file=/fake/dir/prometheus.yml')
+          '-web.listen-address=localhost:9090 -storage.local.path=/fake/dir/data -storage.local.chunk-encoding-version=2 -storage.local.target-heap-size=47689236 -config.file=/fake/dir/prometheus.yml')
       end
     end
   end