Update Prometheus flags

Prometheus needs to be configured for how much memory is used for buffering metric data.
* Memory chunks are 1kb in size.
* The minimum memory needs to be 2x as many chunks as there are unique metrics time-series.
See: https://prometheus.io/docs/operating/storage/#memory-usage

* Update memory allocation controls
  - Use 10k + 2% of memory for memory-chunks.
  - Set half of that for max-chunks-to-persist.
* Set default for chunk encoding version to 2
  - Trades some CPU time for increased compression.

files/gitlab-config-template/gitlab.rb.template

@@ -1255,10 +1255,29 @@ external_url 'GENERATED_EXTERNAL_URL'
 # prometheus['log_directory'] = '/var/log/gitlab/prometheus'
 # prometheus['scrape_interval'] = 15
 # prometheus['scrape_timeout'] = 15
+# prometheus['chunk_encoding_version'] = 2
+#
+### Prometheus Memory Management
+#
+# Prometheus needs to be configured for how much memory is used for buffering metric data.
+# * Memory chunks are 1kb in size.
+# * The minimum memory needs to be 2x as many chunks as there are unique metrics time-series.
+# See: https://prometheus.io/docs/operating/storage/#memory-usage
+#
+# prometheus['memory_chunks'] = (
+#   # Use 10k + 2% of total memory for chunk buffers.
+#   10_000 + (node['memory']['total'].to_i * 0.02)
+# ).to_i
+# prometheus['max_chunks_to_persist'] = (
+#   # Use 5k + 1% of total memory for max dirty chunks. (half of total chunks)
+#   5_000 + (node['memory']['total'].to_i * 0.01)
+# ).to_i
+#
 # prometheus['flags'] = {
 #   'storage.local.path' => "#{node['gitlab']['prometheus']['home']}/data",
-#   'storage.local.memory-chunks' => '50000',
-#   'storage.local.max-chunks-to-persist' => '40000',
+#   'storage.local.chunk-encoding-version' => user_config['chunk-encoding-version'],
+#   'storage.local.memory-chunks' => node['gitlab']['prometheus']['memory-chunks'],
+#   'storage.local.max-chunks-to-persist' => node['gitlab']['prometheus']['max-chunks-to-persist'],
 #   'config.file' => "#{node['gitlab']['prometheus']['home']}/prometheus.yml"
 # }
 

files/gitlab-cookbooks/gitlab/attributes/default.rb

@@ -902,6 +902,15 @@ default['gitlab']['prometheus']['log_directory'] = '/var/log/gitlab/prometheus'
 default['gitlab']['prometheus']['scrape_interval'] = 15
 default['gitlab']['prometheus']['scrape_timeout'] = 15
 default['gitlab']['prometheus']['listen_address'] = 'localhost:9090'
+default['gitlab']['prometheus']['chunk_encoding_version'] = 2
+default['gitlab']['prometheus']['memory_chunks'] = (
+  # Use 10k + 2% of total memory for chunk buffers.
+  10_000 + (node['memory']['total'].to_i * 0.02)
+).to_i
+default['gitlab']['prometheus']['max_chunks_to_persist'] = (
+  # Use 5k + 1% of total memory for max dirty chunks. (half of total chunks)
+  5_000 + (node['memory']['total'].to_i * 0.01)
+).to_i
 
 ####
 # Prometheus Node Exporter

files/gitlab-cookbooks/gitlab/libraries/prometheus.rb

@@ -54,11 +54,15 @@ module Prometheus
 
       home_directory = user_config['home'] || default_config['home']
       listen_address = user_config['listen_address'] || default_config['listen_address']
+      chunk_encoding_version = user_config['chunk_encoding_version'] || default_config['chunk_encoding_version']
+      memory_chunks = user_config['memory_chunks'] || default_config['memory_chunks']
+      max_chunks_to_persist = user_config['max_chunks_to_persist'] || default_config['max_chunks_to_persist']
       default_config['flags'] = {
         'web.listen-address' => listen_address,
         'storage.local.path' => File.join(home_directory, 'data'),
-        'storage.local.memory-chunks' => '50000',
-        'storage.local.max-chunks-to-persist' => '40000',
+        'storage.local.chunk-encoding-version' => chunk_encoding_version.to_s,
+        'storage.local.memory-chunks' => memory_chunks.to_s,
+        'storage.local.max-chunks-to-persist' => max_chunks_to_persist.to_s,
         'config.file' => File.join(home_directory, 'prometheus.yml')
       }
 

spec/chef/recipes/prometheus_spec.rb

@@ -91,7 +91,7 @@ describe 'gitlab::prometheus' do
 
     it 'keeps the defaults that the user did not override' do
       expect(chef_run).to render_file('/opt/gitlab/sv/prometheus/run')
-        .with_content(/storage.local.memory-chunks=5000/)
+        .with_content(/storage.local.memory-chunks=30971/)
       expect(chef_run).to render_file('/opt/gitlab/sv/prometheus/run')
         .with_content(/storage.local.path=foo/)
     end

spec/libraries/prometheus_helper_spec.rb

@@ -26,7 +26,7 @@ describe PrometheusHelper do
       it 'returns the correct default config string' do
         chef_run.converge('gitlab::default')
         expect(subject.flags('prometheus')).to eq(
-          '-web.listen-address=localhost:9090 -storage.local.path=/var/opt/gitlab/prometheus/data -storage.local.memory-chunks=50000 -storage.local.max-chunks-to-persist=40000 -config.file=/var/opt/gitlab/prometheus/prometheus.yml')
+          '-web.listen-address=localhost:9090 -storage.local.path=/var/opt/gitlab/prometheus/data -storage.local.chunk-encoding-version=2 -storage.local.memory-chunks=30971 -storage.local.max-chunks-to-persist=15485 -config.file=/var/opt/gitlab/prometheus/prometheus.yml')
       end
     end
 
@@ -38,7 +38,7 @@ describe PrometheusHelper do
         chef_run.converge('gitlab::default')
 
         expect(subject.flags('prometheus')).to eq(
-          '-web.listen-address=localhost:9090 -storage.local.path=/fake/dir/data -storage.local.memory-chunks=50000 -storage.local.max-chunks-to-persist=40000 -config.file=/fake/dir/prometheus.yml')
+          '-web.listen-address=localhost:9090 -storage.local.path=/fake/dir/data -storage.local.chunk-encoding-version=2 -storage.local.memory-chunks=30971 -storage.local.max-chunks-to-persist=15485 -config.file=/fake/dir/prometheus.yml')
       end
     end
   end