Merge branch 'pages_daemon' into 'master'

Build the daemon for gitlab-pages.

Fixes #1097

See merge request !638

config/software/gitlab-pages.rb

@@ -0,0 +1,29 @@
+#
+# Copyright:: Copyright (c) 2016 GitLab B.V.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+name "gitlab-pages"
+default_version "master"
+
+source :git => "https://gitlab.com/gitlab-org/gitlab-pages.git"
+
+build do
+  # We use the `base_dir`, because the sources are put in `src/gitlab-pages`
+  # This is required for GO15VENDOREXPERIMENT=1 to work properly,
+  # since it requires the package to be in $GOPATH/src/package
+  env = { 'GOPATH' => "#{Omnibus::Config.base_dir}"}
+  make "gitlab-pages", env: env
+  move "gitlab-pages", "#{install_dir}/embedded/bin/gitlab-pages"
+end

config/software/gitlab-rails.rb

@@ -32,9 +32,13 @@ dependency "rsync"
 dependency "libicu"
 dependency "postgresql"
 dependency "python-docutils"
-dependency "mysql-client" if EE
 dependency "krb5"
-dependency "unzip" if EE
+
+if EE
+  dependency "mysql-client"
+  dependency "unzip"
+  dependency "gitlab-pages"
+end
 
 source :git => version.remote
 

files/gitlab-config-template/gitlab.rb.template

@@ -722,6 +722,15 @@ external_url 'GENERATED_EXTERNAL_URL'
 ## Define to enable GitLab Pages
 # pages_external_url "http://pages.example.com/"
 
+# gitlab_pages['enable'] = false
+# gitlab_pages['external_http'] = nil # Configure to expose GitLab Pages on external IP address, serving the HTTP
+# gitlab_pages['external_https'] = nil # Configure to expose GitLab Pages on external IP address, serving the HTTPS
+# gitlab_pages['listen_proxy'] = "localhost:8090"
+# gitlab_pages['redirect_http'] = true
+# gitlab_pages['use_http2'] = true
+# gitlab_pages['dir'] = "/var/opt/gitlab/gitlab-pages"
+# gitlab_pages['log_dir'] = "/var/log/gitlab/gitlab-pages"
+
 ####################
 # GitLab Pages NGINX #
 ####################

files/gitlab-cookbooks/gitlab/attributes/default.rb

@@ -105,13 +105,13 @@ default['gitlab']['gitlab-rails']['lfs_storage_path'] = nil
 default['gitlab']['gitlab-rails']['elasticsearch_enabled'] = false
 default['gitlab']['gitlab-rails']['elasticsearch_host'] = nil
 default['gitlab']['gitlab-rails']['elasticsearch_port'] = nil
+default['gitlab']['gitlab-rails']['ldap_enabled'] = false
+default['gitlab']['gitlab-rails']['ldap_servers'] = []
 default['gitlab']['gitlab-rails']['pages_enabled'] = false
 default['gitlab']['gitlab-rails']['pages_host'] = nil
 default['gitlab']['gitlab-rails']['pages_port'] = nil
 default['gitlab']['gitlab-rails']['pages_https'] = false
 default['gitlab']['gitlab-rails']['pages_path'] = nil
-default['gitlab']['gitlab-rails']['ldap_enabled'] = false
-default['gitlab']['gitlab-rails']['ldap_servers'] = []
 
 ####
 # These LDAP settings are deprecated in favor of the new syntax. They are kept here for backwards compatibility.
@@ -393,6 +393,22 @@ default['gitlab']['mailroom']['enable'] = false
 default['gitlab']['mailroom']['ha'] = false
 default['gitlab']['mailroom']['log_dir'] = "/var/log/gitlab/mailroom"
 
+####
+# GitLab Pages Daemon
+####
+default['gitlab']['gitlab-pages']['enable'] = false
+default['gitlab']['gitlab-pages']['external_http'] = nil
+default['gitlab']['gitlab-pages']['external_https'] = nil
+default['gitlab']['gitlab-pages']['listen_proxy'] = "localhost:8090"
+default['gitlab']['gitlab-pages']['pages_path'] = nil
+default['gitlab']['gitlab-pages']['domain'] = nil
+default['gitlab']['gitlab-pages']['cert'] = nil
+default['gitlab']['gitlab-pages']['cert_key'] = nil
+default['gitlab']['gitlab-pages']['redirect_http'] = true
+default['gitlab']['gitlab-pages']['use_http2'] = true
+default['gitlab']['gitlab-pages']['dir'] = "/var/opt/gitlab/gitlab-pages"
+default['gitlab']['gitlab-pages']['log_dir'] = "/var/log/gitlab/gitlab-pages"
+
 ####
 # Nginx
 ####
@@ -466,7 +482,7 @@ default['gitlab']['remote-syslog']['dir'] = "/var/opt/gitlab/remote-syslog"
 default['gitlab']['remote-syslog']['log_directory'] = "/var/log/gitlab/remote-syslog"
 default['gitlab']['remote-syslog']['destination_host'] = "localhost"
 default['gitlab']['remote-syslog']['destination_port'] = 514
-default['gitlab']['remote-syslog']['services'] = %w{redis nginx unicorn gitlab-rails gitlab-shell postgresql sidekiq ci-redis ci-unicorn ci-sidekiq}
+default['gitlab']['remote-syslog']['services'] = %w{redis nginx unicorn gitlab-rails gitlab-shell postgresql sidekiq gitlab-workhorse gitlab-pages}
 
 ###
 # Logrotate
@@ -475,7 +491,7 @@ default['gitlab']['logrotate']['enable'] = true
 default['gitlab']['logrotate']['ha'] = false
 default['gitlab']['logrotate']['dir'] = "/var/opt/gitlab/logrotate"
 default['gitlab']['logrotate']['log_directory'] = "/var/log/gitlab/logrotate"
-default['gitlab']['logrotate']['services'] = %w{nginx unicorn gitlab-rails gitlab-shell gitlab-ci}
+default['gitlab']['logrotate']['services'] = %w{nginx unicorn gitlab-rails gitlab-shell gitlab-workhorse gitlab-pages}
 default['gitlab']['logrotate']['pre_sleep'] = 600 # sleep 10 minutes before rotating after start-up
 default['gitlab']['logrotate']['post_sleep'] = 3000 # wait 50 minutes after rotating
 

files/gitlab-cookbooks/gitlab/libraries/gitlab.rb

@@ -61,6 +61,7 @@ module Gitlab
   high_availability Mash.new
   web_server Mash.new
   mattermost Mash.new
+  gitlab_pages Mash.new
   node nil
   external_url nil
   pages_external_url nil
@@ -383,6 +384,7 @@ module Gitlab
       return unless pages_external_url
 
       gitlab_rails['pages_enabled'] = true if gitlab_rails['pages_enabled'].nil?
+      gitlab_pages['enable'] = true if gitlab_pages['enable'].nil?
 
       uri = URI(pages_external_url.to_s)
 
@@ -412,6 +414,19 @@ module Gitlab
       Gitlab['pages_nginx']['fqdn_regex'] = uri.host.sub('.', '\.')
     end
 
+    def parse_gitlab_pages_daemon
+      return unless gitlab_pages['enable']
+
+      gitlab_pages['domain'] = Gitlab['gitlab_rails']['pages_host']
+
+      if gitlab_pages['external_https']
+        Gitlab['gitlab_pages']['cert'] ||= "/etc/gitlab/ssl/#{Gitlab['gitlab_pages']['domain']}.crt"
+        Gitlab['gitlab_pages']['cert_key'] ||= "/etc/gitlab/ssl/#{Gitlab['gitlab_pages']['domain']}.key"
+      end
+
+      Gitlab['gitlab_pages']['pages_root'] ||= (gitlab_rails['pages_path'] || File.join(Gitlab['gitlab_rails']['shared_path'], 'pages'))
+    end
+
     def parse_mattermost_external_url
       return unless mattermost_external_url
 
@@ -496,7 +511,8 @@ module Gitlab
         "external_url",
         "ci_external_url",
         "mattermost_external_url",
-        "pages_external_url"
+        "pages_external_url",
+        "gitlab_pages"
       ].each do |key|
         rkey = key.gsub('_', '-')
         results['gitlab'][rkey] = Gitlab[key]
@@ -529,6 +545,7 @@ module Gitlab
       parse_gitlab_ci
       parse_gitlab_mattermost
       parse_incoming_email
+      parse_gitlab_pages_daemon
       disable_gitlab_rails_services
       # The last step is to convert underscores to hyphens in top-level keys
       generate_hash

files/gitlab-cookbooks/gitlab/recipes/default.rb

@@ -101,7 +101,8 @@ include_recipe "runit"
   "remote-syslog",
   "logrotate",
   "bootstrap",
-  "mattermost"
+  "mattermost",
+  "gitlab-pages"
 ].each do |service|
   if node["gitlab"][service]["enable"]
     include_recipe "gitlab::#{service}"

files/gitlab-cookbooks/gitlab/recipes/gitlab-pages.rb

@@ -0,0 +1,40 @@
+#
+# Copyright:: Copyright (c) 2016 GitLab B.V.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+account_helper = AccountHelper.new(node)
+
+working_dir = node['gitlab']['gitlab-pages']['dir']
+log_dir = node['gitlab']['gitlab-pages']['log_dir']
+gitlab_pages_static_etc_dir = "/opt/gitlab/etc/gitlab-pages"
+
+[
+  working_dir,
+  log_dir,
+  gitlab_pages_static_etc_dir
+].each do |dir|
+  directory dir do
+    owner account_helper.gitlab_user
+    mode '0700'
+    recursive true
+  end
+end
+
+runit_service 'gitlab-pages' do
+  options({
+    :log_directory => log_dir
+  }.merge(params))
+  log_options node['gitlab']['logging'].to_hash.merge(node['gitlab']['gitlab-pages'].to_hash)
+end

files/gitlab-cookbooks/gitlab/recipes/gitlab-pages_disable.rb

@@ -0,0 +1,20 @@
+#
+# Copyright:: Copyright (c) 2016 GitLab B.V.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+runit_service "gitlab-pages" do
+  action :disable
+end

files/gitlab-cookbooks/gitlab/recipes/gitlab-rails.rb

@@ -230,7 +230,9 @@ template_symlink File.join(gitlab_rails_etc_dir, "gitlab.yml") do
       gitlab_ci_all_broken_builds: node['gitlab']['gitlab-ci']['gitlab_ci_all_broken_builds'],
       gitlab_ci_add_pusher: node['gitlab']['gitlab-ci']['gitlab_ci_add_pusher'],
       builds_directory: gitlab_ci_builds_dir,
-      git_annex_enabled: node['gitlab']['gitlab-shell']['git_annex_enabled']
+      git_annex_enabled: node['gitlab']['gitlab-shell']['git_annex_enabled'],
+      pages_external_http: node['gitlab']['gitlab-pages']['external_http'],
+      pages_external_https: node['gitlab']['gitlab-pages']['external_https']
     )
   )
   restarts dependent_services

files/gitlab-cookbooks/gitlab/recipes/nginx.rb

@@ -74,7 +74,7 @@ nginx_vars = nginx_vars.to_hash.merge!({
                :gitlab_mattermost_http_config => gitlab_mattermost_enabled ? gitlab_mattermost_http_conf : nil
              })
 
-# Include the config file for gitlab mattermost in nginx.conf later
+# Include the config file for gitlab pages in nginx.conf later
 nginx_vars = nginx_vars.to_hash.merge!({
                                          :gitlab_pages_http_config => gitlab_pages_enabled ? gitlab_pages_http_conf : nil
                                        })
@@ -121,6 +121,7 @@ template gitlab_pages_http_conf do
   variables(pages_nginx_vars.merge(
     {
       pages_path: node['gitlab']['gitlab-rails']['pages_path'],
+      pages_listen_proxy: node['gitlab']['gitlab-pages']['listen_proxy']
     }
   ))
   notifies :restart, 'service[nginx]' if OmnibusHelper.should_notify?("nginx")

files/gitlab-cookbooks/gitlab/templates/default/gitlab.yml.erb

@@ -123,6 +123,8 @@ production: &base
     host: <%= @pages_host %>
     port: <%= @pages_port %>
     https: <%= @pages_https %>
+    external_http: <%= @pages_external_http %>
+    external_https: <%= @pages_external_https %>
 
   ## Elasticsearch (EE only)
   # Enable it if you are going to use elasticsearch instead of

files/gitlab-cookbooks/gitlab/templates/default/nginx-gitlab-pages-http.conf.erb

@@ -25,11 +25,10 @@ server {
 
 server {
 <% @listen_addresses.each do |listen_address| %>
-  listen <%= listen_address %>:<%= @listen_port %><% if @https %> ssl spdy<% end %>;
+  listen <%= listen_address %>:<%= @listen_port %><% if @https %> ssl http2<% end %>;
 <% end %>
   server_name  ~^(?<group>.*)\.<%= @fqdn_regex %>$;
   server_tokens off; ## Don't show the nginx version number, a security best practice
-  root <%= @pages_path %>/${group};
 
   ## Disable symlink traversal
   disable_symlinks on;
@@ -60,14 +59,13 @@ server {
   access_log  <%= @log_directory %>/gitlab_pages_access.log gitlab_access;
   error_log   <%= @log_directory %>/gitlab_pages_error.log;
 
-  # 1. Try to get /project/ to => shared/pages/${group}/public/ or index.html
-  # 2. Try to get / to => shared/pages/${group}/${host}/public/ or index.html
-  location ~ ^/([^/]*)(/.*)?$ {
-    try_files "/$1/public$2"
-              "/$1/public$2/index.html"
-              "/${host}/public/${uri}"
-              "/${host}/public/${uri}/index.html"
-              =404;
+  # Pass everything to pages daemon
+  location / {
+    proxy_set_header    Host                $http_host;
+    proxy_set_header    X-Real-IP           $remote_addr;
+    proxy_set_header    X-Forwarded-For     $proxy_add_x_forwarded_for;
+    proxy_set_header    X-Forwarded-Proto   $scheme;
+    proxy_pass          http://<%= @pages_listen_proxy %>;
   }
 
   # Define custom error pages

files/gitlab-cookbooks/gitlab/templates/default/sv-gitlab-pages-log-config.erb

@@ -0,0 +1,6 @@
+<%= "s#@svlogd_size" if @svlogd_size %>
+<%= "n#@svlogd_num" if @svlogd_num %>
+<%= "t#@svlogd_timeout" if @svlogd_timeout %>
+<%= "!#@svlogd_filter" if @svlogd_filter %>
+<%= "u#@svlogd_udp" if @svlogd_udp %>
+<%= "p#@svlogd_prefix" if @svlogd_prefix %>

files/gitlab-cookbooks/gitlab/templates/default/sv-gitlab-pages-log-run.erb

@@ -0,0 +1,2 @@
+#!/bin/sh
+exec svlogd -tt <%= @options[:log_directory] %>

files/gitlab-cookbooks/gitlab/templates/default/sv-gitlab-pages-run.erb

@@ -0,0 +1,28 @@
+#!/bin/bash
+set -e # fail on errors
+
+# Redirect stderr -> stdout
+exec 2>&1
+
+cd <%= node['gitlab']['gitlab-pages']['dir'] %>
+
+exec /opt/gitlab/embedded/bin/gitlab-pages \
+    <% if node['gitlab']['pages-nginx']['enable'] && node['gitlab']['gitlab-pages']['listen_proxy'] %>
+    -listen-proxy "<%= node['gitlab']['gitlab-pages']['listen_proxy'] %>" \
+    <% end %>
+    <% if node['gitlab']['gitlab-pages']['external_http'] %>
+    -listen-http  "<%= node['gitlab']['gitlab-pages']['external_http'] %>" \
+    <% end %>
+    <% if node['gitlab']['gitlab-pages']['external_https'] %>
+    -listen-https "<%= node['gitlab']['gitlab-pages']['external_https'] %>" \
+    -root-cert "<%= node['gitlab']['gitlab-pages']['cert'] %>" \
+    -root-key  "<%= node['gitlab']['gitlab-pages']['cert_key'] %>" \
+    <% end %>
+    -daemon-uid "$(id -u "<%= node['gitlab']['user']['username'] %>")" \
+    -daemon-gid "$(id -g "<%= node['gitlab']['user']['username'] %>")" \
+    \
+    -pages-domain "<%= node['gitlab']['gitlab-pages']['domain'] %>" \
+    -pages-root "<%= node['gitlab']['gitlab-pages']['pages_root'] %>" \
+    \
+    -redirect-http <%= node['gitlab']['gitlab-pages']['redirect_http'] %> \
+    -use-http2 <%= node['gitlab']['gitlab-pages']['use_http2'] %> \