Merge branch 'gitlab-pages' into 'master'

GitLab Pages support for Omnibus

This adds `pages_external_url` which allows to enable GitLab Pages.

This adds `pages_nginx` section for nginx configuration.
The configuration model is the same as for `nginx` and `mattermost_nginx`.

/cc @jacobvosmaer 

See merge request !565

doc/README.md

@@ -32,6 +32,7 @@
 - [Environment variables](settings/environment-variables.md)
 - [gitlab.yml](settings/gitlab.yml.md)
 - [Backups](settings/backups.md)
+- [Pages (EE-only)](settings/pages.md)
 
 ## Updating
 

doc/settings/pages.md

@@ -0,0 +1,81 @@
+# GitLab Pages (EE-only)
+
+_**Note:** This feature was [introduced][ee-80] in GitLab EE 8.3_
+
+If you are looking for ways to upload your static content in GitLab Pages, you
+probably want to read the [user documentation][user-doc].
+
+## Documentation version
+
+Make sure you view this guide from the tag (version) of GitLab you would like
+to install. In most cases this should be the highest numbered production tag
+(without rc in it). You can select the tag in the version dropdown in the top
+left corner of GitLab (below the menu bar).
+
+If the highest number stable branch is unclear please check the
+[GitLab Blog](https://about.gitlab.com/blog/) for installation guide links by
+version.
+
+## Getting started
+
+GitLab Pages expect to run on their own virtual host. In your DNS you need to
+add a [wildcard DNS A record][wiki-wildcard-dns] pointing to the host that
+GitLab runs. For example, an entry would look like this:
+
+```
+*.gitlabpages.com. 60 IN A 1.2.3.4
+```
+
+where `gitlabpages.com` is the domain under which GitLab Pages will be served
+and `1.2.3.4` is the IP address of your GitLab instance.
+
+It is strongly advised to **not** use the GitLab domain to serve user pages to
+prevent XSS attacks.
+
+GitLab Pages is disabled by default, to enable it just tell omnibus-gitlab what
+the external URL for GitLab Pages is:
+
+```ruby
+# in /etc/gitlab/gitlab.rb
+pages_external_url 'http://gitlabpages.com'
+```
+
+Run `sudo gitlab-ctl reconfigure` for the changes to take effect and read the
+[user documentation][user-doc] to learn how to create a static webpage for your
+project, your user or group.
+
+## Running GitLab Pages with HTTPS
+
+If you want the pages to be served under HTTPS, a wildcard SSL certificate is
+required.
+
+Place the certificate and key inside `/etc/gitlab/ssl` and in
+`/etc/gitlab/gitlab.rb` specify the following configuration:
+
+```ruby
+pages_external_url 'https://gitlabpages.com'
+
+pages_nginx['redirect_http_to_https'] = true
+pages_nginx['ssl_certificate'] = "/etc/gitlab/ssl/pages-nginx.crt"
+pages_nginx['ssl_certificate_key'] = "/etc/gitlab/ssl/pages-nginx.key"
+```
+
+where `pages-nginx.crt` and `pages-nginx.key` are the SSL cert and key,
+respectively. Once the configuration is set, run `sudo gitlab-ctl reconfigure`
+for the changes to take effect.
+
+## Change storage path
+
+Pages are stored by default in `/var/opt/gitlab/gitlab-rails/shared/pages`.
+If you wish to store them in another location you must set it up in
+`/etc/gitlab/gitlab.rb`:
+
+```ruby
+gitlab_rails['pages_path'] = "/mnt/storage/pages"
+```
+
+Run `sudo gitlab-ctl reconfigure` for the changes to take effect.
+
+[user-doc]: http://doc.gitlab.com/pages/README.md
+[ee-80]: https://gitlab.com/gitlab-org/gitlab-ee/merge_requests/80
+[wiki-wildcard-dns]: https://en.wikipedia.org/wiki/Wildcard_DNS_record

files/gitlab-config-template/gitlab.rb.template

@@ -73,6 +73,9 @@ external_url 'GENERATED_EXTERNAL_URL'
 # gitlab_rails['lfs_enabled'] = true
 # gitlab_rails['lfs_storage_path'] = "/mnt/storage/lfs-objects"
 
+## GitLab Pages
+# gitlab_rails['pages_path'] = "/mnt/storage/pages"
+
 ## For setting up LDAP
 ## see https://gitlab.com/gitlab-org/omnibus-gitlab/blob/629def0a7a26e7c2326566f0758d4a27857b52a3/README.md#setting-up-ldap-sign-in
 ## Be careful not to break the identation in the ldap_servers block. It is in
@@ -816,4 +819,35 @@ external_url 'GENERATED_EXTERNAL_URL'
 # mattermost_nginx['gzip_proxied'] = "any"
 # mattermost_nginx['gzip_types'] = [ "text/plain", "text/css", "application/x-javascript", "text/xml", "application/xml", "application/xml+rss", "text/javascript", "application/json" ]
 # mattermost_nginx['keepalive_timeout'] = 65
-# mattermost_nginx['cache_max_size'] = '5000m'
+# mattermost_nginx['cache_max_size'] = '5000m'
+
+####################
+# GitLab Pages
+####################
+
+## Define to enable GitLab Pages
+# pages_external_url "http://pages.example.com/"
+
+####################
+# GitLab Pages NGINX #
+####################
+
+# pages_nginx['enable'] = false
+# pages_nginx['redirect_http_to_https'] = false
+# pages_nginx['redirect_http_to_https_port'] = 80
+# pages_nginx['ssl_certificate'] = "/etc/gitlab/ssl/#{node['fqdn']}.crt"
+# pages_nginx['ssl_certificate_key'] = "/etc/gitlab/ssl/#{node['fqdn']}.key"
+# pages_nginx['ssl_ciphers'] = "ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-GCM-SHA256"
+# pages_nginx['ssl_prefer_server_ciphers'] = "on"
+# pages_nginx['ssl_protocols'] = "TLSv1 TLSv1.1 TLSv1.2" # recommended by https://raymii.org/s/tutorials/Strong_SSL_Security_On_nginx.html & https://cipherli.st/
+# pages_nginx['ssl_session_cache'] = "builtin:1000  shared:SSL:10m" # recommended in http://nginx.org/en/docs/http/ngx_http_ssl_module.html
+# pages_nginx['ssl_session_timeout'] = "5m" # default according to http://nginx.org/en/docs/http/ngx_http_ssl_module.html
+# pages_nginx['ssl_dhparam'] = nil # Path to ci_dhparams.pem, eg. /etc/gitlab/ssl/ci_dhparams.pem
+# pages_nginx['listen_addresses'] = ['*']
+# pages_nginx['listen_port'] = nil # override only if you use a reverse proxy: https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/doc/settings/nginx.md#setting-the-nginx-listen-port
+# pages_nginx['listen_https'] = nil # override only if your reverse proxy internally communicates over HTTP: https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/doc/settings/nginx.md#supporting-proxied-ssl
+# pages_nginx['custom_gitlab_server_config'] = "location ^~ /foo-namespace/bar-project/raw/ {\n deny all;\n}\n"
+
+## Advanced settings
+# pages_nginx['dir'] = "/var/opt/gitlab/nginx"
+# pages_nginx['log_directory'] = "/var/log/gitlab/nginx"

files/gitlab-cookbooks/gitlab/attributes/default.rb

@@ -100,6 +100,11 @@ default['gitlab']['gitlab-rails']['artifacts_enabled'] = true
 default['gitlab']['gitlab-rails']['artifacts_path'] = nil
 default['gitlab']['gitlab-rails']['lfs_enabled'] = false
 default['gitlab']['gitlab-rails']['lfs_storage_path'] = nil
+default['gitlab']['gitlab-rails']['pages_enabled'] = false
+default['gitlab']['gitlab-rails']['pages_host'] = nil
+default['gitlab']['gitlab-rails']['pages_port'] = nil
+default['gitlab']['gitlab-rails']['pages_https'] = false
+default['gitlab']['gitlab-rails']['pages_path'] = nil
 default['gitlab']['gitlab-rails']['ldap_enabled'] = false
 default['gitlab']['gitlab-rails']['ldap_servers'] = []
 
@@ -678,3 +683,9 @@ default['gitlab']['mattermost']['gitlab_user_api_endpoint'] = nil
 ####
 default['gitlab']['mattermost-nginx'] = default['gitlab']['nginx'].dup
 default['gitlab']['mattermost-nginx']['enable'] = false
+
+####
+# GitLab Pages NGINX
+####
+default['gitlab']['pages-nginx'] = default['gitlab']['nginx'].dup
+default['gitlab']['pages-nginx']['enable'] = true

files/gitlab-cookbooks/gitlab/libraries/gitlab.rb

@@ -50,6 +50,7 @@ module Gitlab
   ci_sidekiq Mash.new
   gitlab_workhorse Mash.new
   gitlab_git_http_server Mash.new # legacy from GitLab 7.14, 8.0, 8.1
+  pages_nginx Mash.new
   mailroom Mash.new
   nginx Mash.new
   ci_nginx Mash.new
@@ -62,6 +63,7 @@ module Gitlab
   mattermost Mash.new
   node nil
   external_url nil
+  pages_external_url nil
   ci_external_url nil
   mattermost_external_url nil
   git_data_dir nil
@@ -156,13 +158,20 @@ module Gitlab
     end
 
     def parse_artifacts_dir
+      # This requires the parse_shared_dir to be executed before
       Gitlab['gitlab_rails']['artifacts_path'] ||= File.join(Gitlab['gitlab_rails']['shared_path'], 'artifacts')
     end
 
     def parse_lfs_objects_dir
+      # This requires the parse_shared_dir to be executed before
       Gitlab['gitlab_rails']['lfs_storage_path'] ||= File.join(Gitlab['gitlab_rails']['shared_path'], 'lfs-objects')
     end
 
+    def parse_pages_dir
+      # This requires the parse_shared_dir to be executed before
+      Gitlab['gitlab_rails']['pages_path'] ||= File.join(Gitlab['gitlab_rails']['shared_path'], 'pages')
+    end
+
     def parse_udp_log_shipping
       return unless logging['udp_log_shipping_host']
 
@@ -284,7 +293,8 @@ module Gitlab
       [
         [%w{nginx listen_port}, %w{gitlab_rails gitlab_port}],
         [%w{ci_nginx listen_port}, %w{gitlab_ci gitlab_ci_port}],
-        [%w{mattermost_nginx listen_port}, %w{mattermost port}]
+        [%w{mattermost_nginx listen_port}, %w{mattermost port}],
+        [%w{pages_nginx listen_port}, %w{gitlab_rails pages_port}],
 
       ].each do |left, right|
         if !Gitlab[left.first][left.last].nil?
@@ -338,6 +348,39 @@ module Gitlab
       ci_nginx['enable'] = true if ci_nginx['enable'].nil?
     end
 
+    def parse_pages_external_url
+      return unless pages_external_url
+
+      gitlab_rails['pages_enabled'] = true if gitlab_rails['pages_enabled'].nil?
+
+      uri = URI(pages_external_url.to_s)
+
+      unless uri.host
+        raise "GitLab Pages external URL must must include a schema and FQDN, e.g. http://pages.example.com/"
+      end
+
+      Gitlab['gitlab_rails']['pages_host'] = uri.host
+      Gitlab['gitlab_rails']['pages_port'] = uri.port
+
+      case uri.scheme
+      when "http"
+        Gitlab['gitlab_rails']['pages_https'] = false
+      when "https"
+        Gitlab['gitlab_rails']['pages_https'] = true
+        Gitlab['pages_nginx']['ssl_certificate'] ||= "/etc/gitlab/ssl/#{uri.host}.crt"
+        Gitlab['pages_nginx']['ssl_certificate_key'] ||= "/etc/gitlab/ssl/#{uri.host}.key"
+      else
+        raise "Unsupported GitLab Pages external URL scheme: #{uri.scheme}"
+      end
+
+      unless ["", "/"].include?(uri.path)
+        raise "Unsupported GitLab Pages external URL path: #{uri.path}"
+      end
+
+      # FQDN are prepared to be used as regexp: the dot is escaped
+      Gitlab['pages_nginx']['fqdn_regex'] = uri.host.sub('.', '\.')
+    end
+
     def parse_mattermost_external_url
       return unless mattermost_external_url
 
@@ -411,6 +454,7 @@ module Gitlab
         "nginx",
         "ci_nginx",
         "mattermost_nginx",
+        "pages_nginx",
         "logging",
         "remote_syslog",
         "logrotate",
@@ -437,6 +481,7 @@ module Gitlab
       parse_shared_dir
       parse_artifacts_dir
       parse_lfs_objects_dir
+      parse_pages_dir
       parse_udp_log_shipping
       parse_redis_settings
       parse_postgresql_settings
@@ -444,6 +489,7 @@ module Gitlab
       # Parse ci_external_url _before_ gitlab_ci settings so that the user
       # can turn on gitlab_ci by only specifying ci_external_url
       parse_ci_external_url
+      parse_pages_external_url
       parse_mattermost_external_url
       parse_unicorn_listen_address
       parse_nginx_listen_address

files/gitlab-cookbooks/gitlab/recipes/gitlab-rails.rb

@@ -45,6 +45,14 @@ directory File.dirname(gitlab_rails_log_dir) do
   recursive true
 end
 
+# We create shared_path with 711 allowing other users to enter into the directories
+# It's needed, because by default the shared_path is used to store pages which are served by gitlab-www:gitlab-www
+directory node['gitlab']['gitlab-rails']['shared_path'] do
+  owner gitlab_user
+  mode '0711'
+  recursive true
+end
+
 [
   gitlab_rails_etc_dir,
   gitlab_rails_static_etc_dir,
@@ -52,7 +60,6 @@ end
   gitlab_rails_tmp_dir,
   gitlab_ci_builds_dir,
   node['gitlab']['gitlab-rails']['gitlab_repository_downloads_path'],
-  node['gitlab']['gitlab-rails']['shared_path'],
   node['gitlab']['gitlab-rails']['artifacts_path'],
   node['gitlab']['gitlab-rails']['lfs_storage_path'],
   gitlab_rails_log_dir
@@ -90,6 +97,13 @@ directory gitlab_rails_public_uploads_dir do
   recursive true
 end
 
+directory node['gitlab']['gitlab-rails']['pages_path'] do
+  owner gitlab_user
+  group account_helper.web_server_group
+  mode '0750'
+  recursive true
+end
+
 template File.join(gitlab_rails_static_etc_dir, "gitlab-rails-rc")
 
 dependent_services = []

files/gitlab-cookbooks/gitlab/recipes/nginx.rb

@@ -42,6 +42,7 @@ end
 nginx_config = File.join(nginx_conf_dir, "nginx.conf")
 
 gitlab_rails_http_conf = File.join(nginx_conf_dir, "gitlab-http.conf")
+gitlab_pages_http_conf = File.join(nginx_conf_dir, "gitlab-pages.conf")
 gitlab_mattermost_http_conf = File.join(nginx_conf_dir, "gitlab-mattermost-http.conf")
 
 # If the service is enabled, check if we are using internal nginx
@@ -57,6 +58,12 @@ gitlab_mattermost_enabled = if node['gitlab']['mattermost']['enable']
                               false
                             end
 
+gitlab_pages_enabled = if node['gitlab']['gitlab-rails']['pages_enabled']
+                         node['gitlab']['pages-nginx']['enable']
+                       else
+                         false
+                       end
+
 # Include the config file for gitlab-rails in nginx.conf later
 nginx_vars = node['gitlab']['nginx'].to_hash.merge({
                :gitlab_http_config => gitlab_rails_enabled ? gitlab_rails_http_conf : nil
@@ -67,6 +74,11 @@ nginx_vars = nginx_vars.to_hash.merge!({
                :gitlab_mattermost_http_config => gitlab_mattermost_enabled ? gitlab_mattermost_http_conf : nil
              })
 
+# Include the config file for gitlab mattermost in nginx.conf later
+nginx_vars = nginx_vars.to_hash.merge!({
+                                         :gitlab_pages_http_config => gitlab_pages_enabled ? gitlab_pages_http_conf : nil
+                                       })
+
 if nginx_vars['listen_https'].nil?
   nginx_vars['https'] = node['gitlab']['gitlab-rails']['gitlab_https']
 else
@@ -92,6 +104,28 @@ template gitlab_rails_http_conf do
   action gitlab_rails_enabled ? :create : :delete
 end
 
+pages_nginx_vars = node['gitlab']['pages-nginx'].to_hash
+
+if pages_nginx_vars['listen_https'].nil?
+  pages_nginx_vars['https'] = node['gitlab']['gitlab-rails']['pages_https']
+else
+  pages_nginx_vars['https'] = pages_nginx_vars['listen_https']
+end
+
+template gitlab_pages_http_conf do
+  source "nginx-gitlab-pages-http.conf.erb"
+  owner "root"
+  group "root"
+  mode "0644"
+  variables(pages_nginx_vars.merge(
+    {
+      pages_path: node['gitlab']['gitlab-rails']['pages_path'],
+    }
+  ))
+  notifies :restart, 'service[nginx]' if OmnibusHelper.should_notify?("nginx")
+  action gitlab_pages_enabled ? :create : :delete
+end
+
 mattermost_nginx_vars = node['gitlab']['mattermost-nginx'].to_hash
 
 if mattermost_nginx_vars['listen_https'].nil?

files/gitlab-cookbooks/gitlab/templates/default/gitlab.yml.erb

@@ -120,6 +120,14 @@ production: &base
     # The location where LFS objects are stored (default: shared/lfs-objects).
     storage_path: <%= @lfs_storage_path %>
 
+  ## GitLab Pages
+  pages:
+    enabled: <%= @pages_enabled %>
+    path: <%= @pages_path %>
+    host: <%= @pages_host %>
+    port: <%= @pages_port %>
+    https: <%= @pages_https %>
+
   ## Gravatar
   ## For Libravatar see: http://doc.gitlab.com/ce/customization/libravatar.html
   gravatar:

files/gitlab-cookbooks/gitlab/templates/default/nginx-gitlab-pages-http.conf.erb

@@ -0,0 +1,78 @@
+# This file is managed by gitlab-ctl. Manual changes will be
+# erased! To change the contents below, edit /etc/gitlab/gitlab.rb
+# and run `sudo gitlab-ctl reconfigure`.
+
+## Lines starting with two hashes (##) are comments with information.
+## Lines starting with one hash (#) are configuration parameters that can be uncommented.
+##
+###################################
+##         configuration         ##
+###################################
+
+<% if @https && @redirect_http_to_https %>
+## Redirects all HTTP traffic to the HTTPS host
+server {
+<% @listen_addresses.each do |listen_address| %>
+  listen <%= listen_address %>:<%= @redirect_http_to_https_port %>;
+<% end %>
+  server_name  ~^(?<group>.*)\.<%= @fqdn_regex %>$;
+  server_tokens off; ## Don't show the nginx version number, a security best practice
+  return 301 https://$http_host:<%= @port %>$request_uri;
+  access_log  <%= @log_directory %>/gitlab_pages_access.log gitlab_access;
+  error_log   <%= @log_directory %>/gitlab_pages_error.log;
+}
+<% end %>
+
+server {
+<% @listen_addresses.each do |listen_address| %>
+  listen <%= listen_address %>:<%= @listen_port %><% if @https %> ssl spdy<% end %>;
+<% end %>
+  server_name  ~^(?<group>.*)\.<%= @fqdn_regex %>$;
+  server_tokens off; ## Don't show the nginx version number, a security best practice
+  root <%= @pages_path %>/${group};
+
+  ## Disable symlink traversal
+  disable_symlinks on;
+
+  <% if @https %>
+  ## Strong SSL Security
+  ## https://raymii.org/s/tutorials/Strong_SSL_Security_On_nginx.html & https://cipherli.st/
+  ssl on;
+  ssl_certificate <%= @ssl_certificate %>;
+  ssl_certificate_key <%= @ssl_certificate_key %>;
+  <% if @ssl_client_certificate %>
+  ssl_client_certificate <%= @ssl_client_certificate%>;
+  <% end %>
+
+  # GitLab needs backwards compatible ciphers to retain compatibility with Java IDEs
+  ssl_ciphers '<%= @ssl_ciphers %>';
+  ssl_protocols  <%= @ssl_protocols %>;
+  ssl_prefer_server_ciphers <%= @ssl_prefer_server_ciphers %>;
+  ssl_session_cache  <%= @ssl_session_cache %>;
+  ssl_session_timeout  <%= @ssl_session_timeout %>;
+
+  <% if @ssl_dhparam %>
+  ssl_dhparam <%= @ssl_dhparam %>;
+  <% end %>
+  <% end %>
+
+  ## Individual nginx logs for this GitLab vhost
+  access_log  <%= @log_directory %>/gitlab_pages_access.log gitlab_access;
+  error_log   <%= @log_directory %>/gitlab_pages_error.log;
+
+  # 1. Try to get /project/ to => shared/pages/${group}/public/ or index.html
+  # 2. Try to get / to => shared/pages/${group}/${host}/public/ or index.html
+  location ~ ^/([^/]*)(/.*)?$ {
+    try_files "/$1/public$2"
+              "/$1/public$2/index.html"
+              "/${host}/public/${uri}"
+              "/${host}/public/${uri}/index.html"
+              =404;
+  }
+
+  # Define custom error pages
+  error_page 403 /403.html;
+  error_page 404 /404.html;
+
+  <%= @custom_gitlab_server_config %>
+}

files/gitlab-cookbooks/gitlab/templates/default/nginx.conf.erb

@@ -40,6 +40,10 @@ http {
   include <%= @gitlab_ci_http_config %>;
   <% end %>
 
+  <% if @gitlab_pages_http_config %>
+  include <%= @gitlab_pages_http_config %>;
+  <% end %>
+
   <% if @gitlab_mattermost_http_config %>
   include <%= @gitlab_mattermost_http_config %>;
   <% end %>

files/gitlab-cookbooks/gitlab/templates/default/sv-sidekiq-run.erb

@@ -15,6 +15,7 @@ exec chpst -e /opt/gitlab/etc/<%= @options[:rails_app] %>/env -P \
     -q gitlab_shell \
     -q incoming_email \
     -q common \
+    -q pages \
     -q default \
     -e <%= node['gitlab'][@options[:rails_app]]['environment'] %> \
     -r /opt/gitlab/embedded/service/<%= @options[:rails_app] %> \