Merge branch 'custom_sources' into 'master'

Read custom sources from a file

Fixes gitlab-org/omnibus-gitlab#1525

See merge request !976

.custom_sources.yml

@@ -0,0 +1,14 @@
+gitlab-rails:
+  remote: "git@dev.gitlab.org:gitlab/gitlabhq.git"
+gitlab-rails-ee:
+  remote: "git@dev.gitlab.org:gitlab/gitlab-ee.git"
+gitlab-shell:
+  remote: "git@dev.gitlab.org:gitlab/gitlab-shell.git"
+gitlab-workhorse:
+  remote: "git@dev.gitlab.org:gitlab/gitlab-workhorse.git"
+gitlab-pages:
+  remote: "git@dev.gitlab.org:gitlab/gitlab-pages.git"
+config_guess:
+  remote: "git@dev.gitlab.org:omnibus-mirror/config_guess.git"
+omnibus:
+  remote: "https://dev.gitlab.org/gitlab/omnibus.git"

Gemfile

@@ -1,6 +1,10 @@
 source 'https://rubygems.org'
 
-gem 'omnibus', git: 'https://dev.gitlab.org/gitlab/omnibus.git', branch: 'omnibus-5-4-0'
+require_relative "lib/gitlab/version"
+
+omnibus_gem_version =  Gitlab::Version.new('omnibus', "omnibus-5-4-0")
+
+gem 'omnibus', git: omnibus_gem_version.remote, branch: omnibus_gem_version.print(false)
 gem 'omnibus-software', :git => 'git://github.com/opscode/omnibus-software.git', :branch => 'master'
 gem 'ohai'
 gem 'package_cloud'

config/projects/gitlab.rb

@@ -17,6 +17,7 @@
 #
 
 require "#{Omnibus::Config.project_root}/lib/gitlab/build_iteration"
+require "#{Omnibus::Config.project_root}/lib/gitlab/version"
 
 ee = system("#{Omnibus::Config.project_root}/support/is_gitlab_ee.sh")
 
@@ -60,6 +61,11 @@ else
 end
 build_iteration Gitlab::BuildIteration.new.build_iteration
 
+# Overrides for remote URLs of the software
+#
+# Original git://git.sv.gnu.org/config.git is failing intermittently
+config_guess_version =  Gitlab::Version.new('config_guess', "master")
+
 override :ruby, version: '2.3.1', source: { md5: '0d896c2e7fd54f722b399f407e48a4c6' }
 override :rubygems, version: '2.6.6'
 override :'chef-gem', version: '12.12.15'
@@ -69,7 +75,7 @@ override :liblzma, version: '5.2.2', source: { md5: '7cf6a8544a7dae8e8106fdf7add
 override :libxml2, version: '2.9.4', source: { md5: 'ae249165c173b1ff386ee8ad676815f5' }
 override :pcre, version: '8.38', source: { md5: '8a353fe1450216b6655dfcf3561716d9', url: "http://downloads.sourceforge.net/project/pcre/pcre/8.38/pcre-8.38.tar.gz" }
 override :expat, version: '2.2.0', source: { md5: '2f47841c829facb346eb6e3fab5212e2', url: "http://downloads.sourceforge.net/project/expat/expat/2.2.0/expat-2.2.0.tar.bz2" }
-override :config_guess, source: { git: "git@dev.gitlab.org:omnibus-mirror/config_guess.git" } # Original git://git.sv.gnu.org/config.git is failing intermittently
+override :config_guess, version: config_guess_version.print, source: { git: config_guess_version.remote }
 override :rsync, version: '3.1.2'
 
 # Openssh needs to be installed

config/software/gitlab-pages.rb

@@ -15,7 +15,7 @@
 # limitations under the License.
 #
 require "#{Omnibus::Config.project_root}/lib/gitlab/version"
-version = Gitlab::Version.new("GITLAB_PAGES_VERSION")
+version = Gitlab::Version.new("gitlab-pages")
 
 name "gitlab-pages"
 default_version version.print

config/software/gitlab-rails.rb

@@ -16,10 +16,16 @@
 # limitations under the License.
 #
 require "#{Omnibus::Config.project_root}/lib/gitlab/version"
-version = Gitlab::Version.new("VERSION")
+
+EE = system("#{Omnibus::Config.project_root}/support/is_gitlab_ee.sh")
+
+software_name = EE ? "gitlab-rails-ee":"gitlab-rails"
+version = Gitlab::Version.new(software_name)
 
 name "gitlab-rails"
+
 default_version version.print
+source git: version.remote
 
 combined_licenses_file = "#{install_dir}/embedded/service/gem/gitlab-gem-licenses"
 gems_directory = "#{install_dir}/embedded/service/gem/ruby/2.3.0/gems"
@@ -28,8 +34,6 @@ license "MIT"
 license_file "LICENSE"
 license_file combined_licenses_file
 
-EE = system("#{Omnibus::Config.project_root}/support/is_gitlab_ee.sh")
-
 dependency "ruby"
 dependency "bundler"
 dependency "libxml2"
@@ -48,7 +52,6 @@ if EE
   dependency "gitlab-pages"
 end
 
-source :git => version.remote
 
 build do
   env = with_standard_compiler_flags(with_embedded_path)
@@ -69,7 +72,7 @@ build do
   # This patch makes the github-markup gem use and be compatible with Python3
   # We've sent part of the changes upstream: https://github.com/github/markup/pull/919
   patch source: 'github-markup_gem-markups.patch', target: "#{gems_directory}/github-markup-1.4.0/lib/github/markups.rb"
-  patch source: 'github-markup_gem-rest2html.patch', target: "#{gems_directory}/github-markup-1.4.0/lib/github/commands/rest2html" 
+  patch source: 'github-markup_gem-rest2html.patch', target: "#{gems_directory}/github-markup-1.4.0/lib/github/commands/rest2html"
 
   # In order to precompile the assets, we need to get to a state where rake can
   # load the Rails environment.

config/software/gitlab-shell.rb

@@ -16,7 +16,7 @@
 ##
 #
 require "#{Omnibus::Config.project_root}/lib/gitlab/version"
-version = Gitlab::Version.new("GITLAB_SHELL_VERSION")
+version = Gitlab::Version.new("gitlab-shell")
 
 name "gitlab-shell"
 default_version version.print
@@ -26,7 +26,7 @@ license_file "LICENSE"
 
 dependency "ruby"
 
-source :git => version.remote
+source git: version.remote
 
 build do
   command "mkdir -p #{install_dir}/embedded/service/gitlab-shell"

config/software/gitlab-workhorse.rb

@@ -15,7 +15,7 @@
 # limitations under the License.
 #
 require "#{Omnibus::Config.project_root}/lib/gitlab/version"
-version = Gitlab::Version.new("GITLAB_WORKHORSE_VERSION")
+version = Gitlab::Version.new("gitlab-workhorse")
 
 name "gitlab-workhorse"
 default_version version.print

doc/build/README.md

@@ -41,6 +41,10 @@ code from internal dev.gitlab.org server. The internal server hosts the same
 copy of the source code available elsewhere. Alternative source location is
 necessary in case of an unforeseen circumstances.
 
+All repositories used for building the packages that are not publicly reachable,
+have their remotes listed in  `.custom_sources.yml` file in the root of this
+project.
+
 If you are using these tools to build your own packages, you will have to
 adjust them to your needs.
 

lib/gitlab/version.rb

@@ -1,16 +1,48 @@
+require 'yaml'
+
 module Gitlab
   class Version
 
-    def initialize(filename)
-      @filename = filename
-      filepath = File.expand_path(@filename, Omnibus::Config.project_root)
-      @read_version = File.read(filepath).chomp
+    def initialize(software_name, version = nil)
+      @software = software_name
+
+      @read_version = if version
+                        version
+                      else
+                        read_version_from_file
+                      end
+      @project_root = File.join(File.dirname(__dir__), '../')
+    end
+
+    def read_version_from_file
+      path_to_version_file = version_file
+      if path_to_version_file
+        filepath = File.expand_path(path_to_version_file, @project_root)
+        File.read(filepath).chomp
+      else
+        ""
+      end
     rescue Errno::ENOENT
       # Didn't find the file
       @read_version = ""
     end
 
-    def print
+    def version_file
+      case @software
+      when "gitlab-rails", "gitlab-rails-ee"
+        "VERSION"
+      when "gitlab-shell"
+        "GITLAB_SHELL_VERSION"
+      when "gitlab-workhorse"
+        "GITLAB_WORKHORSE_VERSION"
+      when "gitlab-pages"
+        "GITLAB_PAGES_VERSION"
+      else
+        nil
+      end
+    end
+
+    def print(prepend_version = true)
       if @read_version.include?('.pre') || @read_version == "master"
         "master"
       elsif @read_version.start_with?('buildfrombranch:')
@@ -18,26 +50,22 @@ module Gitlab
       elsif @read_version.empty?
         nil
       else
-        "v#{@read_version}"
+        v = "v" if prepend_version
+        [
+          v,
+          @read_version
+        ].join
       end
     end
 
     def remote
-      case @filename
-      when "VERSION"
-        if @read_version.include?('-ee')
-          "git@dev.gitlab.org:gitlab/gitlab-ee.git"
-        else
-          "git@dev.gitlab.org:gitlab/gitlabhq.git"
-        end
-      when "GITLAB_SHELL_VERSION"
-        "git@dev.gitlab.org:gitlab/gitlab-shell.git"
-      when "GITLAB_WORKHORSE_VERSION"
-        "git@dev.gitlab.org:gitlab/gitlab-workhorse.git"
-      when "GITLAB_PAGES_VERSION"
-        "https://gitlab.com/gitlab-org/gitlab-pages.git"
+      filepath = File.expand_path(".custom_sources.yml", @project_root)
+      software = YAML.load_file(filepath)[@software]
+
+      if software
+        software['remote']
       else
-        nil
+        ""
       end
     end
   end

spec/gitlab/build_iteration_spec.rb

@@ -1,4 +1,5 @@
 require_relative '../../lib/gitlab/build_iteration.rb'
+require 'chef_helper'
 
 describe Gitlab::BuildIteration do
   describe :build_iteration do

spec/gitlab/version_spec.rb

@@ -0,0 +1,59 @@
+require_relative '../../lib/gitlab/version.rb'
+require 'chef_helper'
+
+describe Gitlab::Version do
+  describe :remote do
+    subject { Gitlab::Version.new(software) }
+
+    context 'with a valid software name' do
+      let(:software) { 'gitlab-rails-ee' }
+
+      it 'returns a link from custom_sources yml' do
+        expect(subject.remote).to eq('git@dev.gitlab.org:gitlab/gitlab-ee.git')
+      end
+    end
+
+    context 'with an invalid software name' do
+      let(:software) { 'not a valid software' }
+
+      it 'outputs an empty string' do
+        expect(subject.remote).to eq('')
+      end
+    end
+  end
+
+  describe :print do
+    subject { Gitlab::Version.new(software, version) }
+
+    context 'with a valid software name and version' do
+      let(:software) { 'ruby' }
+      let(:version) { '2.3.1' }
+
+      it 'returns a link from custom_sources yml' do
+        expect(subject.print).to eq('v2.3.1')
+      end
+    end
+
+    context 'with a valid software name and no version' do
+      let(:software) { 'ruby' }
+      let(:version) { nil }
+
+      it 'outputs an empty string' do
+        expect(subject.print).to eq(nil)
+      end
+    end
+
+    context 'with a valid software name and a version' do
+      let(:software) { 'ruby' }
+      let(:version) { '2.3.1' }
+
+      it 'adds a v prefix' do
+        expect(subject.print).to eq("v2.3.1")
+      end
+
+      it 'does not add a v prefix if explicitly set' do
+        expect(subject.print(false)).to eq("2.3.1")
+      end
+    end
+  end
+end