diff --git a/gitlab-ci-config/dev-gitlab-org.yml b/gitlab-ci-config/dev-gitlab-org.yml index 2d4ddca2475c42a119ce540aad79c6c5804ca436_Z2l0bGFiLWNpLWNvbmZpZy9kZXYtZ2l0bGFiLW9yZy55bWw=..43df64705bf656af84c2988a187687f811ad6aa9_Z2l0bGFiLWNpLWNvbmZpZy9kZXYtZ2l0bGFiLW9yZy55bWw= 100644 --- a/gitlab-ci-config/dev-gitlab-org.yml +++ b/gitlab-ci-config/dev-gitlab-org.yml @@ -1045,6 +1045,23 @@ - if: '$PIPELINE_TYPE =~ /_(RC|TAG)_BUILD_PIPELINE$/' when: manual +manifest-upload: + stage: metrics + image: "${BUILDER_IMAGE_REGISTRY}/ubuntu_18.04:${BUILDER_IMAGE_REVISION}" + script: + - bundle exec rake manifest:upload + tags: + - docker-builder + cache: !reference [.tag-cache] + needs: + - Ubuntu-18.04 + allow_failure: true + rules: + - !reference [.default_rules, rules] + - !reference [.skip_release_jobs, rules] + - if: '$PIPELINE_TYPE =~ /_(RC|TAG)_BUILD_PIPELINE$/' + when: manual + RAT-Tag: stage: slow_jobs image: "dev.gitlab.org:5005/cookbooks/gitlab-omnibus-builder/ubuntu_16.04:${BUILDER_IMAGE_REVISION}" diff --git a/gitlab-ci-config/gitlab-com.yml b/gitlab-ci-config/gitlab-com.yml index 2d4ddca2475c42a119ce540aad79c6c5804ca436_Z2l0bGFiLWNpLWNvbmZpZy9naXRsYWItY29tLnltbA==..43df64705bf656af84c2988a187687f811ad6aa9_Z2l0bGFiLWNpLWNvbmZpZy9naXRsYWItY29tLnltbA== 100644 --- a/gitlab-ci-config/gitlab-com.yml +++ b/gitlab-ci-config/gitlab-com.yml @@ -722,6 +722,8 @@ - yard script: - bundle exec rake license:generate_pages + # Remove "|| true" after we confirm this works. + - bundle exec rake manifest:generate_pages || true - mv ${LICENSE_S3_BUCKET} public - cp support/webpages/* public - cp -R yard/* public diff --git a/lib/gitlab/manifest/base.rb b/lib/gitlab/manifest/base.rb new file mode 100644 index 0000000000000000000000000000000000000000..43df64705bf656af84c2988a187687f811ad6aa9_bGliL2dpdGxhYi9tYW5pZmVzdC9iYXNlLnJi --- /dev/null +++ b/lib/gitlab/manifest/base.rb @@ -0,0 +1,23 @@ +require_relative "../util.rb" + +module Manifest + class Base + def s3_sync(source, destination) + system( + { + 'AWS_ACCESS_KEY_ID' => Gitlab::Util.get_env('LICENSE_AWS_ACCESS_KEY_ID'), + 'AWS_SECRET_ACCESS_KEY' => Gitlab::Util.get_env('LICENSE_AWS_SECRET_ACCESS_KEY') + }, + *%W[aws s3 sync --region #{@manifests_bucket_region} #{source} #{destination}] + ) + end + + def s3_fetch + s3_sync("s3://#{@manifests_bucket_path}", @manifests_local_path) + end + + def s3_upload + s3_sync(@manifests_local_path, "s3://#{@manifests_bucket_path}") + end + end +end diff --git a/lib/gitlab/manifest/collector.rb b/lib/gitlab/manifest/collector.rb new file mode 100644 index 0000000000000000000000000000000000000000..43df64705bf656af84c2988a187687f811ad6aa9_bGliL2dpdGxhYi9tYW5pZmVzdC9jb2xsZWN0b3IucmI= --- /dev/null +++ b/lib/gitlab/manifest/collector.rb @@ -0,0 +1,37 @@ +require 'erb' +require 'fileutils' +require_relative '../build/info.rb' +require_relative "../util.rb" +require_relative 'base.rb' + +module Manifest + # Roots through the LICENSE_S3_BUCKET'and creates an html page with links to + # all version manifests. The gitlab.com CI "pages" job is later used to upload + # the results (along with the bucket itself) to the web-site. + class Collector < Base + def initialize + @manifests_bucket = Gitlab::Util.get_env('LICENSE_S3_BUCKET') + @manifests_bucket_path = File.join(@manifests_bucket, 'gitlab-manifests') + @manifests_local_path = File.join(File.absolute_path(@manifests_bucket), 'gitlab-manifests') + @manifests_bucket_region = "eu-west-1" + @json_data = nil + end + + def execute + s3_fetch + generate_index_page + end + + def generate_index_page + template = File.read(File.join(File.dirname(__FILE__), "manifests.html.erb")) + output_text = ERB.new(template).result(binding) + + output_path = File.join(@manifests_local_path, "manifests.html") + FileUtils.mkdir_p(File.dirname(output_path)) + + File.open(output_path, "w") do |f| + f.write(output_text) + end + end + end +end diff --git a/lib/gitlab/manifest/manifests.html.erb b/lib/gitlab/manifest/manifests.html.erb new file mode 100644 index 0000000000000000000000000000000000000000..43df64705bf656af84c2988a187687f811ad6aa9_bGliL2dpdGxhYi9tYW5pZmVzdC9tYW5pZmVzdHMuaHRtbC5lcmI= --- /dev/null +++ b/lib/gitlab/manifest/manifests.html.erb @@ -0,0 +1,39 @@ +<!doctype html> +<head> + <link rel="stylesheet" media="all" href="style.css" /> + <link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.7/css/bootstrap.css"> + <script src="https://ajax.googleapis.com/ajax/libs/jquery/3.3.1/jquery.min.js"></script> + <script src="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.7/js/bootstrap.min.js"></script> + <script src="script.js"></script> + <title>Version manifests | omnibus-gitlab </title> +</head> +<body> + <% editions = {"gitlab-ce" => "GitLab Community Edition", "gitlab-ee" => "GitLab Enterprise Edition"} %> + <h1>Version manifests for the omnibus-gitlab package</h1> + <div style="margin-left:10%;margin-right:10%;width:80%;"> + <div class="row"> + <% editions.each do |slug, edition| %> + <div class="col-md-6"> + <h2><%= edition %></h2> + <table class="table table-bordered"> + <% Dir.glob(File.join(@manifests_local_path, slug, "*")).sort_by { |version_path| Gem::Version.new(File.basename(version_path)) }.reverse.each do |minor_version| %> + <tr class="header clickable"> + <td class="deps"><h3><%= File.basename(minor_version) %></h3></td> + </tr> + <% Dir.glob(File.join(minor_version, "*.version-manifest.json")).sort_by { |version_path| Gem::Version.new(File.basename(version_path).gsub(/-(ce|ee).*$/, "")) }.reverse.each do |manifest_file| %> + <% url = "#{slug}/#{File.basename(minor_version)}/#{File.basename(manifest_file).gsub('+', '%2B')}" %> + <% title = File.basename(manifest_file).gsub(".json", "") %> + <tr> + <td> + <a href='<%= url %>'><%= title %></a> + </td> + </tr> + <% end %> + <% end %> + </table> + </div> + <% end %> + </div> + </div> +</body> +</html> diff --git a/lib/gitlab/manifest/uploader.rb b/lib/gitlab/manifest/uploader.rb new file mode 100644 index 0000000000000000000000000000000000000000..43df64705bf656af84c2988a187687f811ad6aa9_bGliL2dpdGxhYi9tYW5pZmVzdC91cGxvYWRlci5yYg== --- /dev/null +++ b/lib/gitlab/manifest/uploader.rb @@ -0,0 +1,82 @@ +# Regenerate the versions-manifest.json file setting ALTERNATIVE_SOURCES to true +# so we get the public versions of the repositories. This file is then published +# at https://gitlab-org.gitlab.io/omnibus-gitlab/gitlab-manifests/manifests.html +# using the CI "pages" job. + +require 'omnibus' +require 'yaml' +require 'gitlab/version' +require_relative 'base.rb' + +module Manifest + # Creates manifest for one edition based on the value of the "ee" environment + # variable. + class Generator + attr_reader :manifest_dir, :manifest_filename + + def initialize + @manifest_dir = File.join(Omnibus::Config.base_dir, 'manifests') + @manifest_filename = 'version-manifest.json' + end + + def manifest_path + File.join(@manifest_dir, @manifest_filename) + end + + def create_manifest + FileUtils.mkdir_p(manifest_dir) unless File.directory?(manifest_dir) + Gitlab::Util.set_env('ALTERNATIVE_SOURCES', 'true') # We always want public sources. + project = Omnibus::Project.load('gitlab') # project is a singleton. + project.json_manifest_path manifest_path + project.write_json_manifest + end + end + + class Uploader < Base + # Generates and uploads the manifest to the LICENSE_S3_BUCKET. As the name + # implies, this bucket is shared with the licenses. + def initialize + @generator = Generator.new + @edition = Build::Info.edition + @package = Build::Info.package + @manifests_bucket = Gitlab::Util.get_env('LICENSE_S3_BUCKET') + @manifests_bucket_path = File.join(@manifests_bucket, 'gitlab-manifests') + @manifests_local_path = File.join(File.absolute_path(@manifests_bucket), 'gitlab-manifests') + @current_version = Build::Info.release_version.split("+")[0] + @current_minor_version = @current_version.split(".")[0, 2].join(".") + @manifests_bucket_region = "eu-west-1" + @json_data = nil + end + + def execute + @generator.create_manifest + s3_fetch + copy_manifests + s3_upload + end + + def copy_manifests + # The bucket has the following structure + # + # gitlab-manifests + # |-- gitlab-ce + # | |-- 11.0 + # | | |-- 11.0.0-ce.version-manifest.json + # | | |-- 11.0.1-ce.version-manifest.json + # | `-- 11.1 + # | |-- 11.1.0-ce.version-manifest.json + # | |-- 11.1.1-ce.version-manifest.json + # `-- gitlab-ee + # |-- 11.0 + # | |-- 11.0.0-ee.version-manifest.json + # | |-- 11.0.1-ee.version-manifest.json + # `-- 11.1 + # |-- 11.1.0-ee.version-manifest.json + # |-- 11.1.1.ee.version-manifest.json + # + dest_dir = File.join(@manifests_local_path, @package, @current_minor_version) + FileUtils.mkdir_p(dest_dir) + FileUtils.cp(@generator.manifest_path, File.join(dest_dir, "#{@current_version}-#{@edition}.#{@generator.manifest_filename}")) + end + end +end diff --git a/lib/gitlab/tasks/manifest.rake b/lib/gitlab/tasks/manifest.rake new file mode 100644 index 0000000000000000000000000000000000000000..43df64705bf656af84c2988a187687f811ad6aa9_bGliL2dpdGxhYi90YXNrcy9tYW5pZmVzdC5yYWtl --- /dev/null +++ b/lib/gitlab/tasks/manifest.rake @@ -0,0 +1,19 @@ +require_relative '../manifest/uploader.rb' +require_relative '../manifest/collector.rb' + +namespace :manifest do + desc "Generate version manifest file of current release and push to AWS bucket" + task :upload do + Gitlab::Util.section('manifest:upload') do + # This is done on Ubuntu 18.04 non-rc tag pipeline only + Manifest::Uploader.new.execute unless Build::Check.is_rc_tag? + end + end + + desc "Collect all manifest files and generate index page" + task :generate_pages do + Gitlab::Util.section('manifest:generate_pages') do + Manifest::Collector.new.execute + end + end +end diff --git a/lib/gitlab/version.rb b/lib/gitlab/version.rb index 2d4ddca2475c42a119ce540aad79c6c5804ca436_bGliL2dpdGxhYi92ZXJzaW9uLnJi..43df64705bf656af84c2988a187687f811ad6aa9_bGliL2dpdGxhYi92ZXJzaW9uLnJi 100644 --- a/lib/gitlab/version.rb +++ b/lib/gitlab/version.rb @@ -9,6 +9,8 @@ ALTERNATIVE_SOURCE = 'alternative'.freeze SECURITY_SOURCE = 'security'.freeze + CUSTOM_SOURCES_FILENAME = '.custom_sources.yml'.freeze + COMPONENTS_ENV_VARS = { 'gitlab-rails' => 'GITLAB_VERSION', 'gitlab-rails-ee' => 'GITLAB_VERSION', @@ -141,7 +143,7 @@ end def read_remote_from_file(channel = nil) - filepath = File.expand_path(".custom_sources.yml", @project_root) + filepath = File.expand_path(CUSTOM_SOURCES_FILENAME, @project_root) sources = YAML.load_file(filepath)[@software] channel ||= ::Gitlab::Version.sources_channel