Commit 14185a69 authored by Krasimir Angelov's avatar Krasimir Angelov Committed by Fabio Pitino
Browse files

Add project level git depth setting

Introduce default_git_depth in project's CI/CD settings and set it to
50. Use it if there is no GIT_DEPTH variable specified. Apply this
default only to newly created projects and keep it nil for old ones
in order to not break pipelines that rely on non-shallow clones.

default_git_depth can be updated from CI/CD Settings in the UI, must be
either nil or integer between 0 and 1000 (incl).

Inherit default_git_depth from the origin project when forking projects.

MR pipelines are run on a MR ref (refs/merge-requests/:iid/merge) and it
contains unique commit (i.e. merge commit) which doesn't exist in the
other branch/tags refs. We need to add it cause otherwise it may break
pipelines for old projects that have already enabled Pipelines for merge
results and have git depth 0.

Document new default_git_depth project CI/CD setting
parent ce2a83c1f1cb
...@@ -50,7 +50,8 @@ def update_params ...@@ -50,7 +50,8 @@ def update_params
:runners_token, :builds_enabled, :build_allow_git_fetch, :runners_token, :builds_enabled, :build_allow_git_fetch,
:build_timeout_human_readable, :build_coverage_regex, :public_builds, :build_timeout_human_readable, :build_coverage_regex, :public_builds,
:auto_cancel_pending_pipelines, :ci_config_path, :auto_cancel_pending_pipelines, :ci_config_path,
auto_devops_attributes: [:id, :domain, :enabled, :deploy_strategy] auto_devops_attributes: [:id, :domain, :enabled, :deploy_strategy],
ci_cd_settings_attributes: [:default_git_depth]
) )
end end
......
...@@ -292,6 +292,7 @@ class Project < ApplicationRecord ...@@ -292,6 +292,7 @@ class Project < ApplicationRecord
accepts_nested_attributes_for :project_feature, update_only: true accepts_nested_attributes_for :project_feature, update_only: true
accepts_nested_attributes_for :import_data accepts_nested_attributes_for :import_data
accepts_nested_attributes_for :auto_devops, update_only: true accepts_nested_attributes_for :auto_devops, update_only: true
accepts_nested_attributes_for :ci_cd_settings, update_only: true
accepts_nested_attributes_for :remote_mirrors, accepts_nested_attributes_for :remote_mirrors,
allow_destroy: true, allow_destroy: true,
...@@ -310,6 +311,7 @@ class Project < ApplicationRecord ...@@ -310,6 +311,7 @@ class Project < ApplicationRecord
delegate :root_ancestor, to: :namespace, allow_nil: true delegate :root_ancestor, to: :namespace, allow_nil: true
delegate :last_pipeline, to: :commit, allow_nil: true delegate :last_pipeline, to: :commit, allow_nil: true
delegate :external_dashboard_url, to: :metrics_setting, allow_nil: true, prefix: true delegate :external_dashboard_url, to: :metrics_setting, allow_nil: true, prefix: true
delegate :default_git_depth, :default_git_depth=, to: :ci_cd_settings
# Validations # Validations
validates :creator, presence: true, on: :create validates :creator, presence: true, on: :create
......
...@@ -6,6 +6,18 @@ class ProjectCiCdSetting < ApplicationRecord ...@@ -6,6 +6,18 @@ class ProjectCiCdSetting < ApplicationRecord
# The version of the schema that first introduced this model/table. # The version of the schema that first introduced this model/table.
MINIMUM_SCHEMA_VERSION = 20180403035759 MINIMUM_SCHEMA_VERSION = 20180403035759
DEFAULT_GIT_DEPTH = 50
before_create :set_default_git_depth, unless: :default_git_depth?
validates :default_git_depth,
numericality: {
only_integer: true,
greater_than_or_equal_to: 0,
less_than_or_equal_to: 1000
},
allow_nil: true
def self.available? def self.available?
@available ||= @available ||=
ActiveRecord::Migrator.current_version >= MINIMUM_SCHEMA_VERSION ActiveRecord::Migrator.current_version >= MINIMUM_SCHEMA_VERSION
...@@ -15,4 +27,10 @@ def self.reset_column_information ...@@ -15,4 +27,10 @@ def self.reset_column_information
@available = nil @available = nil
super super
end end
private
def set_default_git_depth
self.default_git_depth = DEFAULT_GIT_DEPTH
end
end end
...@@ -25,14 +25,16 @@ def ref_type ...@@ -25,14 +25,16 @@ def ref_type
end end
def git_depth def git_depth
strong_memoize(:git_depth) do if git_depth_variable
git_depth = variables&.find { |variable| variable[:key] == 'GIT_DEPTH' }&.dig(:value) git_depth_variable[:value]
git_depth.to_i else
end project.default_git_depth
end.to_i
end end
def refspecs def refspecs
specs = [] specs = []
specs << refspec_for_merge_request_ref if merge_request_ref?
if git_depth > 0 if git_depth > 0
specs << refspec_for_branch(ref) if branch? || legacy_detached_merge_request_pipeline? specs << refspec_for_branch(ref) if branch? || legacy_detached_merge_request_pipeline?
...@@ -42,8 +44,6 @@ def refspecs ...@@ -42,8 +44,6 @@ def refspecs
specs << refspec_for_tag specs << refspec_for_tag
end end
specs << refspec_for_merge_request_ref if merge_request_ref?
specs specs
end end
...@@ -89,5 +89,11 @@ def refspec_for_tag(ref = '*') ...@@ -89,5 +89,11 @@ def refspec_for_tag(ref = '*')
def refspec_for_merge_request_ref def refspec_for_merge_request_ref
"+#{ref}:#{ref}" "+#{ref}:#{ref}"
end end
def git_depth_variable
strong_memoize(:git_depth_variable) do
variables&.find { |variable| variable[:key] == 'GIT_DEPTH' }
end
end
end end
end end
...@@ -36,18 +36,19 @@ def link_existing_project(fork_to_project) ...@@ -36,18 +36,19 @@ def link_existing_project(fork_to_project)
def fork_new_project def fork_new_project
new_params = { new_params = {
visibility_level: allowed_visibility_level, visibility_level: allowed_visibility_level,
description: @project.description, description: @project.description,
name: target_name, name: target_name,
path: target_path, path: target_path,
shared_runners_enabled: @project.shared_runners_enabled, shared_runners_enabled: @project.shared_runners_enabled,
namespace_id: target_namespace.id, namespace_id: target_namespace.id,
fork_network: fork_network, fork_network: fork_network,
ci_cd_settings_attributes: { default_git_depth: @project.default_git_depth },
# We need to assign the fork network membership after the project has # We need to assign the fork network membership after the project has
# been instantiated to avoid ActiveRecord trying to create it when # been instantiated to avoid ActiveRecord trying to create it when
# initializing the project, as that would cause a foreign key constraint # initializing the project, as that would cause a foreign key constraint
# exception. # exception.
relations_block: -> (project) { build_fork_network_member(project) } relations_block: -> (project) { build_fork_network_member(project) }
} }
if @project.avatar.present? && @project.avatar.image? if @project.avatar.present? && @project.avatar.image?
...@@ -56,7 +57,10 @@ def fork_new_project ...@@ -56,7 +57,10 @@ def fork_new_project
new_params.merge!(@project.object_pool_params) new_params.merge!(@project.object_pool_params)
new_project = CreateService.new(current_user, new_params).execute new_project = CreateService.new(current_user, new_params).execute do |p|
p.build_ci_cd_settings(default_git_depth: @project.default_git_depth)
end
return new_project unless new_project.persisted? return new_project unless new_project.persisted?
# Set the forked_from_project relation after saving to avoid having to # Set the forked_from_project relation after saving to avoid having to
......
...@@ -24,6 +24,14 @@ ...@@ -24,6 +24,14 @@
%span.descr %span.descr
= _("Faster as it re-uses the project workspace (falling back to clone if it doesn't exist)") = _("Faster as it re-uses the project workspace (falling back to clone if it doesn't exist)")
%hr
.form-group
= f.fields_for :ci_cd_settings_attributes, @project.ci_cd_settings do |form|
= form.label :default_git_depth, _('Git shallow clone'), class: 'label-bold'
= form.number_field :default_git_depth, { class: 'form-control', min: 0, max: 1000 }
%p.form-text.text-muted
= _('The number of changes to be fetched from GitLab when cloning a repository. This can speed up Pipelines execution. Keep empty or set to 0 to disable shallow clone by default and make GitLab CI fetch all branches and tags each time.')
%hr %hr
.form-group .form-group
= f.label :build_timeout_human_readable, _('Timeout'), class: 'label-bold' = f.label :build_timeout_human_readable, _('Timeout'), class: 'label-bold'
......
---
title: Add project level git depth CI/CD setting
merge_request: 28919
author:
type: added
# frozen_string_literal: true
class AddDefaultGitDepthToCiCdSettings < ActiveRecord::Migration[5.1]
include Gitlab::Database::MigrationHelpers
DOWNTIME = false
def change
add_column :project_ci_cd_settings, :default_git_depth, :integer
end
end
...@@ -1644,6 +1644,7 @@ ...@@ -1644,6 +1644,7 @@
t.boolean "group_runners_enabled", default: true, null: false t.boolean "group_runners_enabled", default: true, null: false
t.boolean "merge_pipelines_enabled" t.boolean "merge_pipelines_enabled"
t.boolean "merge_trains_enabled", default: false, null: false t.boolean "merge_trains_enabled", default: false, null: false
t.integer "default_git_depth"
t.index ["project_id"], name: "index_project_ci_cd_settings_on_project_id", unique: true, using: :btree t.index ["project_id"], name: "index_project_ci_cd_settings_on_project_id", unique: true, using: :btree
end end
......
...@@ -20,6 +20,22 @@ There are two options. Using: ...@@ -20,6 +20,22 @@ There are two options. Using:
The default Git strategy can be overridden by the [GIT_STRATEGY variable](../../../ci/yaml/README.md#git-strategy) The default Git strategy can be overridden by the [GIT_STRATEGY variable](../../../ci/yaml/README.md#git-strategy)
in `.gitlab-ci.yml`. in `.gitlab-ci.yml`.
## Git shallow clone
> [Introduced](https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/28919) in GitLab 12.0.
NOTE: **Note**: As of GitLab 12.0, newly created projects will automaticallyl have a default
`git depth` value of `50`.
It is possible to limit the number of changes that GitLab CI/CD will fetch when cloning
a repository. Setting a limit to `git depth` can speed up Pipelines execution. Maximum
allowed value is `1000`.
To disable shallow clone and make GitLab CI/CD fetch all branches and tags each time,
keep the value empty or set to `0`.
This value can also be [overridden by `GIT_DEPTH`](../../../ci/large_repositories/index.md#shallow-cloning) variable in `.gitlab-ci.yml` file.
## Timeout ## Timeout
Timeout defines the maximum amount of time in minutes that a job is able run. Timeout defines the maximum amount of time in minutes that a job is able run.
......
...@@ -4698,6 +4698,9 @@ msgstr "" ...@@ -4698,6 +4698,9 @@ msgstr ""
msgid "Git revision" msgid "Git revision"
msgstr "" msgstr ""
msgid "Git shallow clone"
msgstr ""
msgid "Git strategy for pipelines" msgid "Git strategy for pipelines"
msgstr "" msgstr ""
...@@ -9978,6 +9981,9 @@ msgstr "" ...@@ -9978,6 +9981,9 @@ msgstr ""
msgid "The name %{entryName} is already taken in this directory." msgid "The name %{entryName} is already taken in this directory."
msgstr "" msgstr ""
msgid "The number of changes to be fetched from GitLab when cloning a repository. This can speed up Pipelines execution. Keep empty or set to 0 to disable shallow clone by default and make GitLab CI fetch all branches and tags each time."
msgstr ""
msgid "The number of times an upload record could not find its file" msgid "The number of times an upload record could not find its file"
msgstr "" msgstr ""
......
...@@ -200,6 +200,21 @@ ...@@ -200,6 +200,21 @@
expect(response).to redirect_to(namespace_project_settings_ci_cd_path) expect(response).to redirect_to(namespace_project_settings_ci_cd_path)
end end
end end
context 'when default_git_depth is not specified' do
let(:params) { { ci_cd_settings_attributes: { default_git_depth: 10 } } }
before do
project.ci_cd_settings.update!(default_git_depth: nil)
end
it 'set specified git depth' do
subject
project.reload
expect(project.default_git_depth).to eq(10)
end
end
end end
end end
end end
...@@ -3,6 +3,12 @@ ...@@ -3,6 +3,12 @@
# rubocop:disable RSpec/FactoriesInMigrationSpecs # rubocop:disable RSpec/FactoriesInMigrationSpecs
describe Gitlab::BackgroundMigration::DeleteDiffFiles, :migration, :sidekiq, schema: 20180619121030 do describe Gitlab::BackgroundMigration::DeleteDiffFiles, :migration, :sidekiq, schema: 20180619121030 do
describe '#perform' do describe '#perform' do
before do
# This migration was created before we introduced ProjectCiCdSetting#default_git_depth
allow_any_instance_of(ProjectCiCdSetting).to receive(:default_git_depth?).and_return(true)
allow_any_instance_of(ProjectCiCdSetting).to receive(:default_git_depth).and_return(nil)
end
context 'when diff files can be deleted' do context 'when diff files can be deleted' do
let(:merge_request) { create(:merge_request, :merged) } let(:merge_request) { create(:merge_request, :merged) }
let!(:merge_request_diff) do let!(:merge_request_diff) do
......
...@@ -9,6 +9,9 @@ ...@@ -9,6 +9,9 @@
before do before do
# This migration was created before we introduced metadata configs # This migration was created before we introduced metadata configs
stub_feature_flags(ci_build_metadata_config: false) stub_feature_flags(ci_build_metadata_config: false)
# This migration was created before we introduced ProjectCiCdSetting#default_git_depth
allow_any_instance_of(ProjectCiCdSetting).to receive(:default_git_depth?).and_return(true)
allow_any_instance_of(ProjectCiCdSetting).to receive(:default_git_depth).and_return(nil)
end end
let!(:internal_pipeline) { create(:ci_pipeline, source: :web) } let!(:internal_pipeline) { create(:ci_pipeline, source: :web) }
......
...@@ -10,6 +10,12 @@ ...@@ -10,6 +10,12 @@
let(:project) { create(:project) } # rubocop:disable RSpec/FactoriesInMigrationSpecs let(:project) { create(:project) } # rubocop:disable RSpec/FactoriesInMigrationSpecs
let(:label) { create_label } let(:label) { create_label }
before do
# This migration was created before we introduced ProjectCiCdSetting#default_git_depth
allow_any_instance_of(ProjectCiCdSetting).to receive(:default_git_depth?).and_return(true)
allow_any_instance_of(ProjectCiCdSetting).to receive(:default_git_depth).and_return(nil)
end
context 'add foreign key on label_id' do context 'add foreign key on label_id' do
let!(:label_link_with_label) { create_label_link(label_id: label.id) } let!(:label_link_with_label) { create_label_link(label_id: label.id) }
let!(:label_link_without_label) { create_label_link(label_id: nil) } let!(:label_link_without_label) { create_label_link(label_id: nil) }
......
...@@ -21,4 +21,26 @@ ...@@ -21,4 +21,26 @@
2.times { described_class.available? } 2.times { described_class.available? }
end end
end end
describe 'validations' do
it { is_expected.to validate_numericality_of(:default_git_depth).only_integer.is_greater_than_or_equal_to(0).is_less_than_or_equal_to(1000).allow_nil }
end
describe '#default_git_depth' do
let(:default_value) { described_class::DEFAULT_GIT_DEPTH }
it 'sets default value for new records' do
project = create(:project)
expect(project.ci_cd_settings.default_git_depth).to eq(default_value)
end
it 'does not set default value if present' do
project = build(:project)
project.build_ci_cd_settings(default_git_depth: 42)
project.save!
expect(project.reload.ci_cd_settings.default_git_depth).to eq(42)
end
end
end end
...@@ -119,23 +119,23 @@ ...@@ -119,23 +119,23 @@
end end
describe '#git_depth' do describe '#git_depth' do
subject { presenter.git_depth }
let(:build) { create(:ci_build) } let(:build) { create(:ci_build) }
it 'returns the correct git depth' do subject(:git_depth) { presenter.git_depth }
is_expected.to eq(0)
end
context 'when GIT_DEPTH variable is specified' do context 'when GIT_DEPTH variable is specified' do
before do before do
create(:ci_pipeline_variable, key: 'GIT_DEPTH', value: 1, pipeline: build.pipeline) create(:ci_pipeline_variable, key: 'GIT_DEPTH', value: 1, pipeline: build.pipeline)
end end
it 'returns the correct git depth' do it 'returns its value' do
is_expected.to eq(1) expect(git_depth).to eq(1)
end end
end end
it 'defaults to git depth setting for the project' do
expect(git_depth).to eq(build.project.default_git_depth)
end
end end
describe '#refspecs' do describe '#refspecs' do
...@@ -144,24 +144,24 @@ ...@@ -144,24 +144,24 @@
let(:build) { create(:ci_build) } let(:build) { create(:ci_build) }
it 'returns the correct refspecs' do it 'returns the correct refspecs' do
is_expected.to contain_exactly('+refs/tags/*:refs/tags/*', is_expected.to contain_exactly("+refs/heads/#{build.ref}:refs/remotes/origin/#{build.ref}")
'+refs/heads/*:refs/remotes/origin/*')
end end
context 'when GIT_DEPTH variable is specified' do context 'when ref is tag' do
before do let(:build) { create(:ci_build, :tag) }
create(:ci_pipeline_variable, key: 'GIT_DEPTH', value: 1, pipeline: build.pipeline)
end
it 'returns the correct refspecs' do it 'returns the correct refspecs' do
is_expected.to contain_exactly("+refs/heads/#{build.ref}:refs/remotes/origin/#{build.ref}") is_expected.to contain_exactly("+refs/tags/#{build.ref}:refs/tags/#{build.ref}")
end end
context 'when ref is tag' do context 'when GIT_DEPTH is zero' do
let(:build) { create(:ci_build, :tag) } before do
create(:ci_pipeline_variable, key: 'GIT_DEPTH', value: 0, pipeline: build.pipeline)
end
it 'returns the correct refspecs' do it 'returns the correct refspecs' do
is_expected.to contain_exactly("+refs/tags/#{build.ref}:refs/tags/#{build.ref}") is_expected.to contain_exactly('+refs/tags/*:refs/tags/*',
'+refs/heads/*:refs/remotes/origin/*')
end end
end end
end end
...@@ -173,17 +173,27 @@ ...@@ -173,17 +173,27 @@
it 'returns the correct refspecs' do it 'returns the correct refspecs' do
is_expected is_expected
.to contain_exactly('+refs/heads/*:refs/remotes/origin/*', .to contain_exactly('+refs/merge-requests/1/head:refs/merge-requests/1/head')
'+refs/tags/*:refs/tags/*', end
'+refs/merge-requests/1/head:refs/merge-requests/1/head')
context 'when GIT_DEPTH is zero' do
before do
create(:ci_pipeline_variable, key: 'GIT_DEPTH', value: 0, pipeline: build.pipeline)
end
it 'returns the correct refspecs' do
is_expected
.to contain_exactly('+refs/merge-requests/1/head:refs/merge-requests/1/head',
'+refs/heads/*:refs/remotes/origin/*',
'+refs/tags/*:refs/tags/*')
end
end end
context 'when pipeline is legacy detached merge request pipeline' do context 'when pipeline is legacy detached merge request pipeline' do
let(:merge_request) { create(:merge_request, :with_legacy_detached_merge_request_pipeline) } let(:merge_request) { create(:merge_request, :with_legacy_detached_merge_request_pipeline) }
it 'returns the correct refspecs' do it 'returns the correct refspecs' do
is_expected.to contain_exactly('+refs/tags/*:refs/tags/*', is_expected.to contain_exactly("+refs/heads/#{build.ref}:refs/remotes/origin/#{build.ref}")
'+refs/heads/*:refs/remotes/origin/*')
end end
end end
end end
......
...@@ -444,8 +444,8 @@ ...@@ -444,8 +444,8 @@
'sha' => job.sha, 'sha' => job.sha,
'before_sha' => job.before_sha, 'before_sha' => job.before_sha,
'ref_type' => 'branch', 'ref_type' => 'branch',
'refspecs' => %w[+refs/heads/*:refs/remotes/origin/* +refs/tags/*:refs/tags/*], 'refspecs' => ["+refs/heads/#{job.ref}:refs/remotes/origin/#{job.ref}"],
'depth' => 0 } 'depth' => project.default_git_depth }
end end
let(:expected_steps) do let(:expected_steps) do
...@@ -531,7 +531,11 @@ ...@@ -531,7 +531,11 @@
end end
end end
context 'when GIT_DEPTH is not specified' do context 'when GIT_DEPTH is not specified and there is no default git depth for the project' do
before do
project.update!(default_git_depth: nil)
end
it 'specifies refspecs' do it 'specifies refspecs' do
request_job request_job
...@@ -587,7 +591,11 @@ ...@@ -587,7 +591,11 @@
end end
end end
context 'when GIT_DEPTH is not specified' do context 'when GIT_DEPTH is not specified and there is no default git depth for the project' do
before do
project.update!(default_git_depth: nil)
end
it 'specifies refspecs' do it 'specifies refspecs' do
request_job request_job
......
...@@ -145,6 +145,14 @@ ...@@ -145,6 +145,14 @@
end end
end end
context "CI/CD settings" do
it "inherits default_git_depth from the origin project" do
@from_project.update(default_git_depth: 42)
@to_project = fork_project(@from_project, @to_user