Commit 14185a69 authored by Krasimir Angelov's avatar Krasimir Angelov Committed by Fabio Pitino
Browse files

Add project level git depth setting

Introduce default_git_depth in project's CI/CD settings and set it to
50. Use it if there is no GIT_DEPTH variable specified. Apply this
default only to newly created projects and keep it nil for old ones
in order to not break pipelines that rely on non-shallow clones.

default_git_depth can be updated from CI/CD Settings in the UI, must be
either nil or integer between 0 and 1000 (incl).

Inherit default_git_depth from the origin project when forking projects.

MR pipelines are run on a MR ref (refs/merge-requests/:iid/merge) and it
contains unique commit (i.e. merge commit) which doesn't exist in the
other branch/tags refs. We need to add it cause otherwise it may break
pipelines for old projects that have already enabled Pipelines for merge
results and have git depth 0.

Document new default_git_depth project CI/CD setting
parent ce2a83c1f1cb
......@@ -50,7 +50,8 @@ def update_params
:runners_token, :builds_enabled, :build_allow_git_fetch,
:build_timeout_human_readable, :build_coverage_regex, :public_builds,
:auto_cancel_pending_pipelines, :ci_config_path,
auto_devops_attributes: [:id, :domain, :enabled, :deploy_strategy]
auto_devops_attributes: [:id, :domain, :enabled, :deploy_strategy],
ci_cd_settings_attributes: [:default_git_depth]
)
end
......
......@@ -292,6 +292,7 @@ class Project < ApplicationRecord
accepts_nested_attributes_for :project_feature, update_only: true
accepts_nested_attributes_for :import_data
accepts_nested_attributes_for :auto_devops, update_only: true
accepts_nested_attributes_for :ci_cd_settings, update_only: true
accepts_nested_attributes_for :remote_mirrors,
allow_destroy: true,
......@@ -310,6 +311,7 @@ class Project < ApplicationRecord
delegate :root_ancestor, to: :namespace, allow_nil: true
delegate :last_pipeline, to: :commit, allow_nil: true
delegate :external_dashboard_url, to: :metrics_setting, allow_nil: true, prefix: true
delegate :default_git_depth, :default_git_depth=, to: :ci_cd_settings
# Validations
validates :creator, presence: true, on: :create
......
......@@ -6,6 +6,18 @@ class ProjectCiCdSetting < ApplicationRecord
# The version of the schema that first introduced this model/table.
MINIMUM_SCHEMA_VERSION = 20180403035759
DEFAULT_GIT_DEPTH = 50
before_create :set_default_git_depth, unless: :default_git_depth?
validates :default_git_depth,
numericality: {
only_integer: true,
greater_than_or_equal_to: 0,
less_than_or_equal_to: 1000
},
allow_nil: true
def self.available?
@available ||=
ActiveRecord::Migrator.current_version >= MINIMUM_SCHEMA_VERSION
......@@ -15,4 +27,10 @@ def self.reset_column_information
@available = nil
super
end
private
def set_default_git_depth
self.default_git_depth = DEFAULT_GIT_DEPTH
end
end
......@@ -25,14 +25,16 @@ def ref_type
end
def git_depth
strong_memoize(:git_depth) do
git_depth = variables&.find { |variable| variable[:key] == 'GIT_DEPTH' }&.dig(:value)
git_depth.to_i
end
if git_depth_variable
git_depth_variable[:value]
else
project.default_git_depth
end.to_i
end
def refspecs
specs = []
specs << refspec_for_merge_request_ref if merge_request_ref?
if git_depth > 0
specs << refspec_for_branch(ref) if branch? || legacy_detached_merge_request_pipeline?
......@@ -42,8 +44,6 @@ def refspecs
specs << refspec_for_tag
end
specs << refspec_for_merge_request_ref if merge_request_ref?
specs
end
......@@ -89,5 +89,11 @@ def refspec_for_tag(ref = '*')
def refspec_for_merge_request_ref
"+#{ref}:#{ref}"
end
def git_depth_variable
strong_memoize(:git_depth_variable) do
variables&.find { |variable| variable[:key] == 'GIT_DEPTH' }
end
end
end
end
......@@ -36,18 +36,19 @@ def link_existing_project(fork_to_project)
def fork_new_project
new_params = {
visibility_level: allowed_visibility_level,
description: @project.description,
name: target_name,
path: target_path,
shared_runners_enabled: @project.shared_runners_enabled,
namespace_id: target_namespace.id,
fork_network: fork_network,
visibility_level: allowed_visibility_level,
description: @project.description,
name: target_name,
path: target_path,
shared_runners_enabled: @project.shared_runners_enabled,
namespace_id: target_namespace.id,
fork_network: fork_network,
ci_cd_settings_attributes: { default_git_depth: @project.default_git_depth },
# We need to assign the fork network membership after the project has
# been instantiated to avoid ActiveRecord trying to create it when
# initializing the project, as that would cause a foreign key constraint
# exception.
relations_block: -> (project) { build_fork_network_member(project) }
relations_block: -> (project) { build_fork_network_member(project) }
}
if @project.avatar.present? && @project.avatar.image?
......@@ -56,7 +57,10 @@ def fork_new_project
new_params.merge!(@project.object_pool_params)
new_project = CreateService.new(current_user, new_params).execute
new_project = CreateService.new(current_user, new_params).execute do |p|
p.build_ci_cd_settings(default_git_depth: @project.default_git_depth)
end
return new_project unless new_project.persisted?
# Set the forked_from_project relation after saving to avoid having to
......
......@@ -24,6 +24,14 @@
%span.descr
= _("Faster as it re-uses the project workspace (falling back to clone if it doesn't exist)")
%hr
.form-group
= f.fields_for :ci_cd_settings_attributes, @project.ci_cd_settings do |form|
= form.label :default_git_depth, _('Git shallow clone'), class: 'label-bold'
= form.number_field :default_git_depth, { class: 'form-control', min: 0, max: 1000 }
%p.form-text.text-muted
= _('The number of changes to be fetched from GitLab when cloning a repository. This can speed up Pipelines execution. Keep empty or set to 0 to disable shallow clone by default and make GitLab CI fetch all branches and tags each time.')
%hr
.form-group
= f.label :build_timeout_human_readable, _('Timeout'), class: 'label-bold'
......
---
title: Add project level git depth CI/CD setting
merge_request: 28919
author:
type: added
# frozen_string_literal: true
class AddDefaultGitDepthToCiCdSettings < ActiveRecord::Migration[5.1]
include Gitlab::Database::MigrationHelpers
DOWNTIME = false
def change
add_column :project_ci_cd_settings, :default_git_depth, :integer
end
end
......@@ -1644,6 +1644,7 @@
t.boolean "group_runners_enabled", default: true, null: false
t.boolean "merge_pipelines_enabled"
t.boolean "merge_trains_enabled", default: false, null: false
t.integer "default_git_depth"
t.index ["project_id"], name: "index_project_ci_cd_settings_on_project_id", unique: true, using: :btree
end
......
......@@ -20,6 +20,22 @@ There are two options. Using:
The default Git strategy can be overridden by the [GIT_STRATEGY variable](../../../ci/yaml/README.md#git-strategy)
in `.gitlab-ci.yml`.
## Git shallow clone
> [Introduced](https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/28919) in GitLab 12.0.
NOTE: **Note**: As of GitLab 12.0, newly created projects will automaticallyl have a default
`git depth` value of `50`.
It is possible to limit the number of changes that GitLab CI/CD will fetch when cloning
a repository. Setting a limit to `git depth` can speed up Pipelines execution. Maximum
allowed value is `1000`.
To disable shallow clone and make GitLab CI/CD fetch all branches and tags each time,
keep the value empty or set to `0`.
This value can also be [overridden by `GIT_DEPTH`](../../../ci/large_repositories/index.md#shallow-cloning) variable in `.gitlab-ci.yml` file.
## Timeout
Timeout defines the maximum amount of time in minutes that a job is able run.
......
......@@ -4698,6 +4698,9 @@ msgstr ""
msgid "Git revision"
msgstr ""
msgid "Git shallow clone"
msgstr ""
msgid "Git strategy for pipelines"
msgstr ""
......@@ -9978,6 +9981,9 @@ msgstr ""
msgid "The name %{entryName} is already taken in this directory."
msgstr ""
msgid "The number of changes to be fetched from GitLab when cloning a repository. This can speed up Pipelines execution. Keep empty or set to 0 to disable shallow clone by default and make GitLab CI fetch all branches and tags each time."
msgstr ""
msgid "The number of times an upload record could not find its file"
msgstr ""
......
......@@ -200,6 +200,21 @@
expect(response).to redirect_to(namespace_project_settings_ci_cd_path)
end
end
context 'when default_git_depth is not specified' do
let(:params) { { ci_cd_settings_attributes: { default_git_depth: 10 } } }
before do
project.ci_cd_settings.update!(default_git_depth: nil)
end
it 'set specified git depth' do
subject
project.reload
expect(project.default_git_depth).to eq(10)
end
end
end
end
end
......@@ -3,6 +3,12 @@
# rubocop:disable RSpec/FactoriesInMigrationSpecs
describe Gitlab::BackgroundMigration::DeleteDiffFiles, :migration, :sidekiq, schema: 20180619121030 do
describe '#perform' do
before do
# This migration was created before we introduced ProjectCiCdSetting#default_git_depth
allow_any_instance_of(ProjectCiCdSetting).to receive(:default_git_depth?).and_return(true)
allow_any_instance_of(ProjectCiCdSetting).to receive(:default_git_depth).and_return(nil)
end
context 'when diff files can be deleted' do
let(:merge_request) { create(:merge_request, :merged) }
let!(:merge_request_diff) do
......
......@@ -9,6 +9,9 @@
before do
# This migration was created before we introduced metadata configs
stub_feature_flags(ci_build_metadata_config: false)
# This migration was created before we introduced ProjectCiCdSetting#default_git_depth
allow_any_instance_of(ProjectCiCdSetting).to receive(:default_git_depth?).and_return(true)
allow_any_instance_of(ProjectCiCdSetting).to receive(:default_git_depth).and_return(nil)
end
let!(:internal_pipeline) { create(:ci_pipeline, source: :web) }
......
......@@ -10,6 +10,12 @@
let(:project) { create(:project) } # rubocop:disable RSpec/FactoriesInMigrationSpecs
let(:label) { create_label }
before do
# This migration was created before we introduced ProjectCiCdSetting#default_git_depth
allow_any_instance_of(ProjectCiCdSetting).to receive(:default_git_depth?).and_return(true)
allow_any_instance_of(ProjectCiCdSetting).to receive(:default_git_depth).and_return(nil)
end
context 'add foreign key on label_id' do
let!(:label_link_with_label) { create_label_link(label_id: label.id) }
let!(:label_link_without_label) { create_label_link(label_id: nil) }
......
......@@ -21,4 +21,26 @@
2.times { described_class.available? }
end
end
describe 'validations' do
it { is_expected.to validate_numericality_of(:default_git_depth).only_integer.is_greater_than_or_equal_to(0).is_less_than_or_equal_to(1000).allow_nil }
end
describe '#default_git_depth' do
let(:default_value) { described_class::DEFAULT_GIT_DEPTH }
it 'sets default value for new records' do
project = create(:project)
expect(project.ci_cd_settings.default_git_depth).to eq(default_value)
end
it 'does not set default value if present' do
project = build(:project)
project.build_ci_cd_settings(default_git_depth: 42)
project.save!
expect(project.reload.ci_cd_settings.default_git_depth).to eq(42)
end
end
end
......@@ -119,23 +119,23 @@
end
describe '#git_depth' do
subject { presenter.git_depth }
let(:build) { create(:ci_build) }
it 'returns the correct git depth' do
is_expected.to eq(0)
end
subject(:git_depth) { presenter.git_depth }
context 'when GIT_DEPTH variable is specified' do
before do
create(:ci_pipeline_variable, key: 'GIT_DEPTH', value: 1, pipeline: build.pipeline)
end
it 'returns the correct git depth' do
is_expected.to eq(1)
it 'returns its value' do
expect(git_depth).to eq(1)
end
end
it 'defaults to git depth setting for the project' do
expect(git_depth).to eq(build.project.default_git_depth)
end
end
describe '#refspecs' do
......@@ -144,24 +144,24 @@
let(:build) { create(:ci_build) }
it 'returns the correct refspecs' do
is_expected.to contain_exactly('+refs/tags/*:refs/tags/*',
'+refs/heads/*:refs/remotes/origin/*')
is_expected.to contain_exactly("+refs/heads/#{build.ref}:refs/remotes/origin/#{build.ref}")
end
context 'when GIT_DEPTH variable is specified' do
before do
create(:ci_pipeline_variable, key: 'GIT_DEPTH', value: 1, pipeline: build.pipeline)
end
context 'when ref is tag' do
let(:build) { create(:ci_build, :tag) }
it 'returns the correct refspecs' do
is_expected.to contain_exactly("+refs/heads/#{build.ref}:refs/remotes/origin/#{build.ref}")
is_expected.to contain_exactly("+refs/tags/#{build.ref}:refs/tags/#{build.ref}")
end
context 'when ref is tag' do
let(:build) { create(:ci_build, :tag) }
context 'when GIT_DEPTH is zero' do
before do
create(:ci_pipeline_variable, key: 'GIT_DEPTH', value: 0, pipeline: build.pipeline)
end
it 'returns the correct refspecs' do
is_expected.to contain_exactly("+refs/tags/#{build.ref}:refs/tags/#{build.ref}")
is_expected.to contain_exactly('+refs/tags/*:refs/tags/*',
'+refs/heads/*:refs/remotes/origin/*')
end
end
end
......@@ -173,17 +173,27 @@
it 'returns the correct refspecs' do
is_expected
.to contain_exactly('+refs/heads/*:refs/remotes/origin/*',
'+refs/tags/*:refs/tags/*',
'+refs/merge-requests/1/head:refs/merge-requests/1/head')
.to contain_exactly('+refs/merge-requests/1/head:refs/merge-requests/1/head')
end
context 'when GIT_DEPTH is zero' do
before do
create(:ci_pipeline_variable, key: 'GIT_DEPTH', value: 0, pipeline: build.pipeline)
end
it 'returns the correct refspecs' do
is_expected
.to contain_exactly('+refs/merge-requests/1/head:refs/merge-requests/1/head',
'+refs/heads/*:refs/remotes/origin/*',
'+refs/tags/*:refs/tags/*')
end
end
context 'when pipeline is legacy detached merge request pipeline' do
let(:merge_request) { create(:merge_request, :with_legacy_detached_merge_request_pipeline) }
it 'returns the correct refspecs' do
is_expected.to contain_exactly('+refs/tags/*:refs/tags/*',
'+refs/heads/*:refs/remotes/origin/*')
is_expected.to contain_exactly("+refs/heads/#{build.ref}:refs/remotes/origin/#{build.ref}")
end
end
end
......
......@@ -444,8 +444,8 @@
'sha' => job.sha,
'before_sha' => job.before_sha,
'ref_type' => 'branch',
'refspecs' => %w[+refs/heads/*:refs/remotes/origin/* +refs/tags/*:refs/tags/*],
'depth' => 0 }
'refspecs' => ["+refs/heads/#{job.ref}:refs/remotes/origin/#{job.ref}"],
'depth' => project.default_git_depth }
end
let(:expected_steps) do
......@@ -531,7 +531,11 @@
end
end
context 'when GIT_DEPTH is not specified' do
context 'when GIT_DEPTH is not specified and there is no default git depth for the project' do
before do
project.update!(default_git_depth: nil)
end
it 'specifies refspecs' do
request_job
......@@ -587,7 +591,11 @@
end
end
context 'when GIT_DEPTH is not specified' do
context 'when GIT_DEPTH is not specified and there is no default git depth for the project' do
before do
project.update!(default_git_depth: nil)
end
it 'specifies refspecs' do
request_job
......
......@@ -145,6 +145,14 @@
end
end
context "CI/CD settings" do
it "inherits default_git_depth from the origin project" do
@from_project.update(default_git_depth: 42)
@to_project = fork_project(@from_project, @to_user)
expect(@to_project.default_git_depth).to eq(42)
end
end
context "when project has restricted visibility level" do
context "and only one visibility level is restricted" do
before do
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment