Commit 88232b2b authored by GitLab Bot's avatar GitLab Bot
Browse files

Add latest changes from gitlab-org/gitlab@master

parent a1982684866e
......@@ -2,7 +2,7 @@
extends:
- .default-tags
- .default-retry
- .only-docs-changes
- .only:changes-docs
only:
refs:
- merge_requests
......@@ -50,7 +50,7 @@ docs lint:
- .default-tags
- .default-retry
- .default-only
- .only-docs-changes
- .only:changes-docs
image: "registry.gitlab.com/gitlab-org/gitlab-build-images:gitlab-docs-lint"
stage: test
dependencies: []
......@@ -76,7 +76,7 @@ graphql-docs-verify:
- .default-cache
- .default-only
- .default-before_script
- .only-graphql-changes
- .only:changes-graphql
variables:
SETUP_DB: "false"
stage: test
......
......@@ -12,7 +12,7 @@
- .default-only
- .default-before_script
- .assets-compile-cache
- .only-code-qa-changes
- .only:changes-code-backstage-qa
image: registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.6.3-git-2.22-chrome-73.0-node-12.x-yarn-1.16-graphicsmagick-1.3.33-docker-18.06.1
stage: test
dependencies: ["setup-test-env"]
......@@ -73,7 +73,7 @@ gitlab:assets:compile pull-cache:
- .default-only
- .default-before_script
- .assets-compile-cache
- .only-code-qa-changes
- .only:changes-code-backstage-qa
- .use-pg9
stage: prepare
script:
......@@ -128,7 +128,7 @@ compile-assets pull-cache foss:
- .default-cache
- .default-only
- .default-before_script
- .only-code-changes
- .only:changes-code-backstage
- .use-pg9
stage: test
needs: ["setup-test-env", "compile-assets pull-cache"]
......@@ -205,7 +205,7 @@ jest-foss:
- .default-retry
- .default-cache
- .default-only
- .only-code-changes
- .only:changes-code-backstage
stage: test
dependencies: []
cache:
......@@ -238,7 +238,7 @@ webpack-dev-server:
- .default-retry
- .default-cache
- .default-only
- .only-code-changes
- .only:changes-code-backstage
stage: test
needs: ["setup-test-env", "compile-assets pull-cache"]
dependencies: ["setup-test-env", "compile-assets pull-cache"]
......
......@@ -40,14 +40,64 @@
- merge_requests
- tags
.only-code-changes:
.code-patterns: &code-patterns
- ".gitlab/ci/**/*"
- ".{eslintignore,gitattributes,nvmrc,prettierrc,stylelintrc,yamllint}"
- ".{codeclimate,eslintrc,gitlab-ci,haml-lint,haml-lint_todo,rubocop,rubocop_todo,scss-lint}.yml"
- ".csscomb.json"
- "Dockerfile.assets"
- "*_VERSION"
- "Gemfile{,.lock}"
- "Rakefile"
- "{babel.config,jest.config}.js"
- "config.ru"
- "{package.json,yarn.lock}"
- "{,ee/}{app,bin,config,db,haml_lint,lib,locale,public,scripts,symbol,vendor}/**/*"
.backstage-patterns: &backstage-patterns
- "Dangerfile"
- "danger/**/*"
- "{,ee/}fixtures/**/*"
- "{,ee/}rubocop/**/*"
- "{,ee/}spec/**/*"
- "doc/README.md" # Some RSpec test rely on this file
.qa-patterns: &qa-patterns
- ".dockerignore"
- "qa/**/*"
.docs-patterns: &docs-patterns
- ".gitlab/route-map.yml"
- "doc/**/*"
- ".markdownlint.json"
.graphql-patterns: &graphql-patterns
- "{,ee/}app/graphql/**/*"
- "{,ee/}lib/gitlab/graphql/**/*"
.only:changes-code:
only:
changes: *code-patterns
.only:changes-qa:
only:
changes: *qa-patterns
.only:changes-docs:
only:
changes: *docs-patterns
.only:changes-graphql:
only:
changes: *graphql-patterns
.only:changes-code-backstage:
only:
changes:
- ".gitlab/ci/**/*"
- ".{eslintignore,gitattributes,nvmrc,prettierrc,stylelintrc,yamllint}"
- ".{codeclimate,eslintrc,gitlab-ci,haml-lint,haml-lint_todo,rubocop,rubocop_todo,scss-lint}.yml"
- ".csscomb.json"
- "Dangerfile"
- "Dockerfile.assets"
- "*_VERSION"
- "Gemfile{,.lock}"
......@@ -55,36 +105,41 @@
- "{babel.config,jest.config}.js"
- "config.ru"
- "{package.json,yarn.lock}"
- "{app,bin,config,danger,db,ee,fixtures,haml_lint,lib,locale,public,rubocop,scripts,spec,symbol,vendor}/**/*"
- "{,ee/}{app,bin,config,db,haml_lint,lib,locale,public,scripts,symbol,vendor}/**/*"
# Backstage changes
- "Dangerfile"
- "danger/**/*"
- "{,ee/}fixtures/**/*"
- "{,ee/}rubocop/**/*"
- "{,ee/}spec/**/*"
- "doc/README.md" # Some RSpec test rely on this file
.only-qa-changes:
.only:changes-code-qa:
only:
changes:
- ".gitlab/ci/**/*"
- ".{eslintignore,gitattributes,nvmrc,prettierrc,stylelintrc,yamllint}"
- ".{codeclimate,eslintrc,gitlab-ci,haml-lint,haml-lint_todo,rubocop,rubocop_todo,scss-lint}.yml"
- ".csscomb.json"
- "Dockerfile.assets"
- "*_VERSION"
- "Gemfile{,.lock}"
- "Rakefile"
- "{babel.config,jest.config}.js"
- "config.ru"
- "{package.json,yarn.lock}"
- "{,ee/}{app,bin,config,db,haml_lint,lib,locale,public,scripts,symbol,vendor}/**/*"
# QA changes
- ".dockerignore"
- "qa/**/*"
.only-docs-changes:
only:
changes:
- ".gitlab/route-map.yml"
- "doc/**/*"
- ".markdownlint.json"
.only-graphql-changes:
only:
changes:
- "{,ee/}app/graphql/**/*"
- "{,ee/}lib/gitlab/graphql/**/*"
.only-code-qa-changes:
.only:changes-code-backstage-qa:
only:
changes:
- ".gitlab/ci/**/*"
- ".{eslintignore,gitattributes,nvmrc,prettierrc,stylelintrc,yamllint}"
- ".{codeclimate,eslintrc,gitlab-ci,haml-lint,haml-lint_todo,rubocop,rubocop_todo,scss-lint}.yml"
- ".csscomb.json"
- "Dangerfile"
- "Dockerfile.assets"
- "*_VERSION"
- "Gemfile{,.lock}"
......@@ -92,8 +147,15 @@
- "{babel.config,jest.config}.js"
- "config.ru"
- "{package.json,yarn.lock}"
- "{app,bin,config,danger,db,ee,fixtures,haml_lint,lib,locale,public,rubocop,scripts,spec,symbol,vendor}/**/*"
- "{,ee/}{app,bin,config,db,haml_lint,lib,locale,public,scripts,symbol,vendor}/**/*"
# Backstage changes
- "Dangerfile"
- "danger/**/*"
- "{,ee/}fixtures/**/*"
- "{,ee/}rubocop/**/*"
- "{,ee/}spec/**/*"
- "doc/README.md" # Some RSpec test rely on this file
# QA changes
- ".dockerignore"
- "qa/**/*"
......
......@@ -5,7 +5,7 @@
- .default-cache
- .default-only
- .default-before_script
- .only-code-changes
- .only:changes-code
memory-static:
extends: .only-code-memory-job-base
......
......@@ -4,7 +4,7 @@ pages:
- .default-retry
- .default-cache
- .default-only
- .only-code-qa-changes
- .only:changes-code-backstage-qa
only:
refs:
- master
......
......@@ -3,7 +3,7 @@
- .default-tags
- .default-retry
- .default-only
- .only-code-qa-changes
- .only:changes-code-qa
stage: test
dependencies: []
cache:
......@@ -47,7 +47,7 @@ qa:selectors-foss:
package-and-qa-manual:
extends:
- .package-and-qa-base
- .only-code-changes
- .only:changes-code
except:
refs:
- master
......@@ -58,7 +58,7 @@ package-and-qa-manual:
package-and-qa:
extends:
- .package-and-qa-base
- .only-qa-changes
- .only:changes-qa
except:
refs:
- master
......@@ -69,6 +69,6 @@ package-and-qa:
schedule:package-and-qa:
extends:
- .package-and-qa-base
- .only-code-qa-changes
- .only:changes-code-qa
- .only-canonical-schedules
needs: ["build-qa-image", "gitlab:assets:compile pull-cache"]
......@@ -22,7 +22,7 @@
- .default-cache
- .default-only
- .default-before_script
- .only-code-changes
- .only:changes-code-backstage
.only-code-qa-rails-job-base:
extends:
......@@ -31,7 +31,7 @@
- .default-cache
- .default-only
- .default-before_script
- .only-code-qa-changes
- .only:changes-code-backstage-qa
setup-test-env:
extends:
......@@ -251,7 +251,7 @@ static-analysis:
downtime_check:
extends:
- .rake-exec
- .only-code-changes
- .only:changes-code-backstage
except:
refs:
- master
......
......@@ -11,7 +11,7 @@ code_quality:
extends:
- .default-retry
- .default-only
- .only-code-changes
- .only:changes-code-backstage
stage: test
image: docker:stable
allow_failure: true
......@@ -50,7 +50,7 @@ sast:
extends:
- .default-retry
- .default-only
- .only-code-changes
- .only:changes-code-backstage-qa
stage: test
image: docker:stable
variables:
......@@ -132,7 +132,7 @@ dependency_scanning:
extends:
- .default-retry
- .default-only
- .only-code-changes
- .only:changes-code-backstage-qa
stage: test
image: docker:stable
variables:
......@@ -195,7 +195,7 @@ dast:
extends:
- .default-retry
- .default-only
- .only-code-qa-changes
- .only:changes-code-qa
- .only-review
stage: qa
needs: ["review-deploy"]
......
......@@ -23,7 +23,7 @@
build-qa-image:
extends:
- .review-docker
- .only-code-qa-changes
- .only:changes-code-qa
only:
variables:
- $CI_SERVER_HOST == "gitlab.com" && $CI_PROJECT_NAMESPACE == "gitlab-org"
......@@ -40,7 +40,7 @@ build-qa-image:
- .default-tags
- .default-retry
- .default-only
- .only-code-qa-changes
- .only:changes-code-qa
stage: prepare
image: registry.gitlab.com/gitlab-org/gitlab-build-images:gitlab-charts-build-base
allow_failure: true
......@@ -66,7 +66,7 @@ manual:review-cleanup:
.review-build-cng-base:
extends:
- .default-only
- .only-code-qa-changes
- .only:changes-code-qa
- .except-deploys
image: ruby:2.6-alpine
stage: review-prepare
......@@ -95,7 +95,7 @@ schedule:review-build-cng:
- .default-tags
- .default-retry
- .default-only
- .only-code-qa-changes
- .only:changes-code-qa
- .except-deploys
stage: review
image: registry.gitlab.com/gitlab-org/gitlab-build-images:gitlab-charts-build-base
......@@ -189,7 +189,7 @@ review-cleanup-failed-deployment:
extends:
- .review-docker
- .only-review
- .only-code-qa-changes
- .only:changes-code-qa
stage: qa
allow_failure: true
variables:
......@@ -236,7 +236,7 @@ review-qa-all:
.review-performance-base:
extends:
- .review-docker
- .only-code-qa-changes
- .only:changes-code-qa
stage: qa
allow_failure: true
before_script:
......@@ -288,7 +288,7 @@ parallel-spec-reports:
extends:
- .default-tags
- .default-only
- .only-code-qa-changes
- .only:changes-code-qa
- .only-review
- .except-deploys
image: ruby:2.6-alpine
......
......@@ -6,7 +6,7 @@ cache gems:
- .default-retry
- .default-cache
- .default-before_script
- .only-code-qa-changes
- .only:changes-code-backstage-qa
stage: test
dependencies: ["setup-test-env"]
needs: ["setup-test-env"]
......@@ -29,7 +29,7 @@ cache gems:
- .default-tags
- .default-retry
- .default-only
- .only-code-changes
- .only:changes-code-backstage
dependencies: []
gitlab_git_test:
......
.tests-metadata-state:
extends:
- .default-only
- .only-code-changes
- .only:changes-code-backstage
variables:
TESTS_METADATA_S3_BUCKET: "gitlab-ce-cache"
before_script:
......@@ -48,7 +48,7 @@ flaky-examples-check:
- .default-tags
- .default-retry
- .default-only
- .only-code-changes
- .only:changes-code-backstage
image: ruby:2.6-alpine
stage: post-test
variables:
......
......@@ -26,7 +26,8 @@ export default (resolvers = {}, config = {}) => {
createUploadLink(httpOptions),
new BatchHttpLink(httpOptions),
),
cache: new InMemoryCache(config.cacheConfig),
cache: new InMemoryCache({ ...config.cacheConfig, freezeResults: true }),
resolvers,
assumeImmutableResults: true,
});
};
......@@ -23,6 +23,7 @@
.signup-heading h2 {
font-weight: $gl-font-weight-bold;
padding: 0 10px;
@include media-breakpoint-down(md) {
font-size: $gl-font-size-large;
......
......@@ -35,6 +35,10 @@ class Build < CommitStatus
refspecs: -> (build) { build.merge_request_ref? }
}.freeze
DEFAULT_RETRIES = {
scheduler_failure: 2
}.freeze
has_one :deployment, as: :deployable, class_name: 'Deployment'
has_many :trace_sections, class_name: 'Ci::BuildTraceSection'
has_many :trace_chunks, class_name: 'Ci::BuildTraceChunk', foreign_key: :build_id
......@@ -372,18 +376,25 @@ def retries_count
pipeline.builds.retried.where(name: self.name).count
end
def retries_max
normalized_retry.fetch(:max, 0)
def retry_failure?
max_allowed_retries = nil
max_allowed_retries ||= options_retry_max if retry_on_reason_or_always?
max_allowed_retries ||= DEFAULT_RETRIES.fetch(failure_reason.to_sym, 0)
max_allowed_retries > 0 && retries_count < max_allowed_retries
end
def retry_when
normalized_retry.fetch(:when, ['always'])
def options_retry_max
options_retry[:max]
end
def retry_failure?
return false if retries_max.zero? || retries_count >= retries_max
def options_retry_when
options_retry.fetch(:when, ['always'])
end
retry_when.include?('always') || retry_when.include?(failure_reason.to_s)
def retry_on_reason_or_always?
options_retry_when.include?(failure_reason.to_s) ||
options_retry_when.include?('always')
end
def latest?
......@@ -831,6 +842,13 @@ def deployment_status
:creating
end
# Consider this object to have a structural integrity problems
def doom!
update_columns(
status: :failed,
failure_reason: :data_integrity_failure)
end
private
def successful_deployment_status
......@@ -875,8 +893,8 @@ def environment_url
# format, but builds created before GitLab 11.5 and saved in database still
# have the old integer only format. This method returns the retry option
# normalized as a hash in 11.5+ format.
def normalized_retry
strong_memoize(:normalized_retry) do
def options_retry
strong_memoize(:options_retry) do
value = options&.dig(:retry)
value = value.is_a?(Integer) ? { max: value } : value.to_h
value.with_indifferent_access
......
......@@ -15,7 +15,9 @@ def self.failure_reasons
stale_schedule: 7,
job_execution_timeout: 8,
archived_failure: 9,
unmet_prerequisites: 10
unmet_prerequisites: 10,
scheduler_failure: 11,
data_integrity_failure: 12
}
end
end
......
......@@ -11,7 +11,9 @@ class CommitStatusPresenter < Gitlab::View::Presenter::Delegated
stale_schedule: 'Delayed job could not be executed by some reason, please try again',
job_execution_timeout: 'The script exceeded the maximum execution time set for the job',
archived_failure: 'The job is archived and cannot be run',
unmet_prerequisites: 'The job failed to complete prerequisite tasks'
unmet_prerequisites: 'The job failed to complete prerequisite tasks',
scheduler_failure: 'The scheduler failed to assign job to the runner, please try again or contact system administrator',
data_integrity_failure: 'There has been a structural integrity problem detected, please contact system administrator'
}.freeze
private_constant :CALLOUT_FAILURE_MESSAGES
......@@ -33,6 +35,6 @@ def recoverable?
end
def unrecoverable?
script_failure? || missing_dependency_failure? || archived_failure?
script_failure? || missing_dependency_failure? || archived_failure? || scheduler_failure? || data_integrity_failure?
end
end
......@@ -42,26 +42,16 @@ def execute(params = {})
end
builds.each do |build|
next unless runner.can_pick?(build)
begin
# In case when 2 runners try to assign the same build, second runner will be declined
# with StateMachines::InvalidTransition or StaleObjectError when doing run! or save method.
if assign_runner!(build, params)
register_success(build)
return Result.new(build, true)
end
rescue StateMachines::InvalidTransition, ActiveRecord::StaleObjectError
# We are looping to find another build that is not conflicting
# It also indicates that this build can be picked and passed to runner.
# If we don't do it, basically a bunch of runners would be competing for a build
# and thus we will generate a lot of 409. This will increase
# the number of generated requests, also will reduce significantly
# how many builds can be picked by runner in a unit of time.
# In case we hit the concurrency-access lock,
# we still have to return 409 in the end,
# to make sure that this is properly handled by runner.
result = process_build(build, params)
next unless result
if result.valid?
register_success(result.build)
return result
else
# The usage of valid: is described in
# handling of ActiveRecord::StaleObjectError
valid = false
end
end
......@@ -73,6 +63,35 @@ def execute(params = {})
private
def process_build(build, params)
return unless runner.can_pick?(build)
# In case when 2 runners try to assign the same build, second runner will be declined
# with StateMachines::InvalidTransition or StaleObjectError when doing run! or save method.
if assign_runner!(build, params)
Result.new(build, true)
end
rescue StateMachines::InvalidTransition, ActiveRecord::StaleObjectError
# We are looping to find another build that is not conflicting
# It also indicates that this build can be picked and passed to runner.
# If we don't do it, basically a bunch of runners would be competing for a build
# and thus we will generate a lot of 409. This will increase
# the number of generated requests, also will reduce significantly
# how many builds can be picked by runner in a unit of time.
# In case we hit the concurrency-access lock,
# we still have to return 409 in the end,
# to make sure that this is properly handled by runner.
Result.new(nil, false)
rescue => ex
raise ex unless Feature.enabled?(:ci_doom_build, default_enabled: true)
scheduler_failure!(build)
track_exception_for_build(ex, build)
# skip, and move to next one
nil
end
def assign_runner!(build, params)
build.runner_id = runner.id
build.runner_session_attributes = params[:session] if params[:session].present?
......@@ -96,6 +115,28 @@ def assign_runner!(build, params)
true
end
def scheduler_failure!(build)
Gitlab::OptimisticLocking.retry_lock(build, 3) do |subject|
subject.drop!(:scheduler_failure)
end
rescue => ex
build.doom!
# This requires extra exception, otherwise we would loose information
# why we cannot perform `scheduler_failure`
track_exception_for_build(ex, build)