Commit 22b539ed authored by GitLab Bot's avatar GitLab Bot
Browse files

Add latest changes from gitlab-org/gitlab@master

parent aca5a274eed5
---
title: Fix inline rendering of videos for uploads with uppercase file extensions
merge_request: 17581
author:
type: fixed
---
title: 'Project list: Align star icons'
merge_request: 17833
author:
type: other
---
title: Fix project imports for pipelines for merge requests
merge_request: 17799
author:
type: fixed
# frozen_string_literal: true
class AddSuccessfullPagesDeployPartialIndexOnCiBuilds < ActiveRecord::Migration[5.2]
include Gitlab::Database::MigrationHelpers
DOWNTIME = false
disable_ddl_transaction!
INDEX_NAME = 'index_ci_builds_on_project_id_for_successfull_pages_deploy'
def up
add_concurrent_index(
:ci_builds, :project_id,
name: INDEX_NAME,
where: "type='GenericCommitStatus' AND stage='deploy' AND name='pages:deploy' AND status = 'success'"
)
end
def down
remove_concurrent_index_by_name :ci_builds, INDEX_NAME
end
end
......@@ -643,6 +643,7 @@
t.index ["name"], name: "index_ci_builds_on_name_for_security_products_values", where: "((name)::text = ANY (ARRAY[('container_scanning'::character varying)::text, ('dast'::character varying)::text, ('dependency_scanning'::character varying)::text, ('license_management'::character varying)::text, ('sast'::character varying)::text]))"
t.index ["project_id", "id"], name: "index_ci_builds_on_project_id_and_id"
t.index ["project_id", "status"], name: "index_ci_builds_project_id_and_status_for_live_jobs_partial2", where: "(((type)::text = 'Ci::Build'::text) AND ((status)::text = ANY (ARRAY[('running'::character varying)::text, ('pending'::character varying)::text, ('created'::character varying)::text])))"
t.index ["project_id"], name: "index_ci_builds_on_project_id_for_successfull_pages_deploy", where: "(((type)::text = 'GenericCommitStatus'::text) AND ((stage)::text = 'deploy'::text) AND ((name)::text = 'pages:deploy'::text) AND ((status)::text = 'success'::text))"
t.index ["protected"], name: "index_ci_builds_on_protected"
t.index ["queued_at"], name: "index_ci_builds_on_queued_at"
t.index ["runner_id"], name: "index_ci_builds_on_runner_id"
......
......@@ -36,6 +36,9 @@ will be enabled:
### Protected paths throttle
NOTE: **Note:** Omnibus GitLab protected paths throttle is deprecated and is scheduled for removal in
GitLab 13.0. Please refer to [Migrate settings from GitLab 12.3 and earlier](../user/admin_area/settings/protected_paths.md#migrate-settings-from-gitlab-123-and-earlier).
GitLab responds with HTTP status code `429` to POST requests at protected paths
that exceed 10 requests per minute per IP address.
......@@ -124,6 +127,9 @@ The following settings can be configured:
**Installations from source**
NOTE: **Note:** Rack Attack initializer was temporarily renamed to `rack_attack_new`, to
support backwards compatibility with the one [Omnibus initializer](https://docs.gitlab.com/omnibus/settings/configuration.html#setting-up-paths-to-be-protected-by-rack-attack). It'll be renamed back to `rack_attack.rb` once Omnibus throttle is removed. Please see the [GitLab issue](https://gitlab.com/gitlab-org/gitlab/issues/29952) for more information.
These settings can be found in `config/initializers/rack_attack.rb`. If you are
missing `config/initializers/rack_attack.rb`, the following steps need to be
taken in order to enable protection for your GitLab instance:
......
......@@ -20,6 +20,7 @@ include:
- [Visibility and access controls](visibility_and_access_controls.md)
- [User and IP rate limits](user_and_ip_rate_limits.md)
- [Custom templates repository](instance_template_repository.md) **(PREMIUM)**
- [Protected paths](protected_paths.md) **(CORE ONLY)**
NOTE: **Note:**
You can change the [first day of the week](../../profile/preferences.md) for the entire GitLab instance
......
---
type: reference
---
# Protected paths **(CORE ONLY)**
GitLab protects the following paths with Rack Attack by default:
```
'/users/password',
'/users/sign_in',
'/api/#{API::API.version}/session.json',
'/api/#{API::API.version}/session',
'/users',
'/users/confirmation',
'/unsubscribes/',
'/import/github/personal_access_token'
```
GitLab responds with HTTP status code `429` to POST requests at protected paths
that exceed 10 requests per minute per IP address.
This header is included in responses to blocked requests:
```
Retry-After: 60
```
For example, the following are limited to a maximum 10 requests per minute:
- User sign-in
- User sign-up (if enabled)
- User password reset
After 10 requests, the client must wait 60 seconds before it can
try again.
## Configure using GitLab UI
> Introduced in [GitLab 12.4](https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/31246).
Throttling of protected paths is enabled by default and can be disabled or
customized on **Admin > Network > Protected Paths**, along with these options:
- Maximum number of requests per period per user.
- Rate limit period in seconds.
- Paths to be protected.
![protected-paths](img/protected_paths.png)
Requests over the rate limit are logged into `auth.log`.
## Migrate settings from GitLab 12.3 and earlier
Omnibus GitLab protected paths throttle is deprecated and is scheduled for removal in
GitLab 13.0. Please see the [GitLab issue](https://gitlab.com/gitlab-org/gitlab/issues/29952) and the [Omnibus GitLab issue](https://gitlab.com/gitlab-org/omnibus-gitlab/issues/4688) for more information.
NOTE: **Note:** If Omnibus settings are present, applications settings will be automatically ignored to avoid generating multiple requests blocks.
To migrate from Omnibus GitLab 12.3 and earlier settings:
1. Disable the Protected Paths throttle from Omnibus, by changing `rack_attack_enabled` value to `false` on [`rack_attack.rb.erb`](https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/files/gitlab-cookbooks/gitlab/templates/default/rack_attack.rb.erb#L18):
```ruby
rack_attack_enabled = false
```
1. Customize and enable your protected paths settings by following [Configure using GitLab UI](#configure-using-gitlab-ui) section.
1. Restart GitLab:
```bash
sudo gitlab-ctl restart
```
That's it. Protected paths throttle are now managed by GitLab admin settings.
......@@ -316,9 +316,7 @@ This header is included in responses to blocked requests:
Retry-After: 60
```
Source:
- Search for `rate_limit_requests_per_period`, `rate_limit_period`, and `rack_attack_protected_paths` in [GitLab.com's current Rails app settings](https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/files/gitlab-cookbooks/gitlab/attributes/default.rb).
See [Protected Paths](../admin_area/settings/protected_paths.md) for more details.
#### Git and container registry failed authentication ban
......
......@@ -6,7 +6,7 @@ module Helpers
# against the graphql API. Helper code for the graphql server implementation
# should be in app/graphql/ or lib/gitlab/graphql/
module GraphqlHelpers
def conditionally_graphql!(fallback:, query:, context: {}, transform: nil)
def run_graphql!(query:, context: {}, transform: nil)
result = GitlabSchema.execute(query, context: context)
if transform
......
......@@ -19,11 +19,10 @@ class Version < Grape::API
detail 'This feature was introduced in GitLab 8.13.'
end
get '/version' do
conditionally_graphql!(
run_graphql!(
query: METADATA_QUERY,
context: { current_user: current_user },
transform: ->(result) { result.dig('data', 'metadata') },
fallback: -> { { version: Gitlab::VERSION, revision: Gitlab.revision } }
transform: ->(result) { result.dig('data', 'metadata') }
)
end
end
......
......@@ -8,8 +8,8 @@ module Filter
# a "Download" link in the case the video cannot be played.
class VideoLinkFilter < HTML::Pipeline::Filter
def call
doc.xpath('descendant-or-self::img[not(ancestor::a)]').each do |el|
el.replace(video_node(doc, el)) if has_video_extension?(el)
doc.xpath(query).each do |el|
el.replace(video_node(doc, el))
end
doc
......@@ -17,10 +17,22 @@ def call
private
def has_video_extension?(element)
src_attr = context[:asset_proxy_enabled] ? 'data-canonical-src' : 'src'
def query
@query ||= begin
src_query = UploaderHelper::SAFE_VIDEO_EXT.map do |ext|
"'.#{ext}' = substring(@src, string-length(@src) - #{ext.size})"
end
element.attr(src_attr).downcase.end_with?(*UploaderHelper::SAFE_VIDEO_EXT)
if context[:asset_proxy_enabled].present?
src_query.concat(
UploaderHelper::SAFE_VIDEO_EXT.map do |ext|
"'.#{ext}' = substring(@data-canonical-src, string-length(@data-canonical-src) - #{ext.size})"
end
)
end
"descendant-or-self::img[not(ancestor::a) and (#{src_query.join(' or ')})]"
end
end
def video_node(doc, element)
......
# frozen_string_literal: true
module Gitlab
module BackgroundMigration
# Class that will insert record into project_pages_metadata
# for each existing project
class MigratePagesMetadata
def perform(start_id, stop_id)
perform_on_relation(Project.where(id: start_id..stop_id))
end
def perform_on_relation(relation)
successful_pages_deploy = <<~SQL
SELECT TRUE
FROM ci_builds
WHERE ci_builds.type = 'GenericCommitStatus'
AND ci_builds.status = 'success'
AND ci_builds.stage = 'deploy'
AND ci_builds.name = 'pages:deploy'
AND ci_builds.project_id = projects.id
LIMIT 1
SQL
select_from = relation
.select("projects.id", "COALESCE((#{successful_pages_deploy}), FALSE)")
.to_sql
ActiveRecord::Base.connection_pool.with_connection do |connection|
connection.execute <<~SQL
INSERT INTO project_pages_metadata (project_id, deployed)
#{select_from}
ON CONFLICT (project_id) DO NOTHING
SQL
end
end
end
end
end
# frozen_string_literal: true
module Gitlab
module Ci
module Status
class Composite
include Gitlab::Utils::StrongMemoize
# This class accepts an array of arrays/hashes/or objects
def initialize(all_statuses, with_allow_failure: true)
unless all_statuses.respond_to?(:pluck)
raise ArgumentError, "all_statuses needs to respond to `.pluck`"
end
@status_set = Set.new
@status_key = 0
@allow_failure_key = 1 if with_allow_failure
consume_all_statuses(all_statuses)
end
# The status calculation is order dependent,
# 1. In some cases we assume that that status is exact
# if the we only have given statues,
# 2. In other cases we assume that status is of that type
# based on what statuses are no longer valid based on the
# data set that we have
def status
return if none?
strong_memoize(:status) do
if only_of?(:skipped, :ignored)
'skipped'
elsif only_of?(:success, :skipped, :success_with_warnings, :ignored)
'success'
elsif only_of?(:created, :success_with_warnings, :ignored)
'created'
elsif only_of?(:preparing, :success_with_warnings, :ignored)
'preparing'
elsif only_of?(:canceled, :success, :skipped, :success_with_warnings, :ignored)
'canceled'
elsif only_of?(:pending, :created, :skipped, :success_with_warnings, :ignored)
'pending'
elsif any_of?(:running, :pending)
'running'
elsif any_of?(:manual)
'manual'
elsif any_of?(:scheduled)
'scheduled'
elsif any_of?(:preparing)
'preparing'
elsif any_of?(:created)
'running'
else
'failed'
end
end
end
def warnings?
@status_set.include?(:success_with_warnings)
end
private
def none?
@status_set.empty?
end
def any_of?(*names)
names.any? { |name| @status_set.include?(name) }
end
def only_of?(*names)
matching = names.count { |name| @status_set.include?(name) }
matching > 0 &&
matching == @status_set.size
end
def consume_all_statuses(all_statuses)
columns = []
columns[@status_key] = :status
columns[@allow_failure_key] = :allow_failure if @allow_failure_key
all_statuses
.pluck(*columns) # rubocop: disable CodeReuse/ActiveRecord
.each(&method(:consume_status))
end
def consume_status(description)
# convert `"status"` into `["status"]`
description = Array(description)
status =
if success_with_warnings?(description)
:success_with_warnings
elsif ignored_status?(description)
:ignored
else
description[@status_key].to_sym
end
@status_set.add(status)
end
def success_with_warnings?(status)
@allow_failure_key &&
status[@allow_failure_key] &&
HasStatus::PASSED_WITH_WARNINGS_STATUSES.include?(status[@status_key])
end
def ignored_status?(status)
@allow_failure_key &&
status[@allow_failure_key] &&
HasStatus::EXCLUDE_IGNORED_STATUSES.include?(status[@status_key])
end
end
end
end
end
......@@ -52,6 +52,11 @@ def members_mapper
project: restored_project)
end
# A Hash of the imported merge request ID -> imported ID.
def merge_requests_mapping
@merge_requests_mapping ||= {}
end
# Loops through the tree of models defined in import_export.yml and
# finds them in the imported JSON so they can be instantiated and saved
# in the DB. The structure and relationships between models are guessed from
......@@ -80,10 +85,26 @@ def save_relation_hash(relation_key, relation_hash_batch)
@saved = false unless restored_project.append_or_update_attribute(relation_key, relation_hash)
save_id_mappings(relation_key, relation_hash_batch, relation_hash)
# Restore the project again, extra query that skips holding the AR objects in memory
@restored_project = Project.find(@project_id)
end
# Older, serialized CI pipeline exports may only have a
# merge_request_id and not the full hash of the merge request. To
# import these pipelines, we need to preserve the mapping between
# the old and new the merge request ID.
def save_id_mappings(relation_key, relation_hash_batch, relation_hash)
return unless relation_key == 'merge_requests'
relation_hash = Array(relation_hash)
Array(relation_hash_batch).each_with_index do |raw_data, index|
merge_requests_mapping[raw_data['id']] = relation_hash[index]['id']
end
end
# Remove project models that became group models as we found them at group level.
# This no longer required saving them at the root project level.
# For example, in the case of an existing group label that matched the title.
......@@ -222,6 +243,7 @@ def create_relation(relation_key, relation_hash_list)
relation_sym: relation_key.to_sym,
relation_hash: relation_hash,
members_mapper: members_mapper,
merge_requests_mapping: merge_requests_mapping,
user: @user,
project: @restored_project,
excluded_keys: excluded_keys_for_relation(relation_key))
......
......@@ -55,10 +55,11 @@ def self.relation_class(relation_name)
relation_name.to_s.constantize
end
def initialize(relation_sym:, relation_hash:, members_mapper:, user:, project:, excluded_keys: [])
def initialize(relation_sym:, relation_hash:, members_mapper:, merge_requests_mapping:, user:, project:, excluded_keys: [])
@relation_name = self.class.overrides[relation_sym]&.to_sym || relation_sym
@relation_hash = relation_hash.except('noteable_id')
@members_mapper = members_mapper
@merge_requests_mapping = merge_requests_mapping
@user = user
@project = project
@imported_object_retries = 0
......@@ -109,7 +110,10 @@ def setup_models
update_group_references
remove_duplicate_assignees
setup_pipeline if @relation_name == :'Ci::Pipeline'
if @relation_name == :'Ci::Pipeline'
update_merge_request_references
setup_pipeline
end
reset_tokens!
remove_encrypted_attributes!
......@@ -194,6 +198,28 @@ def update_group_references
@relation_hash['group_id'] = @project.namespace_id
end
# This code is a workaround for broken project exports that don't
# export merge requests with CI pipelines (i.e. exports that were
# generated from
# https://gitlab.com/gitlab-org/gitlab/merge_requests/17844).
# This method can be removed in GitLab 12.6.
def update_merge_request_references
# If a merge request was properly created, we don't need to fix
# up this export.
return if @relation_hash['merge_request']
merge_request_id = @relation_hash['merge_request_id']
return unless merge_request_id
new_merge_request_id = @merge_requests_mapping[merge_request_id]
return unless new_merge_request_id
@relation_hash['merge_request_id'] = new_merge_request_id
parsed_relation_hash['merge_request_id'] = new_merge_request_id
end
def reset_tokens!
return unless Gitlab::ImportExport.reset_tokens? && TOKEN_RESET_MODELS.include?(@relation_name)
......
......@@ -15,14 +15,14 @@ function retrieve_tests_metadata() {
function update_tests_metadata() {
echo "{}" > "${KNAPSACK_RSPEC_SUITE_REPORT_PATH}"
scripts/merge-reports "${KNAPSACK_RSPEC_SUITE_REPORT_PATH}" "knapsack/rspec*_pg9_*.json"
scripts/merge-reports "${KNAPSACK_RSPEC_SUITE_REPORT_PATH}" knapsack/rspec*_pg9_*.json
if [[ -n "${TESTS_METADATA_S3_BUCKET}" ]]; then
scripts/sync-reports put "${TESTS_METADATA_S3_BUCKET}" "${KNAPSACK_RSPEC_SUITE_REPORT_PATH}"
fi
rm -f "knapsack/rspec*.json"
rm -f knapsack/rspec*.json
scripts/merge-reports "${FLAKY_RSPEC_SUITE_REPORT_PATH}" "rspec_flaky/all_*.json"
scripts/merge-reports "${FLAKY_RSPEC_SUITE_REPORT_PATH}" rspec_flaky/all_*.json
export FLAKY_RSPEC_GENERATE_REPORT="1"
scripts/prune-old-flaky-specs "${FLAKY_RSPEC_SUITE_REPORT_PATH}"
......@@ -31,7 +31,7 @@ function update_tests_metadata() {
scripts/sync-reports put "${TESTS_METADATA_S3_BUCKET}" "${FLAKY_RSPEC_SUITE_REPORT_PATH}"
fi
rm -f "rspec_flaky/all_*.json" "rspec_flaky/new_*.json"
rm -f rspec_flaky/all_*.json rspec_flaky/new_*.json
scripts/insert-rspec-profiling-data
}
......
......@@ -104,20 +104,20 @@
it 'displays the failure reason' do
wait_for_all_requests
within('.builds-container') do
build_link = first('.build-job > a')
expect(build_link['data-original-title']).to eq('test - failed - (unknown failure)')
expect(page).to have_selector(
".build-job > a[data-original-title='test - failed - (unknown failure)']")
end
end
end
context 'when a failed job has been retried' do
let!(:build) { create(:ci_build, :failed, :retried, :trace_artifact, pipeline: pipeline) }
let!(:build_retried) { create(:ci_build, :failed, :retried, :trace_artifact, pipeline: pipeline) }
it 'displays the failure reason and retried label' do
wait_for_all_requests
within('.builds-container') do
build_link = first('.build-job > a')
expect(build_link['data-original-title']).to eq('test - failed - (unknown failure) (retried)')
expect(page).to have_selector(
".build-job > a[data-original-title='test - failed - (unknown failure) (retried)']")
end
end
end
......
......@@ -6175,6 +6175,8 @@
"finished_at": null,
"user_id": 9999,
"duration": null,
"source": "push",
"merge_request_id": null,
"notes": [
{
"id": 999,
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment