Commit 825ca3a0 authored by GitLab Bot's avatar GitLab Bot
Browse files

Add latest changes from gitlab-org/gitlab@13-1-stable-ee

parent 1c425679c1e0
......@@ -30,7 +30,7 @@ class Projects::BlobController < Projects::ApplicationController
before_action :set_last_commit_sha, only: [:edit, :update]
before_action only: :show do
push_frontend_feature_flag(:code_navigation, @project)
push_frontend_feature_flag(:code_navigation, @project, default_enabled: true)
push_frontend_feature_flag(:suggest_pipeline) if experiment_enabled?(:suggest_pipeline)
end
......
......@@ -29,7 +29,7 @@ class Projects::MergeRequestsController < Projects::MergeRequests::ApplicationCo
push_frontend_feature_flag(:deploy_from_footer, @project, default_enabled: true)
push_frontend_feature_flag(:single_mr_diff_view, @project, default_enabled: true)
push_frontend_feature_flag(:suggest_pipeline) if experiment_enabled?(:suggest_pipeline)
push_frontend_feature_flag(:code_navigation, @project)
push_frontend_feature_flag(:code_navigation, @project, default_enabled: true)
push_frontend_feature_flag(:widget_visibility_polling, @project, default_enabled: true)
push_frontend_feature_flag(:merge_ref_head_comments, @project)
push_frontend_feature_flag(:mr_commit_neighbor_nav, @project, default_enabled: true)
......
---
title: Fix path conflict for Ghost on UpdateRoutesForLostAndFoundGroupAndOrphanedProjects
merge_request: 35425
author:
type: fixed
---
title: Fix existing repository_storages_weighted migrations
merge_request: 35814
author:
type: fixed
---
title: Fix gitlab:*:check Rake tasks
merge_request: 35944
author:
type: fixed
---
title: Fix error 500s creating new projects due to empty weights
merge_request: 35829
author:
type: fixed
......@@ -3,11 +3,37 @@
class AddRepositoryStoragesWeightedToApplicationSettings < ActiveRecord::Migration[6.0]
DOWNTIME = false
class ApplicationSetting < ActiveRecord::Base
serialize :repository_storages
self.table_name = 'application_settings'
end
def up
add_column :application_settings, :repository_storages_weighted, :jsonb, default: {}, null: false
seed_repository_storages_weighted
end
def down
remove_column :application_settings, :repository_storages_weighted
end
private
def seed_repository_storages_weighted
# We need to flush the cache to ensure the newly-added column is loaded
ApplicationSetting.reset_column_information
# There should only be one row here due to
# 20200420162730_remove_additional_application_settings_rows.rb
ApplicationSetting.all.each do |settings|
storages = Gitlab.config.repositories.storages.keys.collect do |storage|
weight = settings.repository_storages.include?(storage) ? 100 : 0
[storage.to_sym, weight]
end
settings.repository_storages_weighted = Hash[storages]
settings.save!
end
end
end
# frozen_string_literal: true
class ReseedRepositoryStoragesWeighted < ActiveRecord::Migration[6.0]
DOWNTIME = false
class ApplicationSetting < ActiveRecord::Base
serialize :repository_storages
self.table_name = 'application_settings'
end
def up
reseed_repository_storages_weighted
end
private
def reseed_repository_storages_weighted
# We need to flush the cache to ensure the newly-added column is loaded
ApplicationSetting.reset_column_information
# There should only be one row here due to
# 20200420162730_remove_additional_application_settings_rows.rb
ApplicationSetting.all.each do |settings|
# Admins may have already tweaked these values, so don't do anything
# if there is data already.
next if settings.repository_storages_weighted.present?
storages = Gitlab.config.repositories.storages.keys.collect do |storage|
weight = settings.repository_storages.include?(storage) ? 100 : 0
[storage.to_sym, weight]
end
settings.repository_storages_weighted = Hash[storages]
settings.save!
end
end
end
# frozen_string_literal: true
class SeedRepositoryStoragesWeighted < ActiveRecord::Migration[6.0]
DOWNTIME = false
class ApplicationSetting < ActiveRecord::Base
serialize :repository_storages
self.table_name = 'application_settings'
end
def up
# We need to flush the cache to ensure the newly-added column is loaded
ApplicationSetting.reset_column_information
# There should only be one row here due to
# 20200420162730_remove_additional_application_settings_rows.rb
ApplicationSetting.all.each do |settings|
storages = Gitlab.config.repositories.storages.keys.collect do |storage|
weight = settings.repository_storages.include?(storage) ? 100 : 0
[storage, weight]
[storage.to_sym, weight]
end
settings.repository_storages_weighted = Hash[storages]
......
......@@ -136,6 +136,7 @@ def up
# to ensure the Active Record's knowledge of the table structure is current
Namespace.reset_column_information
Route.reset_column_information
User.reset_column_information
# Find the ghost user, its namespace and the "lost and found" group
ghost_user = User.ghost
......@@ -158,6 +159,15 @@ def up
'More info: gitlab.com/gitlab-org/gitlab/-/issues/198603'
lost_and_found_group.save!
# make sure that the ghost namespace has a unique path
ghost_namespace.generate_unique_path
if ghost_namespace.path_changed?
ghost_namespace.save!
# If the path changed, also update the Ghost User's username to match the new path.
ghost_user.update!(username: ghost_namespace.path)
end
# Update the routes for the Ghost user, the "lost and found" group
# and all the orphaned projects
ghost_namespace.ensure_route!
......
......@@ -13903,6 +13903,7 @@ COPY "schema_migrations" (version) FROM STDIN;
20200508091106
20200508140959
20200508203901
20200509203901
20200511080113
20200511083541
20200511092246
......
# Doctor Rake tasks **(CORE ONLY)**
This is a collection of tasks to help investigate and repair
problems caused by data integrity issues.
## Verify database values can be decrypted using the current secrets
> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/20069) in GitLab 13.1.
This task runs through all possible encrypted values in the
database, verifying that they are decryptable using the current
secrets file (`gitlab-secrets.json`).
Automatic resolution is not yet implemented. If you have values that
cannot be decrypted, you can follow steps to reset them, see our
docs on what to do [when the secrets file is lost](../../raketasks/backup_restore.md#when-the-secrets-file-is-lost).
NOTE: **Note:**
This can take a very long time, depending on the size of your
database, as it checks all rows in all tables.
**Omnibus Installation**
```shell
sudo gitlab-rake gitlab:doctor:secrets
```
**Source Installation**
```shell
bundle exec rake gitlab:doctor:secrets RAILS_ENV=production
```
**Example output**
<!-- vale gitlab.SentenceSpacing = NO -->
```plaintext
I, [2020-06-11T17:17:54.951815 #27148] INFO -- : Checking encrypted values in the database
I, [2020-06-11T17:18:12.677708 #27148] INFO -- : - ApplicationSetting failures: 0
I, [2020-06-11T17:18:12.823692 #27148] INFO -- : - User failures: 0
[...] other models possibly containing encrypted data
I, [2020-06-11T17:18:14.938335 #27148] INFO -- : - Group failures: 1
I, [2020-06-11T17:18:15.559162 #27148] INFO -- : - Operations::FeatureFlagsClient failures: 0
I, [2020-06-11T17:18:15.575533 #27148] INFO -- : - ScimOauthAccessToken failures: 0
I, [2020-06-11T17:18:15.575678 #27148] INFO -- : Total: 1 row(s) affected
I, [2020-06-11T17:18:15.575711 #27148] INFO -- : Done!
```
<!-- vale gitlab.SentenceSpacing = YES -->
### Verbose mode
In order to get more detailed information about which
rows and columns cannot be decrypted, you can pass a VERBOSE
environment variable:
**Omnibus Installation**
```shell
sudo gitlab-rake gitlab:doctor:secrets VERBOSE=1
```
**Source Installation**
```shell
bundle exec rake gitlab:doctor:secrets RAILS_ENV=production VERBOSE=1
```
**Example verbose output**
<!-- vale gitlab.SentenceSpacing = NO -->
```plaintext
I, [2020-06-11T17:17:54.951815 #27148] INFO -- : Checking encrypted values in the database
I, [2020-06-11T17:18:12.677708 #27148] INFO -- : - ApplicationSetting failures: 0
I, [2020-06-11T17:18:12.823692 #27148] INFO -- : - User failures: 0
[...] other models possibly containing encrypted data
D, [2020-06-11T17:19:53.224344 #27351] DEBUG -- : > Something went wrong for Group[10].runners_token: Validation failed: Route can't be blank
I, [2020-06-11T17:19:53.225178 #27351] INFO -- : - Group failures: 1
D, [2020-06-11T17:19:53.225267 #27351] DEBUG -- : - Group[10]: runners_token
I, [2020-06-11T17:18:15.559162 #27148] INFO -- : - Operations::FeatureFlagsClient failures: 0
I, [2020-06-11T17:18:15.575533 #27148] INFO -- : - ScimOauthAccessToken failures: 0
I, [2020-06-11T17:18:15.575678 #27148] INFO -- : Total: 1 row(s) affected
I, [2020-06-11T17:18:15.575711 #27148] INFO -- : Done!
```
<!-- vale gitlab.SentenceSpacing = YES -->
......@@ -320,23 +320,7 @@ end
### Find mirrors with "bad decrypt" errors
```ruby
total = 0
bad = []
ProjectImportData.find_each do |data|
begin
total += 1
data.credentials
rescue => e
bad << data
end
end
puts "Bad count: #{bad.count} / #{total}"
bad.each do |repo|
puts Project.find(repo.project_id).full_path
end; bad.count
```
This content has been converted to a Rake task, see the [Doctor Rake tasks docs](../raketasks/doctor.md).
### Transfer mirror users and tokens to a single service account
......@@ -755,18 +739,9 @@ area on disk. It remains to be seen exactly how or whether the deletion is usefu
### Bad Decrypt Script (for encrypted variables)
See <https://gitlab.com/snippets/1730735/raw>.
This script will go through all the encrypted variables and count how many are not able
to be decrypted. Might be helpful to run on multiple nodes to see which `gitlab-secrets.json`
file is most up to date:
```shell
wget -O /tmp/bad-decrypt.rb https://gitlab.com/snippets/1730735/raw
gitlab-rails runner /tmp/bad-decrypt.rb
```
This content has been converted to a Rake task, see the [Doctor Rake tasks docs](../raketasks/doctor.md).
If `ProjectImportData Bad count:` is detected and the decision is made to delete the
As an example of repairing, if `ProjectImportData Bad count:` is detected and the decision is made to delete the
encrypted credentials to allow manual reentry:
```ruby
......@@ -797,16 +772,18 @@ encrypted credentials to allow manual reentry:
If `User OTP Secret Bad count:` is detected. For each user listed disable/enable
two-factor authentication.
### Decrypt Script for encrypted tokens
This script will search for all encrypted tokens that are causing decryption errors,
and update or reset as needed:
The following script will search in some of the tables for encrypted tokens that are
causing decryption errors, and update or reset as needed:
```shell
wget -O /tmp/encrypted-tokens.rb https://gitlab.com/snippets/1876342/raw
gitlab-rails runner /tmp/encrypted-tokens.rb
```
### Decrypt Script for encrypted tokens
This content has been converted to a Rake task, see the [Doctor Rake tasks docs](../raketasks/doctor.md).
## Geo
### Artifacts
......
......@@ -20,6 +20,7 @@ The following are available Rake tasks:
| [Back up and restore](backup_restore.md) | Back up, restore, and migrate GitLab instances between servers. |
| [Clean up](cleanup.md) | Clean up unneeded items from GitLab instances. |
| [Development](../development/rake_tasks.md) | Tasks for GitLab contributors. |
| [Doctor tasks](../administration/raketasks/doctor.md) | Checks for data integrity issues. |
| [Elasticsearch](../integration/elasticsearch.md#gitlab-elasticsearch-rake-tasks) **(STARTER ONLY)** | Maintain Elasticsearch in a GitLab instance. |
| [Enable namespaces](features.md) | Enable usernames and namespaces for user projects. |
| [General maintenance](../administration/raketasks/maintenance.md) | General maintenance and self-check tasks. |
......
......@@ -941,6 +941,9 @@ experience some unexpected behavior such as:
- Stuck jobs.
- 500 errors.
You can check whether you have undecryptable values in the database using
the [Secrets Doctor Rake task](../administration/raketasks/doctor.md).
In this case, you are required to reset all the tokens for CI/CD variables
and Runner Authentication, which is described in more detail below. After
resetting the tokens, you should be able to visit your project and the jobs
......
......@@ -71,9 +71,8 @@ def failure(object, message)
# It's already set to Logger::INFO, but acts as if it is set to
# Logger::DEBUG, and this fixes it...
def fix_google_api_logger
if Object.const_defined?('Google::Apis')
Google::Apis.logger.level = Logger::INFO
end
require 'google/apis'
Google::Apis.logger.level = Logger::INFO
end
# This should return an ActiveRecord::Relation suitable for calling #in_batches on
......
......@@ -20,7 +20,7 @@
end
before do
stub_feature_flags(web_ide_default: false, single_mr_diff_view: false, code_navigation: false)
stub_feature_flags(web_ide_default: false, single_mr_diff_view: false)
target_project.add_maintainer(user)
sign_in(user)
......
......@@ -13,10 +13,6 @@ def visit_blob(path, anchor: nil, ref: 'master')
wait_for_requests
end
before do
stub_feature_flags(code_navigation: false)
end
context 'Ruby file' do
before do
visit_blob('files/ruby/popen.rb')
......
......@@ -69,8 +69,6 @@ def fill_editor(content: 'class NextFeature\\nend\\n')
context 'from blob file path' do
before do
stub_feature_flags(code_navigation: false)
visit project_blob_path(project, tree_join(branch, file_path))
end
......
......@@ -8,7 +8,6 @@
shared_examples 'creating a file' do
before do
stub_feature_flags(code_navigation: false)
sign_in(user)
visit project_path(project)
end
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment