Commit ae03a52f authored by Douwe Maan's avatar Douwe Maan
Browse files

Merge branch 'hashed-storage-migration-path' into 'master'

Hashed storage migration path

Closes gitlab-ee#3118

See merge request gitlab-org/gitlab-ce!14067
......@@ -245,6 +245,9 @@ class Project < ActiveRecord::Base
scope :pending_delete, -> { where(pending_delete: true) }
scope :without_deleted, -> { where(pending_delete: false) }
scope :with_hashed_storage, -> { where('storage_version >= 1') }
scope :with_legacy_storage, -> { where(storage_version: [nil, 0]) }
scope :sorted_by_activity, -> { reorder(last_activity_at: :desc) }
scope :sorted_by_stars, -> { reorder('projects.star_count DESC') }
......@@ -1550,18 +1553,44 @@ def forks_count
end
def legacy_storage?
self.storage_version.nil?
[nil, 0].include?(self.storage_version)
end
def hashed_storage?
self.storage_version && self.storage_version >= 1
end
def renamed?
persisted? && path_changed?
end
def migrate_to_hashed_storage!
return if hashed_storage?
update!(repository_read_only: true)
if repo_reference_count > 0 || wiki_reference_count > 0
ProjectMigrateHashedStorageWorker.perform_in(Gitlab::ReferenceCounter::REFERENCE_EXPIRE_TIME, id)
else
ProjectMigrateHashedStorageWorker.perform_async(id)
end
end
def storage_version=(value)
super
@storage = nil if storage_version_changed?
end
def gl_repository(is_wiki:)
Gitlab::GlRepository.gl_repository(self, is_wiki)
end
private
def storage
@storage ||=
if self.storage_version && self.storage_version >= 1
if hashed_storage?
Storage::HashedProject.new(self)
else
Storage::LegacyProject.new(self)
......@@ -1574,6 +1603,14 @@ def use_hashed_storage
end
end
def repo_reference_count
Gitlab::ReferenceCounter.new(gl_repository(is_wiki: false)).value
end
def wiki_reference_count
Gitlab::ReferenceCounter.new(gl_repository(is_wiki: true)).value
end
# set last_activity_at to the same as created_at
def set_last_activity_at
update_column(:last_activity_at, self.created_at)
......
......@@ -4,6 +4,7 @@ class HashedProject
delegate :gitlab_shell, :repository_storage_path, to: :project
ROOT_PATH_PREFIX = '@hashed'.freeze
STORAGE_VERSION = 1
def initialize(project)
@project = project
......
module Projects
class HashedStorageMigrationService < BaseService
include Gitlab::ShellAdapter
attr_reader :old_disk_path, :new_disk_path
def initialize(project, logger = nil)
@project = project
@logger ||= Rails.logger
end
def execute
return if project.hashed_storage?
@old_disk_path = project.disk_path
has_wiki = project.wiki.repository_exists?
project.storage_version = Storage::HashedProject::STORAGE_VERSION
project.ensure_storage_path_exists
@new_disk_path = project.disk_path
result = move_repository(@old_disk_path, @new_disk_path)
if has_wiki
result &&= move_repository("#{@old_disk_path}.wiki", "#{@new_disk_path}.wiki")
end
unless result
rollback_folder_move
return
end
project.repository_read_only = false
project.save!
block_given? ? yield : result
end
private
def move_repository(from_name, to_name)
from_exists = gitlab_shell.exists?(project.repository_storage_path, "#{from_name}.git")
to_exists = gitlab_shell.exists?(project.repository_storage_path, "#{to_name}.git")
# If we don't find the repository on either original or target we should log that as it could be an issue if the
# project was not originally empty.
if !from_exists && !to_exists
logger.warn "Can't find a repository on either source or target paths for #{project.full_path} (ID=#{project.id}) ..."
return false
elsif !from_exists
# Repository have been moved already.
return true
end
gitlab_shell.mv_repository(project.repository_storage_path, from_name, to_name)
end
def rollback_folder_move
move_repository(@new_disk_path, @old_disk_path)
move_repository("#{@new_disk_path}.wiki", "#{@old_disk_path}.wiki")
end
def logger
@logger
end
end
end
class ProjectMigrateHashedStorageWorker
include Sidekiq::Worker
include DedicatedSidekiqQueue
def perform(project_id)
project = Project.find_by(id: project_id)
return if project.nil? || project.pending_delete?
::Projects::HashedStorageMigrationService.new(project, logger).execute
end
end
class StorageMigratorWorker
include Sidekiq::Worker
include DedicatedSidekiqQueue
BATCH_SIZE = 100
def perform(start, finish)
projects = build_relation(start, finish)
projects.with_route.find_each(batch_size: BATCH_SIZE) do |project|
Rails.logger.info "Starting storage migration of #{project.full_path} (ID=#{project.id})..."
begin
project.migrate_to_hashed_storage!
rescue => err
Rails.logger.error("#{err.message} migrating storage of #{project.full_path} (ID=#{project.id}), trace - #{err.backtrace}")
end
end
end
def build_relation(start, finish)
relation = Project
table = Project.arel_table
relation = relation.where(table[:id].gteq(start)) if start
relation = relation.where(table[:id].lteq(finish)) if finish
relation
end
end
---
title: Script to migrate project's repositories to new Hashed Storage
merge_request: 14067
author:
type: added
......@@ -62,3 +62,5 @@
- [update_user_activity, 1]
- [propagate_service_template, 1]
- [background_migration, 1]
- [project_migrate_hashed_storage, 1]
- [storage_migrator, 1]
class AddRepositoryReadOnlyToProjects < ActiveRecord::Migration
include Gitlab::Database::MigrationHelpers
DOWNTIME = false
def change
add_column :projects, :repository_read_only, :boolean
end
end
......@@ -1215,6 +1215,7 @@
t.datetime "last_repository_updated_at"
t.integer "storage_version", limit: 2
t.boolean "resolve_outdated_diff_discussions"
t.boolean "repository_read_only"
end
add_index "projects", ["ci_id"], name: "index_projects_on_ci_id", using: :btree
......
# Repository Storage Rake Tasks
This is a collection of rake tasks you can use to help you list and migrate
existing projects from Legacy storage to the new Hashed storage type.
You can read more about the storage types [here][storage-types].
## List projects on Legacy storage
To have a simple summary of projects using **Legacy** storage:
**Omnibus Installation**
```bash
gitlab-rake gitlab:storage:legacy_projects
```
**Source Installation**
```bash
rake gitlab:storage:legacy_projects
```
------
To list projects using **Legacy** storage:
**Omnibus Installation**
```bash
gitlab-rake gitlab:storage:list_legacy_projects
```
**Source Installation**
```bash
rake gitlab:storage:list_legacy_projects
```
## List projects on Hashed storage
To have a simple summary of projects using **Hashed** storage:
**Omnibus Installation**
```bash
gitlab-rake gitlab:storage:hashed_projects
```
**Source Installation**
```bash
rake gitlab:storage:hashed_projects
```
------
To list projects using **Hashed** storage:
**Omnibus Installation**
```bash
gitlab-rake gitlab:storage:list_hashed_projects
```
**Source Installation**
```bash
rake gitlab:storage:list_hashed_projects
```
## Migrate existing projects to Hashed storage
Before migrating your existing projects, you should
[enable hashed storage][storage-migration] for the new projects as well.
This task will schedule all your existing projects to be migrated to the
**Hashed** storage type:
**Omnibus Installation**
```bash
gitlab-rake gitlab:storage:migrate_to_hashed
```
**Source Installation**
```bash
rake gitlab:storage:migrate_to_hashed
```
You can monitor the progress in the _Admin > Monitoring > Background jobs_ screen.
There is a specific Queue you can watch to see how long it will take to finish: **project_migrate_hashed_storage**
After it reaches zero, you can confirm every project has been migrated by running the commands above.
If you find it necessary, you can run this migration script again to schedule missing projects.
Any error or warning will be logged in the sidekiq log file.
[storage-types]: ../repository_storage_types.md
[storage-migration]: ../repository_storage_types.md#how-to-migrate-to-hashed-storage
# Repository Storage Types
> [Introduced][ce-28283] in GitLab 10.0.
## Legacy Storage
Legacy Storage is the storage behavior prior to version 10.0. For historical reasons, GitLab replicated the same
mapping structure from the projects URLs:
* Project's repository: `#{namespace}/#{project_name}.git`
* Project's wiki: `#{namespace}/#{project_name}.wiki.git`
This structure made simple to migrate from existing solutions to GitLab and easy for Administrators to find where the
repository is stored.
On the other hand this has some drawbacks:
Storage location will concentrate huge amount of top-level namespaces. The impact can be reduced by the introduction of [multiple storage paths][storage-paths].
Because Backups are a snapshot of the same URL mapping, if you try to recover a very old backup, you need to verify
if any project has taken the place of an old removed project sharing the same URL. This means that `mygroup/myproject`
from your backup may not be the same original project that is today in the same URL.
Any change in the URL will need to be reflected on disk (when groups / users or projects are renamed). This can add a lot
of load in big installations, and can be even worst if they are using any type of network based filesystem.
Last, for GitLab Geo, this storage type means we have to synchronize the disk state, replicate renames in the correct
order or we may end-up with wrong repository or missing data temporarily.
## Hashed Storage
Hashed Storage is the new storage behavior we are rolling out with 10.0. It's not enabled by default yet, but we
encourage everyone to try-it and take the time to fix any script you may have that depends on the old behavior.
Instead of coupling project URL and the folder structure where the repository will be stored on disk, we are coupling
a hash, based on the project's ID.
This makes the folder structure immutable, and therefore eliminates any requirement to synchronize state from URLs to
disk structure. This means that renaming a group, user or project will cost only the database transaction, and will take
effect immediately.
The hash also helps to spread the repositories more evenly on the disk, so the top-level directory will contain less
folders than the total amount of top-level namespaces.
Hash format is based on hexadecimal representation of SHA256: `SHA256(project.id)`.
Top-level folder uses first 2 characters, followed by another folder with the next 2 characters. They are both stored in
a special folder `@hashed`, to co-exist with existing Legacy projects:
```ruby
# Project's repository:
"@hashed/#{hash[0..1]}/#{hash[2..3]}/#{hash}.git"
# Wiki's repository:
"@hashed/#{hash[0..1]}/#{hash[2..3]}/#{hash}.wiki.git"
```
This new format also makes possible to restore backups with confidence, as when restoring a repository from the backup,
you will never mistakenly restore a repository in the wrong project (considering the backup is made after the migration).
### How to migrate to Hashed Storage
In GitLab, go to **Admin > Settings**, find the **Repository Storage** section and select
"_Create new projects using hashed storage paths_".
To migrate your existing projects to the new storage type, check the specific [rake tasks].
[ce-28283]: https://gitlab.com/gitlab-org/gitlab-ce/issues/28283
[rake tasks]: raketasks/storage.md#migrate-existing-projects-to-hashed-storage
[storage-paths]: repository_storage_types.md
......@@ -16,7 +16,8 @@ class GitAccess
account_blocked: 'Your account has been blocked.',
command_not_allowed: "The command you're trying to execute is not allowed.",
upload_pack_disabled_over_http: 'Pulling over HTTP is not allowed.',
receive_pack_disabled_over_http: 'Pushing over HTTP is not allowed.'
receive_pack_disabled_over_http: 'Pushing over HTTP is not allowed.',
readonly: 'The repository is temporarily read-only. Please try again later.'
}.freeze
DOWNLOAD_COMMANDS = %w{ git-upload-pack git-upload-archive }.freeze
......@@ -159,6 +160,10 @@ def check_download_access!
end
def check_push_access!(changes)
if project.repository_read_only?
raise UnauthorizedError, ERROR_MESSAGES[:readonly]
end
if deploy_key
check_deploy_key_push_access!
elsif user
......
namespace :gitlab do
namespace :storage do
desc 'GitLab | Storage | Migrate existing projects to Hashed Storage'
task migrate_to_hashed: :environment do
legacy_projects_count = Project.with_legacy_storage.count
if legacy_projects_count == 0
puts 'There are no projects using legacy storage. Nothing to do!'
next
end
print "Enqueuing migration of #{legacy_projects_count} projects in batches of #{batch_size}"
project_id_batches do |start, finish|
StorageMigratorWorker.perform_async(start, finish)
print '.'
end
puts ' Done!'
end
desc 'Gitlab | Storage | Summary of existing projects using Legacy Storage'
task legacy_projects: :environment do
projects_summary(Project.with_legacy_storage)
end
desc 'Gitlab | Storage | List existing projects using Legacy Storage'
task list_legacy_projects: :environment do
projects_list(Project.with_legacy_storage)
end
desc 'Gitlab | Storage | Summary of existing projects using Hashed Storage'
task hashed_projects: :environment do
projects_summary(Project.with_hashed_storage)
end
desc 'Gitlab | Storage | List existing projects using Hashed Storage'
task list_hashed_projects: :environment do
projects_list(Project.with_hashed_storage)
end
def batch_size
ENV.fetch('BATCH', 200).to_i
end
def project_id_batches(&block)
Project.with_legacy_storage.in_batches(of: batch_size, start: ENV['ID_FROM'], finish: ENV['ID_TO']) do |relation| # rubocop: disable Cop/InBatches
ids = relation.pluck(:id)
yield ids.min, ids.max
end
end
def projects_summary(relation)
projects_count = relation.count
puts "* Found #{projects_count} projects".color(:green)
projects_count
end
def projects_list(relation)
projects_count = projects_summary(relation)
projects = relation.with_route
limit = ENV.fetch('LIMIT', 500).to_i
return unless projects_count > 0
puts " ! Displaying first #{limit} projects..." if projects_count > limit
counter = 0
projects.find_in_batches(batch_size: batch_size) do |batch|
batch.each do |project|
counter += 1
puts " - #{project.full_path} (id: #{project.id})".color(:red)
return if counter >= limit # rubocop:disable Lint/NonLocalExitFromIterator
end
end
end
end
end
......@@ -143,6 +143,16 @@
end
end
trait :wiki_repo do
after(:create) do |project|
raise 'Failed to create wiki repository!' unless project.create_wiki
end
end
trait :readonly do
repository_read_only true
end
trait :broken_repo do
after(:create) do |project|
raise "Failed to create repository!" unless project.create_repository
......
......@@ -48,14 +48,35 @@
end
end
describe '#add_key' do
it 'removes trailing garbage' do
allow(gitlab_shell).to receive(:gitlab_shell_keys_path).and_return(:gitlab_shell_keys_path)
expect(gitlab_shell).to receive(:gitlab_shell_fast_execute).with(
[:gitlab_shell_keys_path, 'add-key', 'key-123', 'ssh-rsa foobar']
)
gitlab_shell.add_key('key-123', 'ssh-rsa foobar trailing garbage')
describe 'projects commands' do
let(:gitlab_shell_path) { File.expand_path('tmp/tests/gitlab-shell') }
let(:projects_path) { File.join(gitlab_shell_path, 'bin/gitlab-projects') }
let(:gitlab_shell_hooks_path) { File.join(gitlab_shell_path, 'hooks') }
before do
allow(Gitlab.config.gitlab_shell).to receive(:path).and_return(gitlab_shell_path)
allow(Gitlab.config.gitlab_shell).to receive(:hooks_path).and_return(gitlab_shell_hooks_path)
allow(Gitlab.config.gitlab_shell).to receive(:git_timeout).and_return(800)
end
describe '#mv_repository' do
it 'executes the command' do
expect(gitlab_shell).to receive(:gitlab_shell_fast_execute).with(
[projects_path, 'mv-project', 'storage/path', 'project/path.git', 'new/path.git']
)
gitlab_shell.mv_repository('storage/path', 'project/path', 'new/path')
end
end
describe '#add_key' do
it 'removes trailing garbage' do
allow(gitlab_shell).to receive(:gitlab_shell_keys_path).and_return(:gitlab_shell_keys_path)
expect(gitlab_shell).to receive(:gitlab_shell_fast_execute).with(
[:gitlab_shell_keys_path, 'add-key', 'key-123', 'ssh-rsa foobar']
)
gitlab_shell.add_key('key-123', 'ssh-rsa foobar trailing garbage')
end
end
end
......@@ -136,7 +157,7 @@
it 'returns true when the command succeeds' do
expect(Gitlab::Popen).to receive(:popen)
.with([projects_path, 'rm-project', 'current/storage', 'project/path.git'],
nil, popen_vars).and_return([nil, 0])
nil, popen_vars).and_return([nil, 0])
expect(gitlab_shell.remove_repository('current/storage', 'project/path')).to be true
end
......@@ -144,7 +165,7 @@
it 'returns false when the command fails' do
expect(Gitlab::Popen).to receive(:popen)
.with([projects_path, 'rm-project', 'current/storage', 'project/path.git'],
nil, popen_vars).and_return(["error", 1])
nil, popen_vars).and_return(["error", 1])
expect(gitlab_shell.remove_repository('current/storage', 'project/path')).to be false
end
......
......@@ -2363,10 +2363,22 @@ def enable_lfs
describe '#legacy_storage?' do
it 'returns true when storage_version is nil' do
project = build(:project)
project = build(:project, storage_version: nil)
expect(project.legacy_storage?).to be_truthy
end
it 'returns true when the storage_version is 0' do
project = build(:project, storage_version: 0)
expect(project.legacy_storage?).to be_truthy
end
end
describe '#hashed_storage?' do
it 'returns false' do