Read about our upcoming Code of Conduct on this issue

Commit 9e6fa996 authored by Gabriel Mazetto's avatar Gabriel Mazetto
Browse files

New storage is now "Hashed" instead of "UUID"

parent 186ef4b7330a
module Storage
module UUIDProject
module HashedProject
extend ActiveSupport::Concern
def uuid_dir
%Q(#{uuid[0..1]}/#{uuid[2..3]})
# Base directory
#
# @return [String] directory where repository is stored
def base_dir
%Q(#{disk_hash[0..1]}/#{disk_hash[2..3]}) if disk_hash
end
# Disk path is used to build repository and project's wiki path on disk
#
# @return [String] combination of base_dir and the repository own name without `.git` or `.wiki.git` extensions
def disk_path
%Q(#{uuid_dir}/#{uuid})
%Q(#{base_dir}/#{disk_hash})
end
def ensure_storage_path_exist
gitlab_shell.add_namespace(repository_storage_path, uuid_dir)
gitlab_shell.add_namespace(repository_storage_path, base_dir)
end
def rename_repo
......@@ -54,5 +60,13 @@ def rename_repo
Gitlab::UploadsTransfer.new.rename_project(path_was, path, namespace.full_path)
Gitlab::PagesTransfer.new.rename_project(path_was, path, namespace.full_path)
end
private
# Generates the hash for the project path and name on disk
# If you need to refer to the repository on disk, use the `#disk_path`
def disk_hash
@disk_hash ||= Digest::SHA2.hexdigest(self.id.to_s) if self.id
end
end
end
......@@ -2,12 +2,22 @@ module Storage
module LegacyProject
extend ActiveSupport::Concern
# Base directory
#
# @return [String] directory where repository is stored
def base_dir
namespace.full_path
end
# Disk path is used to build repository and project's wiki path on disk
#
# @return [String] combination of base_dir and the repository own name without `.git` or `.wiki.git` extensions
def disk_path
full_path
end
def ensure_storage_path_exist
gitlab_shell.add_namespace(repository_storage_path, namespace.full_path)
gitlab_shell.add_namespace(repository_storage_path, base_dir)
end
def rename_repo
......
......@@ -1424,10 +1424,12 @@ def forks_count
private
def load_storage
if self.storage_version > 1
self.class.include Storage::UUIDProject
return unless has_attribute?(:storage_version)
if self.storage_version && self.storage_version >= 1
self.extend Storage::HashedProject
else
self.class.include Storage::LegacyProject
self.extend Storage::LegacyProject
end
end
......
......@@ -4,36 +4,16 @@
class AddStorageFieldsToProject < ActiveRecord::Migration
include Gitlab::Database::MigrationHelpers
# Set this constant to true if this migration requires downtime.
DOWNTIME = false
# When a migration requires downtime you **must** uncomment the following
# constant and define a short and easy to understand explanation as to why the
# migration requires downtime.
# DOWNTIME_REASON = ''
# When using the methods "add_concurrent_index", "remove_concurrent_index" or
# "add_column_with_default" you must disable the use of transactions
# as these methods can not run in an existing transaction.
# When using "add_concurrent_index" or "remove_concurrent_index" methods make sure
# that either of them is the _only_ method called in the migration,
# any other changes should go in a separate migration.
# This ensures that upon failure _only_ the index creation or removing fails
# and can be retried or reverted easily.
#
# To disable transactions uncomment the following line and remove these
# comments:
disable_ddl_transaction!
def up
# rubocop:disable Migration/AddColumnWithDefaultToLargeTable
add_column :projects, :uuid, :uuid
add_column_with_default :projects, :storage_version, :integer, default: 0, limit: 1
add_concurrent_index :projects, :uuid
add_column :projects, :storage_version, :integer, limit: 2
add_concurrent_index :projects, :storage_version
end
def down
remove_column :projects, :uuid
remove_column :projects, :storage_version
end
end
......@@ -1208,8 +1208,7 @@
t.datetime "last_repository_updated_at"
t.string "ci_config_path"
t.text "delete_error"
t.uuid "uuid"
t.integer "storage_version", limit: 2, default: 0, null: false
t.integer "storage_version", limit: 2
end
add_index "projects", ["ci_id"], name: "index_projects_on_ci_id", using: :btree
......@@ -1226,7 +1225,7 @@
add_index "projects", ["pending_delete"], name: "index_projects_on_pending_delete", using: :btree
add_index "projects", ["runners_token"], name: "index_projects_on_runners_token", using: :btree
add_index "projects", ["star_count"], name: "index_projects_on_star_count", using: :btree
add_index "projects", ["uuid"], name: "index_projects_on_uuid", using: :btree
add_index "projects", ["storage_version"], name: "index_projects_on_storage_version", using: :btree
add_index "projects", ["visibility_level"], name: "index_projects_on_visibility_level", using: :btree
create_table "protected_branch_merge_access_levels", force: :cascade do |t|
......
......@@ -98,6 +98,7 @@ excluded_attributes:
- :last_activity_at
- :last_repository_updated_at
- :last_repository_check_at
- :storage_version
snippets:
- :expired_at
merge_request_diff:
......
......@@ -81,6 +81,10 @@
archived true
end
trait :hashed do
storage_version 1
end
trait :access_requestable do
request_access_enabled true
end
......
......@@ -1251,60 +1251,6 @@ def create_pipeline
end
end
describe '#rename_repo' do
let(:project) { create(:project, :repository) }
let(:gitlab_shell) { Gitlab::Shell.new }
before do
# Project#gitlab_shell returns a new instance of Gitlab::Shell on every
# call. This makes testing a bit easier.
allow(project).to receive(:gitlab_shell).and_return(gitlab_shell)
allow(project).to receive(:previous_changes).and_return('path' => ['foo'])
end
it 'renames a repository' do
stub_container_registry_config(enabled: false)
expect(gitlab_shell).to receive(:mv_repository)
.ordered
.with(project.repository_storage_path, "#{project.namespace.full_path}/foo", "#{project.full_path}")
.and_return(true)
expect(gitlab_shell).to receive(:mv_repository)
.ordered
.with(project.repository_storage_path, "#{project.namespace.full_path}/foo.wiki", "#{project.full_path}.wiki")
.and_return(true)
expect_any_instance_of(SystemHooksService)
.to receive(:execute_hooks_for)
.with(project, :rename)
expect_any_instance_of(Gitlab::UploadsTransfer)
.to receive(:rename_project)
.with('foo', project.path, project.namespace.full_path)
expect(project).to receive(:expire_caches_before_rename)
expect(project).to receive(:expires_full_path_cache)
project.rename_repo
end
context 'container registry with images' do
let(:container_repository) { create(:container_repository) }
before do
stub_container_registry_config(enabled: true)
stub_container_registry_tags(repository: :any, tags: ['tag'])
project.container_repositories << container_repository
end
subject { project.rename_repo }
it { expect {subject}.to raise_error(StandardError) }
end
end
describe '#expire_caches_before_rename' do
let(:project) { create(:project, :repository) }
let(:repo) { double(:repo, exists?: true) }
......@@ -2367,4 +2313,163 @@ def enable_lfs
expect(project.forks_count).to eq(1)
end
end
context 'legacy storage' do
let(:project) { create(:project, :repository) }
let(:gitlab_shell) { Gitlab::Shell.new }
describe '#base_dir' do
it 'returns base_dir based on namespace only' do
expect(project.base_dir).to eq(project.namespace.full_path)
end
end
describe '#disk_path' do
it 'returns disk_path based on namespace and project path' do
expect(project.disk_path).to eq("#{project.namespace.full_path}/#{project.path}")
end
end
describe '#ensure_storage_path_exist' do
before do
allow(project).to receive(:gitlab_shell).and_return(gitlab_shell)
end
it 'delegates to gitlab_shell to ensure namespace is created' do
expect(gitlab_shell).to receive(:add_namespace).with(project.repository_storage_path, project.base_dir)
project.ensure_storage_path_exist
end
end
describe '#rename_repo' do
before do
# Project#gitlab_shell returns a new instance of Gitlab::Shell on every
# call. This makes testing a bit easier.
allow(project).to receive(:gitlab_shell).and_return(gitlab_shell)
allow(project).to receive(:previous_changes).and_return('path' => ['foo'])
end
it 'renames a repository' do
stub_container_registry_config(enabled: false)
expect(gitlab_shell).to receive(:mv_repository)
.ordered
.with(project.repository_storage_path, "#{project.namespace.full_path}/foo", "#{project.full_path}")
.and_return(true)
expect(gitlab_shell).to receive(:mv_repository)
.ordered
.with(project.repository_storage_path, "#{project.namespace.full_path}/foo.wiki", "#{project.full_path}.wiki")
.and_return(true)
expect_any_instance_of(SystemHooksService)
.to receive(:execute_hooks_for)
.with(project, :rename)
expect_any_instance_of(Gitlab::UploadsTransfer)
.to receive(:rename_project)
.with('foo', project.path, project.namespace.full_path)
expect(project).to receive(:expire_caches_before_rename)
expect(project).to receive(:expires_full_path_cache)
project.rename_repo
end
context 'container registry with images' do
let(:container_repository) { create(:container_repository) }
before do
stub_container_registry_config(enabled: true)
stub_container_registry_tags(repository: :any, tags: ['tag'])
project.container_repositories << container_repository
end
subject { project.rename_repo }
it { expect{subject}.to raise_error(StandardError) }
end
end
end
context 'hashed storage' do
let(:project) { create(:project, :repository, :hashed) }
let(:gitlab_shell) { Gitlab::Shell.new }
let(:hash) { '6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b' }
before do
allow(Digest::SHA2).to receive(:hexdigest) { hash }
end
describe '#base_dir' do
it 'returns base_dir based on hash of project id' do
expect(project.base_dir).to eq('6b/86')
end
end
describe '#disk_path' do
it 'returns disk_path based on has of project id' do
hashed_path = '6b/86/6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b'
expect(project.disk_path).to eq(hashed_path)
end
end
describe '#ensure_storage_path_exist' do
before do
allow(project).to receive(:gitlab_shell).and_return(gitlab_shell)
end
it 'delegates to gitlab_shell to ensure namespace is created' do
expect(gitlab_shell).to receive(:add_namespace).with(project.repository_storage_path, '6b/86')
project.ensure_storage_path_exist
end
end
describe '#rename_repo' do
before do
# Project#gitlab_shell returns a new instance of Gitlab::Shell on every
# call. This makes testing a bit easier.
allow(project).to receive(:gitlab_shell).and_return(gitlab_shell)
allow(project).to receive(:previous_changes).and_return('path' => ['foo'])
end
it 'renames a repository' do
stub_container_registry_config(enabled: false)
expect(gitlab_shell).not_to receive(:mv_repository)
expect_any_instance_of(SystemHooksService)
.to receive(:execute_hooks_for)
.with(project, :rename)
expect_any_instance_of(Gitlab::UploadsTransfer)
.to receive(:rename_project)
.with('foo', project.path, project.namespace.full_path)
expect(project).to receive(:expire_caches_before_rename)
expect(project).to receive(:expires_full_path_cache)
project.rename_repo
end
context 'container registry with images' do
let(:container_repository) { create(:container_repository) }
before do
stub_container_registry_config(enabled: true)
stub_container_registry_tags(repository: :any, tags: ['tag'])
project.container_repositories << container_repository
end
subject { project.rename_repo }
it { expect{subject}.to raise_error(StandardError) }
end
end
end
end
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment