Commit 6c92c623 authored by Alejandro Rodríguez's avatar Alejandro Rodríguez
Browse files

Refactor repository paths handling to allow multiple git mount points

parent d3a68dbbcb06
......@@ -3,6 +3,7 @@
v 8.10.0 (unreleased)
- Fix commit builds API, return all builds for all pipelines for given commit. !4849
- Replace Haml with Hamlit to make view rendering faster. !3666
- Refactor repository paths handling to allow multiple git mount points
- Wrap code blocks on Activies and Todos page. !4783 (winniehell)
- Align flash messages with left side of page content !4959 (winniehell)
- Display last commit of deleted branch in push events !4699 (winniehell)
......
......@@ -327,6 +327,6 @@
end
end
def sanitize_repo_path(message)
def sanitize_repo_path(project, message)
return '' unless message.present?
......@@ -331,5 +331,5 @@
return '' unless message.present?
message.strip.gsub(Gitlab.config.gitlab_shell.repos_path.chomp('/'), "[REPOS PATH]")
message.strip.gsub(project.repository_storage_path.chomp('/'), "[REPOS PATH]")
end
end
......@@ -21,5 +21,4 @@
delegate :name, to: :owner, allow_nil: true, prefix: true
after_create :ensure_dir_exist
after_update :move_dir, if: :path_changed?
......@@ -25,4 +24,7 @@
after_update :move_dir, if: :path_changed?
# Save the storage paths before the projects are destroyed to use them on after destroy
before_destroy(prepend: true) { @old_repository_storage_paths = repository_storage_paths }
after_destroy :rm_dir
scope :root, -> { where('type IS NULL') }
......@@ -87,23 +89,4 @@
owner_name
end
def ensure_dir_exist
gitlab_shell.add_namespace(path)
end
def rm_dir
# Move namespace directory into trash.
# We will remove it later async
new_path = "#{path}+#{id}+deleted"
if gitlab_shell.mv_namespace(path, new_path)
message = "Namespace directory \"#{path}\" moved to \"#{new_path}\""
Gitlab::AppLogger.info message
# Remove namespace directroy async with delay so
# GitLab has time to remove all projects first
GitlabShellWorker.perform_in(5.minutes, :rm_namespace, new_path)
end
end
def move_dir
......@@ -109,8 +92,5 @@
def move_dir
# Ensure old directory exists before moving it
gitlab_shell.add_namespace(path_was)
if any_project_has_container_registry_tags?
raise Exception.new('Namespace cannot be moved, because at least one project has tags in container registry')
end
......@@ -113,7 +93,16 @@
if any_project_has_container_registry_tags?
raise Exception.new('Namespace cannot be moved, because at least one project has tags in container registry')
end
if gitlab_shell.mv_namespace(path_was, path)
Gitlab::UploadsTransfer.new.rename_namespace(path_was, path)
# Move the namespace directory in all storages paths used by member projects
repository_storage_paths.each do |repository_storage_path|
# Ensure old directory exists before moving it
gitlab_shell.add_namespace(repository_storage_path, path_was)
unless gitlab_shell.mv_namespace(repository_storage_path, path_was, path)
# if we cannot move namespace directory we should rollback
# db changes in order to prevent out of sync between db and fs
raise Exception.new('namespace directory cannot be moved')
end
end
......@@ -119,19 +108,16 @@
# If repositories moved successfully we need to
# send update instructions to users.
# However we cannot allow rollback since we moved namespace dir
# So we basically we mute exceptions in next actions
begin
send_update_instructions
rescue
# Returning false does not rollback after_* transaction but gives
# us information about failing some of tasks
false
end
else
# if we cannot move namespace directory we should rollback
# db changes in order to prevent out of sync between db and fs
raise Exception.new('namespace directory cannot be moved')
Gitlab::UploadsTransfer.new.rename_namespace(path_was, path)
# If repositories moved successfully we need to
# send update instructions to users.
# However we cannot allow rollback since we moved namespace dir
# So we basically we mute exceptions in next actions
begin
send_update_instructions
rescue
# Returning false does not rollback after_* transaction but gives
# us information about failing some of tasks
false
end
end
......@@ -152,4 +138,33 @@
def find_fork_of(project)
projects.joins(:forked_project_link).find_by('forked_project_links.forked_from_project_id = ?', project.id)
end
private
def repository_storage_paths
# We need to get the storage paths for all the projects, even the ones that are
# pending delete. Unscoping also get rids of the default order, which causes
# problems with SELECT DISTINCT.
Project.unscoped do
projects.select('distinct(repository_storage)').to_a.map(&:repository_storage_path)
end
end
def rm_dir
# Remove the namespace directory in all storages paths used by member projects
@old_repository_storage_paths.each do |repository_storage_path|
# Move namespace directory into trash.
# We will remove it later async
new_path = "#{path}+#{id}+deleted"
if gitlab_shell.mv_namespace(repository_storage_path, path, new_path)
message = "Namespace directory \"#{path}\" moved to \"#{new_path}\""
Gitlab::AppLogger.info message
# Remove namespace directroy async with delay so
# GitLab has time to remove all projects first
GitlabShellWorker.perform_in(5.minutes, :rm_namespace, repository_storage_path, new_path)
end
end
end
end
......@@ -26,6 +26,9 @@
default_value_for :container_registry_enabled, gitlab_config_features.container_registry
default_value_for(:shared_runners_enabled) { current_application_settings.shared_runners_enabled }
after_create :ensure_dir_exist
after_save :ensure_dir_exist, if: :namespace_id_changed?
# set last_activity_at to the same as created_at
after_create :set_last_activity_at
def set_last_activity_at
......@@ -165,6 +168,9 @@
validate :visibility_level_allowed_by_group
validate :visibility_level_allowed_as_fork
validate :check_wiki_path_conflict
validates :repository_storage,
presence: true,
inclusion: { in: ->(_object) { Gitlab.config.repositories.storages.keys } }
add_authentication_token_field :runners_token
before_save :ensure_runners_token
......@@ -376,6 +382,10 @@
end
end
def repository_storage_path
Gitlab.config.repositories.storages[repository_storage]
end
def team
@team ||= ProjectTeam.new(self)
end
......@@ -842,8 +852,8 @@
raise Exception.new('Project cannot be renamed, because tags are present in its container registry')
end
if gitlab_shell.mv_repository(old_path_with_namespace, new_path_with_namespace)
if gitlab_shell.mv_repository(repository_storage_path, old_path_with_namespace, new_path_with_namespace)
# If repository moved successfully we need to send update instructions to users.
# However we cannot allow rollback since we moved repository
# So we basically we mute exceptions in next actions
begin
......@@ -846,8 +856,8 @@
# If repository moved successfully we need to send update instructions to users.
# However we cannot allow rollback since we moved repository
# So we basically we mute exceptions in next actions
begin
gitlab_shell.mv_repository("#{old_path_with_namespace}.wiki", "#{new_path_with_namespace}.wiki")
gitlab_shell.mv_repository(repository_storage_path, "#{old_path_with_namespace}.wiki", "#{new_path_with_namespace}.wiki")
send_move_instructions(old_path_with_namespace)
reset_events_cache
......@@ -988,7 +998,7 @@
def create_repository
# Forked import is handled asynchronously
unless forked?
if gitlab_shell.add_repository(path_with_namespace)
if gitlab_shell.add_repository(repository_storage_path, path_with_namespace)
repository.after_create
true
else
......@@ -1140,4 +1150,8 @@
_, status = Gitlab::Popen.popen(%W(find #{export_path} -not -path #{export_path} -delete))
status.zero?
end
def ensure_dir_exist
gitlab_shell.add_namespace(repository_storage_path, namespace.path)
end
end
......@@ -159,7 +159,7 @@
private
def init_repo(path_with_namespace)
gitlab_shell.add_repository(path_with_namespace)
gitlab_shell.add_repository(project.repository_storage_path, path_with_namespace)
end
def commit_details(action, message = nil, title = nil)
......@@ -173,7 +173,7 @@
end
def path_to_repo
@path_to_repo ||= File.join(Gitlab.config.gitlab_shell.repos_path, "#{path_with_namespace}.git")
@path_to_repo ||= File.join(project.repository_storage_path, "#{path_with_namespace}.git")
end
def update_project_activity
......
......@@ -39,7 +39,7 @@
# Return absolute path to repository
def path_to_repo
@path_to_repo ||= File.expand_path(
File.join(Gitlab.config.gitlab_shell.repos_path, path_with_namespace + ".git")
File.join(@project.repository_storage_path, path_with_namespace + ".git")
)
end
......
......@@ -51,7 +51,7 @@
return true if params[:skip_repo] == true
# There is a possibility project does not have repository or wiki
return true unless gitlab_shell.exists?(path + '.git')
return true unless gitlab_shell.exists?(project.repository_storage_path, path + '.git')
new_path = removal_path(path)
......@@ -55,5 +55,5 @@
new_path = removal_path(path)
if gitlab_shell.mv_repository(path, new_path)
if gitlab_shell.mv_repository(project.repository_storage_path, path, new_path)
log_info("Repository \"#{path}\" moved to \"#{new_path}\"")
......@@ -59,5 +59,5 @@
log_info("Repository \"#{path}\" moved to \"#{new_path}\"")
GitlabShellWorker.perform_in(5.minutes, :remove_repository, new_path)
GitlabShellWorker.perform_in(5.minutes, :remove_repository, project.repository_storage_path, new_path)
else
false
end
......
......@@ -24,7 +24,7 @@
def execute
raise LeaseTaken unless try_obtain_lease
GitlabShellOneShotWorker.perform_async(:gc, @project.path_with_namespace)
GitlabShellOneShotWorker.perform_async(:gc, @project.repository_storage_path, @project.path_with_namespace)
ensure
Gitlab::Metrics.measure(:reset_pushes_since_gc) do
@project.update_column(:pushes_since_gc, 0)
......
......@@ -42,7 +42,7 @@
def import_repository
begin
gitlab_shell.import_repository(project.path_with_namespace, project.import_url)
gitlab_shell.import_repository(project.repository_storage_path, project.path_with_namespace, project.import_url)
rescue Gitlab::Shell::Error => e
raise Error, "Error importing repository #{project.import_url} into #{project.path_with_namespace} - #{e.message}"
end
......
......@@ -50,8 +50,8 @@
project.send_move_instructions(old_path)
# Move main repository
unless gitlab_shell.mv_repository(old_path, new_path)
unless gitlab_shell.mv_repository(project.repository_storage_path, old_path, new_path)
raise TransferError.new('Cannot move project')
end
# Move wiki repo also if present
......@@ -54,8 +54,8 @@
raise TransferError.new('Cannot move project')
end
# Move wiki repo also if present
gitlab_shell.mv_repository("#{old_path}.wiki", "#{new_path}.wiki")
gitlab_shell.mv_repository(project.repository_storage_path, "#{old_path}.wiki", "#{new_path}.wiki")
# clear project cached events
project.reset_events_cache
......
......@@ -10,7 +10,7 @@
.panel-body
%pre
:preserve
#{sanitize_repo_path(@project.import_error)}
#{sanitize_repo_path(@project, @project.import_error)}
= form_for @project, url: namespace_project_import_path(@project.namespace, @project), method: :post, html: { class: 'form-horizontal' } do |f|
= render "shared/import_form", f: f
......
......@@ -4,6 +4,6 @@
sidekiq_options queue: :post_receive
def perform(repo_path, identifier, changes)
if repo_path.start_with?(Gitlab.config.gitlab_shell.repos_path.to_s)
repo_path.gsub!(Gitlab.config.gitlab_shell.repos_path.to_s, "")
if path = Gitlab.config.repositories.storages.find { |p| repo_path.start_with?(p[1].to_s) }
repo_path.gsub!(path[1].to_s, "")
else
......@@ -9,5 +9,5 @@
else
log("Check gitlab.yml config for correct gitlab_shell.repos_path variable. \"#{Gitlab.config.gitlab_shell.repos_path}\" does not match \"#{repo_path}\"")
log("Check gitlab.yml config for correct repositories.storages values. No repository storage path matches \"#{repo_path}\"")
end
post_received = Gitlab::GitPostReceive.new(repo_path, identifier, changes)
......
......@@ -12,7 +12,7 @@
return
end
result = gitlab_shell.fork_repository(source_path, target_path)
result = gitlab_shell.fork_repository(project.repository_storage_path, source_path, target_path)
unless result
logger.error("Unable to fork project #{project_id} for repository #{source_path} -> #{target_path}")
project.mark_import_as_failed('The project could not be forked.')
......
......@@ -47,6 +47,10 @@
backup:
path: "tmp/backups" # Relative paths are relative to Rails.root (default: tmp/backups/)
repositories:
storages: # REPO PATHS MUST NOT BE A SYMLINK!!!
default: /apps/repositories/
gitlab_shell:
path: /apps/gitlab-shell/
......@@ -50,8 +54,6 @@
gitlab_shell:
path: /apps/gitlab-shell/
# REPOS_PATH MUST NOT BE A SYMLINK!!!
repos_path: /apps/repositories/
hooks_path: /apps/gitlab-shell/hooks/
upload_pack: true
......
......@@ -428,6 +428,13 @@
satellites:
path: /home/git/gitlab-satellites/
## Repositories settings
repositories:
# Paths where repositories can be stored. Give the canonicalized absolute pathname.
# NOTE: REPOS PATHS MUST NOT CONTAIN ANY SYMLINK!!!
storages: # You must have at least a `default` storage path.
default: /home/git/repositories/
## Backup settings
backup:
path: "tmp/backups" # Relative paths are relative to Rails.root (default: tmp/backups/)
......@@ -452,9 +459,6 @@
## GitLab Shell settings
gitlab_shell:
path: /home/git/gitlab-shell/
# REPOS_PATH MUST NOT BE A SYMLINK!!!
repos_path: /home/git/repositories/
hooks_path: /home/git/gitlab-shell/hooks/
# File that contains the secret key for verifying access for gitlab-shell.
......@@ -528,7 +532,10 @@
# user: YOUR_USERNAME
satellites:
path: tmp/tests/gitlab-satellites/
repositories:
storages:
default: tmp/tests/repositories/
backup:
path: tmp/tests/backups
gitlab_shell:
path: tmp/tests/gitlab-shell/
......@@ -531,8 +538,7 @@
backup:
path: tmp/tests/backups
gitlab_shell:
path: tmp/tests/gitlab-shell/
repos_path: tmp/tests/repositories/
hooks_path: tmp/tests/gitlab-shell/hooks/
issues_tracker:
redmine:
......
......@@ -304,7 +304,6 @@
Settings.gitlab_shell['secret_file'] ||= Rails.root.join('.gitlab_shell_secret')
Settings.gitlab_shell['receive_pack'] = true if Settings.gitlab_shell['receive_pack'].nil?
Settings.gitlab_shell['upload_pack'] = true if Settings.gitlab_shell['upload_pack'].nil?
Settings.gitlab_shell['repos_path'] ||= Settings.gitlab['user_home'] + '/repositories/'
Settings.gitlab_shell['ssh_host'] ||= Settings.gitlab.ssh_host
Settings.gitlab_shell['ssh_port'] ||= 22
Settings.gitlab_shell['ssh_user'] ||= Settings.gitlab.user
......@@ -312,6 +311,14 @@
Settings.gitlab_shell['ssh_path_prefix'] ||= Settings.send(:build_gitlab_shell_ssh_path_prefix)
#
# Repositories
#
Settings['repositories'] ||= Settingslogic.new({})
Settings.repositories['storages'] ||= {}
# Setting gitlab_shell.repos_path is DEPRECATED and WILL BE REMOVED in version 9.0
Settings.repositories.storages['default'] ||= Settings.gitlab_shell['repos_path'] || Settings.gitlab['user_home'] + '/repositories/'
#
# Backup
#
Settings['backup'] ||= Settingslogic.new({})
......
def storage_name_valid?(name)
!!(name =~ /\A[a-zA-Z0-9\-_]+\z/)
end
def find_parent_path(name, path)
Gitlab.config.repositories.storages.detect do |n, p|
name != n && path.chomp('/').start_with?(p.chomp('/'))
end
end
def error(message)
raise "#{message}. Please fix this in your gitlab.yml before starting GitLab."
end
error('No repository storage path defined') if Gitlab.config.repositories.storages.empty?
Gitlab.config.repositories.storages.each do |name, path|
error("\"#{name}\" is not a valid storage name") unless storage_name_valid?(name)
parent_name, _parent_path = find_parent_path(name, path)
if parent_name
error("#{name} is a nested path of #{parent_name}. Nested paths are not supported for repository storages")
end
end
class AddRepositoryStorageToProjects < ActiveRecord::Migration
include Gitlab::Database::MigrationHelpers
disable_ddl_transaction!
def up
add_column_with_default(:projects, :repository_storage, :string, default: 'default')
end
def down
remove_column(:projects, :repository_storage)
end
end
......@@ -796,7 +796,7 @@
t.datetime "created_at"
t.datetime "updated_at"
t.integer "creator_id"
t.boolean "issues_enabled", default: true, null: false
t.boolean "merge_requests_enabled", default: true, null: false
t.boolean "wiki_enabled", default: true, null: false
t.boolean "issues_enabled", default: true, null: false
t.boolean "merge_requests_enabled", default: true, null: false
t.boolean "wiki_enabled", default: true, null: false
t.integer "namespace_id"
......@@ -802,4 +802,4 @@
t.integer "namespace_id"
t.boolean "snippets_enabled", default: true, null: false
t.boolean "snippets_enabled", default: true, null: false
t.datetime "last_activity_at"
t.string "import_url"
......@@ -804,7 +804,7 @@
t.datetime "last_activity_at"
t.string "import_url"
t.integer "visibility_level", default: 0, null: false
t.boolean "archived", default: false, null: false
t.integer "visibility_level", default: 0, null: false
t.boolean "archived", default: false, null: false
t.string "avatar"
t.string "import_status"
t.float "repository_size", default: 0.0
......@@ -808,9 +808,9 @@
t.string "avatar"
t.string "import_status"
t.float "repository_size", default: 0.0
t.integer "star_count", default: 0, null: false
t.integer "star_count", default: 0, null: false
t.string "import_type"
t.string "import_source"
t.integer "commit_count", default: 0
t.text "import_error"
t.integer "ci_id"
......@@ -812,9 +812,9 @@
t.string "import_type"
t.string "import_source"
t.integer "commit_count", default: 0
t.text "import_error"
t.integer "ci_id"
t.boolean "builds_enabled", default: true, null: false
t.boolean "shared_runners_enabled", default: true, null: false
t.boolean "builds_enabled", default: true, null: false
t.boolean "shared_runners_enabled", default: true, null: false
t.string "runners_token"
t.string "build_coverage_regex"
......@@ -819,5 +819,5 @@
t.string "runners_token"
t.string "build_coverage_regex"
t.boolean "build_allow_git_fetch", default: true, null: false
t.integer "build_timeout", default: 3600, null: false
t.boolean "build_allow_git_fetch", default: true, null: false
t.integer "build_timeout", default: 3600, null: false
t.boolean "pending_delete", default: false
......@@ -823,6 +823,6 @@
t.boolean "pending_delete", default: false
t.boolean "public_builds", default: true, null: false
t.boolean "public_builds", default: true, null: false
t.integer "pushes_since_gc", default: 0
t.boolean "last_repository_check_failed"
t.datetime "last_repository_check_at"
t.boolean "container_registry_enabled"
......@@ -825,6 +825,6 @@
t.integer "pushes_since_gc", default: 0
t.boolean "last_repository_check_failed"
t.datetime "last_repository_check_at"
t.boolean "container_registry_enabled"
t.boolean "only_allow_merge_if_build_succeeds", default: false, null: false
t.boolean "only_allow_merge_if_build_succeeds", default: false, null: false
t.boolean "has_external_issue_tracker"
......@@ -830,4 +830,5 @@
t.boolean "has_external_issue_tracker"
t.string "repository_storage", default: "default", null: false
end
add_index "projects", ["builds_enabled", "shared_runners_enabled"], name: "index_projects_on_builds_enabled_and_shared_runners_enabled", using: :btree
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment