Commit 86359ec8 authored by Alejandro Rodríguez's avatar Alejandro Rodríguez
Browse files

Refactor repository paths handling to allow multiple git mount points

parent d3a68dbbcb06
......@@ -3,6 +3,7 @@ Please view this file on the master branch, on stable branches it's out of date.
v 8.10.0 (unreleased)
- Fix commit builds API, return all builds for all pipelines for given commit. !4849
- Replace Haml with Hamlit to make view rendering faster. !3666
- Refactor repository paths handling to allow multiple git mount points
- Wrap code blocks on Activies and Todos page. !4783 (winniehell)
- Align flash messages with left side of page content !4959 (winniehell)
- Display last commit of deleted branch in push events !4699 (winniehell)
......
......@@ -327,9 +327,9 @@ def filename_path(project, filename)
end
end
def sanitize_repo_path(message)
def sanitize_repo_path(project, message)
return '' unless message.present?
message.strip.gsub(Gitlab.config.gitlab_shell.repos_path.chomp('/'), "[REPOS PATH]")
message.strip.gsub(project.repository_storage_path.chomp('/'), "[REPOS PATH]")
end
end
......@@ -21,8 +21,10 @@ class Namespace < ActiveRecord::Base
delegate :name, to: :owner, allow_nil: true, prefix: true
after_create :ensure_dir_exist
after_update :move_dir, if: :path_changed?
# Save the storage paths before the projects are destroyed to use them on after destroy
before_destroy(prepend: true) { @old_repository_storage_paths = repository_storage_paths }
after_destroy :rm_dir
scope :root, -> { where('type IS NULL') }
......@@ -87,51 +89,35 @@ def human_name
owner_name
end
def ensure_dir_exist
gitlab_shell.add_namespace(path)
end
def rm_dir
# Move namespace directory into trash.
# We will remove it later async
new_path = "#{path}+#{id}+deleted"
if gitlab_shell.mv_namespace(path, new_path)
message = "Namespace directory \"#{path}\" moved to \"#{new_path}\""
Gitlab::AppLogger.info message
# Remove namespace directroy async with delay so
# GitLab has time to remove all projects first
GitlabShellWorker.perform_in(5.minutes, :rm_namespace, new_path)
end
end
def move_dir
# Ensure old directory exists before moving it
gitlab_shell.add_namespace(path_was)
if any_project_has_container_registry_tags?
raise Exception.new('Namespace cannot be moved, because at least one project has tags in container registry')
end
if gitlab_shell.mv_namespace(path_was, path)
Gitlab::UploadsTransfer.new.rename_namespace(path_was, path)
# If repositories moved successfully we need to
# send update instructions to users.
# However we cannot allow rollback since we moved namespace dir
# So we basically we mute exceptions in next actions
begin
send_update_instructions
rescue
# Returning false does not rollback after_* transaction but gives
# us information about failing some of tasks
false
# Move the namespace directory in all storages paths used by member projects
repository_storage_paths.each do |repository_storage_path|
# Ensure old directory exists before moving it
gitlab_shell.add_namespace(repository_storage_path, path_was)
unless gitlab_shell.mv_namespace(repository_storage_path, path_was, path)
# if we cannot move namespace directory we should rollback
# db changes in order to prevent out of sync between db and fs
raise Exception.new('namespace directory cannot be moved')
end
else
# if we cannot move namespace directory we should rollback
# db changes in order to prevent out of sync between db and fs
raise Exception.new('namespace directory cannot be moved')
end
Gitlab::UploadsTransfer.new.rename_namespace(path_was, path)
# If repositories moved successfully we need to
# send update instructions to users.
# However we cannot allow rollback since we moved namespace dir
# So we basically we mute exceptions in next actions
begin
send_update_instructions
rescue
# Returning false does not rollback after_* transaction but gives
# us information about failing some of tasks
false
end
end
......@@ -152,4 +138,33 @@ def kind
def find_fork_of(project)
projects.joins(:forked_project_link).find_by('forked_project_links.forked_from_project_id = ?', project.id)
end
private
def repository_storage_paths
# We need to get the storage paths for all the projects, even the ones that are
# pending delete. Unscoping also get rids of the default order, which causes
# problems with SELECT DISTINCT.
Project.unscoped do
projects.select('distinct(repository_storage)').to_a.map(&:repository_storage_path)
end
end
def rm_dir
# Remove the namespace directory in all storages paths used by member projects
@old_repository_storage_paths.each do |repository_storage_path|
# Move namespace directory into trash.
# We will remove it later async
new_path = "#{path}+#{id}+deleted"
if gitlab_shell.mv_namespace(repository_storage_path, path, new_path)
message = "Namespace directory \"#{path}\" moved to \"#{new_path}\""
Gitlab::AppLogger.info message
# Remove namespace directroy async with delay so
# GitLab has time to remove all projects first
GitlabShellWorker.perform_in(5.minutes, :rm_namespace, repository_storage_path, new_path)
end
end
end
end
......@@ -26,6 +26,9 @@ class Project < ActiveRecord::Base
default_value_for :container_registry_enabled, gitlab_config_features.container_registry
default_value_for(:shared_runners_enabled) { current_application_settings.shared_runners_enabled }
after_create :ensure_dir_exist
after_save :ensure_dir_exist, if: :namespace_id_changed?
# set last_activity_at to the same as created_at
after_create :set_last_activity_at
def set_last_activity_at
......@@ -165,6 +168,9 @@ def update_forks_visibility_level
validate :visibility_level_allowed_by_group
validate :visibility_level_allowed_as_fork
validate :check_wiki_path_conflict
validates :repository_storage,
presence: true,
inclusion: { in: ->(_object) { Gitlab.config.repositories.storages.keys } }
add_authentication_token_field :runners_token
before_save :ensure_runners_token
......@@ -376,6 +382,10 @@ def remove_gitlab_exports!
end
end
def repository_storage_path
Gitlab.config.repositories.storages[repository_storage]
end
def team
@team ||= ProjectTeam.new(self)
end
......@@ -842,12 +852,12 @@ def rename_repo
raise Exception.new('Project cannot be renamed, because tags are present in its container registry')
end
if gitlab_shell.mv_repository(old_path_with_namespace, new_path_with_namespace)
if gitlab_shell.mv_repository(repository_storage_path, old_path_with_namespace, new_path_with_namespace)
# If repository moved successfully we need to send update instructions to users.
# However we cannot allow rollback since we moved repository
# So we basically we mute exceptions in next actions
begin
gitlab_shell.mv_repository("#{old_path_with_namespace}.wiki", "#{new_path_with_namespace}.wiki")
gitlab_shell.mv_repository(repository_storage_path, "#{old_path_with_namespace}.wiki", "#{new_path_with_namespace}.wiki")
send_move_instructions(old_path_with_namespace)
reset_events_cache
......@@ -988,7 +998,7 @@ def origin_merge_requests
def create_repository
# Forked import is handled asynchronously
unless forked?
if gitlab_shell.add_repository(path_with_namespace)
if gitlab_shell.add_repository(repository_storage_path, path_with_namespace)
repository.after_create
true
else
......@@ -1140,4 +1150,8 @@ def remove_exports
_, status = Gitlab::Popen.popen(%W(find #{export_path} -not -path #{export_path} -delete))
status.zero?
end
def ensure_dir_exist
gitlab_shell.add_namespace(repository_storage_path, namespace.path)
end
end
......@@ -159,7 +159,7 @@ def hook_attrs
private
def init_repo(path_with_namespace)
gitlab_shell.add_repository(path_with_namespace)
gitlab_shell.add_repository(project.repository_storage_path, path_with_namespace)
end
def commit_details(action, message = nil, title = nil)
......@@ -173,7 +173,7 @@ def default_message(action, title)
end
def path_to_repo
@path_to_repo ||= File.join(Gitlab.config.gitlab_shell.repos_path, "#{path_with_namespace}.git")
@path_to_repo ||= File.join(project.repository_storage_path, "#{path_with_namespace}.git")
end
def update_project_activity
......
......@@ -39,7 +39,7 @@ def update_autocrlf_option
# Return absolute path to repository
def path_to_repo
@path_to_repo ||= File.expand_path(
File.join(Gitlab.config.gitlab_shell.repos_path, path_with_namespace + ".git")
File.join(@project.repository_storage_path, path_with_namespace + ".git")
)
end
......
......@@ -51,13 +51,13 @@ def remove_repository(path)
return true if params[:skip_repo] == true
# There is a possibility project does not have repository or wiki
return true unless gitlab_shell.exists?(path + '.git')
return true unless gitlab_shell.exists?(project.repository_storage_path, path + '.git')
new_path = removal_path(path)
if gitlab_shell.mv_repository(path, new_path)
if gitlab_shell.mv_repository(project.repository_storage_path, path, new_path)
log_info("Repository \"#{path}\" moved to \"#{new_path}\"")
GitlabShellWorker.perform_in(5.minutes, :remove_repository, new_path)
GitlabShellWorker.perform_in(5.minutes, :remove_repository, project.repository_storage_path, new_path)
else
false
end
......
......@@ -24,7 +24,7 @@ def initialize(project)
def execute
raise LeaseTaken unless try_obtain_lease
GitlabShellOneShotWorker.perform_async(:gc, @project.path_with_namespace)
GitlabShellOneShotWorker.perform_async(:gc, @project.repository_storage_path, @project.path_with_namespace)
ensure
Gitlab::Metrics.measure(:reset_pushes_since_gc) do
@project.update_column(:pushes_since_gc, 0)
......
......@@ -42,7 +42,7 @@ def create_repository
def import_repository
begin
gitlab_shell.import_repository(project.path_with_namespace, project.import_url)
gitlab_shell.import_repository(project.repository_storage_path, project.path_with_namespace, project.import_url)
rescue Gitlab::Shell::Error => e
raise Error, "Error importing repository #{project.import_url} into #{project.path_with_namespace} - #{e.message}"
end
......
......@@ -50,12 +50,12 @@ def transfer(project, new_namespace)
project.send_move_instructions(old_path)
# Move main repository
unless gitlab_shell.mv_repository(old_path, new_path)
unless gitlab_shell.mv_repository(project.repository_storage_path, old_path, new_path)
raise TransferError.new('Cannot move project')
end
# Move wiki repo also if present
gitlab_shell.mv_repository("#{old_path}.wiki", "#{new_path}.wiki")
gitlab_shell.mv_repository(project.repository_storage_path, "#{old_path}.wiki", "#{new_path}.wiki")
# clear project cached events
project.reset_events_cache
......
......@@ -10,7 +10,7 @@
.panel-body
%pre
:preserve
#{sanitize_repo_path(@project.import_error)}
#{sanitize_repo_path(@project, @project.import_error)}
= form_for @project, url: namespace_project_import_path(@project.namespace, @project), method: :post, html: { class: 'form-horizontal' } do |f|
= render "shared/import_form", f: f
......
......@@ -4,10 +4,10 @@ class PostReceive
sidekiq_options queue: :post_receive
def perform(repo_path, identifier, changes)
if repo_path.start_with?(Gitlab.config.gitlab_shell.repos_path.to_s)
repo_path.gsub!(Gitlab.config.gitlab_shell.repos_path.to_s, "")
if path = Gitlab.config.repositories.storages.find { |p| repo_path.start_with?(p[1].to_s) }
repo_path.gsub!(path[1].to_s, "")
else
log("Check gitlab.yml config for correct gitlab_shell.repos_path variable. \"#{Gitlab.config.gitlab_shell.repos_path}\" does not match \"#{repo_path}\"")
log("Check gitlab.yml config for correct repositories.storages values. No repository storage path matches \"#{repo_path}\"")
end
post_received = Gitlab::GitPostReceive.new(repo_path, identifier, changes)
......
......@@ -12,7 +12,7 @@ def perform(project_id, source_path, target_path)
return
end
result = gitlab_shell.fork_repository(source_path, target_path)
result = gitlab_shell.fork_repository(project.repository_storage_path, source_path, target_path)
unless result
logger.error("Unable to fork project #{project_id} for repository #{source_path} -> #{target_path}")
project.mark_import_as_failed('The project could not be forked.')
......
......@@ -47,11 +47,13 @@ production: &base
backup:
path: "tmp/backups" # Relative paths are relative to Rails.root (default: tmp/backups/)
repositories:
storages: # REPO PATHS MUST NOT BE A SYMLINK!!!
default: /apps/repositories/
gitlab_shell:
path: /apps/gitlab-shell/
# REPOS_PATH MUST NOT BE A SYMLINK!!!
repos_path: /apps/repositories/
hooks_path: /apps/gitlab-shell/hooks/
upload_pack: true
......
......@@ -428,6 +428,13 @@ production: &base
satellites:
path: /home/git/gitlab-satellites/
## Repositories settings
repositories:
# Paths where repositories can be stored. Give the canonicalized absolute pathname.
# NOTE: REPOS PATHS MUST NOT CONTAIN ANY SYMLINK!!!
storages: # You must have at least a `default` storage path.
default: /home/git/repositories/
## Backup settings
backup:
path: "tmp/backups" # Relative paths are relative to Rails.root (default: tmp/backups/)
......@@ -452,9 +459,6 @@ production: &base
## GitLab Shell settings
gitlab_shell:
path: /home/git/gitlab-shell/
# REPOS_PATH MUST NOT BE A SYMLINK!!!
repos_path: /home/git/repositories/
hooks_path: /home/git/gitlab-shell/hooks/
# File that contains the secret key for verifying access for gitlab-shell.
......@@ -528,11 +532,13 @@ test:
# user: YOUR_USERNAME
satellites:
path: tmp/tests/gitlab-satellites/
repositories:
storages:
default: tmp/tests/repositories/
backup:
path: tmp/tests/backups
gitlab_shell:
path: tmp/tests/gitlab-shell/
repos_path: tmp/tests/repositories/
hooks_path: tmp/tests/gitlab-shell/hooks/
issues_tracker:
redmine:
......
......@@ -304,13 +304,20 @@ def host(url)
Settings.gitlab_shell['secret_file'] ||= Rails.root.join('.gitlab_shell_secret')
Settings.gitlab_shell['receive_pack'] = true if Settings.gitlab_shell['receive_pack'].nil?
Settings.gitlab_shell['upload_pack'] = true if Settings.gitlab_shell['upload_pack'].nil?
Settings.gitlab_shell['repos_path'] ||= Settings.gitlab['user_home'] + '/repositories/'
Settings.gitlab_shell['ssh_host'] ||= Settings.gitlab.ssh_host
Settings.gitlab_shell['ssh_port'] ||= 22
Settings.gitlab_shell['ssh_user'] ||= Settings.gitlab.user
Settings.gitlab_shell['owner_group'] ||= Settings.gitlab.user
Settings.gitlab_shell['ssh_path_prefix'] ||= Settings.send(:build_gitlab_shell_ssh_path_prefix)
#
# Repositories
#
Settings['repositories'] ||= Settingslogic.new({})
Settings.repositories['storages'] ||= {}
# Setting gitlab_shell.repos_path is DEPRECATED and WILL BE REMOVED in version 9.0
Settings.repositories.storages['default'] ||= Settings.gitlab_shell['repos_path'] || Settings.gitlab['user_home'] + '/repositories/'
#
# Backup
#
......
def storage_name_valid?(name)
!!(name =~ /\A[a-zA-Z0-9\-_]+\z/)
end
def find_parent_path(name, path)
Gitlab.config.repositories.storages.detect do |n, p|
name != n && path.chomp('/').start_with?(p.chomp('/'))
end
end
def error(message)
raise "#{message}. Please fix this in your gitlab.yml before starting GitLab."
end
error('No repository storage path defined') if Gitlab.config.repositories.storages.empty?
Gitlab.config.repositories.storages.each do |name, path|
error("\"#{name}\" is not a valid storage name") unless storage_name_valid?(name)
parent_name, _parent_path = find_parent_path(name, path)
if parent_name
error("#{name} is a nested path of #{parent_name}. Nested paths are not supported for repository storages")
end
end
class AddRepositoryStorageToProjects < ActiveRecord::Migration
include Gitlab::Database::MigrationHelpers
disable_ddl_transaction!
def up
add_column_with_default(:projects, :repository_storage, :string, default: 'default')
end
def down
remove_column(:projects, :repository_storage)
end
end
......@@ -796,38 +796,39 @@
t.datetime "created_at"
t.datetime "updated_at"
t.integer "creator_id"
t.boolean "issues_enabled", default: true, null: false
t.boolean "merge_requests_enabled", default: true, null: false
t.boolean "wiki_enabled", default: true, null: false
t.boolean "issues_enabled", default: true, null: false
t.boolean "merge_requests_enabled", default: true, null: false
t.boolean "wiki_enabled", default: true, null: false
t.integer "namespace_id"
t.boolean "snippets_enabled", default: true, null: false
t.boolean "snippets_enabled", default: true, null: false
t.datetime "last_activity_at"
t.string "import_url"
t.integer "visibility_level", default: 0, null: false
t.boolean "archived", default: false, null: false
t.integer "visibility_level", default: 0, null: false
t.boolean "archived", default: false, null: false
t.string "avatar"
t.string "import_status"
t.float "repository_size", default: 0.0
t.integer "star_count", default: 0, null: false
t.integer "star_count", default: 0, null: false
t.string "import_type"
t.string "import_source"
t.integer "commit_count", default: 0
t.text "import_error"
t.integer "ci_id"
t.boolean "builds_enabled", default: true, null: false
t.boolean "shared_runners_enabled", default: true, null: false
t.boolean "builds_enabled", default: true, null: false
t.boolean "shared_runners_enabled", default: true, null: false
t.string "runners_token"
t.string "build_coverage_regex"
t.boolean "build_allow_git_fetch", default: true, null: false
t.integer "build_timeout", default: 3600, null: false
t.boolean "build_allow_git_fetch", default: true, null: false
t.integer "build_timeout", default: 3600, null: false
t.boolean "pending_delete", default: false
t.boolean "public_builds", default: true, null: false
t.boolean "public_builds", default: true, null: false
t.integer "pushes_since_gc", default: 0
t.boolean "last_repository_check_failed"
t.datetime "last_repository_check_at"
t.boolean "container_registry_enabled"
t.boolean "only_allow_merge_if_build_succeeds", default: false, null: false
t.boolean "only_allow_merge_if_build_succeeds", default: false, null: false
t.boolean "has_external_issue_tracker"
t.string "repository_storage", default: "default", null: false
end
add_index "projects", ["builds_enabled", "shared_runners_enabled"], name: "index_projects_on_builds_enabled_and_shared_runners_enabled", using: :btree
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment