Read about our upcoming Code of Conduct on this issue

Commit 3f962fdd authored by Georges Racinet's avatar Georges Racinet 🦑
Browse files

hashed storage: adapting path to group HGRC

In the case of the hashed storage, we keep a
subset of the directory hierarchy for namespaces
and the repo HGRC includes from that, hence depending
on the namespace path from the storage root and the
depth of the hashed storage (currently a constant, at
two intermediate subdirectories).

In both the hashed and non-hashed cases, this is provided by
`relative_path_from`, whose result is just `../../hgrc` in
the non hashed case.

The inclusion path is relative to the storage root, hence
this independent of the storage location, which can
change (reconfig, change of host by backup & restore)
with no need to crawl through repositories.

This partially defeats one of the advantages of the
hash storage: avoiding too much top level directories.
This shouldn't be critical for Heptapod because

1. By the time we have instances that big that this cluttering
   is a problem, assuming we just don't use a FS with hashmap
   directories (XFS for instance), we'll be able to come up
   with other mechanisms
2. It's about namespaces having either HGRC settings or subgroups.
   These should be the minority. After all, using subgroups
   reduces the number of toplevel groups.

--HG--
branch : heptapod
parent 8c116884d4d8
......@@ -1354,6 +1354,10 @@ def track_project_repository
repository.update!(shard_name: repository_storage, disk_path: disk_path)
end
def namespace_fs_pathname
Pathname('').join(*namespace.full_path.split('/'))
end
def create_repository(force: false)
return false unless create_git_repository(force: force)
Rails.logger.info("create_repository #{disk_path} vcs_type #{vcs_type} "\
......@@ -1362,7 +1366,9 @@ def create_repository(force: false)
Rails.logger.info("Creating Mercurial repository for #{disk_path}")
if gitlab_shell.add_hg_repository(
repository_storage, disk_path,
repository_storage,
disk_path,
namespace_fs_pathname,
in_subgroup: !group.nil? && !group.parent_id.nil?)
true
else
......
......@@ -200,7 +200,10 @@ def create_repo!(raw_repository)
if project.vcs_type == 'hg_git'
# we don't care about subgroup hgrc, it's the job of the main
# project repository to ensure its consistency
gitlab_shell.add_hg_repository(project.repository_storage, disk_path)
gitlab_shell.add_hg_repository(project.repository_storage,
disk_path,
project.namespace.full_path
)
end
raise CouldNotCreateWikiError unless raw_repository.exists?
......
# frozen_string_literal: true
require 'pathname'
require 'securerandom'
class Repository
......@@ -108,7 +109,8 @@ def inspect
end
def set_hgrc(user, inherit, values)
raw_repository.set_hgrc(user, inherit, values)
namespace_path = Pathname('').join(*project.namespace.full_path.split('/'))
raw_repository.set_hgrc(user, inherit, namespace_path, values)
end
def get_hgrc
......
......@@ -4,6 +4,14 @@
require 'forwardable'
require "rubygems/package"
# Index of the line in lines matching a given regexp
def match_idx(lines, rx)
lines.each_with_index do |line, i|
return i if rx.match(line)
end
nil
end
module Gitlab
module Git
class Repository
......@@ -645,6 +653,7 @@ def hg_env_for_write(user)
'HEPTAPOD_USERINFO_NAME' => user.name,
'HEPTAPOD_USERINFO_EMAIL' => user.email,
'HEPTAPOD_PROJECT_PATH' => self.name.chomp('.git'),
# TODO won't work on hashed storage
'HEPTAPOD_PROJECT_NAMESPACE_FULL_PATH' => @relative_path.rpartition('/')[0],
'HGUSER' => nil,
'HGRCPATH' => Gitlab::Mercurial.hgrc_path
......@@ -905,6 +914,20 @@ def pull_force_topic(url, hgsha, topic)
node_branch.split(':')
end
# Compute path to a repo namespace HGRC, ready for inclusion from repo's HGRC.
#
# all parameters are supposed to be relative from the same root or absolute
#
# Returns: path as a string
def self.compute_hgrc_inherit_path(repo_path, namespace_path)
dot_hg = Pathname(repo_path).join('.hg')
Pathname(namespace_path).join('hgrc').relative_path_from(dot_hg).to_s
end
def hgrc_inherit_path(namespace_path)
self.class.compute_hgrc_inherit_path(@hg_relative_path, namespace_path)
end
def hgrc_main_path
File.join(@hgpath, '.hg', 'hgrc')
end
......@@ -914,7 +937,7 @@ def hgrc_managed
[managed, File.join(@hgpath, '.hg', managed)]
end
HGRC_INCL_INHERIT = "%include ../../hgrc".freeze
HGRC_INCL_INHERIT_RX = /%include .*\/hgrc/.freeze
# write the repository-local HGRC files.
......@@ -932,7 +955,7 @@ def hgrc_managed
#
# Newlines in values are prohibited (use plain whitespace if needed),
# as well as in keys (section and item names).
def set_hgrc(user, inherit, values)
def set_hgrc(user, inherit, namespace_path, values)
key_rx = /^\w*$/
write_main = false
......@@ -950,13 +973,14 @@ def set_hgrc(user, inherit, values)
by_line = "by user #{user.username} (id=#{user.id}) at #{Time.new}"
if inherit.nil?
elsif inherit
if !main_stripped_lines.include?(HGRC_INCL_INHERIT)
main_lines.insert(0, HGRC_INCL_INHERIT)
if match_idx(main_stripped_lines, HGRC_INCL_INHERIT_RX).nil?
main_lines.insert(0,
"%include " + hgrc_inherit_path(namespace_path))
main_lines.insert(0, "# inheritance restored #{by_line}")
write_main = true
end
else
inh_idx = main_stripped_lines.find_index(HGRC_INCL_INHERIT)
inh_idx = match_idx(main_stripped_lines, HGRC_INCL_INHERIT_RX)
if !inh_idx.nil?
main_lines[inh_idx] = "# inheritance removed #{by_line}"
write_main = true
......@@ -1001,7 +1025,7 @@ def get_hgrc
main_lines = File.readlines(hgrc_main_path)
main_stripped_lines = main_lines.map { |l| l.strip }
{
'inherit' => main_stripped_lines.include?(HGRC_INCL_INHERIT),
'inherit' => !match_idx(main_stripped_lines, HGRC_INCL_INHERIT_RX).nil?,
'content' => File.read(hgrc_managed[1])
}
end
......
......@@ -2,6 +2,7 @@
# Gitaly note: SSH key operations are not part of Gitaly so will never be migrated.
require 'pathname'
require 'securerandom'
module Gitlab
......@@ -92,7 +93,7 @@ def create_repository(storage, disk_path, gl_project_path)
"# You may also remove the %include line altogether if you prefer.",
"",
"# group-level configuration",
"%include ../../hgrc",
"%include <GROUP_HGRC_RPATH>",
""].join("\n").freeze
DEFAULT_SUBGROUP_HGRC = [
......@@ -116,25 +117,34 @@ def create_repository(storage, disk_path, gl_project_path)
# Ex.
# add_hg_git_repository("/path/to/storage", "gitlab/gitlab-ci")
#
def add_hg_repository(storage, name, in_subgroup: false)
def add_hg_repository(storage, name, namespace_fs_path, in_subgroup: false)
hg_relative_path = name.dup
hg_relative_path << '.hg' unless hg_relative_path.end_with?('.hg')
Rails.logger.info(
"Creating Mercurial repository at #{hg_relative_path} "\
"namespace path=#{namespace_fs_path} "\
"in_subgroup: #{in_subgroup}")
begin
hg_repo_path = File.join(Gitlab.config.repositories.storages[storage].hg_disk_path,
hg_relative_path)
root = Pathname(
Gitlab.config.repositories.storages[storage].hg_disk_path)
ns_path = root.join(namespace_fs_path)
hg_repo_path = root.join(hg_relative_path)
gitlab_shell_fast_execute_raise_error(
[Gitlab.config.mercurial.bin_path, "init", hg_repo_path],
[Gitlab.config.mercurial.bin_path, "init", hg_repo_path.to_s],
{ 'HGRCPATH' => Gitlab::Mercurial.hgrc_path,
})
File.write(File.join(hg_repo_path, '.hg/hgrc'), DEFAULT_HGRC)
ns_hgrc_incl_path = Gitlab::Git::Repository.compute_hgrc_inherit_path(
hg_repo_path, ns_path)
File.write(hg_repo_path.join('.hg', 'hgrc'),
DEFAULT_HGRC.sub('<GROUP_HGRC_RPATH>', ns_hgrc_incl_path))
if in_subgroup
hgrcf = File.join(hg_repo_path, '../hgrc')
File.write(hgrcf, DEFAULT_SUBGROUP_HGRC) unless File.exist?(hgrcf)
ns_hgrcf = ns_path.join('hgrc')
unless File.exist?(ns_hgrcf)
FileUtils.mkdir_p(ns_path)
File.write(ns_hgrcf, DEFAULT_SUBGROUP_HGRC)
end
end
end
# Note for non-rubyists: up to now we had no need of this because
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment