# Index of the line in lines matching a given regexp def match_idx(lines, rx) lines.each_with_index do |line, i| return i if rx.match(line) end nil end module Gitlab module Mercurial class HgGitRepository < Gitlab::Git::Repository include Gitlab::Popen extend ::Gitlab::Utils::Override attr_accessor :unit_tests_skip_hooks attr_writer :hg_project_for_perms MergeError = Class.new(StandardError) HgError = Class.new(StandardError) InvalidHgSetting = Class.new(StandardError) HgRevisionError = Class.new(StandardError) # In general, GitLab expects `multi_action` to raise IndexError, # so let's subclass it. ActionError = Class.new(Gitlab::Git::Index::IndexError) DEFAULT_HGRC = [ "# This is the specific configuration for this repository", "# You may modify it freely at your will.", "#", "# By default, it includes the configuration from the enclosing group", "# but all the settings defined in the present file will take precedence", "# as long as they are below the %include line", "# You may also remove the %include line altogether if you prefer.", "", "# group-level configuration", "%include ", ""].join("\n").freeze DEFAULT_SUBGROUP_HGRC = [ "# This is the Mercurial configuration for this subgroup.", "# You may modify it freely at your will.", "#", "# By default, it includes the configuration from the enclosing", "# group, but all the settings defined in the present file will", "# take precedence as long as they are below the %include line", "# You may also remove the %include line altogether if you prefer.", "", "# enclosing group configuration", "%include ../hgrc", ""].join("\n").freeze def initialize(*args) super(*args) @hg_relative_path = relative_path.sub(/\.git$/, '.hg') # storage can actually be in VCS-qualified form (see CommitService) # so we need to handle that. bare_storage = Gitlab::GitalyClient.vcs_storage_split(@storage)[0] @hg_storage_root = Gitlab.config.repositories.storages.fetch( bare_storage).hg_disk_path @hgpath = File.join(@hg_storage_root, @hg_relative_path) # `nil` is the language default for instance variables, # but we want to be explicit in that case: it will mean no check # in Mercurial process for write operations (besides what's been done # at the Model level already, and is enough in the Git case). # It should stay nil only for Wikis and similar. @hg_project_for_perms = nil end # Mercurial path will stay until HGitaly is ready def hg_full_path @hgpath end def hg_config_item_bool?(full_dotted_name) env = { 'HGRCPATH' => Gitlab::Mercurial.hgrc_path } val_str, status = popen( [Gitlab.config.mercurial.bin_path, 'config', full_dotted_name], @hgpath, env) return false if status == 1 # status 1 means value is missing raise HgError, "Could not read `#{full_dotted_name}` config item for #{@hgpath}" unless status == 0 HG_CONFIG_TRUE.include?(val_str.downcase.strip) end def create_repository(**named_args) # at the time being all named_args are Mercurial specific. Still, # passing explicitely is more future proof super(**named_args) create_hg_repository(**named_args) end # Create the Mercurial repository only # # This is what a FS based native Mercurial class would do # # - namespace_fs_path: the path of the directory for the parent # namespace config files # - in_subgroup: true is the parent namespace is a subgroup def create_hg_repository(namespace_fs_path:, in_subgroup:) Rails.logger.info( "Creating Mercurial repository at #{@hg_relative_path} "\ "namespace path=#{namespace_fs_path} "\ "in_subgroup: #{in_subgroup}") begin out, status = popen( [Gitlab.config.mercurial.bin_path, "init", @hgpath], # Rails.root can be a Mercurial repo, that can lead to # complications. Let's run from a path that's very unlikely # to be one: @hg_storage_root, { 'HGRCPATH' => Gitlab::Mercurial.hgrc_path, }) if status != 0 raise HgError, "Could not execute `hg init` for #{@hgpath}: #{out}" end File.write(File.join(@hgpath, '.hg', 'hgrc'), DEFAULT_HGRC.sub('', hgrc_inherit_path(namespace_fs_path))) if in_subgroup ns_path = Pathname(@hg_storage_root).join(namespace_fs_path) ns_hgrcf = ns_path.join('hgrc') unless File.exist?(ns_hgrcf) FileUtils.mkdir_p(ns_path) File.write(ns_hgrcf, DEFAULT_SUBGROUP_HGRC) end end end true rescue => err Rails.logger.error("Failed to add Mercurial repository for #{storage}/#{name}: #{err}") false end def import_repository(url, **create_args) raise ArgumentError, "don't use disk paths with import_repository: "\ "#{url.inspect}" if url.start_with?('.', '/') logprefix = "#{self.class.name} import_repository "\ "for #{mask_password_in_url(url)} "\ "into #{@hgpath}" logger = Rails.logger logger.info("#{logprefix} creating repository(ies) before pull") create_repository(**create_args) hg_exe = Gitlab.config.mercurial.bin_path hg_env = {'HGUSER' => 'Heptapod system', 'HEPTAPOD_SKIP_ALL_GITLAB_HOOKS' => 'yes', 'HGRCPATH' => Gitlab::Mercurial.hgrc_path, } Rails.logger.info("#{logprefix} Pulling") pid = Process.spawn(hg_env, hg_exe, "pull", "-q", "--config", "heptapod.initial-import=yes", "-R", @hgpath, url) timeout = Gitlab.config.gitlab_shell.git_timeout begin Timeout.timeout(timeout) do Process.wait(pid) end unless $?.exitstatus.zero? msg = "#{logprefix} import failed" Rails.logger.error(msg) raise HgError, msg end rescue Timeout::Error msg = "#{logprefix} failed due to timeout: pull not finished after "\ "the #{timeout} seconds set in configuration" Process.kill('KILL', pid) Process.wait FileUtils.rm_rf(@hgpath) # Removing the Git auxiliary repo as well for consistency FileUtils.rm_rf(@hgpath[0..-4] + ".git") raise HgError, msg end begin hg_optim rescue StandardError => err # let's not abord the import because optimization failed Rails.logger.error("#{logprefix} Post import optimization "\ "failed: #{err}") end end # Perform Mercurial repository optimizations # # See heptapod#192 for details about implications of various options def hg_optim hg_env = {'HGUSER' => 'Heptapod system', 'HEPTAPOD_SKIP_ALL_GITLAB_HOOKS' => 'yes', 'HGRCPATH' => Gitlab::Mercurial.hgrc_path, } output, status = popen( [Gitlab.config.mercurial.bin_path, "debugupgraderepo", '-o', 're-delta-all', '--run', '--no-backup', ], @hgpath, hg_env) unless status.zero? raise HgError, "Optimization failed for #{@hgpath}: " + output end end # Run `hg recover`. # This is idempotent and reasonably fast if there's nothing to recover def hg_recover hg_env = {'HGUSER' => 'Heptapod system', 'HEPTAPOD_SKIP_ALL_GITLAB_HOOKS' => 'yes', 'HGRCPATH' => Gitlab::Mercurial.hgrc_path, } output, status = popen( [Gitlab.config.mercurial.bin_path, "recover", ], @hgpath, hg_env) # as usual with hg, code 1 means there's nothing to do unless [0, 1].include?(status) raise HgError, "`hg recover` failed for #{@hgpath}: " + output end end def rename(new_git_relative_path) super(new_git_relative_path) new_hg_relative_path = new_git_relative_path.sub(/\.git$/, '.hg') new_hgpath = File.join(@hg_storage_root, new_hg_relative_path) namespace_rpath = rpath_to_namespace_if_hgrc_included( new_hg_relative_path) # here (merge with GitLab 12.10), # it seems to be the caller's responsibility to check existence # and rescue from exceptions. FileUtils.mv(@hgpath, new_hgpath) return if namespace_rpath.nil? @hgpath = new_hgpath @hg_relative_path = new_hg_relative_path check_fix_hgrc_inheritance(namespace_rpath) end def remove super FileUtils.rm_rf(@hgpath) end def hg_size cmd = ["du", "-ks", @hgpath] kilobytes, status = popen(cmd) if status != 0 raise StandardError, "Could not run command #{command}" end # rounded to 2 decimal places is what Git::Repository does (kilobytes.to_f / 1024).round(2) end # Return repo size in megabytes (same as `super` as of this writing) def size super + hg_size end def hg_sha_map @git_sha_map = nil unless @hg_sha_map # Get the entries of the git-mapfile and store them in a git => hg Hash @hg_sha_map ||= File.open(File.join(@hgpath, '.hg', 'git-mapfile')) do |f| f.map do |line| line.match(/^([0-9a-f]{40}) ([0-9a-f]{40})\n$/) do |match| [match[1], match[2]] end end .to_h end rescue Errno::ENOENT # This can happen on rare occasions. # # Logically that means the correspondence is empty, but let's not cache it # so that we won't need exceptional cache invalidations # # For instance, Lfs::FileTransformer.lfs_file? # (called from Files::CreateService) queries tree/blob attributes while # it's not guaranteed yet that the repo isn't empty. In the current state # of Mercurial native projects (before HGitaly2 developments have even # started), this goes through Git Tree/Blob lookups... and a conversion # of Mercurial SHAs to Git that must not raise unexpected exceptions. {} end def git_sha_map # Reverse the entries, so that it is a hg => git Hash @git_sha_map ||= hg_sha_map.to_a.map {|x| x.reverse }.to_h end # Useful for repetitive calls, and will cache only the `true` result # Rationale: repetitive calls are typically for existing repositories. # (disabling the cop because that's really what we want, and there's # another cop preventing us to use a more explicit style) # rubocop:disable Gitlab/PredicateMemoization def cached_exists? @cache_exists ||= exists? end # rubocop:enable Gitlab/PredicateMemoization def sha_from_hgsha(hgsha) return unless cached_exists? git_sha_map[hgsha] end def hgsha_from_sha(sha) return unless cached_exists? hg_sha_map[sha] end override :archive_sha def archive_sha(sha) hgsha_from_sha(sha) end def commit_id_to_display_id(commit_id) hgsha_from_sha(commit_id) end def commit_id_from_display_id(display_id) sha_from_hgsha(display_id) end def batch_by_hgsha(sha_prefixes) # TODO stupid algorithm. Above a certain size, it's more efficient # to call hg with a the `sha_prefixes.join('+')` revset but then, we # actually need a specific command so that ambiguous (too short) # prefixes don't turn in hard errors that make just one ambiguous # among many break the lookup of all. res = [] git_sha_map.each_pair do |hgsha, gitsha| sha_prefixes.each do |pref| res << gitsha if hgsha.start_with?(pref) end end res end def hg_git_invalidate_maps! @git_sha_map = nil @hg_sha_map = nil end # Retrieve the Mercurial SHA from a symbolic revision # # `path` can be specified to use something else than # the main repository path, typically a temporary working dir # obtained through `hg share`, hence taking advantage of working dir # revisions (`.`) def hgsha_from_rev(hgrev, path = nil) hg_env = {'HGRCPATH' => Gitlab::Mercurial.hgrc_path, } path = @hgpath if path.nil? sha, status = popen([Gitlab.config.mercurial.bin_path, 'log', '--limit', '1', '-r', hgrev, '-T', '{node}'], path, hg_env) if status != 0 raise HgRevisionError, "Could not find changeset #{hgrev} in #{@hgpath}" end sha end # Find the named branch of a Mercurial changeset def hg_changeset_branch(hgsha) out, status = popen([Gitlab.config.mercurial.bin_path, 'log', '-r', hgsha, '-T', '{branch}'], @hgpath, {'HGRCPATH' => Gitlab::Mercurial.hgrc_path, }) if status != 0 raise HgError, "Could not get hg branch of changeset #{hgsha}: #{out}" end out.strip end # Return a suitable environment for write operations (merge etc.) # # This method assumes that write permission is already granted, # it adds the publish permission if the given user has the appropriate # role (at least Maintainer as of this writing, but we should have # a new Publisher role soon) def hg_env_for_write(user, force_system_user: false) if user.nil? and !force_system_user raise Gitlab::HgAccess::ForbiddenError, "write operation without user not allowed "\ "without force_system_user flag" end env = { # `name` and `email` are as in `committer` (which # does not have `username`) but the purpose here is # different: we want to pass them # along regardless of what should be a commit authorship. # If that latter information would change for any reason # these would still be directly linked to authenticated # user info 'HEPTAPOD_SKIP_ALL_GITLAB_HOOKS' => "yes", 'HEPTAPOD_USERINFO_ID' => user&.id.to_s, 'HEPTAPOD_USERINFO_USERNAME' => user&.username, 'HEPTAPOD_USERINFO_NAME' => user&.name, 'HEPTAPOD_USERINFO_EMAIL' => user&.email, 'HEPTAPOD_HG_NATIVE' => 'no', 'GL_REPOSITORY' => @gl_repository, 'HGUSER' => nil, 'HGRCPATH' => Gitlab::Mercurial.hgrc_path } unless @hg_project_for_perms.nil? env.update( { 'HEPTAPOD_PROJECT_PATH' => @hg_project_for_perms.path, 'HEPTAPOD_PROJECT_NAMESPACE_FULL_PATH' => @hg_project_for_perms.namespace.full_path }) end if force_system_user # we need a real user for pre-receive hooks to succeed # we'll change the display name for clarity, but `id` must # match for GitLab to retrieve it in the hooks payload # (`username` is probably necessary as well) unless sys_user = User.admins.order_id_asc.first raise StandardError, "Running with force_system_user, "\ "but no system user found" end env.update( { 'HGUSER' => 'Heptapod system', 'REMOTE_USER' => nil, # bypass Mercurial permission hooks 'HEPTAPOD_USERINFO_ID' => sys_user.id.to_s, 'HEPTAPOD_USERINFO_USERNAME' => sys_user.username, 'HEPTAPOD_USERINFO_NAME' => "Heptapod system", 'HEPTAPOD_USERINFO_EMAIL' => Gitlab.config.gitlab.email_from, } ) env.delete('HEPTAPOD_SKIP_ALL_GITLAB_HOOKS') unless unit_tests_skip_hooks elsif !@hg_project_for_perms.nil? env['REMOTE_USER'] = ::Gitlab::UserAccess.new(user, container: @hg_project_for_perms) .hg_can_publish? ? 'heptapod-publish' : 'heptapod-write' end env end def hg_prepare_user_encoding(hg_env, user, message) username = "#{user.name} <#{user.email}>" # we take the encoding from one of the two strings # it would be very surprising for them to differ, and # hg couldn't cope with that anyway (usually, it's UTF8) encname = username.encoding.name hg_env["HGENCODING"] = encname if message.encoding.name != encname # this would be very suprising, but if it ever happens # people debugging the case will be glad to have this log # (Mercurial could cope with that only in cases one of the # strings is actually pure ASCII (decoding is trivial) Rails.logger.error("Encoding of commit message #{message.encoding.name} differs "\ "from user name's #{encname}") end username end # Publish a changeset # # The given `user` information will be passed over to Mercurial # in the usual environment variables def hg_changeset_publish(user, hgsha, notify_gitlab: false, for_mr_iid: nil) env = hg_env_for_write(user) env.delete('HEPTAPOD_SKIP_ALL_GITLAB_HOOKS') if notify_gitlab && !unit_tests_skip_hooks env['HEPTAPOD_ACCEPT_MR_IID'] = for_mr_iid.to_s unless for_mr_iid.nil? out, status = popen([Gitlab.config.mercurial.bin_path, 'phase', '--public', '-r', hgsha], @hgpath, env) raise HgError, "Could not publish changeset #{hgsha}: #{out}" unless status.zero? end # Perform a Mercurial command # # `args` - list of command arguments for `hg` executable # `user` - user object responsible for running the hg command # `path` - the working directory path, typically obtained with `hg share` # `for_write` - if true, we prepare the env of write access for user # `success_code_can_be_one` - flags that success return code for hg command being run, can be 1 # `force_system_user` - force to use system user over Rails user def hg_call(args, user, path, env: nil, for_write: false, success_code_can_be_one: false, force_system_user: false) if env.nil? if for_write env = hg_env_for_write(user, force_system_user: force_system_user) else env = { 'HGRCPATH' => Gitlab::Mercurial.hgrc_path } end end username = hg_prepare_user_encoding(env, user, "") cmd = [Gitlab.config.mercurial.bin_path, '--config', 'ui.username=' + username] + args output, status = popen(cmd, path, env) if for_write hg_git_invalidate_maps! end if status != 0 if status == 1 && success_code_can_be_one # cmd passed with return code 1 else raise HgError, "Command failed (status code #{status}): "\ "'#{output}' command was: #{cmd}" end end end # Perform a Mercurial commit # # `path` - the working directory path, typically obtained with `hg share` # `username` - used as `ui.username` # `env` - preparred Hash of environment variables # # returns Mercurial SHA def hg_commit(path, username, message, env, add_remove = false) cmd = [Gitlab.config.mercurial.bin_path, 'commit', '--config', 'ui.username=' + username, '-m', message] cmd.append('--addremove') if add_remove output, status = popen(cmd, path, env) raise HgError, "Could not commit (status code #{status}): #{output}" if status != 0 hgsha_from_rev('.', path) end # Perform a Mercurial pull # # `gitlab_branches` - optional Array to make a precise pull, using as # many times `--rev` as needed. The remote revs are to be # given in GitLab branch notation, e.g., as `branch/default`. # If `nil`, a full pull will be performed. # If `[]`, nothing will happen # `env` - optionally allows to reuse a precomputed Hash of environment # variables. # `force_system_user` - can be set to true to perform with full Mercurial # permissions and allow `user` to be `nil`. # Ignored if `env` is not `nil` def hg_pull(user, remote_url, gitlab_branches: nil, env: nil, force_system_user: false) return if gitlab_branches&.empty? env = hg_env_for_write(user, force_system_user: force_system_user) if env.nil? cmd = [Gitlab.config.mercurial.bin_path, 'pull', remote_url] unless gitlab_branches.nil? gitlab_branches.each do |glb| cmd << '--rev' cmd << Gitlab::Mercurial::branchmap_key_for_gitlab_branch(glb) end end output, status = popen(cmd, @hgpath, env) if status == 1 prefix, complements = if gitlab_branches.nil? ["Full pull", ""] else ["Partial pull", " for GitLab branches #{gitlab_branches}"] end Rails.logger.info("#{prefix} from #{remote_url}#{complements} "\ "retrieved no changeset") elsif status != 0 raise HgError, "Could not pull (status code #{status}): "\ "'#{output}' command was #{cmd}" end hg_git_invalidate_maps! # there's only one possible field at this point: `divergent_ref` (not # really meaningful for hg) Gitaly::UpdateRemoteMirrorResponse.new end # Low level method to create a tag. # # This is *not* meant to be a full override of Git::Repository::tag. # # Returns the Node ID of the tagging commit. # # Optional parameters: # - env: allows to reuse a Hash of environment variables. # If not specified, the standard one for given user is computed # - tag_parent_revision: (tags have to be created on repository heads) # A revision to update the workdir to, before creating the tag. # It will indeed be the parent of the tagging commit. def hg_tag(user, tag_name, revision, env: nil, tag_parent_revision: "default") env = hg_env_for_write(user) if env.nil? username = hg_prepare_user_encoding(env, user, "") hg_tmp_workdir(tag_parent_revision) do |share_path| cmd = [Gitlab.config.mercurial.bin_path, 'tag', tag_name, '--config', 'ui.username=' + username] output, status = popen(cmd, share_path, env) raise HgError, "Could not tag (status code #{status}): #{output}" unless status == 0 hgsha_from_rev('.', share_path) end end def has_tmp_workdir?(prefix) !Dir.glob("#{@hgpath}tmp-#{prefix}-*").empty? end # Create an independent working directory # # this relies onto `hg share`, of which that's one of the main use # cases. # # `update_rev`: revision to update the new working directoy to # `prefix`: optional prefix used to check if a share is present # for a given operation. # # returns path to the working directory def hg_tmp_workdir(update_rev, prefix: nil) share_path = @hgpath + "tmp" unless prefix.nil? share_path += "-" + prefix + '-' end share_path += SecureRandom.hex begin hg_exe = Gitlab.config.mercurial.bin_path env = { 'HGRCPATH' => Gitlab::Mercurial.hgrc_path } output, status = popen([hg_exe, 'share', @hgpath, share_path, '--noupdate', '--bookmarks'], nil, env) raise HgError, "Could not share repo at #{@hgpath} to #{share_path}: #{output}" if status != 0 %w[hgrc hgrc.managed].each do |fname| src_hgrc_path = File.join(@hgpath, '.hg', fname) if File.exist?(src_hgrc_path) # FileUtils.copy_file happily overrides any existing file, # and is just what FileUtils.cp calls in its loop (but `cp` # for both files would expect them both to exist) FileUtils.copy_file(src_hgrc_path, File.join(share_path, '.hg', fname)) end rescue StandardError => exc raise HgError, "Could not copy HGRC file .hg/#{fname} from #{@hgpath} to #{share_path}: #{exc.message}" end unless update_rev.nil? output, status = popen([hg_exe, 'up', update_rev], share_path, env) raise HgError, "Could not update shared repo to #{update_rev}: #{output}" if status != 0 end yield share_path ensure FileUtils.rm_rf(share_path) end end # Produce a Mercurial merge changeset # # This method works with Mercurial SHAs only, and performs `merge` and `commit` in # insulation, by working in a temporary repository created with `share`. # # For `hg_git` repositories, the resulting writes to the Git repository are # protected by the Mercurial store lock, which is always held on the share source. def hg_merge_changeset(user, source_hgsha, target_hgsha, message, simulate = false) logprefix = "hg_merge_changeset " logprefix += "simulation " if simulate logprefix += "for #{@relative_path} #{source_hgsha} into #{target_hgsha}" logger = Rails.logger # could need to be adapted for gitaly-ruby (see initialize()) hg_exe = Gitlab.config.mercurial.bin_path hg_env = if user.nil? # TODO time to have a method for minimal hg_env { 'HGRCPATH' => Gitlab::Mercurial.hgrc_path } else hg_env_for_write(user) end hg_env.delete('HEPTAPOD_SKIP_ALL_GITLAB_HOOKS') unless unit_tests_skip_hooks hg_tmp_workdir(target_hgsha) do |share_path| logger.info("#{logprefix} calling #{hg_exe} merge -r #{source_hgsha}") output, status = popen([hg_exe, 'merge', '-t', 'internal:merge3', '-r', source_hgsha], share_path, hg_env) if status != 0 raise MergeError, "Could not merge hg #{source_hgsha} onto hg #{target_hgsha}: #{output}" end next if simulate username = hg_prepare_user_encoding(hg_env, user, message) commit_hgsha = hg_commit(share_path, username, message, hg_env) logger.info("#{logprefix} merge done, final changeset is #{commit_hgsha}") commit_hgsha end end # A temporary protection against unwanted publication, see heptapod#284. def forbid_merge_in_topic(target_branch) target_topic = Gitlab::Mercurial.parse_gitlab_branch_for_hg( target_branch)[1] raise ArgumentError, "Merge Requests targeting topics aren't supported yet" unless target_topic.nil? end def merge(user, source_sha, target_branch, message, for_mr_iid: nil) logprefix = "hg_merge for #{@relative_path} "\ "source_sha=#{source_sha} target_branch=#{target_branch}" logger = Rails.logger # could need to be adapted for gitaly-ruby (see initialize()) forbid_merge_in_topic(target_branch) logger.info("#{logprefix} starting") begin target_sha = find_branch(target_branch).target raise 'Invalid merge target' unless target_sha raise 'Invalid merge source' unless source_sha begin source_hgsha = hgsha_from_sha(source_sha) target_hgsha = hgsha_from_sha(target_sha) begin target_hg_branch = hg_changeset_branch(target_hgsha) source_hg_branch = hg_changeset_branch(source_hgsha) rescue HgError => e raise MergeError, e.message end with_merge_changeset = (target_hg_branch != source_hg_branch) || !(ancestor?(target_sha, source_sha)) result_hgsha = if with_merge_changeset hg_merge_changeset(user, source_hgsha, target_hgsha, message) else logger.info("#{logprefix} this is a merge without merge changeset") source_hgsha end logger.info("#{logprefix} publishing changeset #{result_hgsha}") begin hg_changeset_publish(user, result_hgsha, notify_gitlab: true, for_mr_iid: for_mr_iid) rescue HgError => e raise MergeError, e.message end logger.info("#{logprefix} merge done, final changeset is #{result_hgsha}") end hg_git_invalidate_maps! commit_id = sha_from_hgsha result_hgsha yield commit_id Gitlab::Git::OperationService::BranchUpdate.new(commit_id, false, false) end rescue Gitlab::Git::CommitError # when merge_index.conflicts? nil end def can_be_merged?(source_sha, target_branch) target_topic = Gitlab::Mercurial.parse_gitlab_branch_for_hg( target_branch)[1] return false unless target_topic.nil? logprefix = "Mercurial.can_be_merged? for #{@relative_path} "\ "source_sha=#{source_sha} target_branch=#{target_branch}" logger = Rails.logger # could need to be adapted for gitaly-ruby (see initialize()) return false unless source_sha target_sha = find_branch(target_branch)&.target if target_sha.nil? logger.warn("#{logprefix} could not resolve Git branch #{target_branch}") return false end if ancestor?(target_sha, source_sha) # always doable: true fast forward or merge changeset only # involving metadata (e.g., branch change) logger.info("#{logprefix} is mergeable (direct ancestor)") return true end source_hgsha = hgsha_from_sha(source_sha) target_hgsha = hgsha_from_sha(target_sha) if source_hgsha.nil? logger.warn( "#{logprefix} could not find Hg changeset for source Git commit #{source_sha}") return false end if target_hgsha.nil? logger.warn( "#{logprefix} could not find Hg changeset for target Git commit #{target_sha}") return false end begin hg_merge_changeset(nil, source_hgsha, target_hgsha, nil, true) logger.info("#{logprefix} is mergeable (true merge)") true rescue HgError, MergeError => e logger.info("#{logprefix} not mergeable: #{e.class}, #{e.message}") false end end def ff_merge(user, source_sha, target_branch, for_mr_iid: nil) forbid_merge_in_topic(target_branch) source_hgsha = hgsha_from_sha(source_sha) raise ArgumentError, 'Invalid merge target' unless find_branch(target_branch).target hg_branch = begin hg_changeset_branch(source_hgsha) rescue HgError => e raise MergeError, e.message end if target_branch != "branch/" + hg_branch # TODO how to return the proper error 406 seen from API? raise MergeError, "Being on branch #{hg_branch}, "\ "changeset #{source_hgsha} cannot be published "\ "within #{target_branch}" end begin hg_changeset_publish(user, source_hgsha, notify_gitlab: true, for_mr_iid: for_mr_iid) rescue HgError => e raise MergeError, e end Gitlab::Git::OperationService::BranchUpdate.new(source_sha, false, false) rescue Gitlab::Git::CommitError nil end # Squash a linear range of changesets. # # In typical usage (merge requests), start_sha is ill-named: # it is the head of the target branch (perhaps not really the head any # more). def squash(user, squash_id, start_sha:, end_sha:, author:, message:) logprefix = "hg_squash " logger = Rails.logger hg_exe = Gitlab.config.mercurial.bin_path hg_env = hg_env_for_write(user) start_hgsha = hgsha_from_sha(start_sha) end_hgsha = hgsha_from_sha(end_sha) username = hg_prepare_user_encoding(hg_env, author, message) hg_tmp_workdir(nil, prefix: "squash-#{squash_id}") do |share_path| # TODO: gracinet not 100% sure we need a workdir, but I don't see # an explicit "inmemory" option as there is for `hg rebase` # If we update to target_hgsha, then the fold will look like # an extra head and be rejected (probably because it is kept active by being the # working dir parent). Let's not update anywhere. revset = "ancestor(#{start_hgsha}, #{end_hgsha})::#{end_hgsha} "\ "- ancestor(#{start_hgsha}, #{end_hgsha})" logger.info("#{logprefix} calling `hg squash --exact` "\ "on revset `#{revset}` #{message.nil?} for #{username}") # Note that hg fold --exact will fail unless the revset is # "an unbroken linear chain". That fits the idea of a Merge Request # neatly, and should be unsuprising to users: it's natural to expect # squashes to stay simple. # In particular, if there's a merge of a side topic, it will be # unsquashable. # `allowunstable=no` protects us against all instabilities, # in particular against orphaning dependent topics. output, status = popen( [hg_exe, 'fold', '--exact', '-r', revset, '-m', message, '-u', username, '--config', 'experimental.evolution.allowunstable=no'], share_path, hg_env) if status != 0 raise HgError, "Could not fold revset `#{revset}` Error: #{output}" end logger.info("#{logprefix} squash done, finding successor") output, status = popen( [hg_exe, 'hpd-unique-successor', '-r', end_hgsha], share_path, hg_env) if status != 0 raise HgError, "Could not retrieve folded changeset "\ "(successor of #{end_hgsha})" end squash_hgsha = output.strip logger.info("#{logprefix} squash successor changeset #{squash_hgsha}") hg_git_invalidate_maps! sha_from_hgsha(squash_hgsha) end end # Mercurial squashes are atomic. We can pretend there's no squash # in progress. # # Actually at this point, reading the resulting changeset is not, # but it doesn't matter much: there's no risk of repo corruption. def squash_in_progress?(squash_id) has_tmp_workdir?("squash-#{squash_id}") end # Perform a rebase, only for topics. # # We may be subject to the same race condition than explained in # https://gitlab.com/gitlab-org/gitlab/-/issues/5966#note_150894306 # in short, there's a chance that the PostReceive gets executed # before the resulting commit is written in the database. # That's the kind of thing we will be able to fix once these # methods are reimplemented in HGitaly (should be after Heptapod 1.0) def rebase( user, rebase_id, branch:, branch_sha:, remote_repository:, remote_branch:, push_options: []) logprefix = "hg_rebase for #{@relative_path} of git #{branch_sha} "\ "(branch #{branch}) onto #{remote_branch}" logger = Rails.logger raise StandardError, "Rebasing between two different repos is not "\ "supported" unless remote_repository == self topic = Gitlab::Mercurial.parse_gitlab_branch_for_hg(branch)[1] raise HgError, "Only topics can be rebased" if topic.nil? hg_exe = Gitlab.config.mercurial.bin_path hg_env = hg_env_for_write(user) # we need to notify GitLab: the merge request update is done at # the receiving end of the post-receive hook hg_env.delete('HEPTAPOD_SKIP_ALL_GITLAB_HOOKS') unless unit_tests_skip_hooks # don't see an option to rebase with an obsmarker note username = hg_prepare_user_encoding(hg_env, user, "") end_hgsha = hgsha_from_sha(branch_sha) raise HgError, "Could not find hg changeset for Git #{branch_sha}" if end_hgsha.nil? # For consistency in corner cases (multiple heads) let's query the # Git sha for the target branch, even though we could also parse # and use Mercurial dest_sha = find_branch(remote_branch)&.target dest_hgsha = hgsha_from_sha(dest_sha) raise HgError, "Could not find hg changeset for Git #{branch_sha}" if dest_hgsha.nil? # revset insisting on using branch_sha because # - the topic could actually span several named branches (we could also # use the branch knowledge) # - if branch_sha is not the branch/topic head, that means something # is wrong. Instead of rebasing the whole, let's have the error # we get by refusing instabilities # topic naming rules make sure that we need no escaping revset = "topic(#{topic}) and ::#{end_hgsha}" rebase_hgsha = hg_tmp_workdir(nil, prefix: "rebase-#{rebase_id}") do |share_path| # TODO would be nice to experiment with in-memory rebase (wouldn't # need a working dir) but not sure what the good use cases are. # For instance, is a small rebase on a big repo much more efficient # in memory or should that precisely be avoided? # `allowunstable=no` protects us against all instabilities, # in particular against orphaning dependent topics. output, status = popen( [hg_exe, 'rebase', '-r', revset, '-d', dest_hgsha, '--tool', 'internal:merge3', # extension activation should also end up in py-heptapod's # required.hgrc, but let's make it work straight away '--config', 'extensions.rebase=', '--config', 'rebase.singletransaction=yes', '--config', 'experimental.evolution.allowunstable=no', '--config', 'ui.username=' + username, ], share_path, hg_env) if status != 0 raise HgError, "Could not rebase revset `#{revset}` "\ "onto #{dest_hgsha} Error: #{output}" end logger.info("#{logprefix} rebase done, finding successor") output, status = popen( [hg_exe, 'hpd-unique-successor', '-r', end_hgsha], share_path, hg_env) if status != 0 raise HgError, "Could not retrieve topic head after rebase "\ "(successor of #{end_hgsha})" end output.strip end hg_git_invalidate_maps! rebase_sha = sha_from_hgsha(rebase_hgsha) logger.info("#{logprefix} rebase successor changeset #{rebase_hgsha} "\ "(git sha #{rebase_sha})") yield rebase_sha rebase_sha end # first approx, but it'd be better to have a way to tell # TODO use rebase_id in tmp workdir name (same for squash) def rebase_in_progress?(rebase_id) has_tmp_workdir?("rebase-#{rebase_id}") end # Pull a given revision from URL forcing topic on all new changesets # # Return: # [full node, branch name, topic name] # # It is not guaranteed that the returned full node carries the given # topic. This happens for instance if it was already known and public. # In that case, the returned topic is empty def pull_force_topic(url, hgsha, topic) # necessary to amend changesets # TODO do better and think of encoding, as well hg_env = {'HGUSER' => 'Heptapod system', 'HEPTAPOD_SKIP_ALL_GITLAB_HOOKS' => 'yes', 'HGRCPATH' => Gitlab::Mercurial.hgrc_path, } hg_exe = Gitlab.config.mercurial.bin_path _output, status = popen( [hg_exe, 'pull-force-topic', '-q', '-r', hgsha, topic, url, '--config', 'experimental.single-head-per-branch=no', '--config', 'topic.publish-bare-branch=no', '--config', 'experimental.hg-git.accept-slash-in-topic-name=yes', '--config', 'experimental.hg-git.bookmarks-on-named-branches= yes',], @hgpath, hg_env ) if status == 1 query_rev = hgsha elsif status != 0 raise StandardError, "Could not pull-force-topic #{hgsha} from #{url}" else query_rev = topic end hg_git_invalidate_maps! node_branch, status = popen( [hg_exe, 'log', '-T', '{node}:{branch}:{topic}', '-r', query_rev], @hgpath, hg_env) if status != 0 raise HgRevisionError, "Could not log revision #{query_rev}" end node_branch.split(':') end def write_file(abspath, content) if content.is_a?(UploadedFile) # TODO mv would be more efficient, but it's unclear how much the # UploadedFile would be then broken and whether it matters. # (probably not much, the tempfile is an open file anyway). # Another option if on same FS (case where mv is efficient) # would be to make a hardlink. Playing safe for now. FileUtils.cp(content.path, abspath) else # File.write uses text mode, which forces conversion to # Encoding.external_encoding ('UTF-8') and gives an error. # On the other hand, a file explicitely open in binary mode # is happy to receive an UTF-8 string File.open(abspath, "wb") do |f| f.write(content) end end end # perform multiple write actions in the repository in a single commit # # typically multiple file writes or renames, followed by a commit # # actions details (should be the same as with Git::Repository): # # - create: ensures there's not a file at the same `file_path` already # and take care of intermediate directories # - update: ensures the file at `file_path` already exists. def multi_action(user, branch_name:, message:, actions:, author_email: nil, author_name: nil, start_branch_name: nil, start_sha: nil, start_repository: self, force: false) logprefix = "hg_multi_action for #{@relative_path} "\ "start_branch_name=#{start_branch_name} branch_name=#{branch_name}" logger = Rails.logger # could need to be adapted for gitaly-ruby (see initialize()) logger.info("#{logprefix} starting") hg_exe = Gitlab.config.mercurial.bin_path hg_env = hg_env_for_write(user) hg_env.delete('HEPTAPOD_SKIP_ALL_GITLAB_HOOKS') unless unit_tests_skip_hooks username = hg_prepare_user_encoding(hg_env, user, message) branch_creating = !branch_exists?(branch_name) repo_creating = exists? start_branch_name ||= branch_name start_hg_branch, start_topic = Gitlab::Mercurial.parse_gitlab_branch_for_hg( start_branch_name) # using the branchmap notation to perform just one update start_hg = if empty? nil else Gitlab::Mercurial::branchmap_key(start_hg_branch, start_topic) end commit_hgsha = hg_tmp_workdir(start_hg) do |share_path| hg_branch, topic = Gitlab::Mercurial.parse_gitlab_branch_for_hg( branch_name) if hg_branch != start_hg_branch # doing this inconditionally would be more that just a waste of # resources, it could reset the topic output, status = popen([hg_exe, 'branch', hg_branch], share_path, hg_env) if status != 0 raise HgError, "Could not switch working dir to branch #{hg_branch}: #{output}" end end if topic != start_topic if topic.nil? output, status = popen([hg_exe, 'topic', '--clear'], share_path, hg_env) if status != 0 raise HgError, "Could not clear topic: #{output}" end else output, status = popen([hg_exe, 'topic', topic], share_path, hg_env) if status != 0 raise HgError, "Could not set topic to #{topic}: #{output}" end end end logger.info("#{logprefix} Setup done, ready to execute actions "\ "in workdir #{share_path}") actions.each do |action| if !action[:file_path] raise HgError, "All current actions need a file_path argument" end file_path = File.join(*action[:file_path].split('/')) abspath = File.join(share_path, file_path) logger.info( "#{logprefix} Performing '#{action[:action]}' for #{file_path}") case action[:action].to_sym when :create if File.exists?(abspath) raise ActionError, "A file with this name already exists" end parent_abspath = File.split(abspath)[0] if File.file?(parent_abspath) raise ActionError, "A file with same name as parent directory "\ "already exists." end FileUtils.mkdir_p(parent_abspath) write_file(abspath, action[:content]) when :update unless File.file?(abspath) raise ActionError, "A file with this name doesn't exist" end write_file(abspath, action[:content]) when :create_dir if File.file?(abspath) raise ActionError, "A file with this name already exists " end if File.directory?(abspath) raise ActionError, "A directory with this name already exists" end FileUtils.mkdir_p(abspath) File.write(File.join(abspath, '.hg_keep'), '') when :move previous_path = File.join(action[:previous_path].split('/')) new_path = file_path unless File.exists?(File.join(share_path, previous_path)) raise ActionError, "A file with this name doesn't exist" end if File.exists?(abspath) raise ActionError, "A file with this name already exists" end # hg mv will take care of creating parent dir if needed output, status = popen([hg_exe, 'mv', previous_path, new_path], share_path, hg_env) # TODO should check if file is tracked and do a simple `mv` if not if status != 0 raise HgError, "Could not `hg mv #{previous_path} #{new_path}`: #{output}" end when :delete # Let's support chains of actions that could be simplified (use-case: generated). # We're currently never just 'adding' a file, but let's be future proof. unless File.exists?(abspath) raise ActionError, "A file with this name doesn't exist" end output, status = popen([hg_exe, 'rm', '-f', file_path], share_path, hg_env) if ![0, 1].include?(status) # status 1 happens with untracked files raise HgError, "Could not `hg rm -f #{file_path}`: #{output}" end # needed if the rm was actually just a forget: FileUtils.rm_f(abspath) else raise HgError, "Notimplemented action #{action}" end end logger.info("#{logprefix} Actions, done, now committing.") commit_hgsha = hg_commit(share_path, username, message, hg_env, true) logger.info("#{logprefix} commit done, changeset is #{commit_hgsha}") if topic.nil? && hg_config_item_bool?('experimental.topic.publish-bare-branch') hg_changeset_publish(user, commit_hgsha, notify_gitlab: true) logger.info("#{logprefix} published changeset #{commit_hgsha}") end commit_hgsha end hg_git_invalidate_maps! commit_gitsha = sha_from_hgsha commit_hgsha # Same kind of return as gitaly_operation_client.user_commit_files(), # itself called from `multi_action()` Gitlab::Git::OperationService::BranchUpdate.new(commit_gitsha, repo_creating, branch_creating) end # Compute path to a repo namespace HGRC, ready for inclusion from repo's HGRC. # namespace_path: must be relative to the storage root, as @relative_path # is. # # Returns: path as a string def hgrc_inherit_path(namespace_path) Pathname(namespace_path).join('hgrc') .relative_path_from(File.join(@hg_relative_path, '.hg')).to_s end # Absolute path to the repo HGRC directly read by Mercurial def hgrc_main_path File.join(@hgpath, '.hg', 'hgrc') end # Absolute path to the repo HGRC managed by the Rails app def hgrc_managed managed = 'hgrc.managed' [managed, File.join(@hgpath, '.hg', managed)] end HGRC_INCL_INHERIT_RX = /%include (.*)\/hgrc/.freeze # write the repository-local HGRC files. # # all configuration values are written in a separate `hgrc.managed` # file that is completely overwritten. # # in the main `hgrc`, only inclusions are managed # # inherit: if true, include directive for the namespace HGRC is included # first if not already there. # if false, it is removed if present # if nil, nothing happens # values: two-level Hash, with String leaf values # # Newlines in values are prohibited (use plain whitespace if needed), # as well as in keys (section and item names). def set_hgrc(user, inherit, namespace_path, values) key_rx = /^\w*$/ write_main = false main_path = hgrc_main_path managed, managed_path = hgrc_managed incl_managed = "%include " + managed main_lines = File.readlines(main_path) main_stripped_lines = main_lines.map { |l| l.strip } if !main_stripped_lines.include?(incl_managed) main_lines.push("\n", incl_managed + "\n") write_main = true end by_line = "by user #{user.username} (id=#{user.id}) at #{Time.new}" if inherit.nil? elsif inherit if match_idx(main_stripped_lines, HGRC_INCL_INHERIT_RX).nil? main_lines.insert(0, "%include " + hgrc_inherit_path(namespace_path)) main_lines.insert(0, "# inheritance restored #{by_line}") write_main = true end else inh_idx = match_idx(main_stripped_lines, HGRC_INCL_INHERIT_RX) if !inh_idx.nil? main_lines[inh_idx] = "# inheritance removed #{by_line}" write_main = true end end File.open(managed_path, 'w') do |hgrcf| hgrcf.puts( "# This file is entirely managed through the GitLab Rails app") hgrcf.puts("# last update #{by_line}") if !values.nil? values.each do |section, items| raise InvalidHgSetting(section) unless key_rx.match(section) hgrcf.puts("[#{section}]") items.each do |item, value| raise InvalidHgSetting(item) unless key_rx.match(section) raise InvalidHgSetting(value) if value.include?('\n') hgrcf.puts("#{item} = #{value}") end hgrcf.puts() end end end if write_main File.open(main_path, 'w') do |mainf| for line in main_lines # note for not-rubyists: puts() adds "\n" only if needeed mainf.puts(line) end end end end # Read the HGRC configuration files *for this repo* # # Return a Hash: # 'inherit' => bool # 'content' => raw content of the managed file def get_hgrc main_lines = File.readlines(hgrc_main_path) main_stripped_lines = main_lines.map { |l| l.strip } managed_path = hgrc_managed[1] content = File.exist?(managed_path) ? File.read(managed_path) : nil { 'inherit' => !match_idx(main_stripped_lines, HGRC_INCL_INHERIT_RX).nil?, 'content' => content, } end # Check HGRC inheritance from group path and fix it if needed. # # This is especially useful in cases of repository relocation: # - rename # - transfer # - hashed storage migration def check_fix_hgrc_inheritance(namespace_path) hgrc_path = hgrc_main_path Rails.logger.info("check_fix_hgrc_inheritance called with namespace_path=#{namespace_path} for #{hgrc_path}") main_lines = File.readlines(hgrc_path) inherit_idx = match_idx(main_lines, HGRC_INCL_INHERIT_RX) return if inherit_idx.nil? inherit_line = "%include " + hgrc_inherit_path(namespace_path) return unless inherit_line != main_lines[inherit_idx].strip main_lines[inherit_idx] = inherit_line File.open(hgrc_path, 'w') do |mainf| for line in main_lines # note for not-rubyists: puts() adds "\n" only if needeed mainf.puts(line) end end end # Return path to namespace, if namespace's HGRC included from the repo's # # The returned path is relative to storage root. # In case the namespace HGRC is not included from the repo's, the return # value is `nil`. # # TODO inelegant, we're using the regexp twice, only to finally # give back to a caller that will create a new instance and call # check_fix_hgrc_inheritance, redoing the whole scan. def rpath_to_namespace_if_hgrc_included(new_disk_path) hgrc_lines = File.readlines(hgrc_main_path) inherit_idx = match_idx(hgrc_lines, HGRC_INCL_INHERIT_RX) return if inherit_idx.nil? # TODO unnecessary reapplication of the regexp previous_line = hgrc_lines[inherit_idx] previous_namespace_rpath = HGRC_INCL_INHERIT_RX.match(previous_line)[1] Pathname(@hg_relative_path).join(".hg", previous_namespace_rpath) end SAFE_HG_HEPTAPOD_CONFIG = {'allow-multiple-heads' => 'bool', 'allow-bookmarks' => 'bool', 'auto-publish' => 'str'}.freeze HG_CONFIG_TRUE = ["1", "yes", "true", "on"].freeze def get_hg_heptapod_config(local: false) hg_env = {'HGUSER' => 'Heptapod system', 'HEPTAPOD_SKIP_ALL_GITLAB_HOOKS' => 'yes', 'HGRCPATH' => local ? "" : Gitlab::Mercurial.hgrc_path, } hg_exe = Gitlab.config.mercurial.bin_path out, status = popen([hg_exe, 'config', 'heptapod'], @hgpath, hg_env) return {} if status == 1 # as usual, code 1 is not an error for hg raise StandardError, "Can't read hg repository config" unless status.zero? config = {} out.lines.each do |line| key, value = line.split('=', 2) key = key.strip next unless key.start_with?('heptapod.') key = key[9..] # no attempt to convert back, that's up to the user # (we'd have to know the type for that) value = value.strip vtype = SAFE_HG_HEPTAPOD_CONFIG[key] next if vtype.nil? # not a safe item config[key] = converted = case vtype when 'bool' HG_CONFIG_TRUE.include?(value) when 'str' value else next end end config end protected def mask_password_in_url(url) # TODO there must be a generic helper doing the same thing to use # instead somewhere in GitLab and its dependencies! result = URI(url) result.password = "*****" unless result.password.nil? result.user = "*****" unless result.user.nil? #it's needed for oauth access_token result rescue url end end end end