This instance will be upgraded to the latest Heptapod 0.20rc on 2021-02-26 at 11:00 UTC+1

Commit 74c6f1bc authored by Georges Racinet's avatar Georges Racinet 🦑

WIP new mirroring class for native repos

Some consequences:

- `GitLabRefChange` methods for back-conversion in native mode
  and related error cases in its callers had to be removed.
  (actually the `Hook` classes should just accept `GitLabRefChange`
- Specific tests in `git.test_integration` for native mode
  have been dropped

branch : hgitaly2
parent a2dac5544bb7
Pipeline #16951 passed with stages
in 1 minute and 3 seconds
......@@ -39,31 +39,13 @@ class GitLabRefChange(object):
def is_creation(self):
return self.before == ZERO_SHA and self.after != ZERO_SHA
def export_for_hook(self, handler, native=False):
def export_for_hook(self):
"""Prepare for actual hook sending.
This depends on the `heptapod.native` boolean config item.
if native:
def hg_sha(git_sha):
if git_sha == ZERO_SHA:
return git_sha
return handler.map_hg_get(git_sha)
before_hg_sha = hg_sha(self.before)
if before_hg_sha is None:
# before is not known to Git, this is a corruption as in the
# old heptapod/hg-git#3 and we must recover by creating the
# Git branch anyway. On the 'after' side, it would mean
# a severe inconsistency on our side
# TODO in a later version we should
# not convert at all, but we still rely as Git being
# a giant cache of previous state of GitLab branches.
before_hg_sha = ZERO_SHA
after_hg_sha = hg_sha(self.after)
if after_hg_sha is None:
raise KeyError(self.after)
return before_hg_sha, after_hg_sha
This method used to provide back translation to Mercrial SHAs
in the native case, which has become useless with
TODO refactor, simply having the hook classes accept change objects.
return self.before, self.after
......@@ -836,16 +836,8 @@ class GitLabMirrorMixin:
native = ui.configbool(b'heptapod', b'native')
ui.note(b"heptapod_notify_gitlab heptapod.native=%r" % native)
if native and prune_reasons:
prune_reasons = {gl_branch: reason.convert_native(self)
for gl_branch, reason in prune_reasons.items()}
converted_changes = {ref: ch.export_for_hook(self, native=native)
for ref, ch in changes.items()}
except KeyError as exc:
ui.error(b"The after Git SHA %r has no Mercurial "
b"counterpart" % exc.args)
raise error.Abort("Inconsistency in internal conversion")
converted_changes = {ref: ch.export_for_hook()
for ref, ch in changes.items()}
# a verbose log line that will help with issues such as heptapod#278
......@@ -41,11 +41,6 @@ class HeadPruneReason(object):
def convert_native(self, git_handler):
"""Return a new reason, with all SHAs converted back to Mercurial.
return self
class HeadPruneReasonWithSha(HeadPruneReason):
"""Base class for all prune reasons whose info is a single SHA (bytes)
......@@ -54,9 +49,6 @@ class HeadPruneReasonWithSha(HeadPruneReason):
def extract(self):
def convert_native(self, handler):
return self.__class__(handler.map_hg_get(
class TopicPublished(HeadPruneReasonWithSha):
"""The current transaction published the topic GitLab branch.
......@@ -91,16 +83,6 @@ class BranchClosed(HeadPruneReason):
key = 'branch_closed'
def convert_native(self, handler):
if is None:
return self
hg_sha = handler.map_hg_get
return self.__class__(
[[hg_sha(closing), [hg_sha(p) for p in parents]]
for closing, parents in]
def extract(self):
if is None:
return None
# Copyright 2020 Georges Racinet <>
# This software may be used and distributed according to the terms of the
# GNU General Public License version 2 or any later version.
# SPDX-License-Identifier: GPL-2.0-or-later
"""Tests for the GitRefChange object.
(that may later be renamed as GitLabRefChange or other so that it doesn't
sound too weird once we are in full native mode)
TODO move now to heptapod.gitlab.tests
import pytest
from heptapod.gitlab.change import (
GitLabRefChange as RefChange,
class FakeHandler:
def __init__(self, **git2hg):
self.git2hg = git2hg
def map_hg_get(self, git_sha):
return self.git2hg.get(git_sha.decode())
def test_export_for_hook_errors():
handler = FakeHandler(known_git_sha=b'123d12fa45')
ch = RefChange(b'branch/something', b'known_git_sha', b'the_unknown')
with pytest.raises(KeyError) as exc_info:
ch.export_for_hook(handler=handler, native=True)
assert exc_info.value.args[0] == b'the_unknown'
ch = RefChange(b'branch/something', b'unknown', b'known_git_sha')
assert ch.export_for_hook(handler=handler, native=True) == (
ZERO_SHA, b'123d12fa45')
......@@ -38,8 +38,8 @@ def patch_gitlab_hooks(monkeypatch, records, action=None):
class GitLabMirrorFixture:
"""Helper class to create fixtures for GitLab aware hg-git mirroring.
class GitLabStateMaintainerFixture:
"""Helper class to create fixtures for GitLab state maintainers.
The pytest fixture functions themselves will have to be provided with
the tests that use them.
......@@ -47,36 +47,36 @@ class GitLabMirrorFixture:
It is not the role of this class to make decisions about scopes or
the kind of root directory it operates in.
There will be later on a fixture for Mercurial-native Heptapod
repositories, hence GitLab notifications without a Git repository.
This provides
- Mercurial repository test wrapper
- GitLab notifications interception
and is thus usable directly for native repositories.
base_path = attr.ib()
hg_repo_wrapper = attr.ib()
git_repo = attr.ib()
gitlab_notifs = attr.ib()
import heptapod.testhelpers.gitlab
def init(cls, base_path, monkeypatch, hg_config=None,
common_repo_name='repo', **kw):
if hg_config is None:
config = {}
config = deepcopy(hg_config)
config.setdefault('extensions', {})['hggit'] = ''
config['phases'] = dict(publish=False)
hg_repo_wrapper = RepoWrapper.init(
base_path / (common_repo_name + '.hg'),
git_repo = GitRepo.init(base_path / (common_repo_name + '.git'))
notifs = []
patch_gitlab_hooks(monkeypatch, notifs)
return cls(hg_repo_wrapper=hg_repo_wrapper,
def clear_gitlab_notifs(self):
"""Forget about all notifications already sent to GitLab.
......@@ -96,12 +96,6 @@ class GitLabMirrorFixture:
def delete(self):
git_path = self.git_repo.path
except Exception:
logger.exception("Error removing the Git repo at %r", git_path)
hg_path = self.hg_repo_wrapper.repo.root
......@@ -115,3 +109,32 @@ class GitLabMirrorFixture:
def __exit__(self, *exc_args):
return False # no exception handling related to exc_args
class GitLabMirrorFixture(GitLabStateMaintainerFixture):
"""Helper class to create fixtures for GitLab aware hg-git mirroring.
Adds the Git repository to GitLabStateMaintainerFixture
git_repo = attr.ib()
import heptapod.testhelpers.gitlab
def init(cls, base_path, monkeypatch,
common_repo_name='repo', **kw):
git_repo = GitRepo.init(base_path / (common_repo_name + '.git'))
return super(GitLabMirrorFixture, cls).init(
base_path, monkeypatch,
def delete(self):
git_path = self.git_repo.path
except Exception:
logger.exception("Error removing the Git repo at %r", git_path)
super(GitLabMirrorFixture, self).delete()
# Copyright 2021 Georges Racinet <>
# This software may be used and distributed according to the terms of the
# GNU General Public License version 2 or any later version.
# SPDX-License-Identifier: GPL-2.0-or-later
import subprocess
from ..git import GitRepo
def test_branches(tmpdir):
server = tmpdir / 'server.git'
# server_repo is bare, hence we cannot commit directly in there
server_repo = GitRepo.init(server)
client = tmpdir / 'client''git', 'init', str(client)))'git', 'remote', 'add', 'origin', str(server)),
cwd=str(client))'git', 'config', '', 'John Doe'),
cwd=str(client))'git', 'config', '', 'jdoe@heptapod.example'),
(client / 'foo').write('foo')'git', 'add', 'foo'), cwd=str(client))'git', 'commit', '-m', 'init commit', '--no-gpg-sign'),
cwd=str(client))'git', 'push', 'origin', 'master'), cwd=str(client))
assert server_repo.branch_titles() == {b'master': b'init commit'}
sha = server_repo.get_branch_sha('master')
sha_bytes = sha.encode()
assert server_repo.commit_hash_title('master') == [sha_bytes,
b'init commit']
assert server_repo.get_branch_sha(b'master') == sha
......@@ -34,6 +34,7 @@ from heptapod import (
from . import (
topic as topicmod,
# these have conditional imports and subsequent `None` testing in
......@@ -142,7 +143,10 @@ def pull_force_topic(ui, repo, topic, source=b"default",
def gitlab_mirror(ui, repo):
"""Export changesets as Git commits in the GitLab repository."""
git.HeptapodGitHandler(repo, repo.ui).export_commits()
if ui.configbool(b'heptapod', b'native'):
native.NativeGitLabStateMaintainer(ui, repo).sync()
git.HeptapodGitHandler(repo, repo.ui).export_commits()
# Copyright 2021 Georges Racinet <>
# This software may be used and distributed according to the terms of the
# GNU General Public License version 2 or any later version.
# SPDX-License-Identifier: GPL-2.0-or-later
"""Interaction with GitLab for native Mercurial repos.
Although we don't need to convert native repos to Git, we still have to
maintain the GitLab branches, and fire appropriate notifications.
from __future__ import absolute_import
import attr
import collections
import itertools
from mercurial.node import hex
from heptapod.gitlab.branch import (
from heptapod.gitlab.tag import gitlab_tag_ref
from heptapod.gitlab.mirror import GitLabMirrorMixin
from .branch import (
gitlab_tags, # TODO move in separate module?
class RefsByType:
"""Record GitLab refs, with a set for each type of ref.
Types of ref can be `tags`, `refs` or whatever becomes useful to track.
heads = attr.ib(default=attr.Factory(set))
tags = attr.ib(default=attr.Factory(set))
def __iter__(self):
return itertools.chain(self.heads, self.tags)
def __bool__(self):
return bool(self.heads) or bool(self.tags)
class NativeGitLabStateMaintainer(GitLabMirrorMixin):
def __init__(self, ui, repo):
self.ui = ui
self.repo = repo
self.unfiltered_repo = self.repo.unfiltered()
self._default_git_ref = None
gl_branches = gitlab_branches(repo)
if gl_branches is GITLAB_NAMED_REFS_MISSING:
# should mean we are on an empty repo
gl_branches = {}
gl_tags = gitlab_tags(repo)
# should mean we are on an empty repo
gl_tags = {}
self.gitlab_refs = {gitlab_branch_ref(gl_branch): sha
for gl_branch, sha in gl_branches.items()}
self.gitlab_refs.update((gitlab_tag_ref(gl_tag), sha)
for gl_tag, sha in gl_tags.items())
def sync(self):
"""Main entry point for synchronisation of shared state with GitLab."""
def get_default_gitlab_ref(self):
res = self._default_git_ref
if res is None:
gl_branch = get_default_gitlab_branch(self.repo)
if gl_branch is not None:
res = gitlab_branch_ref(gl_branch)
return res
def set_default_gitlab_ref(self, new_default_ref):
new_gl_branch = gitlab_branch_from_ref(new_default_ref)
b"Setting GitLab branch to %s" % new_gl_branch)
set_default_gitlab_branch(self.repo, new_gl_branch)
# cache invalidation
self._default_git_ref = None
def hg_sha_from_gitlab_sha(self, sha):
return sha
def gitlab_sha_from_hg_sha(self, sha):
return sha
def _filter_for_bookmarks(self, bms):
# A simple compatibily with GitHandler, but truly, since we
# don't use the branch_bookmark_suffix, we would be better off
# not calling this method from the base class
return [(bm, bm) for bm in bms]
def extract_bookmarks(self, git_refs):
"""Update git_refs with current bookmarks."""
for name, node in self.repo._bookmarks.items():
def extract_all_gitlab_refs(self):
"""Heptapod version of GitHandler.get_exportable() for native case.
We should be able to do without tags, as they are handled separately
(no persistent state in Mercurial repo, just notifications by
git_refs = collections.defaultdict(RefsByType)
return git_refs
def write_gitlab_branches(self):
def write_gitlab_tags(self):
......@@ -16,15 +16,12 @@ from mercurial import (
import pytest
import re
from dulwich.protocol import ZERO_SHA
from heptapod.testhelpers import (
from heptapod.gitlab import prune_reasons
from heptapod.gitlab.branch import gitlab_branch_ref as git_branch_ref
from heptapod.gitlab.change import GitLabRefChange as RefChange
from heptapod.gitlab.tag import gitlab_tag_ref
from ...git import (
......@@ -32,9 +29,6 @@ from ...git import (
from mercurial import pycompat
from ..utils import common_config
# TODO move to helper module
from .test_integration import patch_gitlab_hooks
def bkey(**d):
"""Replacement for `dict(k=v, ...)` that produces bytes keys"""
......@@ -126,31 +120,6 @@ def test_notify_gitlab_empty_changes(tmpdir):
assert handler.heptapod_notify_gitlab('some-hook', {}, {}) is None
def test_native_unknown_git_shas(tmpdir):
wrapper = RepoWrapper.init(tmpdir.join('repo'),
wrapper.write_commit('foo', message='foo0')
handler = HeptapodGitHandler(wrapper.repo, wrapper.repo.ui)
hg_map = {b'known': b'cafe7654'}
handler.map_hg_get = lambda k: hg_map.get(k)
records = []
def hook(*a):
return 0, b'', b''
ch = RefChange(b'topic/default/zetop', before=b'unknown', after=b'known')
handler.heptapod_notify_gitlab(hook, {}, {b'topic/default/zetop': ch})
_prune, changes = records[0][0]
assert changes == {b'topic/default/zetop': (ZERO_SHA, b'cafe7654')}
ch = RefChange(b'topic/default/zetop', before=b'known', after=b'unknown')
with pytest.raises(error.Abort):
handler.heptapod_notify_gitlab(hook, {}, {b'topic/default/zetop': ch})
def test_analyse_vanished_topic_lookup_error(tmpdir):
wrapper = RepoWrapper.init(tmpdir.join('repo'))
handler = HeptapodGitHandler(wrapper.repo, wrapper.repo.ui)
......@@ -341,32 +310,6 @@ def test_generate_prune_changes_existing_unknown(tmpdir):
assert reasons == {gitlab_branch: closed_reason}
def test_never_prune_default_branch(tmpdir, monkeypatch):
notifs = []
patch_gitlab_hooks(monkeypatch, notifs)
config = common_config()
config['heptapod'] = dict(native=True)
wrapper = RepoWrapper.init(tmpdir.join('repo'), config=config)
handler = HeptapodGitHandler(wrapper.repo, wrapper.repo.ui)
def no_analyse(existing, exportable):
# that's just not the point here
return {}
handler.analyse_vanished_refs = no_analyse
with pytest.raises(error.Abort) as exc_info:
{ZERO_SHA: {b'branch/default': prune_reasons.HeadPruneReason()}})
assert'prune.*default branch', exc_info.value.args[0])
def test_heptapod_compare_tags_inconsistent(tmpdir):
wrapper = RepoWrapper.init(tmpdir.join('repo'),
......@@ -1163,109 +1163,6 @@ def test_heptapod_notify_gitlab(tmpdir, monkeypatch):
def test_heptapod_notify_gitlab_native(main_fixture):
fixture = main_fixture
wrapper = fixture.hg_repo_wrapper
git_repo = fixture.git_repo
notifs = fixture.gitlab_notifs
topic_ctx = wrapper.write_commit('zz', message='in topic', topic='zzetop')
gl_branch_default = b'branch/default'
default_hg_sha = wrapper.repo[1].hex()
default_git_sha = git_repo.get_branch_sha(gl_branch_default).encode()
gl_branch_topic = b'topic/default/zzetop'
topic_hg_sha = topic_ctx.hex()
topic_git_sha = git_repo.get_branch_sha(gl_branch_topic).encode()
handler = HeptapodGitHandler(wrapper.repo, wrapper.repo.ui)
hook1 = hooks.PreReceive(wrapper.repo)
hook2 = hooks.PostReceive(wrapper.repo)
wrapper.repo.ui.setconfig(b'heptapod', b'native', b'yes')
# branch creation, interesting because ZERO_SHA isn't in the mappings
change = GitRefChange(b'something', ZERO_SHA, topic_git_sha)
handler.heptapod_notify_gitlab(hook1, [], {b'something': change})
assert notifs == [
('pre-receive', ([], {b'something': (ZERO_SHA, topic_hg_sha)}))
# branch change
# second hook call to check for side effects in pre/post-receive succession
change = GitRefChange(b'branch/default', default_git_sha, topic_git_sha)
handler.heptapod_notify_gitlab(hook1, [], {b'branch/default': change})
assert notifs == [
('pre-receive', ([],
{b'branch/default': (default_hg_sha, topic_hg_sha)}))
handler.heptapod_notify_gitlab(hook2, [], {b'branch/default': change})
assert notifs == [
('post-receive', ([],
{b'branch/default': (default_hg_sha, topic_hg_sha)}))
# Topic published (making this realistic, but consistency shouldn't matter)
# this tests most of the PruneReasons. If a new one not subclassing
# HeadPruneReasonWithSha would appear, coverage drop would warn us.
prune_change = GitRefChange(gl_branch_topic, topic_git_sha, ZERO_SHA)
prune_reasons = {gl_branch_topic: TopicPublished(topic_git_sha)}
changes = {gl_branch_default: change, gl_branch_topic: prune_change}
expected = ({gl_branch_topic: TopicPublished(topic_hg_sha)},
{gl_branch_default: (default_hg_sha, topic_hg_sha),
gl_branch_topic: (topic_hg_sha, ZERO_SHA),
handler.heptapod_notify_gitlab(hook1, prune_reasons, changes)
assert notifs[-1] == ('pre-receive', expected)
handler.heptapod_notify_gitlab(hook2, prune_reasons, changes)
assert notifs[-1] == ('post-receive', expected)
# BranchClosed.
# This example is less realistic, we pretend the topic changeset
# actually closes the default branch. At this low level, it doesn't matter.
prune_change = GitRefChange(gl_branch_default, default_git_sha, ZERO_SHA)
changes = {gl_branch_default: prune_change}
prune_reasons = {
gl_branch_default: BranchClosed([(topic_git_sha, [default_git_sha])]),
expected = (