Skip to content
Snippets Groups Projects
Commit d2a9a756 authored by Georges Racinet's avatar Georges Racinet
Browse files

Gitaly Comparison tests: generic comparison methods

This follows the development patterns we've been using lately:
a single bunch of request attributes is given to, e.g, the
`assert_compare` method, which translates it to Git for Gitaly.
Request translation is fairly simple, taking care of direct
attributes that are declared to be commit SHAs.

Before comparison, the Mercurial response is also translated to
Git. This is done with a list of attribute dotted paths, with
the special convention that a name ending in `[]` represents a
list-like value on which iteration must happen (see example
in `tests_with_gitaly/test_commit.py`).

For cases where attributes are not comparable, or more generally
acknowledged to differ, the `normalizer` callable can be used.

It's not clear this makes tests easier to write, but it probably
makes them easier to read.

For the conversion of Mercurial SHAs to Git SHAs, we're
accessing the hg-git class, which is a bit heavy-handed but gives
us more chances to understand future incompatibility, as the
on-disk format storage of the mapping is due to change in hg-git.
parent e47f9676
No related branches found
No related tags found
1 merge request!68more helpers for comparison tests
Pipeline #28095 passed
......@@ -7,5 +7,7 @@
"""Fixture for Gitaly comparison tests based on Heptapod's Git mirroring."""
import attr
import contextlib
import functools
import pytest
import random
......@@ -10,5 +12,10 @@
import random
import grpc
from mercurial_testhelpers.util import as_bytes
from hggit import GitHandler
from heptapod.testhelpers.gitlab import GitLabMirrorFixture
from hgitaly.stub.shared_pb2 import Repository
......@@ -32,6 +39,18 @@
def rpc_helper(self, **kw):
return RpcHelper(self, **kw)
@functools.cached_property
def hg_git(self):
"""The hg-git GitHandler instance, with SHA mapping preloaded.
To invalidate this cached property, just delete it::
del comparison.hg_git
"""
hg_repo = self.hg_repo_wrapper.repo
hg_git = GitHandler(hg_repo, hg_repo.ui)
hg_git.load_map()
return hg_git
@contextlib.contextmanager
def gitaly_comparison_fixture(server_repos_root,
......@@ -76,6 +95,10 @@
def __init__(self, comparison, stub_cls, method_name, request_cls,
repository_arg=True,
request_defaults=None,
request_sha_attrs=(),
response_sha_attrs=(),
normalizer=None,
streaming=False):
self.comparison = comparison
self.stubs = dict(git=stub_cls(comparison.gitaly_channel),
......@@ -86,6 +109,12 @@
self.streaming = streaming
self.repository_arg = repository_arg
self.request_defaults = request_defaults
self.request_sha_attrs = request_sha_attrs
self.response_sha_attrs = response_sha_attrs
self.streaming = streaming
self.normalizer = normalizer
def rpc(self, vcs, **kwargs):
if self.repository_arg:
kwargs.setdefault('repository', self.comparison.gitaly_repo)
......@@ -95,3 +124,102 @@
return [resp for resp in meth(request)]
return meth(request)
def hg2git(self, hg_sha):
"""Convert a Mercurial hex SHA to its counterpart SHA in Git repo.
If not found in the Git Repo, the original SHA is returned, which
is useful for tests about non existent commits.
"""
# if hg_sha is None or not 40 bytes long it certainly won't
# be found in the hg-git mapping, we don't need a special case
# for that
git_sha = self.comparison.hg_git.map_git_get(as_bytes(hg_sha))
return hg_sha if git_sha is None else git_sha
def request_kwargs_to_git(self, hg_kwargs):
git_kwargs = hg_kwargs.copy()
for sha_attr in self.request_sha_attrs:
sha = hg_kwargs.get(sha_attr)
if sha is None or len(sha) != 40:
continue
git_kwargs[sha_attr] = self.hg2git(sha)
return git_kwargs
def response_to_git(self, resp):
sha_attr_paths = [path.split('.') for path in self.response_sha_attrs]
if self.streaming:
for message in resp:
self.message_to_git(message, sha_attr_paths)
else:
self.message_to_git(resp, sha_attr_paths)
def message_to_git(self, message, attr_paths):
for attr_path in attr_paths:
obj = message
trav = list(attr_path)
recurse = False
while len(trav) > 1:
attr_name, trav = trav[0], trav[1:]
recurse = attr_name.endswith('[]')
if recurse:
attr_name = attr_name[:-2]
obj = getattr(obj, attr_name)
if recurse:
for msg in obj:
self.message_to_git(msg, [trav])
break
if recurse:
# this attr_path was fully treated by recursion
continue
obj_attr = trav[0]
scalar_list = obj_attr.endswith('[]')
if scalar_list:
obj_attr = obj_attr[:-2]
value = getattr(obj, obj_attr)
if scalar_list:
for i, sha in enumerate(value):
value[i] = self.hg2git(sha)
else:
setattr(obj, obj_attr, self.hg2git(value))
def apply_request_defaults(self, kwargs):
defaults = self.request_defaults
if defaults is not None:
for k, v in defaults.items():
kwargs.setdefault(k, v)
def assert_compare(self, **hg_kwargs):
self.apply_request_defaults(hg_kwargs)
git_kwargs = self.request_kwargs_to_git(hg_kwargs)
hg_response = self.rpc('hg', **hg_kwargs)
git_response = self.rpc('git', **git_kwargs)
self.response_to_git(hg_response)
norm = self.normalizer
if norm is not None:
norm(hg_response)
norm(git_response)
assert hg_response == git_response
def assert_compare_errors(self, same_details=True, **hg_kwargs):
self.apply_request_defaults(hg_kwargs)
git_kwargs = self.request_kwargs_to_git(hg_kwargs)
with pytest.raises(grpc.RpcError) as exc_info_hg:
self.rpc('hg', **hg_kwargs)
with pytest.raises(grpc.RpcError) as exc_info_git:
self.rpc('git', **git_kwargs)
exc_hg = exc_info_hg.value
exc_git = exc_info_git.value
assert exc_hg.code() == exc_git.code()
if same_details:
assert exc_hg.details() == exc_git.details()
return exc_hg, exc_git
......@@ -25,7 +25,6 @@
def test_compare_list_last_commits_for_tree(gitaly_comparison):
fixture = gitaly_comparison
repo_message = fixture.gitaly_repo
git_repo = fixture.git_repo
wrapper = fixture.hg_repo_wrapper
......@@ -56,17 +55,15 @@
git_branches = git_repo.branches()
assert set(git_branches) == {b'branch/default', b'branch/other'}
# TODO check if we can access the hg-git map, would be easier
git_shas[ctx2.hex()] = git_branches[b'branch/default']['sha']
git_shas[ctx3.hex()] = git_branches[b'branch/other']['sha']
hg_shas = {git: hg for hg, git in git_shas.items()}
commit_stubs = dict(git=CommitServiceStub(fixture.gitaly_channel),
hg=CommitServiceStub(fixture.hgitaly_channel))
def do_rpc(vcs, rev, path, offset=0, limit=None):
if vcs == 'git' and len(rev) == 40:
# defaulting useful for tests of unknown revs
rev = git_shas.get(rev, rev)
def response_ignores(responses):
for resp in responses:
for commit_for_tree in resp.commits:
commit = commit_for_tree.commit
# TODO tree_id should be replaced by HGitaly standard value
# once HGitaly2 is the norm
commit.tree_id = ''
# Git adds a line ending
# hg-git adds a branch marker
commit.body = b''
commit.body_size = 0
......@@ -72,22 +69,19 @@
request = ListLastCommitsForTreeRequest(
repository=repo_message,
revision=rev,
offset=offset,
limit=1000 if limit is None else limit,
path=path)
def convert(sha):
sha = sha.encode()
if vcs == 'hg':
return sha
# fallback to incoming value for easier debugging than `None`
return hg_shas.get(sha, sha)
return [(cft.path_bytes, convert(cft.commit.id))
for chunk in commit_stubs[vcs].ListLastCommitsForTree(request)
for cft in chunk.commits]
rpc_helper = fixture.rpc_helper(stub_cls=CommitServiceStub,
method_name='ListLastCommitsForTree',
streaming=True,
request_cls=ListLastCommitsForTreeRequest,
request_defaults=dict(limit=1000),
request_sha_attrs=['revision'],
response_sha_attrs=[
'commits[].commit.id',
'commits[].commit.parent_ids[]',
],
normalizer=response_ignores,
)
assert_compare = rpc_helper.assert_compare
assert_compare_errors = rpc_helper.assert_compare_errors
for path in (b'sub/dir', b'sub/dir/', b'', b'.', b'/', b'./',
b'sub', b'sub/', b'foo'):
for rev in ('branch/default', 'branch/other', ctx2.hex(), ctx3.hex()):
......@@ -90,6 +84,6 @@
for path in (b'sub/dir', b'sub/dir/', b'', b'.', b'/', b'./',
b'sub', b'sub/', b'foo'):
for rev in ('branch/default', 'branch/other', ctx2.hex(), ctx3.hex()):
assert do_rpc('hg', rev, path) == do_rpc('git', rev, path)
assert_compare(revision=rev, path=path)
......@@ -95,10 +89,3 @@
assert (do_rpc('hg', 'branch/default', b'sub', offset=1)
==
do_rpc('git', 'branch/default', b'sub', offset=1))
rev, path = ctx2.hex(), b''
assert (do_rpc('hg', rev, path, limit=0)
==
do_rpc('git', rev, path, limit=0))
assert_compare(revision='branch/default', path=b'sub', offset=1)
......@@ -104,8 +91,7 @@
with pytest.raises(grpc.RpcError) as exc_info_hg:
do_rpc('hg', rev, path, limit=-1)
with pytest.raises(grpc.RpcError) as exc_info_git:
do_rpc('git', rev, path, limit=-1)
assert exc_info_hg.value.code() == exc_info_git.value.code()
assert exc_info_hg.value.details() == exc_info_git.value.details()
# for a bunch of assertions that aren't about revision nor path
common_args = dict(revision=ctx2.hex(), path=b'')
assert_compare(limit=0, **common_args)
assert_compare_errors(limit=-1, **common_args)
assert_compare_errors(limit=10, offset=-1, **common_args)
......@@ -111,20 +97,7 @@
with pytest.raises(grpc.RpcError) as exc_info_hg:
do_rpc('hg', rev, path, offset=-1)
with pytest.raises(grpc.RpcError) as exc_info_git:
do_rpc('git', rev, path, offset=-1)
assert exc_info_hg.value.code() == exc_info_git.value.code()
assert exc_info_hg.value.details() == exc_info_git.value.details()
# error won't be due to invalidity as a SHA
# (let's not depend on Gitaly accepting revisions, here)
rev = b'be0123ef' * 5
with pytest.raises(grpc.RpcError) as exc_info_hg:
do_rpc('hg', rev, path)
with pytest.raises(grpc.RpcError) as exc_info_git:
do_rpc('git', rev, path)
assert exc_info_hg.value.code() == exc_info_git.value.code()
assert exc_info_hg.value.details() == exc_info_git.value.details()
# error won't be due to invalidity as a SHA, but because commit doesn't
# exist (let's not depend on Gitaly accepting symbolic revisions, here)
assert_compare_errors(revision=b'be0123ef' * 5, path=b'')
def test_compare_raw_blame(gitaly_comparison):
......
......@@ -193,7 +193,6 @@
wrapper = fixture.hg_repo_wrapper
gl_default = b'branch/default'
gl_other = b'branch/other'
base_hg_ctx = wrapper.write_commit('foo', message="base")
base_hg_sha = base_hg_ctx.hex()
# TODO get_branch_sha does not work because of PosixPath not having
......@@ -208,12 +207,6 @@
other_hg_sha = wrapper.write_commit('foo', message="other",
branch="other",
parent=base_hg_ctx).hex()
other_git_sha = git_repo.branches()[gl_other]['sha']
hg2git = {base_hg_sha: git_sha0,
default_hg_sha: git_sha1,
other_hg_sha: other_git_sha,
}
rpc_helper = fixture.rpc_helper(stub_cls=RefServiceStub,
method_name='FindRefName',
......@@ -217,5 +210,7 @@
rpc_helper = fixture.rpc_helper(stub_cls=RefServiceStub,
method_name='FindRefName',
request_cls=FindRefNameRequest)
request_cls=FindRefNameRequest,
request_sha_attrs=['commit_id'],
)
......@@ -221,12 +216,5 @@
def assert_compare(commit_hg_id, prefix):
if len(commit_hg_id) == 40:
# defaulting useful for tests of unknown revs
commit_git_id = hg2git.get(commit_hg_id, commit_hg_id)
assert (rpc_helper.rpc('hg', commit_id=commit_hg_id, prefix=prefix)
==
rpc_helper.rpc('git', commit_id=commit_git_id, prefix=prefix))
assert_compare = rpc_helper.assert_compare
# Git returns the first ref in alphabetic order, hence not branch/default
# for the base commit because 'default' < 'other'
......@@ -237,8 +225,8 @@
b'refs/heads/branch/default',
):
assert_compare(base_hg_sha, prefix)
assert_compare(default_hg_sha, prefix)
assert_compare(commit_id=base_hg_sha, prefix=prefix)
assert_compare(commit_id=default_hg_sha, prefix=prefix)
for prefix in (b'refs/heads',
b'refs/heads/',
......@@ -246,9 +234,9 @@
b'refs/heads/branch/',
b'refs/heads/branch/other',
):
assert_compare(other_hg_sha, prefix)
assert_compare(commit_id=other_hg_sha, prefix=prefix)
# cases where response should be empty
for prefix in (b'refs/heads/bra',
b'refs/heads/branch/def',
):
......@@ -250,7 +238,7 @@
# cases where response should be empty
for prefix in (b'refs/heads/bra',
b'refs/heads/branch/def',
):
assert_compare(base_hg_sha, prefix)
assert_compare(default_hg_sha, prefix)
assert_compare(commit_id=base_hg_sha, prefix=prefix)
assert_compare(commit_id=default_hg_sha, prefix=prefix)
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment