diff --git a/tests_with_gitaly/comparison.py b/tests_with_gitaly/comparison.py index eb5ffde16fdd94d2bf2d123370ddd500d31f3daf_dGVzdHNfd2l0aF9naXRhbHkvY29tcGFyaXNvbi5weQ==..026fbefd2e2dd950ac6946656a5c60c2c449d11b_dGVzdHNfd2l0aF9naXRhbHkvY29tcGFyaXNvbi5weQ== 100644 --- a/tests_with_gitaly/comparison.py +++ b/tests_with_gitaly/comparison.py @@ -7,6 +7,7 @@ """Fixture for Gitaly comparison tests based on Heptapod's Git mirroring.""" import attr import contextlib +from copy import deepcopy import functools import pytest import random @@ -115,6 +116,8 @@ feature_flags=(), normalizer=None, error_details_normalizer=None, + chunked_fields_remover=None, + chunks_concatenator=None, streaming=False): self.comparison = comparison self.stubs = dict(git=stub_cls(comparison.gitaly_channel), @@ -132,6 +135,8 @@ self.feature_flags = list(feature_flags) self.normalizer = normalizer self.error_details_normalizer = error_details_normalizer + self.chunked_fields_remover = chunked_fields_remover + self.chunks_concatenator = chunks_concatenator def grpc_metadata(self): return feature.as_grpc_metadata(self.feature_flags) @@ -232,7 +237,14 @@ for k, v in defaults.items(): kwargs.setdefault(k, v) - def assert_compare(self, **hg_kwargs): + def call_backends(self, **hg_kwargs): + """Call Gitaly and HGitaly with uniform request kwargs. + + To be used only if no error is expected. + + :param hg_kwargs: used as-is to construct the request for HGitaly, + converted to Git and then to a request for Gitaly. + """ self.apply_request_defaults(hg_kwargs) git_kwargs = self.request_kwargs_to_git(hg_kwargs) @@ -240,8 +252,11 @@ hg_response = self.rpc('hg', **hg_kwargs) git_response = self.rpc('git', **git_kwargs) + return hg_response, git_response + + def normalize_responses(self, hg_response, git_response): self.response_to_git(hg_response) norm = self.normalizer if norm is not None: norm(self, hg_response, vcs='hg') norm(self, git_response, vcs='git') @@ -243,7 +258,11 @@ self.response_to_git(hg_response) norm = self.normalizer if norm is not None: norm(self, hg_response, vcs='hg') norm(self, git_response, vcs='git') + + def assert_compare(self, **hg_kwargs): + hg_response, git_response = self.call_backends(**hg_kwargs) + self.normalize_responses(hg_response, git_response) assert hg_response == git_response @@ -248,5 +267,53 @@ assert hg_response == git_response + def assert_compare_aggregated(self, + compare_first_chunks=True, + check_both_chunked=True, + **hg_kwargs): + """Compare streaming responses with appropriate concatenation. + + Sometimes, it's unreasonable to expect HGitaly chunking to + exactly match Gitaly's. This method allows to compare after + regrouping the chunks, with the provided :attr:`chunks_concatenator`. + + Usually Gitaly returns small values within the first response only, + to avoid the bandwidth waste of repetiting them. This helper + checks that HGitaly does the same by comparing after applying + :attr: `chunked_fields_remover` to as many responses as possible + (typically the number of responses would differ). + + :param bool compare_first_chunk: if ``True``, the first chunks of + both responses are directly compared (including main content). + :param bool check_both_chunked: if ``True` checks that we get + more than one response for both HGitaly and Gitaly + :return: a pair: the first chunk of responses for Gitaly and HGitaly + respectively, taken before normalization. This can be useful, e.g., + for pagination parameters. + """ + assert self.streaming # for consistency + + hg_resps, git_resps = self.call_backends(**hg_kwargs) + original_first_chunks = deepcopy((git_resps[0], hg_resps[0])) + + self.normalize_responses(hg_resps, git_resps) + if compare_first_chunks: + assert hg_resps[0] == git_resps[0] + + if check_both_chunked: + assert len(hg_resps) > 1 + assert len(git_resps) > 1 + + concatenator = getattr(self, 'chunks_concatenator') + fields_remover = getattr(self, 'chunked_fields_remover') + assert concatenator(hg_resps) == concatenator(git_resps) + + for hg_resp, git_resp in zip(hg_resps, git_resps): + fields_remover(hg_resp) + fields_remover(git_resp) + assert hg_resp == git_resp + + return original_first_chunks + def assert_compare_errors(self, same_details=True, **hg_kwargs): self.apply_request_defaults(hg_kwargs) diff --git a/tests_with_gitaly/test_blob_tree.py b/tests_with_gitaly/test_blob_tree.py index eb5ffde16fdd94d2bf2d123370ddd500d31f3daf_dGVzdHNfd2l0aF9naXRhbHkvdGVzdF9ibG9iX3RyZWUucHk=..026fbefd2e2dd950ac6946656a5c60c2c449d11b_dGVzdHNfd2l0aF9naXRhbHkvdGVzdF9ibG9iX3RyZWUucHk= 100644 --- a/tests_with_gitaly/test_blob_tree.py +++ b/tests_with_gitaly/test_blob_tree.py @@ -4,4 +4,5 @@ # GNU General Public License version 2 or any later version. # # SPDX-License-Identifier: GPL-2.0-or-later +from copy import deepcopy import pytest @@ -7,6 +8,4 @@ import pytest -import grpc - from hgitaly.oid import ( blob_oid, @@ -10,6 +9,10 @@ from hgitaly.oid import ( blob_oid, + tree_oid, +) +from hgitaly.revision import ( + gitlab_revision_changeset, ) from hgitaly.stream import WRITE_BUFFER_SIZE @@ -32,32 +35,6 @@ pytestmark = pytest.mark.skip -class TreeBlobFixture: - - def __init__(self, gitaly_comparison): - self.comparison = gitaly_comparison - self.hg_repo_wrapper = gitaly_comparison.hg_repo_wrapper - self.git_repo = gitaly_comparison.git_repo - - self.gitaly_repo = gitaly_comparison.gitaly_repo - self.commit_stubs = dict( - git=CommitServiceStub(self.comparison.gitaly_channel), - hg=CommitServiceStub(self.comparison.hgitaly_channel)) - self.blob_stubs = dict( - git=BlobServiceStub(self.comparison.gitaly_channel), - hg=BlobServiceStub(self.comparison.hgitaly_channel)) - - def tree_entry(self, vcs, path, revision=b'branch/default', - limit=0, max_size=0): - request = TreeEntryRequest(repository=self.gitaly_repo, - revision=revision, - limit=limit, - max_size=max_size, - path=path) - resp = self.commit_stubs[vcs].TreeEntry(request) - return [r for r in resp] - - def assert_compare_tree_entry(self, path, several_responses=False, **kw): - hg_entries = self.tree_entry('hg', path, **kw) - git_entries = self.tree_entry('git', path, **kw) +def concat_data_fields(responses): + """Callback for RpcHelper.assert_compare_aggregated. @@ -63,31 +40,5 @@ - for entries in (hg_entries, git_entries): - for r in entries: - # oid should be the only difference in comparison - r.oid = '' - - if several_responses: - assert len(hg_entries) > 1 - assert (b''.join(e.data for e in hg_entries) - == - b''.join(e.data for e in git_entries)) - for he, ge in zip(hg_entries, git_entries): - he.data = ge.data = b'' - assert he == ge - else: - assert hg_entries == git_entries - - def assert_error_compare_tree_entry(self, path, **kw): - with pytest.raises(grpc.RpcError) as hg_err_info: - self.tree_entry('hg', path, **kw) - with pytest.raises(grpc.RpcError) as git_err_info: - self.tree_entry('git', path, **kw) - - assert hg_err_info.value.code() == git_err_info.value.code() - assert hg_err_info.value.details() == git_err_info.value.details() - - def get_blob(self, vcs, oid, limit=-1): - request = GetBlobRequest(repository=self.gitaly_repo, - limit=limit, - oid=oid) + Works for all response gRPC messages with a `data` field. + """ + return b''.join(r.data for r in responses) @@ -93,3 +44,2 @@ - return [r for r in self.blob_stubs[vcs].GetBlob(request)] @@ -95,13 +45,4 @@ - def get_blobs(self, vcs, rev_paths, limit=-1, **request_kw): - rev_path_msgs = [ - GetBlobsRequest.RevisionPath(revision=rev, path=path) - for rev, path in rev_paths - ] - request = GetBlobsRequest(repository=self.gitaly_repo, - revision_paths=rev_path_msgs, - limit=limit, - **request_kw) - - return [r for r in self.blob_stubs[vcs].GetBlobs(request)] +def remove_data_field(response): + """Callback for RpcHelper.assert_compare_aggregated. @@ -107,50 +48,6 @@ - def get_tree_entries_raw(self, vcs, path, revision=b'branch/default', - pagination=True, - cursor='', - limit=10, - trees_first=False, - skip_flat_paths=False, - recursive=False): - pagination_params = PaginationParameter( - page_token=cursor, limit=limit) if pagination else None - if trees_first: - sort = GetTreeEntriesRequest.SortBy.TREES_FIRST - else: - sort = GetTreeEntriesRequest.SortBy.DEFAULT - request = GetTreeEntriesRequest(repository=self.gitaly_repo, - revision=revision, - pagination_params=pagination_params, - sort=sort, - skip_flat_paths=skip_flat_paths, - recursive=recursive, - path=path) - - return self.commit_stubs[vcs].GetTreeEntries(request) - - def get_tree_entries(self, vcs, path, **kw): - return [entry - for chunk in self.get_tree_entries_raw(vcs, path, **kw) - for entry in chunk.entries] - - def assert_compare_get_tree_entries(self, path, **kw): - hg_tree_entries = self.get_tree_entries('hg', path, **kw) - git_tree_entries = self.get_tree_entries('git', path, **kw) - - # TODO itertools - for entry in (e for elist in (git_tree_entries, hg_tree_entries) - for e in elist): - entry.oid = entry.root_oid = '' - - assert hg_tree_entries == git_tree_entries - - def assert_error_compare_get_tree_entries(self, *a, **kw): - with pytest.raises(grpc.RpcError) as hg_err_info: - self.get_tree_entries('hg', *a, **kw) - with pytest.raises(grpc.RpcError) as git_err_info: - self.get_tree_entries('git', *a, **kw) - git_err, hg_err = git_err_info.value, hg_err_info.value - assert git_err.code() == hg_err.code() - assert git_err.details() == hg_err.details() + Allows to compare all fields but `data`. + """ + response.data = b'' @@ -155,7 +52,29 @@ -@pytest.fixture -def tree_blob_fixture(gitaly_comparison): - yield TreeBlobFixture(gitaly_comparison) +def oid_mapping(gitaly_comp, rev_paths): + """Provide the OID mappings between Mercurial and Git for blobs. + + :param fixture: an instance of `GitalyComparison` + :param rev_paths: an iterable of tuples (revision, path, type), where + type is 'tree' or 'blob' + :return: a :class:`dict` mapping Mercurial oids to Git oids + + Does not need HGitaly blob/tree methods to work, only those of Gitaly. + """ + res = {} + gitaly_meth = CommitServiceStub(gitaly_comp.gitaly_channel).TreeEntry + hg_repo = gitaly_comp.hg_repo_wrapper.repo + hg_oid_meths = dict(tree=tree_oid, blob=blob_oid) + for rev, path, t in rev_paths: + changeset = gitlab_revision_changeset(hg_repo, rev) + hg_oid = hg_oid_meths[t](hg_repo, changeset.hex().decode(), path) + gitaly_resp = gitaly_meth( + TreeEntryRequest(repository=gitaly_comp.gitaly_repo, + revision=rev, + path=path, + limit=1) + ) + res[hg_oid] = next(iter(gitaly_resp)).oid + return res @@ -160,7 +79,24 @@ -def test_compare_tree_entry_request(tree_blob_fixture): - fixture = tree_blob_fixture +def oid_normalizer(oid2git): + """Return a response normalizer for oid fields. + + :param oid_mapping: :class:`dict` instance mapping HGitaly OIDs to + Gitaly OIDs + """ + def normalizer(rpc_helper, responses, vcs='hg'): + if vcs != 'hg': + return + + for entry in responses: + if entry.oid: + entry.oid = oid2git[entry.oid] + + return normalizer + + +def test_compare_tree_entry_request(gitaly_comparison): + fixture = gitaly_comparison wrapper = fixture.hg_repo_wrapper wrapper.write_commit('foo', message="Some foo") @@ -173,4 +109,28 @@ wrapper.commit(rel_paths=['sub/bar', 'sub/ba2'], message="zebar", add_remove=True) + default_rev = b'branch/default' + oid2git = oid_mapping(fixture, + ((default_rev, b'foo', 'blob'), + (default_rev, b'sub', 'tree'), + (default_rev, b'sub/bar', 'blob'), + (default_rev, b'sub/ba2', 'blob'), + )) + normalizer = oid_normalizer(oid2git) + + rpc_helper = fixture.rpc_helper(stub_cls=CommitServiceStub, + method_name='TreeEntry', + request_cls=TreeEntryRequest, + request_defaults=dict( + revision=default_rev, + limit=0, + max_size=0), + streaming=True, + normalizer=normalizer, + chunked_fields_remover=remove_data_field, + chunks_concatenator=concat_data_fields, + ) + assert_compare = rpc_helper.assert_compare + assert_compare_errors = rpc_helper.assert_compare_errors + # precondition for the test: mirror worked @@ -176,5 +136,5 @@ # precondition for the test: mirror worked - assert fixture.git_repo.branch_titles() == {b'branch/default': b"zebar"} + assert fixture.git_repo.branch_titles() == {default_rev: b"zebar"} # basic case for path in (b'sub', b'sub/bar', b'sub/'): @@ -178,6 +138,6 @@ # basic case for path in (b'sub', b'sub/bar', b'sub/'): - fixture.assert_compare_tree_entry(path) + assert_compare(path=path) for path in (b'.', b'do-not-exist'): @@ -182,5 +142,5 @@ for path in (b'.', b'do-not-exist'): - fixture.assert_error_compare_tree_entry(path) + assert_compare_errors(path=path) # limit and max_size (does not apply to Trees) @@ -185,7 +145,7 @@ # limit and max_size (does not apply to Trees) - fixture.assert_compare_tree_entry(b'foo', limit=4) - fixture.assert_error_compare_tree_entry(b'foo', max_size=4) - fixture.assert_compare_tree_entry(b'sub', max_size=1) + assert_compare(path=b'foo', limit=4) + assert_compare_errors(path=b'foo', max_size=4) + assert_compare(path=b'sub', max_size=1) # unknown revision @@ -190,7 +150,7 @@ # unknown revision - fixture.assert_error_compare_tree_entry(b'sub', revision=b'unknown') + assert_compare_errors(path=b'sub', revision=b'unknown') # chunking for big Blob entry wrapper.write_commit('bigfile', message="A big file with 2 or 3 chunks", content=b'a' * 1023 + b'ff' * 16384) @@ -193,6 +153,18 @@ # chunking for big Blob entry wrapper.write_commit('bigfile', message="A big file with 2 or 3 chunks", content=b'a' * 1023 + b'ff' * 16384) - fixture.assert_compare_tree_entry(b'bigfile', several_responses=True) + # default_rev now resolves to another changeset, hence a different + # HGitaly OID given the current implementation + oid2git.update(oid_mapping(fixture, ((default_rev, b'bigfile', 'blob'), + (default_rev, b'sub', 'tree'), + (default_rev, b'sub/bar', 'blob'), + (default_rev, b'sub/ba2', 'blob'), + (default_rev, b'foo', 'blob'), + ))) + rpc_helper.assert_compare_aggregated(path=b'bigfile') + + # + # reusing content to test GetTreeEntries, hence with a new rpc_helper + # @@ -198,5 +170,22 @@ - # reusing content to test GetTreeEntries + def gte_normalizer(rpc_helper, responses, vcs='hg'): + for resp in responses: + for entry in resp.entries: + if entry.oid and vcs == 'hg': + entry.oid = oid2git[entry.oid] + entry.root_oid = b'' + + rpc_helper = fixture.rpc_helper(stub_cls=CommitServiceStub, + method_name='GetTreeEntries', + request_cls=GetTreeEntriesRequest, + request_defaults=dict( + revision=default_rev, + ), + streaming=True, + normalizer=gte_normalizer, + ) + assert_compare = rpc_helper.assert_compare + for path in (b'.', b'sub'): for recursive in (False, True): for skip_flat in (False, True): @@ -200,9 +189,7 @@ for path in (b'.', b'sub'): for recursive in (False, True): for skip_flat in (False, True): - fixture.assert_compare_get_tree_entries( - path, - skip_flat_paths=skip_flat, - recursive=recursive - ) + assert_compare(path=path, + skip_flat_paths=skip_flat, + recursive=recursive) @@ -208,4 +195,4 @@ - fixture.assert_compare_get_tree_entries(b'.', revision=b'unknown') + assert_compare(path=b'.', revision=b'unknown') # sort parameter @@ -210,3 +197,4 @@ # sort parameter + SortBy = GetTreeEntriesRequest.SortBy for recursive in (False, True): @@ -212,6 +200,6 @@ for recursive in (False, True): - fixture.assert_compare_get_tree_entries(b'.', recursive=recursive, - trees_first=True) + assert_compare(path=b'.', recursive=recursive, + sort=SortBy.TREES_FIRST) # tree first and nested trees nested = sub / 'nested' @@ -219,4 +207,15 @@ (nested / 'deeper').write_text('deep thoughts') wrapper.commit_file('sub/nested/deeper', message='deeper') assert fixture.git_repo.branch_titles() == {b'branch/default': b"deeper"} + + # see comment above about update of OID mapping + oid2git.update(oid_mapping(fixture, ( + (default_rev, b'bigfile', 'blob'), + (default_rev, b'sub', 'tree'), + (default_rev, b'sub/bar', 'blob'), + (default_rev, b'sub/ba2', 'blob'), + (default_rev, b'sub/nested', 'tree'), + (default_rev, b'sub/nested/deeper', 'blob'), + (default_rev, b'foo', 'blob'), + ))) for skip_flat in (False, True): @@ -222,6 +221,7 @@ for skip_flat in (False, True): - fixture.assert_compare_get_tree_entries(b'.', recursive=True, - skip_flat_paths=skip_flat, - trees_first=True) + assert_compare(path=b'.', + recursive=True, + skip_flat_paths=skip_flat, + sort=SortBy.TREES_FIRST) @@ -226,7 +226,7 @@ -def test_compare_get_tree_entries_pagination(tree_blob_fixture): - fixture = tree_blob_fixture +def test_compare_get_tree_entries_pagination(gitaly_comparison): + fixture = gitaly_comparison wrapper = fixture.hg_repo_wrapper wrapper.write_commit('foo', message="Some foo") @@ -244,11 +244,53 @@ wrapper.commit(rel_paths=rel_paths, message="zebar", add_remove=True) - def assert_compare_entries_amount(*resp_collections): - distinct_amounts = set(sum(len(resp.entries) for resp in resps) - for resps in resp_collections) - assert len(distinct_amounts) == 1 - return next(iter(distinct_amounts)) + def normalizer(rpc_helper, responses, **kw): + for resp in responses: + for entry in resp.entries: + entry.oid = entry.root_oid = b'' + resp.pagination_cursor.next_cursor = '' + + def concat_entries(responses): + return list(e for r in responses for e in r.entries) + + def remove_entries(response): + del response.entries[:] + + cursor2git = {} # hg cursor -> git cursor + + rpc_helper = fixture.rpc_helper(stub_cls=CommitServiceStub, + method_name='GetTreeEntries', + request_cls=GetTreeEntriesRequest, + request_defaults=dict( + revision=b'branch/default', + ), + streaming=True, + normalizer=normalizer, + chunked_fields_remover=remove_entries, + chunks_concatenator=concat_entries, + ) + + def request_kwargs_to_git(hg_kwargs): + """Swapping the cursor is too specific for the current RpcHelper. + + We might grow something for all paginated methods, though. + """ + pagination = hg_kwargs.get('pagination_params') + if pagination is None: + return hg_kwargs + + hg_cursor = pagination.page_token + if not hg_cursor: + return hg_kwargs + + git_kwargs = deepcopy(hg_kwargs) + git_kwargs['pagination_params'].page_token = cursor2git[hg_cursor] + return git_kwargs + + rpc_helper.request_kwargs_to_git = request_kwargs_to_git + + assert_compare = rpc_helper.assert_compare + assert_compare_aggregated = rpc_helper.assert_compare_aggregated # asking more than the expected Gitaly first chunk size (2888 entries) # but still less than the total @@ -252,10 +294,10 @@ # asking more than the expected Gitaly first chunk size (2888 entries) # but still less than the total - git_resps, hg_resps = [ - list(fixture.get_tree_entries_raw(vcs, b'sub', - recursive=True, - limit=2890)) - for vcs in ('git', 'hg') - ] + first_resps = assert_compare_aggregated( + path=b'sub', + recursive=True, + pagination_params=PaginationParameter(limit=2890), + compare_first_chunks=False, + ) @@ -261,12 +303,5 @@ - # the page token (aka cursor) being an oid, comparison can only be - # indirect. Chunk sizes are different between Gitaly and HGitaly - assert len(git_resps) > 1 - assert len(hg_resps) > 1 - - assert_compare_entries_amount(git_resps, hg_resps) - - git_cursor, hg_cursor = [resps[0].pagination_cursor.next_cursor - for resps in (git_resps, hg_resps)] + git_cursor, hg_cursor = [resp.pagination_cursor.next_cursor + for resp in first_resps] assert git_cursor assert hg_cursor @@ -271,10 +306,5 @@ assert git_cursor assert hg_cursor - - # cursor is only on first responses (that's probably suboptimal, hence - # prone to change) - assert not any(resp.HasField('pagination_cursor') - for resps in (git_resps, hg_resps) - for resp in resps[1:]) + cursor2git[hg_cursor] = git_cursor # using the cursor @@ -279,13 +309,8 @@ # using the cursor - git_resps, hg_resps = [ - list(fixture.get_tree_entries_raw(vcs, b'sub', - recursive=True, - cursor=cursor, - limit=9000)) - for vcs, cursor in (('git', git_cursor), - ('hg', hg_cursor)) - ] - assert_compare_entries_amount(git_resps, hg_resps) + assert_compare(path=b'sub', + recursive=True, + pagination_params=PaginationParameter(page_token=hg_cursor, + limit=9000)) # negative limit means all results, and there's no cursor if no next page @@ -290,11 +315,10 @@ # negative limit means all results, and there's no cursor if no next page - git_resps, hg_resps = [ - list(fixture.get_tree_entries_raw(vcs, b'sub', - recursive=True, - limit=-1)) - for vcs in ('git', 'hg') - ] - assert_compare_entries_amount(git_resps, hg_resps) - assert git_resps[0].pagination_cursor == hg_resps[0].pagination_cursor + git_resp0, hg_resp0 = assert_compare_aggregated( + path=b'sub', + recursive=True, + pagination_params=PaginationParameter(limit=-1), + compare_first_chunks=False, + ) + assert git_resp0.pagination_cursor == hg_resp0.pagination_cursor @@ -300,10 +324,7 @@ - # case of limit=0 - git_resps, hg_resps = [ - list(fixture.get_tree_entries_raw(vcs, b'sub', - recursive=True, - limit=0)) - for vcs in ('git', 'hg') - ] - assert git_resps == hg_resps # both are empty + # case of limit=0 (empty results) + assert_compare(path=b'sub', + recursive=True, + pagination_params=PaginationParameter(limit=0), + ) @@ -309,13 +330,8 @@ - # case of no params - git_resps, hg_resps = [ - list(fixture.get_tree_entries_raw(vcs, b'sub', - recursive=True, - pagination=False, - limit=0)) - for vcs in ('git', 'hg') - ] - assert_compare_entries_amount(git_resps, hg_resps) + # case of no pagination params + assert_compare_aggregated(path=b'sub', + recursive=True, + compare_first_chunks=False) # case of a cursor that doesn't match any entry (can happen if content # changes between requests) @@ -319,10 +335,13 @@ # case of a cursor that doesn't match any entry (can happen if content # changes between requests) - - fixture.assert_error_compare_get_tree_entries(b'sub', - recursive=True, - cursor="surely not an OID", - limit=10) + cursor = "surely not an OID" + cursor2git[cursor] = cursor + rpc_helper.assert_compare_errors(path=b'sub', + recursive=True, + pagination_params=PaginationParameter( + page_token=cursor, + limit=10) + ) @@ -327,7 +346,14 @@ -def test_compare_get_blob_request(tree_blob_fixture): - fixture = tree_blob_fixture +def rev_path_messages(rev_paths): + """Convert (revision, path) pairs into a list of `RevisionPath` messages. + """ + return [GetBlobsRequest.RevisionPath(revision=rev, path=path) + for rev, path in rev_paths] + + +def test_compare_get_blob_request(gitaly_comparison): + fixture = gitaly_comparison git_repo = fixture.git_repo wrapper = fixture.hg_repo_wrapper @@ -336,5 +362,8 @@ wrapper.commit_file('small', message="Small file") changeset = wrapper.commit_file('foo', message="Large foo", content=large_data) + hg_oid = blob_oid(wrapper.repo, changeset.hex().decode(), b'foo') + + default_rev = b'branch/default' # mirror worked @@ -339,4 +368,4 @@ # mirror worked - assert git_repo.branch_titles() == {b'branch/default': b"Large foo"} + assert git_repo.branch_titles() == {default_rev: b"Large foo"} @@ -342,10 +371,6 @@ - oids = dict( - git=fixture.tree_entry('git', b'foo', limit=1)[0].oid, - hg=blob_oid(wrapper.repo, changeset.hex().decode(), b'foo') - ) - - git_resps = fixture.get_blob('git', oids['git'], limit=12) - # important assumption for hg implementation: - assert git_resps[0].oid == oids['git'] + oid2git = oid_mapping(fixture, + ((default_rev, b'foo', 'blob'), + (default_rev, b'small', 'blob'))) + normalizer = oid_normalizer(oid2git) @@ -351,10 +376,13 @@ - hg_resps = fixture.get_blob('hg', oids['hg'], limit=12) - assert len(hg_resps) == 1 # double-check: already done in direct hg test - assert len(git_resps) == 1 - git_resp, hg_resp = git_resps[0], hg_resps[0] - assert hg_resp.size == git_resp.size - assert hg_resp.data == git_resp.data - - git_resps = fixture.get_blob('git', oids['git']) + rpc_helper = fixture.rpc_helper(stub_cls=BlobServiceStub, + method_name='GetBlob', + request_cls=GetBlobRequest, + request_defaults=dict( + limit=-1, + ), + streaming=True, + normalizer=normalizer, + chunked_fields_remover=remove_data_field, + chunks_concatenator=concat_data_fields, + ) @@ -360,14 +388,4 @@ - hg_resps = fixture.get_blob('hg', oids['hg']) - # Gitaly chunking is not fully deterministic, so the most - # we can check is that chunking occurs for both servers - # and that the first and second responses have the same metadata - assert len(hg_resps) > 1 - assert len(git_resps) > 1 - - assert hg_resps[0].oid == oids['hg'] - assert git_resps[0].oid == oids['git'] - assert hg_resps[1].oid == git_resps[1].oid - for hgr, gitr in zip(hg_resps[:2], git_resps[:2]): - assert hgr.size == gitr.size + def request_kwargs_to_git(hg_kwargs): + """Swapping oid is too specific for the current RpcHelper @@ -373,7 +391,8 @@ - assert ( - b''.join(r.data for r in hg_resps) - == - b''.join(r.data for r in git_resps) - ) + We might provide it with a direct `git` subprocess or with dulwich, + though. + """ + git_kwargs = deepcopy(hg_kwargs) + git_kwargs['oid'] = oid2git[hg_kwargs['oid']] + return git_kwargs @@ -379,7 +398,3 @@ - # now with get_blobs - rev_paths = ((b'branch/default', b'small'), - (b'branch/default', b'does-not-exist'), - (b'no-such-revision', b'small'), - ) + rpc_helper.request_kwargs_to_git = request_kwargs_to_git @@ -385,4 +400,6 @@ - hg_resps = fixture.get_blobs('hg', rev_paths) - git_resps = fixture.get_blobs('git', rev_paths) + rpc_helper.assert_compare(oid=hg_oid, limit=12) + + rpc_helper.assert_compare_aggregated(oid=hg_oid, + compare_first_chunks=False) @@ -388,6 +405,14 @@ - for resp in hg_resps: - resp.oid = '' - for resp in git_resps: - resp.oid = '' + # now with GetBlobs + rpc_helper = fixture.rpc_helper(stub_cls=BlobServiceStub, + method_name='GetBlobs', + request_cls=GetBlobsRequest, + request_defaults=dict( + limit=-1, + ), + streaming=True, + normalizer=normalizer, + chunked_fields_remover=remove_data_field, + chunks_concatenator=concat_data_fields, + ) @@ -393,4 +418,7 @@ - assert hg_resps == git_resps - + rev_paths = rev_path_messages(((b'branch/default', b'small'), + (b'branch/default', b'does-not-exist'), + (b'no-such-revision', b'small'), + )) + rpc_helper.assert_compare(revision_paths=rev_paths) # with limits (the limit is per file) @@ -396,12 +424,4 @@ # with limits (the limit is per file) - hg_resps = fixture.get_blobs('hg', rev_paths, limit=3) - git_resps = fixture.get_blobs('git', rev_paths, limit=3) - - for resp in hg_resps: - resp.oid = '' - for resp in git_resps: - resp.oid = '' - - assert hg_resps == git_resps + rpc_helper.assert_compare(revision_paths=rev_paths, limit=3) # chunking in get_blobs, again non-deterministic for Gitaly @@ -406,23 +426,6 @@ # chunking in get_blobs, again non-deterministic for Gitaly - rev_paths = ((b'branch/default', b'small'), - (b'branch/default', b'foo'), - ) - hg_resps = fixture.get_blobs('hg', rev_paths) - git_resps = fixture.get_blobs('git', rev_paths) - assert len(hg_resps) > 2 - assert len(git_resps) > 2 - assert hg_resps[0].oid != "" - assert git_resps[0].oid != "" - assert hg_resps[1].oid != "" - assert git_resps[1].oid != "" - assert hg_resps[2].oid == "" - assert git_resps[2].oid == "" - for hgr, gitr in zip(hg_resps[:3], git_resps[:3]): - assert hgr.size == gitr.size - - assert ( # content of the big file at 'foo' - b''.join(r.data for r in hg_resps[1:]) - == - b''.join(r.data for r in git_resps[1:]) - ) + rev_paths = rev_path_messages(((b'branch/default', b'small'), + (b'branch/default', b'foo'), + )) + rpc_helper.assert_compare_aggregated(revision_paths=rev_paths)