Skip to content
Snippets Groups Projects
Commit f52d6361 authored by Georges Racinet's avatar Georges Racinet
Browse files

BlobService: implementation of GetBlobs (plural)

Closes #43
parent dedad609
No related branches found
No related tags found
1 merge request!57Blob and Tree related service methods
......@@ -7,8 +7,13 @@
import io
import logging
from hgitaly.errors import (
not_implemented,
from mercurial import (
error,
)
from ..stub.shared_pb2 import (
ObjectType,
)
from ..stub.blob_pb2 import (
GetBlobRequest,
......@@ -19,5 +24,6 @@
from ..stub.blob_pb2_grpc import BlobServiceServicer
from ..oid import (
blob_oid,
extract_blob_oid,
)
......@@ -22,5 +28,7 @@
extract_blob_oid,
)
from ..file_context import git_perms
from ..revision import gitlab_revision_changeset
from ..servicer import HGitalyServicer
logger = logging.getLogger(__name__)
......@@ -60,8 +68,59 @@
yield GetBlobResponse(**resp_dict)
def GetBlobs(self, request: GetBlobsRequest, context) -> GetBlobsResponse:
raise not_implemented(context,
GetBlobsResponse, issue=43) # pragma no cover
revision_paths = [(rev_path.revision, rev_path.path)
for rev_path in request.revision_paths]
logger.debug("GetBlobs repo relative_path=%r, revision_paths=%r",
request.repository.relative_path,
revision_paths,
)
repo = self.load_repo(request.repository, context).unfiltered()
limit = request.limit
for rev, path in revision_paths:
# TODO it's probably an oversight on upstream side that revision
# is `str` and not `bytes`, but then, what would be the encoding?
changeset = gitlab_revision_changeset(repo,
rev.encode('utf-8', 'ignore')
)
if changeset is None:
filectx = None
else:
try:
filectx = changeset.filectx(path)
except error.ManifestLookupError:
filectx = None
if changeset is None or filectx is None:
# missing file is represented by oid=''
# (see comment in blob.proto)
yield GetBlobsResponse(path=path, revision=rev)
continue
filectx = changeset.filectx(path)
oid = blob_oid(repo, changeset.hex().decode(), path)
size = filectx.size()
data = filectx.data()
if limit != -1:
data = data[:limit]
for chunk, first in iter_blob_chunks(data):
# is_submodule will be False, because that's the default
# should at least be documented somewhere though
dict_resp = dict(data=chunk)
if first:
dict_resp.update(size=size,
oid=oid,
path=path,
revision=rev,
type=ObjectType.BLOB,
mode=git_perms(filectx),
)
yield GetBlobsResponse(**dict_resp)
def iter_blob_chunks(data, chunk_size=CHUNK_SIZE):
......
......@@ -18,6 +18,7 @@
from hgitaly.stub.blob_pb2 import (
GetBlobRequest,
GetBlobsRequest,
)
from hgitaly.stub.blob_pb2_grpc import BlobServiceStub
......@@ -62,6 +63,52 @@
assert resps[1].oid == ''
def test_get_blobs(grpc_channel, server_repos_root):
grpc_stub = BlobServiceStub(grpc_channel)
wrapper, grpc_repo = make_empty_repo(server_repos_root)
repo = wrapper.repo
RevisionPath = GetBlobsRequest.RevisionPath
def do_rpc(rev_paths, limit=-1):
request = GetBlobsRequest(
repository=grpc_repo,
revision_paths=[RevisionPath(revision=rev, path=path)
for rev, path in rev_paths],
limit=limit)
return [resp for resp in grpc_stub.GetBlobs(request)]
wrapper.write_commit('foo', content='some content')
two_files = wrapper.write_commit('bar', content='bar content')
tip_hex = two_files.hex().decode()
resps = do_rpc([(tip_hex, b'foo'),
(tip_hex, b'bar'),
(b'unknown-rev', b'foo'),
(tip_hex, b'unknown-file')
])
assert len(resps) == 4
assert resps[0].data == b'some content'
assert resps[0].size == 12
assert resps[0].oid == blob_oid(repo, tip_hex, b'foo')
assert resps[1].data == b'bar content'
assert resps[1].size == 11
assert resps[1].oid == blob_oid(repo, tip_hex, b'bar')
assert not resps[2].oid
assert not resps[3].oid
resps = do_rpc([(tip_hex, b'foo'), (tip_hex, b'bar')], limit=4)
assert len(resps) == 2
assert resps[0].data == b'some'
assert resps[0].size == 12
assert resps[0].oid == blob_oid(repo, tip_hex, b'foo')
assert resps[1].data == b'bar '
assert resps[1].size == 11
assert resps[1].oid == blob_oid(repo, tip_hex, b'bar')
def test_iter_blob_chunks():
"""Complements: edge cases etc."""
......
......@@ -15,6 +15,7 @@
from hgitaly.stub.blob_pb2 import (
GetBlobRequest,
GetBlobsRequest,
)
from hgitaly.stub.commit_pb2 import (
TreeEntryRequest,
......@@ -83,6 +84,18 @@
return [r for r in self.blob_stubs[vcs].GetBlob(request)]
def get_blobs(self, vcs, rev_paths, limit=-1, **request_kw):
rev_path_msgs = [
GetBlobsRequest.RevisionPath(revision=rev, path=path)
for rev, path in rev_paths
]
request = GetBlobsRequest(repository=self.gitaly_repo,
revision_paths=rev_path_msgs,
limit=limit,
**request_kw)
return [r for r in self.blob_stubs[vcs].GetBlobs(request)]
@pytest.fixture
def tree_blob_fixture(gitaly_comparison):
......@@ -130,8 +143,9 @@
wrapper = fixture.hg_repo_wrapper
large_data = b'\xbe\xef' * CHUNK_SIZE # twice as CHUNK_SIZE
changeset = wrapper.write_commit('foo', message="Large foo",
content=large_data)
wrapper.commit_file('small', message="Small file")
changeset = wrapper.commit_file('foo', message="Large foo",
content=large_data)
# mirror worked
assert git_repo.branch_titles() == {b'branch/default': b"Large foo"}
......@@ -167,3 +181,30 @@
assert hg_resp.data == git_resp.data
assert all(git_resps[i].size == hg_resps[i].size for i in (0, 1))
# now with get_blobs
rev_paths = ((b'branch/default', b'small'),
(b'branch/default', b'does-not-exist'),
(b'no-such-revision', b'small'),
(b'branch/default', b'foo'))
hg_resps = fixture.get_blobs('hg', rev_paths)
git_resps = fixture.get_blobs('git', rev_paths)
for resp in hg_resps:
resp.oid = ''
for resp in git_resps:
resp.oid = ''
assert hg_resps == git_resps
# with limits (the limit is per file)
hg_resps = fixture.get_blobs('hg', rev_paths, limit=3)
git_resps = fixture.get_blobs('git', rev_paths, limit=3)
for resp in hg_resps:
resp.oid = ''
for resp in git_resps:
resp.oid = ''
assert hg_resps == git_resps
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment