Skip to content
Snippets Groups Projects
Commit 00744973 authored by Georges Racinet's avatar Georges Racinet
Browse files

WIP RepositoryService: GetArchive method initial implementation

Closes #6

There are still a few TODOs.

Works in end-to-end development context. We have a long way to
go for proper testing.
parent d79a7c23
No related branches found
No related tags found
No related merge requests found
Pipeline #11470 passed
......@@ -6,4 +6,7 @@
# SPDX-License-Identifier: GPL-2.0-or-later
from grpc import StatusCode
import logging
import os
import tempfile
from mercurial import (
......@@ -9,4 +12,6 @@
from mercurial import (
archival,
scmutil,
error,
)
......@@ -19,6 +24,8 @@
from ..stub.repository_service_pb2 import (
RepositoryExistsRequest,
RepositoryExistsResponse,
GetArchiveRequest,
GetArchiveResponse,
HasLocalBranchesRequest,
HasLocalBranchesResponse,
WriteRefRequest,
......@@ -30,6 +37,43 @@
logger = logging.getLogger(__name__)
DEFAULT_BRANCH_FILE_NAME = b'default_gitlab_branch'
ARCHIVE_FORMATS = {
GetArchiveRequest.Format.Value('ZIP'): b'zip',
GetArchiveRequest.Format.Value('TAR'): b'tar',
GetArchiveRequest.Format.Value('TAR_GZ'): b'tgz',
GetArchiveRequest.Format.Value('TAR_BZ2'): b'tbz2',
}
# Quoting from Gitaly 13.4:
# // WriteBufferSize is the largest []byte that Write() will pass
# // to its underlying send function. This value can be changed
# // at runtime using the GITALY_STREAMIO_WRITE_BUFFER_SIZE environment
# // variable.
#
# var WriteBufferSize = 128 * 1024
#
# As of GitLab 13.4, the environment variable is parsed with
# `strconv.ParseInt(value, 0, 32)`.
# Quoting https://golang.org/pkg/strconv/#ParseInt:
# If the base argument is 0, the true base is implied by
# the string's prefix: 2 for "0b", 8 for "0" or "0o", 16 for "0x",
# and 10 otherwise. Also, for argument base 0 only,
# underscore characters are permitted as defined by the
# Go syntax for integer literals.
# TODO maybe factorize with other services
def _env_write_buffer_size():
str_val = os.environ.get('GITALY_STREAMIO_WRITE_BUFFER_SIZE')
if not str_val:
return 128 << 10 # 128kB
# TODO check at least octal and hex syntaxes
return int(str_val)
WRITE_BUFFER_SIZE = _env_write_buffer_size()
class RepositoryServiceServicer(RepositoryServiceServicer, HGitalyServicer):
"""RepositoryServiceService implementation.
......@@ -46,6 +90,39 @@
return RepositoryExistsResponse(exists=exists)
def GetArchive(self,
request: GetArchiveRequest,
context) -> GetArchiveResponse:
logger.debug("GetArchive request=%r", request)
repo = self.load_repo(request.repository, context)
ctx = repo[request.commit_id]
# TODO translate 'path' as an include rule
match = scmutil.match(ctx, [], {})
# using an anonymous (not linked) temporary file
# TODO OPTIM check if archive is not by any chance
# using a tempfile already…
with tempfile.TemporaryFile(
mode='wb+', # the default, but let's insist on binary here
buffering=WRITE_BUFFER_SIZE) as tmpf:
archival.archive(
repo,
tmpf,
ctx.node(),
ARCHIVE_FORMATS[request.format],
True, # decode (TODO this is the default but what is this?)
match,
request.prefix.encode(),
subrepos=False # maybe later, check what GitLab's standard is
)
tmpf.seek(0)
while True:
data = tmpf.read(WRITE_BUFFER_SIZE)
if not data:
break
yield GetArchiveResponse(data=data)
def HasLocalBranches(self,
request: HasLocalBranchesRequest,
context) -> HasLocalBranchesResponse:
......
......@@ -4,4 +4,6 @@
# GNU General Public License version 2 or any later version.
#
# SPDX-License-Identifier: GPL-2.0-or-later
from contextlib import contextmanager
from io import BytesIO
import grpc
......@@ -7,2 +9,3 @@
import grpc
import os
import shutil
......@@ -8,6 +11,8 @@
import shutil
import tarfile
import pytest
from hgitaly.tests.common import make_empty_repo
from hgitaly.stub.repository_service_pb2 import (
......@@ -9,10 +14,11 @@
import pytest
from hgitaly.tests.common import make_empty_repo
from hgitaly.stub.repository_service_pb2 import (
GetArchiveRequest,
HasLocalBranchesRequest,
RepositoryExistsRequest,
WriteRefRequest,
)
from hgitaly.stub.repository_service_pb2_grpc import RepositoryServiceStub
......@@ -14,8 +20,9 @@
HasLocalBranchesRequest,
RepositoryExistsRequest,
WriteRefRequest,
)
from hgitaly.stub.repository_service_pb2_grpc import RepositoryServiceStub
from .. import repository_service
def test_repository_exists(grpc_channel, server_repos_root):
......@@ -72,3 +79,90 @@
ref=b'HEAD',
revision=b'topic/default/wont-last'))
assert exc_info.value.code() == grpc.StatusCode.INVALID_ARGUMENT
@contextmanager
def get_archive_tarfile(stub, grpc_repo, commit_id):
with BytesIO() as fobj:
for chunk_index, chunk_response in enumerate(
stub.GetArchive(GetArchiveRequest(
repository=grpc_repo,
format=GetArchiveRequest.Format.Value('TAR'),
commit_id=commit_id,
prefix='archive-dir',
))):
fobj.write(chunk_response.data)
fobj.seek(0)
with tarfile.open(fileobj=fobj) as tarf:
yield tarf, chunk_index + 1
def test_get_archive(grpc_channel, server_repos_root, tmpdir):
repo_stub = RepositoryServiceStub(grpc_channel)
wrapper, grpc_repo = make_empty_repo(server_repos_root)
ctx = wrapper.write_commit('foo', content="Foo")
(wrapper.path / 'sub').mkdir()
ctx2 = wrapper.write_commit('sub/bar', content="Bar")
node_str = ctx.hex().decode()
with get_archive_tarfile(repo_stub, grpc_repo, node_str) as (tarf, _nb):
assert set(tarf.getnames()) == {'archive-dir/.hg_archival.txt',
'archive-dir/foo'}
extract_dir = tmpdir.join('extract')
tarf.extractall(path=extract_dir)
metadata_lines = extract_dir.join('archive-dir',
'.hg_archival.txt').readlines()
assert 'node: %s\n' % node_str in metadata_lines
assert extract_dir.join('archive-dir', 'foo').read() == "Foo"
node2_str = ctx2.hex().decode()
with get_archive_tarfile(repo_stub, grpc_repo, node2_str) as (tarf, _nb):
assert set(tarf.getnames()) == {'archive-dir/.hg_archival.txt',
'archive-dir/foo',
'archive-dir/sub/bar'}
extract_dir = tmpdir.join('extract-2')
tarf.extractall(path=extract_dir)
metadata_lines = extract_dir.join('archive-dir',
'.hg_archival.txt').readlines()
assert 'node: %s\n' % node2_str in metadata_lines
assert extract_dir.join('archive-dir', 'sub', 'bar').read() == "Bar"
def test_get_archive_multiple_chunks(grpc_channel, server_repos_root,
tmpdir, monkeypatch):
repo_stub = RepositoryServiceStub(grpc_channel)
wrapper, grpc_repo = make_empty_repo(server_repos_root)
large_content = "Foo" * 200000 # should be enough
ctx = wrapper.write_commit('foo', content=large_content)
node_str = ctx.hex().decode()
with get_archive_tarfile(repo_stub, grpc_repo, node_str) as (tarf, count):
assert count > 1
assert set(tarf.getnames()) == {'archive-dir/.hg_archival.txt',
'archive-dir/foo'}
extract_dir = tmpdir.join('extract')
tarf.extractall(path=extract_dir)
metadata_lines = extract_dir.join('archive-dir',
'.hg_archival.txt').readlines()
assert 'node: %s\n' % node_str in metadata_lines
assert extract_dir.join('archive-dir', 'foo').read() == large_content
del large_content
def test_env_write_buffer_size(monkeypatch):
monkeypatch.setitem(os.environ, 'GITALY_STREAMIO_WRITE_BUFFER_SIZE', '10')
assert repository_service._env_write_buffer_size() == 10
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment