Skip to content
Snippets Groups Projects
Commit f37a44b7 authored by Sushil Khanchi's avatar Sushil Khanchi :koala:
Browse files

RepositoryService: implement CreateBundle and CreateRepositoryFromBundle

Rebased and amended by gracinet from the original version by Sushil.

Things have changed upstream, with better uniformity in error treatment.
The test case where the repo creation fails after the initial path checks
is now a bit more convoluted, as explained in comments.

Also some simplifications in tests.
parent b1ef4099
No related branches found
No related tags found
1 merge request!71repository-service: implement CreateBundle and CreateRepositoryFromBundle
......@@ -11,6 +11,7 @@
from mercurial import (
archival,
pycompat,
scmutil,
hg,
)
......@@ -14,6 +15,10 @@
scmutil,
hg,
)
from mercurial.commands import (
bundle,
unbundle,
)
from heptapod.gitlab.branch import gitlab_branch_from_ref
from hgext3rd.heptapod.branch import set_default_gitlab_branch
......@@ -46,5 +51,7 @@
gitlab_revision_changeset,
)
from ..stub.repository_service_pb2 import (
CreateBundleRequest,
CreateBundleResponse,
CreateRepositoryRequest,
CreateRepositoryResponse,
......@@ -49,5 +56,7 @@
CreateRepositoryRequest,
CreateRepositoryResponse,
CreateRepositoryFromBundleRequest,
CreateRepositoryFromBundleResponse,
FindMergeBaseRequest,
FindMergeBaseResponse,
GetRawChangesRequest,
......@@ -277,6 +286,7 @@
if err is not None:
return err
# TODO refactor with hg_init_repository
try:
hg.peer(self.ui, opts={}, path=repo_path, create=True)
except Exception as exc:
......@@ -329,6 +339,81 @@
set_gitlab_project_full_path(repo, request.path.encode('utf-8'))
return SetFullPathResponse()
def CreateBundle(self, request: CreateBundleRequest,
context) -> CreateBundleResponse:
repo = self.load_repo(request.repository, context).unfiltered()
# overrides makes sure that 1) phases info 2) obsmarkers are bundled
overrides = {
(b'experimental', b'bundle-phases'): True,
(b'experimental', b'evolution.bundle-obsmarker'): True,
}
with tempfile.NamedTemporaryFile(
mode='wb+',
buffering=WRITE_BUFFER_SIZE) as tmpf:
with repo.ui.configoverride(overrides, b'CreateBundle'):
# also bundle the hidden changesets
opts = dict(all=True, hidden=True)
bundle(repo.ui, repo, pycompat.sysbytes(tmpf.name), **opts)
tmpf.seek(0)
while True:
data = tmpf.read(WRITE_BUFFER_SIZE)
if not data:
break
yield CreateBundleResponse(data=data)
def CreateRepositoryFromBundle(
self, request: CreateRepositoryFromBundleRequest,
context) -> CreateRepositoryFromBundleResponse:
"""Create repository from bundle.
param `request`: is an iterator streaming sub-requests where first
sub-request contains repository+data and subsequent sub-requests
contains only data (i.e. the actual bundle sent in parts).
"""
def init_repo(req):
"""Resolve the path and create the repository for good.
provided as a nested function because error treatment is
specific to this gRPC method
"""
repo_path, err = self.repo_path_for_creation(
req.repository,
CreateRepositoryFromBundleResponse,
context)
if err is not None:
return err
err = self.hg_init_repository(
repo_path, CreateRepositoryFromBundleResponse,
context)
if err is not None:
return err
with tempfile.NamedTemporaryFile(
mode='wb+',
buffering=WRITE_BUFFER_SIZE) as tmpf:
first_request = True
for req in request:
if first_request:
err = init_repo(req)
if err is not None:
return err
repo = self.load_repo(req.repository, context)
first_request = False
tmpf.write(req.data)
tmpf.seek(0)
# make sure obsmarker creation is allowed while unbundle
overrides = {(b'experimental', b'evolution'): b'all'}
# TODO it would be nice to have UPSTREAM a method
# to unbundle from an arbitrary file-like object rather than
# paths forcing us to dump to disk
with repo.ui.configoverride(
overrides, b'CreateRepositoryFromBundle'):
unbundle(repo.ui, repo, pycompat.sysbytes(tmpf.name))
# TODO in case of error, we must remove the created repo,
# so that a later attempt has a chance to succeed.
return CreateRepositoryFromBundleResponse()
def repo_path_for_creation(self, repository: Repository,
response_class,
context):
......@@ -332,6 +417,7 @@
def repo_path_for_creation(self, repository: Repository,
response_class,
context):
try:
repo_path = self.repo_disk_path(repository, context)
except KeyError:
......@@ -344,3 +430,16 @@
"repository exists already")
return repo_path, None
def hg_init_repository(self, repo_path, response_cls, context):
"""Initialize a mercuiral repository from a request object.
:return: `None` if no error encountered, otherwise set the error
in `context` and return the empty response.
"""
try:
hg.peer(self.ui, opts={}, path=repo_path, create=True)
except OSError as exc:
return internal_error(context, response_cls,
message="hg_init_repository(%r): %r" % (
repo_path, exc))
......@@ -7,6 +7,7 @@
from contextlib import contextmanager
from io import BytesIO
import grpc
from pathlib import Path
import shutil
import tarfile
from mercurial_testhelpers.util import as_bytes
......@@ -26,6 +27,8 @@
from hgitaly.stub.shared_pb2 import Repository
from hgitaly.stub.repository_service_pb2 import (
CreateBundleRequest,
CreateRepositoryFromBundleRequest,
CreateRepositoryRequest,
FindMergeBaseRequest,
GetArchiveRequest,
......@@ -408,3 +411,116 @@
grpc_repo=Repository(
relative_path=grpc_repo.relative_path,
storage_name='unknown'))
def test_create_bundle_and_create_repository_from_bundle(
grpc_channel, server_repos_root, tmpdir):
repo_stub = RepositoryServiceStub(grpc_channel)
wrapper, grpc_repo = make_empty_repo(server_repos_root)
# repo structure:
#
# @ 3 zoo (phase: draft) (amended)
# |
# | o 1 bar (phase: public) (tag: v1.2.3)
# |/
# o 0 foo (phase: public)
#
ctx0 = wrapper.commit_file('foo')
sha0 = ctx0.hex()
sha1 = wrapper.commit_file('bar').hex()
wrapper.commit_file('zoo', parent=ctx0)
wrapper.amend_file('zoo')
wrapper.set_phase('public', [sha0, sha1])
wrapper.update(sha1)
wrapper.command('tag', b'v1.2.3', rev=sha1)
bundle_path = tmpdir / 'my.bundle'
default_storage = 'default'
target_rel_path = 'target_repo'
target_repo_path = server_repos_root / default_storage / target_rel_path
target_repo_msg = Repository(relative_path=target_rel_path,
storage_name=default_storage)
def rpc_create_bundle(repo):
request = CreateBundleRequest(repository=repo)
response = repo_stub.CreateBundle(request)
with open(bundle_path, 'wb') as fobj:
for chunk in response:
fobj.write(chunk.data)
def rpc_create_repository_from_bundle(repo):
with open(bundle_path, 'rb') as fobj:
data = fobj.read()
first_req_data_size = len(data) // 2
request1 = CreateRepositoryFromBundleRequest(
repository=repo, data=data[:first_req_data_size])
request2 = CreateRepositoryFromBundleRequest(
data=data[first_req_data_size:])
# create an iterator of requests
request = (req for req in [request1, request2])
return repo_stub.CreateRepositoryFromBundle(request)
def assert_orig_and_target_repo_content(orig, target):
for rev_str in ('draft()', 'public()', 'obsolete()'):
assert (
[r for r in orig.revs(rev_str)]
==
[r for r in target.revs(rev_str)])
# assert empty incoming/outgoing changes between orig and target repo
no_changes = 1 # code returned when no changes to transmit
# XXX: extra args bundle, force should not be required once
# upstream Mercurial use opts.get() instead of [] access
incoming = wrapper.command('incoming', target.root,
bundle=None, force=None)
outgoing = wrapper.command('outgoing', target.root,
bundle=None, force=None)
assert incoming == no_changes
assert outgoing == no_changes
# assert branches
orig_br = [br[0] for br in orig.branchmap().iterbranches()]
target_br = [br[0] for br in target.branchmap().iterbranches()]
assert orig_br == target_br
# assert tags
assert orig.tags() == target.tags()
# Bundling and creating new repo from bundle
rpc_create_bundle(grpc_repo)
rpc_create_repository_from_bundle(target_repo_msg)
# compare the created repository content with original one
orig_repo = wrapper.repo.unfiltered()
target_repo = LocalRepoWrapper.load(target_repo_path).repo.unfiltered()
assert target_repo.path.endswith(b'default/target_repo/.hg')
assert_orig_and_target_repo_content(orig_repo, target_repo)
# error case of repository creation when target repo already exists
with pytest.raises(grpc.RpcError) as exc_info:
rpc_create_repository_from_bundle(target_repo_msg)
assert exc_info.value.code() == grpc.StatusCode.ALREADY_EXISTS
details = exc_info.value.details()
assert 'exists already' in details
# when storage name doesn't exists
new_repo_msg = Repository(relative_path='new_target_repo',
storage_name='hippoship')
with pytest.raises(grpc.RpcError) as exc_info:
rpc_create_repository_from_bundle(new_repo_msg)
assert exc_info.value.code() == grpc.StatusCode.INVALID_ARGUMENT
assert 'no such storage' in exc_info.value.details()
# test errors while creating repo itself
# Because the main code refuses existing paths, we make a parent dir that
# is a broken symlink (which points to itself). This way the repo path
# does not exist *and* its creation fails.
br_dir = Path("my_broken_symlink")
br_path = (server_repos_root / default_storage / br_dir)
br_path.symlink_to(br_path)
trepo_msg = Repository(relative_path=str(br_dir / 'repo'),
storage_name=default_storage)
with pytest.raises(grpc.RpcError) as exc_info:
rpc_create_repository_from_bundle(trepo_msg)
assert exc_info.value.code() == grpc.StatusCode.INTERNAL
assert 'Too many levels of symbolic links' in exc_info.value.details()
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment