Commit 7db892e8 authored by Georges Racinet's avatar Georges Racinet 🦑
Browse files

Heptapod 0.21 is the new oldstable

Part of heptapod#496

--HG--
branch : oldstable
Pipeline #28835 passed with stage
in 2 minutes and 55 seconds
......@@ -14,3 +14,6 @@ d82e4c4d3de109fb1d116a58320e3b1e89594220 0.8.0
b22dcf5dcf7a205276e79ff4f4a729a51cd15fa8 0.11.0
39a6480f55014e3c3586d8a601b6258cd3b6c072 0.11.1
dafcae2d9a1626ce8ef01e8338bda53acaa516df 0.11.2
df53b71c17a4bbd1a85ab72e9752fc0b70ea77ab 0.12.0
9058efa8be3929d711cd80806f22785d92ac345a 0.12.1
fd405714088c2746874f119e16e899ee3f1f30e2 0.12.2
import pytest
from hgitaly.service.blob import BlobServicer
from hgitaly.service.commit import CommitServicer
from hgitaly.service.ref import RefServicer
from hgitaly.service.diff import DiffServicer
from hgitaly.service.repository_service import RepositoryServiceServicer
from hgitaly.service.server import ServerServicer
from hgitaly.stub.blob_pb2_grpc import add_BlobServiceServicer_to_server
from hgitaly.stub.commit_pb2_grpc import add_CommitServiceServicer_to_server
from hgitaly.stub.ref_pb2_grpc import add_RefServiceServicer_to_server
from hgitaly.stub.diff_pb2_grpc import add_DiffServiceServicer_to_server
from hgitaly.stub.repository_service_pb2_grpc import (
add_RepositoryServiceServicer_to_server
)
from hgitaly.stub.server_pb2_grpc import add_ServerServiceServicer_to_server
@pytest.fixture(scope='module')
def grpc_server(_grpc_server, grpc_addr, server_repos_root):
storages = dict(default=str(server_repos_root / 'default').encode())
add_BlobServiceServicer_to_server(BlobServicer(storages),
_grpc_server)
add_CommitServiceServicer_to_server(CommitServicer(storages),
_grpc_server)
add_RefServiceServicer_to_server(RefServicer(storages),
......@@ -25,6 +31,8 @@ def grpc_server(_grpc_server, grpc_addr, server_repos_root):
add_RepositoryServiceServicer_to_server(
RepositoryServiceServicer(storages),
_grpc_server)
add_ServerServiceServicer_to_server(ServerServicer(storages),
_grpc_server)
_grpc_server.add_insecure_port(grpc_addr)
_grpc_server.start()
yield _grpc_server
......
......@@ -5,7 +5,7 @@ set -eu
PROTOS_DIR=./protos
STUBS_DIR=hgitaly/stub
for PROTO in lint shared blob commit diff ref repository-service; do
for PROTO in lint shared blob commit diff ref repository-service server; do
python -m grpc_tools.protoc \
-I${PROTOS_DIR} \
--python_out=${STUBS_DIR} \
......
# Copyright 2020 Georges Racinet <georges.racinet@octobus.net>
# Copyright 2020-2021 Georges Racinet <georges.racinet@octobus.net>
#
# This software may be used and distributed according to the terms of the
# GNU General Public License version 2 or any later version.
#
# SPDX-License-Identifier: GPL-2.0-or-later
from pathlib import Path
__version__ = Path(__file__).with_name('VERSION').read_text().strip() # noqa F401
......@@ -88,3 +88,28 @@ def unknown_error(context, response_cls, message, log_level=logging.ERROR):
logger.log(log_level, message)
context.set_details(message)
return response_cls()
def unavailable_error(context, response_cls, message, log_level=logging.ERROR):
"""Return grpc UNAVAILABLE code with message.
Similar to :func:`not_found`, except for the default ``log_level``
"""
context.set_code(StatusCode.UNAVAILABLE)
logger.log(log_level, message)
context.set_details(message)
return response_cls()
def failed_precondition(context, response_cls, message,
log_level=logging.INFO):
"""Return grpc FAILED_PRECONDITION code with message.
Similar to :func:`not_found`, except for the default ``log_level``,
which is ``INFO`` because it seems to be used for routine stuff, like
fetching data only if size is smaller than a given limit.
"""
context.set_code(StatusCode.FAILED_PRECONDITION)
logger.log(log_level, message)
context.set_details(message)
return response_cls()
# Copyright 2020-2021 Georges Racinet <georges.racinet@octobus.net>
#
# This software may be used and distributed according to the terms of the
# GNU General Public License version 2 or any later version.
#
# SPDX-License-Identifier: GPL-2.0-or-later
from .git import (
OBJECT_MODE_EXECUTABLE,
OBJECT_MODE_LINK,
OBJECT_MODE_NON_EXECUTABLE,
)
def git_perms(filectx):
"""Return Git representation for file permissions (aka mode).
Reference: https://git-scm.com/book/en/v2/Git-Internals-Git-Objects
"""
if filectx.islink():
return OBJECT_MODE_LINK
elif filectx.isexec():
return OBJECT_MODE_EXECUTABLE
else:
return OBJECT_MODE_NON_EXECUTABLE
......@@ -29,7 +29,7 @@ class ManifestMiner:
changeset = attr.ib()
def ls_dir(self, path):
"""List directory contents of path at given changeset.
"""List direct directory contents of path at given changeset.
Anything from inside subdirectories of ``path`` is ignored.
......@@ -57,6 +57,143 @@ class ManifestMiner:
file_paths.sort()
return sorted(subtrees), file_paths
def iter_dir_recursive(self, path):
"""Iterate on recursive directory contents of path in order.
:returns: yields pairs (path, is_directory)
"""
prefix = path.rstrip(b'/') + b'/' if path else path
prefix_len = len(prefix)
changeset = self.changeset
in_dir = False
seen_subdirs = set()
for file_path in changeset.manifest().iterkeys():
if not file_path.startswith(prefix):
if in_dir:
break
continue # pragma no cover (see coverage#198 and PEP626)
in_dir = True
split = file_path[prefix_len:].rsplit(b'/', 1)
# accumulate ancestor dirs that need to be yielded
acc = []
while len(split) > 1:
subdir = split[0]
if subdir in seen_subdirs:
# if yielded yet, all its ancestors also are
break
acc.append(subdir)
seen_subdirs.add(subdir)
split = subdir.rsplit(b'/', 1)
for subdir in reversed(acc):
yield (prefix + subdir, True)
yield (file_path, False)
def iter_dir_with_flat_paths(self, path):
"""Iterate on directory direct contents with "flat_path" information.
:returns: yields triplets (full path, is_dir, flat_path) where
``full_path`` is the path of a file or directory from
the repo root, ``is_dir`` indicates whether it is a
directory and ``flat_path`` is as explained below.
About ``flat_path``, here is a comment from the current version of
commit.proto::
// Relative path of the first subdir that doesn't have only
// one directory descendant
Gitaly reference implementation (Golang)::
func populateFlatPath(ctx context.Context, c catfile.Batch,
entries []*gitalypb.TreeEntry) error {
for _, entry := range entries {
entry.FlatPath = entry.Path
if entry.Type != gitalypb.TreeEntry_TREE {
continue
}
for i := 1; i < defaultFlatTreeRecursion; i++ {
subEntries, err := treeEntries(
ctx, c, entry.CommitOid,
string(entry.FlatPath), "", false)
if err != nil {
return err
}
if (len(subEntries) != 1 ||
subEntries[0].Type != gitalypb.TreeEntry_TREE) {
break
}
entry.FlatPath = subEntries[0].Path
}
}
return nil
}
Implementation for the standard Mercurial manifest has of course
to be very different, since it lists full paths to leaf
(non-directory) files. In particular, there are no empty directories.
The interpretation (would have to be formally proven) is that
the "flat path" is the longest common directory ancestor of all file
entries that are inside the considered directory entry.
This implementation relies on manifest to emit paths in sorted manner.
"""
prefix = path.rstrip(b'/') + b'/' if path else path
prefix_len = len(prefix)
changeset = self.changeset
in_dir = False
subdir, flat_path = None, ()
for file_path in changeset.manifest().iterkeys():
if not file_path.startswith(prefix):
if in_dir:
break
continue # pragma no cover (see coverage#198 and PEP626)
in_dir = True
split = file_path[prefix_len:].split(b'/')
if subdir is not None and split[0] != subdir:
# we are leaving subdir, yield it
dir_path = prefix + subdir
yield (dir_path, True,
prefix + b'/'.join(flat_path) if flat_path else dir_path
)
subdir, flat_path = None, ()
if len(split) == 1:
yield (file_path, False, file_path)
subdir, flat_path = None, ()
elif subdir is None:
subdir, flat_path = split[0], split[:-1]
continue
flat_path = [
segments[0] for segments in zip(flat_path, split)
if segments[0] == segments[1]
]
if subdir is not None:
dir_path = prefix + subdir
yield (dir_path, True,
prefix + b'/'.join(flat_path) if flat_path else dir_path
)
def miner(changeset):
"""Return an appropriate manifest extractor for the given changeset.
......
......@@ -20,17 +20,21 @@ import signal
import socket
from urllib.parse import urlparse
from .service.blob import BlobServicer
from .service.commit import CommitServicer
from .service.ref import RefServicer
from .service.diff import DiffServicer
from .service.repository_service import RepositoryServiceServicer
from .service.server import ServerServicer
from .stub.blob_pb2_grpc import add_BlobServiceServicer_to_server
from .stub.commit_pb2_grpc import add_CommitServiceServicer_to_server
from .stub.ref_pb2_grpc import add_RefServiceServicer_to_server
from .stub.diff_pb2_grpc import add_DiffServiceServicer_to_server
from .stub.repository_service_pb2_grpc import (
add_RepositoryServiceServicer_to_server
)
from .stub.server_pb2_grpc import add_ServerServiceServicer_to_server
logger = logging.getLogger(__name__)
......@@ -60,11 +64,13 @@ def init(listen_urls, storages):
server = grpc.server(futures.ThreadPoolExecutor(max_workers=1),
options=(('grpc.so_reuseport', 1),
))
add_BlobServiceServicer_to_server(BlobServicer(storages), server)
add_CommitServiceServicer_to_server(CommitServicer(storages), server)
add_RefServiceServicer_to_server(RefServicer(storages), server)
add_DiffServiceServicer_to_server(DiffServicer(storages), server)
add_RepositoryServiceServicer_to_server(
RepositoryServiceServicer(storages), server)
add_ServerServiceServicer_to_server(ServerServicer(storages), server)
for url in listen_urls:
try:
......
# Copyright 2021 Georges Racinet <georges.racinet@octobus.net>
#
# This software may be used and distributed according to the terms of the
# GNU General Public License version 2 or any later version.
#
# SPDX-License-Identifier: GPL-2.0-or-later
import io
import logging
from mercurial import (
error,
)
from ..stub.shared_pb2 import (
ObjectType,
)
from ..stub.blob_pb2 import (
GetBlobRequest,
GetBlobResponse,
GetBlobsRequest,
GetBlobsResponse,
)
from ..stub.blob_pb2_grpc import BlobServiceServicer
from ..oid import (
blob_oid,
extract_blob_oid,
)
from ..file_context import git_perms
from ..revision import gitlab_revision_changeset
from ..servicer import HGitalyServicer
from ..stream import WRITE_BUFFER_SIZE
logger = logging.getLogger(__name__)
class BlobServicer(BlobServiceServicer, HGitalyServicer):
"""BlobService implementation.
The ordering of methods in this source file is the same as in the proto
file.
"""
def GetBlob(self, request: GetBlobRequest,
context) -> GetBlobResponse:
logger.info("GetBlob request=%r", request)
# TODO return Unavailable for readLimit = 0, as Gitaly does
repo = self.load_repo(request.repository, context).unfiltered()
chgs_id, path = extract_blob_oid(repo, request.oid)
changeset = repo[chgs_id.encode()] # TODO error treatment
filectx = changeset.filectx(path) # TODO error treatment
size = filectx.size()
data = filectx.data()
if request.limit != -1:
data = data[:request.limit]
for chunk, is_first in iter_blob_chunks(data):
resp_dict = dict(data=chunk)
if is_first:
resp_dict.update(size=size, oid=request.oid)
yield GetBlobResponse(**resp_dict)
def GetBlobs(self, request: GetBlobsRequest, context) -> GetBlobsResponse:
revision_paths = [(rev_path.revision, rev_path.path)
for rev_path in request.revision_paths]
logger.debug("GetBlobs repo relative_path=%r, revision_paths=%r",
request.repository.relative_path,
revision_paths,
)
repo = self.load_repo(request.repository, context).unfiltered()
limit = request.limit
for rev, path in revision_paths:
# TODO it's probably an oversight on upstream side that revision
# is `str` and not `bytes`, but then, what would be the encoding?
changeset = gitlab_revision_changeset(repo,
rev.encode('utf-8', 'ignore')
)
if changeset is None:
filectx = None
else:
try:
filectx = changeset.filectx(path)
except error.ManifestLookupError:
filectx = None
if changeset is None or filectx is None:
# missing file is represented by oid=''
# (see comment in blob.proto)
yield GetBlobsResponse(path=path, revision=rev)
continue
filectx = changeset.filectx(path)
oid = blob_oid(repo, changeset.hex().decode(), path)
size = filectx.size()
data = filectx.data()
if limit != -1:
data = data[:limit]
for chunk, first in iter_blob_chunks(data):
# is_submodule will be False, because that's the default
# should at least be documented somewhere though
dict_resp = dict(data=chunk)
if first:
dict_resp.update(size=size,
oid=oid,
path=path,
revision=rev,
type=ObjectType.BLOB,
mode=git_perms(filectx),
)
yield GetBlobsResponse(**dict_resp)
def iter_blob_chunks(data, chunk_size=WRITE_BUFFER_SIZE):
"""Generator for chunks of Blob data.
The first chunk always bear the Blob oid and the full Blob size.
The next ones don't have them (i.e., they have the gRPC default values)
"""
# better than reimplementing our own cursor system
reader = io.BytesIO(data)
is_first = True
while True:
chunk = reader.read(chunk_size)
# send first response even if empty
if is_first or chunk:
yield chunk, is_first
else:
# the break statement would be equivalent, but would appears
# uncovered. If someone has the courage, it could be a
# bug to report in coverage 5.5
return
is_first = False
......@@ -18,6 +18,7 @@ from mercurial import (
from .. import manifest
from ..errors import (
failed_precondition,
internal_error,
invalid_argument,
not_implemented,
......@@ -35,6 +36,7 @@ from ..stub.commit_pb2 import (
CountDivergingCommitsResponse,
GetTreeEntriesRequest,
GetTreeEntriesResponse,
TreeEntry,
ListFilesRequest,
ListFilesResponse,
FindCommitRequest,
......@@ -69,8 +71,18 @@ from ..stub.commit_pb2 import (
from ..stub.commit_pb2_grpc import CommitServiceServicer
from .. import message
from ..oid import (
tree_oid,
blob_oid,
)
from ..revision import gitlab_revision_changeset
from ..servicer import HGitalyServicer
from ..git import (
OBJECT_MODE_TREE,
)
from ..file_context import (
git_perms,
)
from ..util import chunked
from ..stream import (
concat_resplit,
......@@ -108,8 +120,96 @@ class CommitServicer(CommitServiceServicer, HGitalyServicer):
def TreeEntry(self, request: TreeEntryRequest,
context) -> TreeEntryResponse:
return not_implemented(context, TreeEntryResponse,
issue=16) # pragma no cover
"""Return an entry of a tree.
The name could be confusing with the entry for a tree.
Actually, it always yields one response message, using the empty
response in case the given path does not resolve.
"""
logger.info("TreeEntry request=%r", request)
repo = self.load_repo(request.repository, context).unfiltered()
changeset = gitlab_revision_changeset(repo, request.revision)
if changeset is None:
# unknown revision is not an error, and expected in some cases.
yield TreeEntryResponse()
return
sha = changeset.hex().decode('ascii')
# early testing shows that even for leaf files, Gitaly ignores
# trailing slashes
path = request.path.rstrip(b'/')
try:
filectx = changeset.filectx(path)
except error.ManifestLookupError:
filectx = None
if filectx is not None:
otype = TreeEntryResponse.ObjectType.BLOB
oid = blob_oid(repo, sha, path)
mode = git_perms(filectx)
size = filectx.size()
max_size = request.max_size
if max_size != 0 and size > request.max_size:
yield failed_precondition(
context, TreeEntryResponse,
"TreeEntry: object size (%d) is bigger than the maximum "
"allowed size (%d)" % (size, max_size))
return
data = filectx.data()
limit = request.limit
if limit != 0:
data = data[:limit]
offset = 0
while offset <= size:
yield TreeEntryResponse(
type=otype,
oid=oid,
size=size,
data=data[offset:offset+WRITE_BUFFER_SIZE],
mode=mode)
offset += WRITE_BUFFER_SIZE
return
subtrees, file_paths = manifest.miner(changeset).ls_dir(path)
if not subtrees and not file_paths:
yield TreeEntryResponse()
return
# path is an actual directory
# size computation to match Git response.
# The formula for size computation is meant to match the size
# returned by Git, which is actually the size of the raw Git Tree
# object as returned by `git-cat-file` with `<type>` parameter.
#
# The raw Git Tree object is a simple concatenation of entries, each
# one being made of
# - mode (octal representation): 6 bytes for blobs (e.g 100644),
# 5 bytes for subtrees (40000)
# - 0x20 (separator): 1 byte
# - name of the entry
# - 0x00 (separator): 1 byte
# - binary SHA-1 of the object referenced by the entry: 20 bytes
# Hence the total length per entry is 27 + (name length) for subtrees
# and 28 + (name length) for blobs.
# Finally, our `ls_dir` returns full paths from the root, so the
# have to substract `len(path + b'/')`, hence `len(path) + 1`
# for each entry.
size = (
28 * len(file_paths) + 27 * len(subtrees)
+ sum(len(s) for s in subtrees) + sum(len(f) for f in file_paths)
- (len(subtrees) + len(file_paths)) * (len(path) + 1)
)
# max_size does not apply here (see Gitaly comparison test)
yield TreeEntryResponse(
type=TreeEntryResponse.ObjectType.TREE,
oid=tree_oid(repo, sha, path),
mode=OBJECT_MODE_TREE,
size=size,
)
def CommitsBetween(self,
request: CommitsBetweenRequest,
......@@ -232,13 +332,53 @@ class CommitServicer(CommitServiceServicer, HGitalyServicer):
def GetTreeEntries(self, request: GetTreeEntriesRequest,
context) -> GetTreeEntriesResponse:
return not_implemented(context, GetTreeEntriesResponse,
issue=16) # pragma no cover
logger.info("GetTreeEntries request=%r", request)
repo = self.load_repo(request.repository, context).unfiltered()
revision = request.revision
changeset = gitlab_revision_changeset(repo, revision)
sha = changeset.hex().decode('ascii')
path = request.path.rstrip(b'/') # same as in TreeEntry
if path == b'.': # special case, means the top directory
path = b''
blob_type = TreeEntry.EntryType.BLOB
tree_type = TreeEntry.EntryType.TREE
root_oid = tree_oid(repo, sha, b'')
miner = manifest.miner(changeset)
if request.recursive:
entries = ((epath, is_dir, b'')
for epath, is_dir in miner.iter_dir_recursive(path))
else:
entries = miner.iter_dir_with_flat_paths(path)
for chunk in chunked(entries):
yield GetTreeEntriesResponse(
entries=(
TreeEntry(
path=path,
type=tree_type if is_dir else blob_type,
oid=(tree_oid(repo, sha, path) if is_dir
else blob_oid(repo, sha, path)),
commit_oid=revision,
root_oid=root_oid,
mode=(OBJECT_MODE_TREE if is_dir else