Skip to content
GitLab
Menu
Projects
Groups
Snippets
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
Menu
Open sidebar
heptapod
hgitaly
Commits
f038573efd78
Commit
f038573e
authored
Apr 24, 2021
by
Georges Racinet
🦑
Browse files
Heptapod 0.21 is the new stable
parents
07c23fc4e28a
796f924ea568
Pipeline
#23017
passed with stage
in 1 minute and 31 seconds
Changes
34
Pipelines
6
Expand all
Hide whitespace changes
Inline
Side-by-side
.hgtags
View file @
f038573e
...
...
@@ -14,3 +14,6 @@
b22dcf5dcf7a205276e79ff4f4a729a51cd15fa8 0.11.0
39a6480f55014e3c3586d8a601b6258cd3b6c072 0.11.1
dafcae2d9a1626ce8ef01e8338bda53acaa516df 0.11.2
df53b71c17a4bbd1a85ab72e9752fc0b70ea77ab 0.12.0
9058efa8be3929d711cd80806f22785d92ac345a 0.12.1
fd405714088c2746874f119e16e899ee3f1f30e2 0.12.2
conftest.py
View file @
f038573e
import
pytest
from
hgitaly.service.blob
import
BlobServicer
from
hgitaly.service.commit
import
CommitServicer
from
hgitaly.service.ref
import
RefServicer
from
hgitaly.service.diff
import
DiffServicer
from
hgitaly.service.repository_service
import
RepositoryServiceServicer
...
...
@@ -3,10 +4,12 @@
from
hgitaly.service.commit
import
CommitServicer
from
hgitaly.service.ref
import
RefServicer
from
hgitaly.service.diff
import
DiffServicer
from
hgitaly.service.repository_service
import
RepositoryServiceServicer
from
hgitaly.service.server
import
ServerServicer
from
hgitaly.stub.blob_pb2_grpc
import
add_BlobServiceServicer_to_server
from
hgitaly.stub.commit_pb2_grpc
import
add_CommitServiceServicer_to_server
from
hgitaly.stub.ref_pb2_grpc
import
add_RefServiceServicer_to_server
from
hgitaly.stub.diff_pb2_grpc
import
add_DiffServiceServicer_to_server
from
hgitaly.stub.repository_service_pb2_grpc
import
(
add_RepositoryServiceServicer_to_server
)
...
...
@@ -7,12 +10,13 @@
from
hgitaly.stub.commit_pb2_grpc
import
add_CommitServiceServicer_to_server
from
hgitaly.stub.ref_pb2_grpc
import
add_RefServiceServicer_to_server
from
hgitaly.stub.diff_pb2_grpc
import
add_DiffServiceServicer_to_server
from
hgitaly.stub.repository_service_pb2_grpc
import
(
add_RepositoryServiceServicer_to_server
)
from
hgitaly.stub.server_pb2_grpc
import
add_ServerServiceServicer_to_server
@
pytest
.
fixture
(
scope
=
'module'
)
def
grpc_server
(
_grpc_server
,
grpc_addr
,
server_repos_root
):
storages
=
dict
(
default
=
str
(
server_repos_root
/
'default'
).
encode
())
...
...
@@ -13,9 +17,11 @@
@
pytest
.
fixture
(
scope
=
'module'
)
def
grpc_server
(
_grpc_server
,
grpc_addr
,
server_repos_root
):
storages
=
dict
(
default
=
str
(
server_repos_root
/
'default'
).
encode
())
add_BlobServiceServicer_to_server
(
BlobServicer
(
storages
),
_grpc_server
)
add_CommitServiceServicer_to_server
(
CommitServicer
(
storages
),
_grpc_server
)
add_RefServiceServicer_to_server
(
RefServicer
(
storages
),
...
...
@@ -25,6 +31,8 @@
add_RepositoryServiceServicer_to_server
(
RepositoryServiceServicer
(
storages
),
_grpc_server
)
add_ServerServiceServicer_to_server
(
ServerServicer
(
storages
),
_grpc_server
)
_grpc_server
.
add_insecure_port
(
grpc_addr
)
_grpc_server
.
start
()
yield
_grpc_server
...
...
generate-stubs
View file @
f038573e
...
...
@@ -5,7 +5,7 @@
PROTOS_DIR
=
./protos
STUBS_DIR
=
hgitaly/stub
for
PROTO
in
lint shared blob commit diff ref repository-service
;
do
for
PROTO
in
lint shared blob commit diff ref repository-service
server
;
do
python
-m
grpc_tools.protoc
\
-I
${
PROTOS_DIR
}
\
--python_out
=
${
STUBS_DIR
}
\
...
...
hgitaly/VERSION
0 → 100644
View file @
f038573e
0.12.3dev0
hgitaly/__init__.py
View file @
f038573e
# Copyright 2020 Georges Racinet <georges.racinet@octobus.net>
# Copyright 2020
-2021
Georges Racinet <georges.racinet@octobus.net>
#
# This software may be used and distributed according to the terms of the
# GNU General Public License version 2 or any later version.
#
# SPDX-License-Identifier: GPL-2.0-or-later
...
...
@@ -2,5 +2,9 @@
#
# This software may be used and distributed according to the terms of the
# GNU General Public License version 2 or any later version.
#
# SPDX-License-Identifier: GPL-2.0-or-later
from
pathlib
import
Path
__version__
=
Path
(
__file__
).
with_name
(
'VERSION'
).
read_text
().
strip
()
# noqa F401
hgitaly/errors.py
View file @
f038573e
...
...
@@ -88,3 +88,28 @@
logger
.
log
(
log_level
,
message
)
context
.
set_details
(
message
)
return
response_cls
()
def
unavailable_error
(
context
,
response_cls
,
message
,
log_level
=
logging
.
ERROR
):
"""Return grpc UNAVAILABLE code with message.
Similar to :func:`not_found`, except for the default ``log_level``
"""
context
.
set_code
(
StatusCode
.
UNAVAILABLE
)
logger
.
log
(
log_level
,
message
)
context
.
set_details
(
message
)
return
response_cls
()
def
failed_precondition
(
context
,
response_cls
,
message
,
log_level
=
logging
.
INFO
):
"""Return grpc FAILED_PRECONDITION code with message.
Similar to :func:`not_found`, except for the default ``log_level``,
which is ``INFO`` because it seems to be used for routine stuff, like
fetching data only if size is smaller than a given limit.
"""
context
.
set_code
(
StatusCode
.
FAILED_PRECONDITION
)
logger
.
log
(
log_level
,
message
)
context
.
set_details
(
message
)
return
response_cls
()
hgitaly/file_context.py
0 → 100644
View file @
f038573e
# Copyright 2020-2021 Georges Racinet <georges.racinet@octobus.net>
#
# This software may be used and distributed according to the terms of the
# GNU General Public License version 2 or any later version.
#
# SPDX-License-Identifier: GPL-2.0-or-later
from
.git
import
(
OBJECT_MODE_EXECUTABLE
,
OBJECT_MODE_LINK
,
OBJECT_MODE_NON_EXECUTABLE
,
)
def
git_perms
(
filectx
):
"""Return Git representation for file permissions (aka mode).
Reference: https://git-scm.com/book/en/v2/Git-Internals-Git-Objects
"""
if
filectx
.
islink
():
return
OBJECT_MODE_LINK
elif
filectx
.
isexec
():
return
OBJECT_MODE_EXECUTABLE
else
:
return
OBJECT_MODE_NON_EXECUTABLE
hgitaly/manifest.py
View file @
f038573e
...
...
@@ -29,7 +29,7 @@
changeset
=
attr
.
ib
()
def
ls_dir
(
self
,
path
):
"""List directory contents of path at given changeset.
"""List
direct
directory contents of path at given changeset.
Anything from inside subdirectories of ``path`` is ignored.
...
...
@@ -57,6 +57,143 @@
file_paths
.
sort
()
return
sorted
(
subtrees
),
file_paths
def
iter_dir_recursive
(
self
,
path
):
"""Iterate on recursive directory contents of path in order.
:returns: yields pairs (path, is_directory)
"""
prefix
=
path
.
rstrip
(
b
'/'
)
+
b
'/'
if
path
else
path
prefix_len
=
len
(
prefix
)
changeset
=
self
.
changeset
in_dir
=
False
seen_subdirs
=
set
()
for
file_path
in
changeset
.
manifest
().
iterkeys
():
if
not
file_path
.
startswith
(
prefix
):
if
in_dir
:
break
continue
# pragma no cover (see coverage#198 and PEP626)
in_dir
=
True
split
=
file_path
[
prefix_len
:].
rsplit
(
b
'/'
,
1
)
# accumulate ancestor dirs that need to be yielded
acc
=
[]
while
len
(
split
)
>
1
:
subdir
=
split
[
0
]
if
subdir
in
seen_subdirs
:
# if yielded yet, all its ancestors also are
break
acc
.
append
(
subdir
)
seen_subdirs
.
add
(
subdir
)
split
=
subdir
.
rsplit
(
b
'/'
,
1
)
for
subdir
in
reversed
(
acc
):
yield
(
prefix
+
subdir
,
True
)
yield
(
file_path
,
False
)
def
iter_dir_with_flat_paths
(
self
,
path
):
"""Iterate on directory direct contents with "flat_path" information.
:returns: yields triplets (full path, is_dir, flat_path) where
``full_path`` is the path of a file or directory from
the repo root, ``is_dir`` indicates whether it is a
directory and ``flat_path`` is as explained below.
About ``flat_path``, here is a comment from the current version of
commit.proto::
// Relative path of the first subdir that doesn't have only
// one directory descendant
Gitaly reference implementation (Golang)::
func populateFlatPath(ctx context.Context, c catfile.Batch,
entries []*gitalypb.TreeEntry) error {
for _, entry := range entries {
entry.FlatPath = entry.Path
if entry.Type != gitalypb.TreeEntry_TREE {
continue
}
for i := 1; i < defaultFlatTreeRecursion; i++ {
subEntries, err := treeEntries(
ctx, c, entry.CommitOid,
string(entry.FlatPath), "", false)
if err != nil {
return err
}
if (len(subEntries) != 1 ||
subEntries[0].Type != gitalypb.TreeEntry_TREE) {
break
}
entry.FlatPath = subEntries[0].Path
}
}
return nil
}
Implementation for the standard Mercurial manifest has of course
to be very different, since it lists full paths to leaf
(non-directory) files. In particular, there are no empty directories.
The interpretation (would have to be formally proven) is that
the "flat path" is the longest common directory ancestor of all file
entries that are inside the considered directory entry.
This implementation relies on manifest to emit paths in sorted manner.
"""
prefix
=
path
.
rstrip
(
b
'/'
)
+
b
'/'
if
path
else
path
prefix_len
=
len
(
prefix
)
changeset
=
self
.
changeset
in_dir
=
False
subdir
,
flat_path
=
None
,
()
for
file_path
in
changeset
.
manifest
().
iterkeys
():
if
not
file_path
.
startswith
(
prefix
):
if
in_dir
:
break
continue
# pragma no cover (see coverage#198 and PEP626)
in_dir
=
True
split
=
file_path
[
prefix_len
:].
split
(
b
'/'
)
if
subdir
is
not
None
and
split
[
0
]
!=
subdir
:
# we are leaving subdir, yield it
dir_path
=
prefix
+
subdir
yield
(
dir_path
,
True
,
prefix
+
b
'/'
.
join
(
flat_path
)
if
flat_path
else
dir_path
)
subdir
,
flat_path
=
None
,
()
if
len
(
split
)
==
1
:
yield
(
file_path
,
False
,
file_path
)
subdir
,
flat_path
=
None
,
()
elif
subdir
is
None
:
subdir
,
flat_path
=
split
[
0
],
split
[:
-
1
]
continue
flat_path
=
[
segments
[
0
]
for
segments
in
zip
(
flat_path
,
split
)
if
segments
[
0
]
==
segments
[
1
]
]
if
subdir
is
not
None
:
dir_path
=
prefix
+
subdir
yield
(
dir_path
,
True
,
prefix
+
b
'/'
.
join
(
flat_path
)
if
flat_path
else
dir_path
)
def
miner
(
changeset
):
"""Return an appropriate manifest extractor for the given changeset.
...
...
hgitaly/server.py
View file @
f038573e
...
...
@@ -20,7 +20,8 @@
import
socket
from
urllib.parse
import
urlparse
from
.service.blob
import
BlobServicer
from
.service.commit
import
CommitServicer
from
.service.ref
import
RefServicer
from
.service.diff
import
DiffServicer
from
.service.repository_service
import
RepositoryServiceServicer
...
...
@@ -23,5 +24,6 @@
from
.service.commit
import
CommitServicer
from
.service.ref
import
RefServicer
from
.service.diff
import
DiffServicer
from
.service.repository_service
import
RepositoryServiceServicer
from
.service.server
import
ServerServicer
...
...
@@ -27,7 +29,8 @@
from
.stub.blob_pb2_grpc
import
add_BlobServiceServicer_to_server
from
.stub.commit_pb2_grpc
import
add_CommitServiceServicer_to_server
from
.stub.ref_pb2_grpc
import
add_RefServiceServicer_to_server
from
.stub.diff_pb2_grpc
import
add_DiffServiceServicer_to_server
from
.stub.repository_service_pb2_grpc
import
(
add_RepositoryServiceServicer_to_server
)
...
...
@@ -28,9 +31,10 @@
from
.stub.commit_pb2_grpc
import
add_CommitServiceServicer_to_server
from
.stub.ref_pb2_grpc
import
add_RefServiceServicer_to_server
from
.stub.diff_pb2_grpc
import
add_DiffServiceServicer_to_server
from
.stub.repository_service_pb2_grpc
import
(
add_RepositoryServiceServicer_to_server
)
from
.stub.server_pb2_grpc
import
add_ServerServiceServicer_to_server
logger
=
logging
.
getLogger
(
__name__
)
...
...
@@ -60,8 +64,9 @@
server
=
grpc
.
server
(
futures
.
ThreadPoolExecutor
(
max_workers
=
1
),
options
=
((
'grpc.so_reuseport'
,
1
),
))
add_BlobServiceServicer_to_server
(
BlobServicer
(
storages
),
server
)
add_CommitServiceServicer_to_server
(
CommitServicer
(
storages
),
server
)
add_RefServiceServicer_to_server
(
RefServicer
(
storages
),
server
)
add_DiffServiceServicer_to_server
(
DiffServicer
(
storages
),
server
)
add_RepositoryServiceServicer_to_server
(
RepositoryServiceServicer
(
storages
),
server
)
...
...
@@ -63,8 +68,9 @@
add_CommitServiceServicer_to_server
(
CommitServicer
(
storages
),
server
)
add_RefServiceServicer_to_server
(
RefServicer
(
storages
),
server
)
add_DiffServiceServicer_to_server
(
DiffServicer
(
storages
),
server
)
add_RepositoryServiceServicer_to_server
(
RepositoryServiceServicer
(
storages
),
server
)
add_ServerServiceServicer_to_server
(
ServerServicer
(
storages
),
server
)
for
url
in
listen_urls
:
try
:
...
...
hgitaly/service/blob.py
0 → 100644
View file @
f038573e
# Copyright 2021 Georges Racinet <georges.racinet@octobus.net>
#
# This software may be used and distributed according to the terms of the
# GNU General Public License version 2 or any later version.
#
# SPDX-License-Identifier: GPL-2.0-or-later
import
io
import
logging
from
mercurial
import
(
error
,
)
from
..stub.shared_pb2
import
(
ObjectType
,
)
from
..stub.blob_pb2
import
(
GetBlobRequest
,
GetBlobResponse
,
GetBlobsRequest
,
GetBlobsResponse
,
)
from
..stub.blob_pb2_grpc
import
BlobServiceServicer
from
..oid
import
(
blob_oid
,
extract_blob_oid
,
)
from
..file_context
import
git_perms
from
..revision
import
gitlab_revision_changeset
from
..servicer
import
HGitalyServicer
from
..stream
import
WRITE_BUFFER_SIZE
logger
=
logging
.
getLogger
(
__name__
)
class
BlobServicer
(
BlobServiceServicer
,
HGitalyServicer
):
"""BlobService implementation.
The ordering of methods in this source file is the same as in the proto
file.
"""
def
GetBlob
(
self
,
request
:
GetBlobRequest
,
context
)
->
GetBlobResponse
:
logger
.
info
(
"GetBlob request=%r"
,
request
)
# TODO return Unavailable for readLimit = 0, as Gitaly does
repo
=
self
.
load_repo
(
request
.
repository
,
context
).
unfiltered
()
chgs_id
,
path
=
extract_blob_oid
(
repo
,
request
.
oid
)
changeset
=
repo
[
chgs_id
.
encode
()]
# TODO error treatment
filectx
=
changeset
.
filectx
(
path
)
# TODO error treatment
size
=
filectx
.
size
()
data
=
filectx
.
data
()
if
request
.
limit
!=
-
1
:
data
=
data
[:
request
.
limit
]
for
chunk
,
is_first
in
iter_blob_chunks
(
data
):
resp_dict
=
dict
(
data
=
chunk
)
if
is_first
:
resp_dict
.
update
(
size
=
size
,
oid
=
request
.
oid
)
yield
GetBlobResponse
(
**
resp_dict
)
def
GetBlobs
(
self
,
request
:
GetBlobsRequest
,
context
)
->
GetBlobsResponse
:
revision_paths
=
[(
rev_path
.
revision
,
rev_path
.
path
)
for
rev_path
in
request
.
revision_paths
]
logger
.
debug
(
"GetBlobs repo relative_path=%r, revision_paths=%r"
,
request
.
repository
.
relative_path
,
revision_paths
,
)
repo
=
self
.
load_repo
(
request
.
repository
,
context
).
unfiltered
()
limit
=
request
.
limit
for
rev
,
path
in
revision_paths
:
# TODO it's probably an oversight on upstream side that revision
# is `str` and not `bytes`, but then, what would be the encoding?
changeset
=
gitlab_revision_changeset
(
repo
,
rev
.
encode
(
'utf-8'
,
'ignore'
)
)
if
changeset
is
None
:
filectx
=
None
else
:
try
:
filectx
=
changeset
.
filectx
(
path
)
except
error
.
ManifestLookupError
:
filectx
=
None
if
changeset
is
None
or
filectx
is
None
:
# missing file is represented by oid=''
# (see comment in blob.proto)
yield
GetBlobsResponse
(
path
=
path
,
revision
=
rev
)
continue
filectx
=
changeset
.
filectx
(
path
)
oid
=
blob_oid
(
repo
,
changeset
.
hex
().
decode
(),
path
)
size
=
filectx
.
size
()
data
=
filectx
.
data
()
if
limit
!=
-
1
:
data
=
data
[:
limit
]
for
chunk
,
first
in
iter_blob_chunks
(
data
):
# is_submodule will be False, because that's the default
# should at least be documented somewhere though
dict_resp
=
dict
(
data
=
chunk
)
if
first
:
dict_resp
.
update
(
size
=
size
,
oid
=
oid
,
path
=
path
,
revision
=
rev
,
type
=
ObjectType
.
BLOB
,
mode
=
git_perms
(
filectx
),
)
yield
GetBlobsResponse
(
**
dict_resp
)
def
iter_blob_chunks
(
data
,
chunk_size
=
WRITE_BUFFER_SIZE
):
"""Generator for chunks of Blob data.
The first chunk always bear the Blob oid and the full Blob size.
The next ones don't have them (i.e., they have the gRPC default values)
"""
# better than reimplementing our own cursor system
reader
=
io
.
BytesIO
(
data
)
is_first
=
True
while
True
:
chunk
=
reader
.
read
(
chunk_size
)
# send first response even if empty
if
is_first
or
chunk
:
yield
chunk
,
is_first
else
:
# the break statement would be equivalent, but would appears
# uncovered. If someone has the courage, it could be a
# bug to report in coverage 5.5
return
is_first
=
False
hgitaly/service/commit.py
View file @
f038573e
...
...
@@ -18,6 +18,7 @@
from
..
import
manifest
from
..errors
import
(
failed_precondition
,
internal_error
,
invalid_argument
,
not_implemented
,
...
...
@@ -35,6 +36,7 @@
CountDivergingCommitsResponse
,
GetTreeEntriesRequest
,
GetTreeEntriesResponse
,
TreeEntry
,
ListFilesRequest
,
ListFilesResponse
,
FindCommitRequest
,
...
...
@@ -69,5 +71,9 @@
from
..stub.commit_pb2_grpc
import
CommitServiceServicer
from
..
import
message
from
..oid
import
(
tree_oid
,
blob_oid
,
)
from
..revision
import
gitlab_revision_changeset
from
..servicer
import
HGitalyServicer
...
...
@@ -72,5 +78,11 @@
from
..revision
import
gitlab_revision_changeset
from
..servicer
import
HGitalyServicer
from
..git
import
(
OBJECT_MODE_TREE
,
)
from
..file_context
import
(
git_perms
,
)
from
..util
import
chunked
from
..stream
import
(
concat_resplit
,
...
...
@@ -108,8 +120,96 @@
def
TreeEntry
(
self
,
request
:
TreeEntryRequest
,
context
)
->
TreeEntryResponse
:
return
not_implemented
(
context
,
TreeEntryResponse
,
issue
=
16
)
# pragma no cover
"""Return an entry of a tree.
The name could be confusing with the entry for a tree.
Actually, it always yields one response message, using the empty
response in case the given path does not resolve.
"""
logger
.
info
(
"TreeEntry request=%r"
,
request
)
repo
=
self
.
load_repo
(
request
.
repository
,
context
).
unfiltered
()
changeset
=
gitlab_revision_changeset
(
repo
,
request
.
revision
)
if
changeset
is
None
:
# unknown revision is not an error, and expected in some cases.
yield
TreeEntryResponse
()
return
sha
=
changeset
.
hex
().
decode
(
'ascii'
)
# early testing shows that even for leaf files, Gitaly ignores
# trailing slashes
path
=
request
.
path
.
rstrip
(
b
'/'
)
try
:
filectx
=
changeset
.
filectx
(
path
)
except
error
.
ManifestLookupError
:
filectx
=
None
if
filectx
is
not
None
:
otype
=
TreeEntryResponse
.
ObjectType
.
BLOB
oid
=
blob_oid
(
repo
,
sha
,
path
)
mode
=
git_perms
(
filectx
)
size
=
filectx
.
size
()
max_size
=
request
.
max_size
if
max_size
!=
0
and
size
>
request
.
max_size
:
yield
failed_precondition
(
context
,
TreeEntryResponse
,
"TreeEntry: object size (%d) is bigger than the maximum "
"allowed size (%d)"
%
(
size
,
max_size
))
return
data
=
filectx
.
data
()
limit
=
request
.
limit
if
limit
!=
0
:
data
=
data
[:
limit
]
offset
=
0
while
offset
<=
size
:
yield
TreeEntryResponse
(
type
=
otype
,
oid
=
oid
,
size
=
size
,
data
=
data
[
offset
:
offset
+
WRITE_BUFFER_SIZE
],
mode
=
mode
)
offset
+=
WRITE_BUFFER_SIZE
return
subtrees
,
file_paths
=
manifest
.
miner
(
changeset
).
ls_dir
(
path
)
if
not
subtrees
and
not
file_paths
:
yield
TreeEntryResponse
()
return
# path is an actual directory
# size computation to match Git response.
# The formula for size computation is meant to match the size
# returned by Git, which is actually the size of the raw Git Tree
# object as returned by `git-cat-file` with `<type>` parameter.
#
# The raw Git Tree object is a simple concatenation of entries, each
# one being made of
# - mode (octal representation): 6 bytes for blobs (e.g 100644),
# 5 bytes for subtrees (40000)
# - 0x20 (separator): 1 byte
# - name of the entry
# - 0x00 (separator): 1 byte
# - binary SHA-1 of the object referenced by the entry: 20 bytes
# Hence the total length per entry is 27 + (name length) for subtrees
# and 28 + (name length) for blobs.
# Finally, our `ls_dir` returns full paths from the root, so the
# have to substract `len(path + b'/')`, hence `len(path) + 1`
# for each entry.
size
=
(
28
*
len
(
file_paths
)
+
27
*
len
(
subtrees
)
+
sum
(
len
(
s
)
for
s
in
subtrees
)
+
sum
(
len
(
f
)
for
f
in
file_paths
)
-
(
len
(
subtrees
)
+
len
(
file_paths
))
*
(
len
(
path
)
+
1
)
)
# max_size does not apply here (see Gitaly comparison test)
yield
TreeEntryResponse
(
type
=
TreeEntryResponse
.
ObjectType
.
TREE
,
oid
=
tree_oid
(
repo
,
sha
,
path
),
mode
=
OBJECT_MODE_TREE
,
size
=
size
,