Skip to content
Snippets Groups Projects
Commit d274bd92 authored by Georges Racinet's avatar Georges Racinet
Browse files

RHGitaly: Blob gRPC messages definitions and adapters

This handles the fact that `GetBlobResponse` and `GetBlobsResponse` have
some similar logic (including metadata only on the first message for a
given blob) and some differences (more metadata in the latter).

We enclose the common logic in the new `message::BlobResponseChunk` trait,
and implement it for both response types.

The intermixing with data is not very natural in the Rust ownership
context, we try and make it not too painful. There is possibly room
for improvement in reducing cloning/allocations, perhaps using the
`Bytes`-oriented variant of `prost`.
parent 76f69a84
No related branches found
No related tags found
3 merge requests!186Merging stable branch into default,!177Merged oldstable branch for RHGitaly blob methods,!175RHGitaly Blob requests
......@@ -22,6 +22,7 @@
.protoc_arg("--experimental_allow_proto3_optional")
.compile(
&[
"proto/blob.proto",
"proto/commit.proto",
"proto/ref.proto",
"proto/repository.proto",
......
syntax = "proto3";
package gitaly;
import "lint.proto";
import "shared.proto";
option go_package = "gitlab.com/gitlab-org/gitaly/v15/proto/go/gitalypb";
// This comment is left unintentionally blank.
message GetBlobRequest {
// This comment is left unintentionally blank.
Repository repository = 1[(target_repository)=true];
// Object ID (SHA1) of the blob we want to get
string oid = 2;
// Maximum number of bytes we want to receive. Use '-1' to get the full blob no matter how big.
int64 limit = 3;
}
// This comment is left unintentionally blank.
message GetBlobResponse {
// Blob size; present only in first response message
int64 size = 1;
// Chunk of blob data
bytes data = 2;
// Object ID of the actual blob returned. Empty if no blob was found.
string oid = 3;
}
// This comment is left unintentionally blank.
message GetBlobsRequest {
// This comment is left unintentionally blank.
message RevisionPath {
// This comment is left unintentionally blank.
string revision = 1;
// This comment is left unintentionally blank.
bytes path = 2;
}
// This comment is left unintentionally blank.
Repository repository = 1[(target_repository)=true];
// Revision/Path pairs of the blobs we want to get.
repeated RevisionPath revision_paths = 2;
// Maximum number of bytes we want to receive. Use '-1' to get the full blobs no matter how big.
int64 limit = 3;
}
// This comment is left unintentionally blank.
message GetBlobsResponse {
// Blob size; present only on the first message per blob
int64 size = 1;
// Chunk of blob data, could span over multiple messages.
bytes data = 2;
// Object ID of the current blob. Only present on the first message per blob. Empty if no blob was found.
string oid = 3;
// This comment is left unintentionally blank.
bool is_submodule = 4;
// This comment is left unintentionally blank.
int32 mode = 5;
// This comment is left unintentionally blank.
string revision = 6;
// This comment is left unintentionally blank.
bytes path = 7;
// This comment is left unintentionally blank.
ObjectType type = 8;
}
// ListBlobsRequest is a request for the ListBlobs RPC.
message ListBlobsRequest {
// Repository is the repository in which blobs should be enumerated.
Repository repository = 1 [(target_repository)=true];
// Revisions is the list of revisions to retrieve blobs from. These revisions
// will be walked. Supports pseudo-revisions `--all` and `--not` as well as
// negated revisions via `^revision`. Revisions cannot start with a leading
// dash. Please consult gitrevisions(7) for more info. Must not be empty.
repeated string revisions = 2;
// Limit is the maximum number of blobs to return. If set to its default
// (`0`), then all found blobs wll be returned.
uint32 limit = 3;
// BytesLimit is the maximum number of bytes to receive for each blob. If set
// to `0`, then no blob data will be sent. If `-1`, then all blob data will
// be sent without any limits.
int64 bytes_limit = 4;
// WithPaths determines whether paths of blobs should be returned. When
// set to `true`, paths are returned on a best-effort basis: a path will only
// exist if the blob was traversed via a tree.
bool with_paths = 5;
}
// ListBlobsResponse is a response for the ListBlobs RPC.
message ListBlobsResponse {
// Blob represents a Git blob object.
message Blob {
// Oid is the object ID of the blob. Will only be set for the first
// message of each specific blob.
string oid = 1;
// Size is the size of the blob. Will only be set for the first message
// of each specific blob.
int64 size = 2;
// Data is the contents of the blob. This field is optional and depends on
// the BytesLimit in the original request.
bytes data = 3;
// Path is the path of the blob. May be unset depending on how the blob had
// been traversed.
bytes path = 4;
}
// Blobs is the blobs which have been found. In case blob contents were
// requested and contents of a blob exceed the maximum gRPC message size,
// then this blob will be split up into multiple blob messages which span
// across multiple responses. In that case, metadata of the blob will only be
// sent on the first such message for this specific blob.
repeated Blob blobs = 1;
}
// ListAllBlobsRequest is a request for the ListAllBlobs RPC.
message ListAllBlobsRequest {
// Repository is the repository in which blobs should be enumerated.
Repository repository = 1 [(target_repository)=true];
// Limit is the maximum number of blobs to return. If set to its default
// (`0`), then all found blobs wll be returned.
uint32 limit = 2;
// BytesLimit is the maximum number of bytes to receive for each blob. If set
// to `0`, then no blob data will be sent. If `-1`, then all blob data will
// be sent without any limits.
int64 bytes_limit = 3;
}
// ListAllBlobsResponse is a response for the ListAllBlobs RPC.
message ListAllBlobsResponse {
// Blob represents a Git blob object.
message Blob {
// Oid is the object ID of the blob. Will only be set for the first
// message of each specific blob.
string oid = 1;
// Size is the size of the blob. Will only be set for the first message
// of each specific blob.
int64 size = 2;
// Data is the contents of the blob. This field is optional and depends on
// the BytesLimit in the original request.
bytes data = 3;
}
// Blobs is the blobs which have been found. In case blob contents were
// requested and contents of a blob exceed the maximum gRPC message size,
// then this blob will be split up into multiple blob messages which span
// across multiple responses. In that case, metadata of the blob will only be
// sent on the first such message for this specific blob.
repeated Blob blobs = 1;
}
// LFSPointer is a git blob which points to an LFS object.
message LFSPointer {
// Size is the size of the blob. This is not the size of the LFS object
// pointed to.
int64 size = 1;
// Data is the bare data of the LFS pointer blob. It contains the pointer to
// the LFS data in the format specified by the LFS project.
bytes data = 2;
// Oid is the object ID of the blob.
string oid = 3;
}
// This comment is left unintentionally blank.
message NewBlobObject {
// This comment is left unintentionally blank.
int64 size = 1;
// This comment is left unintentionally blank.
string oid = 2;
// This comment is left unintentionally blank.
bytes path = 3;
}
// GetLFSPointersRequest is a request for the GetLFSPointers RPC.
message GetLFSPointersRequest {
// Repository is the repository for which LFS pointers should be retrieved
// from.
Repository repository = 1[(target_repository)=true];
// BlobIds is the list of blobs to retrieve LFS pointers from. Must be a
// non-empty list of blobs IDs to fetch.
repeated string blob_ids = 2;
}
// GetLFSPointersResponse is a response for the GetLFSPointers RPC.
message GetLFSPointersResponse {
// LfsPointers is the list of LFS pointers which were requested.
repeated LFSPointer lfs_pointers = 1;
}
// ListLFSPointersRequest is a request for the ListLFSPointers RPC.
message ListLFSPointersRequest {
// Repository is the repository for which LFS pointers should be retrieved
// from.
Repository repository = 1[(target_repository)=true];
// Revisions is the list of revisions to retrieve LFS pointers from. Must be
// a non-empty list.
repeated string revisions = 2;
// Limit limits the number of LFS pointers returned.
int32 limit = 3;
}
// ListLFSPointersResponse is a response for the ListLFSPointers RPC.
message ListLFSPointersResponse {
// LfsPointers is the list of LFS pointers which were requested.
repeated LFSPointer lfs_pointers = 1;
}
// ListAllLFSPointersRequest is a request for the ListAllLFSPointers RPC.
message ListAllLFSPointersRequest {
// Repository is the repository for which LFS pointers should be retrieved
// from.
Repository repository = 1[(target_repository)=true];
// Limit limits the number of LFS pointers returned.
int32 limit = 3;
}
// ListAllLFSPointersResponse is a response for the ListAllLFSPointers RPC.
message ListAllLFSPointersResponse {
// LfsPointers is the list of LFS pointers which were requested.
repeated LFSPointer lfs_pointers = 1;
}
......@@ -10,7 +10,10 @@
use hg::revlog::{NodePrefix, RevlogError, NULL_REVISION};
use std::fmt;
use crate::gitaly::{CommitAuthor, GitCommit};
use crate::gitaly::{
get_blobs_request::RevisionPath, CommitAuthor, GetBlobResponse, GetBlobsResponse, GitCommit,
};
use crate::mercurial::ObjectMetadata;
use prost_types::Timestamp;
use std::str::FromStr;
......@@ -251,6 +254,70 @@
Ok(Some(commit(cl.entry_for_rev(rev)?)?))
}
/// Common methods to producde blob responses, with or without metadata.
///
/// The usual Gitaly protocol convention when streaming several chunks of data for a single
/// object is to include metadata in the first response only for that object.
///
/// When several objects are streamed, as is, e.g, the case of the `GetBlobs` gRPC method,
/// the presence of metadata tells clients that the stream started treating another object.
pub trait BlobResponseChunk {
fn with_metadata(chunk: &[u8], md: ObjectMetadata) -> Self;
fn only_data(chunk: &[u8]) -> Self;
}
impl BlobResponseChunk for GetBlobsResponse {
fn with_metadata(chunk: &[u8], md: ObjectMetadata) -> Self {
let rev_path = md
.revision_path
.unwrap_or_else(|| (String::new(), Vec::new()));
GetBlobsResponse {
data: chunk.to_vec(),
mode: md.mode,
revision: rev_path.0,
path: rev_path.1,
is_submodule: false,
oid: md.oid,
size: md.size,
r#type: md.obj_type as i32,
}
}
fn only_data(chunk: &[u8]) -> Self {
GetBlobsResponse {
data: chunk.to_vec(),
..Default::default()
}
}
}
impl BlobResponseChunk for GetBlobResponse {
fn with_metadata(chunk: &[u8], md: ObjectMetadata) -> Self {
GetBlobResponse {
data: chunk.to_vec(),
oid: md.oid,
size: md.size,
}
}
fn only_data(chunk: &[u8]) -> Self {
GetBlobResponse {
data: chunk.to_vec(),
..Default::default()
}
}
}
/// Return a suitable empty [`GetBlobsResponse`] for unresolved [`RevisionPath`]
///
/// The protocol specifies that all fields are to be empty, except for `revision` and `path`
/// that are echoes of the request.
pub fn empty_blobs_response(rev_path: RevisionPath) -> GetBlobsResponse {
GetBlobsResponse {
revision: rev_path.revision,
path: rev_path.path,
..Default::default()
}
}
#[cfg(test)]
mod tests {
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment