diff --git a/rust/rhgitaly/build.rs b/rust/rhgitaly/build.rs index 76f69a84e83aa23c6a943e5f6c700a86bae4440b_cnVzdC9yaGdpdGFseS9idWlsZC5ycw==..d274bd92c3bd391cbf8326361ba7c5f3cb68c7f6_cnVzdC9yaGdpdGFseS9idWlsZC5ycw== 100644 --- a/rust/rhgitaly/build.rs +++ b/rust/rhgitaly/build.rs @@ -22,6 +22,7 @@ .protoc_arg("--experimental_allow_proto3_optional") .compile( &[ + "proto/blob.proto", "proto/commit.proto", "proto/ref.proto", "proto/repository.proto", diff --git a/rust/rhgitaly/proto/blob.proto b/rust/rhgitaly/proto/blob.proto new file mode 100644 index 0000000000000000000000000000000000000000..d274bd92c3bd391cbf8326361ba7c5f3cb68c7f6_cnVzdC9yaGdpdGFseS9wcm90by9ibG9iLnByb3Rv --- /dev/null +++ b/rust/rhgitaly/proto/blob.proto @@ -0,0 +1,221 @@ +syntax = "proto3"; + +package gitaly; + +import "lint.proto"; +import "shared.proto"; + +option go_package = "gitlab.com/gitlab-org/gitaly/v15/proto/go/gitalypb"; + +// This comment is left unintentionally blank. +message GetBlobRequest { + // This comment is left unintentionally blank. + Repository repository = 1[(target_repository)=true]; + // Object ID (SHA1) of the blob we want to get + string oid = 2; + // Maximum number of bytes we want to receive. Use '-1' to get the full blob no matter how big. + int64 limit = 3; +} + +// This comment is left unintentionally blank. +message GetBlobResponse { + // Blob size; present only in first response message + int64 size = 1; + // Chunk of blob data + bytes data = 2; + // Object ID of the actual blob returned. Empty if no blob was found. + string oid = 3; +} + +// This comment is left unintentionally blank. +message GetBlobsRequest { + // This comment is left unintentionally blank. + message RevisionPath { + // This comment is left unintentionally blank. + string revision = 1; + // This comment is left unintentionally blank. + bytes path = 2; + } + + // This comment is left unintentionally blank. + Repository repository = 1[(target_repository)=true]; + // Revision/Path pairs of the blobs we want to get. + repeated RevisionPath revision_paths = 2; + // Maximum number of bytes we want to receive. Use '-1' to get the full blobs no matter how big. + int64 limit = 3; +} + +// This comment is left unintentionally blank. +message GetBlobsResponse { + // Blob size; present only on the first message per blob + int64 size = 1; + // Chunk of blob data, could span over multiple messages. + bytes data = 2; + // Object ID of the current blob. Only present on the first message per blob. Empty if no blob was found. + string oid = 3; + // This comment is left unintentionally blank. + bool is_submodule = 4; + // This comment is left unintentionally blank. + int32 mode = 5; + // This comment is left unintentionally blank. + string revision = 6; + // This comment is left unintentionally blank. + bytes path = 7; + // This comment is left unintentionally blank. + ObjectType type = 8; +} + +// ListBlobsRequest is a request for the ListBlobs RPC. +message ListBlobsRequest { + // Repository is the repository in which blobs should be enumerated. + Repository repository = 1 [(target_repository)=true]; + // Revisions is the list of revisions to retrieve blobs from. These revisions + // will be walked. Supports pseudo-revisions `--all` and `--not` as well as + // negated revisions via `^revision`. Revisions cannot start with a leading + // dash. Please consult gitrevisions(7) for more info. Must not be empty. + repeated string revisions = 2; + // Limit is the maximum number of blobs to return. If set to its default + // (`0`), then all found blobs wll be returned. + uint32 limit = 3; + // BytesLimit is the maximum number of bytes to receive for each blob. If set + // to `0`, then no blob data will be sent. If `-1`, then all blob data will + // be sent without any limits. + int64 bytes_limit = 4; + // WithPaths determines whether paths of blobs should be returned. When + // set to `true`, paths are returned on a best-effort basis: a path will only + // exist if the blob was traversed via a tree. + bool with_paths = 5; +} + +// ListBlobsResponse is a response for the ListBlobs RPC. +message ListBlobsResponse { + // Blob represents a Git blob object. + message Blob { + // Oid is the object ID of the blob. Will only be set for the first + // message of each specific blob. + string oid = 1; + // Size is the size of the blob. Will only be set for the first message + // of each specific blob. + int64 size = 2; + // Data is the contents of the blob. This field is optional and depends on + // the BytesLimit in the original request. + bytes data = 3; + // Path is the path of the blob. May be unset depending on how the blob had + // been traversed. + bytes path = 4; + } + + // Blobs is the blobs which have been found. In case blob contents were + // requested and contents of a blob exceed the maximum gRPC message size, + // then this blob will be split up into multiple blob messages which span + // across multiple responses. In that case, metadata of the blob will only be + // sent on the first such message for this specific blob. + repeated Blob blobs = 1; +} + +// ListAllBlobsRequest is a request for the ListAllBlobs RPC. +message ListAllBlobsRequest { + // Repository is the repository in which blobs should be enumerated. + Repository repository = 1 [(target_repository)=true]; + // Limit is the maximum number of blobs to return. If set to its default + // (`0`), then all found blobs wll be returned. + uint32 limit = 2; + // BytesLimit is the maximum number of bytes to receive for each blob. If set + // to `0`, then no blob data will be sent. If `-1`, then all blob data will + // be sent without any limits. + int64 bytes_limit = 3; +} + +// ListAllBlobsResponse is a response for the ListAllBlobs RPC. +message ListAllBlobsResponse { + // Blob represents a Git blob object. + message Blob { + // Oid is the object ID of the blob. Will only be set for the first + // message of each specific blob. + string oid = 1; + // Size is the size of the blob. Will only be set for the first message + // of each specific blob. + int64 size = 2; + // Data is the contents of the blob. This field is optional and depends on + // the BytesLimit in the original request. + bytes data = 3; + } + + // Blobs is the blobs which have been found. In case blob contents were + // requested and contents of a blob exceed the maximum gRPC message size, + // then this blob will be split up into multiple blob messages which span + // across multiple responses. In that case, metadata of the blob will only be + // sent on the first such message for this specific blob. + repeated Blob blobs = 1; +} + +// LFSPointer is a git blob which points to an LFS object. +message LFSPointer { + // Size is the size of the blob. This is not the size of the LFS object + // pointed to. + int64 size = 1; + // Data is the bare data of the LFS pointer blob. It contains the pointer to + // the LFS data in the format specified by the LFS project. + bytes data = 2; + // Oid is the object ID of the blob. + string oid = 3; +} + +// This comment is left unintentionally blank. +message NewBlobObject { + // This comment is left unintentionally blank. + int64 size = 1; + // This comment is left unintentionally blank. + string oid = 2; + // This comment is left unintentionally blank. + bytes path = 3; +} + +// GetLFSPointersRequest is a request for the GetLFSPointers RPC. +message GetLFSPointersRequest { + // Repository is the repository for which LFS pointers should be retrieved + // from. + Repository repository = 1[(target_repository)=true]; + // BlobIds is the list of blobs to retrieve LFS pointers from. Must be a + // non-empty list of blobs IDs to fetch. + repeated string blob_ids = 2; +} + +// GetLFSPointersResponse is a response for the GetLFSPointers RPC. +message GetLFSPointersResponse { + // LfsPointers is the list of LFS pointers which were requested. + repeated LFSPointer lfs_pointers = 1; +} + +// ListLFSPointersRequest is a request for the ListLFSPointers RPC. +message ListLFSPointersRequest { + // Repository is the repository for which LFS pointers should be retrieved + // from. + Repository repository = 1[(target_repository)=true]; + // Revisions is the list of revisions to retrieve LFS pointers from. Must be + // a non-empty list. + repeated string revisions = 2; + // Limit limits the number of LFS pointers returned. + int32 limit = 3; +} + +// ListLFSPointersResponse is a response for the ListLFSPointers RPC. +message ListLFSPointersResponse { + // LfsPointers is the list of LFS pointers which were requested. + repeated LFSPointer lfs_pointers = 1; +} + +// ListAllLFSPointersRequest is a request for the ListAllLFSPointers RPC. +message ListAllLFSPointersRequest { + // Repository is the repository for which LFS pointers should be retrieved + // from. + Repository repository = 1[(target_repository)=true]; + // Limit limits the number of LFS pointers returned. + int32 limit = 3; +} + +// ListAllLFSPointersResponse is a response for the ListAllLFSPointers RPC. +message ListAllLFSPointersResponse { + // LfsPointers is the list of LFS pointers which were requested. + repeated LFSPointer lfs_pointers = 1; +} diff --git a/rust/rhgitaly/src/message.rs b/rust/rhgitaly/src/message.rs index 76f69a84e83aa23c6a943e5f6c700a86bae4440b_cnVzdC9yaGdpdGFseS9zcmMvbWVzc2FnZS5ycw==..d274bd92c3bd391cbf8326361ba7c5f3cb68c7f6_cnVzdC9yaGdpdGFseS9zcmMvbWVzc2FnZS5ycw== 100644 --- a/rust/rhgitaly/src/message.rs +++ b/rust/rhgitaly/src/message.rs @@ -10,7 +10,10 @@ use hg::revlog::{NodePrefix, RevlogError, NULL_REVISION}; use std::fmt; -use crate::gitaly::{CommitAuthor, GitCommit}; +use crate::gitaly::{ + get_blobs_request::RevisionPath, CommitAuthor, GetBlobResponse, GetBlobsResponse, GitCommit, +}; +use crate::mercurial::ObjectMetadata; use prost_types::Timestamp; use std::str::FromStr; @@ -251,6 +254,70 @@ Ok(Some(commit(cl.entry_for_rev(rev)?)?)) } +/// Common methods to producde blob responses, with or without metadata. +/// +/// The usual Gitaly protocol convention when streaming several chunks of data for a single +/// object is to include metadata in the first response only for that object. +/// +/// When several objects are streamed, as is, e.g, the case of the `GetBlobs` gRPC method, +/// the presence of metadata tells clients that the stream started treating another object. +pub trait BlobResponseChunk { + fn with_metadata(chunk: &[u8], md: ObjectMetadata) -> Self; + fn only_data(chunk: &[u8]) -> Self; +} + +impl BlobResponseChunk for GetBlobsResponse { + fn with_metadata(chunk: &[u8], md: ObjectMetadata) -> Self { + let rev_path = md + .revision_path + .unwrap_or_else(|| (String::new(), Vec::new())); + GetBlobsResponse { + data: chunk.to_vec(), + mode: md.mode, + revision: rev_path.0, + path: rev_path.1, + is_submodule: false, + oid: md.oid, + size: md.size, + r#type: md.obj_type as i32, + } + } + fn only_data(chunk: &[u8]) -> Self { + GetBlobsResponse { + data: chunk.to_vec(), + ..Default::default() + } + } +} + +impl BlobResponseChunk for GetBlobResponse { + fn with_metadata(chunk: &[u8], md: ObjectMetadata) -> Self { + GetBlobResponse { + data: chunk.to_vec(), + oid: md.oid, + size: md.size, + } + } + fn only_data(chunk: &[u8]) -> Self { + GetBlobResponse { + data: chunk.to_vec(), + ..Default::default() + } + } +} + +/// Return a suitable empty [`GetBlobsResponse`] for unresolved [`RevisionPath`] +/// +/// The protocol specifies that all fields are to be empty, except for `revision` and `path` +/// that are echoes of the request. +pub fn empty_blobs_response(rev_path: RevisionPath) -> GetBlobsResponse { + GetBlobsResponse { + revision: rev_path.revision, + path: rev_path.path, + ..Default::default() + } +} + #[cfg(test)] mod tests {