diff --git a/rust/rhgitaly/proto/commit.proto b/rust/rhgitaly/proto/commit.proto index eb6763d7756800f0c20f133e6156233262565705_cnVzdC9yaGdpdGFseS9wcm90by9jb21taXQucHJvdG8=..1a19cbb5876f2c87f0be87499cf8107f55cf6ae7_cnVzdC9yaGdpdGFseS9wcm90by9jb21taXQucHJvdG8= 100644 --- a/rust/rhgitaly/proto/commit.proto +++ b/rust/rhgitaly/proto/commit.proto @@ -13,6 +13,13 @@ service CommitService { // This comment is left unintentionally blank. + rpc TreeEntry(TreeEntryRequest) returns (stream TreeEntryResponse) { + option (op_type) = { + op: ACCESSOR + }; + } + + // This comment is left unintentionally blank. rpc FindCommit(FindCommitRequest) returns (FindCommitResponse) { option (op_type) = { op: ACCESSOR diff --git a/rust/rhgitaly/src/service/commit.rs b/rust/rhgitaly/src/service/commit.rs index eb6763d7756800f0c20f133e6156233262565705_cnVzdC9yaGdpdGFseS9zcmMvc2VydmljZS9jb21taXQucnM=..1a19cbb5876f2c87f0be87499cf8107f55cf6ae7_cnVzdC9yaGdpdGFseS9zcmMvc2VydmljZS9jb21taXQucnM= 100644 --- a/rust/rhgitaly/src/service/commit.rs +++ b/rust/rhgitaly/src/service/commit.rs @@ -5,4 +5,5 @@ use tracing::{field, info, instrument}; use hg::changelog::Changelog; +use hg::repo::Repo; use hg::revlog::{NodePrefix, RevlogError, NULL_REVISION}; @@ -8,2 +9,3 @@ use hg::revlog::{NodePrefix, RevlogError, NULL_REVISION}; +use hg::utils::hg_path::HgPath; @@ -9,2 +11,3 @@ +use super::blob::stream_blob; use crate::config::Config; @@ -10,3 +13,4 @@ use crate::config::Config; +use crate::git; use crate::gitaly::commit_service_server::{CommitService, CommitServiceServer}; use crate::gitaly::{ @@ -11,6 +15,7 @@ use crate::gitaly::commit_service_server::{CommitService, CommitServiceServer}; use crate::gitaly::{ - FindCommitRequest, FindCommitResponse, ListCommitsByOidRequest, ListCommitsByOidResponse, - Repository, + tree_entry_response::ObjectType, FindCommitRequest, FindCommitResponse, + ListCommitsByOidRequest, ListCommitsByOidResponse, Repository, TreeEntryRequest, + TreeEntryResponse, }; use crate::gitlab::revision::gitlab_revision_node_prefix; @@ -15,4 +20,5 @@ }; use crate::gitlab::revision::gitlab_revision_node_prefix; +use crate::mercurial::{changelog_entry_manifest, ls_path, read_blob, PathContent}; use crate::message; use crate::metadata::correlation_id; @@ -17,6 +23,7 @@ use crate::message; use crate::metadata::correlation_id; +use crate::oid::tree_oid; use crate::repository::{ default_repo_spec_error_status, load_repo_and_stream, load_repo_and_then, repo_store_vfs, RequestWithRepo, }; @@ -19,8 +26,8 @@ use crate::repository::{ default_repo_spec_error_status, load_repo_and_stream, load_repo_and_then, repo_store_vfs, RequestWithRepo, }; -use crate::streaming::ResultResponseStream; +use crate::streaming::{BlockingResponseSender, ResultResponseStream}; #[derive(Debug)] pub struct CommitServiceImpl { @@ -64,6 +71,85 @@ Ok(ListCommitsByOidResponse { commits }) } +fn not_found_for_path(path: &[u8]) -> Status { + Status::not_found(format!("not found: {}", &String::from_utf8_lossy(path))) +} + +fn stream_tree_entry( + req: TreeEntryRequest, + repo: &Repo, + node_prefix: NodePrefix, + tx: &BlockingResponseSender<TreeEntryResponse>, +) -> Result<(), Status> { + let changelog = repo + .changelog() + .map_err(|e| Status::internal(format!("Error getting changelog: {:?}", e)))?; + let manifestlog = repo + .manifestlog() + .map_err(|e| Status::internal(format!("Error getting manifestlog: {:?}", e)))?; + let (cs_node, manifest) = changelog_entry_manifest(&changelog, &manifestlog, node_prefix) + .map_err(|e| Status::internal(format!("Error obtaining manifest: {:?}", e)))? + .ok_or_else(|| not_found_for_path(&req.path))?; + + let mut pl = req.path.len(); + while pl > 0 && req.path[pl - 1] == b'/' { + pl -= 1; + } + let trimmed = &req.path[0..pl]; + + match ls_path(&manifest, trimmed) + .map_err(|e| Status::internal(format!("Error reading manifest {:?}", e)))? + { + PathContent::File(manifest_entry) => { + let hg_path = HgPath::new(&req.path); + let (mut data, metadata) = read_blob(repo, &cs_node, manifest_entry, hg_path) + .map_err(|e| Status::internal(format!("Error reading file {:?}", e)))?; + if req.max_size != 0 && metadata.size > req.max_size { + tx.send(Err(Status::failed_precondition(format!( + "TreeEntry: object size ({}) is bigger than the maximum \ + allowed size ({})", + metadata.size, req.max_size + )))); + } else { + if req.limit > 0 { + data.truncate(req.limit as usize); + } + stream_blob(tx, data, metadata) + } + } + PathContent::NotFound => { + return Err(not_found_for_path(&req.path)); + } + PathContent::Directory(listing) => { + let mut size: usize = 0; + // See Python impl for explanation (note that the Rust implementation is using + // the relative path instead of the absolute path). + for elt in listing.iter() { + if elt.is_file() { + size += 28 + } else { + size += 27 + } + size += elt.relative_path.len(); + } + tx.send(Ok(TreeEntryResponse { + r#type: ObjectType::Tree as i32, + oid: tree_oid(&cs_node, trimmed), + mode: git::OBJECT_MODE_TREE, + size: size as i64, + ..Default::default() + })); + } + } + Ok(()) +} + +impl RequestWithRepo for TreeEntryRequest { + fn repository_ref(&self) -> Option<&Repository> { + self.repository.as_ref() + } +} + impl RequestWithRepo for ListCommitsByOidRequest { fn repository_ref(&self) -> Option<&Repository> { self.repository.as_ref() @@ -79,6 +165,7 @@ #[tonic::async_trait] impl CommitService for CommitServiceImpl { type ListCommitsByOidStream = ReceiverStream<Result<ListCommitsByOidResponse, Status>>; + type TreeEntryStream = ReceiverStream<Result<TreeEntryResponse, Status>>; async fn list_commits_by_oid( &self, @@ -99,6 +186,30 @@ self.inner_find_commit(inner, correlation_id(&metadata)) .await } + + async fn tree_entry( + &self, + request: Request<TreeEntryRequest>, + ) -> Result<Response<Self::TreeEntryStream>, Status> { + let (metadata, _ext, inner) = request.into_parts(); + + self.inner_tree_entry(inner, correlation_id(&metadata)) + .await + } +} + +struct TreeEntryTracingRequest<'a>(&'a TreeEntryRequest); + +impl Debug for TreeEntryTracingRequest<'_> { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.debug_struct("TreeEntryRequest") + .field("repository", &self.0.repository) + .field("revision", &String::from_utf8_lossy(&self.0.revision)) + .field("path", &String::from_utf8_lossy(&self.0.path)) + .field("limit", &self.0.limit) + .field("max_size", &self.0.max_size) + .finish() + } } struct FindCommitTracingRequest<'a>(&'a FindCommitRequest); @@ -113,6 +224,50 @@ } impl CommitServiceImpl { + #[instrument(name = "tree_entry", + skip(self, request), + fields(request=field::debug(TreeEntryTracingRequest(&request))))] + async fn inner_tree_entry( + &self, + request: TreeEntryRequest, + correlation_id: Option<&MetadataValue<Ascii>>, + ) -> ResultResponseStream<TreeEntryResponse> { + info!("Processing"); + + if request.revision.is_empty() { + return Err(Status::invalid_argument("empty revision")); + } + + let config = self.config.clone(); + let store_vfs = repo_store_vfs(&config, &request.repository) + .await + .map_err(default_repo_spec_error_status)?; + + match gitlab_revision_node_prefix(&store_vfs, &request.revision) + .await + .map_err(|e| Status::internal(format!("Error resolving revision: {:?}", e)))? + { + None => { + info!("Revision not resolved"); + Err(not_found_for_path(&request.path)) + } + Some(node_prefix) => { + info!("Revision resolved as {:x}", &node_prefix); + + load_repo_and_stream( + self.config.clone(), + request, + default_repo_spec_error_status, + move |req, repo, tx| { + if let Err(e) = stream_tree_entry(req, &repo, node_prefix, &tx) { + tx.send(Err(e)); + } + }, + ) + } + } + } + #[instrument(name = "list_commits_by_oid", skip(self))] async fn inner_list_commits_by_oid( &self, diff --git a/tests_with_gitaly/test_blob_tree.py b/tests_with_gitaly/test_blob_tree.py index eb6763d7756800f0c20f133e6156233262565705_dGVzdHNfd2l0aF9naXRhbHkvdGVzdF9ibG9iX3RyZWUucHk=..1a19cbb5876f2c87f0be87499cf8107f55cf6ae7_dGVzdHNfd2l0aF9naXRhbHkvdGVzdF9ibG9iX3RyZWUucHk= 100644 --- a/tests_with_gitaly/test_blob_tree.py +++ b/tests_with_gitaly/test_blob_tree.py @@ -13,6 +13,7 @@ ) from hgitaly.revision import ( gitlab_revision_changeset, + ZERO_SHA, ZERO_SHA_STR, ) from hgitaly.stream import WRITE_BUFFER_SIZE @@ -99,8 +100,9 @@ return normalizer -def test_compare_tree_entry_request(gitaly_comparison): - fixture = gitaly_comparison +@parametrize('hg_server', ('hgitaly', 'rhgitaly')) +def test_compare_tree_entry_request(gitaly_rhgitaly_comparison, hg_server): + fixture = gitaly_rhgitaly_comparison wrapper = fixture.hg_repo_wrapper wrapper.write_commit('foo', message="Some foo") @@ -123,6 +125,7 @@ normalizer = oid_normalizer(oid2git) rpc_helper = fixture.rpc_helper(stub_cls=CommitServiceStub, + hg_server=hg_server, method_name='TreeEntry', request_cls=TreeEntryRequest, request_defaults=dict( @@ -154,6 +157,7 @@ # unknown revision assert_compare_errors(path=b'sub', revision=b'unknown') + assert_compare_errors(path=b'sub', revision=ZERO_SHA) # chunking for big Blob entry wrapper.write_commit('bigfile', message="A big file with 2 or 3 chunks", @@ -157,7 +161,7 @@ # chunking for big Blob entry wrapper.write_commit('bigfile', message="A big file with 2 or 3 chunks", - content=b'a' * 1023 + b'ff' * 16384) + content=b'a' * 1023 + b'ff' * 65536) # default_rev now resolves to another changeset, hence a different # HGitaly OID given the current implementation oid2git.update(oid_mapping(fixture, ((default_rev, b'bigfile', 'blob'), @@ -173,6 +177,9 @@ # reusing content to test GetTreeEntries, hence with a new rpc_helper # + if hg_server == 'rhgitaly': # TODO split with a module fixture + return + def gte_normalizer(rpc_helper, responses, vcs='hg'): for resp in responses: for entry in resp.entries: