# HG changeset patch # User Georges Racinet <georges.racinet@octobus.net> # Date 1693435598 -7200 # Thu Aug 31 00:46:38 2023 +0200 # Branch stable # Node ID aba798503268b844524b9db31003f95ef754a370 # Parent 8e8a7ac85c156b634e4ff6d0801c9a5dd01dbaab rhgitaly::mercurial::DirIteratorWithFlatPaths This is the engine behind the non-recursive case of `CommitService.GetTreeEntries`, if `skip_flat_paths` is `false`. Like the Python reference HGitaly implementation, we are interpreting the "flat path" to be the greatest common path of all entries equal or inside the given entry (see `hgitaly.manifest` Python module for details about this) diff --git a/rust/rhgitaly/src/mercurial.rs b/rust/rhgitaly/src/mercurial.rs --- a/rust/rhgitaly/src/mercurial.rs +++ b/rust/rhgitaly/src/mercurial.rs @@ -300,7 +300,6 @@ } /// Derive from `self.path` a string prefix ready for concatenation with a relative path. - #[allow(dead_code)] fn prefix(&self) -> Vec<u8> { let mut v = self.path.to_vec(); if !v.is_empty() { @@ -351,7 +350,6 @@ /// Result of splitting a path for first and last segment struct PathWithTopLevelDir<'a> { /// The full path, for flat_path computation - #[allow(dead_code)] path: &'a [u8], /// The top level directory in the path (first segment) top_level: &'a [u8], @@ -605,6 +603,154 @@ } } +/// Non recursive iterator over a directory, including "flat path" information +pub struct DirIteratorWithFlatPaths<'a, 'm, IM> +where + IM: Iterator<Item = Result<ManifestEntry<'m>, HgError>>, +{ + /// Is expected to be the repetition of an incoming request "oid" field, + /// often something like `branch/default` and not a real OID + commit_oid: String, + changeset_node: Node, + root_oid: String, + manifest_dir_iter: ManifestDirIterator<'a, 'm, IM>, + /// This iterator also has the potential to yield several elements for a single + /// read manifest entry, however this queue is overvill, because it can at most yield two + /// TreeEntries (a Tree and a Blob), so we should later on replace with something lighter. + to_yield: VecDeque<Result<TreeEntry, HgError>>, +} + +impl<'a, 'm, IM> DirIteratorWithFlatPaths<'a, 'm, IM> +where + IM: Iterator<Item = Result<ManifestEntry<'m>, HgError>>, +{ + pub fn new(commit_oid: String, changeset_node: Node, manifest: IM, path: &'a [u8]) -> Self { + let root_oid = tree_oid(&changeset_node, b""); + Self { + commit_oid, + changeset_node, + root_oid, + manifest_dir_iter: ManifestDirIterator::new(manifest, path), + /// Queue of `TreeEntry` messages that have to be emitted. + /// + /// Each manifest entry can give rise to several `TreeEntry` messages to yield, + /// because of intermediate directories. Hence we store them in this queue. + /// + /// each call to `next()` either pops an element from the queue or reads next manifest + /// line. + to_yield: VecDeque::new(), + } + } + + fn dir_tree_entry(&self, rel_path: &[u8], rel_flat_path: &[u8]) -> Result<TreeEntry, HgError> { + let mut path = self.manifest_dir_iter.prefix(); + path.extend_from_slice(rel_path); + let mut flat_path = self.manifest_dir_iter.prefix(); + flat_path.extend_from_slice(rel_flat_path); + Ok(TreeEntry { + oid: tree_oid(&self.changeset_node, &path), + mode: git::OBJECT_MODE_TREE, + commit_oid: self.commit_oid.clone(), + root_oid: self.root_oid.clone(), + path, + r#type: EntryType::Tree as i32, + flat_path, + }) + } + + fn file_tree_entry(&self, path: &[u8], mode: i32) -> Result<TreeEntry, HgError> { + Ok(TreeEntry { + mode, + oid: blob_oid(&self.changeset_node, path), + commit_oid: self.commit_oid.clone(), + root_oid: self.root_oid.clone(), + path: path.to_vec(), + r#type: EntryType::Blob as i32, + flat_path: path.to_vec(), + }) + } + + fn enqueue_file_entry(&mut self, entry: &ManifestEntry) { + match git_perms(entry) { + Err(e) => { + self.to_yield.push_back(Err(e)); + } + Ok(mode) => { + self.to_yield + .push_back(self.file_tree_entry(entry.path.as_bytes(), mode)); + } + } + } + + /// Main scanning loop for [`Iterator`] implementation + /// + /// Returns `Result<()>` whereas `next()` returns instead of `Option<Result<T>>` + /// to make error treatment easier (question-mark operator). Therefore this method + /// only pushes to `self.to_yield`. + fn inner_next(&mut self) -> Result<(), HgError> { + let mut current_subdir: Option<&[u8]> = None; + let mut current_flat_path: &[u8] = &[]; + + while let Some(entry_res) = self.manifest_dir_iter.next() { + let entry = entry_res?; + let ep = entry.path.as_bytes(); + let sp = &ep[self.manifest_dir_iter.prefix_len()..]; + + if let Some(subdir) = current_subdir { + let (common, rem_idx) = common_subpath_split(subdir, sp); + if rem_idx != 0 { + current_flat_path = common; + } else { + // we are leaving current_subdir, so schedule to yield it and clear it + // so that the later check whether we are entering a new one or simply have + // a top-level file runs. + self.to_yield + .push_back(self.dir_tree_entry(subdir, current_flat_path)); + current_subdir = None + } + } + + if current_subdir.is_none() { + match split_dir_and_top_level(sp, ep)? { + None => { + self.enqueue_file_entry(&entry); + break; + } + Some(split) => { + current_subdir = Some(split.top_level); + current_flat_path = split.path; + } + } + } + } + // If the last entry is not a top-level file, then the loop ends without yielding + // `current_subdir`, hence we need to do it now. + if let Some(subdir) = current_subdir { + self.to_yield + .push_back(self.dir_tree_entry(subdir, current_flat_path)); + } + Ok(()) + } +} + +impl<'a, 'm, IM> Iterator for DirIteratorWithFlatPaths<'a, 'm, IM> +where + IM: Iterator<Item = Result<ManifestEntry<'m>, HgError>>, +{ + type Item = Result<TreeEntry, HgError>; + + fn next(&mut self) -> Option<Self::Item> { + if let Some(entry) = self.to_yield.pop_front() { + return Some(entry); + } + + match self.inner_next() { + Ok(()) => self.to_yield.pop_front(), + Err(e) => Some(Err(e)), + } + } +} + #[cfg(test)] mod tests { @@ -928,4 +1074,59 @@ ] ); } + #[test] + fn test_dir_iter_flat_paths() { + let cs_node = Node::from_hex(b"1234567812345678123456781234567812345678").unwrap(); + + let manifest = paths_manifest(vec!["foo/a"]); + let iter = DirIteratorWithFlatPaths::new( + "branch/test".to_owned(), + cs_node, + manifest.into_iter(), + &[], + ); + let res: Vec<TreeEntry> = iter.map(|r| r.unwrap()).collect(); + assert_eq!(res, vec![testing_tree_flat_path(&cs_node, b"foo", b"foo"),]); + + let manifest = paths_manifest(vec![ + "foo.", + "foo/a", + "foo/sub/a", + "foo/sub/b", + "foo/subb", + "foo0", + ]); + let iter = DirIteratorWithFlatPaths::new( + "branch/test".to_owned(), + cs_node, + manifest.into_iter(), + b"foo", + ); + let res: Vec<TreeEntry> = iter.map(|r| r.unwrap()).collect(); + assert_eq!( + res, + vec![ + testing_blob_flat_path(&cs_node, b"foo/a", b"foo/a"), + testing_tree_flat_path(&cs_node, b"foo/sub", b"foo/sub"), + testing_blob_flat_path(&cs_node, b"foo/subb", b"foo/subb"), + ] + ); + + let manifest = paths_manifest(vec!["foo.", "foo/a", "foo/sub/ssub/b", "foo/subb", "foo0"]); + let iter = DirIteratorWithFlatPaths::new( + "branch/test".to_owned(), + cs_node, + manifest.into_iter(), + b"foo", + ); + let res: Vec<TreeEntry> = iter.map(|r| r.unwrap()).collect(); + assert_eq!( + res, + vec![ + testing_blob_flat_path(&cs_node, b"foo/a", b"foo/a"), + testing_tree_flat_path(&cs_node, b"foo/sub", b"foo/sub/ssub"), + testing_blob_flat_path(&cs_node, b"foo/subb", b"foo/subb"), + ] + ); + } }