# HG changeset patch # User Georges Racinet <georges.racinet@cloudcrane.io> # Date 1737404802 -3600 # Mon Jan 20 21:26:42 2025 +0100 # Branch heptapod-17-6 # Node ID adb5d49653d610f2d40aa1e675bcc36b6216adc2 # Parent 216426df9b7fbad5a77aff8cf9fb9f2f7b18a664 Moved Git repos: fixup GitLab pools relative paths Should solve heptapod#1984, especially given that the migration to native Projects calls `hg move-hg-git-repo-out-of-gitaly-reach`. Given that this is a one shot and we happen to have the same Python dependencies in the default and stable series of Heptapod, we're hardcoding the series in the CI configuration. Obviously to be reverted upon merge in the default branch. diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -4,7 +4,7 @@ - downstream variables: - BASE_IMAGES_TAG: $CI_COMMIT_HG_BRANCH + BASE_IMAGES_TAG: default BASE_IMAGES_COLLECTION: registry.heptapod.net:443/heptapod/ci-images BASE_IMAGES_MERCURIAL: $BASE_IMAGES_COLLECTION/mercurial diff --git a/hgext3rd/heptapod/git.py b/hgext3rd/heptapod/git.py --- a/hgext3rd/heptapod/git.py +++ b/hgext3rd/heptapod/git.py @@ -11,6 +11,7 @@ """ from __future__ import absolute_import +from mercurial import error try: from hgext3rd.hggit.git_handler import GitHandler except ImportError: # pragma: no cover (fallback for hg-git < 0.11) @@ -99,6 +100,41 @@ if os.path.exists(old_gitdir) and not os.path.exists(self.gitdir): os.rename(old_gitdir, self.gitdir) + self.fixup_git_pool() + + def fixup_git_pool(self): + alternates_path = os.path.join(self.gitdir, + b'objects/info/alternates') + if not os.path.exists(alternates_path): + return + + with open(alternates_path, 'rb') as alternates: + lines = alternates.readlines() + + new_lines = [] + for line in lines: + line = line.strip() + tgt_path = os.path.join(self.gitdir, line) + if os.path.exists(tgt_path): + new_lines.append(line) + else: + repositories_root = self.repo.ui.config(b'heptapod', + b'repositories-root') + idx = tgt_path.find(b'@pools/') + if idx == -1: + raise error.Abort( + b"Unreachable Git alternates path that is not " + b"under the expected GitLab pools: %r" % tgt_path) + abs_tgt = os.path.join(repositories_root, tgt_path[idx:]) + if not os.path.exists(abs_tgt): + raise error.Abort( + b"After fix, path to shared Git inside GitLab pools " + b"does not exist: %r" % abs_tgt) + new_lines.append(os.path.relpath(abs_tgt, + self.gitdir + b'/objects')) + with open(alternates_path, 'wb') as alternates: + alternates.write(b'\n'.join(new_lines)) + @property def gitlab_refs(self): return self.git.refs diff --git a/hgext3rd/heptapod/tests/git/test_inner.py b/hgext3rd/heptapod/tests/git/test_inner.py --- a/hgext3rd/heptapod/tests/git/test_inner.py +++ b/hgext3rd/heptapod/tests/git/test_inner.py @@ -15,6 +15,7 @@ from mercurial import ( error, ) +from pathlib import Path import pytest import re @@ -22,6 +23,7 @@ from heptapod.testhelpers import ( RepoWrapper, ) +from heptapod.testhelpers.git import GitRepo from heptapod.gitlab import prune_reasons from heptapod.gitlab.branch import gitlab_branch_ref as git_branch_ref from heptapod.gitlab.change import GitLabRefChange as RefChange @@ -39,6 +41,8 @@ # TODO move to helper module from .test_integration import patch_gitlab_hooks +parametrize = pytest.mark.parametrize + @pytest.fixture def wrapper(tmpdir): @@ -449,3 +453,35 @@ fallback=REF_TARGET_UNKNOWN_RAISE) ) assert exc_info.value.args == (unknown_git_sha, ) + + +@parametrize('special', + ('abspath-not-found', 'relpath-not-found', 'other-alt')) +def test_fixup_git_pool_special_cases(wrapper, special, tmpdir): + wrapper.repo.ui.setconfig(b'heptapod', b'native', True) + git_repo = GitRepo.init(wrapper.path / '../repo.git') + if special == 'abspath-not-found': + alt_path = b'/no/such/path' + elif special == 'relpath-not-found': + alt_path = b'../@pools/does/not/exist/either' + else: + alt_path = tmpdir / 'existing' + alt_path.mkdir() + alt_path = str(alt_path).encode('utf-8') + conf_relpath = 'objects/info/alternates' + (git_repo.path / conf_relpath).write_binary(alt_path) + + if special == 'other-alt': + handler = HeptapodGitHandler(wrapper.repo, wrapper.repo.ui) + moved_path = Path(handler.gitdir.decode()) + assert moved_path != git_repo.path + assert (moved_path / conf_relpath).read_bytes() == alt_path + return + + with pytest.raises(error.Abort) as exc_info: + HeptapodGitHandler(wrapper.repo, wrapper.repo.ui) + if special == 'abspath-not-found': + expected_msg = b'Unreachable Git alternates' + else: + expected_msg = b'path to shared Git inside GitLab pools' + assert expected_msg in exc_info.value.args[0] diff --git a/hgext3rd/heptapod/tests/git/test_integration.py b/hgext3rd/heptapod/tests/git/test_integration.py --- a/hgext3rd/heptapod/tests/git/test_integration.py +++ b/hgext3rd/heptapod/tests/git/test_integration.py @@ -16,6 +16,7 @@ import pytest import re import shutil +import subprocess from heptapod.gitlab import hooks from heptapod.gitlab.prune_reasons import ( @@ -154,22 +155,52 @@ } -def test_moved_out_of_gitaly_reach(main_fixture): +@parametrize('pool', ('with-pool', 'without_pool')) +def test_moved_out_of_gitaly_reach(main_fixture, pool): + with_pool = pool == 'with-pool' fixture = main_fixture wrapper = fixture.hg_repo_wrapper wrapper.command('gitlab-mirror') orig_git_path = fixture.git_repo.path assert fixture.git_repo.branch_titles() == {b'branch/default': b'default1'} + alternates_relpath = 'objects/info/alternates' + if with_pool: + pools_path = orig_git_path / '../@pools' + pools_path.mkdir() + in_pool_path = pools_path / 'shared.git' + orig_git_path.rename(in_pool_path) + # git clone --shared normalizes the 'alternates' with absolute paths. + # There is now point trying to give it relative paths, we will + # edit the alternates later on to match what GitLab does + pool_rel_path = b'../../@pools/shared.git/objects' + subprocess.check_call(('git', 'clone', '--bare', '--shared', + in_pool_path, orig_git_path)) + (orig_git_path / alternates_relpath).write_binary(pool_rel_path) + # we did not break it by setting up the alternate objects store. + assert fixture.git_repo.branch_titles() == { + b'branch/default': b'default1'} wrapper.set_config('heptapod', 'native', True) fixture.reload_git_repo() assert fixture.git_repo.path != orig_git_path + # this performs the move wrapper.commit_file('foo', content='foo2', message="default2") wrapper.command('gitlab-mirror') assert fixture.git_repo.branch_titles() == {b'branch/default': b'default2'} + # subsequent updates (not going through the moving code) also work + wrapper.commit_file('foo', + content='foo3', + message="default3") + wrapper.command('gitlab-mirror') + assert fixture.git_repo.branch_titles() == {b'branch/default': b'default3'} + + if with_pool: + # just to be sure + assert (fixture.git_repo.path / alternates_relpath + ).read_binary() != pool_rel_path @parametrize('pushvar', ('skip-ci', 'mr-create')) # HG changeset patch # User Georges Racinet <georges.racinet@cloudcrane.io> # Date 1737405785 -3600 # Mon Jan 20 21:43:05 2025 +0100 # Node ID 619676e43dc4f241bfcd4d6a55d0f376fd7f3255 # Parent c0860aa1c0bb9a54df0c84d3431a4d6e701d0bbf # Parent adb5d49653d610f2d40aa1e675bcc36b6216adc2 Merged Git pools fixup from heptapod-17-6 branch. In the default branch, we have already stopped calling by mere instantiation of the handler the "ensure" method and the resync of the Git repo (supporting the mirrors for native projects). It is now done from the commands themselves, hence the tests have to be slightly adapted to call them as well. diff --git a/hgext3rd/heptapod/git.py b/hgext3rd/heptapod/git.py --- a/hgext3rd/heptapod/git.py +++ b/hgext3rd/heptapod/git.py @@ -11,6 +11,7 @@ """ from __future__ import absolute_import +from mercurial import error try: from hgext3rd.hggit.git_handler import GitHandler except ImportError: # pragma: no cover (fallback for hg-git < 0.11) @@ -96,6 +97,41 @@ if os.path.exists(old_gitdir) and not os.path.exists(self.gitdir): os.rename(old_gitdir, self.gitdir) + self.fixup_git_pool() + + def fixup_git_pool(self): + alternates_path = os.path.join(self.gitdir, + b'objects/info/alternates') + if not os.path.exists(alternates_path): + return + + with open(alternates_path, 'rb') as alternates: + lines = alternates.readlines() + + new_lines = [] + for line in lines: + line = line.strip() + tgt_path = os.path.join(self.gitdir, line) + if os.path.exists(tgt_path): + new_lines.append(line) + else: + repositories_root = self.repo.ui.config(b'heptapod', + b'repositories-root') + idx = tgt_path.find(b'@pools/') + if idx == -1: + raise error.Abort( + b"Unreachable Git alternates path that is not " + b"under the expected GitLab pools: %r" % tgt_path) + abs_tgt = os.path.join(repositories_root, tgt_path[idx:]) + if not os.path.exists(abs_tgt): + raise error.Abort( + b"After fix, path to shared Git inside GitLab pools " + b"does not exist: %r" % abs_tgt) + new_lines.append(os.path.relpath(abs_tgt, + self.gitdir + b'/objects')) + with open(alternates_path, 'wb') as alternates: + alternates.write(b'\n'.join(new_lines)) + @property def gitlab_refs(self): return self.git.refs diff --git a/hgext3rd/heptapod/tests/git/test_inner.py b/hgext3rd/heptapod/tests/git/test_inner.py --- a/hgext3rd/heptapod/tests/git/test_inner.py +++ b/hgext3rd/heptapod/tests/git/test_inner.py @@ -15,6 +15,7 @@ from mercurial import ( error, ) +from pathlib import Path import pytest import re @@ -22,6 +23,7 @@ from heptapod.testhelpers import ( RepoWrapper, ) +from heptapod.testhelpers.git import GitRepo from heptapod.gitlab import prune_reasons from heptapod.gitlab.branch import gitlab_branch_ref as git_branch_ref from heptapod.gitlab.change import GitLabRefChange as RefChange @@ -39,6 +41,8 @@ # TODO move to helper module from .test_integration import patch_gitlab_hooks +parametrize = pytest.mark.parametrize + @pytest.fixture def wrapper(tmpdir): @@ -449,3 +453,36 @@ fallback=REF_TARGET_UNKNOWN_RAISE) ) assert exc_info.value.args == (unknown_git_sha, ) + + +@parametrize('special', + ('abspath-not-found', 'relpath-not-found', 'other-alt')) +def test_fixup_git_pool_special_cases(wrapper, special, tmpdir): + wrapper.repo.ui.setconfig(b'heptapod', b'native', True) + git_repo = GitRepo.init(wrapper.path / '../repo.git') + if special == 'abspath-not-found': + alt_path = b'/no/such/path' + elif special == 'relpath-not-found': + alt_path = b'../@pools/does/not/exist/either' + else: + alt_path = tmpdir / 'existing' + alt_path.mkdir() + alt_path = str(alt_path).encode('utf-8') + conf_relpath = 'objects/info/alternates' + (git_repo.path / conf_relpath).write_binary(alt_path) + + handler = HeptapodGitHandler(wrapper.repo, wrapper.repo.ui) + if special == 'other-alt': + handler.native_project_ensure_set_git_dir() + moved_path = Path(handler.gitdir.decode()) + assert moved_path != git_repo.path + assert (moved_path / conf_relpath).read_bytes() == alt_path + return + + with pytest.raises(error.Abort) as exc_info: + handler.native_project_ensure_set_git_dir() + if special == 'abspath-not-found': + expected_msg = b'Unreachable Git alternates' + else: + expected_msg = b'path to shared Git inside GitLab pools' + assert expected_msg in exc_info.value.args[0] diff --git a/hgext3rd/heptapod/tests/git/test_integration.py b/hgext3rd/heptapod/tests/git/test_integration.py --- a/hgext3rd/heptapod/tests/git/test_integration.py +++ b/hgext3rd/heptapod/tests/git/test_integration.py @@ -16,6 +16,7 @@ import pytest import re import shutil +import subprocess from heptapod.gitlab import hooks from heptapod.gitlab.prune_reasons import ( @@ -154,23 +155,54 @@ } -def test_moved_out_of_gitaly_reach(main_fixture): +@parametrize('pool', ('with-pool', 'without_pool')) +def test_moved_out_of_gitaly_reach(main_fixture, pool): + with_pool = pool == 'with-pool' fixture = main_fixture wrapper = fixture.hg_repo_wrapper wrapper.command('gitlab-mirror') orig_git_path = fixture.git_repo.path assert fixture.git_repo.branch_titles() == {b'branch/default': b'default1'} + alternates_relpath = 'objects/info/alternates' + if with_pool: + pools_path = orig_git_path / '../@pools' + pools_path.mkdir() + in_pool_path = pools_path / 'shared.git' + orig_git_path.rename(in_pool_path) + # git clone --shared normalizes the 'alternates' with absolute paths. + # There is now point trying to give it relative paths, we will + # edit the alternates later on to match what GitLab does + pool_rel_path = b'../../@pools/shared.git/objects' + subprocess.check_call(('git', 'clone', '--bare', '--shared', + in_pool_path, orig_git_path)) + (orig_git_path / alternates_relpath).write_binary(pool_rel_path) + # we did not break it by setting up the alternate objects store. + assert fixture.git_repo.branch_titles() == { + b'branch/default': b'default1'} wrapper.set_config('heptapod', 'native', True) fixture.reload_git_repo() assert fixture.git_repo.path != orig_git_path + # this performs the move wrapper.commit_file('foo', content='foo2', message="default2") wrapper.command('gitlab-mirror') wrapper.command('hpd-export-native-to-git') assert fixture.git_repo.branch_titles() == {b'branch/default': b'default2'} + # subsequent updates (not going through the moving code) also work + wrapper.commit_file('foo', + content='foo3', + message="default3") + wrapper.command('gitlab-mirror') + wrapper.command('hpd-export-native-to-git') + assert fixture.git_repo.branch_titles() == {b'branch/default': b'default3'} + + if with_pool: + # just to be sure + assert (fixture.git_repo.path / alternates_relpath + ).read_binary() != pool_rel_path @parametrize('pushvar', ('skip-ci', 'mr-create'))