Skip to content
Snippets Groups Projects
Commit 6dcf0eb13b41 authored by Georges Racinet's avatar Georges Racinet
Browse files

Backups: adopting additional files backup/restore

This takes care of all configuration and state files that
should be routinely backuped for Heptapod.

Rather than painfully maintaining correspondence in the Rails
application by spawning a `tar` executable, it is much better
to use the various constants we have around in py-heptapod,
and it is a much easier setup for comprehensive testing than
in the Rails app. We also have functional tests to confirm at
the opposite of the spectrum.
parent db930a1ee453
No related branches found
Tags 3.0.0
1 merge request!54Backups: adopting additional files backup/restore
Pipeline #19223 passed
......@@ -26,4 +26,5 @@
ui as uimod,
util,
)
import os
import sys
......@@ -29,4 +30,5 @@
import sys
import tarfile
from heptapod import (
obsutil,
......@@ -35,6 +37,8 @@
topic as topicmod,
git,
)
from .branch import DEFAULT_GITLAB_BRANCH_FILE_NAME
# these have conditional imports and subsequent `None` testing in
# (urllib3 and/or requests). In other words, hgdemandimport breaks `requests`
......@@ -276,6 +280,57 @@
ui.write(pycompat.sysbytes(json.dumps(versions)))
@command(b'hpd-backup-additional', [], _(b'OUTPUT_PATH'))
def backup_additional(ui, repo, output_path):
"""Produce a tar file with everything needing backup and not in bundle.
The result is a plain, uncompressed tar at the given OUTPUT_PATH. The
file format is totally independent from the file name.
"""
with tarfile.open(output_path, 'w') as tarf:
def add_file(fpath):
relpath = os.fsdecode(os.path.relpath(fpath, repo.path))
print(relpath)
tarf.add(fpath, arcname=relpath)
# locking, as we will be reading only but we need a consistent state
# ideally should be also consistent with the bundle, but that's
# something we can improve on later (GitLab locks the project anyway
# when its backup starts TODO recheck that).
with repo.wlock(), repo.lock():
vfs, svfs = repo.vfs, repo.svfs
vfs_file_names = [
DEFAULT_GITLAB_BRANCH_FILE_NAME,
git.HeptapodGitHandler.map_file,
]
vfs_file_names.extend(fname for fname in vfs.listdir()
if fname.startswith(b'hgrc'))
for fname in vfs_file_names:
fpath = vfs.join(fname)
if not os.path.exists(fpath):
# can happen, e.g, for default branch file on empty repo
continue
add_file(fpath)
for fname in svfs.listdir():
if fname.startswith(b'gitlab.'):
add_file(svfs.join(fname))
@command(b'hpd-restore-additional', [], _(b'ADDITIONAL_BACKUP_PATH'))
def restore_additional(ui, repo, additional_backup_path):
"""Restore from a tar file produced by hpd-backup-additional.
"""
with repo.wlock(), repo.lock():
with tarfile.open(additional_backup_path) as tarf:
for mname in tarf.getnames():
if mname.startswith('/') or '..' in mname:
raise error.Abort(
b"Prohibited member '%s' in tar file at '%s'" % (
os.fsencode(mname), additional_backup_path))
tarf.extractall(os.fsdecode(repo.path))
def runsystem(orig, ui, cmd, environ, cwd, out):
heptapod_env = {k: v for k, v in ui.environ.items()
if k.startswith(b'HEPTAPOD_')}
......
# Copyright 2021 Georges Racinet <georges.racinet@octobus.net>
#
# This software may be used and distributed according to the terms of the
# GNU General Public License version 2 or any later version.
#
# SPDX-License-Identifier: GPL-2.0-or-later
import pytest
import tarfile
from io import BytesIO
from mercurial import (
error,
scmutil,
)
from heptapod.testhelpers import (
RepoWrapper,
as_bytes,
)
from .utils import common_config
from .. import (
backup_additional,
restore_additional,
)
from ..branch import (
gitlab_branches,
get_default_gitlab_branch,
)
from ..git import HeptapodGitHandler
from ..keep_around import (
create_keep_around,
iter_keep_arounds,
)
from ..special_ref import (
special_refs,
write_gitlab_special_ref,
)
from ..tag import gitlab_tags
from heptapod.testhelpers.gitlab import GitLabMirrorFixture
def test_additional_backup_restore(tmpdir, monkeypatch):
fixture = GitLabMirrorFixture.init(tmpdir, monkeypatch,
hg_config=common_config())
src_wrapper = fixture.hg_repo_wrapper
src_repo = src_wrapper.repo
main_hgrc = b'[foo]\nsome=bar\n'
managed_hgrc = b'[heptapod]\nallow-bookmars=true\n'
src_repo.vfs.write(b'hgrc', main_hgrc)
src_repo.vfs.write(b'hgrc.managed', managed_hgrc)
# Backup/restore for the empty repo to check that config
# is still ok (even though default branch file is missing)
tar_for_empty_path = tmpdir / 'additional_empty_repo.tar'
src_wrapper.command('hpd-backup-additional', as_bytes(tar_for_empty_path))
empty_dst_wrapper = RepoWrapper.init(tmpdir / 'empty_restored')
empty_dst_wrapper.command('hpd-restore-additional',
as_bytes(tar_for_empty_path))
empty_dst_repo = empty_dst_wrapper.repo
assert empty_dst_repo.vfs(b'hgrc').read() == main_hgrc
assert empty_dst_repo.vfs(b'hgrc.managed').read() == managed_hgrc
# Now with repo content and GitLab refs
sha = src_wrapper.write_commit('foo', branch='surprise').hex()
src_wrapper.command('tag', b'v3.2.1')
tagging_sha = scmutil.revsingle(src_wrapper.repo, b'.').hex()
src_wrapper.command('gitlab-mirror')
write_gitlab_special_ref(src_wrapper.repo, b'merge_requests/5/head', sha)
create_keep_around(src_repo, sha)
tar_path = as_bytes(tmpdir / 'additional.tar')
backup_additional(src_repo.ui, src_repo, tar_path)
dst_wrapper = RepoWrapper.init(tmpdir / 'restored')
dst_repo = dst_wrapper.repo
dst_wrapper.command('pull', src_repo.root)
restore_additional(dst_repo.ui, dst_repo, tar_path)
# Default GitLab branch (cannot pass by accident thanks to branch name)
# Assertions for all kinds of GitLab refs
assert get_default_gitlab_branch(src_repo) == b'branch/surprise'
assert special_refs(dst_repo) == {b'merge_requests/5/head': sha}
assert list(iter_keep_arounds(dst_repo)) == [sha]
assert gitlab_branches(dst_repo) == {b'branch/surprise': tagging_sha}
assert gitlab_tags(dst_repo) == {b'v3.2.1': sha}
# Assertions for hg-git mapping
src_handler = HeptapodGitHandler(src_repo, src_repo.ui)
dst_handler = HeptapodGitHandler(dst_repo, dst_repo.ui)
dst_git_sha = dst_handler.map_git_get(sha)
assert dst_git_sha # avoid traps
assert dst_git_sha == src_handler.map_git_get(sha)
def test_restore_bad_tar(tmpdir):
dst_wrapper = RepoWrapper.init(tmpdir / 'repo')
tar_path = tmpdir / 'bad.tar'
# str() because we support Python 3.7 (for CI)
# and path-like objects are accepted in tarfile from Python 3.8 onwards
with tarfile.open(str(tar_path), 'w') as tarf:
tinfo = tarfile.TarInfo(name='/etc/shadow')
tarf.addfile(tinfo, BytesIO(b'evil input'))
with pytest.raises(error.Abort) as exc_info:
dst_wrapper.command('hpd-restore-additional', as_bytes(tar_path))
assert b"Prohibited member" in exc_info.value.message
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment