Skip to content
Snippets Groups Projects
Commit 181e4f6b authored by Pierre-Yves David's avatar Pierre-Yves David :octopus:
Browse files

script: introduce a setup-repos script that download reference

This will eventually replace the makefile to setup repository
parent b705f733
No related branches found
No related tags found
No related merge requests found
#!/usr/bin/python
#
# Script to setup the reference repository used for benchmarl
DEFAULT_REPO_SOURCE = "https://static.octobus.net/asv/"
import hashlib
import os
import shutil
import subprocess
import sys
from os.path import (
abspath,
dirname,
join,
)
parent_dir = abspath(join(dirname(abspath(__file__)), ".."))
sys.path.insert(0, join(parent_dir, "lib"))
from scmperf_lib import read_yaml
class SetupError(RuntimeError):
pass
def setup_all_repos(repodir, source, repos):
"""setup all repository in the `repos` list
Repository will be fetch from `source` and setup in `repodir`
"""
print("%d repositories to setup in %s" % (len(repos), repodir))
for r in repos:
setup_one_repo(repodir, source, r)
def setup_one_repo(repodir, source, repo):
"""Setup all items for one "atomic" repo
This include the main reference repository and the partial variants. This also setup the associated data file.
Repository will be fetch from `source` and setup in `repodir`
"""
print "Setting up reference repository: %s" % repo
# setup one
# download reference
# extract partial from `.benchrepo`
# setup partial
# download partial
# hg update the reference
print ' main reference'
setup_repo_variant(repodir, source, repo, None, update=True)
refpath = join(repodir, repo + '.benchrepo')
if not os.path.exists(refpath):
print ' missing data file'
setup_repo_variant(repodir, source, repo, None, update=True, force=True)
refdata = read_yaml(refpath)
partial_sets = refdata.get("partial-sets")
if partial_sets is None:
msg = "not partials-sets declared: %s" % refdata
raise SetupError(msg)
for variant in sorted(partial_sets):
print ' partial reference: ', variant
setup_repo_variant(repodir, source, repo, variant)
def repobasename(repo_name, partial_id):
"""return the directory name for the partial variant of a repository
When partial_id is set to None, the main reference repository is used.
"""
if partial_id is None:
remote_base_name = "%s-reference" % repo_name
else:
remote_base_name = "%s-partial-%s" % (repo_name, partial_id)
return remote_base_name
def remotepath(source, repo_name, partial_id):
"""return the remote URL to fetch the given partial variant for a repository
The data will be fetch from `source`. When partial_id is set to None, the
main reference repository is used."""
remote_base_name = repobasename(repo_name, partial_id)
remote_tar_name = "%s.tar" % remote_base_name
return "%s/%s" % (source.rstrip('/'), remote_tar_name)
def repodir(repo_dir, repo_name, partial_id):
"""return the appropriate directory to put a repository variants"""
if partial_id is None:
return repo_dir
return os.path.join(repo_dir, 'partial-references')
def repopath(repo_dir, repo_name, partial_id):
"""return the full path to a repository"""
repo_dir = repodir(repo_dir, repo_name, partial_id)
base_name = repobasename(repo_name, partial_id)
return os.path.join(repo_dir, base_name)
def setupfile(repo_path):
"""return the full path to the "repo-digest" file
This file indicate the repository restoration has completed."""
return os.path.join(repo_path, '.hg', 'scm-perf-repo-digest')
def download_repo(repo_dir, source, repo_name, partial_id):
"""download a repository in the appropriate location"""
remote_path = remotepath(source, repo_name, partial_id)
repo_dir = repodir(repo_dir, repo_name, partial_id)
cmd_fetch = ["curl", remote_path]
if os.environ.get("HGUSER") == "test":
cmd_fetch.insert(1, "--silent")
cmd_untar = ["tar", "x", "--touch"]
proc_fetch = subprocess.Popen(cmd_fetch, stdout=subprocess.PIPE)
proc_untar = subprocess.Popen(cmd_untar, stdin=proc_fetch.stdout, cwd=repo_dir)
proc_fetch.stdout.close()
if proc_fetch.wait() != 0:
msg = "error downloading: [%s] %s" % (proc_fetch.returncode, remote_path)
raise SetupError(msg)
if proc_untar.wait() != 0:
msg = "error extracting tar: [%s] %s"
msg %= (proc_untar.returncode, remote_path)
raise SetupError(msg)
def update_repo(repo_path):
"""update a repository to tip"""
cmd_update = ["hg", "-R", repo_path, "update", "--clean", "tip"]
proc_update = subprocess.Popen(cmd_update)
if proc_update.wait() != 0:
msg = "error updating repo: [%s] %s"
msg %= (proc_update.returncode, repo_path)
raise SetupError(msg)
def conclude_setup(repo_path):
"""write the content file that mark a restoration as completed"""
cmd_conclude = ["hg", "-R", repo_path, "log", "--template", "{node}\n"]
proc_conclude = subprocess.Popen(cmd_conclude, stdout=subprocess.PIPE)
out = proc_conclude.communicate()[0]
if proc_conclude.returncode != 0:
msg = "error gathering repo fingerprint: [%s] %s"
msg %= (proc_conclude.returncode, repo_path)
digest = hashlib.sha512(out).hexdigest()
filepath = setupfile(repo_path)
with open(filepath + '.tmp', 'w') as f:
f.write(digest)
os.rename(filepath + '.tmp', filepath)
def setup_repo_variant(repo_dir, source, repo_name, partial_id,
update=False, force=False):
"""setup a specific variant of a repository
When partial_id is set to None, the main reference repository is used."""
repo_path = repopath(repo_dir, repo_name, partial_id)
setup_file = setupfile(repo_path)
if os.path.exists(setup_file) and not force:
print ' already up to date'
return
if os.path.exists(repo_path):
shutil.rmtree(repo_path)
print ' downloading repo'
download_repo(repo_dir, source, repo_name, partial_id)
if update:
print ' updating repo'
update_repo(repo_path)
conclude_setup(repo_path)
def parse_reposfile(path):
"""read a repos-file that list repo to setup"""
repos = []
with open(path) as reposfile:
for l in reposfile:
l = l.strip()
if l.startswith('#') or not l:
continue
repos.append(l)
return repos
if __name__ == "__main__":
if not (2 <= len(sys.argv) <= 4):
print >> sys.stderr, "usage: %s REPOS-FILE [DIRECTORY [SOURCE]]"
print >> sys.stderr, ""
print >> sys.stderr, "Make sure all item in REPOS-FILE are setup in DIRECTORY."
print >> sys.stderr, "(default `.`)."
print >> sys.stderr, "When missing, data are fetched from SOURCE"
print >> sys.stderr, "(default: %s)" % DEFAULT_REPO_SOURCE
sys.exit(128)
reposfile = sys.argv[1]
repo_dir = '.'
if len(sys.argv) >= 3:
repo_dir = sys.argv[2]
repo_dir = os.path.abspath(repo_dir)
partialdir = os.path.join(repo_dir, 'partial-references')
if not os.path.exists(partialdir):
os.makedirs(partialdir)
source = DEFAULT_REPO_SOURCE
if len(sys.argv) >= 4:
source = sys.argv[3]
repos = parse_reposfile(reposfile)
try:
setup_all_repos(repo_dir, source, repos)
except SetupError as exc:
print >> sys.stderr, 'abort:', str(exc)
sys.exit(1)
sys.exit(0)
===================================
test the reference repository setup
===================================
initial setup
(locally cache wheel)
$ mkdir -p /tmp/.scm-perf-test/pip-cache
$ export XDG_CACHE_HOME=/tmp/.scm-perf-test/pip-cache
$ export PATH=$TESTDIR/../repo-scripts/:${PATH}
$ export PATH=$TESTDIR/../script/:${PATH}
generate basic tar
------------------
$ mkdir some-dir
$ hg init some-dir/test1
$ hg -R some-dir/test1 debugbuilddag --new-file '.+5:brancha$.+11:branchb$.+30:branchc<brancha+2<branchb+2'
$ hg init some-dir/test2
$ hg -R some-dir/test2 debugbuilddag --new-file '.+6:brancha$.+15:branchb$.+14:branchc<brancha+3<branchb+7'
Define partial set
$ cat << EOF >> p-config.yaml
> partial-sets:
> same:
> missing-last-10:
> remove: "last(all(), 10)"
> roles:
> pull:
> noop:
> same:
> source: "reference"
> target: "same"
> EOF
make the tarbal
$ mkdir reference-tmp
$ cd reference-tmp
$ make-all 5.0 ../some-dir/test1 test1repo ../p-config.yaml
hg cloning repository from: ../some-dir/test1
building a reference tarball
result available at: test1repo-4ac61281-reference.tar
Cloning test1repo (test1repo-4ac61281-reference stripped of last(all(), 10)) into ./test1repo-4ac61281-partial-missing-last-10 (for exchange benchmarks)
Cloning test1repo (test1repo-4ac61281-reference stripped of None) into ./test1repo-4ac61281-partial-same (for exchange benchmarks)
$ make-all 5.0 ../some-dir/test2 test2repo ../p-config.yaml
hg cloning repository from: ../some-dir/test2
building a reference tarball
result available at: test2repo-b7c3972a-reference.tar
Cloning test2repo (test2repo-b7c3972a-reference stripped of last(all(), 10)) into ./test2repo-b7c3972a-partial-missing-last-10 (for exchange benchmarks)
Cloning test2repo (test2repo-b7c3972a-reference stripped of None) into ./test2repo-b7c3972a-partial-same (for exchange benchmarks)
$ cd ..
$ mkdir reference-tar
$ mv reference-tmp/*.tar reference-tar
setup an appropriate repos file
-------------------------------
$ cat << EOF >> test.repos
> # first repo
> test1repo-4ac61281
>
> # second repo
> # with longer comment
> test2repo-b7c3972a
> EOF
Test the setup-script
=====================
From scratch
------------
$ setup-repos test.repos repos "file://$TESTTMP/reference-tar/"
14 files updated, 0 files merged, 0 files removed, 0 files unresolved
23 files updated, 0 files merged, 0 files removed, 0 files unresolved
2 repositories to setup in $TESTTMP/repos
Setting up reference repository: test1repo-4ac61281
main reference
downloading repo
updating repo
partial reference: missing-last-10
downloading repo
partial reference: same
downloading repo
Setting up reference repository: test2repo-b7c3972a
main reference
downloading repo
updating repo
partial reference: missing-last-10
downloading repo
partial reference: same
downloading repo
$ ls -d1 $TESTTMP/repos/* $TESTTMP/repos/*/.hg/scm-perf-repo-digest $TESTTMP/repos/partial-references/* $TESTTMP/repos/partial-references/*/.hg/scm-perf-repo-digest
$TESTTMP/repos/partial-references
$TESTTMP/repos/partial-references/test1repo-4ac61281-partial-missing-last-10
$TESTTMP/repos/partial-references/test1repo-4ac61281-partial-missing-last-10/.hg/scm-perf-repo-digest
$TESTTMP/repos/partial-references/test1repo-4ac61281-partial-same
$TESTTMP/repos/partial-references/test1repo-4ac61281-partial-same/.hg/scm-perf-repo-digest
$TESTTMP/repos/partial-references/test2repo-b7c3972a-partial-missing-last-10
$TESTTMP/repos/partial-references/test2repo-b7c3972a-partial-missing-last-10/.hg/scm-perf-repo-digest
$TESTTMP/repos/partial-references/test2repo-b7c3972a-partial-same
$TESTTMP/repos/partial-references/test2repo-b7c3972a-partial-same/.hg/scm-perf-repo-digest
$TESTTMP/repos/test1repo-4ac61281-reference
$TESTTMP/repos/test1repo-4ac61281-reference/.hg/scm-perf-repo-digest
$TESTTMP/repos/test1repo-4ac61281.benchrepo
$TESTTMP/repos/test2repo-b7c3972a-reference
$TESTTMP/repos/test2repo-b7c3972a-reference/.hg/scm-perf-repo-digest
$TESTTMP/repos/test2repo-b7c3972a.benchrepo
Check the reference repository are updated
$ ls -1 $TESTTMP/repos/*
$TESTTMP/repos/test1repo-4ac61281.benchrepo
$TESTTMP/repos/test2repo-b7c3972a.benchrepo
$TESTTMP/repos/partial-references:
test1repo-4ac61281-partial-missing-last-10
test1repo-4ac61281-partial-same
test2repo-b7c3972a-partial-missing-last-10
test2repo-b7c3972a-partial-same
$TESTTMP/repos/test1repo-4ac61281-reference:
nf10
nf11
nf12
nf13
nf14
nf15
nf16
nf17
nf51
nf52
nf6
nf7
nf8
nf9
$TESTTMP/repos/test2repo-b7c3972a-reference:
nf10
nf11
nf12
nf13
nf14
nf15
nf16
nf17
nf18
nf19
nf20
nf21
nf22
nf41
nf42
nf43
nf44
nf45
nf46
nf47
nf7
nf8
nf9
Check the partial variant has not been updated
$ ls -1 $TESTTMP/repos/partial-references/*
$TESTTMP/repos/partial-references/test1repo-4ac61281-partial-missing-last-10:
$TESTTMP/repos/partial-references/test1repo-4ac61281-partial-same:
$TESTTMP/repos/partial-references/test2repo-b7c3972a-partial-missing-last-10:
$TESTTMP/repos/partial-references/test2repo-b7c3972a-partial-same:
Does not do any work if all is already there
--------------------------------------------
$ setup-repos test.repos repos "file://$TESTTMP/reference-tar/"
2 repositories to setup in $TESTTMP/repos
Setting up reference repository: test1repo-4ac61281
main reference
already up to date
partial reference: missing-last-10
already up to date
partial reference: same
already up to date
Setting up reference repository: test2repo-b7c3972a
main reference
already up to date
partial reference: missing-last-10
already up to date
partial reference: same
already up to date
Does setup missing variants if any
----------------------------------
$ rm -rf $TESTTMP/repos/partial-references/test2*same
$ setup-repos test.repos repos "file://$TESTTMP/reference-tar/"
2 repositories to setup in $TESTTMP/repos
Setting up reference repository: test1repo-4ac61281
main reference
already up to date
partial reference: missing-last-10
already up to date
partial reference: same
already up to date
Setting up reference repository: test2repo-b7c3972a
main reference
already up to date
partial reference: missing-last-10
already up to date
partial reference: same
downloading repo
Does setup missing main if any
------------------------------
$ rm -rf $TESTTMP/repos/partial-references/test1* $TESTTMP/repos/test1*
$ setup-repos test.repos repos "file://$TESTTMP/reference-tar/"
14 files updated, 0 files merged, 0 files removed, 0 files unresolved
2 repositories to setup in $TESTTMP/repos
Setting up reference repository: test1repo-4ac61281
main reference
downloading repo
updating repo
partial reference: missing-last-10
downloading repo
partial reference: same
downloading repo
Setting up reference repository: test2repo-b7c3972a
main reference
already up to date
partial reference: missing-last-10
already up to date
partial reference: same
already up to date
Partial download get reset
--------------------------
$ rm $TESTTMP/repos/partial-references/test2*same/.hg/scm-perf-repo-digest
$ touch $TESTTMP/repos/partial-references/test2*same/.hg/foo
touch: cannot touch '$TESTTMP/repos/partial-references/test2*same/.hg/foo': No such file or directory
[1]
$ touch $TESTTMP/repos/partial-references/test2*same/bar
touch: cannot touch '$TESTTMP/repos/partial-references/test2*same/bar': No such file or directory
[1]
$ setup-repos test.repos repos "file://$TESTTMP/reference-tar/"
2 repositories to setup in $TESTTMP/repos
Setting up reference repository: test1repo-4ac61281
main reference
already up to date
partial reference: missing-last-10
already up to date
partial reference: same
already up to date
Setting up reference repository: test2repo-b7c3972a
main reference
already up to date
partial reference: missing-last-10
already up to date
partial reference: same
downloading repo
$ ls $TESTTMP/repos/partial-references/test2*same/.hg/scm-perf-repo-digest
$TESTTMP/repos/partial-references/test2repo-b7c3972a-partial-same/.hg/scm-perf-repo-digest
$ ls $TESSTTMP/repos/partial-references/test2*same/.hg/foo
ls: cannot access '/repos/partial-references/test2*same/.hg/foo': No such file or directory
[2]
$ ls $TESTTMP/repos/partial-references/test2*same/bar
ls: cannot access '$TESTTMP/repos/partial-references/test2*same/bar': No such file or directory
[2]
Removing a benchrepo file get it added again
--------------------------------------------
$ rm $TESTTMP/repos/test1*.benchrepo
$ setup-repos test.repos repos "file://$TESTTMP/reference-tar/"
14 files updated, 0 files merged, 0 files removed, 0 files unresolved
2 repositories to setup in $TESTTMP/repos
Setting up reference repository: test1repo-4ac61281
main reference
already up to date
missing data file
downloading repo
updating repo
partial reference: missing-last-10
already up to date
partial reference: same
already up to date
Setting up reference repository: test2repo-b7c3972a
main reference
already up to date
partial reference: missing-last-10
already up to date
partial reference: same
already up to date
wrong invocation
----------------
$ setup-repos
usage: %s REPOS-FILE [DIRECTORY [SOURCE]]
Make sure all item in REPOS-FILE are setup in DIRECTORY.
(default `.`).
When missing, data are fetched from SOURCE
(default: https://static.octobus.net/asv/)
[128]
$ setup-repos foo bar baz fuzz
usage: %s REPOS-FILE [DIRECTORY [SOURCE]]
Make sure all item in REPOS-FILE are setup in DIRECTORY.
(default `.`).
When missing, data are fetched from SOURCE
(default: https://static.octobus.net/asv/)
[128]
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment