#!/usr/bin/env python # # Script to setup the reference repository used for benchmarl DEFAULT_REPO_SOURCE = "https://static.octobus.net/asv/" import hashlib import os import shutil import subprocess import sys from os.path import ( abspath, dirname, join, ) parent_dir = abspath(join(dirname(abspath(__file__)), "..")) sys.path.insert(0, join(parent_dir, "lib")) from scmperf_lib import read_yaml class SetupError(RuntimeError): pass def setup_all_repos(repodir, source, repos): """setup all repository in the `repos` list Repository will be fetch from `source` and setup in `repodir` """ print("%d repositories to setup in %s" % (len(repos), repodir)) for r in repos: setup_one_repo(repodir, source, r) def setup_one_repo(repodir, source, repo): """Setup all items for one "atomic" repo This include the main reference repository and the partial variants. This also setup the associated data file. Repository will be fetch from `source` and setup in `repodir` """ print "Setting up reference repository: %s" % repo # setup one # download reference # extract partial from `.benchrepo` # setup partial # download partial # hg update the reference print ' main reference' setup_repo_variant(repodir, source, repo, None, update=True) refpath = join(repodir, repo + '.benchrepo') if not os.path.exists(refpath): print ' missing data file' setup_repo_variant(repodir, source, repo, None, update=True, force=True) refdata = read_yaml(refpath) partial_sets = refdata.get("partial-sets") if partial_sets is None: msg = "not partial-sets declared: %s" % refpath raise SetupError(msg) for variant in sorted(partial_sets): print ' partial reference: ', variant setup_repo_variant(repodir, source, repo, variant) def repobasename(repo_name, partial_id): """return the directory name for the partial variant of a repository When partial_id is set to None, the main reference repository is used. """ if partial_id is None: remote_base_name = "%s-reference" % repo_name else: remote_base_name = "%s-partial-%s" % (repo_name, partial_id) return remote_base_name def remotepath(source, repo_name, partial_id): """return the remote URL to fetch the given partial variant for a repository The data will be fetch from `source`. When partial_id is set to None, the main reference repository is used.""" remote_base_name = repobasename(repo_name, partial_id) remote_tar_name = "%s.tar" % remote_base_name return "%s/%s" % (source.rstrip('/'), remote_tar_name) def repodir(repo_dir, repo_name, partial_id): """return the appropriate directory to put a repository variants""" if partial_id is None: return repo_dir return os.path.join(repo_dir, 'partial-references') def repopath(repo_dir, repo_name, partial_id): """return the full path to a repository""" repo_dir = repodir(repo_dir, repo_name, partial_id) base_name = repobasename(repo_name, partial_id) return os.path.join(repo_dir, base_name) def setupfile(repo_path): """return the full path to the "repo-digest" file This file indicate the repository restoration has completed.""" return os.path.join(repo_path, '.hg', 'scm-perf-repo-digest') def download_repo(repo_dir, source, repo_name, partial_id): """download a repository in the appropriate location""" remote_path = remotepath(source, repo_name, partial_id) repo_dir = repodir(repo_dir, repo_name, partial_id) cmd_fetch = ["curl", remote_path] if os.environ.get("HGUSER") == "test": cmd_fetch.insert(1, "--silent") cmd_untar = ["tar", "x", "--touch"] proc_fetch = subprocess.Popen(cmd_fetch, stdout=subprocess.PIPE) proc_untar = subprocess.Popen(cmd_untar, stdin=proc_fetch.stdout, cwd=repo_dir) proc_fetch.stdout.close() if proc_fetch.wait() != 0: msg = "error downloading: [%s] %s" % (proc_fetch.returncode, remote_path) raise SetupError(msg) if proc_untar.wait() != 0: msg = "error extracting tar: [%s] %s" msg %= (proc_untar.returncode, remote_path) raise SetupError(msg) def update_repo(repo_path): """update a repository to tip""" cmd_update = ["hg", "-R", repo_path, "update", "--clean", "tip"] proc_update = subprocess.Popen(cmd_update) if proc_update.wait() != 0: msg = "error updating repo: [%s] %s" msg %= (proc_update.returncode, repo_path) raise SetupError(msg) def conclude_setup(repo_path): """write the content file that mark a restoration as completed""" cmd_conclude = ["hg", "-R", repo_path, "log", "--template", "{node}\n"] proc_conclude = subprocess.Popen(cmd_conclude, stdout=subprocess.PIPE) out = proc_conclude.communicate()[0] if proc_conclude.returncode != 0: msg = "error gathering repo fingerprint: [%s] %s" msg %= (proc_conclude.returncode, repo_path) digest = hashlib.sha512(out).hexdigest() filepath = setupfile(repo_path) with open(filepath + '.tmp', 'w') as f: f.write(digest) os.rename(filepath + '.tmp', filepath) def setup_repo_variant(repo_dir, source, repo_name, partial_id, update=False, force=False): """setup a specific variant of a repository When partial_id is set to None, the main reference repository is used.""" repo_path = repopath(repo_dir, repo_name, partial_id) setup_file = setupfile(repo_path) if os.path.exists(setup_file) and not force: print ' already up to date' return if os.path.exists(repo_path): shutil.rmtree(repo_path) print ' downloading repo' download_repo(repo_dir, source, repo_name, partial_id) if update: print ' updating repo' update_repo(repo_path) conclude_setup(repo_path) def parse_reposfile(path): """read a repos-file that list repo to setup""" repos = [] with open(path) as reposfile: for l in reposfile: l = l.strip() if l.startswith('#') or not l: continue repos.append(l) return repos if __name__ == "__main__": if not (2 <= len(sys.argv) <= 4): print >> sys.stderr, "usage: %s REPOS-FILE [DIRECTORY [SOURCE]]" print >> sys.stderr, "" print >> sys.stderr, "Make sure all item in REPOS-FILE are setup in DIRECTORY." print >> sys.stderr, "(default `.`)." print >> sys.stderr, "When missing, data are fetched from SOURCE" print >> sys.stderr, "(default: %s)" % DEFAULT_REPO_SOURCE sys.exit(128) reposfile = sys.argv[1] repo_dir = '.' if len(sys.argv) >= 3: repo_dir = sys.argv[2] repo_dir = os.path.abspath(repo_dir) partialdir = os.path.join(repo_dir, 'partial-references') if not os.path.exists(partialdir): os.makedirs(partialdir) source = DEFAULT_REPO_SOURCE if len(sys.argv) >= 4: source = sys.argv[3] repos = parse_reposfile(reposfile) try: setup_all_repos(repo_dir, source, repos) except SetupError as exc: print >> sys.stderr, 'abort:', str(exc) sys.exit(1) sys.exit(0)