Skip to content
Snippets Groups Projects
setup-repos 7.14 KiB
Newer Older
#!/usr/bin/env python
#
# Script to setup the reference repository used for benchmarl

DEFAULT_REPO_SOURCE = "https://static.octobus.net/asv/"

import hashlib
import os
import shutil
import subprocess
import sys

from os.path import (
    abspath,
    dirname,
    join,
)

parent_dir = abspath(join(dirname(abspath(__file__)), ".."))
sys.path.insert(0, join(parent_dir, "lib"))

from scmperf_lib import read_yaml

class SetupError(RuntimeError):
    pass

def setup_all_repos(repodir, source, repos):
    """setup all repository in the `repos` list

    Repository will be fetch from `source` and setup in `repodir`
    """
    print("%d repositories to setup in %s" % (len(repos), repodir))
    for r in repos:
        setup_one_repo(repodir, source, r)

def setup_one_repo(repodir, source, repo):
    """Setup all items for one "atomic" repo

    This include the main reference repository and the partial variants. This also setup the associated data file.

    Repository will be fetch from `source` and setup in `repodir`
    """
    print "Setting up reference repository: %s" % repo
    # setup one
    #   download reference
    #   extract partial from `.benchrepo`
    #   setup partial
    #     download partial
    #   hg update the reference
    print '  main reference'
    setup_repo_variant(repodir, source, repo, None, update=True)
    refpath = join(repodir, repo + '.benchrepo')
    if not os.path.exists(refpath):
        print '  missing data file'
        setup_repo_variant(repodir, source, repo, None, update=True, force=True)
    refdata = read_yaml(refpath)
    partial_sets = refdata.get("partial-sets")
    if partial_sets is None:
        msg = "not partial-sets declared: %s" % refpath
        raise SetupError(msg)

    for variant in sorted(partial_sets):
        print '  partial reference: ', variant
        setup_repo_variant(repodir, source, repo, variant)

def repobasename(repo_name, partial_id):
    """return the directory name for the partial variant of a repository

    When partial_id is set to None, the main reference repository is used.
    """
    if partial_id is None:
        remote_base_name = "%s-reference" % repo_name
    else:
        remote_base_name = "%s-partial-%s" % (repo_name, partial_id)
    return remote_base_name

def remotepath(source, repo_name, partial_id):
    """return the remote URL to fetch the given partial variant for a repository

    The data will be fetch from `source`. When partial_id is set to None, the
    main reference repository is used."""
    remote_base_name = repobasename(repo_name, partial_id)
    remote_tar_name = "%s.tar" % remote_base_name
    return "%s/%s" % (source.rstrip('/'), remote_tar_name)

def repodir(repo_dir, repo_name, partial_id):
    """return the appropriate directory to put a repository variants"""
    if partial_id is None:
        return repo_dir
    return os.path.join(repo_dir, 'partial-references')

def repopath(repo_dir, repo_name, partial_id):
    """return the full path to a repository"""
    repo_dir = repodir(repo_dir, repo_name, partial_id)
    base_name = repobasename(repo_name, partial_id)
    return os.path.join(repo_dir, base_name)

def setupfile(repo_path):
    """return the full path to the "repo-digest" file

    This file indicate the repository restoration has completed."""
    return os.path.join(repo_path, '.hg', 'scm-perf-repo-digest')

def download_repo(repo_dir, source, repo_name, partial_id):
    """download a repository in the appropriate location"""
    remote_path = remotepath(source, repo_name, partial_id)
    repo_dir = repodir(repo_dir, repo_name, partial_id)
    cmd_fetch = ["curl", remote_path]
    if os.environ.get("HGUSER") == "test":
        cmd_fetch.insert(1, "--silent")
    cmd_untar = ["tar", "x", "--touch"]
    proc_fetch = subprocess.Popen(cmd_fetch, stdout=subprocess.PIPE)
    proc_untar = subprocess.Popen(cmd_untar, stdin=proc_fetch.stdout, cwd=repo_dir)
    proc_fetch.stdout.close()
    if proc_fetch.wait() != 0:
        msg = "error downloading: [%s] %s" % (proc_fetch.returncode, remote_path)
        raise SetupError(msg)
    if proc_untar.wait() != 0:
        msg = "error extracting tar: [%s] %s"
        msg %= (proc_untar.returncode, remote_path)
        raise SetupError(msg)

def update_repo(repo_path):
    """update a repository to tip"""
    cmd_update = ["hg", "-R", repo_path, "update", "--clean", "tip"]
    proc_update = subprocess.Popen(cmd_update)
    if proc_update.wait() != 0:
        msg = "error updating repo: [%s] %s"
        msg %= (proc_update.returncode, repo_path)
        raise SetupError(msg)

def conclude_setup(repo_path):
    """write the content file that mark a restoration as completed"""
    cmd_conclude = ["hg", "-R", repo_path, "log", "--template", "{node}\n"]
    proc_conclude = subprocess.Popen(cmd_conclude, stdout=subprocess.PIPE)
    out = proc_conclude.communicate()[0]
    if proc_conclude.returncode != 0:
        msg = "error gathering repo fingerprint: [%s] %s"
        msg %= (proc_conclude.returncode, repo_path)
    digest = hashlib.sha512(out).hexdigest()

    filepath = setupfile(repo_path)
    with open(filepath + '.tmp', 'w') as f:
        f.write(digest)
    os.rename(filepath + '.tmp', filepath)

def setup_repo_variant(repo_dir, source, repo_name, partial_id,
                       update=False, force=False):
    """setup a specific variant of a repository

    When partial_id is set to None, the main reference repository is used."""
    repo_path = repopath(repo_dir, repo_name, partial_id)
    setup_file = setupfile(repo_path)
    if os.path.exists(setup_file) and not force:
        print '    already up to date'
        return
    if os.path.exists(repo_path):
        shutil.rmtree(repo_path)
    print '    downloading repo'
    download_repo(repo_dir, source, repo_name, partial_id)
    if update:
        print '    updating repo'
        update_repo(repo_path)
    conclude_setup(repo_path)

def parse_reposfile(path):
    """read a repos-file that list repo to setup"""
    repos = []
    with open(path) as reposfile:
        for l in reposfile:
            l = l.strip()
            if l.startswith('#') or not l:
                continue
            repos.append(l)
    return repos


if __name__ == "__main__":
    if not (2 <= len(sys.argv) <= 4):
        print >> sys.stderr, "usage: %s REPOS-FILE [DIRECTORY [SOURCE]]"
        print >> sys.stderr, ""
        print >> sys.stderr, "Make sure all item in REPOS-FILE are setup in DIRECTORY."
        print >> sys.stderr, "(default `.`)."
        print >> sys.stderr, "When missing, data are fetched from SOURCE"
        print >> sys.stderr, "(default: %s)" % DEFAULT_REPO_SOURCE
        sys.exit(128)

    reposfile = sys.argv[1]
    repo_dir = '.'
    if len(sys.argv) >= 3:
        repo_dir = sys.argv[2]
    repo_dir = os.path.abspath(repo_dir)

    partialdir = os.path.join(repo_dir, 'partial-references')

    if not os.path.exists(partialdir):
        os.makedirs(partialdir)

    source = DEFAULT_REPO_SOURCE
    if len(sys.argv) >= 4:
        source = sys.argv[3]
    repos = parse_reposfile(reposfile)
    try:
        setup_all_repos(repo_dir, source, repos)
    except SetupError as exc:
        print >> sys.stderr, 'abort:', str(exc)
        sys.exit(1)
    sys.exit(0)