diff --git a/benchmarks/basic_commands.py b/benchmarks/basic_commands.py index 8bec24508fbe3bd049617310a36e3097f53c32fb_YmVuY2htYXJrcy9iYXNpY19jb21tYW5kcy5weQ==..096662f9d12a831edd0ee8ad7749e960be694322_YmVuY2htYXJrcy9iYXNpY19jb21tYW5kcy5weQ== 100644 --- a/benchmarks/basic_commands.py +++ b/benchmarks/basic_commands.py @@ -1,6 +1,5 @@ from __future__ import print_function -import base64 import errno import os import os.path @@ -14,6 +13,7 @@ import stat import sys import tempfile +import urllib import pipes from .utils import (BaseNChangesetsTestSuite, BaseTestSuite, @@ -240,4 +240,5 @@ self._hgserve = None if repo_type == 'http' and self.get_asv_rev() in self.get_skip()['hgweb']: raise NotImplementedError + repo_suffix = urllib.quote_plus(strip) self.clone_path = os.path.join(REPOS_DIR, '.cache', 'partial-{}-{}'.format( @@ -243,5 +244,5 @@ self.clone_path = os.path.join(REPOS_DIR, '.cache', 'partial-{}-{}'.format( - repo_name, base64.b64encode(strip))) + repo_name, repo_suffix)) if revset is not None: self.rev = self.hg('identify', '-i', '-r', revset).strip() else: diff --git a/prepare_repos.py b/clone_repos.py similarity index 28% rename from prepare_repos.py rename to clone_repos.py index 8bec24508fbe3bd049617310a36e3097f53c32fb_cHJlcGFyZV9yZXBvcy5weQ==..096662f9d12a831edd0ee8ad7749e960be694322_Y2xvbmVfcmVwb3MucHk= 100644 --- a/prepare_repos.py +++ b/clone_repos.py @@ -1,5 +1,3 @@ """ Ensure that all repository are cloned and at their tip """ -import base64 -import errno import hglib @@ -5,7 +3,4 @@ import hglib -import json -import os -import subprocess from os.path import isdir, join import yaml @@ -9,9 +4,6 @@ from os.path import isdir, join import yaml -# changesets stripped in partial clones -PARTIAL_REVSET = ("same", 'last(all(), 10)', 'last(all(), 100)', 'last(all(), 1000)') - def read_configuration(config_path): with open(config_path) as config_file: @@ -19,12 +11,6 @@ def clone_repositories(config, repos_dir): - cachedir = join(repos_dir, '.cache') - try: - os.makedirs(cachedir) - except OSError as exc: - if exc.errno != errno.EEXIST: - raise - for repo_name, repo in config['repos'].items(): + for repo_name, repo in config["repos"].items(): clonedir = join(repos_dir, repo_name) if not isdir(clonedir): @@ -29,37 +15,10 @@ clonedir = join(repos_dir, repo_name) if not isdir(clonedir): - print("Cloning %s (%s) into %s" % (repo_name, repo['url'], clonedir)) - hglib.clone(repo['url'], clonedir) - for revset in PARTIAL_REVSET: - partialdir = join(cachedir, 'partial-{}-{}'.format( - repo_name, base64.b64encode(revset))) - if not isdir(partialdir): - print("Cloning %s (%s%s) into %s (for exchange benchmarks)" % ( - repo_name, clonedir, - " stripped of %s" % (revset,) if revset != "same" else "", partialdir)) - subprocess.check_call(['hg', 'clone', '--noupdate', clonedir, - partialdir]) - if revset != "same": - subprocess.check_call(['hg', '--cwd', partialdir, - '--config', 'extensions.strip=', 'strip', - '--no-backup', '-r', revset]) - subprocess.check_call(['hg', '--cwd', partialdir, 'debugupdatecache']) - - -def generate_skip_file(config): - skip = {} - for key, revsets in config['skip'].items(): - skiprev = set() - for revset in revsets: - for rev in subprocess.check_output([ - 'hg', 'log', '--cwd', 'mercurial', - '-r', revset, '-T', '{node|short}\n', - ]).splitlines(): - skiprev.add(rev) - skip[key] = list(skiprev) - with open(os.path.join('repos', 'skip.json'), 'w') as f: - json.dump(skip, f) + print("Cloning %s (%s) into %s" % (repo_name, repo["url"], clonedir)) + hglib.clone(repo["url"], clonedir) + else: + print("Repository %s is already cloned" % repo_name) config = read_configuration("config.yaml") clone_repositories(config, "repos") @@ -62,5 +21,4 @@ config = read_configuration("config.yaml") clone_repositories(config, "repos") -skip = generate_skip_file(config) diff --git a/prepare_repos.py b/create_skip_file.py similarity index 30% copy from prepare_repos.py copy to create_skip_file.py index 8bec24508fbe3bd049617310a36e3097f53c32fb_cHJlcGFyZV9yZXBvcy5weQ==..096662f9d12a831edd0ee8ad7749e960be694322_Y3JlYXRlX3NraXBfZmlsZS5weQ== 100644 --- a/prepare_repos.py +++ b/create_skip_file.py @@ -1,8 +1,5 @@ """ Ensure that all repository are cloned and at their tip """ -import base64 -import errno -import hglib import json import os import subprocess @@ -6,6 +3,5 @@ import json import os import subprocess -from os.path import isdir, join import yaml @@ -10,11 +6,8 @@ import yaml -# changesets stripped in partial clones -PARTIAL_REVSET = ("same", 'last(all(), 10)', 'last(all(), 100)', 'last(all(), 1000)') - def read_configuration(config_path): with open(config_path) as config_file: return yaml.load(config_file.read()) @@ -15,36 +8,8 @@ def read_configuration(config_path): with open(config_path) as config_file: return yaml.load(config_file.read()) -def clone_repositories(config, repos_dir): - cachedir = join(repos_dir, '.cache') - try: - os.makedirs(cachedir) - except OSError as exc: - if exc.errno != errno.EEXIST: - raise - for repo_name, repo in config['repos'].items(): - clonedir = join(repos_dir, repo_name) - if not isdir(clonedir): - print("Cloning %s (%s) into %s" % (repo_name, repo['url'], clonedir)) - hglib.clone(repo['url'], clonedir) - for revset in PARTIAL_REVSET: - partialdir = join(cachedir, 'partial-{}-{}'.format( - repo_name, base64.b64encode(revset))) - if not isdir(partialdir): - print("Cloning %s (%s%s) into %s (for exchange benchmarks)" % ( - repo_name, clonedir, - " stripped of %s" % (revset,) if revset != "same" else "", partialdir)) - subprocess.check_call(['hg', 'clone', '--noupdate', clonedir, - partialdir]) - if revset != "same": - subprocess.check_call(['hg', '--cwd', partialdir, - '--config', 'extensions.strip=', 'strip', - '--no-backup', '-r', revset]) - subprocess.check_call(['hg', '--cwd', partialdir, 'debugupdatecache']) - - def generate_skip_file(config): skip = {} @@ -49,5 +14,5 @@ def generate_skip_file(config): skip = {} - for key, revsets in config['skip'].items(): + for key, revsets in config["skip"].items(): skiprev = set() for revset in revsets: @@ -52,8 +17,16 @@ skiprev = set() for revset in revsets: - for rev in subprocess.check_output([ - 'hg', 'log', '--cwd', 'mercurial', - '-r', revset, '-T', '{node|short}\n', - ]).splitlines(): + for rev in subprocess.check_output( + [ + "hg", + "log", + "--cwd", + "mercurial", + "-r", + revset, + "-T", + "{node|short}\n", + ] + ).splitlines(): skiprev.add(rev) skip[key] = list(skiprev) @@ -58,7 +31,7 @@ skiprev.add(rev) skip[key] = list(skiprev) - with open(os.path.join('repos', 'skip.json'), 'w') as f: + with open(os.path.join("repos", "skip.json"), "w") as f: json.dump(skip, f) config = read_configuration("config.yaml") @@ -61,6 +34,5 @@ json.dump(skip, f) config = read_configuration("config.yaml") -clone_repositories(config, "repos") skip = generate_skip_file(config) diff --git a/prepare_repos.py b/create_stripped_cached_repos.py similarity index 31% copy from prepare_repos.py copy to create_stripped_cached_repos.py index 8bec24508fbe3bd049617310a36e3097f53c32fb_cHJlcGFyZV9yZXBvcy5weQ==..096662f9d12a831edd0ee8ad7749e960be694322_Y3JlYXRlX3N0cmlwcGVkX2NhY2hlZF9yZXBvcy5weQ== 100644 --- a/prepare_repos.py +++ b/create_stripped_cached_repos.py @@ -1,4 +1,3 @@ """ Ensure that all repository are cloned and at their tip """ -import base64 import errno @@ -4,7 +3,5 @@ import errno -import hglib -import json import os import subprocess from os.path import isdir, join import yaml @@ -7,6 +4,7 @@ import os import subprocess from os.path import isdir, join import yaml +import urllib # changesets stripped in partial clones @@ -11,6 +9,11 @@ # changesets stripped in partial clones -PARTIAL_REVSET = ("same", 'last(all(), 10)', 'last(all(), 100)', 'last(all(), 1000)') +PARTIAL_REVSET = ( + "same", + "last(all(), 10)", + "last(all(), 100)", + "last(all(), 1000)", +) def read_configuration(config_path): @@ -19,9 +22,9 @@ def clone_repositories(config, repos_dir): - cachedir = join(repos_dir, '.cache') + cachedir = join(repos_dir, ".cache") try: os.makedirs(cachedir) except OSError as exc: if exc.errno != errno.EEXIST: raise @@ -23,7 +26,7 @@ try: os.makedirs(cachedir) except OSError as exc: if exc.errno != errno.EEXIST: raise - for repo_name, repo in config['repos'].items(): + for repo_name, repo in config["repos"].items(): clonedir = join(repos_dir, repo_name) @@ -29,5 +32,2 @@ clonedir = join(repos_dir, repo_name) - if not isdir(clonedir): - print("Cloning %s (%s) into %s" % (repo_name, repo['url'], clonedir)) - hglib.clone(repo['url'], clonedir) for revset in PARTIAL_REVSET: @@ -33,4 +33,4 @@ for revset in PARTIAL_REVSET: - partialdir = join(cachedir, 'partial-{}-{}'.format( - repo_name, base64.b64encode(revset))) + repo_suffix = urllib.quote_plus(revset) + partialdir = join(cachedir, "partial-{}-{}".format(repo_name, repo_suffix)) if not isdir(partialdir): @@ -36,7 +36,14 @@ if not isdir(partialdir): - print("Cloning %s (%s%s) into %s (for exchange benchmarks)" % ( - repo_name, clonedir, - " stripped of %s" % (revset,) if revset != "same" else "", partialdir)) - subprocess.check_call(['hg', 'clone', '--noupdate', clonedir, - partialdir]) + print( + "Cloning %s (%s%s) into %s (for exchange benchmarks)" + % ( + repo_name, + clonedir, + " stripped of %s" % (revset,) if revset != "same" else "", + partialdir, + ) + ) + subprocess.check_call( + ["hg", "clone", "--noupdate", clonedir, partialdir] + ) if revset != "same": @@ -42,24 +49,21 @@ if revset != "same": - subprocess.check_call(['hg', '--cwd', partialdir, - '--config', 'extensions.strip=', 'strip', - '--no-backup', '-r', revset]) - subprocess.check_call(['hg', '--cwd', partialdir, 'debugupdatecache']) - - -def generate_skip_file(config): - skip = {} - for key, revsets in config['skip'].items(): - skiprev = set() - for revset in revsets: - for rev in subprocess.check_output([ - 'hg', 'log', '--cwd', 'mercurial', - '-r', revset, '-T', '{node|short}\n', - ]).splitlines(): - skiprev.add(rev) - skip[key] = list(skiprev) - with open(os.path.join('repos', 'skip.json'), 'w') as f: - json.dump(skip, f) + subprocess.check_call( + [ + "hg", + "--cwd", + partialdir, + "--config", + "extensions.strip=", + "strip", + "--no-backup", + "-r", + revset, + ] + ) + subprocess.check_call( + ["hg", "--cwd", partialdir, "debugupdatecache"] + ) config = read_configuration("config.yaml") clone_repositories(config, "repos") @@ -62,5 +66,4 @@ config = read_configuration("config.yaml") clone_repositories(config, "repos") -skip = generate_skip_file(config) diff --git a/launch.sh b/launch.sh index 8bec24508fbe3bd049617310a36e3097f53c32fb_bGF1bmNoLnNo..096662f9d12a831edd0ee8ad7749e960be694322_bGF1bmNoLnNo 100755 --- a/launch.sh +++ b/launch.sh @@ -8,7 +8,9 @@ python -m perf system show | grep "OK!" # Ensure all repository are here -python prepare_repos.py +python clone_repos.py +python create_stripped_cached_repos.py +python create_skip_file.py # Launch asv time taskset -c 2,3 asv run --show-stderr --skip-existing-successful "${1-1.9:: and tagged()}"