Skip to content
Snippets Groups Projects
Commit 71963760 authored by Philippe Pepiot's avatar Philippe Pepiot
Browse files

Enable clone benchmarks for all reference repositories

This is a bit tricky because there are several constraints:

* we must cleanup after clone (in setup/teardown) to limit disk space usage
* we must limit benchmark duration (avoid to run too much time on big repositories)
* we must have more stable results for small repositories

The whole algorithm is in asv.benchmark.benchmark_timing().

To ensure setup/teardown is called before/after each clone we *must* have
number = 1 (so timer.timeit() will be called with 1 as argument).
Then it's important to have a proper sample_time set, because it will control
how long the benchmark will run. timeit() is called `repeat` (default 10) times
unless the whole benchmark (= one benchmark function with one set of parameter)
takes more than `repeat` * 1.3 * `sample_time` where `sample_time` has a default value of 0.1

This can produce very short times, for instance for a clone duration of 20s
this will only run one clone even if repeat is set 10...

So use sample_time = (max_time_we_want_the_benchmark_to_run  / (repeat * 1.3))
to control maximum whole benchmark duration and to ensure timeit() will be
called close to `repeat` times.
parent 38865651
No related branches found
No related tags found
No related merge requests found
...@@ -404,5 +404,4 @@ ...@@ -404,5 +404,4 @@
# class CloneTimeSuite(BaseExchangeMixin, BaseTestSuite): # class CloneTimeSuite(BaseExchangeMixin, BaseTestSuite):
# # skip other repos since they are too big
# param_names = BaseTestSuite.param_names + ['repo_type', 'revset'] # param_names = BaseTestSuite.param_names + ['repo_type', 'revset']
...@@ -408,4 +407,4 @@ ...@@ -408,4 +407,4 @@
# param_names = BaseTestSuite.param_names + ['repo_type', 'revset'] # param_names = BaseTestSuite.param_names + ['repo_type', 'revset']
# params = [['mercurial-2017'], ['local', 'ssh', 'http'], [None, 'tip']] # params = BaseTestSuite.params + [['local', 'ssh', 'http'], [None, 'tip']]
# timer = timeit.default_timer # timer = timeit.default_timer
...@@ -410,7 +409,8 @@ ...@@ -410,7 +409,8 @@
# timer = timeit.default_timer # timer = timeit.default_timer
# sample_time = 10 # number = 1
# timeout = 300 # warmup_time = 0
# timeout = 2700
# def setup(self, *args, **kwargs): # def setup(self, *args, **kwargs):
# args = list(args) # args = list(args)
...@@ -418,5 +418,6 @@ ...@@ -418,5 +418,6 @@
# revset = args.pop(-1) # revset = args.pop(-1)
# repo_type = args.pop(-1) # repo_type = args.pop(-1)
# super(CloneTimeSuite, self).setup(*args, **kwargs) # super(CloneTimeSuite, self).setup(*args, **kwargs)
# self._cleanup_paths = []
# self._setup_repo_type(repo_type) # self._setup_repo_type(repo_type)
# self._setup_revset(revset) # self._setup_revset(revset)
...@@ -421,9 +422,3 @@ ...@@ -421,9 +422,3 @@
# self._setup_repo_type(repo_type) # self._setup_repo_type(repo_type)
# self._setup_revset(revset) # self._setup_revset(revset)
# tmpdir = os.path.join(REPOS_DIR, '.tmp')
# try:
# os.makedirs(tmpdir)
# except OSError as exc:
# if exc.errno != errno.EEXIST:
# raise
# self.clone_path = os.path.join(REPOS_DIR, repo_name) # self.clone_path = os.path.join(REPOS_DIR, repo_name)
...@@ -429,7 +424,16 @@ ...@@ -429,7 +424,16 @@
# self.clone_path = os.path.join(REPOS_DIR, repo_name) # self.clone_path = os.path.join(REPOS_DIR, repo_name)
# self.tmp_clone_path = os.path.join(tmpdir, 'clone-{}'.format( # if repo_name in ('mercurial-2017', 'pypy-2017'):
# os.path.basename(self.clone_path))) # # small repositories
# shutil.rmtree(self.tmp_clone_path, ignore_errors=True) # # run 20 times or less than 5 minutes
# self.repeat = 20
# self.sample_time = (5 * 60) / (self.repeat * 1.3)
# elif repo_name in ('mozilla-central-2017', 'netbeans-2017'):
# # big repositories
# # run 3 times or less than 45 minutes
# self.repeat = 3
# self.sample_time = (45 * 60) / (self.repeat * 1.3)
# else:
# raise NotImplementedError('unconfigured timeout for repository %s' % repo_name)
# def teardown(self, *args, **kwargs): # def teardown(self, *args, **kwargs):
# self._teardown_repo_type() # self._teardown_repo_type()
...@@ -433,7 +437,9 @@ ...@@ -433,7 +437,9 @@
# def teardown(self, *args, **kwargs): # def teardown(self, *args, **kwargs):
# self._teardown_repo_type() # self._teardown_repo_type()
# shutil.rmtree(self.tmp_clone_path, ignore_errors=True) # for path in self._cleanup_paths:
# shutil.rmtree(path, ignore_errors=True)
# self._cleanup_paths = []
# def time_clone(self, *args, **kwargs): # def time_clone(self, *args, **kwargs):
# cmd = ['clone', '--pull'] # cmd = ['clone', '--pull']
...@@ -437,4 +443,6 @@ ...@@ -437,4 +443,6 @@
# def time_clone(self, *args, **kwargs): # def time_clone(self, *args, **kwargs):
# cmd = ['clone', '--pull'] # cmd = ['clone', '--pull']
# tmp_clone_path = os.path.abspath(tempfile.mkdtemp(dir='.'))
# self._cleanup_paths.append(tmp_clone_path)
# cmd.extend(self._remote_path_cmd(self.clone_path)) # cmd.extend(self._remote_path_cmd(self.clone_path))
...@@ -440,5 +448,5 @@ ...@@ -440,5 +448,5 @@
# cmd.extend(self._remote_path_cmd(self.clone_path)) # cmd.extend(self._remote_path_cmd(self.clone_path))
# cmd.append(self.tmp_clone_path) # cmd.append(tmp_clone_path)
# if self.rev: # if self.rev:
# cmd.extend(['-r', self.rev]) # cmd.extend(['-r', self.rev])
# self.hg(*cmd) # self.hg(*cmd)
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment