# HG changeset patch # User Philippe Pepiot <phil@philpep.org> # Date 1517402184 -3600 # Wed Jan 31 13:36:24 2018 +0100 # Node ID d2d2c44a228ff4c17446b47832b6dd7e86c4be45 # Parent b1407f61bf1b6d9f165f49899de238a692c3b525 Fix pull/push benchmarks In ASV setup is run for each repeat but benchmark function is run `number` of time within the same repetition (calling it a "sample"). So for push / pull benchmarks, number should be set to 1 (call setup before each call of the benchmark method), also setup isn't called during "warmup" time, so disable it. Set repeat to 20 (instead of the default 10) this should be enough to have stable results while keeping benchmark time reasonable. Also move benchmark params (number, timer) to class variable. diff --git a/benchmarks/basic_commands.py b/benchmarks/basic_commands.py --- a/benchmarks/basic_commands.py +++ b/benchmarks/basic_commands.py @@ -142,10 +142,10 @@ class BaseExchangeTimeSuite(BaseTimeTestSuite): param_names = BaseTimeTestSuite.param_names + ['strip', 'revset'] params = BaseTimeTestSuite.params + [['same', 'last(all(), 10)', 'last(all(), 100)', 'last(all(), 1000)']] + [[None, 'default']] + timer = timeit.default_timer + sample_time = 10 def setup(self, repo_name, strip, revset): - self.timer = timeit.default_timer - self.sample_time = 10 super(BaseExchangeTimeSuite, self).setup(repo_name) self.clone_path = os.path.join(REPOS_DIR, '.cache', 'partial-{}-{}'.format( repo_name, base64.b64encode(strip))) @@ -181,6 +181,10 @@ class PushPullTimeSuite(BaseExchangeTimeSuite): timeout = 300 + # Force setup to be called between two push or pull + number = 1 + repeat = 20 + warmup_time = 0 def setup(self, repo_name, strip, revset): super(PushPullTimeSuite, self).setup(repo_name, strip, revset) @@ -209,7 +213,7 @@ self.repo_path] if self.rev: cmd.extend(['-r', self.rev]) - self._single_execute(cmd, expected_return_code=1 if strip == "same" else 0) + self._single_execute(cmd) class CloneTimeSuite(BaseExchangeTimeSuite): diff --git a/benchmarks/utils.py b/benchmarks/utils.py --- a/benchmarks/utils.py +++ b/benchmarks/utils.py @@ -227,6 +227,9 @@ except subprocess.CalledProcessError as exc: if exc.returncode != expected_return_code: raise + else: + if expected_return_code != 0: + raise RuntimeError('unexpected return code 0 for %s' % (cmd,)) class BaseNChangesetsTestSuite(BaseTimeTestSuite):