diff --git a/docs/benchmarks.rst b/docs/benchmarks.rst index a582cef992a703864562933e22ff81c5492933f7_ZG9jcy9iZW5jaG1hcmtzLnJzdA==..68de0f5ec747a9f19e7fc6a43a424b46f67b51b2_ZG9jcy9iZW5jaG1hcmtzLnJzdA== 100644 --- a/docs/benchmarks.rst +++ b/docs/benchmarks.rst @@ -129,3 +129,45 @@ ~~~~ Arguments to pass to the command (a list of string). + +variants +-------- + +Variants allow for slight variation of a benchmark. For example, they allow to change the +input data, of to turn some feature on and off. + +Variants are organised within independant "dimensions". + +Each dimension has a set of values, each value can modify the benchmark. Each +dimension must have a default value. + +defining a variant +~~~~~~~~~~~~~~~~~~ + +Variants are defined under keys following this patterns : + + hg-perf-ext.variants.dimensions.<dimension-name>.<variant-key> + +The `dimension-name` and `variant-key` are used as identifier in the benchmark +result and for selecting variant at run time. + +variants variables +~~~~~~~~~~~~~~~~~~ + +default +~~~~~~~ + +Exactly one variant must have it set to `true` per dimension. + +When set and no variant have been explicitly selected for this dimension, this +variant will be used. + +args +~~~~~~~~~~~~~~ + +A list of args to add to the command + +cwd +~~~ + +overwrite the global `hg-perf-ext.cwd` variable. Behave the same. diff --git a/python-libs/poulpe/runner.py b/python-libs/poulpe/runner.py index a582cef992a703864562933e22ff81c5492933f7_cHl0aG9uLWxpYnMvcG91bHBlL3J1bm5lci5weQ==..68de0f5ec747a9f19e7fc6a43a424b46f67b51b2_cHl0aG9uLWxpYnMvcG91bHBlL3J1bm5lci5weQ== 100644 --- a/python-libs/poulpe/runner.py +++ b/python-libs/poulpe/runner.py @@ -81,6 +81,19 @@ variants = {} self._selected_variants = variants + # find all variant and pick default one when needed. + for dimension, variants in self.all_dimensions.items(): + selected = self._selected_variants.get(dimension) + if selected is None: + for name, values in variants.items(): + if values.get("default", False): + selected = name + break + else: + msg = "missing default value for dimensions: %s" + assert False, msg % dimension + self._selected_variants[dimension] = selected + @property def name(self): return self._data['meta']['name'] @@ -110,17 +123,7 @@ assert self._raw_data['meta']['method'] == "simple-command" for dimension, variants in self.all_dimensions.items(): - # find the variants we should use. - selected = self._selected_variants.get(dimension) - if selected is None: - for name, values in variants.items(): - if values.get("default", False): - selected = name - break - else: - msg = "missing default value for dimensions: %s" - assert False, msg % dimension - self._selected_variants[dimension] = selected + selected = self._selected_variants[dimension] # gather the changes to the benchmark new_cwd = variants[selected].get('cwd') @@ -241,6 +244,30 @@ (This might live in a pluging in the future.) """ + def __init__(self, data, variants=None): + super().__init__(data, variants) + + assert self._raw_data['meta']['method'] == "mercurial-perf-extension" + + self._data['hg-perf-ext'].setdefault('args', []) + + for dimension, variants in self.all_dimensions.items(): + selected = self._selected_variants[dimension] + + # gather the changes to the benchmark + new_cwd = variants[selected].get('cwd') + if new_cwd is not None: + self._data['hg-perf-ext']['cwd'] = new_cwd + args = variants[selected].get('args') + if args: + self._data['hg-perf-ext']['args'].extend(args) + + @property + def all_dimensions(self): + l_1 = self._data.get('hg-perf-ext', {}) + l_2 = l_1.get('variants', {}) + all_dimensions = l_2.get('dimensions', {}) + return all_dimensions.copy() def run_one(self, bin_env_path, data_env_path): bin_env_path = os.path.abspath(bin_env_path) diff --git a/suites/hg/benchmarks/perf-bundle.pbd b/suites/hg/benchmarks/perf-bundle.pbd index a582cef992a703864562933e22ff81c5492933f7_c3VpdGVzL2hnL2JlbmNobWFya3MvcGVyZi1idW5kbGUucGJk..68de0f5ec747a9f19e7fc6a43a424b46f67b51b2_c3VpdGVzL2hnL2JlbmNobWFya3MvcGVyZi1idW5kbGUucGJk 100644 --- a/suites/hg/benchmarks/perf-bundle.pbd +++ b/suites/hg/benchmarks/perf-bundle.pbd @@ -6,4 +6,23 @@ [hg-perf-ext] command="perf::bundle" cwd="DATA-VARS:mercurial.main-repo-path" -args=["--rev", "last(all(), 40000)", "--type", "none-v2"] +args=["--type", "none-v2"] + +[hg-perf-ext.variants.dimensions.revs.last-40000] +default=true +args=["--rev", "last(all(), 40000)"] + +[hg-perf-ext.variants.dimensions.revs.last-10000] +args=["--rev", "last(all(), 10000)"] + +[hg-perf-ext.variants.dimensions.revs.last-1000] +args=["--rev", "last(all(), 1000)"] + +[hg-perf-ext.variants.dimensions.revs.last-100] +args=["--rev", "last(all(), 100)"] + +[hg-perf-ext.variants.dimensions.revs.last-10] +args=["--rev", "last(all(), 10)"] + +[hg-perf-ext.variants.dimensions.revs.last-1] +args=["--rev", "last(all(), 1)"]