diff --git a/python-libs/poulpe/benchmarks.py b/python-libs/poulpe/benchmarks.py index cf4d47dfbed0ec90943dbcfe834afb8dfb818db8_cHl0aG9uLWxpYnMvcG91bHBlL2JlbmNobWFya3MucHk=..4470c318092c139313d54df2859a807c90b70975_cHl0aG9uLWxpYnMvcG91bHBlL2JlbmNobWFya3MucHk= 100644 --- a/python-libs/poulpe/benchmarks.py +++ b/python-libs/poulpe/benchmarks.py @@ -82,6 +82,10 @@ def all_dimensions(self): return {} + @property + def needed_vars(self): + return set() + class SimpleCommandBenchmark(Benchmark): @@ -108,6 +112,17 @@ all_dimensions = l_2.get('dimensions', {}) return all_dimensions.copy() + + @property + def needed_vars(self): + variables = set() + cwd = self.get_var('simple-command.cwd') + if cwd is not None and cwd.startswith('DATA-VARS:'): + key = cwd.split(':', 1)[1] + variables.add(key) + # XXX missing the "variable" part + return variables + def run_one(self, bin_env_path, data_env_path): data_env = data_mod.get_data_env(data_env_path) @@ -244,6 +259,21 @@ all_dimensions = l_2.get('dimensions', {}) return all_dimensions.copy() + @property + def needed_vars(self): + variables = set() + cwd = self.get_var('simple-command.cwd') + if cwd is not None and cwd.startswith('DATA-VARS:'): + key = cwd.split(':', 1)[1] + variables.add(key) + args = self.get_var('hg-perf-ext.args') + if args is not None: + for value in args: + if value.startswith('DATA-VARS:'): + key = value.split(':', 1)[1] + variables.add(key) + return variables + def run_one(self, bin_env_path, data_env_path): bin_env_path = os.path.abspath(bin_env_path) data_env_path = os.path.abspath(data_env_path) diff --git a/suites/hg/scheduling/auto-cases b/suites/hg/scheduling/auto-cases index cf4d47dfbed0ec90943dbcfe834afb8dfb818db8_c3VpdGVzL2hnL3NjaGVkdWxpbmcvYXV0by1jYXNlcw==..4470c318092c139313d54df2859a807c90b70975_c3VpdGVzL2hnL3NjaGVkdWxpbmcvYXV0by1jYXNlcw== 100755 --- a/suites/hg/scheduling/auto-cases +++ b/suites/hg/scheduling/auto-cases @@ -14,7 +14,11 @@ sys.path.insert(0, os.path.join(poulpe_root, 'python-libs')) import poulpe -from poulpe import benchmarks as bench_mod +from poulpe import ( + benchmarks as bench_mod, + dataenvs as data_env_lib, + errors, +) TARGET_PYTHON_VERSIONS = [ "3.7" @@ -53,6 +57,7 @@ f" {self.benchmark}" ) + def mercurial_repository_path(base_dir): return base_dir / "repos" / "mercurial" @@ -56,5 +61,6 @@ def mercurial_repository_path(base_dir): return base_dir / "repos" / "mercurial" + def list_mercurial_variants(base_dir, changeset): """return a list of possible mercurial variants""" @@ -59,6 +65,6 @@ def list_mercurial_variants(base_dir, changeset): """return a list of possible mercurial variants""" - variants = ["default", "pure"] # lets keep it simple for now + variants = ["default", "pure"] # let's keep it simple for now for k, v in FLAVOR_COMPATIBILITY.items(): revs = list_mercurial_hashes(base_dir, v) if changeset in revs: @@ -66,6 +72,5 @@ return variants - def list_mercurial_hashes(base_dir, interval="all()"): """return a list of possible mercurial hashes""" @@ -70,6 +75,6 @@ def list_mercurial_hashes(base_dir, interval="all()"): """return a list of possible mercurial hashes""" - repo_dir = mercurial_repository_path(base_dir) + repo_dir = mercurial_repository_path(base_dir) cmd = [ "hg", "log", @@ -83,5 +88,6 @@ p = subprocess.run(cmd, stdout=subprocess.PIPE, encoding="utf8") return p.stdout.splitlines() + def list_data_envs(base_dir): """return a list of possible data_envs""" @@ -86,6 +92,6 @@ def list_data_envs(base_dir): """return a list of possible data_envs""" - data_dir = base_dir / "data-envs" + data_dir = base_dir / "data-envs" cmd = [ "find", "-L", @@ -95,11 +101,16 @@ "-name", "data-env.poulpe", ] - p = subprocess.run(cmd, stdout=subprocess.PIPE, encoding="utf8", cwd=str(data_dir)) - raws = [Path(l) for l in p.stdout.splitlines()] + p = subprocess.run( + cmd, + stdout=subprocess.PIPE, + encoding="utf8", + cwd=str(data_dir) + ) + raws = [Path(line) for line in p.stdout.splitlines()] return [Path(*p.parts[:-1]) for p in raws] compat_cache = {} @@ -100,10 +111,10 @@ return [Path(*p.parts[:-1]) for p in raws] compat_cache = {} -def compatible_with(base_dir, data_env, changeset): +def data_compatible_with_bin(base_dir, data_env, changeset): """check if a data_env is compatible with the selected revision""" path = base_dir / "data-envs" / data_env / "data-env.poulpe" data_args = poulpe.get_data(path) @@ -117,6 +128,15 @@ return True +def benchmark_compatible_with(base_dir, data_env, changeset, benchmark): + for v in benchmark.needed_vars: + try: + data_env.get_benchmark_input_vars(v) + except errors.MissingDataEnvInputVars: + return False + return True + + def list_benchmarks(base_dir): """return a list of possible benchmarks""" bench_dir = base_dir / "benchmarks" @@ -147,9 +167,12 @@ def _main(base_dir, size=10): + bench_dir = base_dir / "benchmarks" + data_dir = base_dir / "data-envs" + python_version = random.choice(TARGET_PYTHON_VERSIONS) mercurial_intervals = PY_VERSION_COMPATIBILITY.get(python_version, "all()") hashes = list_mercurial_hashes(base_dir, mercurial_intervals) datas = list_data_envs(base_dir) @@ -150,7 +173,8 @@ python_version = random.choice(TARGET_PYTHON_VERSIONS) mercurial_intervals = PY_VERSION_COMPATIBILITY.get(python_version, "all()") hashes = list_mercurial_hashes(base_dir, mercurial_intervals) datas = list_data_envs(base_dir) + datas = [(data_env_lib.get_data_env(data_dir / d), d) for d in datas] benchmarks = list_benchmarks(base_dir) @@ -156,5 +180,9 @@ benchmarks = list_benchmarks(base_dir) + benchmarks = [ + (bench_mod.get_benchmark(bench_dir / b), b) + for b in benchmarks + ] for i in range(size): h = random.choice(hashes) variants = list_mercurial_variants(base_dir, h) for v in variants: @@ -157,12 +185,12 @@ for i in range(size): h = random.choice(hashes) variants = list_mercurial_variants(base_dir, h) for v in variants: - for d in datas: - if not compatible_with(base_dir, d, h): - continue - for b in benchmarks: - yield Case(h, v, d, b) + for d, d_path in datas: + if data_compatible_with_bin(base_dir, d_path, h): + for b, b_path in benchmarks: + if benchmark_compatible_with(base_dir, d, h, b): + yield Case(h, v, d_path, b_path) if __name__ == "__main__":