# HG changeset patch # User Pierre-Yves David <pierre-yves.david@octobus.net> # Date 1656935016 -7200 # Mon Jul 04 13:43:36 2022 +0200 # Node ID a09e8a9a6d668a981852a0380f8db9522275d0f3 # Parent f6d7ddad1feac5ecbd4da8ed58eff1d766051cbb poulpe: add the first version of scheduler for poulpe diff --git a/poulpe-scheduler-bin/auto-cases b/poulpe-scheduler-bin/auto-cases new file mode 100755 --- /dev/null +++ b/poulpe-scheduler-bin/auto-cases @@ -0,0 +1,96 @@ +#!/usr/bin/env python3 + +from pathlib import Path +import random +import subprocess +import sys + +class Case: + + def __init__( + self, + hg_hash, + hg_variant, + data_env, + benchmark, + ): + self.hg_hash = hg_hash + self.hg_variant = hg_variant + self.data_env = data_env + self.benchmark = benchmark + + def to_scheduler_line(self): + return ( + f"RUN" + f" {self.hg_hash}" + f" {self.hg_variant}" + f" {self.data_env}" + f" {self.benchmark}" + ) + + +def list_mercurial_variants(base_dir): + """return a list of possible mercurial variants""" + return ["default", "pure"] # lets keep it simple for now + +def list_mercurial_hashes(base_dir): + """return a list of possible mercurial hashes""" + repo_dir = base_dir / "repos" / "mercurial" + cmd = [ + "hg", + "log", + "-T", + "{node}\n", + "-R", + str(repo_dir), + ] + p = subprocess.run(cmd, stdout=subprocess.PIPE, encoding="utf8") + return p.stdout.splitlines() + +def list_data_envs(base_dir): + """return a list of possible data_envs""" + data_dir = base_dir / "data-envs" + cmd = [ + "find", + "-L", + ".", + "-maxdepth", + "3", + "-name", + "data-env.poulpe", + ] + p = subprocess.run(cmd, stdout=subprocess.PIPE, encoding="utf8", cwd=str(data_dir)) + raws = [Path(l) for l in p.stdout.splitlines()] + return [Path(*p.parts[:-1]) for p in raws] + +def list_benchmarks(base_dir): + """return a list of possible benchmarks""" + bench_dir = base_dir / "benchmarks" + prefix = len(bench_dir.parts) + return [Path(*p.parts[prefix:]) for p in bench_dir.rglob("*.pbd")] + + +def main(base_dir=".", size="100"): + size = int(size) + base_dir = Path(base_dir).resolve() + for case in _main(base_dir, size): + print(case.to_scheduler_line()) + return 0 + + +def _main(base_dir, size=100): + variants = list_mercurial_variants(base_dir) + hashes = list_mercurial_hashes(base_dir) + datas = list_data_envs(base_dir) + benchmarks = list_benchmarks(base_dir) + for i in range(size): + yield Case( + random.choice(hashes), + random.choice(variants), + random.choice(datas), + random.choice(benchmarks), + ) + + +if __name__ == "__main__": + sys.exit(main(*sys.argv[1:])) diff --git a/poulpe-scheduler-bin/loop b/poulpe-scheduler-bin/loop new file mode 100755 --- /dev/null +++ b/poulpe-scheduler-bin/loop @@ -0,0 +1,165 @@ +#!/usr/bin/env python3 + +import hashlib +import os +from pathlib import Path +import shutil +import subprocess +import sys + +ACTION_FILE = "current.poulpe-schedule" + +RESULT_DIR = "results" +BIN_ENV_DIR = "bin-envs" +DATA_ENV_DIR = "data-envs" +BENCHMARK_DIR = "benchmarks" + +def get_first(base_dir): + return next(iter(_get_all_actions(base_dir)), None) + +def drop_first(base_dir, action): + # only overwrite if no race are detected + # (A small race windows still exists within this function) + actions = _get_all_actions(base_dir) + if actions[0] == action: + actions.pop(0) + _write_all_actions(base_dir, actions) + +def _get_all_actions(base_dir): + actions_file = base_dir / ACTION_FILE + actions = actions_file.read_text() + return [a.strip() for a in actions.splitlines()] + +def _write_all_actions(base_dir, actions): + actions_file = base_dir / ACTION_FILE + actions_file.write_text('\n'.join(actions) + '\n') + + +def process_action(base_dir, action): + parts = action.split(' ', 1) + cmd = parts[0] + if len(parts) > 1: + args = parts[1] + else: + args = '' + if cmd == "#": + pass + elif cmd == "RUN": + action_run(base_dir, *args.split()) + elif cmd == "AUTOFILL": + action_autofill(base_dir, *[int(x) for x in args.split()]) + else: + print("unknown action %s" % action, file=sys.stderr) + + +def action_run(base_dir, mercurial_nodeid, mercurial_variant, data_env, benchmark): + bin_env = ensure_bin_env(base_dir, mercurial_nodeid, mercurial_variant) + if bin_env is None: + print("could not get running bin-env", file=sys.stderr) + return + + result_desc = f"{bin_env} {mercurial_variant} {data_env} {benchmark}" + result_id = hashlib.sha256(result_desc.encode("utf8")).hexdigest() + result = base_dir / RESULT_DIR / f"{result_id}.pbr" + + data_env = base_dir / DATA_ENV_DIR / data_env + benchmark = base_dir / BENCHMARK_DIR / benchmark + + run_script = base_dir / "bin/run-util" + cmd = [ + str(run_script), + str(bin_env), + str(data_env), + str(benchmark), + str(result), + ] + subprocess.call(cmd) + + +def is_ready(bin_env): + """wtf, need better""" + desc = (bin_env / "bin-env.poulpe") + if not desc.exists(): + return False + return "ready = true" in desc.read_text() + + +MAX_BIN_ENV = 50 + +def ensure_bin_env(base_dir, nodeid, variant): + bin_env_dir = base_dir / BIN_ENV_DIR + target = bin_env_dir / f"mercurial-{nodeid}-{variant}" + + all_existing = [p for p in bin_env_dir.iterdir()] + all_existing.sort(key=lambda x: x.stat().st_mtime) + + assert MAX_BIN_ENV > 1 + if len(all_existing) > MAX_BIN_ENV: + for delete_me in all_existing: + if delete_me != target: + break + print(f"Bin-envs {MAX_BIN_ENV} limit, deleting: {delete_me}") + shutil.rmtree(delete_me) + + # that check is too simple, we need to make sure this is a valid bin-env too + if target.exists() and not is_ready(target): + shutil.rmtree(target) + if not target.exists(): + setup_cmd = base_dir / "bin" / "bin-env-util" + setup_script = base_dir / "bin" / "mercurial.poulpe-setup.sh" + os.makedirs(target, exist_ok=True) + + env = os.environ.copy() + env['MERCURIAL_VERSION'] = nodeid + env['MERCURIAL_FLAVOR'] = variant + # XXX check the result of this at some point… + cmd = [ + setup_cmd, + "setup-one", + target, + setup_script, + ] + subprocess.call(cmd, env=env) + + if is_ready(target): + return target + else: + shutil.rmtree(target) + return None + + +def action_autofill(base_dir, count=None): + fill_script = base_dir / "bin/auto-cases" + cmd = [str(fill_script)] + if count: + cmd.append(str(count)) + p = subprocess.run(cmd, stdout=subprocess.PIPE, encoding="utf8") + new_actions = p.stdout.splitlines() + loop = ["AUTOFILL"] + if count: + loop.append(str(count)) + new_actions.append(' '.join(loop)) + # tiny race window here + actions = _get_all_actions(base_dir) + actions.extend(new_actions) + _write_all_actions(base_dir, actions) + + +def loop(base_dir="."): + base_dir = Path(base_dir).resolve() + _loop(base_dir) + return 0 + + +def _loop(base_dir): + + action = get_first(base_dir) + while action is not None: + if action: + process_action(base_dir, action) + drop_first(base_dir, action) + action = get_first(base_dir) + + +if __name__ == "__main__": + sys.exit(loop(*sys.argv[1:]))