# HG changeset patch
# User Pierre-Yves David <pierre-yves.david@octobus.net>
# Date 1656935016 -7200
#      Mon Jul 04 13:43:36 2022 +0200
# Node ID a09e8a9a6d668a981852a0380f8db9522275d0f3
# Parent  f6d7ddad1feac5ecbd4da8ed58eff1d766051cbb
poulpe: add the first version of scheduler for poulpe

diff --git a/poulpe-scheduler-bin/auto-cases b/poulpe-scheduler-bin/auto-cases
new file mode 100755
--- /dev/null
+++ b/poulpe-scheduler-bin/auto-cases
@@ -0,0 +1,96 @@
+#!/usr/bin/env python3
+
+from pathlib import Path
+import random
+import subprocess
+import sys
+
+class Case:
+
+    def __init__(
+        self,
+        hg_hash,
+        hg_variant,
+        data_env,
+        benchmark,
+    ):
+        self.hg_hash = hg_hash
+        self.hg_variant = hg_variant
+        self.data_env = data_env
+        self.benchmark = benchmark
+
+    def to_scheduler_line(self):
+        return (
+            f"RUN"
+            f" {self.hg_hash}"
+            f" {self.hg_variant}"
+            f" {self.data_env}"
+            f" {self.benchmark}"
+        )
+
+    
+def list_mercurial_variants(base_dir):
+    """return a list of possible mercurial variants"""
+    return ["default", "pure"] # lets keep it simple for now
+
+def list_mercurial_hashes(base_dir):
+    """return a list of possible mercurial hashes"""
+    repo_dir =  base_dir / "repos" / "mercurial"
+    cmd = [
+        "hg",
+        "log",
+        "-T",
+        "{node}\n",
+        "-R",
+        str(repo_dir),
+    ]
+    p = subprocess.run(cmd, stdout=subprocess.PIPE, encoding="utf8")
+    return p.stdout.splitlines()
+
+def list_data_envs(base_dir):
+    """return a list of possible data_envs"""
+    data_dir =  base_dir / "data-envs"
+    cmd = [
+        "find",
+        "-L",
+        ".",
+        "-maxdepth",
+        "3",
+        "-name",
+        "data-env.poulpe",
+    ]
+    p = subprocess.run(cmd, stdout=subprocess.PIPE, encoding="utf8", cwd=str(data_dir))
+    raws = [Path(l) for l in p.stdout.splitlines()]
+    return [Path(*p.parts[:-1]) for p in raws]
+
+def list_benchmarks(base_dir):
+    """return a list of possible benchmarks"""
+    bench_dir =  base_dir / "benchmarks"
+    prefix = len(bench_dir.parts)
+    return [Path(*p.parts[prefix:]) for p in bench_dir.rglob("*.pbd")]
+
+
+def main(base_dir=".", size="100"):
+    size = int(size)
+    base_dir = Path(base_dir).resolve()
+    for case in _main(base_dir, size):
+        print(case.to_scheduler_line())
+    return 0
+
+
+def _main(base_dir, size=100):
+    variants = list_mercurial_variants(base_dir)
+    hashes = list_mercurial_hashes(base_dir)
+    datas = list_data_envs(base_dir)
+    benchmarks = list_benchmarks(base_dir)
+    for i in range(size):
+        yield Case(
+            random.choice(hashes),
+            random.choice(variants),
+            random.choice(datas),
+            random.choice(benchmarks),
+        )
+
+
+if __name__ == "__main__":
+    sys.exit(main(*sys.argv[1:]))
diff --git a/poulpe-scheduler-bin/loop b/poulpe-scheduler-bin/loop
new file mode 100755
--- /dev/null
+++ b/poulpe-scheduler-bin/loop
@@ -0,0 +1,165 @@
+#!/usr/bin/env python3
+
+import hashlib
+import os
+from pathlib import Path
+import shutil
+import subprocess
+import sys
+
+ACTION_FILE = "current.poulpe-schedule"
+
+RESULT_DIR = "results"
+BIN_ENV_DIR = "bin-envs"
+DATA_ENV_DIR = "data-envs"
+BENCHMARK_DIR = "benchmarks"
+
+def get_first(base_dir):
+    return next(iter(_get_all_actions(base_dir)), None)
+
+def drop_first(base_dir, action):
+    # only overwrite if no race are detected
+    # (A small race windows still exists within this function)
+    actions = _get_all_actions(base_dir)
+    if actions[0] == action:
+        actions.pop(0)
+        _write_all_actions(base_dir, actions)
+
+def _get_all_actions(base_dir):
+    actions_file = base_dir / ACTION_FILE
+    actions = actions_file.read_text()
+    return [a.strip() for a in actions.splitlines()]
+
+def _write_all_actions(base_dir, actions):
+    actions_file = base_dir / ACTION_FILE
+    actions_file.write_text('\n'.join(actions) + '\n')
+
+
+def process_action(base_dir, action):
+    parts = action.split(' ', 1)
+    cmd = parts[0]
+    if len(parts) > 1:
+        args = parts[1]
+    else:
+        args = ''
+    if cmd == "#":
+        pass
+    elif cmd == "RUN":
+        action_run(base_dir, *args.split())
+    elif cmd == "AUTOFILL":
+        action_autofill(base_dir, *[int(x) for x in args.split()])
+    else:
+        print("unknown action %s" % action, file=sys.stderr)
+
+
+def action_run(base_dir, mercurial_nodeid, mercurial_variant, data_env, benchmark):
+    bin_env = ensure_bin_env(base_dir, mercurial_nodeid, mercurial_variant)
+    if bin_env is None:
+        print("could not get running bin-env", file=sys.stderr)
+        return
+
+    result_desc = f"{bin_env} {mercurial_variant} {data_env} {benchmark}"
+    result_id = hashlib.sha256(result_desc.encode("utf8")).hexdigest()
+    result = base_dir / RESULT_DIR / f"{result_id}.pbr"
+
+    data_env = base_dir / DATA_ENV_DIR / data_env
+    benchmark = base_dir / BENCHMARK_DIR / benchmark
+
+    run_script = base_dir / "bin/run-util"
+    cmd = [
+            str(run_script),
+            str(bin_env),
+            str(data_env),
+            str(benchmark),
+            str(result),
+    ]
+    subprocess.call(cmd)
+
+
+def is_ready(bin_env):
+    """wtf, need better"""
+    desc = (bin_env / "bin-env.poulpe")
+    if not desc.exists():
+        return False
+    return "ready = true" in desc.read_text()
+
+
+MAX_BIN_ENV = 50
+
+def ensure_bin_env(base_dir, nodeid, variant):
+    bin_env_dir = base_dir / BIN_ENV_DIR
+    target = bin_env_dir / f"mercurial-{nodeid}-{variant}"
+
+    all_existing = [p for p in bin_env_dir.iterdir()]
+    all_existing.sort(key=lambda x: x.stat().st_mtime)
+
+    assert MAX_BIN_ENV > 1
+    if len(all_existing) >  MAX_BIN_ENV:
+        for delete_me in all_existing:
+            if delete_me != target:
+                break
+        print(f"Bin-envs {MAX_BIN_ENV} limit, deleting: {delete_me}")
+        shutil.rmtree(delete_me)
+
+    # that check is too simple, we need to make sure this is a valid bin-env too
+    if target.exists() and not is_ready(target):
+        shutil.rmtree(target)
+    if not target.exists():
+        setup_cmd = base_dir / "bin" / "bin-env-util"
+        setup_script = base_dir / "bin" / "mercurial.poulpe-setup.sh"
+        os.makedirs(target, exist_ok=True)
+
+        env = os.environ.copy()
+        env['MERCURIAL_VERSION'] = nodeid
+        env['MERCURIAL_FLAVOR'] = variant
+        # XXX check the result of this at some point…
+        cmd = [
+            setup_cmd,
+            "setup-one",
+            target,
+            setup_script,
+        ]
+        subprocess.call(cmd, env=env)
+
+    if is_ready(target):
+        return target
+    else:
+        shutil.rmtree(target)
+        return None
+
+
+def action_autofill(base_dir, count=None):
+    fill_script = base_dir / "bin/auto-cases"
+    cmd = [str(fill_script)]
+    if count:
+        cmd.append(str(count))
+    p = subprocess.run(cmd, stdout=subprocess.PIPE, encoding="utf8")
+    new_actions = p.stdout.splitlines()
+    loop = ["AUTOFILL"]
+    if count:
+        loop.append(str(count))
+    new_actions.append(' '.join(loop))
+    # tiny race window here
+    actions = _get_all_actions(base_dir)
+    actions.extend(new_actions)
+    _write_all_actions(base_dir, actions)
+
+
+def loop(base_dir="."):
+    base_dir = Path(base_dir).resolve()
+    _loop(base_dir)
+    return 0
+
+
+def _loop(base_dir):
+
+    action = get_first(base_dir)
+    while action is not None:
+        if action:
+            process_action(base_dir, action)
+            drop_first(base_dir, action)
+        action = get_first(base_dir)
+
+
+if __name__ == "__main__":
+    sys.exit(loop(*sys.argv[1:]))