Skip to content
Snippets Groups Projects
Commit a09e8a9a authored by Pierre-Yves David's avatar Pierre-Yves David :octopus:
Browse files

poulpe: add the first version of scheduler for poulpe

parent f6d7ddad
No related branches found
No related tags found
No related merge requests found
#!/usr/bin/env python3
from pathlib import Path
import random
import subprocess
import sys
class Case:
def __init__(
self,
hg_hash,
hg_variant,
data_env,
benchmark,
):
self.hg_hash = hg_hash
self.hg_variant = hg_variant
self.data_env = data_env
self.benchmark = benchmark
def to_scheduler_line(self):
return (
f"RUN"
f" {self.hg_hash}"
f" {self.hg_variant}"
f" {self.data_env}"
f" {self.benchmark}"
)
def list_mercurial_variants(base_dir):
"""return a list of possible mercurial variants"""
return ["default", "pure"] # lets keep it simple for now
def list_mercurial_hashes(base_dir):
"""return a list of possible mercurial hashes"""
repo_dir = base_dir / "repos" / "mercurial"
cmd = [
"hg",
"log",
"-T",
"{node}\n",
"-R",
str(repo_dir),
]
p = subprocess.run(cmd, stdout=subprocess.PIPE, encoding="utf8")
return p.stdout.splitlines()
def list_data_envs(base_dir):
"""return a list of possible data_envs"""
data_dir = base_dir / "data-envs"
cmd = [
"find",
"-L",
".",
"-maxdepth",
"3",
"-name",
"data-env.poulpe",
]
p = subprocess.run(cmd, stdout=subprocess.PIPE, encoding="utf8", cwd=str(data_dir))
raws = [Path(l) for l in p.stdout.splitlines()]
return [Path(*p.parts[:-1]) for p in raws]
def list_benchmarks(base_dir):
"""return a list of possible benchmarks"""
bench_dir = base_dir / "benchmarks"
prefix = len(bench_dir.parts)
return [Path(*p.parts[prefix:]) for p in bench_dir.rglob("*.pbd")]
def main(base_dir=".", size="100"):
size = int(size)
base_dir = Path(base_dir).resolve()
for case in _main(base_dir, size):
print(case.to_scheduler_line())
return 0
def _main(base_dir, size=100):
variants = list_mercurial_variants(base_dir)
hashes = list_mercurial_hashes(base_dir)
datas = list_data_envs(base_dir)
benchmarks = list_benchmarks(base_dir)
for i in range(size):
yield Case(
random.choice(hashes),
random.choice(variants),
random.choice(datas),
random.choice(benchmarks),
)
if __name__ == "__main__":
sys.exit(main(*sys.argv[1:]))
#!/usr/bin/env python3
import hashlib
import os
from pathlib import Path
import shutil
import subprocess
import sys
ACTION_FILE = "current.poulpe-schedule"
RESULT_DIR = "results"
BIN_ENV_DIR = "bin-envs"
DATA_ENV_DIR = "data-envs"
BENCHMARK_DIR = "benchmarks"
def get_first(base_dir):
return next(iter(_get_all_actions(base_dir)), None)
def drop_first(base_dir, action):
# only overwrite if no race are detected
# (A small race windows still exists within this function)
actions = _get_all_actions(base_dir)
if actions[0] == action:
actions.pop(0)
_write_all_actions(base_dir, actions)
def _get_all_actions(base_dir):
actions_file = base_dir / ACTION_FILE
actions = actions_file.read_text()
return [a.strip() for a in actions.splitlines()]
def _write_all_actions(base_dir, actions):
actions_file = base_dir / ACTION_FILE
actions_file.write_text('\n'.join(actions) + '\n')
def process_action(base_dir, action):
parts = action.split(' ', 1)
cmd = parts[0]
if len(parts) > 1:
args = parts[1]
else:
args = ''
if cmd == "#":
pass
elif cmd == "RUN":
action_run(base_dir, *args.split())
elif cmd == "AUTOFILL":
action_autofill(base_dir, *[int(x) for x in args.split()])
else:
print("unknown action %s" % action, file=sys.stderr)
def action_run(base_dir, mercurial_nodeid, mercurial_variant, data_env, benchmark):
bin_env = ensure_bin_env(base_dir, mercurial_nodeid, mercurial_variant)
if bin_env is None:
print("could not get running bin-env", file=sys.stderr)
return
result_desc = f"{bin_env} {mercurial_variant} {data_env} {benchmark}"
result_id = hashlib.sha256(result_desc.encode("utf8")).hexdigest()
result = base_dir / RESULT_DIR / f"{result_id}.pbr"
data_env = base_dir / DATA_ENV_DIR / data_env
benchmark = base_dir / BENCHMARK_DIR / benchmark
run_script = base_dir / "bin/run-util"
cmd = [
str(run_script),
str(bin_env),
str(data_env),
str(benchmark),
str(result),
]
subprocess.call(cmd)
def is_ready(bin_env):
"""wtf, need better"""
desc = (bin_env / "bin-env.poulpe")
if not desc.exists():
return False
return "ready = true" in desc.read_text()
MAX_BIN_ENV = 50
def ensure_bin_env(base_dir, nodeid, variant):
bin_env_dir = base_dir / BIN_ENV_DIR
target = bin_env_dir / f"mercurial-{nodeid}-{variant}"
all_existing = [p for p in bin_env_dir.iterdir()]
all_existing.sort(key=lambda x: x.stat().st_mtime)
assert MAX_BIN_ENV > 1
if len(all_existing) > MAX_BIN_ENV:
for delete_me in all_existing:
if delete_me != target:
break
print(f"Bin-envs {MAX_BIN_ENV} limit, deleting: {delete_me}")
shutil.rmtree(delete_me)
# that check is too simple, we need to make sure this is a valid bin-env too
if target.exists() and not is_ready(target):
shutil.rmtree(target)
if not target.exists():
setup_cmd = base_dir / "bin" / "bin-env-util"
setup_script = base_dir / "bin" / "mercurial.poulpe-setup.sh"
os.makedirs(target, exist_ok=True)
env = os.environ.copy()
env['MERCURIAL_VERSION'] = nodeid
env['MERCURIAL_FLAVOR'] = variant
# XXX check the result of this at some point…
cmd = [
setup_cmd,
"setup-one",
target,
setup_script,
]
subprocess.call(cmd, env=env)
if is_ready(target):
return target
else:
shutil.rmtree(target)
return None
def action_autofill(base_dir, count=None):
fill_script = base_dir / "bin/auto-cases"
cmd = [str(fill_script)]
if count:
cmd.append(str(count))
p = subprocess.run(cmd, stdout=subprocess.PIPE, encoding="utf8")
new_actions = p.stdout.splitlines()
loop = ["AUTOFILL"]
if count:
loop.append(str(count))
new_actions.append(' '.join(loop))
# tiny race window here
actions = _get_all_actions(base_dir)
actions.extend(new_actions)
_write_all_actions(base_dir, actions)
def loop(base_dir="."):
base_dir = Path(base_dir).resolve()
_loop(base_dir)
return 0
def _loop(base_dir):
action = get_first(base_dir)
while action is not None:
if action:
process_action(base_dir, action)
drop_first(base_dir, action)
action = get_first(base_dir)
if __name__ == "__main__":
sys.exit(loop(*sys.argv[1:]))
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment