diff --git a/bin/run-util b/bin/run-util index 757d43cfa220ce7f5cf12bd050a31107789c689e_YmluL3J1bi11dGls..6ed5c99bcfe74fb4c4cc3767fe7415ef4889b8f8_YmluL3J1bi11dGls 100755 --- a/bin/run-util +++ b/bin/run-util @@ -2,5 +2,4 @@ # # Small tool to manipulaote import argparse -import json import os @@ -6,3 +5,2 @@ import os -import subprocess import sys @@ -8,7 +6,5 @@ import sys -import tempfile -import time poulpe_root = os.path.dirname(os.path.dirname(sys.argv[0])) sys.path.insert(0, os.path.join(poulpe_root, 'python-libs')) @@ -11,196 +7,8 @@ poulpe_root = os.path.dirname(os.path.dirname(sys.argv[0])) sys.path.insert(0, os.path.join(poulpe_root, 'python-libs')) -import poulpe - -class Benchmark: - - def __init__(self, data, variants=None): - self._raw_data = data - - assert self._raw_data['meta']['format'] == "0" - assert self._raw_data['meta']['method'] == "simple-command" - - self._data = self._raw_data.copy() - if variants is None: - variants = {} - self._selected_variants = variants - - dimensions_key = 'simple-command.variants.dimensions' - l_1 = self._data.get('simple-command', {}) - l_2 = l_1.get('variants', {}) - all_dimensions = l_2.get('dimensions', {}) - - for dimension, variants in all_dimensions.items(): - # find the variants we should use. - selected = self._selected_variants.get(dimension) - if selected is None: - for name, values in variants.items(): - if values.get("default", False): - selected = name - break - else: - msg = "missing default value for dimensions: %s" - assert False, msg % dimension - self._selected_variants[dimension] = selected - - # gather the changes to the benchmark - new_cwd = variants[selected].get('cwd') - if new_cwd is not None: - self._data['simple-command']['cwd'] = new_cwd - extend_command= variants[selected].get('extend-command') - if extend_command is not None: - self._data['simple-command']['command'] += extend_command - - - @property - def name(self): - return self._data['meta']['name'] - - def get_var(self, key): - return poulpe.get_one_value(self._data, key) - - @property - def selected_variants(self): - return self._selected_variants.copy() - - -def get_benchmark(path_def): - """get a benchmark from path applying possible variant selection on the way - - Expected syntax is PATH[;dimension_name=variant_name][;d=v]... - """ - parts = path_def.split(';') - path = parts[0] - variants_definitions = parts[1:] - variants = {} - for d in variants_definitions: - k, v = d.split("=") - variants[k] = v - - data = poulpe.get_data(path) - if data is None: - return None - return Benchmark(data, variants) - - -class SimpleCommand(object): - - def __init__(self, benchmark_data): - self._benchmark_data = benchmark_data - - def get_var(key): - key = f'simple-command.{key}' - return benchmark_data.get_var(key) - - self.base_command = get_var('command') - assert self.base_command is not None - self.command = None - - # XXX having a list of acceptable status would be better (default to [0]) - self.accept_failure = bool(get_var('accept-failure')) - - self.cwd = get_var('cwd') - - def apply_data_env(self, data_env_data): - variables_data = self._benchmark_data.get_var('simple-command').get('variables') - - def get_var(key): - full_key = f'bench-input-vars.{key}' - return poulpe.get_one_value(data_env_data, full_key) - - variables = {} - if variables_data is not None: - for name, value in variables_data.items(): - if value.startswith('DATA-VARS:'): - key = value.split(':', 1)[1] - value = get_var(key) - assert value is not None - variables[name] = value - - self.command = self.base_command.format(**variables) - - if self.cwd.startswith('DATA-VARS:'): - key = self.cwd.split(':', 1)[1] - self.cwd = get_var(key) - - -def run_one(bin_env_path, data_env_path, benchmark, result): - result_data = {} - result_data['run'] = {} - result_data['run']["timestamp"] = time.time() - - # gather info about the binary environment - bin_env_desc = poulpe.bin_env_file(bin_env_path) - bin_env_data = poulpe.get_data(bin_env_desc) - result_data['bin-env-vars'] = bin_env_data['bin-env-vars'] - - # gather info about the data environment - data_env_desc = poulpe.data_env_file(data_env_path) - data_env_data = poulpe.get_data(data_env_desc) - result_data['data-env-vars'] = data_env_data['data-env-vars'] - - benchmark_data = get_benchmark(benchmark) - assert benchmark_data is not None, benchmark - result_data['benchmark'] = {} - result_data['benchmark']['name'] = benchmark_data.name - variants = benchmark_data.selected_variants - if variants: - result_data['benchmark']['variants'] = variants - - # building the benchmark scenarion, that will likely changes often and quickly - - cmd = SimpleCommand(benchmark_data) - cmd.apply_data_env(data_env_data) - - r = _time_command(bin_env_path, data_env_path, cmd) - - # we should store more - result_data['result'] = {} - result_data['result']['time'] = {} - result_data['result']['time']['median'] = r['results'][0]['median'] - result_data['result']['time']['mean'] = r['results'][0]['mean'] - result_data['result']['time']['standard-deviation'] = r['results'][0]['stddev'] - result_data['result']['time']['min'] = r['results'][0]['min'] - result_data['result']['time']['max'] = r['results'][0]['max'] - - result_data['run']["duration"] = time.time() - result_data['run']["timestamp"] - - poulpe.write_data(result, result_data) - return 0 - - -def _time_command(bin_env_path, data_env_path, cmd): - bin_env_path = os.path.abspath(bin_env_path) - data_env_path = os.path.abspath(data_env_path) - shell_path = poulpe.bin_env_script(bin_env_path) - - with tempfile.NamedTemporaryFile('w') as tmp_result: - time_cmd = [ - shell_path, - "hyperfine", - "--export-json", - tmp_result.name, - "--", - cmd.command, - ] - if cmd.accept_failure: - time_cmd.insert(-2, "--ignore-failure") - - cwd = data_env_path - if cmd.cwd is not None: - cwd = os.path.join(cwd, cmd.cwd) - - subprocess.check_call( - time_cmd, - cwd=cwd, - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, - ) - - with open(tmp_result.name) as f: - return json.load(f) +from poulpe import runner def _parsers(): @@ -231,7 +39,12 @@ def main(args): parser = _parsers() param = parser.parse_args(args) - ret = run_one(param.BIN_ENV, param.DATA_ENV, param.BENCHMARK, param.RESULT) + ret = runner.run_one( + param.BIN_ENV, + param.DATA_ENV, + param.BENCHMARK, + param.RESULT, + ) return ret diff --git a/python-libs/poulpe/__init__.py b/python-libs/poulpe/__init__.py index 757d43cfa220ce7f5cf12bd050a31107789c689e_cHl0aG9uLWxpYnMvcG91bHBlL19faW5pdF9fLnB5..6ed5c99bcfe74fb4c4cc3767fe7415ef4889b8f8_cHl0aG9uLWxpYnMvcG91bHBlL19faW5pdF9fLnB5 100644 --- a/python-libs/poulpe/__init__.py +++ b/python-libs/poulpe/__init__.py @@ -1,122 +1,1 @@ -import os -import toml -import tempfile -import sys - -BIN_ENV_FILE = 'bin-env.poulpe' -SHELL_FILE = 'bin-env.shell' -DATA_ENV_FILE = 'data-env.poulpe' - - -def err(*args, **kwargs): - """print something on stderr""" - print(*args, **kwargs, file=sys.stderr) - - -def bin_env_file(env_path): - """given the path of env, return the file of description file""" - return os.path.join(env_path, BIN_ENV_FILE) - - -def bin_env_script(bin_env_path): - """given the path of env, return the file of description file""" - return os.path.join(bin_env_path, SHELL_FILE) - - -def data_env_file(data_env_path): - """given the path of env, return the file of description file""" - return os.path.join(data_env_path, DATA_ENV_FILE) - - -def get_data(path): - """Read a descript file at <path>""" - try: - with open(path) as f: - return toml.load(f) - except FileNotFoundError: - return None - - -def write_data(path, data): - """write description data at <path>""" - directory = os.path.dirname(path) - basename = os.path.basename(path) - tmp_path = os.path.join(directory, f".{basename}.tmp") - with open(tmp_path, "w") as f: - toml.dump(data, f) - os.replace(tmp_path, path) - - -def show(data, indent=''): - """call to display data""" - if not isinstance(data, dict): - print(f"{indent}{data}") - return - for k, v in sorted(data.items()): - if isinstance(v, dict): - print(f"{indent}{k}:") - show(v, indent + ' ') - elif isinstance(v, list): - print(f"{indent}{k}:") - for i in v: - show(i, indent + '- ') - else: - print(f"{indent}{k} = {v}") - - -def get_one_value(data, key): - key_path = key.split('.') - sub = data - for k in key_path: - sub = sub.get(k) - if sub is None: - break - return sub - - -def set_one_value(data, key, value): - key_path = key.split('.') - sub = data - for k in key_path[:-1]: - val = sub.setdefault(k, {}) - sub = val - sub[key_path[-1]] = value - - -def del_one_value(data, key): - key_path = key.split('.') - sub = data - for k in key_path[:-1]: - sub = sub.get(k) - if sub is None: - break - if sub is not None: - sub.pop(key_path[-1], None) - - -def set_value(path, key, value): - data = get_data(path) - if data is None: - err(f'creating new file: "{path}"') - data = {} - - set_one_value(data, key, value) - - write_data(path, data) - return 0 - - -def del_value(path, key): - data = get_data(path) - if data is None: - err(f'creating new file: "{path}"') - data = {} - - key_path = key.split('.') - sub = data - for k in key_path[:-1]: - sub = sub.setdefault(k, {}) - sub.pop(key_path[-1], None) - - write_data(path, data) - return 0 +from .basics import * diff --git a/python-libs/poulpe/basics.py b/python-libs/poulpe/basics.py new file mode 100644 index 0000000000000000000000000000000000000000..6ed5c99bcfe74fb4c4cc3767fe7415ef4889b8f8_cHl0aG9uLWxpYnMvcG91bHBlL2Jhc2ljcy5weQ== --- /dev/null +++ b/python-libs/poulpe/basics.py @@ -0,0 +1,122 @@ +import os +import toml +import tempfile +import sys + +BIN_ENV_FILE = 'bin-env.poulpe' +SHELL_FILE = 'bin-env.shell' +DATA_ENV_FILE = 'data-env.poulpe' + + +def err(*args, **kwargs): + """print something on stderr""" + print(*args, **kwargs, file=sys.stderr) + + +def bin_env_file(env_path): + """given the path of env, return the file of description file""" + return os.path.join(env_path, BIN_ENV_FILE) + + +def bin_env_script(bin_env_path): + """given the path of env, return the file of description file""" + return os.path.join(bin_env_path, SHELL_FILE) + + +def data_env_file(data_env_path): + """given the path of env, return the file of description file""" + return os.path.join(data_env_path, DATA_ENV_FILE) + + +def get_data(path): + """Read a descript file at <path>""" + try: + with open(path) as f: + return toml.load(f) + except FileNotFoundError: + return None + + +def write_data(path, data): + """write description data at <path>""" + directory = os.path.dirname(path) + basename = os.path.basename(path) + tmp_path = os.path.join(directory, f".{basename}.tmp") + with open(tmp_path, "w") as f: + toml.dump(data, f) + os.replace(tmp_path, path) + + +def show(data, indent=''): + """call to display data""" + if not isinstance(data, dict): + print(f"{indent}{data}") + return + for k, v in sorted(data.items()): + if isinstance(v, dict): + print(f"{indent}{k}:") + show(v, indent + ' ') + elif isinstance(v, list): + print(f"{indent}{k}:") + for i in v: + show(i, indent + '- ') + else: + print(f"{indent}{k} = {v}") + + +def get_one_value(data, key): + key_path = key.split('.') + sub = data + for k in key_path: + sub = sub.get(k) + if sub is None: + break + return sub + + +def set_one_value(data, key, value): + key_path = key.split('.') + sub = data + for k in key_path[:-1]: + val = sub.setdefault(k, {}) + sub = val + sub[key_path[-1]] = value + + +def del_one_value(data, key): + key_path = key.split('.') + sub = data + for k in key_path[:-1]: + sub = sub.get(k) + if sub is None: + break + if sub is not None: + sub.pop(key_path[-1], None) + + +def set_value(path, key, value): + data = get_data(path) + if data is None: + err(f'creating new file: "{path}"') + data = {} + + set_one_value(data, key, value) + + write_data(path, data) + return 0 + + +def del_value(path, key): + data = get_data(path) + if data is None: + err(f'creating new file: "{path}"') + data = {} + + key_path = key.split('.') + sub = data + for k in key_path[:-1]: + sub = sub.setdefault(k, {}) + sub.pop(key_path[-1], None) + + write_data(path, data) + return 0 diff --git a/python-libs/poulpe/runner.py b/python-libs/poulpe/runner.py new file mode 100644 index 0000000000000000000000000000000000000000..6ed5c99bcfe74fb4c4cc3767fe7415ef4889b8f8_cHl0aG9uLWxpYnMvcG91bHBlL3J1bm5lci5weQ== --- /dev/null +++ b/python-libs/poulpe/runner.py @@ -0,0 +1,193 @@ +import json +import os +import subprocess +import tempfile +import time +import poulpe + + +class Benchmark: + + def __init__(self, data, variants=None): + self._raw_data = data + + assert self._raw_data['meta']['format'] == "0" + assert self._raw_data['meta']['method'] == "simple-command" + + self._data = self._raw_data.copy() + if variants is None: + variants = {} + self._selected_variants = variants + + l_1 = self._data.get('simple-command', {}) + l_2 = l_1.get('variants', {}) + all_dimensions = l_2.get('dimensions', {}) + + for dimension, variants in all_dimensions.items(): + # find the variants we should use. + selected = self._selected_variants.get(dimension) + if selected is None: + for name, values in variants.items(): + if values.get("default", False): + selected = name + break + else: + msg = "missing default value for dimensions: %s" + assert False, msg % dimension + self._selected_variants[dimension] = selected + + # gather the changes to the benchmark + new_cwd = variants[selected].get('cwd') + if new_cwd is not None: + self._data['simple-command']['cwd'] = new_cwd + extend_command = variants[selected].get('extend-command') + if extend_command is not None: + self._data['simple-command']['command'] += extend_command + + @property + def name(self): + return self._data['meta']['name'] + + def get_var(self, key): + return poulpe.get_one_value(self._data, key) + + @property + def selected_variants(self): + return self._selected_variants.copy() + + +def get_benchmark(path_def): + """get a benchmark from path applying possible variant selection on the way + + Expected syntax is PATH[;dimension_name=variant_name][;d=v]... + """ + parts = path_def.split(';') + path = parts[0] + variants_definitions = parts[1:] + variants = {} + for d in variants_definitions: + k, v = d.split("=") + variants[k] = v + + data = poulpe.get_data(path) + if data is None: + return None + return Benchmark(data, variants) + + +class SimpleCommand(object): + + def __init__(self, benchmark_data): + self._benchmark_data = benchmark_data + + def get_var(key): + key = f'simple-command.{key}' + return benchmark_data.get_var(key) + + self.base_command = get_var('command') + assert self.base_command is not None + self.command = None + + # XXX having a list of acceptable status would be better (default to [0]) + self.accept_failure = bool(get_var('accept-failure')) + + self.cwd = get_var('cwd') + + def apply_data_env(self, data_env_data): + variables_data = self._benchmark_data.get_var('simple-command').get('variables') + + def get_var(key): + full_key = f'bench-input-vars.{key}' + return poulpe.get_one_value(data_env_data, full_key) + + variables = {} + if variables_data is not None: + for name, value in variables_data.items(): + if value.startswith('DATA-VARS:'): + key = value.split(':', 1)[1] + value = get_var(key) + assert value is not None + variables[name] = value + + self.command = self.base_command.format(**variables) + + if self.cwd.startswith('DATA-VARS:'): + key = self.cwd.split(':', 1)[1] + self.cwd = get_var(key) + + +def run_one(bin_env_path, data_env_path, benchmark, result): + result_data = {} + result_data['run'] = {} + result_data['run']["timestamp"] = time.time() + + # gather info about the binary environment + bin_env_desc = poulpe.bin_env_file(bin_env_path) + bin_env_data = poulpe.get_data(bin_env_desc) + result_data['bin-env-vars'] = bin_env_data['bin-env-vars'] + + # gather info about the data environment + data_env_desc = poulpe.data_env_file(data_env_path) + data_env_data = poulpe.get_data(data_env_desc) + result_data['data-env-vars'] = data_env_data['data-env-vars'] + + benchmark_data = get_benchmark(benchmark) + assert benchmark_data is not None, benchmark + result_data['benchmark'] = {} + result_data['benchmark']['name'] = benchmark_data.name + variants = benchmark_data.selected_variants + if variants: + result_data['benchmark']['variants'] = variants + + # building the benchmark scenarion, that will likely changes often and quickly + + cmd = SimpleCommand(benchmark_data) + cmd.apply_data_env(data_env_data) + + r = _time_command(bin_env_path, data_env_path, cmd) + + # we should store more + result_data['result'] = {} + result_data['result']['time'] = {} + result_data['result']['time']['median'] = r['results'][0]['median'] + result_data['result']['time']['mean'] = r['results'][0]['mean'] + result_data['result']['time']['standard-deviation'] = r['results'][0]['stddev'] + result_data['result']['time']['min'] = r['results'][0]['min'] + result_data['result']['time']['max'] = r['results'][0]['max'] + + result_data['run']["duration"] = time.time() - result_data['run']["timestamp"] + + poulpe.write_data(result, result_data) + return 0 + + +def _time_command(bin_env_path, data_env_path, cmd): + bin_env_path = os.path.abspath(bin_env_path) + data_env_path = os.path.abspath(data_env_path) + shell_path = poulpe.bin_env_script(bin_env_path) + + with tempfile.NamedTemporaryFile('w') as tmp_result: + time_cmd = [ + shell_path, + "hyperfine", + "--export-json", + tmp_result.name, + "--", + cmd.command, + ] + if cmd.accept_failure: + time_cmd.insert(-2, "--ignore-failure") + + cwd = data_env_path + if cmd.cwd is not None: + cwd = os.path.join(cwd, cmd.cwd) + + subprocess.check_call( + time_cmd, + cwd=cwd, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + + with open(tmp_result.name) as f: + return json.load(f)