diff --git a/bin/bin-env-util b/bin/bin-env-util index f8b889a5fb2d225d35616cb95df5995d6f14adde_YmluL2Jpbi1lbnYtdXRpbA==..d00bf0f38e5526b6af264db436226697dd008282_YmluL2Jpbi1lbnYtdXRpbA== 100755 --- a/bin/bin-env-util +++ b/bin/bin-env-util @@ -5,8 +5,11 @@ import os import subprocess import sys -import tempfile -import toml + +poulpe_root = os.path.dirname(os.path.dirname(sys.argv[0])) +sys.path.insert(0, os.path.join(poulpe_root, 'python-libs')) + +import poulpe_helper as poulpe ERR_CODE_NO_KEY = 1 ERR_CODE_NO_BIN_ENV_SHELL = 2 @@ -15,13 +18,7 @@ ERR_CODE_NO_SETUP_SCRIPT = 6 ERR_CODE_SETUP_SCRIPT_FAILED = 7 -BIN_ENV_FILE = 'bin-env.poulpe' -SHELL_FILE = 'bin-env.shell' - - -def _bin_env_file(env_path): - """given the path of env, return the file of description file""" - return os.path.join(env_path, BIN_ENV_FILE) +err = poulpe.err def create_empty(env_path, method="manual"): @@ -35,7 +32,7 @@ }, 'ready': False, } - file_path = _bin_env_file(env_path) + file_path = poulpe.bin_env_file(env_path) if os.path.exists(file_path): err('This is already a bin-env, aborting') return ERR_CODE_EXISTS @@ -39,7 +36,7 @@ if os.path.exists(file_path): err('This is already a bin-env, aborting') return ERR_CODE_EXISTS - _write_data(file_path, base_data) + poulpe.write_data(file_path, base_data) return 0 @@ -55,8 +52,8 @@ err(f'script returned with status {exc.returncode}: {script}') return ERR_CODE_SETUP_SCRIPT_FAILED - env_file = _bin_env_file(env_path) - data = _get_data(env_file) + env_file = poulpe.bin_env_file(env_path) + data = poulpe.get_data(env_file) vars_file = os.path.join(env_path, "POULPE-VARS") if os.path.exists(vars_file): with open(vars_file) as f: @@ -65,9 +62,9 @@ k = k.strip() v = v.strip() k = f'bin-env-vars.{k}' - _set_one_value(data, k, v) - _write_data(env_file, data) + poulpe.set_one_value(data, k, v) + poulpe.write_data(env_file, data) return mark_ready(env_path) def mark_ready(env_path): @@ -70,9 +67,9 @@ return mark_ready(env_path) def mark_ready(env_path): - shell_path = os.path.join(env_path, SHELL_FILE) + shell_path = os.path.join(env_path, poulpe.SHELL_FILE) if not os.access(shell_path, os.X_OK): err(f'cannot find an executable file at: "{shell_path}"') return ERR_CODE_NO_BIN_ENV_SHELL @@ -75,10 +72,10 @@ if not os.access(shell_path, os.X_OK): err(f'cannot find an executable file at: "{shell_path}"') return ERR_CODE_NO_BIN_ENV_SHELL - path = _bin_env_file(env_path) - data = _get_data(path) + path = poulpe.bin_env_file(env_path) + data = poulpe.get_data(path) if data is None: err(f'missing file: "{path}"') return ERR_CODE_NO_BIN_ENV_DESC data['ready'] = True @@ -81,8 +78,8 @@ if data is None: err(f'missing file: "{path}"') return ERR_CODE_NO_BIN_ENV_DESC data['ready'] = True - _write_data(path, data) + poulpe.write_data(path, data) return 0 @@ -86,27 +83,4 @@ return 0 -def _get_data(path): - try: - with open(path) as f: - return toml.load(f) - except FileNotFoundError: - return None - - -def _write_data(path, data): - with tempfile.NamedTemporaryFile(mode='w', delete=False) as f: - toml.dump(data, f) - os.replace(f.name, path) - - -def _show_one_level(data, indent=''): - for k, v in sorted(data.items()): - if not isinstance(v, dict): - print(f"{indent}{k} = {v}") - else: - print(f"{indent}{k}:") - _show_one_level(v, indent + ' ') - - def show(env_path): @@ -112,15 +86,6 @@ def show(env_path): - path = _bin_env_file(env_path) - data = _get_data(path) - if data is None: - err(f'missing file: "{path}"') - return ERR_CODE_NO_BIN_ENV_DESC - _show_one_level(data) - return 0 - - -def get_value(path, key): - data = _get_data(path) + path = poulpe.bin_env_file(env_path) + data = poulpe.get_data(path) if data is None: err(f'missing file: "{path}"') return ERR_CODE_NO_BIN_ENV_DESC @@ -124,13 +89,7 @@ if data is None: err(f'missing file: "{path}"') return ERR_CODE_NO_BIN_ENV_DESC - - key_path = key.split('.') - sub = data - for k in key_path: - sub = sub.get(k) - if sub is None: - return ERR_CODE_NO_KEY + poulpe.show(data) return 0 @@ -134,48 +93,6 @@ return 0 -def _set_one_value(data, key, value): - key_path = key.split('.') - sub = data - for k in key_path[:-1]: - val = sub.setdefault(k, {}) - sub = val - sub[key_path[-1]] = value - - -def set_value(path, key, value): - data = _get_data(path) - if data is None: - err(f'creating new file: "{path}"') - data = {} - - _set_one_value(data, key, value) - - _write_data(path, data) - return 0 - - -def del_value(path, key): - data = _get_data(path) - if data is None: - err(f'creating new file: "{path}"') - data = {} - - key_path = key.split('.') - sub = data - for k in key_path[:-1]: - sub = sub.setdefault(k, {}) - sub.pop(key_path[-1], None) - - _write_data(path, data) - return 0 - - -def err(*args, **kwargs): - """print something on stderr""" - print(*args, **kwargs, file=sys.stderr) - - def _parsers(): top_parser = argparse.ArgumentParser(prog='poulpe-bin-env-util') subparsers = top_parser.add_subparsers( @@ -291,8 +208,17 @@ elif param.command == 'show': ret = show(param.PATH) elif param.command == 'get': - ret = get_value( - _bin_env_file(param.PATH), - f'bin-env-vars.{param.KEY}', - ) + f = poulpe.bin_env_file(param.PATH) + data = poulpe.get_data(f) + if data is None: + err(f'missing file: "{f}"') + ret = ERR_CODE_NO_BIN_ENV_DESC + else: + k = f'bin-env-vars.{param.KEY}' + value = poulpe.get_one_value(data, k) + if value is None: + ret = ERR_CODE_NO_KEY + else: + print(value) + ret = 0 elif param.command == 'set': @@ -298,7 +224,11 @@ elif param.command == 'set': - ret = set_value( - _bin_env_file(param.PATH), - f'bin-env-vars.{param.KEY}', - param.VALUE, - ) + f = poulpe.bin_env_file(param.PATH) + k = f'bin-env-vars.{param.KEY}' + data = poulpe.get_data(f) + if data is None: + err(f'creating new file: "{f}"') + data = {} + poulpe.set_one_value(data, k, param.VALUE) + poulpe.write_data(f, data) + return 0 elif param.command == 'del': @@ -304,8 +234,7 @@ elif param.command == 'del': - ret = del_value( - _bin_env_file(param.PATH), - f'bin-env-vars.{param.KEY}', - ) + f = poulpe.bin_env_file(param.PATH) + k = f'bin-env-vars.{param.KEY}' + ret = poulpe.del_value(f, k) else: assert False return ret diff --git a/bin/diff-result b/bin/diff-result index f8b889a5fb2d225d35616cb95df5995d6f14adde_YmluL2RpZmYtcmVzdWx0..d00bf0f38e5526b6af264db436226697dd008282_YmluL2RpZmYtcmVzdWx0 100755 --- a/bin/diff-result +++ b/bin/diff-result @@ -2,5 +2,4 @@ # # Small tool to manipulaote import argparse -import json import os @@ -6,4 +5,2 @@ import os -import stat -import subprocess import sys @@ -9,31 +6,2 @@ import sys -import tempfile -import toml - - -ERR_CODE_NO_KEY = 1 -ERR_CODE_NO_BIN_ENV_SHELL = 2 -ERR_CODE_NO_BIN_ENV_DESC = 3 -ERR_CODE_EXISTS = 4 -ERR_CODE_NO_SETUP_SCRIPT = 6 -ERR_CODE_SETUP_SCRIPT_FAILED = 7 - -BIN_ENV_FILE = 'bin-env.poulpe' -DATA_ENV_FILE = 'data-env.poulpe' -SHELL_FILE = 'bin-env.shell' - - -def _bin_env_file(bin_env_path): - """given the path of env, return the file of description file""" - return os.path.join(bin_env_path, BIN_ENV_FILE) - -def _bin_env_script(bin_env_path): - """given the path of env, return the file of description file""" - return os.path.join(bin_env_path, SHELL_FILE) - - -def _data_env_file(data_env_path): - """given the path of env, return the file of description file""" - return os.path.join(data_env_path, DATA_ENV_FILE) - @@ -39,22 +7,4 @@ -def _get_data(path): - try: - with open(path) as f: - return toml.load(f) - except FileNotFoundError: - return None - -def _write_data(path, data): - with tempfile.NamedTemporaryFile(mode='w', delete=False) as f: - toml.dump(data, f) - os.replace(f.name, path) - -def _get_one_value(data, key): - key_path = key.split('.') - sub = data - for k in key_path: - sub = sub.get(k) - if sub is None: - break - return sub +poulpe_root = os.path.dirname(os.path.dirname(sys.argv[0])) +sys.path.insert(0, os.path.join(poulpe_root, 'python-libs')) @@ -60,28 +10,5 @@ -def _set_one_value(data, key, value): - key_path = key.split('.') - sub = data - for k in key_path[:-1]: - val = sub.setdefault(k, {}) - sub = val - sub[key_path[-1]] = value - - -def set_value(path, key, value): - data = _get_data(path) - if data is None: - err(f'creating new file: "{path}"') - data = {} - - _set_one_value(data, key, value) - - _write_data(path, data) - return 0 - - -def err(*args, **kwargs): - """print something on stderr""" - print(*args, **kwargs, file=sys.stderr) +import poulpe_helper as poulpe def compare(old_path, new_path): @@ -85,5 +12,5 @@ def compare(old_path, new_path): - old = _get_data(old_path) + old = poulpe.get_data(old_path) assert old is not None @@ -89,5 +16,5 @@ assert old is not None - new = _get_data(new_path) + new = poulpe.get_data(new_path) assert new is not None old_median = old['result']['time']['median'] @@ -121,6 +48,7 @@ ret = compare(param.OLD_RESULT, param.NEW_RESULT) return ret + if __name__ == "__main__": ret = main(sys.argv[1:]) assert ret is not None diff --git a/bin/env-desc b/bin/env-desc index f8b889a5fb2d225d35616cb95df5995d6f14adde_YmluL2Vudi1kZXNj..d00bf0f38e5526b6af264db436226697dd008282_YmluL2Vudi1kZXNj 100755 --- a/bin/env-desc +++ b/bin/env-desc @@ -3,8 +3,6 @@ # Small tool to manipulaote import os import sys -import tempfile -import toml ACTION_SHOW = 'show' ACTION_SET = 'set' @@ -17,17 +15,6 @@ ACTION_DEL, } - -def _get_data(path): - try: - with open(path) as f: - return toml.load(f) - except FileNotFoundError: - return None - -def _write_data(path, data): - with tempfile.NamedTemporaryFile(mode='w', delete=False) as f: - toml.dump(data, f) - os.replace(f.name, path) - +poulpe_root = os.path.dirname(os.path.dirname(sys.argv[0])) +sys.path.insert(0, os.path.join(poulpe_root, 'python-libs')) @@ -33,18 +20,7 @@ -def _show_one_level(data, indent=''): - if not isinstance(data, dict): - print(f"{indent}{data}") - return - for k, v in sorted(data.items()): - if isinstance(v, dict): - print(f"{indent}{k}:") - _show_one_level(v, indent + ' ') - elif isinstance(v, list): - print(f"{indent}{k}:") - for i in v: - _show_one_level(i, indent + '- ') - else: - print(f"{indent}{k} = {v}") +import poulpe_helper as poulpe + +err = poulpe.err def show(path): @@ -48,15 +24,7 @@ def show(path): - data = _get_data(path) - if data is None: - err(f'missing file: "{path}"') - return 3 - _show_one_level(data) - - -def get_value(path, key): - data = _get_data(path) + data = poulpe.get_data(path) if data is None: err(f'missing file: "{path}"') return 3 @@ -60,4 +28,5 @@ if data is None: err(f'missing file: "{path}"') return 3 + poulpe.show(data) @@ -63,12 +32,16 @@ - key_path = key.split('.') - sub = data - for k in key_path: - sub = sub.get(k) - if sub is None: - return 1 - print(sub) - return 0 + +def get_value(path, key): + data = poulpe.get_data(path) + if data is None: + err(f'missing file: "{path}"') + return 3 + value = poulpe.get_one_value(data, key) + if value is None: + return 1 + else: + print(value) + return 0 def set_value(path, key, value): @@ -72,8 +45,8 @@ def set_value(path, key, value): - data = _get_data(path) + data = poulpe.get_data(path) if data is None: err(f'creating new file: "{path}"') data = {} @@ -76,10 +49,6 @@ if data is None: err(f'creating new file: "{path}"') data = {} - key_path = key.split('.') - sub = data - for k in key_path[:-1]: - sub = sub.setdefault(k, {}) - sub[key_path[-1]] = value + value = poulpe.set_one_value(data, key, value) @@ -85,6 +54,6 @@ - _write_data(path, data) + poulpe.write_data(path, data) return 0 def del_value(path, key): @@ -87,9 +56,9 @@ return 0 def del_value(path, key): - data = _get_data(path) + data = poulpe.get_data(path) if data is None: err(f'creating new file: "{path}"') data = {} @@ -92,10 +61,6 @@ if data is None: err(f'creating new file: "{path}"') data = {} - key_path = key.split('.') - sub = data - for k in key_path[:-1]: - sub = sub.setdefault(k, {}) - sub.pop(key_path[-1], None) + poulpe.del_one_value(data, key) @@ -101,4 +66,4 @@ - _write_data(path, data) + poulpe.write_data(path, data) return 0 @@ -103,8 +68,5 @@ return 0 -def err(*args, **kwargs): - print(*args, **kwargs, file=sys.stderr) - if __name__ == "__main__": if len(sys.argv) < 3: diff --git a/bin/run-util b/bin/run-util index f8b889a5fb2d225d35616cb95df5995d6f14adde_YmluL3J1bi11dGls..d00bf0f38e5526b6af264db436226697dd008282_YmluL3J1bi11dGls 100755 --- a/bin/run-util +++ b/bin/run-util @@ -4,7 +4,6 @@ import argparse import json import os -import stat import subprocess import sys import tempfile @@ -8,39 +7,4 @@ import subprocess import sys import tempfile -import toml - - -ERR_CODE_NO_KEY = 1 -ERR_CODE_NO_BIN_ENV_SHELL = 2 -ERR_CODE_NO_BIN_ENV_DESC = 3 -ERR_CODE_EXISTS = 4 -ERR_CODE_NO_SETUP_SCRIPT = 6 -ERR_CODE_SETUP_SCRIPT_FAILED = 7 - -BIN_ENV_FILE = 'bin-env.poulpe' -DATA_ENV_FILE = 'data-env.poulpe' -SHELL_FILE = 'bin-env.shell' - - -def _bin_env_file(bin_env_path): - """given the path of env, return the file of description file""" - return os.path.join(bin_env_path, BIN_ENV_FILE) - -def _bin_env_script(bin_env_path): - """given the path of env, return the file of description file""" - return os.path.join(bin_env_path, SHELL_FILE) - - -def _data_env_file(data_env_path): - """given the path of env, return the file of description file""" - return os.path.join(data_env_path, DATA_ENV_FILE) - - -def _get_data(path): - try: - with open(path) as f: - return toml.load(f) - except FileNotFoundError: - return None @@ -46,15 +10,4 @@ -def _write_data(path, data): - with tempfile.NamedTemporaryFile(mode='w', delete=False) as f: - toml.dump(data, f) - os.replace(f.name, path) - -def _get_one_value(data, key): - key_path = key.split('.') - sub = data - for k in key_path: - sub = sub.get(k) - if sub is None: - break - return sub +poulpe_root = os.path.dirname(os.path.dirname(sys.argv[0])) +sys.path.insert(0, os.path.join(poulpe_root, 'python-libs')) @@ -60,31 +13,8 @@ -def _set_one_value(data, key, value): - key_path = key.split('.') - sub = data - for k in key_path[:-1]: - val = sub.setdefault(k, {}) - sub = val - sub[key_path[-1]] = value - - -def set_value(path, key, value): - data = _get_data(path) - if data is None: - err(f'creating new file: "{path}"') - data = {} - - _set_one_value(data, key, value) - - _write_data(path, data) - return 0 - - -def err(*args, **kwargs): - """print something on stderr""" - print(*args, **kwargs, file=sys.stderr) +import poulpe_helper as poulpe def run_one(bin_env_path, data_env_path, benchmark, result): result_data = {} # gather info about the binary environment @@ -85,11 +15,11 @@ def run_one(bin_env_path, data_env_path, benchmark, result): result_data = {} # gather info about the binary environment - bin_env_desc = _bin_env_file(bin_env_path) - bin_env_data = _get_data(bin_env_desc) + bin_env_desc = poulpe.bin_env_file(bin_env_path) + bin_env_data = poulpe.get_data(bin_env_desc) result_data['bin-env-vars'] = bin_env_data['bin-env-vars'] # gather info about the data environment @@ -93,7 +23,7 @@ result_data['bin-env-vars'] = bin_env_data['bin-env-vars'] # gather info about the data environment - data_env_desc = _data_env_file(data_env_path) - data_env_data = _get_data(data_env_desc) + data_env_desc = poulpe.data_env_file(data_env_path) + data_env_data = poulpe.get_data(data_env_desc) result_data['data-env-vars'] = data_env_data['data-env-vars'] @@ -98,6 +28,6 @@ result_data['data-env-vars'] = data_env_data['data-env-vars'] - benchmark_data = _get_data(benchmark) + benchmark_data = poulpe.get_data(benchmark) assert benchmark_data is not None, benchmark result_data['benchmark'] = {} result_data['benchmark']['name'] = benchmark_data['meta']['name'] @@ -112,7 +42,7 @@ for name, value in variables_data.items(): if value.startswith('DATA-VARS:'): key = value.split(':', 1)[1] - value = _get_one_value(data_env_data, f'bench-input-vars.{key}') + value = poulpe.get_one_value(data_env_data, f'bench-input-vars.{key}') assert value is not None variables[name] = value @@ -125,10 +55,10 @@ result_data['result']['time'] = {} result_data['result']['time']['median'] = r['results'][0]['median'] - _write_data(result, result_data) + poulpe.write_data(result, result_data) return 0 def _time_command(bin_env_path, data_env_path, cmd): bin_env_path = os.path.abspath(bin_env_path) data_env_path = os.path.abspath(data_env_path) @@ -129,10 +59,10 @@ return 0 def _time_command(bin_env_path, data_env_path, cmd): bin_env_path = os.path.abspath(bin_env_path) data_env_path = os.path.abspath(data_env_path) - shell_path = _bin_env_script(bin_env_path) + shell_path = poulpe.bin_env_script(bin_env_path) with tempfile.NamedTemporaryFile('w') as tmp_result: time_cmd = [ @@ -183,6 +113,7 @@ ret = run_one(param.BIN_ENV, param.DATA_ENV, param.BENCHMARK, param.RESULT) return ret + if __name__ == "__main__": ret = main(sys.argv[1:]) assert ret is not None diff --git a/python-libs/poulpe_helper.py b/python-libs/poulpe_helper.py new file mode 100644 index 0000000000000000000000000000000000000000..d00bf0f38e5526b6af264db436226697dd008282_cHl0aG9uLWxpYnMvcG91bHBlX2hlbHBlci5weQ== --- /dev/null +++ b/python-libs/poulpe_helper.py @@ -0,0 +1,119 @@ +import os +import toml +import tempfile +import sys + +BIN_ENV_FILE = 'bin-env.poulpe' +SHELL_FILE = 'bin-env.shell' +DATA_ENV_FILE = 'data-env.poulpe' + + +def err(*args, **kwargs): + """print something on stderr""" + print(*args, **kwargs, file=sys.stderr) + + +def bin_env_file(env_path): + """given the path of env, return the file of description file""" + return os.path.join(env_path, BIN_ENV_FILE) + + +def bin_env_script(bin_env_path): + """given the path of env, return the file of description file""" + return os.path.join(bin_env_path, SHELL_FILE) + + +def data_env_file(data_env_path): + """given the path of env, return the file of description file""" + return os.path.join(data_env_path, DATA_ENV_FILE) + + +def get_data(path): + """Read a descript file at <path>""" + try: + with open(path) as f: + return toml.load(f) + except FileNotFoundError: + return None + + +def write_data(path, data): + """write description data at <path>""" + with tempfile.NamedTemporaryFile(mode='w', delete=False) as f: + toml.dump(data, f) + os.replace(f.name, path) + + +def show(data, indent=''): + """call to display data""" + if not isinstance(data, dict): + print(f"{indent}{data}") + return + for k, v in sorted(data.items()): + if isinstance(v, dict): + print(f"{indent}{k}:") + show(v, indent + ' ') + elif isinstance(v, list): + print(f"{indent}{k}:") + for i in v: + show(i, indent + '- ') + else: + print(f"{indent}{k} = {v}") + + +def get_one_value(data, key): + key_path = key.split('.') + sub = data + for k in key_path: + sub = sub.get(k) + if sub is None: + break + return sub + + +def set_one_value(data, key, value): + key_path = key.split('.') + sub = data + for k in key_path[:-1]: + val = sub.setdefault(k, {}) + sub = val + sub[key_path[-1]] = value + + +def del_one_value(data, key): + key_path = key.split('.') + sub = data + for k in key_path[:-1]: + sub = sub.get(k) + if sub is None: + break + if sub is not None: + sub.pop(key_path[-1], None) + + +def set_value(path, key, value): + data = get_data(path) + if data is None: + err(f'creating new file: "{path}"') + data = {} + + set_one_value(data, key, value) + + write_data(path, data) + return 0 + + +def del_value(path, key): + data = get_data(path) + if data is None: + err(f'creating new file: "{path}"') + data = {} + + key_path = key.split('.') + sub = data + for k in key_path[:-1]: + sub = sub.setdefault(k, {}) + sub.pop(key_path[-1], None) + + write_data(path, data) + return 0