Skip to content
Snippets Groups Projects
Commit 6ed5c99b authored by Pierre-Yves David's avatar Pierre-Yves David :octopus:
Browse files

runner: move most of the runner logic inside the python module

parent 757d43cf
No related branches found
No related tags found
1 merge request!12runner: factor out some part of the benchmark class
...@@ -2,5 +2,4 @@ ...@@ -2,5 +2,4 @@
# #
# Small tool to manipulaote # Small tool to manipulaote
import argparse import argparse
import json
import os import os
...@@ -6,3 +5,2 @@ ...@@ -6,3 +5,2 @@
import os import os
import subprocess
import sys import sys
...@@ -8,7 +6,5 @@ ...@@ -8,7 +6,5 @@
import sys import sys
import tempfile
import time
poulpe_root = os.path.dirname(os.path.dirname(sys.argv[0])) poulpe_root = os.path.dirname(os.path.dirname(sys.argv[0]))
sys.path.insert(0, os.path.join(poulpe_root, 'python-libs')) sys.path.insert(0, os.path.join(poulpe_root, 'python-libs'))
...@@ -11,196 +7,8 @@ ...@@ -11,196 +7,8 @@
poulpe_root = os.path.dirname(os.path.dirname(sys.argv[0])) poulpe_root = os.path.dirname(os.path.dirname(sys.argv[0]))
sys.path.insert(0, os.path.join(poulpe_root, 'python-libs')) sys.path.insert(0, os.path.join(poulpe_root, 'python-libs'))
import poulpe from poulpe import runner
class Benchmark:
def __init__(self, data, variants=None):
self._raw_data = data
assert self._raw_data['meta']['format'] == "0"
assert self._raw_data['meta']['method'] == "simple-command"
self._data = self._raw_data.copy()
if variants is None:
variants = {}
self._selected_variants = variants
dimensions_key = 'simple-command.variants.dimensions'
l_1 = self._data.get('simple-command', {})
l_2 = l_1.get('variants', {})
all_dimensions = l_2.get('dimensions', {})
for dimension, variants in all_dimensions.items():
# find the variants we should use.
selected = self._selected_variants.get(dimension)
if selected is None:
for name, values in variants.items():
if values.get("default", False):
selected = name
break
else:
msg = "missing default value for dimensions: %s"
assert False, msg % dimension
self._selected_variants[dimension] = selected
# gather the changes to the benchmark
new_cwd = variants[selected].get('cwd')
if new_cwd is not None:
self._data['simple-command']['cwd'] = new_cwd
extend_command= variants[selected].get('extend-command')
if extend_command is not None:
self._data['simple-command']['command'] += extend_command
@property
def name(self):
return self._data['meta']['name']
def get_var(self, key):
return poulpe.get_one_value(self._data, key)
@property
def selected_variants(self):
return self._selected_variants.copy()
def get_benchmark(path_def):
"""get a benchmark from path applying possible variant selection on the way
Expected syntax is PATH[;dimension_name=variant_name][;d=v]...
"""
parts = path_def.split(';')
path = parts[0]
variants_definitions = parts[1:]
variants = {}
for d in variants_definitions:
k, v = d.split("=")
variants[k] = v
data = poulpe.get_data(path)
if data is None:
return None
return Benchmark(data, variants)
class SimpleCommand(object):
def __init__(self, benchmark_data):
self._benchmark_data = benchmark_data
def get_var(key):
key = f'simple-command.{key}'
return benchmark_data.get_var(key)
self.base_command = get_var('command')
assert self.base_command is not None
self.command = None
# XXX having a list of acceptable status would be better (default to [0])
self.accept_failure = bool(get_var('accept-failure'))
self.cwd = get_var('cwd')
def apply_data_env(self, data_env_data):
variables_data = self._benchmark_data.get_var('simple-command').get('variables')
def get_var(key):
full_key = f'bench-input-vars.{key}'
return poulpe.get_one_value(data_env_data, full_key)
variables = {}
if variables_data is not None:
for name, value in variables_data.items():
if value.startswith('DATA-VARS:'):
key = value.split(':', 1)[1]
value = get_var(key)
assert value is not None
variables[name] = value
self.command = self.base_command.format(**variables)
if self.cwd.startswith('DATA-VARS:'):
key = self.cwd.split(':', 1)[1]
self.cwd = get_var(key)
def run_one(bin_env_path, data_env_path, benchmark, result):
result_data = {}
result_data['run'] = {}
result_data['run']["timestamp"] = time.time()
# gather info about the binary environment
bin_env_desc = poulpe.bin_env_file(bin_env_path)
bin_env_data = poulpe.get_data(bin_env_desc)
result_data['bin-env-vars'] = bin_env_data['bin-env-vars']
# gather info about the data environment
data_env_desc = poulpe.data_env_file(data_env_path)
data_env_data = poulpe.get_data(data_env_desc)
result_data['data-env-vars'] = data_env_data['data-env-vars']
benchmark_data = get_benchmark(benchmark)
assert benchmark_data is not None, benchmark
result_data['benchmark'] = {}
result_data['benchmark']['name'] = benchmark_data.name
variants = benchmark_data.selected_variants
if variants:
result_data['benchmark']['variants'] = variants
# building the benchmark scenarion, that will likely changes often and quickly
cmd = SimpleCommand(benchmark_data)
cmd.apply_data_env(data_env_data)
r = _time_command(bin_env_path, data_env_path, cmd)
# we should store more
result_data['result'] = {}
result_data['result']['time'] = {}
result_data['result']['time']['median'] = r['results'][0]['median']
result_data['result']['time']['mean'] = r['results'][0]['mean']
result_data['result']['time']['standard-deviation'] = r['results'][0]['stddev']
result_data['result']['time']['min'] = r['results'][0]['min']
result_data['result']['time']['max'] = r['results'][0]['max']
result_data['run']["duration"] = time.time() - result_data['run']["timestamp"]
poulpe.write_data(result, result_data)
return 0
def _time_command(bin_env_path, data_env_path, cmd):
bin_env_path = os.path.abspath(bin_env_path)
data_env_path = os.path.abspath(data_env_path)
shell_path = poulpe.bin_env_script(bin_env_path)
with tempfile.NamedTemporaryFile('w') as tmp_result:
time_cmd = [
shell_path,
"hyperfine",
"--export-json",
tmp_result.name,
"--",
cmd.command,
]
if cmd.accept_failure:
time_cmd.insert(-2, "--ignore-failure")
cwd = data_env_path
if cmd.cwd is not None:
cwd = os.path.join(cwd, cmd.cwd)
subprocess.check_call(
time_cmd,
cwd=cwd,
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
)
with open(tmp_result.name) as f:
return json.load(f)
def _parsers(): def _parsers():
...@@ -231,7 +39,12 @@ ...@@ -231,7 +39,12 @@
def main(args): def main(args):
parser = _parsers() parser = _parsers()
param = parser.parse_args(args) param = parser.parse_args(args)
ret = run_one(param.BIN_ENV, param.DATA_ENV, param.BENCHMARK, param.RESULT) ret = runner.run_one(
param.BIN_ENV,
param.DATA_ENV,
param.BENCHMARK,
param.RESULT,
)
return ret return ret
......
import os from .basics import *
import toml
import tempfile
import sys
BIN_ENV_FILE = 'bin-env.poulpe'
SHELL_FILE = 'bin-env.shell'
DATA_ENV_FILE = 'data-env.poulpe'
def err(*args, **kwargs):
"""print something on stderr"""
print(*args, **kwargs, file=sys.stderr)
def bin_env_file(env_path):
"""given the path of env, return the file of description file"""
return os.path.join(env_path, BIN_ENV_FILE)
def bin_env_script(bin_env_path):
"""given the path of env, return the file of description file"""
return os.path.join(bin_env_path, SHELL_FILE)
def data_env_file(data_env_path):
"""given the path of env, return the file of description file"""
return os.path.join(data_env_path, DATA_ENV_FILE)
def get_data(path):
"""Read a descript file at <path>"""
try:
with open(path) as f:
return toml.load(f)
except FileNotFoundError:
return None
def write_data(path, data):
"""write description data at <path>"""
directory = os.path.dirname(path)
basename = os.path.basename(path)
tmp_path = os.path.join(directory, f".{basename}.tmp")
with open(tmp_path, "w") as f:
toml.dump(data, f)
os.replace(tmp_path, path)
def show(data, indent=''):
"""call to display data"""
if not isinstance(data, dict):
print(f"{indent}{data}")
return
for k, v in sorted(data.items()):
if isinstance(v, dict):
print(f"{indent}{k}:")
show(v, indent + ' ')
elif isinstance(v, list):
print(f"{indent}{k}:")
for i in v:
show(i, indent + '- ')
else:
print(f"{indent}{k} = {v}")
def get_one_value(data, key):
key_path = key.split('.')
sub = data
for k in key_path:
sub = sub.get(k)
if sub is None:
break
return sub
def set_one_value(data, key, value):
key_path = key.split('.')
sub = data
for k in key_path[:-1]:
val = sub.setdefault(k, {})
sub = val
sub[key_path[-1]] = value
def del_one_value(data, key):
key_path = key.split('.')
sub = data
for k in key_path[:-1]:
sub = sub.get(k)
if sub is None:
break
if sub is not None:
sub.pop(key_path[-1], None)
def set_value(path, key, value):
data = get_data(path)
if data is None:
err(f'creating new file: "{path}"')
data = {}
set_one_value(data, key, value)
write_data(path, data)
return 0
def del_value(path, key):
data = get_data(path)
if data is None:
err(f'creating new file: "{path}"')
data = {}
key_path = key.split('.')
sub = data
for k in key_path[:-1]:
sub = sub.setdefault(k, {})
sub.pop(key_path[-1], None)
write_data(path, data)
return 0
import os
import toml
import tempfile
import sys
BIN_ENV_FILE = 'bin-env.poulpe'
SHELL_FILE = 'bin-env.shell'
DATA_ENV_FILE = 'data-env.poulpe'
def err(*args, **kwargs):
"""print something on stderr"""
print(*args, **kwargs, file=sys.stderr)
def bin_env_file(env_path):
"""given the path of env, return the file of description file"""
return os.path.join(env_path, BIN_ENV_FILE)
def bin_env_script(bin_env_path):
"""given the path of env, return the file of description file"""
return os.path.join(bin_env_path, SHELL_FILE)
def data_env_file(data_env_path):
"""given the path of env, return the file of description file"""
return os.path.join(data_env_path, DATA_ENV_FILE)
def get_data(path):
"""Read a descript file at <path>"""
try:
with open(path) as f:
return toml.load(f)
except FileNotFoundError:
return None
def write_data(path, data):
"""write description data at <path>"""
directory = os.path.dirname(path)
basename = os.path.basename(path)
tmp_path = os.path.join(directory, f".{basename}.tmp")
with open(tmp_path, "w") as f:
toml.dump(data, f)
os.replace(tmp_path, path)
def show(data, indent=''):
"""call to display data"""
if not isinstance(data, dict):
print(f"{indent}{data}")
return
for k, v in sorted(data.items()):
if isinstance(v, dict):
print(f"{indent}{k}:")
show(v, indent + ' ')
elif isinstance(v, list):
print(f"{indent}{k}:")
for i in v:
show(i, indent + '- ')
else:
print(f"{indent}{k} = {v}")
def get_one_value(data, key):
key_path = key.split('.')
sub = data
for k in key_path:
sub = sub.get(k)
if sub is None:
break
return sub
def set_one_value(data, key, value):
key_path = key.split('.')
sub = data
for k in key_path[:-1]:
val = sub.setdefault(k, {})
sub = val
sub[key_path[-1]] = value
def del_one_value(data, key):
key_path = key.split('.')
sub = data
for k in key_path[:-1]:
sub = sub.get(k)
if sub is None:
break
if sub is not None:
sub.pop(key_path[-1], None)
def set_value(path, key, value):
data = get_data(path)
if data is None:
err(f'creating new file: "{path}"')
data = {}
set_one_value(data, key, value)
write_data(path, data)
return 0
def del_value(path, key):
data = get_data(path)
if data is None:
err(f'creating new file: "{path}"')
data = {}
key_path = key.split('.')
sub = data
for k in key_path[:-1]:
sub = sub.setdefault(k, {})
sub.pop(key_path[-1], None)
write_data(path, data)
return 0
import json
import os
import subprocess
import tempfile
import time
import poulpe
class Benchmark:
def __init__(self, data, variants=None):
self._raw_data = data
assert self._raw_data['meta']['format'] == "0"
assert self._raw_data['meta']['method'] == "simple-command"
self._data = self._raw_data.copy()
if variants is None:
variants = {}
self._selected_variants = variants
l_1 = self._data.get('simple-command', {})
l_2 = l_1.get('variants', {})
all_dimensions = l_2.get('dimensions', {})
for dimension, variants in all_dimensions.items():
# find the variants we should use.
selected = self._selected_variants.get(dimension)
if selected is None:
for name, values in variants.items():
if values.get("default", False):
selected = name
break
else:
msg = "missing default value for dimensions: %s"
assert False, msg % dimension
self._selected_variants[dimension] = selected
# gather the changes to the benchmark
new_cwd = variants[selected].get('cwd')
if new_cwd is not None:
self._data['simple-command']['cwd'] = new_cwd
extend_command = variants[selected].get('extend-command')
if extend_command is not None:
self._data['simple-command']['command'] += extend_command
@property
def name(self):
return self._data['meta']['name']
def get_var(self, key):
return poulpe.get_one_value(self._data, key)
@property
def selected_variants(self):
return self._selected_variants.copy()
def get_benchmark(path_def):
"""get a benchmark from path applying possible variant selection on the way
Expected syntax is PATH[;dimension_name=variant_name][;d=v]...
"""
parts = path_def.split(';')
path = parts[0]
variants_definitions = parts[1:]
variants = {}
for d in variants_definitions:
k, v = d.split("=")
variants[k] = v
data = poulpe.get_data(path)
if data is None:
return None
return Benchmark(data, variants)
class SimpleCommand(object):
def __init__(self, benchmark_data):
self._benchmark_data = benchmark_data
def get_var(key):
key = f'simple-command.{key}'
return benchmark_data.get_var(key)
self.base_command = get_var('command')
assert self.base_command is not None
self.command = None
# XXX having a list of acceptable status would be better (default to [0])
self.accept_failure = bool(get_var('accept-failure'))
self.cwd = get_var('cwd')
def apply_data_env(self, data_env_data):
variables_data = self._benchmark_data.get_var('simple-command').get('variables')
def get_var(key):
full_key = f'bench-input-vars.{key}'
return poulpe.get_one_value(data_env_data, full_key)
variables = {}
if variables_data is not None:
for name, value in variables_data.items():
if value.startswith('DATA-VARS:'):
key = value.split(':', 1)[1]
value = get_var(key)
assert value is not None
variables[name] = value
self.command = self.base_command.format(**variables)
if self.cwd.startswith('DATA-VARS:'):
key = self.cwd.split(':', 1)[1]
self.cwd = get_var(key)
def run_one(bin_env_path, data_env_path, benchmark, result):
result_data = {}
result_data['run'] = {}
result_data['run']["timestamp"] = time.time()
# gather info about the binary environment
bin_env_desc = poulpe.bin_env_file(bin_env_path)
bin_env_data = poulpe.get_data(bin_env_desc)
result_data['bin-env-vars'] = bin_env_data['bin-env-vars']
# gather info about the data environment
data_env_desc = poulpe.data_env_file(data_env_path)
data_env_data = poulpe.get_data(data_env_desc)
result_data['data-env-vars'] = data_env_data['data-env-vars']
benchmark_data = get_benchmark(benchmark)
assert benchmark_data is not None, benchmark
result_data['benchmark'] = {}
result_data['benchmark']['name'] = benchmark_data.name
variants = benchmark_data.selected_variants
if variants:
result_data['benchmark']['variants'] = variants
# building the benchmark scenarion, that will likely changes often and quickly
cmd = SimpleCommand(benchmark_data)
cmd.apply_data_env(data_env_data)
r = _time_command(bin_env_path, data_env_path, cmd)
# we should store more
result_data['result'] = {}
result_data['result']['time'] = {}
result_data['result']['time']['median'] = r['results'][0]['median']
result_data['result']['time']['mean'] = r['results'][0]['mean']
result_data['result']['time']['standard-deviation'] = r['results'][0]['stddev']
result_data['result']['time']['min'] = r['results'][0]['min']
result_data['result']['time']['max'] = r['results'][0]['max']
result_data['run']["duration"] = time.time() - result_data['run']["timestamp"]
poulpe.write_data(result, result_data)
return 0
def _time_command(bin_env_path, data_env_path, cmd):
bin_env_path = os.path.abspath(bin_env_path)
data_env_path = os.path.abspath(data_env_path)
shell_path = poulpe.bin_env_script(bin_env_path)
with tempfile.NamedTemporaryFile('w') as tmp_result:
time_cmd = [
shell_path,
"hyperfine",
"--export-json",
tmp_result.name,
"--",
cmd.command,
]
if cmd.accept_failure:
time_cmd.insert(-2, "--ignore-failure")
cwd = data_env_path
if cmd.cwd is not None:
cwd = os.path.join(cwd, cmd.cwd)
subprocess.check_call(
time_cmd,
cwd=cwd,
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
)
with open(tmp_result.name) as f:
return json.load(f)
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment