Skip to content
Snippets Groups Projects
Commit 0284e06d authored by Pierre-Yves David's avatar Pierre-Yves David :octopus:
Browse files

Add basic support for variants

This is a bit raw, but should do the job for now.
parent 3c5fe756
No related branches found
No related tags found
1 merge request!10Add basic support for variants
Pipeline #56140 passed
......@@ -93,6 +93,11 @@
sys.exit(2)
key = sys.argv[3]
if action == ACTION_SET:
if path == '--bool':
sys.argv.pop(2)
path = sys.argv[2]
key = sys.argv[3]
sys.argv[4] = bool(sys.argv[4])
if len(sys.argv) < 5:
err("missing value")
sys.exit(2)
......
......@@ -14,6 +14,76 @@
import poulpe_helper as poulpe
class Benchmark:
def __init__(self, data, variants=None):
self._raw_data = data
assert self._raw_data['meta']['format'] == "0"
assert self._raw_data['meta']['method'] == "simple-command"
self._data = self._raw_data.copy()
if variants is None:
variants = {}
self._selected_variants = variants
dimensions_key = 'simple-command.variants.dimensions'
l_1 = self._data.get('simple-command', {})
l_2 = l_1.get('variants', {})
all_dimensions = l_2.get('dimensions', {})
for dimension, variants in all_dimensions.items():
# find the variants we should use.
selected = self._selected_variants.get(dimension)
if selected is None:
for name, values in variants.items():
if values.get("default", False):
selected = name
break
else:
msg = "missing default value for dimensions: %s"
assert False, msg % dimension
self._selected_variants[dimension] = selected
# gather the changes to the benchmark
new_cwd = variants[selected].get('cwd')
if new_cwd is not None:
self._data['simple-command']['cwd'] = new_cwd
extend_command= variants[selected].get('extend-command')
if extend_command is not None:
self._data['simple-command']['command'] += extend_command
@property
def name(self):
return self._data['meta']['name']
def get_var(self, key):
return poulpe.get_one_value(self._data, key)
@property
def selected_variants(self):
return self._selected_variants.copy()
def get_benchmark(path_def):
"""get a benchmark from path applying possible variant selection on the way
Expected syntax is PATH[;dimension_name=variant_name][;d=v]...
"""
parts = path_def.split(';')
path = parts[0]
variants_definitions = parts[1:]
variants = {}
for d in variants_definitions:
k, v = d.split("=")
variants[k] = v
data = poulpe.get_data(path)
if data is None:
return None
return Benchmark(data, variants)
class SimpleCommand(object):
......@@ -22,6 +92,6 @@
def get_var(key):
key = f'simple-command.{key}'
return poulpe.get_one_value(benchmark_data, key)
return benchmark_data.get_var(key)
self.base_command = get_var('command')
......@@ -26,5 +96,6 @@
self.base_command = get_var('command')
assert self.base_command is not None
self.command = None
# XXX having a list of acceptable status would be better (default to [0])
......@@ -33,7 +104,7 @@
self.cwd = get_var('cwd')
def apply_data_env(self, data_env_data):
variables_data = self._benchmark_data['simple-command'].get('variables')
variables_data = self._benchmark_data.get_var('simple-command').get('variables')
def get_var(key):
full_key = f'bench-input-vars.{key}'
......@@ -70,6 +141,6 @@
data_env_data = poulpe.get_data(data_env_desc)
result_data['data-env-vars'] = data_env_data['data-env-vars']
benchmark_data = poulpe.get_data(benchmark)
benchmark_data = get_benchmark(benchmark)
assert benchmark_data is not None, benchmark
result_data['benchmark'] = {}
......@@ -74,6 +145,9 @@
assert benchmark_data is not None, benchmark
result_data['benchmark'] = {}
result_data['benchmark']['name'] = benchmark_data['meta']['name']
result_data['benchmark']['name'] = benchmark_data.name
variants = benchmark_data.selected_variants
if variants:
result_data['benchmark']['variants'] = variants
# building the benchmark scenarion, that will likely changes often and quickly
......
......@@ -35,7 +35,11 @@
A `xxx.pdb` file that describe how to run a specific benchmark and its variant.
The variable from the "data-envs" can be used to configure the benchmark.
The variable from the "data-env" can be used to configure the benchmark.
Benchmarks can have variants. Variants exists in independent **dimensions**.
Each dimension has a set of values, each value can modify the benchmark. Each
dimension must have a default value.
Running benchmarks
==================
......
Check we have enough simple pieces together to do a simple run
--------------------------------------------------------------
$ PATH="${TESTDIR}/../bin:$PATH"
Setup the bin-env
-----------------
$ BLACK_VERSION="18.6b4" bin-env-util setup-one bin-env-black-18.6b4 \
> $TESTDIR/test-data/setup-black.sh
$ bin-env-util show bin-env-black-18.6b4
bin-env-vars:
black:
install-method = pip
version = 18.6b4
python:
version = 3.* (glob)
poulpe-environment:
environment-type = binary
format-version = 0
setup-method = script
ready = True
Setup a data-env
----------------
(currently built by hand as it simple and mostly innert)
$ mkdir data-env
$ env-desc set data-env/data-env.poulpe poulpe-environment.environment-type data
creating new file: "data-env/data-env.poulpe"
$ env-desc set data-env/data-env.poulpe poulpe-environment.format-version 0
$ env-desc set data-env/data-env.poulpe poulpe-environment.setup-method manual
$ env-desc set data-env/data-env.poulpe data-env-vars.name black-bench
$ mkdir data-env/py-files
$ cat << EOF > data-env/py-files/good.py
> foo = [1, 2, 3, 4, 5]
> EOF
$ cat << EOF > data-env/py-files/bad.py
> foo = [1,
> 2,
> 3,
> 4,
> 5]
> EOF
$ env-desc set data-env/data-env.poulpe bench-input-vars.black.check.tiny.good py-files/good.py
$ env-desc set data-env/data-env.poulpe bench-input-vars.black.check.tiny.bad py-files/bad.py
(that one will be a string, so its not good.)
$ env-desc set data-env/data-env.poulpe ready 1
$ env-desc show data-env/data-env.poulpe
bench-input-vars:
black:
check:
tiny:
bad = py-files/bad.py
good = py-files/good.py
data-env-vars:
name = black-bench
poulpe-environment:
environment-type = data
format-version = 0
setup-method = manual
ready = 1
Define a benchmark
------------------
$ env-desc set black-tiny-bad.pbd meta.format 0
creating new file: "black-tiny-bad.pbd"
$ env-desc set black-tiny-bad.pbd meta.name black.check.tiny
$ env-desc set black-tiny-bad.pbd meta.method simple-command
$ env-desc set black-tiny-bad.pbd simple-command.command "black --check {file}"
$ env-desc set black-tiny-bad.pbd simple-command.accept-failure "1"
$ env-desc set black-tiny-bad.pbd simple-command.variables.file DATA-VARS:black.check.tiny.bad
$ env-desc set black-tiny-bad.pbd simple-command.cwd DATA-VARS:.
$ env-desc set --bool black-tiny-bad.pbd simple-command.variants.dimensions.string-normalization.yes.default 1
$ env-desc set black-tiny-bad.pbd simple-command.variants.dimensions.string-normalization.no.extend-command " --skip-string-normalization"
$ env-desc show black-tiny-bad.pbd
meta:
format = 0
method = simple-command
name = black.check.tiny
simple-command:
accept-failure = 1
command = black --check {file}
cwd = DATA-VARS:.
variables:
file = DATA-VARS:black.check.tiny.bad
variants:
dimensions:
string-normalization:
no:
extend-command = --skip-string-normalization
yes:
default = True
Run the benchmark with default variant
--------------------------------------
$ run-util bin-env-black-18.6b4 data-env black-tiny-bad.pbd test-result-18.6b4-bad.pbr
$ env-desc show test-result-18.6b4-bad.pbr
benchmark:
name = black.check.tiny
variants:
string-normalization = yes
bin-env-vars:
black:
install-method = pip
version = 18.6b4
python:
version = 3.* (glob)
data-env-vars:
name = black-bench
result:
time:
max = * (glob)
mean = * (glob)
median = * (glob)
min = * (glob)
standard-deviation = * (glob)
run:
duration = * (glob)
timestamp = * (glob)
Run the benchmark with a selected variant
-----------------------------------------
$ run-util bin-env-black-18.6b4 data-env 'black-tiny-bad.pbd;string-normalization=no' test-result-18.6b4-bad-no-string.pbr
$ env-desc show test-result-18.6b4-bad-no-string.pbr
benchmark:
name = black.check.tiny
variants:
string-normalization = no
bin-env-vars:
black:
install-method = pip
version = 18.6b4
python:
version = 3.* (glob)
data-env-vars:
name = black-bench
result:
time:
max = * (glob)
mean = * (glob)
median = * (glob)
min = * (glob)
standard-deviation = * (glob)
run:
duration = * (glob)
timestamp = * (glob)
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment