# HG changeset patch # User Pierre-Yves David <pierre-yves.david@octobus.net> # Date 1679625657 0 # Fri Mar 24 02:40:57 2023 +0000 # Node ID 5f01c23e26e62b132b48cae8f1eb9977fe9396c3 # Parent 021dc2cdec6e761dc27fbb6fea204459d70085d7 runner: add a --debug mode The debug mode display information about the run, the run output, and reduce the number of run to only two diff --git a/python-libs/poulpe/benchmarks.py b/python-libs/poulpe/benchmarks.py --- a/python-libs/poulpe/benchmarks.py +++ b/python-libs/poulpe/benchmarks.py @@ -1,6 +1,7 @@ import json import os import subprocess +import sys import tempfile from . import ( @@ -197,7 +198,7 @@ # XXX missing the "variable" part return variables - def run_one(self, bin_env_path, data_env_path): + def run_one(self, bin_env_path, data_env_path, debug=False): data_env = data_mod.get_data_env(data_env_path) cmd = SimpleCommand(self) @@ -205,7 +206,7 @@ prepare = self.get_var('simple-command.prepare-run') - r = self._time_command(bin_env_path, data_env_path, cmd, prepare) + r = self._time_command(bin_env_path, data_env_path, cmd, prepare, debug=debug) res = {} # we should store more res['time'] = {} @@ -216,7 +217,7 @@ res['time']['max'] = r['results'][0]['max'] return res - def _time_command(self, bin_env_path, data_env_path, cmd, prepare=None): + def _time_command(self, bin_env_path, data_env_path, cmd, prepare=None, debug=False): bin_env_path = os.path.abspath(bin_env_path) data_env_path = os.path.abspath(data_env_path) shell_path = poulpe.bin_env_script(bin_env_path) @@ -237,22 +238,39 @@ if cmd.accept_failure: time_cmd.append("--ignore-failure") + if debug: + time_cmd.append('--runs') + time_cmd.append('2') + if prepare is not None: + prepare = f'echo "### starting prepare ###" ; date ; {prepare}' + if prepare is not None: time_cmd.append('--prepare') time_cmd.append(prepare) + command = cmd.command + if debug: + command = f'echo "### starting command ###" ; date ; {command}' + time_cmd.append("--") - time_cmd.append(cmd.command) + time_cmd.append(command) cwd = data_env_path if cmd.cwd is not None: cwd = os.path.join(cwd, cmd.cwd) + + if debug: + stdout=sys.stdout + stderr=sys.stderr + else: + stdout = subprocess.PIPE + stderr = subprocess.PIPE r = subprocess.run( time_cmd, cwd=cwd, env=env, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, + stdout=stdout, + stderr=stderr ) if r.returncode != 0: raise errors.BenchmarkRunFailure( @@ -370,7 +388,7 @@ variables.add(key) return variables - def run_one(self, bin_env_path, data_env_path): + def run_one(self, bin_env_path, data_env_path, debug=False): bin_env_path = os.path.abspath(bin_env_path) data_env_path = os.path.abspath(data_env_path) shell_path = poulpe.bin_env_script(bin_env_path) @@ -437,12 +455,18 @@ if args: cmd.extend(args) + if debug: + stdout=sys.stdout + stderr=sys.stderr + else: + stdout = subprocess.PIPE + stderr = subprocess.PIPE r = subprocess.run( cmd, cwd=cwd, env=env, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, + stdout=stdout, + stderr=stderr, ) if r.returncode != 0: raise errors.BenchmarkRunFailure( diff --git a/python-libs/poulpe/bin/run-util b/python-libs/poulpe/bin/run-util --- a/python-libs/poulpe/bin/run-util +++ b/python-libs/poulpe/bin/run-util @@ -7,8 +7,9 @@ @click.argument("bin_env") @click.argument('data_env') @click.argument('benchmark') +@click.option("--debug", is_flag=True) @click.argument('result') -def run_util(bin_env, data_env, benchmark, result): +def run_util(bin_env, data_env, benchmark, result, debug=False): """Run an "atomic" benchmark unit from already setup environments. \b @@ -23,6 +24,7 @@ data_env, benchmark, result, + debug=debug, ) return ret diff --git a/python-libs/poulpe/runner.py b/python-libs/poulpe/runner.py --- a/python-libs/poulpe/runner.py +++ b/python-libs/poulpe/runner.py @@ -36,7 +36,7 @@ shutil.rmtree(tmp_data, ignore_errors=True) -def run_one_core(bin_env_path, data_env_path, benchmark_path): +def run_one_core(bin_env_path, data_env_path, benchmark_path, debug=False): result_data = {} result_data['run'] = {} result_data['run']["timestamp"] = time.time() @@ -68,7 +68,7 @@ result_data['data-env-vars'] = data_env_data.data_env_vars try: - bench_result = benchmark.run_one(bin_env_path, data_env_path) + bench_result = benchmark.run_one(bin_env_path, data_env_path, debug=debug) result_data['result'] = bench_result finally: cleanup_data_env(tmp_data) @@ -77,12 +77,13 @@ return result_data -def run_one(bin_env_path, data_env_path, benchmark_path, result): +def run_one(bin_env_path, data_env_path, benchmark_path, result, debug=False): try: result_data = run_one_core( bin_env_path, data_env_path, benchmark_path, + debug=debug, ) except errors.MissingDataEnvInputVars as exc: # TODO it's not the lib's responsibility to write to stderr diff --git a/tests/test-run-debug.t b/tests/test-run-debug.t new file mode 100644 --- /dev/null +++ b/tests/test-run-debug.t @@ -0,0 +1,164 @@ + + +Check we have enough simple pieces together to do a simple run +-------------------------------------------------------------- + + $ python -m venv $TESTTMP/ + $ . $TESTTMP/bin/activate + $ pip install $TESTDIR/.. --quiet + +Setup the bin-env +----------------- + + $ BLACK_VERSION="18.6b4" poulpe bin-env-util setup-one bin-env-black-18.6b4 \ + > $TESTDIR/test-data/setup-black.sh + $ poulpe bin-env-util show bin-env-black-18.6b4 + bin-env-vars: + black: + install-method = pip + version = 18.6b4 + python: + version = 3.* (glob) + poulpe-environment: + environment-type = binary + format-version = 0 + setup-method = script + ready = True + + +Setup a data-env +---------------- + +(currently built by hand as it simple and mostly innert) + + $ mkdir data-env + $ poulpe env-desc set data-env/data-env.poulpe poulpe-environment.environment-type data + creating new file: "data-env/data-env.poulpe" + $ poulpe env-desc set data-env/data-env.poulpe poulpe-environment.format-version 0 + $ poulpe env-desc set data-env/data-env.poulpe poulpe-environment.setup-method manual + $ poulpe env-desc set data-env/data-env.poulpe data-env-vars.name black-bench + $ mkdir data-env/py-files + $ cat << EOF > data-env/py-files/good.py + > foo = [1, 2, 3, 4, 5] + > EOF + $ cat << EOF > data-env/py-files/bad.py + > foo = [1, + > 2, + > 3, + > 4, + > 5] + > EOF + $ poulpe env-desc set data-env/data-env.poulpe bench-input-vars.black.check.tiny.good py-files/good.py + $ poulpe env-desc set data-env/data-env.poulpe bench-input-vars.black.check.tiny.bad py-files/bad.py + +(that one will be a string, so its not good.) + + $ poulpe env-desc set data-env/data-env.poulpe ready 1 + + $ poulpe env-desc show data-env/data-env.poulpe + bench-input-vars: + black: + check: + tiny: + bad = py-files/bad.py + good = py-files/good.py + data-env-vars: + name = black-bench + poulpe-environment: + environment-type = data + format-version = 0 + setup-method = manual + ready = 1 + +Define a benchmark +------------------ + + $ poulpe env-desc set black-tiny-bad.pbd meta.format 0 + creating new file: "black-tiny-bad.pbd" + $ poulpe env-desc set black-tiny-bad.pbd meta.name black.check.tiny + $ poulpe env-desc set black-tiny-bad.pbd meta.method simple-command + $ poulpe env-desc set black-tiny-bad.pbd simple-command.command "black --check {file}" + $ poulpe env-desc set black-tiny-bad.pbd simple-command.accept-failure "1" + $ poulpe env-desc set black-tiny-bad.pbd simple-command.variables.file DATA-VARS:black.check.tiny.bad + $ poulpe env-desc set black-tiny-bad.pbd simple-command.cwd . + + $ poulpe env-desc show black-tiny-bad.pbd + meta: + format = 0 + method = simple-command + name = black.check.tiny + simple-command: + accept-failure = 1 + command = black --check {file} + cwd = . + variables: + file = DATA-VARS:black.check.tiny.bad + +Run the benchmark +----------------- + $ poulpe run-util bin-env-black-18.6b4 data-env black-tiny-bad.pbd test-result-18.6b4-bad.pbr + $ poulpe env-desc show test-result-18.6b4-bad.pbr + benchmark: + name = black.check.tiny + bin-env-vars: + black: + install-method = pip + version = 18.6b4 + python: + version = 3.* (glob) + data-env-vars: + name = black-bench + result: + time: + max = * (glob) + mean = * (glob) + median = * (glob) + min = * (glob) + standard-deviation = * (glob) + run: + duration = * (glob) + timestamp = * (glob) + +Run the benchmark with --debug +------------------------------ + +It should display details about the run and limit the number of iteration + + $ poulpe run-util --debug bin-env-black-18.6b4 data-env black-tiny-bad.pbd test-result-18.6b4-bad.pbr + Benchmark #1: echo "### starting command ###" ; date ; black --check py-files/bad.py + ### starting command ### + ??? ??? * (glob) + would reformat py-files/bad.py + All done! * (glob) + 1 file would be reformatted. + ### starting command ### + ??? ??? * (glob) + would reformat py-files/bad.py + All done! * (glob) + 1 file would be reformatted. + Time * (glob) + Range * 2 runs (glob) + + Warning: Ignoring non-zero exit code. + + $ poulpe env-desc show test-result-18.6b4-bad.pbr + benchmark: + name = black.check.tiny + bin-env-vars: + black: + install-method = pip + version = 18.6b4 + python: + version = 3.* (glob) + data-env-vars: + name = black-bench + result: + time: + max = * (glob) + mean = * (glob) + median = * (glob) + min = * (glob) + standard-deviation = * (glob) + run: + duration = * (glob) + timestamp = * (glob)