# HG changeset patch
# User Pierre-Yves David <pierre-yves.david@octobus.net>
# Date 1679625657 0
#      Fri Mar 24 02:40:57 2023 +0000
# Node ID 5f01c23e26e62b132b48cae8f1eb9977fe9396c3
# Parent  021dc2cdec6e761dc27fbb6fea204459d70085d7
runner: add a --debug mode

The debug mode display information about the run, the run output, and reduce the
number of run to only two

diff --git a/python-libs/poulpe/benchmarks.py b/python-libs/poulpe/benchmarks.py
--- a/python-libs/poulpe/benchmarks.py
+++ b/python-libs/poulpe/benchmarks.py
@@ -1,6 +1,7 @@
 import json
 import os
 import subprocess
+import sys
 import tempfile
 
 from . import (
@@ -197,7 +198,7 @@
         # XXX missing the "variable" part
         return variables
 
-    def run_one(self, bin_env_path, data_env_path):
+    def run_one(self, bin_env_path, data_env_path, debug=False):
         data_env = data_mod.get_data_env(data_env_path)
 
         cmd = SimpleCommand(self)
@@ -205,7 +206,7 @@
 
         prepare = self.get_var('simple-command.prepare-run')
 
-        r = self._time_command(bin_env_path, data_env_path, cmd, prepare)
+        r = self._time_command(bin_env_path, data_env_path, cmd, prepare, debug=debug)
         res = {}
         # we should store more
         res['time'] = {}
@@ -216,7 +217,7 @@
         res['time']['max'] = r['results'][0]['max']
         return res
 
-    def _time_command(self, bin_env_path, data_env_path, cmd, prepare=None):
+    def _time_command(self, bin_env_path, data_env_path, cmd, prepare=None, debug=False):
         bin_env_path = os.path.abspath(bin_env_path)
         data_env_path = os.path.abspath(data_env_path)
         shell_path = poulpe.bin_env_script(bin_env_path)
@@ -237,22 +238,39 @@
             if cmd.accept_failure:
                 time_cmd.append("--ignore-failure")
 
+            if debug:
+                time_cmd.append('--runs')
+                time_cmd.append('2')
+                if prepare is not None:
+                    prepare = f'echo "### starting prepare ###" ; date ; {prepare}'
+
             if prepare is not None:
                 time_cmd.append('--prepare')
                 time_cmd.append(prepare)
 
+            command = cmd.command
+            if debug:
+                command = f'echo "### starting command ###" ; date ; {command}'
+
             time_cmd.append("--")
-            time_cmd.append(cmd.command)
+            time_cmd.append(command)
 
             cwd = data_env_path
             if cmd.cwd is not None:
                 cwd = os.path.join(cwd, cmd.cwd)
+
+            if debug:
+                stdout=sys.stdout
+                stderr=sys.stderr
+            else:
+                stdout = subprocess.PIPE
+                stderr = subprocess.PIPE
             r = subprocess.run(
                 time_cmd,
                 cwd=cwd,
                 env=env,
-                stdout=subprocess.PIPE,
-                stderr=subprocess.PIPE,
+                stdout=stdout,
+                stderr=stderr
             )
             if r.returncode != 0:
                 raise errors.BenchmarkRunFailure(
@@ -370,7 +388,7 @@
                     variables.add(key)
         return variables
 
-    def run_one(self, bin_env_path, data_env_path):
+    def run_one(self, bin_env_path, data_env_path, debug=False):
         bin_env_path = os.path.abspath(bin_env_path)
         data_env_path = os.path.abspath(data_env_path)
         shell_path = poulpe.bin_env_script(bin_env_path)
@@ -437,12 +455,18 @@
         if args:
             cmd.extend(args)
 
+        if debug:
+            stdout=sys.stdout
+            stderr=sys.stderr
+        else:
+            stdout = subprocess.PIPE
+            stderr = subprocess.PIPE
         r = subprocess.run(
             cmd,
             cwd=cwd,
             env=env,
-            stdout=subprocess.PIPE,
-            stderr=subprocess.PIPE,
+            stdout=stdout,
+            stderr=stderr,
         )
         if r.returncode != 0:
             raise errors.BenchmarkRunFailure(
diff --git a/python-libs/poulpe/bin/run-util b/python-libs/poulpe/bin/run-util
--- a/python-libs/poulpe/bin/run-util
+++ b/python-libs/poulpe/bin/run-util
@@ -7,8 +7,9 @@
 @click.argument("bin_env")
 @click.argument('data_env')
 @click.argument('benchmark')
+@click.option("--debug", is_flag=True)
 @click.argument('result')
-def run_util(bin_env, data_env, benchmark, result):
+def run_util(bin_env, data_env, benchmark, result, debug=False):
     """Run an "atomic" benchmark unit from already setup environments.
 
     \b
@@ -23,6 +24,7 @@
         data_env,
         benchmark,
         result,
+        debug=debug,
     )
     return ret
 
diff --git a/python-libs/poulpe/runner.py b/python-libs/poulpe/runner.py
--- a/python-libs/poulpe/runner.py
+++ b/python-libs/poulpe/runner.py
@@ -36,7 +36,7 @@
         shutil.rmtree(tmp_data, ignore_errors=True)
 
 
-def run_one_core(bin_env_path, data_env_path, benchmark_path):
+def run_one_core(bin_env_path, data_env_path, benchmark_path, debug=False):
     result_data = {}
     result_data['run'] = {}
     result_data['run']["timestamp"] = time.time()
@@ -68,7 +68,7 @@
     result_data['data-env-vars'] = data_env_data.data_env_vars
 
     try:
-        bench_result = benchmark.run_one(bin_env_path, data_env_path)
+        bench_result = benchmark.run_one(bin_env_path, data_env_path, debug=debug)
         result_data['result'] = bench_result
     finally:
         cleanup_data_env(tmp_data)
@@ -77,12 +77,13 @@
     return result_data
 
 
-def run_one(bin_env_path, data_env_path, benchmark_path, result):
+def run_one(bin_env_path, data_env_path, benchmark_path, result, debug=False):
     try:
         result_data = run_one_core(
             bin_env_path,
             data_env_path,
             benchmark_path,
+            debug=debug,
         )
     except errors.MissingDataEnvInputVars as exc:
         # TODO it's not the lib's responsibility to write to stderr
diff --git a/tests/test-run-debug.t b/tests/test-run-debug.t
new file mode 100644
--- /dev/null
+++ b/tests/test-run-debug.t
@@ -0,0 +1,164 @@
+
+
+Check we have enough simple pieces together to do a simple run
+--------------------------------------------------------------
+
+  $ python -m venv $TESTTMP/
+  $ . $TESTTMP/bin/activate
+  $ pip install $TESTDIR/.. --quiet
+
+Setup the bin-env
+-----------------
+
+  $ BLACK_VERSION="18.6b4" poulpe bin-env-util setup-one  bin-env-black-18.6b4 \
+  > $TESTDIR/test-data/setup-black.sh
+  $ poulpe bin-env-util show bin-env-black-18.6b4
+  bin-env-vars:
+    black:
+      install-method = pip
+      version = 18.6b4
+    python:
+      version = 3.* (glob)
+  poulpe-environment:
+    environment-type = binary
+    format-version = 0
+    setup-method = script
+  ready = True
+
+
+Setup a data-env
+----------------
+
+(currently built by hand as it simple and mostly innert)
+
+  $ mkdir data-env
+  $ poulpe env-desc set data-env/data-env.poulpe poulpe-environment.environment-type data
+  creating new file: "data-env/data-env.poulpe"
+  $ poulpe env-desc set data-env/data-env.poulpe poulpe-environment.format-version 0
+  $ poulpe env-desc set data-env/data-env.poulpe poulpe-environment.setup-method manual
+  $ poulpe env-desc set data-env/data-env.poulpe data-env-vars.name black-bench
+  $ mkdir data-env/py-files
+  $ cat << EOF > data-env/py-files/good.py
+  > foo = [1, 2, 3, 4, 5]
+  > EOF
+  $ cat << EOF > data-env/py-files/bad.py
+  > foo = [1,
+  >    2,
+  >   3,
+  >     4,
+  >  5]
+  > EOF
+  $ poulpe env-desc set data-env/data-env.poulpe bench-input-vars.black.check.tiny.good py-files/good.py
+  $ poulpe env-desc set data-env/data-env.poulpe bench-input-vars.black.check.tiny.bad py-files/bad.py
+
+(that one will be a string, so its not good.)
+
+  $ poulpe env-desc set data-env/data-env.poulpe ready 1
+
+  $ poulpe env-desc show data-env/data-env.poulpe
+  bench-input-vars:
+    black:
+      check:
+        tiny:
+          bad = py-files/bad.py
+          good = py-files/good.py
+  data-env-vars:
+    name = black-bench
+  poulpe-environment:
+    environment-type = data
+    format-version = 0
+    setup-method = manual
+  ready = 1
+
+Define a benchmark
+------------------
+
+  $ poulpe env-desc set black-tiny-bad.pbd meta.format 0
+  creating new file: "black-tiny-bad.pbd"
+  $ poulpe env-desc set black-tiny-bad.pbd meta.name black.check.tiny
+  $ poulpe env-desc set black-tiny-bad.pbd meta.method simple-command
+  $ poulpe env-desc set black-tiny-bad.pbd simple-command.command "black --check {file}"
+  $ poulpe env-desc set black-tiny-bad.pbd simple-command.accept-failure "1"
+  $ poulpe env-desc set black-tiny-bad.pbd simple-command.variables.file DATA-VARS:black.check.tiny.bad
+  $ poulpe env-desc set black-tiny-bad.pbd simple-command.cwd .
+
+  $ poulpe env-desc show black-tiny-bad.pbd
+  meta:
+    format = 0
+    method = simple-command
+    name = black.check.tiny
+  simple-command:
+    accept-failure = 1
+    command = black --check {file}
+    cwd = .
+    variables:
+      file = DATA-VARS:black.check.tiny.bad
+
+Run the benchmark
+-----------------
+  $ poulpe run-util bin-env-black-18.6b4 data-env black-tiny-bad.pbd test-result-18.6b4-bad.pbr
+  $ poulpe env-desc show test-result-18.6b4-bad.pbr
+  benchmark:
+    name = black.check.tiny
+  bin-env-vars:
+    black:
+      install-method = pip
+      version = 18.6b4
+    python:
+      version = 3.* (glob)
+  data-env-vars:
+    name = black-bench
+  result:
+    time:
+      max = * (glob)
+      mean = * (glob)
+      median = * (glob)
+      min = * (glob)
+      standard-deviation = * (glob)
+  run:
+    duration = * (glob)
+    timestamp = * (glob)
+
+Run the benchmark with --debug
+------------------------------
+
+It should display details about the run and limit the number of iteration
+
+  $ poulpe run-util --debug bin-env-black-18.6b4 data-env black-tiny-bad.pbd test-result-18.6b4-bad.pbr
+  Benchmark #1: echo "### starting command ###" ; date ; black --check py-files/bad.py
+  ### starting command ###
+  ??? ??? * (glob)
+  would reformat py-files/bad.py
+  All done! * (glob)
+  1 file would be reformatted.
+  ### starting command ###
+  ??? ??? * (glob)
+  would reformat py-files/bad.py
+  All done! * (glob)
+  1 file would be reformatted.
+    Time * (glob)
+    Range * 2 runs (glob)
+   
+    Warning: Ignoring non-zero exit code.
+   
+  $ poulpe env-desc show test-result-18.6b4-bad.pbr
+  benchmark:
+    name = black.check.tiny
+  bin-env-vars:
+    black:
+      install-method = pip
+      version = 18.6b4
+    python:
+      version = 3.* (glob)
+  data-env-vars:
+    name = black-bench
+  result:
+    time:
+      max = * (glob)
+      mean = * (glob)
+      median = * (glob)
+      min = * (glob)
+      standard-deviation = * (glob)
+  run:
+    duration = * (glob)
+    timestamp = * (glob)