# HG changeset patch
# User Boris Feld <boris.feld@octobus.net>
# Date 1548360006 18000
#      Thu Jan 24 15:00:06 2019 -0500
# Node ID 2e0076fa9c33b4d4a8283dd1dd90554492c78b76
# Parent  4f897764425c4026c21117ae0c6ff673e847d0bc
repo-script: rewrite make-reference in Python

The implementation and outputs are kept as close as possible from the previous
bash implementation.

diff --git a/repo-scripts/make-reference b/repo-scripts/make-reference
--- a/repo-scripts/make-reference
+++ b/repo-scripts/make-reference
@@ -1,66 +1,117 @@
-#!/bin/bash
-set -euo pipefail
-
-# This make a standard clone, register some data about it, make sure the
-# cache are warm and build a tarball.
-
-# Possible extensions:
-# * allow optimization (maybe from another script)
-# * records more format data (requirement and config)
+#!/usr/bin/env python
 
-if [ $# -lt 2 ]; then
-     echo 'usage: make-reference SOURCE ID'
-     exit 64
-fi
+import argparse
+import os
+import os.path
+import shutil
+import subprocess
+import sys
+import tarfile
+from glob import glob
+from os.path import join
 
-repo_source="$1"
-repo_id="$2"
+import yaml
 
-repo_main="${repo_id}"-reference
-repo_details="${repo_id}".benchrepo
-repo_tarball="${repo_id}"-reference.tar
 
-### initial clone
-
-if [ -d "${repo_main}"/.hg/ ]; then
-    echo 'target repo already exists, skipping clone'
-    echo '(make sure this was clone with the right version)'
-    rm -rf "${repo_main}"/.hg/upgradebackup*
-else
-    echo 'hg cloning repository from:' "${repo_source}"
-    hg clone --noupdate "${repo_source}" "${repo_main}"
-fi
+def shell_exec(command, env=None):
+    # Simulate bash script printing the commands it runs
+    process = subprocess.Popen(
+        command, stdout=subprocess.PIPE, shell=False, env=env
+    )
+    output, unused_err = process.communicate()
+    retcode = process.poll()
+    if retcode:
+        raise subprocess.CalledProcessError(retcode, command)
+    return output
 
-### Generate various data
 
-hg -R "${repo_main}" debugupdatecache
-
-hgversion=$(hg version -T '{ver}\n')
-numrevs=$(hg -R "${repo_main}" log -T '{rev}\n' | wc -l)
-numrevshidden=$(hg --hidden -R "${repo_main}" log -T '{rev}\n' | wc -l)
-numheads=$(hg -R "${repo_main}" heads -t -T '{rev}\n' | wc -l)
-numheadshidden=$(hg --hidden -R "${repo_main}" heads -t -T '{rev}\n' | wc -l)
-
-numbranches=$(hg -R "${repo_main}" branches -c -T '{rev}\n' | wc -l)
-numbrancheshidden=$(hg --hidden -R "${repo_main}" branches -c -T '{rev}\n' | wc -l)
+def count_line_exec(command, env=None):
+    output = shell_exec(command, env)
+    return len(output.splitlines())
 
 
-echo 'reference-repo:' > "${repo_details}"
-echo '  id:' "${repo_id}" >> "${repo_details}"
-echo '  enabled: Yes' >> "${repo_details}"
-echo '  source:' "${repo_source}" >> "${repo_details}"
-echo '  hg-version:' "${hgversion}" >> "${repo_details}"
-echo '  number-revisions:' >> "${repo_details}"
-echo '    visible:' $numrevs >> "${repo_details}"
-echo '    all:' $numrevshidden >> "${repo_details}"
-echo '  number-heads:' >> "${repo_details}"
-echo '    visible:' $numheads >> "${repo_details}"
-echo '    all:' $numheadshidden >> "${repo_details}"
-echo '  number-names-branch:' >> "${repo_details}"
-echo '    visible:' $numbranches >> "${repo_details}"
-echo '    all:' $numbranches >> "${repo_details}"
+def main(args):
+    parser = argparse.ArgumentParser()
+    parser.add_argument("source")
+    parser.add_argument("id")
+    args = parser.parse_args(args)
+
+    repo_source = args.source
+    repo_id = args.id
+
+    repo_main = "{0}-reference".format(repo_id)
+    repo_details = "{0}.benchrepo".format(repo_id)
+    repo_tarball = "{0}-reference.tar".format(repo_id)
+
+    repo_hg = join(repo_main, ".hg")
+
+    ### initial clone
+
+    if os.path.isdir(repo_hg):
+        print("target repo already exists, skipping clone")
+        print("(make sure this was clone with the right version)")
+        for backup_directory in glob(join(repo_hg, "upgradebackup*")):
+            if os.path.isdir(join(repo_hg, backup_directory)):
+                shutil.rmtree(join(repo_hg, backup_directory))
+            else:
+                os.remove(join(repo_hg, backup_directory))
+
+    else:
+        print("hg cloning repository from: {0}".format(repo_source))
+        shell_exec(["hg", "clone", "--noupdate", repo_source, repo_main])
+
+    ### Clean caches
+    shell_exec(["hg", "-R", repo_main, "debugupdatecache"])
+
+    ### Compute stats
+    hgversion = shell_exec(["hg", "version", "-T", "{ver}"])
+
+    numrevs = count_line_exec(["hg", "-R", repo_main, "log", "-T", "{rev}\n"])
+    numrevshidden = count_line_exec(
+        ["hg", "-R", repo_main, "--hidden", "log", "-T", "{rev}\n"]
+    )
 
-### final tarball
-echo 'building a reference tarball'
-tar cf "${repo_tarball}" "${repo_details}" "${repo_main}"
-echo "result available at:" "${repo_tarball}"
+    numheads = count_line_exec(
+        ["hg", "-R", repo_main, "heads", "-t", "-T", "{rev}\n"]
+    )
+    numheadshidden = count_line_exec(
+        ["hg", "-R", repo_main, "--hidden", "heads", "-t", "-T", "{rev}\n"]
+    )
+
+    numbranches = count_line_exec(
+        ["hg", "-R", repo_main, "branches", "-c", "-T", "{rev}\n"]
+    )
+    numbrancheshidden = count_line_exec(
+        ["hg", "-R", repo_main, "--hidden", "branches", "-c", "-T", "{rev}\n"]
+    )
+
+    # Generate YAML file
+    data = {
+        "reference-repo": {
+            "id": repo_id,
+            "enabled": True,
+            "source": repo_source,
+            "hg-version": hgversion,
+            "number-revisions": {"visible": numrevs, "all": numrevshidden},
+            "number-heads": {"visible": numheads, "all": numheadshidden},
+            "number-named-branch": {
+                "visible": numbranches,
+                "all": numbrancheshidden,
+            },
+        }
+    }
+
+    # Write data to the benchrepo file
+    with open(repo_details, "w") as repo_details_file:
+        repo_details_file.write(yaml.dump(data, default_flow_style=False))
+
+    # Tar everything
+    print("building a reference tarball")
+    with tarfile.open(repo_tarball, "w") as repo_tar_file:
+        repo_tar_file.add(repo_details)
+        repo_tar_file.add(repo_main)
+    print("result available at: {0}".format(repo_tarball))
+
+
+if __name__ == "__main__":
+    main(sys.argv[1:])
diff --git a/tests/test-make-all.t b/tests/test-make-all.t
--- a/tests/test-make-all.t
+++ b/tests/test-make-all.t
@@ -42,16 +42,16 @@
   testrepo-reference.tar
   $ tar -axf testrepo-reference.tar testrepo.benchrepo -O
   reference-repo:
+    enabled: true
+    hg-version: * (glob)
     id: testrepo
-    enabled: Yes
-    source: some-dir/my-source-repo
-    hg-version: * (glob)
+    number-heads:
+      all: 3
+      visible: 3
+    number-named-branch:
+      all: 1
+      visible: 1
     number-revisions:
-      visible: 53
       all: 53
-    number-heads:
-      visible: 3
-      all: 3
-    number-names-branch:
-      visible: 1
-      all: 1
+      visible: 53
+    source: some-dir/my-source-repo
diff --git a/tests/test-make-partials.t b/tests/test-make-partials.t
--- a/tests/test-make-partials.t
+++ b/tests/test-make-partials.t
@@ -156,10 +156,10 @@
   $ cd partials-disabled
 
   $ grep enabled ../local-references/repo4test_A.benchrepo
-    enabled: Yes
-  $ sed -i 's/enabled: Yes/enabled: No/' ../local-references/repo4test_A.benchrepo
+    enabled: true
+  $ sed -i 's/enabled: true/enabled: false/' ../local-references/repo4test_A.benchrepo
   $ grep enabled ../local-references/repo4test_A.benchrepo
-    enabled: No
+    enabled: false
   $ make-partial-repos ../p-config.yaml ../local-references/*.benchrepo
   skipping disabled reference repository: repo4test_A
   Cloning repo4test_B (../local-references/repo4test_B-reference stripped of last(all(), 10)) into ./repo4test_B-partial-missing-last-10 (for exchange benchmarks)
@@ -171,7 +171,7 @@
   repo4test_B-partial-same.tar
   $ sed -i 's/enabled: No/enabled: Yes/' ../local-references/repo4test_A.benchrepo
   $ grep enabled ../local-references/repo4test_A.benchrepo
-    enabled: Yes
+    enabled: false
   $ cd ..
 
 Test buiding partial from a special format (one)
diff --git a/tests/test-make-reference.t b/tests/test-make-reference.t
--- a/tests/test-make-reference.t
+++ b/tests/test-make-reference.t
@@ -14,14 +14,17 @@
 ----------
 
   $ make-reference
-  usage: make-reference SOURCE ID
-  [64]
+  usage: make-reference [-h] source id
+  make-reference: error: too few arguments
+  [2]
   $ make-reference some-dir/my-source-repo
-  usage: make-reference SOURCE ID
-  [64]
+  usage: make-reference [-h] source id
+  make-reference: error: too few arguments
+  [2]
   $ make-reference my-id
-  usage: make-reference SOURCE ID
-  [64]
+  usage: make-reference [-h] source id
+  make-reference: error: too few arguments
+  [2]
 
 Call from scratch
 -----------------
@@ -37,19 +40,19 @@
   some-dir
   $ cat my-reference.benchrepo
   reference-repo:
+    enabled: true
+    hg-version: * (glob)
     id: my-reference
-    enabled: Yes
-    source: some-dir/my-source-repo
-    hg-version: ?.* (glob)
+    number-heads:
+      all: 3
+      visible: 3
+    number-named-branch:
+      all: 1
+      visible: 1
     number-revisions:
-      visible: 53
       all: 53
-    number-heads:
-      visible: 3
-      all: 3
-    number-names-branch:
-      visible: 1
-      all: 1
+      visible: 53
+    source: some-dir/my-source-repo
 
 Call with existing data
 -----------------
@@ -67,19 +70,19 @@
   some-dir
   $ cat my-reference.benchrepo
   reference-repo:
+    enabled: true
+    hg-version: * (glob)
     id: my-reference
-    enabled: Yes
-    source: some-dir/my-source-repo
-    hg-version: ?.* (glob)
+    number-heads:
+      all: 3
+      visible: 3
+    number-named-branch:
+      all: 1
+      visible: 1
     number-revisions:
-      visible: 53
       all: 53
-    number-heads:
-      visible: 3
-      all: 3
-    number-names-branch:
-      visible: 1
-      all: 1
+      visible: 53
+    source: some-dir/my-source-repo
   $ tar tf my-reference-reference.tar | sort
   my-reference-reference/
   my-reference-reference/.hg/
@@ -155,19 +158,19 @@
 
   $ EDITOR=cat update-reference-config my-reference-reference.tar
   reference-repo:
+    enabled: true
+    hg-version: * (glob)
     id: my-reference
-    enabled: Yes
-    source: some-dir/my-source-repo
-    hg-version: * (glob)
+    number-heads:
+      all: 3
+      visible: 3
+    number-named-branch:
+      all: 1
+      visible: 1
     number-revisions:
-      visible: 53
       all: 53
-    number-heads:
-      visible: 3
-      all: 3
-    number-names-branch:
-      visible: 1
-      all: 1
+      visible: 53
+    source: some-dir/my-source-repo
   $ cat << EOF > editor.sh
   > #!/bin/bash
   > sed -i 's,some-dir/my-source-repo,some-dir/my-other-source-repo,' \$1
@@ -179,16 +182,16 @@
   $ tar xf ../my-reference-reference.tar my-reference.benchrepo
   $ cat my-reference.benchrepo
   reference-repo:
+    enabled: true
+    hg-version: * (glob)
     id: my-reference
-    enabled: Yes
-    source: some-dir/my-other-source-repo
-    hg-version: * (glob)
+    number-heads:
+      all: 3
+      visible: 3
+    number-named-branch:
+      all: 1
+      visible: 1
     number-revisions:
-      visible: 53
       all: 53
-    number-heads:
-      visible: 3
-      all: 3
-    number-names-branch:
-      visible: 1
-      all: 1
+      visible: 53
+    source: some-dir/my-other-source-repo