# HG changeset patch # User Boris Feld <boris.feld@octobus.net> # Date 1548360006 18000 # Thu Jan 24 15:00:06 2019 -0500 # Node ID 2e0076fa9c33b4d4a8283dd1dd90554492c78b76 # Parent 4f897764425c4026c21117ae0c6ff673e847d0bc repo-script: rewrite make-reference in Python The implementation and outputs are kept as close as possible from the previous bash implementation. diff --git a/repo-scripts/make-reference b/repo-scripts/make-reference --- a/repo-scripts/make-reference +++ b/repo-scripts/make-reference @@ -1,66 +1,117 @@ -#!/bin/bash -set -euo pipefail - -# This make a standard clone, register some data about it, make sure the -# cache are warm and build a tarball. - -# Possible extensions: -# * allow optimization (maybe from another script) -# * records more format data (requirement and config) +#!/usr/bin/env python -if [ $# -lt 2 ]; then - echo 'usage: make-reference SOURCE ID' - exit 64 -fi +import argparse +import os +import os.path +import shutil +import subprocess +import sys +import tarfile +from glob import glob +from os.path import join -repo_source="$1" -repo_id="$2" +import yaml -repo_main="${repo_id}"-reference -repo_details="${repo_id}".benchrepo -repo_tarball="${repo_id}"-reference.tar -### initial clone - -if [ -d "${repo_main}"/.hg/ ]; then - echo 'target repo already exists, skipping clone' - echo '(make sure this was clone with the right version)' - rm -rf "${repo_main}"/.hg/upgradebackup* -else - echo 'hg cloning repository from:' "${repo_source}" - hg clone --noupdate "${repo_source}" "${repo_main}" -fi +def shell_exec(command, env=None): + # Simulate bash script printing the commands it runs + process = subprocess.Popen( + command, stdout=subprocess.PIPE, shell=False, env=env + ) + output, unused_err = process.communicate() + retcode = process.poll() + if retcode: + raise subprocess.CalledProcessError(retcode, command) + return output -### Generate various data -hg -R "${repo_main}" debugupdatecache - -hgversion=$(hg version -T '{ver}\n') -numrevs=$(hg -R "${repo_main}" log -T '{rev}\n' | wc -l) -numrevshidden=$(hg --hidden -R "${repo_main}" log -T '{rev}\n' | wc -l) -numheads=$(hg -R "${repo_main}" heads -t -T '{rev}\n' | wc -l) -numheadshidden=$(hg --hidden -R "${repo_main}" heads -t -T '{rev}\n' | wc -l) - -numbranches=$(hg -R "${repo_main}" branches -c -T '{rev}\n' | wc -l) -numbrancheshidden=$(hg --hidden -R "${repo_main}" branches -c -T '{rev}\n' | wc -l) +def count_line_exec(command, env=None): + output = shell_exec(command, env) + return len(output.splitlines()) -echo 'reference-repo:' > "${repo_details}" -echo ' id:' "${repo_id}" >> "${repo_details}" -echo ' enabled: Yes' >> "${repo_details}" -echo ' source:' "${repo_source}" >> "${repo_details}" -echo ' hg-version:' "${hgversion}" >> "${repo_details}" -echo ' number-revisions:' >> "${repo_details}" -echo ' visible:' $numrevs >> "${repo_details}" -echo ' all:' $numrevshidden >> "${repo_details}" -echo ' number-heads:' >> "${repo_details}" -echo ' visible:' $numheads >> "${repo_details}" -echo ' all:' $numheadshidden >> "${repo_details}" -echo ' number-names-branch:' >> "${repo_details}" -echo ' visible:' $numbranches >> "${repo_details}" -echo ' all:' $numbranches >> "${repo_details}" +def main(args): + parser = argparse.ArgumentParser() + parser.add_argument("source") + parser.add_argument("id") + args = parser.parse_args(args) + + repo_source = args.source + repo_id = args.id + + repo_main = "{0}-reference".format(repo_id) + repo_details = "{0}.benchrepo".format(repo_id) + repo_tarball = "{0}-reference.tar".format(repo_id) + + repo_hg = join(repo_main, ".hg") + + ### initial clone + + if os.path.isdir(repo_hg): + print("target repo already exists, skipping clone") + print("(make sure this was clone with the right version)") + for backup_directory in glob(join(repo_hg, "upgradebackup*")): + if os.path.isdir(join(repo_hg, backup_directory)): + shutil.rmtree(join(repo_hg, backup_directory)) + else: + os.remove(join(repo_hg, backup_directory)) + + else: + print("hg cloning repository from: {0}".format(repo_source)) + shell_exec(["hg", "clone", "--noupdate", repo_source, repo_main]) + + ### Clean caches + shell_exec(["hg", "-R", repo_main, "debugupdatecache"]) + + ### Compute stats + hgversion = shell_exec(["hg", "version", "-T", "{ver}"]) + + numrevs = count_line_exec(["hg", "-R", repo_main, "log", "-T", "{rev}\n"]) + numrevshidden = count_line_exec( + ["hg", "-R", repo_main, "--hidden", "log", "-T", "{rev}\n"] + ) -### final tarball -echo 'building a reference tarball' -tar cf "${repo_tarball}" "${repo_details}" "${repo_main}" -echo "result available at:" "${repo_tarball}" + numheads = count_line_exec( + ["hg", "-R", repo_main, "heads", "-t", "-T", "{rev}\n"] + ) + numheadshidden = count_line_exec( + ["hg", "-R", repo_main, "--hidden", "heads", "-t", "-T", "{rev}\n"] + ) + + numbranches = count_line_exec( + ["hg", "-R", repo_main, "branches", "-c", "-T", "{rev}\n"] + ) + numbrancheshidden = count_line_exec( + ["hg", "-R", repo_main, "--hidden", "branches", "-c", "-T", "{rev}\n"] + ) + + # Generate YAML file + data = { + "reference-repo": { + "id": repo_id, + "enabled": True, + "source": repo_source, + "hg-version": hgversion, + "number-revisions": {"visible": numrevs, "all": numrevshidden}, + "number-heads": {"visible": numheads, "all": numheadshidden}, + "number-named-branch": { + "visible": numbranches, + "all": numbrancheshidden, + }, + } + } + + # Write data to the benchrepo file + with open(repo_details, "w") as repo_details_file: + repo_details_file.write(yaml.dump(data, default_flow_style=False)) + + # Tar everything + print("building a reference tarball") + with tarfile.open(repo_tarball, "w") as repo_tar_file: + repo_tar_file.add(repo_details) + repo_tar_file.add(repo_main) + print("result available at: {0}".format(repo_tarball)) + + +if __name__ == "__main__": + main(sys.argv[1:]) diff --git a/tests/test-make-all.t b/tests/test-make-all.t --- a/tests/test-make-all.t +++ b/tests/test-make-all.t @@ -42,16 +42,16 @@ testrepo-reference.tar $ tar -axf testrepo-reference.tar testrepo.benchrepo -O reference-repo: + enabled: true + hg-version: * (glob) id: testrepo - enabled: Yes - source: some-dir/my-source-repo - hg-version: * (glob) + number-heads: + all: 3 + visible: 3 + number-named-branch: + all: 1 + visible: 1 number-revisions: - visible: 53 all: 53 - number-heads: - visible: 3 - all: 3 - number-names-branch: - visible: 1 - all: 1 + visible: 53 + source: some-dir/my-source-repo diff --git a/tests/test-make-partials.t b/tests/test-make-partials.t --- a/tests/test-make-partials.t +++ b/tests/test-make-partials.t @@ -156,10 +156,10 @@ $ cd partials-disabled $ grep enabled ../local-references/repo4test_A.benchrepo - enabled: Yes - $ sed -i 's/enabled: Yes/enabled: No/' ../local-references/repo4test_A.benchrepo + enabled: true + $ sed -i 's/enabled: true/enabled: false/' ../local-references/repo4test_A.benchrepo $ grep enabled ../local-references/repo4test_A.benchrepo - enabled: No + enabled: false $ make-partial-repos ../p-config.yaml ../local-references/*.benchrepo skipping disabled reference repository: repo4test_A Cloning repo4test_B (../local-references/repo4test_B-reference stripped of last(all(), 10)) into ./repo4test_B-partial-missing-last-10 (for exchange benchmarks) @@ -171,7 +171,7 @@ repo4test_B-partial-same.tar $ sed -i 's/enabled: No/enabled: Yes/' ../local-references/repo4test_A.benchrepo $ grep enabled ../local-references/repo4test_A.benchrepo - enabled: Yes + enabled: false $ cd .. Test buiding partial from a special format (one) diff --git a/tests/test-make-reference.t b/tests/test-make-reference.t --- a/tests/test-make-reference.t +++ b/tests/test-make-reference.t @@ -14,14 +14,17 @@ ---------- $ make-reference - usage: make-reference SOURCE ID - [64] + usage: make-reference [-h] source id + make-reference: error: too few arguments + [2] $ make-reference some-dir/my-source-repo - usage: make-reference SOURCE ID - [64] + usage: make-reference [-h] source id + make-reference: error: too few arguments + [2] $ make-reference my-id - usage: make-reference SOURCE ID - [64] + usage: make-reference [-h] source id + make-reference: error: too few arguments + [2] Call from scratch ----------------- @@ -37,19 +40,19 @@ some-dir $ cat my-reference.benchrepo reference-repo: + enabled: true + hg-version: * (glob) id: my-reference - enabled: Yes - source: some-dir/my-source-repo - hg-version: ?.* (glob) + number-heads: + all: 3 + visible: 3 + number-named-branch: + all: 1 + visible: 1 number-revisions: - visible: 53 all: 53 - number-heads: - visible: 3 - all: 3 - number-names-branch: - visible: 1 - all: 1 + visible: 53 + source: some-dir/my-source-repo Call with existing data ----------------- @@ -67,19 +70,19 @@ some-dir $ cat my-reference.benchrepo reference-repo: + enabled: true + hg-version: * (glob) id: my-reference - enabled: Yes - source: some-dir/my-source-repo - hg-version: ?.* (glob) + number-heads: + all: 3 + visible: 3 + number-named-branch: + all: 1 + visible: 1 number-revisions: - visible: 53 all: 53 - number-heads: - visible: 3 - all: 3 - number-names-branch: - visible: 1 - all: 1 + visible: 53 + source: some-dir/my-source-repo $ tar tf my-reference-reference.tar | sort my-reference-reference/ my-reference-reference/.hg/ @@ -155,19 +158,19 @@ $ EDITOR=cat update-reference-config my-reference-reference.tar reference-repo: + enabled: true + hg-version: * (glob) id: my-reference - enabled: Yes - source: some-dir/my-source-repo - hg-version: * (glob) + number-heads: + all: 3 + visible: 3 + number-named-branch: + all: 1 + visible: 1 number-revisions: - visible: 53 all: 53 - number-heads: - visible: 3 - all: 3 - number-names-branch: - visible: 1 - all: 1 + visible: 53 + source: some-dir/my-source-repo $ cat << EOF > editor.sh > #!/bin/bash > sed -i 's,some-dir/my-source-repo,some-dir/my-other-source-repo,' \$1 @@ -179,16 +182,16 @@ $ tar xf ../my-reference-reference.tar my-reference.benchrepo $ cat my-reference.benchrepo reference-repo: + enabled: true + hg-version: * (glob) id: my-reference - enabled: Yes - source: some-dir/my-other-source-repo - hg-version: * (glob) + number-heads: + all: 3 + visible: 3 + number-named-branch: + all: 1 + visible: 1 number-revisions: - visible: 53 all: 53 - number-heads: - visible: 3 - all: 3 - number-names-branch: - visible: 1 - all: 1 + visible: 53 + source: some-dir/my-other-source-repo