Newer
Older
#!/usr/bin/env python
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
#
# Script to setup the reference repository used for benchmarl
DEFAULT_REPO_SOURCE = "https://static.octobus.net/asv/"
import hashlib
import os
import shutil
import subprocess
import sys
from os.path import (
abspath,
dirname,
join,
)
parent_dir = abspath(join(dirname(abspath(__file__)), ".."))
sys.path.insert(0, join(parent_dir, "lib"))
from scmperf_lib import read_yaml
class SetupError(RuntimeError):
pass
def setup_all_repos(repodir, source, repos):
"""setup all repository in the `repos` list
Repository will be fetch from `source` and setup in `repodir`
"""
print("%d repositories to setup in %s" % (len(repos), repodir))
for r in repos:
setup_one_repo(repodir, source, r)
def setup_one_repo(repodir, source, repo):
"""Setup all items for one "atomic" repo
This include the main reference repository and the partial variants. This also setup the associated data file.
Repository will be fetch from `source` and setup in `repodir`
"""
print "Setting up reference repository: %s" % repo
# setup one
# download reference
# extract partial from `.benchrepo`
# setup partial
# download partial
# hg update the reference
print ' main reference'
setup_repo_variant(repodir, source, repo, None, update=True)
refpath = join(repodir, repo + '.benchrepo')
if not os.path.exists(refpath):
print ' missing data file'
setup_repo_variant(repodir, source, repo, None, update=True, force=True)
refdata = read_yaml(refpath)
partial_sets = refdata.get("partial-sets")
if partial_sets is None:
msg = "not partial-sets declared: %s" % refpath
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
raise SetupError(msg)
for variant in sorted(partial_sets):
print ' partial reference: ', variant
setup_repo_variant(repodir, source, repo, variant)
def repobasename(repo_name, partial_id):
"""return the directory name for the partial variant of a repository
When partial_id is set to None, the main reference repository is used.
"""
if partial_id is None:
remote_base_name = "%s-reference" % repo_name
else:
remote_base_name = "%s-partial-%s" % (repo_name, partial_id)
return remote_base_name
def remotepath(source, repo_name, partial_id):
"""return the remote URL to fetch the given partial variant for a repository
The data will be fetch from `source`. When partial_id is set to None, the
main reference repository is used."""
remote_base_name = repobasename(repo_name, partial_id)
remote_tar_name = "%s.tar" % remote_base_name
return "%s/%s" % (source.rstrip('/'), remote_tar_name)
def repodir(repo_dir, repo_name, partial_id):
"""return the appropriate directory to put a repository variants"""
if partial_id is None:
return repo_dir
return os.path.join(repo_dir, 'partial-references')
def repopath(repo_dir, repo_name, partial_id):
"""return the full path to a repository"""
repo_dir = repodir(repo_dir, repo_name, partial_id)
base_name = repobasename(repo_name, partial_id)
return os.path.join(repo_dir, base_name)
def setupfile(repo_path):
"""return the full path to the "repo-digest" file
This file indicate the repository restoration has completed."""
return os.path.join(repo_path, '.hg', 'scm-perf-repo-digest')
def download_repo(repo_dir, source, repo_name, partial_id):
"""download a repository in the appropriate location"""
remote_path = remotepath(source, repo_name, partial_id)
repo_dir = repodir(repo_dir, repo_name, partial_id)
cmd_fetch = ["curl", remote_path]
if os.environ.get("HGUSER") == "test":
cmd_fetch.insert(1, "--silent")
cmd_untar = ["tar", "x", "--touch"]
proc_fetch = subprocess.Popen(cmd_fetch, stdout=subprocess.PIPE)
proc_untar = subprocess.Popen(cmd_untar, stdin=proc_fetch.stdout, cwd=repo_dir)
proc_fetch.stdout.close()
if proc_fetch.wait() != 0:
msg = "error downloading: [%s] %s" % (proc_fetch.returncode, remote_path)
raise SetupError(msg)
if proc_untar.wait() != 0:
msg = "error extracting tar: [%s] %s"
msg %= (proc_untar.returncode, remote_path)
raise SetupError(msg)
def update_repo(repo_path):
"""update a repository to tip"""
cmd_update = ["hg", "-R", repo_path, "update", "--clean", "tip"]
proc_update = subprocess.Popen(cmd_update)
if proc_update.wait() != 0:
msg = "error updating repo: [%s] %s"
msg %= (proc_update.returncode, repo_path)
raise SetupError(msg)
def conclude_setup(repo_path):
"""write the content file that mark a restoration as completed"""
cmd_conclude = ["hg", "-R", repo_path, "log", "--template", "{node}\n"]
proc_conclude = subprocess.Popen(cmd_conclude, stdout=subprocess.PIPE)
out = proc_conclude.communicate()[0]
if proc_conclude.returncode != 0:
msg = "error gathering repo fingerprint: [%s] %s"
msg %= (proc_conclude.returncode, repo_path)
digest = hashlib.sha512(out).hexdigest()
filepath = setupfile(repo_path)
with open(filepath + '.tmp', 'w') as f:
f.write(digest)
os.rename(filepath + '.tmp', filepath)
def setup_repo_variant(repo_dir, source, repo_name, partial_id,
update=False, force=False):
"""setup a specific variant of a repository
When partial_id is set to None, the main reference repository is used."""
repo_path = repopath(repo_dir, repo_name, partial_id)
setup_file = setupfile(repo_path)
if os.path.exists(setup_file) and not force:
print ' already up to date'
return
if os.path.exists(repo_path):
shutil.rmtree(repo_path)
print ' downloading repo'
download_repo(repo_dir, source, repo_name, partial_id)
if update:
print ' updating repo'
update_repo(repo_path)
conclude_setup(repo_path)
def parse_reposfile(path):
"""read a repos-file that list repo to setup"""
repos = []
with open(path) as reposfile:
for l in reposfile:
l = l.strip()
if l.startswith('#') or not l:
continue
repos.append(l)
return repos
if __name__ == "__main__":
if not (2 <= len(sys.argv) <= 4):
print >> sys.stderr, "usage: %s REPOS-FILE [DIRECTORY [SOURCE]]"
print >> sys.stderr, ""
print >> sys.stderr, "Make sure all item in REPOS-FILE are setup in DIRECTORY."
print >> sys.stderr, "(default `.`)."
print >> sys.stderr, "When missing, data are fetched from SOURCE"
print >> sys.stderr, "(default: %s)" % DEFAULT_REPO_SOURCE
sys.exit(128)
reposfile = sys.argv[1]
repo_dir = '.'
if len(sys.argv) >= 3:
repo_dir = sys.argv[2]
repo_dir = os.path.abspath(repo_dir)
partialdir = os.path.join(repo_dir, 'partial-references')
if not os.path.exists(partialdir):
os.makedirs(partialdir)
source = DEFAULT_REPO_SOURCE
if len(sys.argv) >= 4:
source = sys.argv[3]
repos = parse_reposfile(reposfile)
try:
setup_all_repos(repo_dir, source, repos)
except SetupError as exc:
print >> sys.stderr, 'abort:', str(exc)
sys.exit(1)
sys.exit(0)