Newer
Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
#!/usr/bin/env python3
import argparse
import json
import os
import re
import subprocess
import sys
import tarfile
DEFAULT_PYTHON_VERSION = '3'
HOME = os.environ['HOME']
# ! Docker context directory for the *heptapod-dev* image!
DOCKER_CONTEXT_DIR = os.path.realpath(os.path.dirname(__file__))
OMNIBUS_DIR = os.path.dirname(DOCKER_CONTEXT_DIR)
DOCKER_BASE_CONTEXT_DIR = os.path.join(OMNIBUS_DIR, 'heptapod')
SOURCES_DIR = os.path.join(OMNIBUS_DIR, 'repos')
PREFETCH_SOURCES = { # name in heptapod_revisions -> repo slug
'workhorse': 'heptapod-workhorse',
'shell': 'heptapod-shell',
}
def read_docker_file_from_field(path, image_tag_rx='.*?'):
"""Read the FROM of Dockerfile at given path."""
rx = re.compile(r'^FROM\s*(' + image_tag_rx + r')\s*$')
with open(path) as dockerfile:
for line in dockerfile.readlines():
line = line.strip().split('#', 1)[0]
match = rx.match(line)
if match is not None:
return match.group(1)
return LookupError("Cant find FROM statement in Dockerfile %r" % path)
def dev_from_base_image_tag(base_image_tag):
split = base_image_tag.split(':', 1)
if len(split) == 1:
image, tag = split[0], 'latest'
else:
image, tag = split
# for the dev image, `testing` tags don't bring much. Instead, they
# could a source of confusion if both `testing` and `latest` images
# happen to have been pushed with inconsistent versions (especially if
# `testing` is older than `latest`)
tag = tag.replace('testing', 'latest')
return ':'.join((image + '-dev', tag))
def default_volumes_dir():
branch = subprocess.check_output(
('hg', 'log', '-T', '{branch}', '-r', '.'),
cwd=DOCKER_CONTEXT_DIR).decode()
return os.path.join(os.environ['HOME'], 'heptapod-volumes', branch)
BASE_IMAGE_TAG = 'octobus/heptapod:testing'
DEFAULT_IMAGE_TAG = dev_from_base_image_tag(BASE_IMAGE_TAG)
def expand_join(*segments):
return os.path.expanduser(os.path.join(*segments))
def print_header(header, underline_char='-'):
print(header)
print(underline_char * len(header))
print()
def print_conf_section(header, elements):
print_header(header)
fmt = '%-{}s%s'.format(2 + max(len(le[0]) for le in elements))
for line_elts in elements:
print(fmt % (line_elts[0] + ':', line_elts[1]))
print()
def hg_rev_sha(repo_path, revision, fail=False):
"""Return full sha for revision, or ``None`` if it doesn't exist."""
p = subprocess.Popen(['hg', 'log', '-T', '{node}', '-r', revision],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
cwd=repo_path)
out = p.communicate()[0]
if p.returncode != 0:
if fail:
raise LookupError((repo_path, revision))
return
return out.decode().strip()
def inspect_archive(tarball):
"""Return the contents of archival metadata as a dict."""
with tarfile.open(tarball) as tar:
archival = tar.extractfile('.hg_archival.txt')
res = dict([token.strip() for token in line.decode().split(':', 1)]
for line in archival.readlines())
archival.close()
return res
def docker_build_args(py_version):
python = 'python' if py_version == '2' else 'omnibus'
return ('--build-arg', 'heptapod_python=' + python)
def py_version_image_tag(full_tag, py_version):
"""Make a derivative of given full tag for a given Python version."""
return '%s-py%s' % (full_tag, py_version)
def main():
parser = argparse.ArgumentParser(
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
description=__doc__)
parser.add_argument('-s', '--simulate', action='store_true',
help="Output configuration instead of "
"executing commands")
vol_group = parser.add_argument_group("Docker volumes options")
vol_group.add_argument('--volumes-dir',
default=default_volumes_dir(),
help="Base directory for the three standard "
"GitLab Docker "
"volumes")
vol_group.add_argument('--volume-conf', default="etc",
help="Name of the volumes subdirectory for "
"configuration")
vol_group.add_argument('--volume-data', default="var",
help="Name of the volumes subdirectory for data")
vol_group.add_argument('--volume-logs', default="log",
help="Name of the volumes subdirectory for logs")
vol_group.add_argument('--volume-src',
default=os.path.dirname(OMNIBUS_DIR),
help="Additional directory bind-mounted to "
"VOLUME_SRC_TARGET, defaulting to the parent "
"directory of the present omnibus working copy. "
"Assuming your working copies of heptapod, hg-git, "
"etc. are in this VOLUMES_SRC, this makes it easy "
"to update the code running in the container "
"from them."
)
vol_group.add_argument('--volume-src-target', default='/home/heptapod',
help="Where to mount VOLUME_SRC inside the "
"container.")
run_group = parser.add_argument_group("Other Docker run options")
run_group.add_argument('--no-run', action="store_true")
run_group.add_argument('--http-port', type=int, default=81)
run_group.add_argument('--ssh-port', type=int, default=2022)
run_group.add_argument('--container-name', default='heptapod')
img_group = parser.add_argument_group("Docker image options")
img_group.add_argument('--no-cache', action='store_true',
help="Build image(s) without cache.")
img_group.add_argument('--image-tag',
help="Tag for the produced and/or run image. "
"The default %r is suitable "
"for --push" % DEFAULT_IMAGE_TAG)
img_group.add_argument('--squash', action="store_true",
help="If True and the base image is to be built, "
"use `docker build --squash` to build the base "
"image")
img_group.add_argument('--push', action="store_true",
help="Push all built images (requires Docker Hub "
"credentials)")
img_group.add_argument('--python-version', choices=('2', '3'),
default=DEFAULT_PYTHON_VERSION,
)
parsed_args = parser.parse_args()
simulate = parsed_args.simulate
volumes_dir = parsed_args.volumes_dir
etc_vol = expand_join(volumes_dir, parsed_args.volume_conf)
var_vol = expand_join(volumes_dir, parsed_args.volume_data)
log_vol = expand_join(volumes_dir, parsed_args.volume_logs)
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
src_vol = os.path.expanduser(parsed_args.volume_src)
do_build_dev = do_build_base = False
do_push = parsed_args.push
do_run = not parsed_args.no_run
image_tag = parsed_args.image_tag
if image_tag is None:
image_tag = BASE_IMAGE_TAG
no_cache = parsed_args.no_cache
run_image_tag = image_tag
if simulate:
print_header("Simulation summary", underline_char='=')
actions = [("Build dev image", do_build_dev),
("Build base image", do_build_base),
("Run container", do_run),
]
if do_run:
actions.append((" with image", image_tag))
actions.append(("Push image(s)", do_push))
print_conf_section("Actions", actions)
print_conf_section("Images",
(("Base", BASE_IMAGE_TAG),
("Development", image_tag),
))
if do_run:
print_conf_section("Volume bind mounts",
(('Configuration', etc_vol),
('Data', var_vol),
('Logs', log_vol),
('Sources', src_vol),
))
print_header("Commands")
if not do_run:
return
run_cmd = ['docker', 'run', '--detach',
'--hostname', 'heptapod',
'--publish', '%d:80' % parsed_args.http_port,
'--publish', '%d:22' % parsed_args.ssh_port,
'--name', parsed_args.container_name,
]
bind_mounts = ((var_vol, '/var/opt/gitlab'),
(log_vol, '/var/log/gitlab'),
(etc_vol, '/etc/gitlab'),
(src_vol, parsed_args.volume_src_target),
)
for bm in bind_mounts:
run_cmd.extend(('--volume', '%s:%s' % bm))
run_cmd.append(run_image_tag)
print("Docker run command: %r" % run_cmd)
if not simulate:
subprocess.check_call(run_cmd)
print()
if __name__ == '__main__':
main()