project.py 41.2 KB
Newer Older
1
import attr
2
from base64 import b64decode
3
import hashlib
4
from io import BytesIO
5
import logging
6
7
import os
import py
8
import re
9
import requests
10
import time
11
from urllib.parse import urlencode
12

13
14
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
15
16
from selenium.webdriver.support.ui import WebDriverWait

17
from . import wait_assert
18
19
from .constants import DATA_DIR
from .hg import LocalRepo
20
from .group import UserNameSpace
21
from .selenium import could_click_element
22

23
24
logger = logging.getLogger(__name__)

25

26
27
28
29
30
class ProjectAccess:
    NO_ACCESS = 0
    GUEST = 10
    REPORTER = 20
    DEVELOPER = 30
31
    MAINTAINER = 40
32
33
34
    OWNER = 50


35
36
37
38
39
def extract_gitlab_branch_titles(branches):
    return {name: info['commit']['title']
            for name, info in branches.items()}


40
41
42
43
44
@attr.s
class Project(object):
    heptapod = attr.ib()
    name = attr.ib()
    group = attr.ib()
45
    # owner=None or id=None means it's not known yet, this should happen only
46
47
    # in the present module functions
    owner = attr.ib(default=None)
48
    id = attr.ib(default=None)
49
50
51
52
53
    # If vcs_type is None, this means Mercurial.
    # These functional tests should only exceptionally make a difference
    # between various ways Mercurial is supported ('hg_git', 'hgitaly'), and
    # preferably for temporary workarounds.
    vcs_type = attr.ib(default=None)
54
    is_legacy = attr.ib(default=False)
55
56
57
58
59
60

    def owner_get(self, **kwargs):
        """A shortcut to perform a simple GET, with BasicAuth as the owner.

        All `kwargs` are passed directly to `requests.get()`
        """
61
        return requests.get(self.url, auth=self.owner_credentials, **kwargs)
62

63
64
65
66
67
68
    def get_session_cookie(self, webdriver):
        for cookie in webdriver.get_cookies():
            if cookie['name'].startswith('_gitlab_session'):
                return cookie
        raise LookupError("Could not find GitLab session cookie")

69
70
71
72
73
74
    def session_api_get(self, webdriver, subpath='', **kwargs):
        """Perform a simple GET, with the session cookie found in webdriver.

        The full URL is made of the API URL of the project, together with
        the given subpath (example '/merge_requests/1')
        """
75
        cookie = self.get_session_cookie(webdriver)
76
77
        return requests.get(
            '/'.join((self.api_url, subpath)),
78
            cookies={cookie['name']: cookie['value']})
79

80
81
    def api_edit(self, **params):
        """Perform a project API edit."""
82
        return self.owner_api_put(data=params)
83

84
85
86
87
    @property
    def owner_user(self):
        return self.heptapod.get_user(self.owner)

88
89
90
    def owner_api_hgrc_set(self, **values):
        return self.owner_api_put(subpath='hgrc', data=values)

91
92
93
    def owner_api_hgrc_get(self):
        return self.owner_api_get(subpath='hgrc')

Georges Racinet's avatar
Georges Racinet committed
94
95
96
    def owner_api_hgrc_reset(self):
        return self.owner_api_delete(subpath='hgrc')

97
98
99
    def owner_api_hg_heptapod_config(self):
        return self.owner_api_get(subpath='hg_heptapod_config')

100
101
    def api_request(self, method, user, subpath='', **kwargs):
        """Perform a simple API HTTP request as the given user.
102
103

        `method` is the HTTP method to use, same as in `requests.request`.
104
105

        The full URL is made of the API URL of the project, together with
106
        the given subpath (example 'merge_requests/1').
107

108
109
110
        Appropriate authentication headers are added on the fly.

        All kwargs are passed to `requests.request()`
111
112
        """
        headers = kwargs.pop('headers', {})
113
        headers['Private-Token'] = user.token
114
115
116
117
        return requests.request(method,
                                '/'.join((self.api_url, subpath)),
                                headers=headers,
                                **kwargs)
118

119
120
121
122
123
124
125
    def owner_api_request(self, method, *args, **kwargs):
        """Perform a simple API HTTP request as the owner.

        See :meth:`api_request` for more details.
        """
        return self.api_request(method, self.owner_user, *args, **kwargs)

126
127
    def owner_api_get(self, **kwargs):
        """Perform a simple HTTP API GET as the owner.
128

129
        See :meth:`api_request` for details.
130
        """
131
        return self.owner_api_request('GET', **kwargs)
132

133
134
    def owner_api_put(self, **kwargs):
        """Perform a simple HTTP API PUT as the owner.
Georges Racinet's avatar
Georges Racinet committed
135

136
        See :meth:`api_request` for details.
137
138
        """
        return self.owner_api_request('PUT', **kwargs)
Georges Racinet's avatar
Georges Racinet committed
139

140
141
142
    def owner_api_post(self, **kwargs):
        """Perform a simple HTTP API POST as the owner.

143
        See :meth:`api_request` for details.
Georges Racinet's avatar
Georges Racinet committed
144
        """
145
        return self.owner_api_request('POST', **kwargs)
Georges Racinet's avatar
Georges Racinet committed
146

147
148
149
    def owner_api_delete(self, **kwargs):
        """Perform a simple HTTP API DELETE as the owner.

150
        See :meth:`api_request` for details.
151
152
153
        """
        return self.owner_api_request('DELETE', **kwargs)

154
155
156
157
158
159
160
161
162
    def api_get_field(self, key):
        """Return the value of a field by performing an API GET request.

        The request is made with full owner credentials.
        """
        resp = self.owner_api_get()
        assert resp.status_code < 400
        return resp.json().get(key)

163
164
165
    @property
    def owner_credentials(self):
        """Return (user, password)."""
166
167
        user = self.owner_user
        return user.name, user.password
168
169
170

    @property
    def owner_token(self):
171
        return self.owner_user.token
172

Georges Racinet's avatar
Georges Racinet committed
173
174
    @property
    def owner_webdriver(self):
175
        return self.heptapod.get_user_webdriver(self.owner)
Georges Racinet's avatar
Georges Racinet committed
176

177
178
179
180
    @property
    def relative_uri(self):
        return '/'.join((self.group.full_path, self.name))

181
182
    @property
    def url(self):
183
        return '/'.join((self.heptapod.url, self.relative_uri))
184

Georges Racinet's avatar
Georges Racinet committed
185
    @property
Georges Racinet's avatar
Georges Racinet committed
186
187
188
189
    def owner_ssh_params(self):
        """See `ssh_params()`
        """
        return self.ssh_params(self.owner)
Georges Racinet's avatar
Georges Racinet committed
190

Georges Racinet's avatar
Georges Racinet committed
191
192
    def ssh_params(self, user_name):
        """Provide command and URL to perform SSH operations as `user_name`
Georges Racinet's avatar
Georges Racinet committed
193
194
195
196
197
198
        Example::

           ('ssh -i /tmp/id_rsa', 'git@localhost:root/test_project.hg')

        """
        heptapod = self.heptapod
199
        ext = '.git' if self.vcs_type == 'git' else '.hg'
200
        url = '/'.join((heptapod.ssh_url, self.relative_uri + ext))
201
        return self.heptapod.get_user(user_name).ssh_command, url
Georges Racinet's avatar
Georges Racinet committed
202

203
204
205
206
207
    def git_ssh_params(self, user_name):
        """Similar to ssh_params, tailored for Git.
        """
        heptapod = self.heptapod
        cmd = self.ssh_params(user_name)[0] + ' -p %d' % heptapod.ssh_port
208
        address = '{heptapod.ssh_user}@{heptapod.host}:{path}'.format(
209
            heptapod=heptapod,
210
            path=self.relative_uri + '.git',
211
        )
212
213
        return cmd, address

214
215
216
217
218
219
    def basic_auth_url(self, user_name, pwd=None):
        """Produce an URL suitable for basic authentication, hence hg CLI.

        :param pwd: if not supplied, will be read from the permanent users
                    known of :attr:`heptapod`
        """
220
        heptapod = self.heptapod
221
        if pwd is None:
222
            pwd = self.heptapod.get_user(user_name).password
223
        url = heptapod.parsed_url
224
        return "{scheme}://{auth}@{netloc}/{path}".format(
225
226
            scheme=url.scheme,
            netloc=url.netloc,
227
            auth=':'.join((user_name, pwd)),
228
            path=self.relative_uri,
229
230
        )

231
232
233
234
    @property
    def owner_basic_auth_url(self):
        return self.basic_auth_url(self.owner)

235
236
237
    def deploy_token_url(self, token):
        return self.basic_auth_url(token['username'], pwd=token['token'])

238
239
    @property
    def api_url(self):
240
241
242
243
244
        return '/'.join((
            self.heptapod.url,
            'api', 'v4', 'projects',
            '/'.join((self.group.full_path, self.name)).replace('/', '%2F')
        ))
245

246
    @property
247
    def fs_common_path(self):
248
249
250
251
252
        """Common abspath on Heptapod server FS (not ending with .hg nor .git)

        Meaningful only for those tests that require file system access.
        Relies on knowledge of internal GitLab details that may well change.
        (since these are tests, we would notice quickly).
253
        """
254
255
256
257
        disk_path = getattr(self, '_disk_path', None)
        if disk_path is not None:
            return disk_path

258
        if not self.is_legacy:
259
260
261
262
263
264
            sha2 = hashlib.sha256(b'%d' % self.id).hexdigest()
            rpath = '@hashed/%s/%s/%s' % (sha2[:2], sha2[2:4], sha2)
            disk_path = os.path.join(self.heptapod.repositories_root, rpath)
        else:
            disk_path = '/'.join((self.heptapod.repositories_root,
                                  self.group.full_path, self.name))
265
266
        self._disk_path = disk_path
        return disk_path
267
268
269
270
271

    @property
    def fs_path(self):
        """Path to the Mercurial repo on Heptapod server file system."""
        return self.fs_common_path + '.hg'
272

273
274
275
276
277
    @property
    def fs_path_git(self):
        """Path to the Git repo on Heptapod server file system."""
        return self.fs_common_path + '.git'

278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
    def _change_storage(self, legacy):
        label = 'legacy' if legacy else 'hashed'
        if legacy is self.is_legacy:
            logger.warn("_change_storage: project %d is already on %s storage",
                        self.id, label)
        rake_cmd = 'rollback_to_legacy' if legacy else 'migrate_to_hashed'
        self.heptapod.rake('gitlab:storage:' + rake_cmd,
                           'ID_FROM=%d' % self.id,
                           'ID_TO=%d' % self.id)
        self.is_legacy = legacy
        self._disk_path = None

        # we're not inconsistent
        assert ('@hashed' in self.fs_path) is (not legacy)

        wait_assert(
            lambda: self.heptapod.execute(('test', '-e', self.fs_path))[0],
            lambda code: code == 0,
            timeout=120,
            retry_wait=2,
            msg="Repository %r not found in %s storage" % (self.fs_path, label)
        )

    def make_storage_legacy(self):
        self._change_storage(True)

    def make_storage_hashed(self):
        self._change_storage(False)

307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
    def archive_url(self, gitlab_branch, file_ext):
        base_file_name = '-'.join([self.name] + gitlab_branch.split('/'))
        return '/'.join((self.url,
                         '-', 'archive',
                         gitlab_branch,
                         base_file_name + '.' + file_ext
                         ))

    def get_archive(self, gitlab_branch, fmt='tar'):
        """Retrieve a repository archive by URL.

        :param fmt: the wished format. At this point, this is directly
            mapped as a file extension in the request, and only the `tar`
            value is tested.
        :returns: the archive content, as a files-like object
        """
        resp = requests.get(self.archive_url(gitlab_branch, fmt))
        assert resp.status_code == 200
        return BytesIO(resp.content)

327
328
    def api_branches(self):
        """Retrieve and pre-sort branches info through the REST API."""
329
        resp = self.owner_api_get(subpath='repository/branches')
330
331
332
333
        assert resp.status_code == 200
        branches = resp.json()
        return dict((branch.pop('name'), branch) for branch in branches)

334
    def api_default_branch(self):
335
336
337
        branch = self.api_get_field('default_branch')
        assert branch is not None
        return branch
338

Georges Racinet's avatar
Georges Racinet committed
339
340
341
342
343
344
    def api_branch_titles(self):
        """Keep only commit titles from `api_branches()`

        With a test scenario in which those titles are characterizing the
        commit uniquely, this is what's very often needed for assertions.
        """
345
        return extract_gitlab_branch_titles(self.api_branches())
Georges Racinet's avatar
Georges Racinet committed
346

347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
    def wait_assert_api_branches(self, condition, msg=None, timeout=10, **kw):
        """Assert some condition to become True on GitLab branches.

        Since the update of pushed or pruned branches is asynchronous and
        becomes even more so as GitLab progresses, this provides the means
        to retry several calls to :meth:`api_branches`.

        :param condition: any callable returning boolean that can take a
                          single argument, the payload of :meth:`api_branches`
        :returns: branches after the wait
        :raises: AssertionError if the condition doesn't become True before
                 timeout
        """
        if msg is None:
            msg = ("The given condition on GitLab branches was still not "
                   "fulfilled after retrying for %s seconds" % timeout)
        return wait_assert(lambda: self.api_branches(), condition,
                           msg=msg, timeout=timeout, **kw)

366
367
368
369
370
371
372
    def api_tags(self):
        """Retrieve and pre-sort tags info through the REST API."""
        resp = self.owner_api_get(subpath='repository/tags')
        assert resp.status_code == 200
        tags = resp.json()
        return dict((tag.pop('name'), tag) for tag in tags)

373
374
375
376
377
    def api_protected_branches(self):
        resp = self.owner_api_get(subpath='protected_branches')
        assert resp.status_code == 200
        return {br['name']: br for br in resp.json()}

378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
    def api_commit(self, sha, check=True):
        """Retrieve a commit by its SHA.

        The SHA is the native one to GitLab, typically obtained through the
        API. For Mercurial SHAs, it's usually simpler to just perform a pull.
        """
        resp = self.owner_api_get(subpath='repository/commits/' + sha)
        if not check:
            return resp

        assert resp.status_code == 200
        return resp.json()

    def api_file_create(self, path, check=True, **data):
        """data is transferred directly into the JSON expected by the API."""
        data['file_path'] = path
        resp = self.owner_api_post(
            subpath='repository/files/' + path,
            data=data)
        if not check:
            return resp

        assert resp.status_code < 400
        return resp.json()

    def api_file_update(self, path, check=True, **data):
        """data is transferred directly into the JSON expected by the API."""
        data['file_path'] = path
        resp = self.owner_api_put(
            subpath='repository/files/' + path,
            data=data)
        if not check:
            return resp

        assert resp.status_code < 400
        return resp.json()

Georges Racinet's avatar
Georges Racinet committed
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
    def api_wiki_page_create(self, check=True, **data):
        resp = self.owner_api_post(subpath='wikis', data=data)
        if not check:
            return resp
        assert resp.status_code == 201
        return resp.json()

    def api_wiki_page_get(self, slug, check=True):
        resp = self.owner_api_get(subpath='wikis/' + slug)
        if not check:
            return resp
        assert resp.status_code == 200
        return resp.json()

    def api_wiki_page_update(self, slug, check=True, **data):
        resp = self.owner_api_put(subpath='wikis/' + slug, data=data)
        if not check:
            return resp
        assert resp.status_code == 200
        return resp.json()

    def api_wiki_pages_list(self, check=True):
        resp = self.owner_api_get(subpath='wikis')
        if not check:
            return resp
        assert resp.status_code == 200
        return resp.json()

    def hg_wiki_url(self, user_name=None):
        """Return an authenticated URL suitage for hg pull/push.

        :param user_name: any user name known to :attr:`heptapod`
        """
        if user_name is None:
            user_name = self.owner
        return self.basic_auth_url(user_name) + '.wiki'

452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
    def api_file_get(self, path, ref, content_only=True):
        """Retrieve a repository file through API.

        :param content_only: if ``True``, the response status code is checked
           and the content is extracted and returned as
           bytes. Otherwise the full HTTP response is returned.
        """
        resp = self.owner_api_get(subpath='repository/files/' + path,
                                  params=dict(ref=ref))
        if not content_only:
            return resp

        assert resp.status_code == 200
        return b64decode(resp.json()['content'])

    def api_file_delete(self, path, check=True, **data):
        data['file_path'] = path
        resp = self.owner_api_delete(subpath='repository/files/' + path,
                                     data=data)
        if not check:
            return resp

        assert resp.status_code < 400

476
477
478
    def webdriver_update_merge_request_settings(self, merge_method):
        driver = self.owner_webdriver
        driver.get('{url}/{group_path}/{project_name}/edit'.format(
479
480
481
482
            group_path=self.group.full_path,
            url=self.heptapod.url,
            project_name=self.name,
        ))
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498

        elem = driver.find_element_by_xpath(
            '//section[contains(@class, "merge-requests-feature")]'
            '//button[contains(text(), "Expand")]'
        )
        elem.click()

        wait = WebDriverWait(driver, 10)
        wait.until(could_click_element(lambda d: d.find_element_by_id(
            'project_merge_method_' + merge_method)))

        submit = driver.find_element_by_xpath(
            '//section[contains(@class, "merge-requests-feature")]'
            '//input[@type="submit"]')
        submit.click()

499
    def webdriver_create_merge_request(self, source, target):
500
        """Create a merge request through the Web UI and return its id.
501
502
503
        """
        heptapod = self.heptapod
        driver = self.owner_webdriver
504
505
506
507
508
        compare_qs = {
            'merge_request[source_project_id]': self.id,
            'merge_request[source_branch]': source,
            'merge_request[target_project_id]': self.id,
            'merge_request[target_branch]': target,
509
        }
Georges Racinet's avatar
flake8    
Georges Racinet committed
510
        driver.get(
511
            '{url}/{group_path}/{project_name}/merge_requests/new?{qs}'.format(
512
                group_path=self.group.full_path,
Georges Racinet's avatar
flake8    
Georges Racinet committed
513
                url=heptapod.url,
514
                project_name=self.name,
515
                qs=urlencode(compare_qs),
516
            ))
517
        assert 'New Merge Request' in driver.title
518
519
        assert self.name in driver.title

520
521
        wait = WebDriverWait(driver, 10)
        wait.until(could_click_element(lambda d: d.find_element_by_xpath(
522
523
            '//input[@type="submit" '
            'and @value="Submit merge request"]')
524
        ))
525

526
527
528
529
        split_url = driver.current_url.rsplit('/', 2)
        assert split_url[-2] == 'merge_requests'
        return int(split_url[-1])

530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
    def api_create_merge_request(self, source,
                                 target='branch/default',
                                 title='Created through API'):
        """As the project owner, create a merge request through API.

        TODO support creation as another user.
        :param source: name of the source branch for GitLab, e.g.,
                    'topic/default/foo'
        :param target: name of the target branch for GitLab
        :returns: numeric iid of the newly created MR
        """
        resp = self.owner_api_post(subpath='merge_requests',
                                   data=dict(id=self.id,
                                             source_branch=source,
                                             target_branch=target,
                                             title=title,
                                             ))
        assert resp.status_code in (200, 201)
        return resp.json()['iid']

550
551
552
553
554
555
556
557
558
559
    def api_rebase_merge_request(self, mr_iid):
        resp = self.owner_api_put(subpath='merge_requests/%d/rebase' % mr_iid)
        assert resp.status_code == 202  # Accepted, this is async

        mr_info = self.wait_assert_merge_request(
            mr_iid,
            lambda info: not info['rebase_in_progress'],
            with_rebase=True)
        assert mr_info['merge_error'] is None

560
561
562
563
564
565
    def api_update_merge_request(self, mr_iid, **data):
        resp = self.owner_api_put(subpath='merge_requests/%d' % mr_iid,
                                  data=data)
        assert resp.status_code == 200  # this is synchronous
        return resp.json()

566
567
568
    def api_get_merge_request(self, mr_id, **params):
        resp = self.owner_api_get(subpath='merge_requests/%s' % mr_id,
                                  params=params)
569
570
571
        assert resp.status_code == 200
        return resp.json()

572
    def wait_assert_merge_request(self, mr_id, condition,
573
574
                                  with_rebase=False,
                                  with_diverged_count=False,
575
576
577
578
579
                                  timeout=10, msg=None, **kw):
        if msg is None:
            msg = (
                "The given condition on Merge Request %s was still not "
                "fulfilled after retrying for %s seconds" % (mr_id, timeout))
580
581
582
583
        mr_opts = dict(include_diverged_commits_count=with_diverged_count,
                       include_rebase_in_progress=with_rebase)
        return wait_assert(lambda: self.api_get_merge_request(mr_id,
                                                              **mr_opts),
584
585
586
                           condition,
                           msg=msg, timeout=timeout, **kw)

587
588
589
590
591
    def webdriver_get_merge_request_commits(self, mr_id):
        """Retrieve the commit links from the 'commits' panel of the MR."""
        webdriver = self.owner_webdriver
        # this is the 'commits' pane, loading is dynamic, hence the wait
        webdriver.get(self.url + '/merge_requests/%d/commits' % mr_id)
592
        WebDriverWait(webdriver, 10).until(
593
594
595
            EC.presence_of_all_elements_located(
                (By.CSS_SELECTOR, 'a.commit-row-message')))

596
597
        return webdriver.find_elements_by_css_selector('li.commit')

598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
    def api_create_deploy_token(self, name, scopes=('read_repository',)):
        """Will be available with GitLab 12.9."""
        resp = self.owner_api_post(subpath='deploy_tokens',
                                   data=dict(name=name,
                                             scopes=scopes))
        assert resp.status_code == 200
        return resp.json()

    def api_delete_deploy_token(self, token):
        """Will be available with GitLab 12.9."""
        self.owner_api_delete(subpath='deploy_tokens',
                              data=dict(token_id=token['id']))

    def webdriver_create_deploy_token(self, name):
        """Create a deploy token with Selenium

        The API doesn't exist before GitLab 12.9.

        :param name: required by GitLab, is only a human intended description
        """
        driver = self.owner_webdriver
        driver.get(self.url + '/-/settings/repository')
        wait = WebDriverWait(driver, 10)

        section_xpath = '//section[contains(@class, "deploy-tokens-settings")]'
        expand_button = driver.find_element_by_xpath(
            section_xpath + '//button[contains(text(), "Expand")]')
        expand_button.click()
        wait.until(lambda d: d.find_element_by_xpath(
            section_xpath + '//button[contains(text(), "Collapse")]'
        ).is_displayed)

        def name_elt(d):
            return d.find_element_by_id('deploy_token_name')

        wait.until(lambda d: name_elt(d).is_displayed())
        name_elt(driver).send_keys(name)
        read_repo_scope = driver.find_element_by_id(
            'deploy_token_read_repository')
        read_repo_scope.click()
        submit = driver.find_element_by_xpath(
            '//form[@id="new_deploy_token"]'
            '//input[@type="submit"]')
        submit.click()

        def value_for_id(elt_id):
            return driver.find_element_by_id(elt_id).get_attribute('value')

        return dict(
            username=value_for_id('deploy-token-user'),
            token=value_for_id('deploy-token'),
        )

651
652
653
654
655
656
657
658
    def grant_member_access(self, user_name, level):
        """Grant given user the given access level.

        It doesn't matter whether the user is already a member or not: this
        method abstracts over it.

        This method is idempotent.
        """
659
        user_id = self.heptapod.get_user(user_name).id
660
661
662
663
664
665
666
667
668
669
670
671
        resp = self.owner_api_get(subpath='members/%d' % user_id)
        if resp.status_code == 404:
            subpath = 'members'
            meth = self.owner_api_post
        else:
            subpath = 'members/%d' % user_id
            meth = self.owner_api_put

        resp = meth(subpath=subpath,
                    data=dict(user_id=user_id, access_level=level))
        assert resp.status_code < 400

672
673
674
675
676
677
    def load_tarball(self, tarball_path):
        """Replace server-side repository files by the contents of tarball.

        This should be used right after the project creation.

        :param tarball_path: path (relative to DATA_DIR) to an uncompressed tar
678
679
680
            archive, containing `hg` and `git`. These
            will be renamed to the values of `self.fs_path`` and
            ``self.fs_path_git``.
681
        """
682
683
684
        if not self.heptapod.fs_access:
            raise NotImplementedError(
                "Can't use load_tarball() without filesystem access")
685
686
687
688
689
690
691
692
        tarball_path = os.path.join(DATA_DIR, tarball_path)

        # initialize repository
        # GitLab needs a first clone or push to consider the Git repository to
        # exist. Otherwise, local pushes such as what the heptapod sync hook
        # does just fail, with an error about the Git repo existence.
        tmpdir = py.path.local.mkdtemp()
        try:
693
            LocalRepo.clone(self.owner_basic_auth_url, tmpdir.join('clone'))
694
695
696
        finally:
            tmpdir.remove()

697
698
699
700
        heptapod = self.heptapod
        # using a temporary space in same mount point and unique enough
        srvtmpdir = self.fs_common_path + '.tmp'
        heptapod.run_shell(['rm', '-rf',
701
                            self.fs_path, self.fs_path_git, srvtmpdir])
702
703
704
705
706
        heptapod.run_shell(['mkdir', '-p', srvtmpdir])
        heptapod.put_archive(srvtmpdir, tarball_path)
        heptapod.run_shell(['mv', srvtmpdir + '/hg', self.fs_path])
        heptapod.run_shell(['mv', srvtmpdir + '/git', self.fs_path_git])

707
708
709
710
711
712
713
714
715
    def get_hgrc(self, managed=False):
        """Return repo's server-side HGRC, as lines, uid and gid

        :param managed: if ``True``, the contents returned are those of the
                        file managed by the Rails app, introduced
                        for heptapod#165
        """
        hgrc_path = '/'.join((self.fs_path, '.hg',
                              'hgrc.managed' if managed else 'hgrc'))
716
717
718
        return self.heptapod.get_file_lines(hgrc_path)

    def put_hgrc(self, lines):
719
720
721
722
        """Replace repo's server-side HGRC with given lines.

        The lines have to include LF, same as with `writelines()`.
        """
723
        repo_inner_path = '/'.join((self.fs_path, '.hg'))
724

725
726
        return self.heptapod.put_file_lines(
            os.path.join(repo_inner_path, 'hgrc'), lines)
727
728
729
730
731
732

    def extend_hgrc(self, *lines):
        """Append given lines to repo's server-side HGRC

        The lines don't have to be newline-terminated.
        """
733
        hgrc_lines = self.get_hgrc()
734
735
736
        # just in case original hgrc isn't new-line terminated
        hgrc_lines.append('\n')
        hgrc_lines.extend(l + '\n' for l in lines)
737
        self.put_hgrc(hgrc_lines)
738

739
740
741
742
743
744
745
746
747
    def hg_config(self, section=None):
        """Return Mercurial configuration item, as really seen by hg process.

        In other words, this isn't inference on the contents of the various
        HGRC, it can be used to test propagation of config entries.

        :return: if ``section`` is passed, a simple ``dict``, otherwise a
                 ``dict`` of ``dicts``. End values are always strings.
        """
748
        cmd = [self.heptapod.hg_executable, '-R', self.fs_path,
749
750
751
752
753
               '--pager', 'false',
               'config']
        if section is not None:
            cmd.append(section)

754
        code, out = self.heptapod.execute(cmd, user='git')
755
        config = {}
756
757
        if out is None:
            return config
758
        for l in out.splitlines():
759
            print(l)
760
761
762
763
764
765
766
767
            fullkey, val = l.split('=', 1)
            section, key = fullkey.split('.', 1)
            if section is not None:
                config[key] = val
            else:
                config.setdefault(section, {})[key] = val
        return config

Georges Racinet's avatar
Georges Racinet committed
768
769
770
771
772
773
774
775
776
    def api_destroy(self, allow_missing=False, timeout=10):
        resp = self.owner_api_request('DELETE')
        if allow_missing and resp.status_code == 404:
            return
        print("DELETE request response: %r" % resp.json())

        # Even though the deletion meaning that the repos are just
        # mv'ed on the filesystem, it is still async
        assert resp.status_code == 202
777
        start = time.time()
Georges Racinet's avatar
Georges Racinet committed
778
779
780
        duration = 0.2
        while self.owner_api_get().status_code != 404:
            time.sleep(duration)
781
            assert time.time() - start < timeout
Georges Racinet's avatar
Georges Racinet committed
782
783
784
785

        return resp

    def webdriver_destroy(self, skip_missing=False):
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
        driver = self.owner_webdriver
        driver.get(self.url + '/edit')
        if skip_missing and '404' in driver.title:
            return

        assert 'General' in driver.title
        assert 'Settings' in driver.title
        assert self.name in driver.title

        wait = WebDriverWait(driver, 10)

        elem = driver.find_element_by_xpath(
            '//section[contains(@class, "advanced-settings")]'
            '//button[contains(text(), "Expand")]'
        )
        elem.click()
        wait.until(lambda d: d.find_element_by_xpath(
            '//section[contains(@class, "advanced-settings")]'
            '//button[contains(text(), "Collapse")]'
        ).is_displayed)

        wait.until(could_click_element(lambda d: d.find_element_by_xpath(
808
809
810
            # a <button> inside a <form> that contains
            # an <input name="_method" value="delete">
            '//form[input[@name="_method" and @value="delete"]]/button'
811
812
        )))

813
814
815
816
817
818
819
820
821
822
823
        # a global find by name or id retrieves the wrong element (despite
        # id being supposed to be unique)
        modal = driver.find_element_by_xpath(
            '//div[contains(@id, "delete-project-modal")]')
        confirm_input = modal.find_element_by_name('confirm_name_input')

        confirm_input.send_keys(modal.find_element_by_tag_name('code').text)
        confirm_button = modal.find_element_by_css_selector(
            'button.js-modal-action-primary')
        confirm_button.click()

824
        deletion_banner_re = r"Project '\w+ / {}' will be deleted.".format(
825
            self.name)
826
827
828
829
830
831
832

        # storing page source in a variable to allow for investigation
        # in case the regexp doesn't match and we've not been redirected
        # yet. (we don't know at this point if the regexp is appropriate
        # for GitLab 13.3)
        html = driver.page_source
        if re.search(deletion_banner_re, html) is None:
833
834
835
836
837
            # It can happen that the deletion has already been started
            # and we have been redirected to the project list.
            assert 'dashboard/projects' in driver.current_url

            if '500' in driver.title:
Georges Racinet's avatar
flake8    
Georges Racinet committed
838
839
                # We follow redirects a bit too fast for the deletion always
                # to be fully done. Watching the dashbord page while the repos
840
841
842
843
                # are already deleted on the filesystem, yet the project
                # still exists in database is an error cause.
                time.sleep(1)
                driver.get(driver.current_url)
844
            assert self.owner_api_get().status_code == 404
845
846
847
848
849
850
851

    def cleanup_teardown(self):
        """Cleanup files even if destroy() failed.

        This is to ensure that other tests can proceed even in face of failure
        of Mercurial specific filesystem removal.
        """
852
853
854
        heptapod = self.heptapod
        if not (heptapod.shell_access and heptapod.repositories_root):
            return
855
        repo_path = self.fs_path
856
        wiki_path = repo_path[:-3] + '.wiki.hg'
857
        heptapod.run_shell(('rm', '-rf', repo_path, wiki_path))
858

859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
    @classmethod
    def api_retrieve(cls, heptapod, user_name, group, name, check=True):
        """Retrieve Project instance if `id` is not known.

        :param user_name: name of an user having enough access to retrieve
                          project info and in particular owner name
        """
        project = cls(heptapod=heptapod,
                      owner=user_name,  # temporary
                      group=group,
                      name=name)
        resp = project.owner_api_get()  # TODO introduce api_get(user=...)
        if check:
            assert resp.status_code == 200
        elif resp.status_code == 404:
            return

        data = resp.json()
        project.id = data['id']
878
879
880
881
882
883
884
885
        creator_id = data.get('creator_id')
        owner = data.get('owner')
        if owner is not None:
            project.owner = data['owner']['username']
        elif creator_id is not None:
            project.creator_id = creator_id
        else:
            raise LookupError("No owner/creator information for %r", project)
886
887
        return project

888
    @classmethod
889
    def webdriver_create(cls, heptapod, user_name, project_name):
890
891
        """Create a new project with the webdriver for given user
        """
892
        group = UserNameSpace(heptapod=heptapod, user_name=user_name)
893
        driver = heptapod.get_user_webdriver(user_name)
894
        driver.get('{url}/projects/new'.format(url=heptapod.url))
895
        cls.webdriver_new_project_submit(driver, project_name)
896
897
        project = cls.api_retrieve(heptapod, user_name, group, project_name)

898
899
900
901
902
903
        project.wait_hg_availability()
        return project

    def wait_hg_availability(self):
        wait_assert(lambda: self.owner_get(params=dict(cmd='capabilities')),
                    lambda resp: resp.status_code == 200)
904
905
906
907
908
909
910
911

    @classmethod
    def webdriver_new_project_submit(cls, driver, project_name):
        """Create a new project while already on the "New project" page.

        This will put the project in the default namespace proposed by
        the "New project" page. We could add a `group` kwarg later on.
        """
912
913
914
        assert 'GitLab' in driver.title
        assert 'New Project' in driver.title

915
916
917
918
919
920
921
922
        # the project path (slug) attribute is derived automatically from
        # its (human readable) name. The names given in these tests all
        # transform identically into paths.
        # In GitLab 12.10 the converse is true: if only the path is given,
        # the name is derived from it with some prettyfication (capitalization,
        # spaces). Therefore it is better for our needs to provide the name
        # only.
        elem = driver.find_element_by_name('project[name]')
923
        elem.send_keys(project_name)
924
925
926
927
928
        # If we wanted later on to make sure the path is the intended
        # one, we'd have to clear the `project[path]` field first, instead
        # of *adding* path after what was just prefilled due to
        # the setting of `project[name]`

929
930
931
932
933
934
935
936
        elem = driver.find_element_by_xpath('//input[@type="submit" '
                                            'and @value="Create project"]')
        elem.click()
        WebDriverWait(driver, 10).until(
            lambda d: 'will automatically refresh' not in d.page_source)
        assert (
            'Project \'{name}\' was successfully created.'.format(
                name=project_name) in driver.page_source)
937

938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
    def webdriver_retrieve(cls, driver, heptapod, group, name,
                           wait_availability=False):
        """Use a driver to retrieve the project if it exists.

        If the project is not found (could happen with authnz problems),
        then `None` is returned.
        """
        project = Project(heptapod=heptapod, group=group, name=name)
        resp = project.session_api_get(driver)

        if resp.status_code == 404:
            return

        data = resp.json()
        project.owner = data['owner']['username']
        project.id = data['id']
        return project

956
    @classmethod
957
958
    def api_create(cls, heptapod, user_name, project_name,
                   group=None, timeout=30, **data):
959
960
961
962
963
964
965
966
967
968
        """Create project using the RESTful API.

        :param user_name: name of the user to run the import with
        :param group: name of the project group (by default, will be
                      user personal space.
        :param data: for extra data in the POST API request that performs
                     the creation
        :returns: Project instance
        """
        if group is None:
969
            group = UserNameSpace(heptapod=heptapod, user_name=user_name)
Georges Racinet's avatar
Georges Racinet committed
970
971
        else:
            data['namespace_id'] = group.id
972
        group_path = group.full_path
Georges Racinet's avatar
Georges Racinet committed
973

974
        headers = heptapod.get_user(user_name).token_headers()
975
976

        data['name'] = project_name
977
        url = heptapod.url + '/api/v4/projects'
978

979
980
981
        if heptapod.hg_native:
            data.setdefault('vcs_type', 'hg')

982
983
984
985
986
987
988
989
990
991
992
993
994
995
        def post():
            return requests.post(url, headers=headers, data=data)

        def needs_retry(response):
            if response is None:
                return True
            if response.status_code != 400:
                return False
            structured = response.json()
            message = structured.get('message', {})

            if ['has already been taken'] in (message.get('name'),
                                              message.get('path')):
                # clean up if we can
996
997
998
999
1000
1001
1002
                try:
                    route = '/'.join((group_path, project_name))
                    logger.warn("GitLab Route %r has already been taken, "
                                "attempting to remove it", route)
                    heptapod.force_remove_route(route)
                except NotImplementedError:
                    pass
1003

1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
                # trigger retry even if we can't clean up from here:
                # on GitLab 12, there's a short timespan after it stops
                # telling us to retry but before it has fully cleaned up routes
                return True

            if any('try again' in l for l in message.get('base', ())):
                logger.info(
                    "Got explicit retry response: %r with headers %r",
                    structured, response.headers)
                return True

        start = time.time()
        while time.time() - start < timeout:
            resp = post()
            if not needs_retry(resp):
                break
            time.sleep(1)

        logger.debug("Creation request response: %r", resp.json())
1023
1024
1025
        assert resp.status_code in (200, 201)

        proj_info = resp.json()
1026
1027
1028
        owner_info = proj_info.get('owner')
        if owner_info is not None:
            assert owner_info['username'] == user_name
1029
1030
1031
        return cls(heptapod=heptapod,
                   group=group, name=project_name,
                   id=proj_info['id'],
1032
                   vcs_type=proj_info.get('vcs_type'),
1033
1034
                   owner=user_name)

1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
    @classmethod
    def api_import_url(cls, heptapod, user_name, project_name, url,
                       group=None, timeout=30):
        """Import project from URL, wait for completion and returns it.

        :param user_name: name of the user to run the import with
        :param group: name of the project group (by default, will be
                      user personal space.
        :returns: import_status ('failed' or 'finished'), import_error,
                  Project instance
        """
        project = cls.api_create(heptapod, user_name, project_name,
                                 group=group,
                                 import_url=url)

        unexpected_statuses = (None, 'none')
        terminal_statuses = ('finished', 'failed')

1053
1054
        start = time.time()
        while time.time() - start < timeout:
1055
1056
1057
1058
1059
1060
1061
1062
            # dedicated PROJECT_URL/import not implemented in our
            # current GitLab 10.1.x version but the general API GET
            # has the `import_status` in this and future versions.
            resp = project.owner_api_get()
            assert resp.status_code == 200
            proj_info = resp.json()
            status = proj_info['import_status']
            if status in terminal_statuses:
1063
                err = proj_info.get('import_error')
1064
                break
1065
            assert status not in unexpected_statuses
1066
            time.sleep(1)
1067
1068
        else:
            err = "Timed out (decision by tests) after %d seconds" % timeout
1069
1070

        print("Project information after import or timeout: \n%r" % proj_info)
1071
        return status, err, project
1072

Georges Racinet's avatar
Georges Racinet committed
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
    @classmethod
    def api_list(cls, heptapod, search_as='root', **params):
        """List the projects owned by the given user

        Does not support batching at this point, hence will work only
        if the result set fits in the first page.

        Various filtering criteria can be applied, hence it is more a search
        than a listing (notably ``search=something`` is possible)

        :param search_as: login name of the user to perform the listing.
                          Must be one of those registered to the
                          :class:`Heptapod` instance.
        """
        resp = requests.get(
            heptapod.api_url + '/projects',
1089
            headers=heptapod.get_user(search_as).token_headers(),
Georges Racinet's avatar
Georges Racinet committed
1090
            params=params
1091
        )
Georges Racinet's avatar
Georges Racinet committed
1092
1093
        return resp

Georges Racinet's avatar
Georges Racinet committed
1094
1095
1096
1097
1098
1099
1100
1101
    def api_transfer(self, group):
        # API for regular users (on the project) has been introduced in
        # GitLab 11.1 (see https://docs.gitlab.com/ce/api/projects.html)
        # but in current GitLab, root can do it
        heptapod = self.heptapod
        resp = requests.post(
            '{api}/groups/{group.id}/projects/{proj.id}'.format(
                api=heptapod.api_url, group=group, proj=self),
1102
1103
            headers=heptapod.get_user('root').token_headers(),
        )
Georges Racinet's avatar
Georges Racinet committed
1104
1105
1106
        if resp.status_code < 400:
            # let's have in particular correct subsequent URLs
            self.group = group
1107
1108
1109
            if self.is_legacy:
                # path on disk actually changes for legacy projects
                self._disk_path = None
Georges Racinet's avatar
Georges Racinet committed
1110
        return resp