procutil.py 19.9 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
# procutil.py - utility for managing processes and executable environment
#
#  Copyright 2005 K. Thananchayan <thananck@yahoo.com>
#  Copyright 2005-2007 Matt Mackall <mpm@selenic.com>
#  Copyright 2006 Vadim Gelfer <vadim.gelfer@gmail.com>
#
# This software may be used and distributed according to the terms of the
# GNU General Public License version 2 or any later version.

from __future__ import absolute_import

12
import contextlib
13
import errno
14
15
16
17
18
import io
import os
import signal
import subprocess
import sys
19
import threading
20
21
22
import time

from ..i18n import _
23
24
25
26
from ..pycompat import (
    getattr,
    open,
)
27
28
29
30
31
32
33
34

from .. import (
    encoding,
    error,
    policy,
    pycompat,
)

35
36
37
# Import like this to keep import-checker happy
from ..utils import resourceutil

38
osutil = policy.importmod('osutil')
39
40
41
42
43

stderr = pycompat.stderr
stdin = pycompat.stdin
stdout = pycompat.stdout

44

45
46
47
48
49
50
def isatty(fp):
    try:
        return fp.isatty()
    except AttributeError:
        return False

51

52
53
54
55
56
# Python 2 uses the C library's standard I/O streams. Glibc determines
# buffering on first write to stdout - if we replace a TTY destined stdout with
# a pipe destined stdout (e.g. pager), we want line buffering (or unbuffered,
# on Windows).
# Python 3 rolls its own standard I/O streams.
57
if isatty(stdout):
58
59
    if pycompat.iswindows:
        # Windows doesn't support line buffering
60
        stdout = os.fdopen(stdout.fileno(), 'wb', 0)
61
62
63
    elif not pycompat.ispy3:
        # on Python 3, stdout (sys.stdout.buffer) is already line buffered and
        # buffering=1 is not handled in binary mode
64
        stdout = os.fdopen(stdout.fileno(), 'wb', 1)
65
66
67

if pycompat.iswindows:
    from .. import windows as platform
68

69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
    stdout = platform.winstdout(stdout)
else:
    from .. import posix as platform

findexe = platform.findexe
_gethgcmd = platform.gethgcmd
getuser = platform.getuser
getpid = os.getpid
hidewindow = platform.hidewindow
quotecommand = platform.quotecommand
readpipe = platform.readpipe
setbinary = platform.setbinary
setsignalhandler = platform.setsignalhandler
shellquote = platform.shellquote
shellsplit = platform.shellsplit
spawndetached = platform.spawndetached
sshargs = platform.sshargs
testpid = platform.testpid

try:
    setprocname = osutil.setprocname
except AttributeError:
    pass
try:
    unblocksignal = osutil.unblocksignal
except AttributeError:
    pass

closefds = pycompat.isposix

99

100
def explainexit(code):
101
    """return a message describing a subprocess status
102
103
    (codes from kill are negative - not os.system/wait encoding)"""
    if code >= 0:
104
105
        return _(b"exited with status %d") % code
    return _(b"killed by signal %d") % -code
106

107

108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
class _pfile(object):
    """File-like wrapper for a stream opened by subprocess.Popen()"""

    def __init__(self, proc, fp):
        self._proc = proc
        self._fp = fp

    def close(self):
        # unlike os.popen(), this returns an integer in subprocess coding
        self._fp.close()
        return self._proc.wait()

    def __iter__(self):
        return iter(self._fp)

    def __getattr__(self, attr):
        return getattr(self._fp, attr)

    def __enter__(self):
        return self

    def __exit__(self, exc_type, exc_value, exc_tb):
        self.close()

132

133
134
def popen(cmd, mode=b'rb', bufsize=-1):
    if mode == b'rb':
135
        return _popenreader(cmd, bufsize)
136
    elif mode == b'wb':
137
        return _popenwriter(cmd, bufsize)
138
    raise error.ProgrammingError(b'unsupported mode: %r' % mode)
139

140

141
def _popenreader(cmd, bufsize):
142
143
144
145
146
147
148
    p = subprocess.Popen(
        tonativestr(quotecommand(cmd)),
        shell=True,
        bufsize=bufsize,
        close_fds=closefds,
        stdout=subprocess.PIPE,
    )
149
150
    return _pfile(p, p.stdout)

151

152
def _popenwriter(cmd, bufsize):
153
154
155
156
157
158
159
    p = subprocess.Popen(
        tonativestr(quotecommand(cmd)),
        shell=True,
        bufsize=bufsize,
        close_fds=closefds,
        stdin=subprocess.PIPE,
    )
160
161
    return _pfile(p, p.stdin)

162

163
def popen2(cmd, env=None):
164
165
166
    # Setting bufsize to -1 lets the system decide the buffer size.
    # The default for bufsize is 0, meaning unbuffered. This leads to
    # poor performance on Mac OS X: http://bugs.python.org/issue4194
167
168
169
170
171
172
173
174
175
    p = subprocess.Popen(
        tonativestr(cmd),
        shell=True,
        bufsize=-1,
        close_fds=closefds,
        stdin=subprocess.PIPE,
        stdout=subprocess.PIPE,
        env=tonativeenv(env),
    )
176
177
    return p.stdin, p.stdout

178

179
180
def popen3(cmd, env=None):
    stdin, stdout, stderr, p = popen4(cmd, env)
181
182
    return stdin, stdout, stderr

183

184
def popen4(cmd, env=None, bufsize=-1):
185
186
187
188
189
190
191
192
193
194
    p = subprocess.Popen(
        tonativestr(cmd),
        shell=True,
        bufsize=bufsize,
        close_fds=closefds,
        stdin=subprocess.PIPE,
        stdout=subprocess.PIPE,
        stderr=subprocess.PIPE,
        env=tonativeenv(env),
    )
195
196
    return p.stdin, p.stdout, p.stderr, p

197

198
199
def pipefilter(s, cmd):
    '''filter string S through command CMD, returning its output'''
200
201
202
203
204
205
206
    p = subprocess.Popen(
        tonativestr(cmd),
        shell=True,
        close_fds=closefds,
        stdin=subprocess.PIPE,
        stdout=subprocess.PIPE,
    )
207
208
209
    pout, perr = p.communicate(s)
    return pout

210

211
212
213
214
215
216
217
def tempfilter(s, cmd):
    '''filter string S through a pair of temporary files with CMD.
    CMD is used as a template to create the real command to be run,
    with the strings INFILE and OUTFILE replaced by the real names of
    the temporary files generated.'''
    inname, outname = None, None
    try:
218
        infd, inname = pycompat.mkstemp(prefix=b'hg-filter-in-')
219
        fp = os.fdopen(infd, 'wb')
220
221
        fp.write(s)
        fp.close()
222
        outfd, outname = pycompat.mkstemp(prefix=b'hg-filter-out-')
223
        os.close(outfd)
224
225
        cmd = cmd.replace(b'INFILE', inname)
        cmd = cmd.replace(b'OUTFILE', outname)
226
        code = system(cmd)
227
        if pycompat.sysplatform == b'OpenVMS' and code & 1:
228
229
            code = 0
        if code:
230
            raise error.Abort(
231
                _(b"command '%s' failed: %s") % (cmd, explainexit(code))
232
            )
233
        with open(outname, b'rb') as fp:
234
235
236
237
238
239
240
241
242
243
244
245
246
            return fp.read()
    finally:
        try:
            if inname:
                os.unlink(inname)
        except OSError:
            pass
        try:
            if outname:
                os.unlink(outname)
        except OSError:
            pass

247

248
_filtertable = {
249
250
    b'tempfile:': tempfilter,
    b'pipe:': pipefilter,
251
252
}

253

254
def filter(s, cmd):
255
256
    """filter a string through a command that transforms its input to its
    output"""
257
    for name, fn in pycompat.iteritems(_filtertable):
258
        if cmd.startswith(name):
259
            return fn(s, cmd[len(name) :].lstrip())
260
261
    return pipefilter(s, cmd)

262

263
264
_hgexecutable = None

265

266
267
268
269
270
271
def hgexecutable():
    """return location of the 'hg' executable.

    Defaults to $HG or 'hg' in the search path.
    """
    if _hgexecutable is None:
272
        hg = encoding.environ.get(b'HG')
273
        mainmod = sys.modules['__main__']
274
275
        if hg:
            _sethgexecutable(hg)
276
        elif resourceutil.mainfrozen():
277
            if getattr(sys, 'frozen', None) == 'macosx_app':
278
                # Env variable set by py2app
279
                _sethgexecutable(encoding.environ[b'EXECUTABLEPATH'])
280
281
            else:
                _sethgexecutable(pycompat.sysexecutable)
282
283
        elif (
            not pycompat.iswindows
284
            and os.path.basename(getattr(mainmod, '__file__', '')) == 'hg'
285
        ):
286
287
            _sethgexecutable(pycompat.fsencode(mainmod.__file__))
        else:
288
            _sethgexecutable(
289
                findexe(b'hg') or os.path.basename(pycompat.sysargv[0])
290
            )
291
292
    return _hgexecutable

293

294
295
296
297
298
def _sethgexecutable(path):
    """set location of the 'hg' executable"""
    global _hgexecutable
    _hgexecutable = path

299

300
301
302
303
304
def _testfileno(f, stdf):
    fileno = getattr(f, 'fileno', None)
    try:
        return fileno and fileno() == stdf.fileno()
    except io.UnsupportedOperation:
305
306
        return False  # fileno() raised UnsupportedOperation

307
308
309
310

def isstdin(f):
    return _testfileno(f, sys.__stdin__)

311

312
313
314
def isstdout(f):
    return _testfileno(f, sys.__stdout__)

315

316
def protectstdio(uin, uout):
317
318
319
320
    """Duplicate streams and redirect original if (uin, uout) are stdio

    If uin is stdin, it's redirected to /dev/null. If uout is stdout, it's
    redirected to stderr so the output is still readable.
321
322
323
324
325
326

    Returns (fin, fout) which point to the original (uin, uout) fds, but
    may be copy of (uin, uout). The returned streams can be considered
    "owned" in that print(), exec(), etc. never reach to them.
    """
    uout.flush()
327
    fin, fout = uin, uout
328
    if _testfileno(uin, stdin):
329
        newfd = os.dup(uin.fileno())
330
        nullfd = os.open(os.devnull, os.O_RDONLY)
331
        os.dup2(nullfd, uin.fileno())
332
        os.close(nullfd)
333
        fin = os.fdopen(newfd, 'rb')
334
    if _testfileno(uout, stdout):
335
        newfd = os.dup(uout.fileno())
336
        os.dup2(stderr.fileno(), uout.fileno())
337
        fout = os.fdopen(newfd, 'wb')
338
    return fin, fout
339

340

341
342
343
344
345
346
347
348
def restorestdio(uin, uout, fin, fout):
    """Restore (uin, uout) streams from possibly duplicated (fin, fout)"""
    uout.flush()
    for f, uif in [(fin, uin), (fout, uout)]:
        if f is not uif:
            os.dup2(f.fileno(), uif.fileno())
            f.close()

349

350
351
def shellenviron(environ=None):
    """return environ with optional override, useful for shelling out"""
352

353
    def py2shell(val):
354
        """convert python object into string that is useful to shell"""
355
        if val is None or val is False:
356
            return b'0'
357
        if val is True:
358
            return b'1'
359
        return pycompat.bytestr(val)
360

361
362
    env = dict(encoding.environ)
    if environ:
363
        env.update((k, py2shell(v)) for k, v in pycompat.iteritems(environ))
364
    env[b'HG'] = hgexecutable()
365
366
    return env

367

368
if pycompat.iswindows:
369

370
    def shelltonative(cmd, env):
371
372
373
        return platform.shelltocmdexe(  # pytype: disable=module-attr
            cmd, shellenviron(env)
        )
374
375

    tonativestr = encoding.strfromlocal
376
else:
377

378
379
380
    def shelltonative(cmd, env):
        return cmd

381
382
    tonativestr = pycompat.identity

383

384
385
386
387
388
def tonativeenv(env):
    '''convert the environment from bytes to strings suitable for Popen(), etc.
    '''
    return pycompat.rapply(tonativestr, env)

389

390
391
392
393
394
395
396
397
398
399
400
401
402
def system(cmd, environ=None, cwd=None, out=None):
    '''enhanced shell command execution.
    run with environment maybe modified, maybe in different dir.

    if out is specified, it is assumed to be a file-like object that has a
    write() method. stdout and stderr will be redirected to out.'''
    try:
        stdout.flush()
    except Exception:
        pass
    cmd = quotecommand(cmd)
    env = shellenviron(environ)
    if out is None or isstdout(out):
403
404
405
406
407
408
409
        rc = subprocess.call(
            tonativestr(cmd),
            shell=True,
            close_fds=closefds,
            env=tonativeenv(env),
            cwd=pycompat.rapply(tonativestr, cwd),
        )
410
    else:
411
412
413
414
415
416
417
418
419
        proc = subprocess.Popen(
            tonativestr(cmd),
            shell=True,
            close_fds=closefds,
            env=tonativeenv(env),
            cwd=pycompat.rapply(tonativestr, cwd),
            stdout=subprocess.PIPE,
            stderr=subprocess.STDOUT,
        )
420
        for line in iter(proc.stdout.readline, b''):
421
422
423
            out.write(line)
        proc.wait()
        rc = proc.returncode
424
    if pycompat.sysplatform == b'OpenVMS' and rc & 1:
425
426
427
        rc = 0
    return rc

428

429
430
431
432
_is_gui = None


def _gui():
433
434
    '''Are we running in a GUI?'''
    if pycompat.isdarwin:
435
        if b'SSH_CONNECTION' in encoding.environ:
436
437
438
439
440
441
442
443
444
            # handle SSH access to a box where the user is logged in
            return False
        elif getattr(osutil, 'isgui', None):
            # check if a CoreGraphics session is available
            return osutil.isgui()
        else:
            # pure build; use a safe default
            return True
    else:
445
        return pycompat.iswindows or encoding.environ.get(b"DISPLAY")
446

447

448
449
450
451
452
453
454
def gui():
    global _is_gui
    if _is_gui is None:
        _is_gui = _gui()
    return _is_gui


455
456
457
458
459
460
461
def hgcmd():
    """Return the command used to execute current hg

    This is different from hgexecutable() because on Windows we want
    to avoid things opening new shell windows like batch files, so we
    get either the python call or current executable.
    """
462
    if resourceutil.mainfrozen():
463
        if getattr(sys, 'frozen', None) == 'macosx_app':
464
            # Env variable set by py2app
465
            return [encoding.environ[b'EXECUTABLEPATH']]
466
467
468
469
        else:
            return [pycompat.sysexecutable]
    return _gethgcmd()

470

471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
def rundetached(args, condfn):
    """Execute the argument list in a detached process.

    condfn is a callable which is called repeatedly and should return
    True once the child process is known to have started successfully.
    At this point, the child process PID is returned. If the child
    process fails to start or finishes before condfn() evaluates to
    True, return -1.
    """
    # Windows case is easier because the child process is either
    # successfully starting and validating the condition or exiting
    # on failure. We just poll on its PID. On Unix, if the child
    # process fails to start, it will be left in a zombie state until
    # the parent wait on it, which we cannot do since we expect a long
    # running process on success. Instead we listen for SIGCHLD telling
    # us our child process terminated.
    terminated = set()
488

489
490
    def handler(signum, frame):
        terminated.add(os.wait())
491

492
493
494
495
496
497
498
    prevhandler = None
    SIGCHLD = getattr(signal, 'SIGCHLD', None)
    if SIGCHLD is not None:
        prevhandler = signal.signal(SIGCHLD, handler)
    try:
        pid = spawndetached(args)
        while not condfn():
499
            if (pid in terminated or not testpid(pid)) and not condfn():
500
501
502
503
504
505
                return -1
            time.sleep(0.1)
        return pid
    finally:
        if prevhandler is not None:
            signal.signal(signal.SIGCHLD, prevhandler)
506

507

508
@contextlib.contextmanager
509
def uninterruptible(warn):
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
    """Inhibit SIGINT handling on a region of code.

    Note that if this is called in a non-main thread, it turns into a no-op.

    Args:
      warn: A callable which takes no arguments, and returns True if the
            previous signal handling should be restored.
    """

    oldsiginthandler = [signal.getsignal(signal.SIGINT)]
    shouldbail = []

    def disabledsiginthandler(*args):
        if warn():
            signal.signal(signal.SIGINT, oldsiginthandler[0])
            del oldsiginthandler[0]
        shouldbail.append(True)

    try:
        try:
            signal.signal(signal.SIGINT, disabledsiginthandler)
        except ValueError:
            # wrong thread, oh well, we tried
            del oldsiginthandler[0]
        yield
    finally:
        if oldsiginthandler:
            signal.signal(signal.SIGINT, oldsiginthandler[0])
        if shouldbail:
            raise KeyboardInterrupt
540

541

542
543
544
545
546
if pycompat.iswindows:
    # no fork on Windows, but we can create a detached process
    # https://msdn.microsoft.com/en-us/library/windows/desktop/ms684863.aspx
    # No stdlib constant exists for this value
    DETACHED_PROCESS = 0x00000008
547
548
549
    # Following creation flags might create a console GUI window.
    # Using subprocess.CREATE_NEW_CONSOLE might helps.
    # See https://phab.mercurial-scm.org/D1701 for discussion
550
551
552
553
    _creationflags = (
        DETACHED_PROCESS
        | subprocess.CREATE_NEW_PROCESS_GROUP  # pytype: disable=module-attr
    )
554

555
    def runbgcommand(
556
557
558
559
560
561
562
        script,
        env,
        shell=False,
        stdout=None,
        stderr=None,
        ensurestart=True,
        record_wait=None,
563
    ):
564
565
566
        '''Spawn a command without waiting for it to finish.'''
        # we can't use close_fds *and* redirect stdin. I'm not sure that we
        # need to because the detached process has no console connection.
567
        p = subprocess.Popen(
568
            tonativestr(script),
569
570
571
572
573
574
575
            shell=shell,
            env=tonativeenv(env),
            close_fds=True,
            creationflags=_creationflags,
            stdout=stdout,
            stderr=stderr,
        )
576
577
        if record_wait is not None:
            record_wait(p.wait)
578
579


580
else:
581

582
    def runbgcommand(
583
584
585
586
587
588
589
        cmd,
        env,
        shell=False,
        stdout=None,
        stderr=None,
        ensurestart=True,
        record_wait=None,
590
    ):
591
592
593
594
595
596
597
598
        '''Spawn a command without waiting for it to finish.


        When `record_wait` is not None, the spawned process will not be fully
        detached and the `record_wait` argument will be called with a the
        `Subprocess.wait` function for the spawned process.  This is mostly
        useful for developers that need to make sure the spawned process
        finished before a certain point. (eg: writing test)'''
599
600
601
602
603
        if pycompat.isdarwin:
            # avoid crash in CoreFoundation in case another thread
            # calls gui() while we're calling fork().
            gui()

604
605
        # double-fork to completely detach from the parent process
        # based on http://code.activestate.com/recipes/278731
606
607
608
609
        if record_wait is None:
            pid = os.fork()
            if pid:
                if not ensurestart:
610
611
612
613
614
615
616
                    # Even though we're not waiting on the child process,
                    # we still must call waitpid() on it at some point so
                    # it's not a zombie/defunct. This is especially relevant for
                    # chg since the parent process won't die anytime soon.
                    # We use a thread to make the overhead tiny.
                    def _do_wait():
                        os.waitpid(pid, 0)
617

618
                    threading.Thread(target=_do_wait, daemon=True).start()
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
                    return
                # Parent process
                (_pid, status) = os.waitpid(pid, 0)
                if os.WIFEXITED(status):
                    returncode = os.WEXITSTATUS(status)
                else:
                    returncode = -(os.WTERMSIG(status))
                if returncode != 0:
                    # The child process's return code is 0 on success, an errno
                    # value on failure, or 255 if we don't have a valid errno
                    # value.
                    #
                    # (It would be slightly nicer to return the full exception info
                    # over a pipe as the subprocess module does.  For now it
                    # doesn't seem worth adding that complexity here, though.)
                    if returncode == 255:
                        returncode = errno.EINVAL
                    raise OSError(
                        returncode,
                        b'error running %r: %s'
                        % (cmd, os.strerror(returncode)),
                    )
641
                return
642
643
644

        returncode = 255
        try:
645
646
647
            if record_wait is None:
                # Start a new session
                os.setsid()
648

649
            stdin = open(os.devnull, b'r')
650
            if stdout is None:
651
                stdout = open(os.devnull, b'w')
652
            if stderr is None:
653
                stderr = open(os.devnull, b'w')
654
655
656

            # connect stdin to devnull to make sure the subprocess can't
            # muck up that stream for mercurial.
657
            p = subprocess.Popen(
658
659
660
661
662
663
664
665
                cmd,
                shell=shell,
                env=env,
                close_fds=True,
                stdin=stdin,
                stdout=stdout,
                stderr=stderr,
            )
666
667
            if record_wait is not None:
                record_wait(p.wait)
668
669
            returncode = 0
        except EnvironmentError as ex:
670
            returncode = ex.errno & 0xFF
671
672
673
674
675
676
677
678
679
            if returncode == 0:
                # This shouldn't happen, but just in case make sure the
                # return code is never 0 here.
                returncode = 255
        except Exception:
            returncode = 255
        finally:
            # mission accomplished, this child needs to exit and not
            # continue the hg process here.
680
681
            if record_wait is None:
                os._exit(returncode)