procutil.py 21.4 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
# procutil.py - utility for managing processes and executable environment
#
#  Copyright 2005 K. Thananchayan <thananck@yahoo.com>
#  Copyright 2005-2007 Matt Mackall <mpm@selenic.com>
#  Copyright 2006 Vadim Gelfer <vadim.gelfer@gmail.com>
#
# This software may be used and distributed according to the terms of the
# GNU General Public License version 2 or any later version.

from __future__ import absolute_import

12
import contextlib
13
import errno
14
15
16
17
18
import io
import os
import signal
import subprocess
import sys
19
import threading
20
21
22
import time

from ..i18n import _
23
24
25
26
from ..pycompat import (
    getattr,
    open,
)
27
28
29
30
31
32
33
34

from .. import (
    encoding,
    error,
    policy,
    pycompat,
)

35
36
37
# Import like this to keep import-checker happy
from ..utils import resourceutil

38
osutil = policy.importmod('osutil')
39

40
41
42
43
44
if pycompat.iswindows:
    from .. import windows as platform
else:
    from .. import posix as platform

45

46
47
48
49
50
51
def isatty(fp):
    try:
        return fp.isatty()
    except AttributeError:
        return False

52

53
54
55
class LineBufferedWrapper(object):
    def __init__(self, orig):
        self.orig = orig
56

57
58
    def __getattr__(self, attr):
        return getattr(self.orig, attr)
59

60
61
62
63
64
65
    def write(self, s):
        orig = self.orig
        res = orig.write(s)
        if s.endswith(b'\n'):
            orig.flush()
        return res
66

67
68

io.BufferedIOBase.register(LineBufferedWrapper)
69
70


71
72
73
74
75
76
77
78
79
80
81
82
def make_line_buffered(stream):
    if pycompat.ispy3 and not isinstance(stream, io.BufferedIOBase):
        # On Python 3, buffered streams can be expected to subclass
        # BufferedIOBase. This is definitively the case for the streams
        # initialized by the interpreter. For unbuffered streams, we don't need
        # to emulate line buffering.
        return stream
    if isinstance(stream, LineBufferedWrapper):
        return stream
    return LineBufferedWrapper(stream)


83
84
85
86
87
88
89
90
91
92
if pycompat.ispy3:
    # TODO: .buffer might not exist if std streams were replaced; we'll need
    # a silly wrapper to make a bytes stream backed by a unicode one.
    stdin = sys.stdin.buffer
    stdout = sys.stdout.buffer
    stderr = sys.stderr.buffer
else:
    stdin = sys.stdin
    stdout = sys.stdout
    stderr = sys.stderr
93

94
95
96
if pycompat.iswindows:
    stdout = platform.winstdout(stdout)

97
98
# glibc determines buffering on first write to stdout - if we replace a TTY
# destined stdout with a pipe destined stdout (e.g. pager), we want line
99
# buffering.
100
if isatty(stdout):
101
    if pycompat.ispy3 or pycompat.iswindows:
102
        # On Python 3, buffered binary streams can't be set line-buffered.
103
        # On Python 2, Windows doesn't support line buffering.
104
        # Therefore we have a wrapper that implements line buffering.
105
        stdout = make_line_buffered(stdout)
106
    else:
107
        stdout = os.fdopen(stdout.fileno(), 'wb', 1)
108

109
110
111
112
113
114
115
116
117
118
119
120
# stderr should be unbuffered
if pycompat.ispy3:
    # On Python 3, buffered streams may expose an underlying raw stream. This is
    # definitively the case for the streams initialized by the interpreter. If
    # the attribute isn't present, the stream is already unbuffered or doesn't
    # expose an underlying raw stream, in which case we use the stream as-is.
    stderr = getattr(stderr, 'raw', stderr)
elif pycompat.iswindows:
    # On Windows, stderr is buffered at least when connected to a pipe.
    stderr = os.fdopen(stderr.fileno(), 'wb', 0)
# On other platforms, stderr is always unbuffered.

121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146

findexe = platform.findexe
_gethgcmd = platform.gethgcmd
getuser = platform.getuser
getpid = os.getpid
hidewindow = platform.hidewindow
readpipe = platform.readpipe
setbinary = platform.setbinary
setsignalhandler = platform.setsignalhandler
shellquote = platform.shellquote
shellsplit = platform.shellsplit
spawndetached = platform.spawndetached
sshargs = platform.sshargs
testpid = platform.testpid

try:
    setprocname = osutil.setprocname
except AttributeError:
    pass
try:
    unblocksignal = osutil.unblocksignal
except AttributeError:
    pass

closefds = pycompat.isposix

147

148
def explainexit(code):
149
    """return a message describing a subprocess status
150
151
    (codes from kill are negative - not os.system/wait encoding)"""
    if code >= 0:
152
153
        return _(b"exited with status %d") % code
    return _(b"killed by signal %d") % -code
154

155

156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
class _pfile(object):
    """File-like wrapper for a stream opened by subprocess.Popen()"""

    def __init__(self, proc, fp):
        self._proc = proc
        self._fp = fp

    def close(self):
        # unlike os.popen(), this returns an integer in subprocess coding
        self._fp.close()
        return self._proc.wait()

    def __iter__(self):
        return iter(self._fp)

    def __getattr__(self, attr):
        return getattr(self._fp, attr)

    def __enter__(self):
        return self

    def __exit__(self, exc_type, exc_value, exc_tb):
        self.close()

180

181
182
def popen(cmd, mode=b'rb', bufsize=-1):
    if mode == b'rb':
183
        return _popenreader(cmd, bufsize)
184
    elif mode == b'wb':
185
        return _popenwriter(cmd, bufsize)
186
    raise error.ProgrammingError(b'unsupported mode: %r' % mode)
187

188

189
def _popenreader(cmd, bufsize):
190
    p = subprocess.Popen(
191
        tonativestr(cmd),
192
193
194
195
196
        shell=True,
        bufsize=bufsize,
        close_fds=closefds,
        stdout=subprocess.PIPE,
    )
197
198
    return _pfile(p, p.stdout)

199

200
def _popenwriter(cmd, bufsize):
201
    p = subprocess.Popen(
202
        tonativestr(cmd),
203
204
205
206
207
        shell=True,
        bufsize=bufsize,
        close_fds=closefds,
        stdin=subprocess.PIPE,
    )
208
209
    return _pfile(p, p.stdin)

210

211
def popen2(cmd, env=None):
212
213
214
    # Setting bufsize to -1 lets the system decide the buffer size.
    # The default for bufsize is 0, meaning unbuffered. This leads to
    # poor performance on Mac OS X: http://bugs.python.org/issue4194
215
216
217
218
219
220
221
222
223
    p = subprocess.Popen(
        tonativestr(cmd),
        shell=True,
        bufsize=-1,
        close_fds=closefds,
        stdin=subprocess.PIPE,
        stdout=subprocess.PIPE,
        env=tonativeenv(env),
    )
224
225
    return p.stdin, p.stdout

226

227
228
def popen3(cmd, env=None):
    stdin, stdout, stderr, p = popen4(cmd, env)
229
230
    return stdin, stdout, stderr

231

232
def popen4(cmd, env=None, bufsize=-1):
233
234
235
236
237
238
239
240
241
242
    p = subprocess.Popen(
        tonativestr(cmd),
        shell=True,
        bufsize=bufsize,
        close_fds=closefds,
        stdin=subprocess.PIPE,
        stdout=subprocess.PIPE,
        stderr=subprocess.PIPE,
        env=tonativeenv(env),
    )
243
244
    return p.stdin, p.stdout, p.stderr, p

245

246
247
def pipefilter(s, cmd):
    '''filter string S through command CMD, returning its output'''
248
249
250
251
252
253
254
    p = subprocess.Popen(
        tonativestr(cmd),
        shell=True,
        close_fds=closefds,
        stdin=subprocess.PIPE,
        stdout=subprocess.PIPE,
    )
255
256
257
    pout, perr = p.communicate(s)
    return pout

258

259
260
261
262
263
264
265
def tempfilter(s, cmd):
    '''filter string S through a pair of temporary files with CMD.
    CMD is used as a template to create the real command to be run,
    with the strings INFILE and OUTFILE replaced by the real names of
    the temporary files generated.'''
    inname, outname = None, None
    try:
266
        infd, inname = pycompat.mkstemp(prefix=b'hg-filter-in-')
267
        fp = os.fdopen(infd, 'wb')
268
269
        fp.write(s)
        fp.close()
270
        outfd, outname = pycompat.mkstemp(prefix=b'hg-filter-out-')
271
        os.close(outfd)
272
273
        cmd = cmd.replace(b'INFILE', inname)
        cmd = cmd.replace(b'OUTFILE', outname)
274
        code = system(cmd)
275
        if pycompat.sysplatform == b'OpenVMS' and code & 1:
276
277
            code = 0
        if code:
278
            raise error.Abort(
279
                _(b"command '%s' failed: %s") % (cmd, explainexit(code))
280
            )
281
        with open(outname, b'rb') as fp:
282
283
284
285
286
287
288
289
290
291
292
293
294
            return fp.read()
    finally:
        try:
            if inname:
                os.unlink(inname)
        except OSError:
            pass
        try:
            if outname:
                os.unlink(outname)
        except OSError:
            pass

295

296
_filtertable = {
297
298
    b'tempfile:': tempfilter,
    b'pipe:': pipefilter,
299
300
}

301

302
def filter(s, cmd):
303
304
    """filter a string through a command that transforms its input to its
    output"""
305
    for name, fn in pycompat.iteritems(_filtertable):
306
        if cmd.startswith(name):
307
            return fn(s, cmd[len(name) :].lstrip())
308
309
    return pipefilter(s, cmd)

310

311
312
_hgexecutable = None

313

314
315
316
317
318
319
def hgexecutable():
    """return location of the 'hg' executable.

    Defaults to $HG or 'hg' in the search path.
    """
    if _hgexecutable is None:
320
        hg = encoding.environ.get(b'HG')
321
        mainmod = sys.modules['__main__']
322
323
        if hg:
            _sethgexecutable(hg)
324
        elif resourceutil.mainfrozen():
325
            if getattr(sys, 'frozen', None) == 'macosx_app':
326
                # Env variable set by py2app
327
                _sethgexecutable(encoding.environ[b'EXECUTABLEPATH'])
328
329
            else:
                _sethgexecutable(pycompat.sysexecutable)
330
331
        elif (
            not pycompat.iswindows
332
            and os.path.basename(getattr(mainmod, '__file__', '')) == 'hg'
333
        ):
334
335
            _sethgexecutable(pycompat.fsencode(mainmod.__file__))
        else:
336
            _sethgexecutable(
337
                findexe(b'hg') or os.path.basename(pycompat.sysargv[0])
338
            )
339
340
    return _hgexecutable

341

342
343
344
345
346
def _sethgexecutable(path):
    """set location of the 'hg' executable"""
    global _hgexecutable
    _hgexecutable = path

347

348
349
350
351
352
def _testfileno(f, stdf):
    fileno = getattr(f, 'fileno', None)
    try:
        return fileno and fileno() == stdf.fileno()
    except io.UnsupportedOperation:
353
354
        return False  # fileno() raised UnsupportedOperation

355
356
357
358

def isstdin(f):
    return _testfileno(f, sys.__stdin__)

359

360
361
362
def isstdout(f):
    return _testfileno(f, sys.__stdout__)

363

364
def protectstdio(uin, uout):
365
366
367
368
    """Duplicate streams and redirect original if (uin, uout) are stdio

    If uin is stdin, it's redirected to /dev/null. If uout is stdout, it's
    redirected to stderr so the output is still readable.
369
370
371
372
373
374

    Returns (fin, fout) which point to the original (uin, uout) fds, but
    may be copy of (uin, uout). The returned streams can be considered
    "owned" in that print(), exec(), etc. never reach to them.
    """
    uout.flush()
375
    fin, fout = uin, uout
376
    if _testfileno(uin, stdin):
377
        newfd = os.dup(uin.fileno())
378
        nullfd = os.open(os.devnull, os.O_RDONLY)
379
        os.dup2(nullfd, uin.fileno())
380
        os.close(nullfd)
381
        fin = os.fdopen(newfd, 'rb')
382
    if _testfileno(uout, stdout):
383
        newfd = os.dup(uout.fileno())
384
        os.dup2(stderr.fileno(), uout.fileno())
385
        fout = os.fdopen(newfd, 'wb')
386
    return fin, fout
387

388

389
390
391
392
393
394
395
396
def restorestdio(uin, uout, fin, fout):
    """Restore (uin, uout) streams from possibly duplicated (fin, fout)"""
    uout.flush()
    for f, uif in [(fin, uin), (fout, uout)]:
        if f is not uif:
            os.dup2(f.fileno(), uif.fileno())
            f.close()

397

398
399
def shellenviron(environ=None):
    """return environ with optional override, useful for shelling out"""
400

401
    def py2shell(val):
402
        """convert python object into string that is useful to shell"""
403
        if val is None or val is False:
404
            return b'0'
405
        if val is True:
406
            return b'1'
407
        return pycompat.bytestr(val)
408

409
410
    env = dict(encoding.environ)
    if environ:
411
        env.update((k, py2shell(v)) for k, v in pycompat.iteritems(environ))
412
    env[b'HG'] = hgexecutable()
413
414
    return env

415

416
if pycompat.iswindows:
417

418
    def shelltonative(cmd, env):
419
420
421
        return platform.shelltocmdexe(  # pytype: disable=module-attr
            cmd, shellenviron(env)
        )
422
423

    tonativestr = encoding.strfromlocal
424
else:
425

426
427
428
    def shelltonative(cmd, env):
        return cmd

429
430
    tonativestr = pycompat.identity

431

432
433
434
435
436
def tonativeenv(env):
    '''convert the environment from bytes to strings suitable for Popen(), etc.
    '''
    return pycompat.rapply(tonativestr, env)

437

438
439
440
441
442
443
444
445
446
447
448
449
def system(cmd, environ=None, cwd=None, out=None):
    '''enhanced shell command execution.
    run with environment maybe modified, maybe in different dir.

    if out is specified, it is assumed to be a file-like object that has a
    write() method. stdout and stderr will be redirected to out.'''
    try:
        stdout.flush()
    except Exception:
        pass
    env = shellenviron(environ)
    if out is None or isstdout(out):
450
451
452
453
454
455
456
        rc = subprocess.call(
            tonativestr(cmd),
            shell=True,
            close_fds=closefds,
            env=tonativeenv(env),
            cwd=pycompat.rapply(tonativestr, cwd),
        )
457
    else:
458
459
460
461
462
463
464
465
466
        proc = subprocess.Popen(
            tonativestr(cmd),
            shell=True,
            close_fds=closefds,
            env=tonativeenv(env),
            cwd=pycompat.rapply(tonativestr, cwd),
            stdout=subprocess.PIPE,
            stderr=subprocess.STDOUT,
        )
467
        for line in iter(proc.stdout.readline, b''):
468
469
470
            out.write(line)
        proc.wait()
        rc = proc.returncode
471
    if pycompat.sysplatform == b'OpenVMS' and rc & 1:
472
473
474
        rc = 0
    return rc

475

476
477
478
479
_is_gui = None


def _gui():
480
481
    '''Are we running in a GUI?'''
    if pycompat.isdarwin:
482
        if b'SSH_CONNECTION' in encoding.environ:
483
484
485
486
487
488
489
490
491
            # handle SSH access to a box where the user is logged in
            return False
        elif getattr(osutil, 'isgui', None):
            # check if a CoreGraphics session is available
            return osutil.isgui()
        else:
            # pure build; use a safe default
            return True
    else:
492
        return pycompat.iswindows or encoding.environ.get(b"DISPLAY")
493

494

495
496
497
498
499
500
501
def gui():
    global _is_gui
    if _is_gui is None:
        _is_gui = _gui()
    return _is_gui


502
503
504
505
506
507
508
def hgcmd():
    """Return the command used to execute current hg

    This is different from hgexecutable() because on Windows we want
    to avoid things opening new shell windows like batch files, so we
    get either the python call or current executable.
    """
509
    if resourceutil.mainfrozen():
510
        if getattr(sys, 'frozen', None) == 'macosx_app':
511
            # Env variable set by py2app
512
            return [encoding.environ[b'EXECUTABLEPATH']]
513
514
515
516
        else:
            return [pycompat.sysexecutable]
    return _gethgcmd()

517

518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
def rundetached(args, condfn):
    """Execute the argument list in a detached process.

    condfn is a callable which is called repeatedly and should return
    True once the child process is known to have started successfully.
    At this point, the child process PID is returned. If the child
    process fails to start or finishes before condfn() evaluates to
    True, return -1.
    """
    # Windows case is easier because the child process is either
    # successfully starting and validating the condition or exiting
    # on failure. We just poll on its PID. On Unix, if the child
    # process fails to start, it will be left in a zombie state until
    # the parent wait on it, which we cannot do since we expect a long
    # running process on success. Instead we listen for SIGCHLD telling
    # us our child process terminated.
    terminated = set()
535

536
537
    def handler(signum, frame):
        terminated.add(os.wait())
538

539
540
541
542
543
544
545
    prevhandler = None
    SIGCHLD = getattr(signal, 'SIGCHLD', None)
    if SIGCHLD is not None:
        prevhandler = signal.signal(SIGCHLD, handler)
    try:
        pid = spawndetached(args)
        while not condfn():
546
            if (pid in terminated or not testpid(pid)) and not condfn():
547
548
549
550
551
552
                return -1
            time.sleep(0.1)
        return pid
    finally:
        if prevhandler is not None:
            signal.signal(signal.SIGCHLD, prevhandler)
553

554

555
@contextlib.contextmanager
556
def uninterruptible(warn):
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
    """Inhibit SIGINT handling on a region of code.

    Note that if this is called in a non-main thread, it turns into a no-op.

    Args:
      warn: A callable which takes no arguments, and returns True if the
            previous signal handling should be restored.
    """

    oldsiginthandler = [signal.getsignal(signal.SIGINT)]
    shouldbail = []

    def disabledsiginthandler(*args):
        if warn():
            signal.signal(signal.SIGINT, oldsiginthandler[0])
            del oldsiginthandler[0]
        shouldbail.append(True)

    try:
        try:
            signal.signal(signal.SIGINT, disabledsiginthandler)
        except ValueError:
            # wrong thread, oh well, we tried
            del oldsiginthandler[0]
        yield
    finally:
        if oldsiginthandler:
            signal.signal(signal.SIGINT, oldsiginthandler[0])
        if shouldbail:
            raise KeyboardInterrupt
587

588

589
590
591
592
593
if pycompat.iswindows:
    # no fork on Windows, but we can create a detached process
    # https://msdn.microsoft.com/en-us/library/windows/desktop/ms684863.aspx
    # No stdlib constant exists for this value
    DETACHED_PROCESS = 0x00000008
594
595
596
    # Following creation flags might create a console GUI window.
    # Using subprocess.CREATE_NEW_CONSOLE might helps.
    # See https://phab.mercurial-scm.org/D1701 for discussion
597
598
599
600
    _creationflags = (
        DETACHED_PROCESS
        | subprocess.CREATE_NEW_PROCESS_GROUP  # pytype: disable=module-attr
    )
601

602
    def runbgcommand(
603
604
605
606
607
608
609
        script,
        env,
        shell=False,
        stdout=None,
        stderr=None,
        ensurestart=True,
        record_wait=None,
610
    ):
611
612
613
        '''Spawn a command without waiting for it to finish.'''
        # we can't use close_fds *and* redirect stdin. I'm not sure that we
        # need to because the detached process has no console connection.
614
        p = subprocess.Popen(
615
            tonativestr(script),
616
617
618
619
620
621
622
            shell=shell,
            env=tonativeenv(env),
            close_fds=True,
            creationflags=_creationflags,
            stdout=stdout,
            stderr=stderr,
        )
623
624
        if record_wait is not None:
            record_wait(p.wait)
625
626


627
else:
628

629
    def runbgcommand(
630
631
632
633
634
635
636
        cmd,
        env,
        shell=False,
        stdout=None,
        stderr=None,
        ensurestart=True,
        record_wait=None,
637
    ):
638
639
640
641
642
643
644
645
        '''Spawn a command without waiting for it to finish.


        When `record_wait` is not None, the spawned process will not be fully
        detached and the `record_wait` argument will be called with a the
        `Subprocess.wait` function for the spawned process.  This is mostly
        useful for developers that need to make sure the spawned process
        finished before a certain point. (eg: writing test)'''
646
647
648
649
650
        if pycompat.isdarwin:
            # avoid crash in CoreFoundation in case another thread
            # calls gui() while we're calling fork().
            gui()

651
652
        # double-fork to completely detach from the parent process
        # based on http://code.activestate.com/recipes/278731
653
654
655
656
        if record_wait is None:
            pid = os.fork()
            if pid:
                if not ensurestart:
657
658
659
660
661
662
663
                    # Even though we're not waiting on the child process,
                    # we still must call waitpid() on it at some point so
                    # it's not a zombie/defunct. This is especially relevant for
                    # chg since the parent process won't die anytime soon.
                    # We use a thread to make the overhead tiny.
                    def _do_wait():
                        os.waitpid(pid, 0)
664

665
666
667
                    t = threading.Thread(target=_do_wait)
                    t.daemon = True
                    t.start()
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
                    return
                # Parent process
                (_pid, status) = os.waitpid(pid, 0)
                if os.WIFEXITED(status):
                    returncode = os.WEXITSTATUS(status)
                else:
                    returncode = -(os.WTERMSIG(status))
                if returncode != 0:
                    # The child process's return code is 0 on success, an errno
                    # value on failure, or 255 if we don't have a valid errno
                    # value.
                    #
                    # (It would be slightly nicer to return the full exception info
                    # over a pipe as the subprocess module does.  For now it
                    # doesn't seem worth adding that complexity here, though.)
                    if returncode == 255:
                        returncode = errno.EINVAL
                    raise OSError(
                        returncode,
                        b'error running %r: %s'
                        % (cmd, os.strerror(returncode)),
                    )
690
                return
691
692
693

        returncode = 255
        try:
694
695
696
            if record_wait is None:
                # Start a new session
                os.setsid()
697

698
            stdin = open(os.devnull, b'r')
699
            if stdout is None:
700
                stdout = open(os.devnull, b'w')
701
            if stderr is None:
702
                stderr = open(os.devnull, b'w')
703
704
705

            # connect stdin to devnull to make sure the subprocess can't
            # muck up that stream for mercurial.
706
            p = subprocess.Popen(
707
708
709
710
711
712
713
714
                cmd,
                shell=shell,
                env=env,
                close_fds=True,
                stdin=stdin,
                stdout=stdout,
                stderr=stderr,
            )
715
716
            if record_wait is not None:
                record_wait(p.wait)
717
718
            returncode = 0
        except EnvironmentError as ex:
719
            returncode = ex.errno & 0xFF
720
721
722
723
724
725
726
727
728
            if returncode == 0:
                # This shouldn't happen, but just in case make sure the
                # return code is never 0 here.
                returncode = 255
        except Exception:
            returncode = 255
        finally:
            # mission accomplished, this child needs to exit and not
            # continue the hg process here.
729
730
            if record_wait is None:
                os._exit(returncode)