procutil.py 21 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
# procutil.py - utility for managing processes and executable environment
#
#  Copyright 2005 K. Thananchayan <thananck@yahoo.com>
#  Copyright 2005-2007 Matt Mackall <mpm@selenic.com>
#  Copyright 2006 Vadim Gelfer <vadim.gelfer@gmail.com>
#
# This software may be used and distributed according to the terms of the
# GNU General Public License version 2 or any later version.

from __future__ import absolute_import

12
import contextlib
13
import errno
14
15
16
17
18
import io
import os
import signal
import subprocess
import sys
19
import threading
20
21
22
import time

from ..i18n import _
23
24
25
26
from ..pycompat import (
    getattr,
    open,
)
27
28
29
30
31
32
33
34

from .. import (
    encoding,
    error,
    policy,
    pycompat,
)

35
36
37
# Import like this to keep import-checker happy
from ..utils import resourceutil

38
osutil = policy.importmod('osutil')
39

40
41
42
43
44
if pycompat.iswindows:
    from .. import windows as platform
else:
    from .. import posix as platform

45

46
47
48
49
50
51
def isatty(fp):
    try:
        return fp.isatty()
    except AttributeError:
        return False

52

53
54
55
class LineBufferedWrapper(object):
    def __init__(self, orig):
        self.orig = orig
56

57
58
    def __getattr__(self, attr):
        return getattr(self.orig, attr)
59

60
61
62
63
64
65
    def write(self, s):
        orig = self.orig
        res = orig.write(s)
        if s.endswith(b'\n'):
            orig.flush()
        return res
66

67
68

io.BufferedIOBase.register(LineBufferedWrapper)
69
70


71
72
73
74
75
76
77
78
79
80
81
82
def make_line_buffered(stream):
    if pycompat.ispy3 and not isinstance(stream, io.BufferedIOBase):
        # On Python 3, buffered streams can be expected to subclass
        # BufferedIOBase. This is definitively the case for the streams
        # initialized by the interpreter. For unbuffered streams, we don't need
        # to emulate line buffering.
        return stream
    if isinstance(stream, LineBufferedWrapper):
        return stream
    return LineBufferedWrapper(stream)


83
if pycompat.ispy3:
84
    # Python 3 implements its own I/O streams.
85
86
87
88
89
    # TODO: .buffer might not exist if std streams were replaced; we'll need
    # a silly wrapper to make a bytes stream backed by a unicode one.
    stdin = sys.stdin.buffer
    stdout = sys.stdout.buffer
    stderr = sys.stderr.buffer
90
91
92
    if isatty(stdout):
        # The standard library doesn't offer line-buffered binary streams.
        stdout = make_line_buffered(stdout)
93
else:
94
    # Python 2 uses the I/O streams provided by the C library.
95
96
97
    stdin = sys.stdin
    stdout = sys.stdout
    stderr = sys.stderr
98
99
100
101
102
103
104
105
106
107
108
    if isatty(stdout):
        if pycompat.iswindows:
            # Work around size limit when writing to console.
            stdout = platform.winstdout(stdout)
            # The Windows C runtime library doesn't support line buffering.
            stdout = make_line_buffered(stdout)
        else:
            # glibc determines buffering on first write to stdout - if we
            # replace a TTY destined stdout with a pipe destined stdout (e.g.
            # pager), we want line buffering.
            stdout = os.fdopen(stdout.fileno(), 'wb', 1)
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135


findexe = platform.findexe
_gethgcmd = platform.gethgcmd
getuser = platform.getuser
getpid = os.getpid
hidewindow = platform.hidewindow
readpipe = platform.readpipe
setbinary = platform.setbinary
setsignalhandler = platform.setsignalhandler
shellquote = platform.shellquote
shellsplit = platform.shellsplit
spawndetached = platform.spawndetached
sshargs = platform.sshargs
testpid = platform.testpid

try:
    setprocname = osutil.setprocname
except AttributeError:
    pass
try:
    unblocksignal = osutil.unblocksignal
except AttributeError:
    pass

closefds = pycompat.isposix

136

137
def explainexit(code):
138
    """return a message describing a subprocess status
139
140
    (codes from kill are negative - not os.system/wait encoding)"""
    if code >= 0:
141
142
        return _(b"exited with status %d") % code
    return _(b"killed by signal %d") % -code
143

144

145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
class _pfile(object):
    """File-like wrapper for a stream opened by subprocess.Popen()"""

    def __init__(self, proc, fp):
        self._proc = proc
        self._fp = fp

    def close(self):
        # unlike os.popen(), this returns an integer in subprocess coding
        self._fp.close()
        return self._proc.wait()

    def __iter__(self):
        return iter(self._fp)

    def __getattr__(self, attr):
        return getattr(self._fp, attr)

    def __enter__(self):
        return self

    def __exit__(self, exc_type, exc_value, exc_tb):
        self.close()

169

170
171
def popen(cmd, mode=b'rb', bufsize=-1):
    if mode == b'rb':
172
        return _popenreader(cmd, bufsize)
173
    elif mode == b'wb':
174
        return _popenwriter(cmd, bufsize)
175
    raise error.ProgrammingError(b'unsupported mode: %r' % mode)
176

177

178
def _popenreader(cmd, bufsize):
179
    p = subprocess.Popen(
180
        tonativestr(cmd),
181
182
183
184
185
        shell=True,
        bufsize=bufsize,
        close_fds=closefds,
        stdout=subprocess.PIPE,
    )
186
187
    return _pfile(p, p.stdout)

188

189
def _popenwriter(cmd, bufsize):
190
    p = subprocess.Popen(
191
        tonativestr(cmd),
192
193
194
195
196
        shell=True,
        bufsize=bufsize,
        close_fds=closefds,
        stdin=subprocess.PIPE,
    )
197
198
    return _pfile(p, p.stdin)

199

200
def popen2(cmd, env=None):
201
202
203
    # Setting bufsize to -1 lets the system decide the buffer size.
    # The default for bufsize is 0, meaning unbuffered. This leads to
    # poor performance on Mac OS X: http://bugs.python.org/issue4194
204
205
206
207
208
209
210
211
212
    p = subprocess.Popen(
        tonativestr(cmd),
        shell=True,
        bufsize=-1,
        close_fds=closefds,
        stdin=subprocess.PIPE,
        stdout=subprocess.PIPE,
        env=tonativeenv(env),
    )
213
214
    return p.stdin, p.stdout

215

216
217
def popen3(cmd, env=None):
    stdin, stdout, stderr, p = popen4(cmd, env)
218
219
    return stdin, stdout, stderr

220

221
def popen4(cmd, env=None, bufsize=-1):
222
223
224
225
226
227
228
229
230
231
    p = subprocess.Popen(
        tonativestr(cmd),
        shell=True,
        bufsize=bufsize,
        close_fds=closefds,
        stdin=subprocess.PIPE,
        stdout=subprocess.PIPE,
        stderr=subprocess.PIPE,
        env=tonativeenv(env),
    )
232
233
    return p.stdin, p.stdout, p.stderr, p

234

235
236
def pipefilter(s, cmd):
    '''filter string S through command CMD, returning its output'''
237
238
239
240
241
242
243
    p = subprocess.Popen(
        tonativestr(cmd),
        shell=True,
        close_fds=closefds,
        stdin=subprocess.PIPE,
        stdout=subprocess.PIPE,
    )
244
245
246
    pout, perr = p.communicate(s)
    return pout

247

248
249
250
251
252
253
254
def tempfilter(s, cmd):
    '''filter string S through a pair of temporary files with CMD.
    CMD is used as a template to create the real command to be run,
    with the strings INFILE and OUTFILE replaced by the real names of
    the temporary files generated.'''
    inname, outname = None, None
    try:
255
        infd, inname = pycompat.mkstemp(prefix=b'hg-filter-in-')
256
        fp = os.fdopen(infd, 'wb')
257
258
        fp.write(s)
        fp.close()
259
        outfd, outname = pycompat.mkstemp(prefix=b'hg-filter-out-')
260
        os.close(outfd)
261
262
        cmd = cmd.replace(b'INFILE', inname)
        cmd = cmd.replace(b'OUTFILE', outname)
263
        code = system(cmd)
264
        if pycompat.sysplatform == b'OpenVMS' and code & 1:
265
266
            code = 0
        if code:
267
            raise error.Abort(
268
                _(b"command '%s' failed: %s") % (cmd, explainexit(code))
269
            )
270
        with open(outname, b'rb') as fp:
271
272
273
274
275
276
277
278
279
280
281
282
283
            return fp.read()
    finally:
        try:
            if inname:
                os.unlink(inname)
        except OSError:
            pass
        try:
            if outname:
                os.unlink(outname)
        except OSError:
            pass

284

285
_filtertable = {
286
287
    b'tempfile:': tempfilter,
    b'pipe:': pipefilter,
288
289
}

290

291
def filter(s, cmd):
292
293
    """filter a string through a command that transforms its input to its
    output"""
294
    for name, fn in pycompat.iteritems(_filtertable):
295
        if cmd.startswith(name):
296
            return fn(s, cmd[len(name) :].lstrip())
297
298
    return pipefilter(s, cmd)

299

300
301
_hgexecutable = None

302

303
304
305
306
307
308
def hgexecutable():
    """return location of the 'hg' executable.

    Defaults to $HG or 'hg' in the search path.
    """
    if _hgexecutable is None:
309
        hg = encoding.environ.get(b'HG')
310
        mainmod = sys.modules['__main__']
311
312
        if hg:
            _sethgexecutable(hg)
313
        elif resourceutil.mainfrozen():
314
            if getattr(sys, 'frozen', None) == 'macosx_app':
315
                # Env variable set by py2app
316
                _sethgexecutable(encoding.environ[b'EXECUTABLEPATH'])
317
318
            else:
                _sethgexecutable(pycompat.sysexecutable)
319
320
        elif (
            not pycompat.iswindows
321
            and os.path.basename(getattr(mainmod, '__file__', '')) == 'hg'
322
        ):
323
324
            _sethgexecutable(pycompat.fsencode(mainmod.__file__))
        else:
325
            _sethgexecutable(
326
                findexe(b'hg') or os.path.basename(pycompat.sysargv[0])
327
            )
328
329
    return _hgexecutable

330

331
332
333
334
335
def _sethgexecutable(path):
    """set location of the 'hg' executable"""
    global _hgexecutable
    _hgexecutable = path

336

337
338
339
340
341
def _testfileno(f, stdf):
    fileno = getattr(f, 'fileno', None)
    try:
        return fileno and fileno() == stdf.fileno()
    except io.UnsupportedOperation:
342
343
        return False  # fileno() raised UnsupportedOperation

344
345
346
347

def isstdin(f):
    return _testfileno(f, sys.__stdin__)

348

349
350
351
def isstdout(f):
    return _testfileno(f, sys.__stdout__)

352

353
def protectstdio(uin, uout):
354
355
356
357
    """Duplicate streams and redirect original if (uin, uout) are stdio

    If uin is stdin, it's redirected to /dev/null. If uout is stdout, it's
    redirected to stderr so the output is still readable.
358
359
360
361
362
363

    Returns (fin, fout) which point to the original (uin, uout) fds, but
    may be copy of (uin, uout). The returned streams can be considered
    "owned" in that print(), exec(), etc. never reach to them.
    """
    uout.flush()
364
    fin, fout = uin, uout
365
    if _testfileno(uin, stdin):
366
        newfd = os.dup(uin.fileno())
367
        nullfd = os.open(os.devnull, os.O_RDONLY)
368
        os.dup2(nullfd, uin.fileno())
369
        os.close(nullfd)
370
        fin = os.fdopen(newfd, 'rb')
371
    if _testfileno(uout, stdout):
372
        newfd = os.dup(uout.fileno())
373
        os.dup2(stderr.fileno(), uout.fileno())
374
        fout = os.fdopen(newfd, 'wb')
375
    return fin, fout
376

377

378
379
380
381
382
383
384
385
def restorestdio(uin, uout, fin, fout):
    """Restore (uin, uout) streams from possibly duplicated (fin, fout)"""
    uout.flush()
    for f, uif in [(fin, uin), (fout, uout)]:
        if f is not uif:
            os.dup2(f.fileno(), uif.fileno())
            f.close()

386

387
388
def shellenviron(environ=None):
    """return environ with optional override, useful for shelling out"""
389

390
    def py2shell(val):
391
        """convert python object into string that is useful to shell"""
392
        if val is None or val is False:
393
            return b'0'
394
        if val is True:
395
            return b'1'
396
        return pycompat.bytestr(val)
397

398
399
    env = dict(encoding.environ)
    if environ:
400
        env.update((k, py2shell(v)) for k, v in pycompat.iteritems(environ))
401
    env[b'HG'] = hgexecutable()
402
403
    return env

404

405
if pycompat.iswindows:
406

407
    def shelltonative(cmd, env):
408
409
410
        return platform.shelltocmdexe(  # pytype: disable=module-attr
            cmd, shellenviron(env)
        )
411
412

    tonativestr = encoding.strfromlocal
413
else:
414

415
416
417
    def shelltonative(cmd, env):
        return cmd

418
419
    tonativestr = pycompat.identity

420

421
422
423
424
425
def tonativeenv(env):
    '''convert the environment from bytes to strings suitable for Popen(), etc.
    '''
    return pycompat.rapply(tonativestr, env)

426

427
428
429
430
431
432
433
434
435
436
437
438
def system(cmd, environ=None, cwd=None, out=None):
    '''enhanced shell command execution.
    run with environment maybe modified, maybe in different dir.

    if out is specified, it is assumed to be a file-like object that has a
    write() method. stdout and stderr will be redirected to out.'''
    try:
        stdout.flush()
    except Exception:
        pass
    env = shellenviron(environ)
    if out is None or isstdout(out):
439
440
441
442
443
444
445
        rc = subprocess.call(
            tonativestr(cmd),
            shell=True,
            close_fds=closefds,
            env=tonativeenv(env),
            cwd=pycompat.rapply(tonativestr, cwd),
        )
446
    else:
447
448
449
450
451
452
453
454
455
        proc = subprocess.Popen(
            tonativestr(cmd),
            shell=True,
            close_fds=closefds,
            env=tonativeenv(env),
            cwd=pycompat.rapply(tonativestr, cwd),
            stdout=subprocess.PIPE,
            stderr=subprocess.STDOUT,
        )
456
        for line in iter(proc.stdout.readline, b''):
457
458
459
            out.write(line)
        proc.wait()
        rc = proc.returncode
460
    if pycompat.sysplatform == b'OpenVMS' and rc & 1:
461
462
463
        rc = 0
    return rc

464

465
466
467
468
_is_gui = None


def _gui():
469
470
    '''Are we running in a GUI?'''
    if pycompat.isdarwin:
471
        if b'SSH_CONNECTION' in encoding.environ:
472
473
474
475
476
477
478
479
480
            # handle SSH access to a box where the user is logged in
            return False
        elif getattr(osutil, 'isgui', None):
            # check if a CoreGraphics session is available
            return osutil.isgui()
        else:
            # pure build; use a safe default
            return True
    else:
481
        return pycompat.iswindows or encoding.environ.get(b"DISPLAY")
482

483

484
485
486
487
488
489
490
def gui():
    global _is_gui
    if _is_gui is None:
        _is_gui = _gui()
    return _is_gui


491
492
493
494
495
496
497
def hgcmd():
    """Return the command used to execute current hg

    This is different from hgexecutable() because on Windows we want
    to avoid things opening new shell windows like batch files, so we
    get either the python call or current executable.
    """
498
    if resourceutil.mainfrozen():
499
        if getattr(sys, 'frozen', None) == 'macosx_app':
500
            # Env variable set by py2app
501
            return [encoding.environ[b'EXECUTABLEPATH']]
502
503
504
505
        else:
            return [pycompat.sysexecutable]
    return _gethgcmd()

506

507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
def rundetached(args, condfn):
    """Execute the argument list in a detached process.

    condfn is a callable which is called repeatedly and should return
    True once the child process is known to have started successfully.
    At this point, the child process PID is returned. If the child
    process fails to start or finishes before condfn() evaluates to
    True, return -1.
    """
    # Windows case is easier because the child process is either
    # successfully starting and validating the condition or exiting
    # on failure. We just poll on its PID. On Unix, if the child
    # process fails to start, it will be left in a zombie state until
    # the parent wait on it, which we cannot do since we expect a long
    # running process on success. Instead we listen for SIGCHLD telling
    # us our child process terminated.
    terminated = set()
524

525
526
    def handler(signum, frame):
        terminated.add(os.wait())
527

528
529
530
531
532
533
534
    prevhandler = None
    SIGCHLD = getattr(signal, 'SIGCHLD', None)
    if SIGCHLD is not None:
        prevhandler = signal.signal(SIGCHLD, handler)
    try:
        pid = spawndetached(args)
        while not condfn():
535
            if (pid in terminated or not testpid(pid)) and not condfn():
536
537
538
539
540
541
                return -1
            time.sleep(0.1)
        return pid
    finally:
        if prevhandler is not None:
            signal.signal(signal.SIGCHLD, prevhandler)
542

543

544
@contextlib.contextmanager
545
def uninterruptible(warn):
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
    """Inhibit SIGINT handling on a region of code.

    Note that if this is called in a non-main thread, it turns into a no-op.

    Args:
      warn: A callable which takes no arguments, and returns True if the
            previous signal handling should be restored.
    """

    oldsiginthandler = [signal.getsignal(signal.SIGINT)]
    shouldbail = []

    def disabledsiginthandler(*args):
        if warn():
            signal.signal(signal.SIGINT, oldsiginthandler[0])
            del oldsiginthandler[0]
        shouldbail.append(True)

    try:
        try:
            signal.signal(signal.SIGINT, disabledsiginthandler)
        except ValueError:
            # wrong thread, oh well, we tried
            del oldsiginthandler[0]
        yield
    finally:
        if oldsiginthandler:
            signal.signal(signal.SIGINT, oldsiginthandler[0])
        if shouldbail:
            raise KeyboardInterrupt
576

577

578
579
580
581
582
if pycompat.iswindows:
    # no fork on Windows, but we can create a detached process
    # https://msdn.microsoft.com/en-us/library/windows/desktop/ms684863.aspx
    # No stdlib constant exists for this value
    DETACHED_PROCESS = 0x00000008
583
584
585
    # Following creation flags might create a console GUI window.
    # Using subprocess.CREATE_NEW_CONSOLE might helps.
    # See https://phab.mercurial-scm.org/D1701 for discussion
586
587
588
589
    _creationflags = (
        DETACHED_PROCESS
        | subprocess.CREATE_NEW_PROCESS_GROUP  # pytype: disable=module-attr
    )
590

591
    def runbgcommand(
592
593
594
595
596
597
598
        script,
        env,
        shell=False,
        stdout=None,
        stderr=None,
        ensurestart=True,
        record_wait=None,
599
    ):
600
601
602
        '''Spawn a command without waiting for it to finish.'''
        # we can't use close_fds *and* redirect stdin. I'm not sure that we
        # need to because the detached process has no console connection.
603
        p = subprocess.Popen(
604
            tonativestr(script),
605
606
607
608
609
610
611
            shell=shell,
            env=tonativeenv(env),
            close_fds=True,
            creationflags=_creationflags,
            stdout=stdout,
            stderr=stderr,
        )
612
613
        if record_wait is not None:
            record_wait(p.wait)
614
615


616
else:
617

618
    def runbgcommand(
619
620
621
622
623
624
625
        cmd,
        env,
        shell=False,
        stdout=None,
        stderr=None,
        ensurestart=True,
        record_wait=None,
626
    ):
627
628
629
630
631
632
633
634
        '''Spawn a command without waiting for it to finish.


        When `record_wait` is not None, the spawned process will not be fully
        detached and the `record_wait` argument will be called with a the
        `Subprocess.wait` function for the spawned process.  This is mostly
        useful for developers that need to make sure the spawned process
        finished before a certain point. (eg: writing test)'''
635
636
637
638
639
        if pycompat.isdarwin:
            # avoid crash in CoreFoundation in case another thread
            # calls gui() while we're calling fork().
            gui()

640
641
        # double-fork to completely detach from the parent process
        # based on http://code.activestate.com/recipes/278731
642
643
644
645
        if record_wait is None:
            pid = os.fork()
            if pid:
                if not ensurestart:
646
647
648
649
650
651
652
                    # Even though we're not waiting on the child process,
                    # we still must call waitpid() on it at some point so
                    # it's not a zombie/defunct. This is especially relevant for
                    # chg since the parent process won't die anytime soon.
                    # We use a thread to make the overhead tiny.
                    def _do_wait():
                        os.waitpid(pid, 0)
653

654
655
656
                    t = threading.Thread(target=_do_wait)
                    t.daemon = True
                    t.start()
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
                    return
                # Parent process
                (_pid, status) = os.waitpid(pid, 0)
                if os.WIFEXITED(status):
                    returncode = os.WEXITSTATUS(status)
                else:
                    returncode = -(os.WTERMSIG(status))
                if returncode != 0:
                    # The child process's return code is 0 on success, an errno
                    # value on failure, or 255 if we don't have a valid errno
                    # value.
                    #
                    # (It would be slightly nicer to return the full exception info
                    # over a pipe as the subprocess module does.  For now it
                    # doesn't seem worth adding that complexity here, though.)
                    if returncode == 255:
                        returncode = errno.EINVAL
                    raise OSError(
                        returncode,
                        b'error running %r: %s'
                        % (cmd, os.strerror(returncode)),
                    )
679
                return
680
681
682

        returncode = 255
        try:
683
684
685
            if record_wait is None:
                # Start a new session
                os.setsid()
686

687
            stdin = open(os.devnull, b'r')
688
            if stdout is None:
689
                stdout = open(os.devnull, b'w')
690
            if stderr is None:
691
                stderr = open(os.devnull, b'w')
692
693
694

            # connect stdin to devnull to make sure the subprocess can't
            # muck up that stream for mercurial.
695
            p = subprocess.Popen(
696
697
698
699
700
701
702
703
                cmd,
                shell=shell,
                env=env,
                close_fds=True,
                stdin=stdin,
                stdout=stdout,
                stderr=stderr,
            )
704
705
            if record_wait is not None:
                record_wait(p.wait)
706
707
            returncode = 0
        except EnvironmentError as ex:
708
            returncode = ex.errno & 0xFF
709
710
711
712
713
714
715
716
717
            if returncode == 0:
                # This shouldn't happen, but just in case make sure the
                # return code is never 0 here.
                returncode = 255
        except Exception:
            returncode = 255
        finally:
            # mission accomplished, this child needs to exit and not
            # continue the hg process here.
718
719
            if record_wait is None:
                os._exit(returncode)