Read about our upcoming Code of Conduct on this issue

Commit 7280a7c3 authored by Matti Picus's avatar Matti Picus
Browse files

merge packaging branch into default, which updates command line parsing

and extends LICENSE file to reflect third-party contributions
This software is copyrighted by the Regents of the University of
California, Sun Microsystems, Inc., and other parties. The following
terms apply to all files associated with the software unless explicitly
disclaimed in individual files.
The authors hereby grant permission to use, copy, modify, distribute,
and license this software and its documentation for any purpose, provided
that existing copyright notices are retained in all copies and that this
notice is included verbatim in any distributions. No written agreement,
license, or royalty fee is required for any of the authorized uses.
Modifications to this software may be copyrighted by their authors
and need not follow the licensing terms described here, provided that
the new terms are clearly indicated on the first page of each file where
they apply.
IN NO EVENT SHALL THE AUTHORS OR DISTRIBUTORS BE LIABLE TO ANY PARTY
FOR DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES
ARISING OUT OF THE USE OF THIS SOFTWARE, ITS DOCUMENTATION, OR ANY
DERIVATIVES THEREOF, EVEN IF THE AUTHORS HAVE BEEN ADVISED OF THE
POSSIBILITY OF SUCH DAMAGE.
THE AUTHORS AND DISTRIBUTORS SPECIFICALLY DISCLAIM ANY WARRANTIES,
INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE, AND NON-INFRINGEMENT. THIS SOFTWARE
IS PROVIDED ON AN "AS IS" BASIS, AND THE AUTHORS AND DISTRIBUTORS HAVE
NO OBLIGATION TO PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR
MODIFICATIONS.
GOVERNMENT USE: If you are acquiring this software on behalf of the
U.S. government, the Government shall have only "Restricted Rights"
in the software and related documentation as defined in the Federal
Acquisition Regulations (FARs) in Clause 52.227.19 (c) (2). If you
are acquiring the software on behalf of the Department of Defense, the
software shall be classified as "Commercial Computer Software" and the
Government shall have only "Restricted Rights" as defined in Clause
252.227-7013 (c) (1) of DFARs. Notwithstanding the foregoing, the
authors grant the U.S. Government and others acting in its behalf
permission to use and distribute the software in accordance with the
terms specified in this license.
......@@ -12,3 +12,9 @@ Annotator cleanups
.. branch: release-2.3.x
.. branch: unify-call-ops
.. branch packaging
Use argparse for packaging.py, and add third-party components to LICENSE file.
Also mention that gdbm is GPL.
Do not crash the packaging process on failure in CFFI or license-building,
rather complete the build step and return -1.
#!/usr/bin/env python
""" A sample script that packages PyPy, provided that it's already built.
""" packages PyPy, provided that it's already built.
It uses 'pypy/goal/pypy-c' and parts of the rest of the working
copy. Usage:
package.py [--nostrip] [--without-tk] root-pypy-dir [name-of-archive] [name-of-pypy-c] [destination-for-tarball] [pypy-c-path]
package.py [--options]
Usually you would do: package.py ../../.. pypy-VER-PLATFORM
The output is found in the directory /tmp/usession-YOURNAME/build/.
Usually you would do: package.py --version-name pypy-VER-PLATFORM
The output is found in the directory from --builddir,
by default /tmp/usession-YOURNAME/build/.
"""
import shutil
import sys
import os
#Add toplevel repository dir to sys.path
sys.path.insert(0,os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))))
basedir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
sys.path.insert(0,basedir)
import py
import fnmatch
from rpython.tool.udir import udir
import subprocess
import glob
if sys.version_info < (2,6): py.test.skip("requires 2.6 so far")
......@@ -40,17 +42,122 @@ def ignore_patterns(*patterns):
class PyPyCNotFound(Exception):
pass
def fix_permissions(basedir):
class MissingDependenciesError(Exception):
pass
def fix_permissions(dirname):
if sys.platform != 'win32':
os.system("chmod -R a+rX %s" % basedir)
os.system("chmod -R g-w %s" % basedir)
os.system("chmod -R a+rX %s" % dirname)
os.system("chmod -R g-w %s" % dirname)
sep_template = "\nThis copy of PyPy includes a copy of %s, which is licensed under the following terms:\n\n"
def generate_license_linux(base_file, options):
with open(base_file) as fid:
txt = fid.read()
searches = [("bzip2","libbz2-*", "copyright", '---------'),
("openssl", "openssl*", "copyright", 'LICENSE ISSUES'),
]
if not options.no_tk:
name = 'Tcl/Tk'
txt += "License for '%s'" %name
txt += '\n' + "="*(14 + len(name)) + '\n'
txt += sep_template % name
with open('lib_pypy/_tkinter/license.terms', 'r') as fid:
txt += fid.read()
for name, pat, fname, first_line in searches:
txt += "License for '" + name + "'"
txt += '\n' + "="*(14 + len(name)) + '\n'
txt += sep_template % name
dirs = glob.glob(options.license_base + "/" +pat)
if not dirs:
raise ValueError, "Could not find "+ options.license_base + "/" + pat
if len(dirs) > 2:
raise ValueError, "Multiple copies of "+pat
dir = dirs[0]
with open(os.path.join(dir, fname)) as fid:
# Read up to the line dividing the packaging header from the actual copyright
for line in fid:
if first_line in line:
break
txt += line
for line in fid:
txt += line
if len(line.strip())<1:
txt += '\n'
txt += third_party_header
# Do something for gdbm, which is GPL
txt += gdbm_bit
return txt
def generate_license_windows(base_file, options):
with open(base_file) as fid:
txt = fid.read()
# shutil.copyfileobj(open("crtlicense.txt"), out) # We do not ship msvc runtime files
if not options.no_tk:
name = 'Tcl/Tk'
txt += "License for '%s'" %name
txt += '\n' + "="*(14 + len(name)) + '\n'
txt += sep_template % name
with open('lib_pypy/_tkinter/license.terms', 'r') as fid:
txt += fid.read()
for name, pat, file in (("bzip2","bzip2-*", "LICENSE"),
("openssl", "openssl-*", "LICENSE")):
txt += sep_template % name
dirs = glob.glob(options.license_base + "/" +pat)
if not dirs:
raise ValueError, "Could not find "+ options.license_base + "/" + pat
if len(dirs) > 2:
raise ValueError, "Multiple copies of "+pat
dir = dirs[0]
with open(os.path.join(dir, file)) as fid:
txt += fid.read()
return txt
def generate_license_darwin(base_file, options):
# where are copyright files on macos?
try:
return generate_license_linux(base_file, options)
except:
import traceback; traceback.print_exc()
pass
with open(base_file) as fid:
txt = fid.read()
return txt
if sys.platform == 'win32':
generate_license = generate_license_windows
elif sys.platform == 'darwin':
generate_license = generate_license_darwin
else:
generate_license = generate_license_linux
def create_cffi_import_libraries(pypy_c, options):
modules = ['_sqlite3']
subprocess.check_call([str(pypy_c), '-c', 'import _sqlite3'])
if not sys.platform == 'win32':
modules += ['_curses', 'syslog', 'gdbm', '_sqlite3']
if not options.no_tk:
modules.append(('_tkinter'))
for module in modules:
try:
subprocess.check_call([str(pypy_c), '-c', 'import ' + module])
except subprocess.CalledProcessError:
print >>sys.stderr, """Building {0} bindings failed.
You can either install development headers package or
add --without-{0} option to skip packaging binary CFFI extension.""".format(module)
raise MissingDependenciesError(module)
def create_package(basedir, options):
retval = 0
name = options.name
if not name:
name = 'pypy-nightly'
rename_pypy_c = options.pypy_c
override_pypy_c = options.override_pypy_c
def package(basedir, name='pypy-nightly', rename_pypy_c='pypy',
copy_to_dir=None, override_pypy_c=None, nostrip=False,
withouttk=False):
assert '/' not in rename_pypy_c
basedir = py.path.local(basedir)
if override_pypy_c is None:
if not override_pypy_c:
basename = 'pypy-c'
if sys.platform == 'win32':
basename += '.exe'
......@@ -68,28 +175,18 @@ def package(basedir, name='pypy-nightly', rename_pypy_c='pypy',
raise PyPyCNotFound(
'Bogus path: %r does not exist (see docstring for more info)'
% (os.path.dirname(str(pypy_c)),))
win_extras = ['libpypy-c.dll', 'libexpat.dll', 'sqlite3.dll',
'libeay32.dll', 'ssleay32.dll']
subprocess.check_call([str(pypy_c), '-c', 'import _sqlite3'])
if not sys.platform == 'win32':
subprocess.check_call([str(pypy_c), '-c', 'import _curses'])
subprocess.check_call([str(pypy_c), '-c', 'import syslog'])
subprocess.check_call([str(pypy_c), '-c', 'import gdbm'])
if not withouttk:
if not options.no_cffi:
try:
subprocess.check_call([str(pypy_c), '-c', 'import _tkinter'])
except subprocess.CalledProcessError:
print >>sys.stderr, """Building Tk bindings failed.
You can either install Tk development headers package or
add --without-tk option to skip packaging binary CFFI extension."""
sys.exit(1)
#Can the dependencies be found from cffi somehow?
win_extras += ['tcl85.dll', 'tk85.dll']
create_cffi_import_libraries(pypy_c, options)
except MissingDependenciesError:
# This is a non-fatal error
retval = -1
if sys.platform == 'win32' and not rename_pypy_c.lower().endswith('.exe'):
rename_pypy_c += '.exe'
binaries = [(pypy_c, rename_pypy_c)]
#
builddir = udir.ensure("build", dir=True)
builddir = options.builddir
pypydir = builddir.ensure(name, dir=True)
includedir = basedir.join('include')
# Recursively copy all headers, shutil has only ignore
......@@ -102,10 +199,11 @@ add --without-tk option to skip packaging binary CFFI extension."""
pypydir.ensure('include', dir=True)
if sys.platform == 'win32':
#Don't include a mscvrXX.dll, users should get their own.
#Instructions are provided on the website.
# Can't rename a DLL: it is always called 'libpypy-c.dll'
win_extras = ['libpypy-c.dll', 'libexpat.dll', 'sqlite3.dll',
'libeay32.dll', 'ssleay32.dll']
if not options.no_tk:
win_extras += ['tcl85.dll', 'tk85.dll']
for extra in win_extras:
p = pypy_c.dirpath().join(extra)
......@@ -116,7 +214,7 @@ add --without-tk option to skip packaging binary CFFI extension."""
continue
print "Picking %s" % p
binaries.append((p, p.basename))
importlib_name = 'python27.lib'
importlib_name = 'python27.lib'
if pypy_c.dirpath().join(importlib_name).check():
shutil.copyfile(str(pypy_c.dirpath().join(importlib_name)),
str(pypydir.join('include/python27.lib')))
......@@ -127,7 +225,7 @@ add --without-tk option to skip packaging binary CFFI extension."""
# XXX users will complain that they cannot compile cpyext
# modules for windows, has the lib moved or are there no
# exported functions in the dll so no import library is created?
if not withouttk:
if not options.no_tk:
try:
p = pypy_c.dirpath().join('tcl85.dll')
if not p.check():
......@@ -139,7 +237,7 @@ add --without-tk option to skip packaging binary CFFI extension."""
tk85.dll and tcl85.dll found, expecting to find runtime in ..\\lib
directory next to the dlls, as per build instructions."""
import traceback;traceback.print_exc()
sys.exit(1)
raise MissingDependenciesError('Tk runtime')
# Careful: to copy lib_pypy, copying just the hg-tracked files
# would not be enough: there are also ctypes_config_cache/_*_cache.py.
......@@ -150,11 +248,19 @@ directory next to the dlls, as per build instructions."""
str(pypydir.join('lib_pypy')),
ignore=ignore_patterns('.svn', 'py', '*.pyc', '*~',
'*.c', '*.o'))
for file in ['LICENSE', 'README.rst']:
for file in ['README.rst',]:
shutil.copy(str(basedir.join(file)), str(pypydir))
for file in ['_testcapimodule.c', '_ctypes_test.c']:
shutil.copyfile(str(basedir.join('lib_pypy', file)),
shutil.copyfile(str(basedir.join('lib_pypy', file)),
str(pypydir.join('lib_pypy', file)))
try:
license = generate_license(str(basedir.join('LICENSE')), options)
with open(str(pypydir.join('LICENSE')), 'w') as LICENSE:
LICENSE.write(license)
except:
# Non-fatal error, use original LICENCE file
import traceback;traceback.print_exc()
retval = -1
#
spdir = pypydir.ensure('site-packages', dir=True)
shutil.copy(str(basedir.join('site-packages', 'README')), str(spdir))
......@@ -167,17 +273,17 @@ directory next to the dlls, as per build instructions."""
for source, target in binaries:
archive = bindir.join(target)
shutil.copy(str(source), str(archive))
old_dir = os.getcwd()
fix_permissions(builddir)
old_dir = os.getcwd()
try:
os.chdir(str(builddir))
#
# 'strip' fun: see issue #587
if not nostrip:
if not options.nostrip:
for source, target in binaries:
if sys.platform == 'win32':
pass
elif sys.platform == 'darwin':
# 'strip' fun: see issue #587 for why -x
os.system("strip -x " + str(bindir.join(target))) # ignore errors
else:
os.system("strip " + str(bindir.join(target))) # ignore errors
......@@ -208,41 +314,91 @@ using another platform..."""
raise OSError('"tar" returned exit status %r' % e)
finally:
os.chdir(old_dir)
if copy_to_dir is not None:
print "Copying %s to %s" % (archive, copy_to_dir)
shutil.copy(archive, str(copy_to_dir))
if options.targetdir:
print "Copying %s to %s" % (archive, options.targetdir)
shutil.copy(archive, options.targetdir)
else:
print "Ready in %s" % (builddir,)
return builddir # for tests
return retval, builddir # for tests
def package(*args):
try:
import argparse
except ImportError:
import imp
argparse = imp.load_source('argparse', 'lib-python/2.7/argparse.py')
if sys.platform == 'win32':
pypy_exe = 'pypy.exe'
license_base = os.path.join(basedir, r'..\..\..\local') # as on buildbot YMMV
else:
pypy_exe = 'pypy'
license_base = '/usr/share/doc'
parser = argparse.ArgumentParser()
args = list(args)
args[0] = str(args[0])
parser.add_argument('--without-tk', dest='no_tk', action='store_true',
help='build and package the cffi tkinter module')
parser.add_argument('--without-cffi', dest='no_cffi', action='store_true',
help='do not pre-import any cffi modules')
parser.add_argument('--nostrip', dest='nostrip', action='store_true',
help='do not strip the exe, making it ~10MB larger')
parser.add_argument('--rename_pypy_c', dest='pypy_c', type=str, default=pypy_exe,
help='target executable name, defaults to "pypy"')
parser.add_argument('--archive-name', dest='name', type=str, default='',
help='pypy-VER-PLATFORM')
parser.add_argument('--license_base', type=str, default=license_base,
help='where to start looking for third party upstream licensing info')
parser.add_argument('--builddir', type=str, default='',
help='tmp dir for packaging')
parser.add_argument('--targetdir', type=str, default='',
help='destination dir for archive')
parser.add_argument('--override_pypy_c', type=str, default='',
help='use as pypy exe instead of pypy/goal/pypy-c')
# Positional arguments, for backward compatability with buldbots
parser.add_argument('extra_args', help='optional interface to positional arguments', nargs=argparse.REMAINDER,
metavar='[root-pypy-dir] [name-of-archive] [name-of-pypy-c] [destination-for-tarball] [pypy-c-path]',
)
options = parser.parse_args(args)
def print_usage():
print >>sys.stderr, __doc__
sys.exit(1)
# Handle positional arguments, choke if both methods are used
for i,target, default in ([1, 'name', ''], [2, 'pypy_c', pypy_exe],
[3, 'targetdir', ''], [4,'override_pypy_c', '']):
if len(options.extra_args)>i:
if getattr(options, target) != default:
print 'positional argument',i,target,'already has value',getattr(options, target)
parser.print_help()
return
setattr(options, target, options.extra_args[i])
if os.environ.has_key("PYPY_PACKAGE_NOSTRIP"):
options.nostrip = True
if os.environ.has_key("PYPY_PACKAGE_WITHOUTTK"):
options.tk = True
if not options.builddir:
# The import actually creates the udir directory
from rpython.tool.udir import udir
options.builddir = udir.ensure("build", dir=True)
assert '/' not in options.pypy_c
return create_package(basedir, options)
if __name__ == '__main__':
if len(sys.argv) == 1:
print_usage()
args = sys.argv[1:]
kw = {}
for i, arg in enumerate(args):
if arg == '--nostrip':
kw['nostrip'] = True
elif arg == '--without-tk':
kw['withouttk'] = True
elif not arg.startswith('--'):
break
else:
print_usage()
if os.environ.has_key("PYPY_PACKAGE_NOSTRIP"):
kw['nostrip'] = True
third_party_header = '''\n\nLicenses and Acknowledgements for Incorporated Software
=======================================================
if os.environ.has_key("PYPY_PACKAGE_WITHOUTTK"):
kw['withouttk'] = True
This section is an incomplete, but growing list of licenses and acknowledgements
for third-party software incorporated in the PyPy distribution.
'''
args = args[i:]
package(*args, **kw)
gdbm_bit = '''gdbm
----
The gdbm module includes code from gdbm.h, which is distributed under the terms
of the GPL license version 2 or any later version.
'''
if __name__ == '__main__':
import sys
retval, _ = package(*sys.argv[1:])
sys.exit(retval)
import py
from pypy.conftest import pypydir
from pypy.tool.release import package
from pypy.tool.release import package, package
from pypy.module.sys.version import CPYTHON_VERSION
import tarfile, zipfile, sys
......@@ -25,8 +25,9 @@ def test_dir_structure(test='test'):
else:
fake_pypy_c = False
try:
builddir = package.package(py.path.local(pypydir).dirpath(), test,
retval, builddir = package.package(py.path.local(pypydir).dirpath(), test,
rename_pypy_c)
assert retval == 0
prefix = builddir.join(test)
cpyver = '%d.%d' % CPYTHON_VERSION[:2]
assert prefix.join('lib-python', cpyver, 'test').check()
......@@ -74,7 +75,6 @@ def test_dir_structure(test='test'):
pypy_c.remove()
def test_with_zipfile_module():
from pypy.tool.release import package
prev = package.USE_ZIPFILE_MODULE
try:
package.USE_ZIPFILE_MODULE = True
......@@ -106,3 +106,22 @@ def test_fix_permissions(tmpdir):
check(file1, 0644)
check(file2, 0644)
check(pypy, 0755)
def test_generate_license():
from os.path import dirname, abspath, join
class Options(object):
pass
options = Options()
basedir = dirname(dirname(dirname(dirname(dirname(abspath(__file__))))))
options.no_tk = False
if sys.platform == 'win32':
# Following recommended build setup at
# http://doc.pypy.org/en/latest/windows.html#abridged-method-for-ojit-builds-using-visual-studio-2008
options.license_base = dirname(basedir) + '/local'
else:
options.license_base = '/usr/share/doc'
license = package.generate_license(join(basedir,'LICENSE'), options)
assert 'bzip2' in license
assert 'openssl' in license
assert 'Tcl' in license
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment