Read about our upcoming Code of Conduct on this issue

Commit d4fe83da authored by Matti Picus's avatar Matti Picus
Browse files

merge default into branch

--HG--
branch : packaging
......@@ -422,7 +422,8 @@ def _compile(pathname, timestamp):
saved back to the filesystem for future imports. The source file's
modification timestamp must be provided as a Long value.
"""
codestring = open(pathname, 'rU').read()
with open(pathname, 'rU') as fp:
codestring = fp.read()
if codestring and codestring[-1] != '\n':
codestring = codestring + '\n'
code = __builtin__.compile(codestring, pathname, 'exec')
......@@ -603,8 +604,8 @@ class DynLoadSuffixImporter:
self.desc = desc
def import_file(self, filename, finfo, fqname):
fp = open(filename, self.desc[1])
module = imp.load_module(fqname, fp, filename, self.desc)
with open(filename, self.desc[1]) as fp:
module = imp.load_module(fqname, fp, filename, self.desc)
module.__file__ = filename
return 0, module, { }
......
......@@ -109,16 +109,16 @@ class ModuleFinder:
def run_script(self, pathname):
self.msg(2, "run_script", pathname)
fp = open(pathname, READ_MODE)
stuff = ("", "r", imp.PY_SOURCE)
self.load_module('__main__', fp, pathname, stuff)
with open(pathname, READ_MODE) as fp:
stuff = ("", "r", imp.PY_SOURCE)
self.load_module('__main__', fp, pathname, stuff)
def load_file(self, pathname):
dir, name = os.path.split(pathname)
name, ext = os.path.splitext(name)
fp = open(pathname, READ_MODE)
stuff = (ext, "r", imp.PY_SOURCE)
self.load_module(name, fp, pathname, stuff)
with open(pathname, READ_MODE) as fp:
stuff = (ext, "r", imp.PY_SOURCE)
self.load_module(name, fp, pathname, stuff)
def import_hook(self, name, caller=None, fromlist=None, level=-1):
self.msg(3, "import_hook", name, caller, fromlist, level)
......@@ -461,6 +461,8 @@ class ModuleFinder:
fp, buf, stuff = self.find_module("__init__", m.__path__)
self.load_module(fqname, fp, buf, stuff)
self.msgout(2, "load_package ->", m)
if fp:
fp.close()
return m
def add_module(self, fqname):
......
......@@ -48,6 +48,9 @@ class TempDirMixin(object):
def tearDown(self):
os.chdir(self.old_dir)
import gc
# Force a collection which should close FileType() options
gc.collect()
for root, dirs, files in os.walk(self.temp_dir, topdown=False):
for name in files:
os.chmod(os.path.join(self.temp_dir, name), stat.S_IWRITE)
......
......@@ -25,13 +25,13 @@ sys.path.append(os.path.abspath('.'))
on_rtd = os.environ.get('READTHEDOCS', None) == 'True'
if not on_rtd: # only import and set the theme if we're building docs locally
try:
import sphinx_rtd_theme
html_theme = 'sphinx_rtd_theme'
html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]
except ImportError:
print('sphinx_rtd_theme is not installed')
html_theme = 'default'
try:
import sphinx_rtd_theme
html_theme = 'sphinx_rtd_theme'
html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]
except ImportError:
print('sphinx_rtd_theme is not installed')
html_theme = 'default'
# otherwise, readthedocs.org uses their theme by default, so no need to specify it
......
......@@ -11,3 +11,4 @@ Annotator cleanups
.. branch: release-2.3.x
.. branch: unify-call-ops
......@@ -613,6 +613,8 @@ class ArrayBuffer(Buffer):
array._charbuf_stop()
def getslice(self, start, stop, step, size):
if size == 0:
return ''
if step == 1:
data = self.array._charbuf_start()
try:
......
......@@ -1029,6 +1029,9 @@ class AppTestArray(BaseArrayTests):
raises(TypeError, "a[MyInt(0)]")
raises(TypeError, "a[MyInt(0):MyInt(5)]")
def test_fresh_array_buffer_str(self):
assert str(buffer(self.array('i'))) == ''
class AppTestArrayBuiltinShortcut(AppTestArray):
spaceconfig = AppTestArray.spaceconfig.copy()
......
......@@ -208,6 +208,9 @@ PyGILState_STATE = rffi.INT
@cpython_api([], PyGILState_STATE, error=CANNOT_FAIL)
def PyGILState_Ensure(space):
# XXX XXX XXX THIS IS A VERY MINIMAL IMPLEMENTATION THAT WILL HAPPILY
# DEADLOCK IF CALLED TWICE ON THE SAME THREAD, OR CRASH IF CALLED IN A
# NEW THREAD. We should very carefully follow what CPython does instead.
if rffi.aroundstate.after:
# After external call is before entering Python
rffi.aroundstate.after()
......@@ -215,6 +218,7 @@ def PyGILState_Ensure(space):
@cpython_api([PyGILState_STATE], lltype.Void)
def PyGILState_Release(space, state):
# XXX XXX XXX We should very carefully follow what CPython does instead.
if rffi.aroundstate.before:
# Before external call is after running Python
rffi.aroundstate.before()
......
......@@ -748,11 +748,11 @@ class ImportRLock:
self.lockcounter = 0
def lock_held_by_someone_else(self):
return self.lockowner is not None and not self.lock_held()
def lock_held(self):
me = self.space.getexecutioncontext() # used as thread ident
return self.lockowner is me
return self.lockowner is not None and self.lockowner is not me
def lock_held_by_anyone(self):
return self.lockowner is not None
def acquire_lock(self):
# this function runs with the GIL acquired so there is no race
......
......@@ -165,7 +165,7 @@ def is_frozen(space, w_name):
def lock_held(space):
if space.config.objspace.usemodules.thread:
return space.wrap(importing.getimportlock(space).lock_held())
return space.wrap(importing.getimportlock(space).lock_held_by_anyone())
else:
return space.w_False
......
......@@ -15,6 +15,7 @@ class MultiArrayModule(MixedModule):
'empty': 'ctors.zeros',
'empty_like': 'ctors.empty_like',
'fromstring': 'ctors.fromstring',
'frombuffer': 'ctors.frombuffer',
'concatenate': 'arrayops.concatenate',
'count_nonzero': 'arrayops.count_nonzero',
......
from pypy.interpreter.error import OperationError, oefmt
from pypy.interpreter.gateway import unwrap_spec, WrappedDefault
from rpython.rlib.buffer import SubBuffer
from rpython.rlib.rstring import strip_spaces
from rpython.rtyper.lltypesystem import lltype, rffi
from pypy.module.micronumpy import descriptor, loop
......@@ -191,3 +192,62 @@ def fromstring(space, s, w_dtype=None, count=-1, sep=''):
return _fromstring_bin(space, s, count, length, dtype)
else:
return _fromstring_text(space, s, count, sep, length, dtype)
def _getbuffer(space, w_buffer):
try:
return space.writebuf_w(w_buffer)
except OperationError as e:
if not e.match(space, space.w_TypeError):
raise
return space.readbuf_w(w_buffer)
@unwrap_spec(count=int, offset=int)
def frombuffer(space, w_buffer, w_dtype=None, count=-1, offset=0):
dtype = space.interp_w(descriptor.W_Dtype,
space.call_function(space.gettypefor(descriptor.W_Dtype), w_dtype))
if dtype.elsize == 0:
raise oefmt(space.w_ValueError, "itemsize cannot be zero in type")
try:
buf = _getbuffer(space, w_buffer)
except OperationError as e:
if not e.match(space, space.w_TypeError):
raise
w_buffer = space.getattr(w_buffer, space.wrap('__buffer__'))
buf = _getbuffer(space, w_buffer)
ts = buf.getlength()
if offset < 0 or offset > ts:
raise oefmt(space.w_ValueError,
"offset must be non-negative and no greater than "
"buffer length (%d)", ts)
s = ts - offset
if offset:
buf = SubBuffer(buf, offset, s)
n = count
itemsize = dtype.elsize
assert itemsize > 0
if n < 0:
if s % itemsize != 0:
raise oefmt(space.w_ValueError,
"buffer size must be a multiple of element size")
n = s / itemsize
else:
if s < n * itemsize:
raise oefmt(space.w_ValueError,
"buffer is smaller than requested size")
try:
storage = buf.get_raw_address()
except ValueError:
a = W_NDimArray.from_shape(space, [n], dtype=dtype)
loop.fromstring_loop(space, a, dtype, itemsize, buf.as_str())
return a
else:
writable = not buf.readonly
return W_NDimArray.from_shape_and_storage(space, [n], storage, dtype=dtype,
w_base=w_buffer, writable=writable)
......@@ -3132,6 +3132,8 @@ class AppTestMultiDim(BaseNumpyAppTest):
class AppTestSupport(BaseNumpyAppTest):
spaceconfig = {'usemodules': ['micronumpy', 'array']}
def setup_class(cls):
import struct
BaseNumpyAppTest.setup_class.im_func(cls)
......@@ -3142,6 +3144,44 @@ class AppTestSupport(BaseNumpyAppTest):
cls.w_float64val = cls.space.wrap(struct.pack('d', 300.4))
cls.w_ulongval = cls.space.wrap(struct.pack('L', 12))
def test_frombuffer(self):
import numpy as np
exc = raises(AttributeError, np.frombuffer, None)
assert str(exc.value) == "'NoneType' object has no attribute '__buffer__'"
exc = raises(AttributeError, np.frombuffer, memoryview(self.data))
assert str(exc.value) == "'memoryview' object has no attribute '__buffer__'"
exc = raises(ValueError, np.frombuffer, self.data, 'S0')
assert str(exc.value) == "itemsize cannot be zero in type"
exc = raises(ValueError, np.frombuffer, self.data, offset=-1)
assert str(exc.value) == "offset must be non-negative and no greater than buffer length (32)"
exc = raises(ValueError, np.frombuffer, self.data, count=100)
assert str(exc.value) == "buffer is smaller than requested size"
for data in [self.data, buffer(self.data)]:
a = np.frombuffer(data)
for i in range(4):
assert a[i] == i + 1
import array
data = array.array('c', 'testing')
a = np.frombuffer(data, 'c')
assert a.base is data
a[2] = 'Z'
assert data.tostring() == 'teZting'
data = buffer(data)
a = np.frombuffer(data, 'c')
assert a.base is data
exc = raises(ValueError, "a[2] = 'Z'")
assert str(exc.value) == "assignment destination is read-only"
class A(object):
__buffer__ = 'abc'
data = A()
a = np.frombuffer(data, 'c')
#assert a.base is data.__buffer__
assert a.tostring() == 'abc'
def test_fromstring(self):
import sys
from numpypy import fromstring, dtype
......
......@@ -13,7 +13,7 @@ class Module(MixedModule):
"""NOT_RPYTHON""" # because parent __init__ isn't
if space.config.translating:
del self.__class__.interpleveldefs['pypy_getudir']
super(Module, self).__init__(space, w_name)
super(Module, self).__init__(space, w_name)
self.recursionlimit = 100
self.w_default_encoder = None
self.defaultencoding = "ascii"
......@@ -21,13 +21,13 @@ class Module(MixedModule):
self.debug = True
interpleveldefs = {
'__name__' : '(space.wrap("sys"))',
'__doc__' : '(space.wrap("PyPy sys module"))',
'__name__' : '(space.wrap("sys"))',
'__doc__' : '(space.wrap("PyPy sys module"))',
'platform' : 'space.wrap(sys.platform)',
'platform' : 'space.wrap(sys.platform)',
'maxint' : 'space.wrap(sys.maxint)',
'maxsize' : 'space.wrap(sys.maxint)',
'byteorder' : 'space.wrap(sys.byteorder)',
'byteorder' : 'space.wrap(sys.byteorder)',
'maxunicode' : 'space.wrap(vm.MAXUNICODE)',
'stdin' : 'state.getio(space).w_stdin',
'__stdin__' : 'state.getio(space).w_stdin',
......@@ -36,35 +36,35 @@ class Module(MixedModule):
'stderr' : 'state.getio(space).w_stderr',
'__stderr__' : 'state.getio(space).w_stderr',
'pypy_objspaceclass' : 'space.wrap(repr(space))',
#'prefix' : # added by pypy_initial_path() when it
#'prefix' : # added by pypy_initial_path() when it
#'exec_prefix' : # succeeds, pointing to trunk or /usr
'path' : 'state.get(space).w_path',
'modules' : 'state.get(space).w_modules',
'modules' : 'state.get(space).w_modules',
'argv' : 'state.get(space).w_argv',
'py3kwarning' : 'space.w_False',
'warnoptions' : 'state.get(space).w_warnoptions',
'warnoptions' : 'state.get(space).w_warnoptions',
'builtin_module_names' : 'space.w_None',
'pypy_getudir' : 'state.pypy_getudir', # not translated
'pypy_find_stdlib' : 'initpath.pypy_find_stdlib',
'pypy_find_executable' : 'initpath.pypy_find_executable',
'pypy_resolvedirof' : 'initpath.pypy_resolvedirof',
'_getframe' : 'vm._getframe',
'_current_frames' : 'currentframes._current_frames',
'setrecursionlimit' : 'vm.setrecursionlimit',
'getrecursionlimit' : 'vm.getrecursionlimit',
'setcheckinterval' : 'vm.setcheckinterval',
'getcheckinterval' : 'vm.getcheckinterval',
'exc_info' : 'vm.exc_info',
'exc_clear' : 'vm.exc_clear',
'_getframe' : 'vm._getframe',
'_current_frames' : 'currentframes._current_frames',
'setrecursionlimit' : 'vm.setrecursionlimit',
'getrecursionlimit' : 'vm.getrecursionlimit',
'setcheckinterval' : 'vm.setcheckinterval',
'getcheckinterval' : 'vm.getcheckinterval',
'exc_info' : 'vm.exc_info',
'exc_clear' : 'vm.exc_clear',
'settrace' : 'vm.settrace',
'gettrace' : 'vm.gettrace',
'setprofile' : 'vm.setprofile',
'getprofile' : 'vm.getprofile',
'call_tracing' : 'vm.call_tracing',
'getsizeof' : 'vm.getsizeof',
'executable' : 'space.wrap("py.py")',
'executable' : 'space.wrap("py.py")',
'api_version' : 'version.get_api_version(space)',
'version_info' : 'version.get_version_info(space)',
'version' : 'version.get_version(space)',
......@@ -73,14 +73,14 @@ class Module(MixedModule):
'_mercurial' : 'version.get_repo_info(space)',
'hexversion' : 'version.get_hexversion(space)',
'displayhook' : 'hook.displayhook',
'__displayhook__' : 'hook.__displayhook__',
'displayhook' : 'hook.displayhook',
'__displayhook__' : 'hook.__displayhook__',
'meta_path' : 'space.wrap([])',
'path_hooks' : 'space.wrap([])',
'path_importer_cache' : 'space.wrap({})',
'dont_write_bytecode' : 'space.w_False',
'getdefaultencoding' : 'interp_encoding.getdefaultencoding',
'getdefaultencoding' : 'interp_encoding.getdefaultencoding',
'setdefaultencoding' : 'interp_encoding.setdefaultencoding',
'getfilesystemencoding' : 'interp_encoding.getfilesystemencoding',
......@@ -119,21 +119,21 @@ class Module(MixedModule):
w_modules = self.get('modules')
try:
return space.getitem(w_modules, space.wrap(name))
except OperationError, e:
if not e.match(space, space.w_KeyError):
raise
return None
except OperationError, e:
if not e.match(space, space.w_KeyError):
raise
return None
def setmodule(self, w_module):
def setmodule(self, w_module):
space = self.space
w_name = self.space.getattr(w_module, space.wrap('__name__'))
w_modules = self.get('modules')
self.space.setitem(w_modules, w_name, w_module)
def getdictvalue(self, space, attr):
""" specialize access to dynamic exc_* attributes. """
value = MixedModule.getdictvalue(self, space, attr)
if value is not None:
""" specialize access to dynamic exc_* attributes. """
value = MixedModule.getdictvalue(self, space, attr)
if value is not None:
return value
if attr == 'exc_type':
operror = space.getexecutioncontext().sys_exc_info()
......@@ -153,7 +153,7 @@ class Module(MixedModule):
return space.w_None
else:
return space.wrap(operror.get_traceback())
return None
return None
def get_w_default_encoder(self):
if self.w_default_encoder is not None:
......
......@@ -62,6 +62,28 @@ class AppTestThread(GenericTestThread):
self.waitfor(lambda: done)
assert done
def test_lock_held_by_another_thread(self):
import thread, imp
lock_held = thread.allocate_lock()
test_complete = thread.allocate_lock()
lock_released = thread.allocate_lock()
def other_thread():
imp.acquire_lock() # 3
assert imp.lock_held()
lock_held.release() # 4
test_complete.acquire() # 7
imp.release_lock() # 8
lock_released.release() # 9
lock_held.acquire()
test_complete.acquire()
lock_released.acquire()
#
thread.start_new_thread(other_thread, ()) # 1
lock_held.acquire() # 2
assert imp.lock_held() # 5
test_complete.release() # 6
lock_released.acquire() # 10
class TestImportLock:
def test_lock(self, space, monkeypatch):
from pypy.module.imp.importing import getimportlock, importhook
......
......@@ -5,7 +5,7 @@ import operator
from rpython.rlib.buffer import Buffer, SubBuffer
from pypy.interpreter.baseobjspace import W_Root
from pypy.interpreter.error import OperationError
from pypy.interpreter.error import OperationError, oefmt
from pypy.interpreter.gateway import interp2app
from pypy.interpreter.typedef import TypeDef, GetSetProperty
......@@ -61,15 +61,6 @@ class W_MemoryView(W_Root):
def getlength(self):
return self.buf.getlength()
def getslice(self, start, stop):
if start < 0:
start = 0
size = stop - start
if size < 0:
size = 0
buf = SubBuffer(self.buf, start, size)
return W_MemoryView(buf)
def descr_tobytes(self, space):
return space.wrap(self.as_str())
......@@ -81,25 +72,25 @@ class W_MemoryView(W_Root):
return space.newlist(result)
def descr_getitem(self, space, w_index):
start, stop, step = space.decode_index(w_index, self.getlength())
start, stop, step, size = space.decode_index4(w_index, self.getlength())
if step not in (0, 1):
raise OperationError(space.w_NotImplementedError, space.wrap(""))
raise oefmt(space.w_NotImplementedError, "")
if step == 0: # index only
return space.wrap(self.buf.getitem(start))
res = self.getslice(start, stop)
return space.wrap(res)
else:
buf = SubBuffer(self.buf, start, size)
return W_MemoryView(buf)
def descr_setitem(self, space, w_index, w_obj):
if self.buf.readonly:
raise OperationError(space.w_TypeError, space.wrap(
"cannot modify read-only memory"))
start, stop, step, size = space.decode_index4(w_index, self.buf.getlength())
raise oefmt(space.w_TypeError, "cannot modify read-only memory")
start, stop, step, size = space.decode_index4(w_index, self.getlength())
if step not in (0, 1):
raise OperationError(space.w_NotImplementedError, space.wrap(""))
raise oefmt(space.w_NotImplementedError, "")
value = space.buffer_w(w_obj, space.BUF_CONTIG_RO)
if value.getlength() != size:
raise OperationError(space.w_ValueError, space.wrap(
"cannot modify size of memoryview object"))
raise oefmt(space.w_ValueError,
"cannot modify size of memoryview object")
if step == 0: # index only
self.buf.setitem(start, value.getitem(0))
elif step == 1:
......
......@@ -2,15 +2,14 @@
Binary operations between SomeValues.
"""
import py
import operator
from rpython.tool.pairtype import pair, pairtype
from rpython.annotator.model import (
SomeObject, SomeInteger, SomeBool, s_Bool, SomeString, SomeChar, SomeList,
SomeDict, SomeOrderedDict, SomeUnicodeCodePoint, SomeUnicodeString,
SomeDict, SomeUnicodeCodePoint, SomeUnicodeString,
SomeTuple, SomeImpossibleValue, s_ImpossibleValue, SomeInstance,
SomeBuiltinMethod, SomeIterator, SomePBC, SomeFloat, s_None, SomeByteArray,
SomeWeakRef, SomeSingleFloat,
SomeBuiltinMethod, SomeIterator, SomePBC, SomeNone, SomeFloat, s_None,
SomeByteArray, SomeWeakRef, SomeSingleFloat,
SomeLongFloat, SomeType, SomeConstantType, unionof, UnionError,
read_can_only_throw, add_knowntypedata,
merge_knowntypedata,)
......@@ -768,57 +767,51 @@ class __extend__(pairtype(SomeObject, SomeImpossibleValue)):
# mixing Nones with other objects
def _make_none_union(classname, constructor_args='', glob=None):
if glob is None:
glob = globals()
loc = locals()
source = py.code.Source("""
class __extend__(pairtype(%(classname)s, SomePBC)):
def union((obj, pbc)):
if pbc.isNone():
return %(classname)s(%(constructor_args)s)
else:
raise UnionError(pbc, obj)
class __extend__(pairtype(SomeObject, SomeNone)):
def union((obj, none)):
return obj.noneify()
class __extend__(pairtype(SomePBC, %(classname)s)):
def union((pbc, obj)):
if pbc.isNone():
return %(classname)s(%(constructor_args)s)
else:
raise UnionError(pbc, obj)
""" % loc)
exec source.compile() in glob
class __extend__(pairtype(SomeNone, SomeObject)):
def union((none, obj)):
return obj.noneify()
class __extend__(pairtype(SomeImpossibleValue, SomeNone)):
def union((imp1, none)):
return s_None
_make_none_union('SomeInstance', 'classdef=obj.classdef, can_be_None=True')
_make_none_union('SomeString', 'no_nul=obj.no_nul, can_be_None=True')
_make_none_union('SomeUnicodeString', 'can_be_None=True')
_make_none_union('SomeList', 'obj.listdef')
_make_none_union('SomeOrderedDict', 'obj.dictdef')
_make_none_union('SomeDict', 'obj.dictdef')
_make_none_union('SomeWeakRef', 'obj.classdef')
class __extend__(pairtype(SomeNone, SomeImpossibleValue)):
def union((none, imp2)):
return s_None
# getitem on SomePBCs, in particular None fails
class __extend__(pairtype(SomePBC, SomeObject)):
def getitem((pbc, o)):
if not pbc.isNone():
raise AnnotatorError("getitem on %r" % pbc)
return s_ImpossibleValue
raise AnnotatorError("getitem on %r" % pbc)
def setitem((pbc, o), s_value):
if not pbc.isNone():
raise AnnotatorError("setitem on %r" % pbc)
raise AnnotatorError("setitem on %r" % pbc)
class __extend__(pairtype(SomeNone, SomeObject)):
def getitem((none, o)):
return s_ImpossibleValue