Commit 92ba093e authored by Armin Rigo's avatar Armin Rigo
Browse files

Port the new_allocator() work from pypy

parent 68008f1b02e3
......@@ -238,7 +238,9 @@ typedef struct {
typedef struct {
CDataObject head;
PyObject *origobj, *destructor;
Py_ssize_t length; /* same as CDataObject_own_length up to here */
PyObject *origobj;
PyObject *destructor;
} CDataObject_gcp;
typedef struct {
......@@ -281,6 +283,8 @@ static void init_errno(void) { }
# include "wchar_helper.h"
#endif
typedef PyObject *const cffi_allocator_t[3];
static cffi_allocator_t default_allocator = { NULL, NULL, NULL };
static PyObject *FFIError;
/************************************************************/
......@@ -1643,14 +1647,17 @@ static void cdatagcp_dealloc(CDataObject_gcp *cd)
PyObject *origobj = cd->origobj;
cdata_dealloc((CDataObject *)cd);
result = PyObject_CallFunctionObjArgs(destructor, origobj, NULL);
if (result != NULL) {
Py_DECREF(result);
}
else {
_my_PyErr_WriteUnraisable("From callback for ffi.gc ", origobj, NULL);
if (destructor != NULL) {
result = PyObject_CallFunctionObjArgs(destructor, origobj, NULL);
if (result != NULL) {
Py_DECREF(result);
}
else {
_my_PyErr_WriteUnraisable("From callback for ffi.gc ",
origobj, NULL);
}
Py_DECREF(destructor);
}
Py_DECREF(destructor);
Py_DECREF(origobj);
}
......@@ -2918,7 +2925,79 @@ convert_struct_to_owning_object(char *data, CTypeDescrObject *ct)
return (PyObject *)cd;
}
static PyObject *direct_newp(CTypeDescrObject *ct, PyObject *init)
static CDataObject *allocate_gcp_object(CDataObject *origobj,
CTypeDescrObject *ct,
PyObject *destructor)
{
CDataObject_gcp *cd = PyObject_GC_New(CDataObject_gcp, &CDataGCP_Type);
if (cd == NULL)
return NULL;
Py_XINCREF(destructor);
Py_INCREF(origobj);
Py_INCREF(ct);
cd->head.c_data = origobj->c_data;
cd->head.c_type = ct;
cd->head.c_weakreflist = NULL;
cd->origobj = (PyObject *)origobj;
cd->destructor = destructor;
PyObject_GC_Track(cd);
return (CDataObject *)cd;
}
static CDataObject *allocate_with_allocator(Py_ssize_t basesize,
Py_ssize_t datasize,
CTypeDescrObject *ct,
cffi_allocator_t allocator)
{
CDataObject *cd;
PyObject *my_alloc = allocator[0];
PyObject *my_free = allocator[1];
PyObject *dont_clear_after_alloc = allocator[2];
if (my_alloc == NULL) { /* alloc */
cd = allocate_owning_object(basesize + datasize, ct);
if (cd == NULL)
return NULL;
cd->c_data = ((char *)cd) + basesize;
}
else {
PyObject *res = PyObject_CallFunction(my_alloc, "n", datasize);
if (res == NULL)
return NULL;
if (!CData_Check(res)) {
PyErr_Format(PyExc_TypeError,
"alloc() must return a cdata object (got %.200s)",
Py_TYPE(res)->tp_name);
Py_DECREF(res);
return NULL;
}
cd = (CDataObject *)res;
if (!(cd->c_type->ct_flags & (CT_POINTER|CT_ARRAY))) {
PyErr_Format(PyExc_TypeError,
"alloc() must return a cdata pointer, not '%s'",
cd->c_type->ct_name);
Py_DECREF(res);
return NULL;
}
if (!cd->c_data) {
PyErr_SetString(PyExc_MemoryError, "alloc() returned NULL");
Py_DECREF(res);
return NULL;
}
cd = allocate_gcp_object(cd, ct, my_free);
Py_DECREF(res);
}
if (dont_clear_after_alloc == NULL)
memset(cd->c_data, 0, datasize);
return cd;
}
static PyObject *direct_newp(CTypeDescrObject *ct, PyObject *init,
cffi_allocator_t allocator)
{
CTypeDescrObject *ctitem;
CDataObject *cd;
......@@ -2982,7 +3061,8 @@ static PyObject *direct_newp(CTypeDescrObject *ct, PyObject *init)
having a strong reference to it */
CDataObject *cds;
cds = allocate_owning_object(dataoffset + datasize, ct->ct_itemdescr);
cds = allocate_with_allocator(dataoffset, datasize, ct->ct_itemdescr,
allocator);
if (cds == NULL)
return NULL;
......@@ -2995,19 +3075,17 @@ static PyObject *direct_newp(CTypeDescrObject *ct, PyObject *init)
((CDataObject_own_structptr *)cd)->structobj = (PyObject *)cds;
assert(explicitlength < 0);
cds->c_data = cd->c_data = ((char *)cds) + dataoffset;
cd->c_data = cds->c_data;
}
else {
cd = allocate_owning_object(dataoffset + datasize, ct);
cd = allocate_with_allocator(dataoffset, datasize, ct, allocator);
if (cd == NULL)
return NULL;
cd->c_data = ((char *)cd) + dataoffset;
if (explicitlength >= 0)
((CDataObject_own_length*)cd)->length = explicitlength;
}
memset(cd->c_data, 0, datasize);
if (init != Py_None) {
if (convert_from_object(cd->c_data,
(ct->ct_flags & CT_POINTER) ? ct->ct_itemdescr : ct, init) < 0) {
......@@ -3024,7 +3102,7 @@ static PyObject *b_newp(PyObject *self, PyObject *args)
PyObject *init = Py_None;
if (!PyArg_ParseTuple(args, "O!|O:newp", &CTypeDescr_Type, &ct, &init))
return NULL;
return direct_newp(ct, init);
return direct_newp(ct, init, default_allocator);
}
static int
......@@ -5630,7 +5708,7 @@ static PyObject *b__get_types(PyObject *self, PyObject *noarg)
static PyObject *b_gcp(PyObject *self, PyObject *args, PyObject *kwds)
{
CDataObject_gcp *cd;
CDataObject *cd;
CDataObject *origobj;
PyObject *destructor;
static char *keywords[] = {"cdata", "destructor", NULL};
......@@ -5639,20 +5717,7 @@ static PyObject *b_gcp(PyObject *self, PyObject *args, PyObject *kwds)
&CData_Type, &origobj, &destructor))
return NULL;
cd = PyObject_GC_New(CDataObject_gcp, &CDataGCP_Type);
if (cd == NULL)
return NULL;
Py_INCREF(destructor);
Py_INCREF(origobj);
Py_INCREF(origobj->c_type);
cd->head.c_data = origobj->c_data;
cd->head.c_type = origobj->c_type;
cd->head.c_weakreflist = NULL;
cd->origobj = (PyObject *)origobj;
cd->destructor = destructor;
PyObject_GC_Track(cd);
cd = allocate_gcp_object(origobj, origobj->c_type, destructor);
return (PyObject *)cd;
}
......
......@@ -331,7 +331,8 @@ PyDoc_STRVAR(ffi_new_doc,
"used for a longer time. Be careful about that when copying the\n"
"pointer to the memory somewhere else, e.g. into another structure.");
static PyObject *ffi_new(FFIObject *self, PyObject *args, PyObject *kwds)
static PyObject *_ffi_new(FFIObject *self, PyObject *args, PyObject *kwds,
cffi_allocator_t allocator)
{
CTypeDescrObject *ct;
PyObject *arg, *init = Py_None;
......@@ -344,7 +345,70 @@ static PyObject *ffi_new(FFIObject *self, PyObject *args, PyObject *kwds)
if (ct == NULL)
return NULL;
return direct_newp(ct, init);
return direct_newp(ct, init, allocator);
}
static PyObject *ffi_new(FFIObject *self, PyObject *args, PyObject *kwds)
{
return _ffi_new(self, args, kwds, default_allocator);
}
static PyObject *_ffi_new_with_allocator(PyObject *allocator, PyObject *args,
PyObject *kwds)
{
return _ffi_new((FFIObject *)PyTuple_GET_ITEM(allocator, 0),
args, kwds,
&PyTuple_GET_ITEM(allocator, 1));
}
PyDoc_STRVAR(ffi_new_allocator_doc, "XXX");
static PyObject *ffi_new_allocator(FFIObject *self, PyObject *args,
PyObject *kwds)
{
PyObject *allocator, *result;
PyObject *my_alloc = Py_None, *my_free = Py_None;
int should_clear_after_alloc = 1;
static char *keywords[] = {"alloc", "free", "should_clear_after_alloc",
NULL};
if (!PyArg_ParseTupleAndKeywords(args, kwds, "|OOi:new_allocator", keywords,
&my_alloc, &my_free,
&should_clear_after_alloc))
return NULL;
if (my_alloc == Py_None && my_free != Py_None) {
PyErr_SetString(PyExc_TypeError, "cannot pass 'free' without 'alloc'");
return NULL;
}
allocator = PyTuple_New(4);
if (allocator == NULL)
return NULL;
Py_INCREF(self);
PyTuple_SET_ITEM(allocator, 0, (PyObject *)self);
if (my_alloc != Py_None) {
Py_INCREF(my_alloc);
PyTuple_SET_ITEM(allocator, 1, my_alloc);
}
if (my_free != Py_None) {
Py_INCREF(my_free);
PyTuple_SET_ITEM(allocator, 2, my_free);
}
if (!should_clear_after_alloc) {
Py_INCREF(Py_True);
PyTuple_SET_ITEM(allocator, 3, Py_True); /* dont_clear_after_alloc */
}
{
static PyMethodDef md = {"allocator",
(PyCFunction)_ffi_new_with_allocator,
METH_VARARGS | METH_KEYWORDS};
result = PyCFunction_New(&md, allocator);
}
Py_DECREF(allocator);
return result;
}
PyDoc_STRVAR(ffi_cast_doc,
......@@ -805,6 +869,7 @@ static PyMethodDef ffi_methods[] = {
#endif
{"integer_const",(PyCFunction)ffi_int_const,METH_VKW, ffi_int_const_doc},
{"new", (PyCFunction)ffi_new, METH_VKW, ffi_new_doc},
{"new_allocator",(PyCFunction)ffi_new_allocator,METH_VKW,ffi_new_allocator_doc},
{"new_handle", (PyCFunction)ffi_new_handle, METH_O, ffi_new_handle_doc},
{"offsetof", (PyCFunction)ffi_offsetof, METH_VARARGS, ffi_offsetof_doc},
{"sizeof", (PyCFunction)ffi_sizeof, METH_O, ffi_sizeof_doc},
......
......@@ -236,6 +236,30 @@ class FFI(object):
cdecl = self._typeof(cdecl)
return self._backend.newp(cdecl, init)
def new_allocator(self, alloc=None, free=None,
should_clear_after_alloc=True):
"""Return a new allocator, i.e. a function that behaves like ffi.new()
but uses the provided low-level 'alloc' and 'free' functions.
'alloc' is called with the size as argument. If it returns NULL, a
MemoryError is raised. 'free' is called with the result of 'alloc'
as argument. Both can be either Python function or directly C
functions. If 'free' is None, then no free function is called.
If both 'alloc' and 'free' are None, the default is used.
If 'should_clear_after_alloc' is set to False, then the memory
returned by 'alloc' is assumed to be already cleared (or you are
fine with garbage); otherwise CFFI will clear it.
"""
compiled_ffi = self._backend.FFI()
allocator = compiled_ffi.new_allocator(alloc, free,
should_clear_after_alloc)
def allocate(cdecl, init=None):
if isinstance(cdecl, basestring):
cdecl = self._typeof(cdecl)
return allocator(cdecl, init)
return allocate
def cast(self, cdecl, source):
"""Similar to a C cast: returns an instance of the named C
type initialized with the given 'source'. The source is
......
......@@ -57,6 +57,77 @@ class TestFFI(backend_tests.BackendTests,
assert tb.tb_frame.f_code.co_name == 'cb'
assert tb.tb_frame.f_locals['n'] == 234
def test_ffi_new_allocator_2(self):
ffi = FFI(backend=self.Backend())
seen = []
def myalloc(size):
seen.append(size)
return ffi.new("char[]", "X" * size)
def myfree(raw):
seen.append(raw)
alloc1 = ffi.new_allocator(myalloc, myfree)
alloc2 = ffi.new_allocator(alloc=myalloc, free=myfree,
should_clear_after_alloc=False)
p1 = alloc1("int[10]")
p2 = alloc2("int[]", 10)
assert seen == [40, 40]
assert ffi.typeof(p1) == ffi.typeof("int[10]")
assert ffi.sizeof(p1) == 40
assert ffi.typeof(p2) == ffi.typeof("int[]")
assert ffi.sizeof(p2) == 40
assert p1[5] == 0
assert p2[6] == ord('X') * 0x01010101
raw1 = ffi.cast("char *", p1)
raw2 = ffi.cast("char *", p2)
del p1, p2
retries = 0
while len(seen) != 4:
retries += 1
assert retries <= 5
import gc; gc.collect()
assert seen == [40, 40, raw1, raw2]
assert repr(seen[2]) == "<cdata 'char[]' owning 41 bytes>"
assert repr(seen[3]) == "<cdata 'char[]' owning 41 bytes>"
def test_ffi_new_allocator_3(self):
ffi = FFI(backend=self.Backend())
seen = []
def myalloc(size):
seen.append(size)
return ffi.new("char[]", "X" * size)
alloc1 = ffi.new_allocator(myalloc) # no 'free'
p1 = alloc1("int[10]")
assert seen == [40]
assert ffi.typeof(p1) == ffi.typeof("int[10]")
assert ffi.sizeof(p1) == 40
assert p1[5] == 0
def test_ffi_new_allocator_4(self):
ffi = FFI(backend=self.Backend())
py.test.raises(TypeError, ffi.new_allocator, free=lambda x: None)
#
def myalloc2(size):
raise LookupError
alloc2 = ffi.new_allocator(myalloc2)
py.test.raises(LookupError, alloc2, "int[5]")
#
def myalloc3(size):
return 42
alloc3 = ffi.new_allocator(myalloc3)
e = py.test.raises(TypeError, alloc3, "int[5]")
assert str(e.value) == "alloc() must return a cdata object (got int)"
#
def myalloc4(size):
return ffi.cast("int", 42)
alloc4 = ffi.new_allocator(myalloc4)
e = py.test.raises(TypeError, alloc4, "int[5]")
assert str(e.value) == "alloc() must return a cdata pointer, not 'int'"
#
def myalloc5(size):
return ffi.NULL
alloc5 = ffi.new_allocator(myalloc5)
py.test.raises(MemoryError, alloc5, "int[5]")
class TestBitfield:
def check(self, source, expected_ofs_y, expected_align, expected_size):
......
......@@ -224,3 +224,95 @@ def test_ffi_getwinerror():
n = (1 << 29) + 42
code, message = ffi.getwinerror(code=n)
assert code == n
def test_ffi_new_allocator_1():
ffi = _cffi1_backend.FFI()
alloc1 = ffi.new_allocator()
alloc2 = ffi.new_allocator(should_clear_after_alloc=False)
for retry in range(100):
p1 = alloc1("int[10]")
p2 = alloc2("int[10]")
combination = 0
for i in range(10):
assert p1[i] == 0
combination |= p2[i]
p1[i] = -42
p2[i] = -43
if combination != 0:
break
del p1, p2
import gc; gc.collect()
else:
raise AssertionError("cannot seem to get an int[10] not "
"completely cleared")
def test_ffi_new_allocator_2():
ffi = _cffi1_backend.FFI()
seen = []
def myalloc(size):
seen.append(size)
return ffi.new("char[]", "X" * size)
def myfree(raw):
seen.append(raw)
alloc1 = ffi.new_allocator(myalloc, myfree)
alloc2 = ffi.new_allocator(alloc=myalloc, free=myfree,
should_clear_after_alloc=False)
p1 = alloc1("int[10]")
p2 = alloc2("int[]", 10)
assert seen == [40, 40]
assert ffi.typeof(p1) == ffi.typeof("int[10]")
assert ffi.sizeof(p1) == 40
assert ffi.typeof(p2) == ffi.typeof("int[]")
assert ffi.sizeof(p2) == 40
assert p1[5] == 0
assert p2[6] == ord('X') * 0x01010101
raw1 = ffi.cast("char *", p1)
raw2 = ffi.cast("char *", p2)
del p1, p2
retries = 0
while len(seen) != 4:
retries += 1
assert retries <= 5
import gc; gc.collect()
assert seen == [40, 40, raw1, raw2]
assert repr(seen[2]) == "<cdata 'char[]' owning 41 bytes>"
assert repr(seen[3]) == "<cdata 'char[]' owning 41 bytes>"
def test_ffi_new_allocator_3():
ffi = _cffi1_backend.FFI()
seen = []
def myalloc(size):
seen.append(size)
return ffi.new("char[]", "X" * size)
alloc1 = ffi.new_allocator(myalloc) # no 'free'
p1 = alloc1("int[10]")
assert seen == [40]
assert ffi.typeof(p1) == ffi.typeof("int[10]")
assert ffi.sizeof(p1) == 40
assert p1[5] == 0
def test_ffi_new_allocator_4():
ffi = _cffi1_backend.FFI()
py.test.raises(TypeError, ffi.new_allocator, free=lambda x: None)
#
def myalloc2(size):
raise LookupError
alloc2 = ffi.new_allocator(myalloc2)
py.test.raises(LookupError, alloc2, "int[5]")
#
def myalloc3(size):
return 42
alloc3 = ffi.new_allocator(myalloc3)
e = py.test.raises(TypeError, alloc3, "int[5]")
assert str(e.value) == "alloc() must return a cdata object (got int)"
#
def myalloc4(size):
return ffi.cast("int", 42)
alloc4 = ffi.new_allocator(myalloc4)
e = py.test.raises(TypeError, alloc4, "int[5]")
assert str(e.value) == "alloc() must return a cdata pointer, not 'int'"
#
def myalloc5(size):
return ffi.NULL
alloc5 = ffi.new_allocator(myalloc5)
py.test.raises(MemoryError, alloc5, "int[5]")
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment