This instance will be upgraded to Heptapod 0.31.0 (final) on 2022-05-24 at 14:00 UTC+2 (a few minutes of down time)

Commit 44786d64 authored by Amaury Forgeot d'Arc's avatar Amaury Forgeot d'Arc
Browse files

Merge branch/interplevel-codecs:

Rewrite all encodings implemented at applevel,
move most of them to rlib.runicode where they may be useful for RPython programs.

- This fixes translation with -O0: this option disables geninterp,
and uses the pypy compiler instead to compile applevel code.
But unicode literals need to be decoded with the codec module...

- This also removes some huge geninterp'd code: app_codecs.py used to contain 3
or 4 functions each rendered with 20000 lines of C code!

+ use StringBuilder everywhere
parent e965f3e6baa9
......@@ -72,12 +72,12 @@ def parsestr(space, encoding, s):
bufp = 0
bufq = len(buf)
assert 0 <= bufp <= bufq
w_substr = space.wrap(buf[bufp : bufq])
substr = buf[bufp:bufq]
if rawmode:
w_v = unicodehelper.PyUnicode_DecodeRawUnicodeEscape(space, w_substr)
v = unicodehelper.PyUnicode_DecodeRawUnicodeEscape(space, substr)
else:
w_v = unicodehelper.PyUnicode_DecodeUnicodeEscape(space, w_substr)
return w_v
v = unicodehelper.PyUnicode_DecodeUnicodeEscape(space, substr)
return space.wrap(v)
need_encoding = (encoding is not None and
encoding != "utf-8" and encoding != "iso-8859-1")
......@@ -86,7 +86,7 @@ def parsestr(space, encoding, s):
substr = s[ps : q]
if rawmode or '\\' not in s[ps:]:
if need_encoding:
w_u = unicodehelper.PyUnicode_DecodeUTF8(space, space.wrap(substr))
w_u = space.wrap(unicodehelper.PyUnicode_DecodeUTF8(space, substr))
#w_v = space.wrap(space.unwrap(w_u).encode(encoding)) this works
w_v = unicodehelper.PyUnicode_AsEncodedString(space, w_u, space.wrap(encoding))
return w_v
......@@ -96,7 +96,7 @@ def parsestr(space, encoding, s):
enc = None
if need_encoding:
enc = encoding
v = PyString_DecodeEscape(space, substr, unicode, enc)
v = PyString_DecodeEscape(space, substr, enc)
return space.wrap(v)
def hexbyte(val):
......@@ -105,10 +105,9 @@ def hexbyte(val):
result = "0" + result
return result
def PyString_DecodeEscape(space, s, unicode, recode_encoding):
def PyString_DecodeEscape(space, s, recode_encoding):
"""
Unescape a backslash-escaped string. If unicode is non-zero,
the string is a u-literal. If recode_encoding is non-zero,
Unescape a backslash-escaped string. If recode_encoding is non-zero,
the string is UTF-8 encoded and should be re-encoded in the
specified encoding.
"""
......@@ -171,9 +170,6 @@ def PyString_DecodeEscape(space, s, unicode, recode_encoding):
raise_app_valueerror(space, 'invalid \\x escape')
# ignored replace and ignore for now
elif unicode and (ch == 'u' or ch == 'U' or ch == 'N'):
raise_app_valueerror(space, 'Unicode escapes not legal '
'when Unicode disabled')
else:
# this was not an escape, so the backslash
# has to be added, and we start over in
......@@ -200,7 +196,7 @@ def decode_utf8(space, s, ps, end, encoding):
# while (s < end && *s != '\\') s++; */ /* inefficient for u".."
while ps < end and ord(s[ps]) & 0x80:
ps += 1
w_u = unicodehelper.PyUnicode_DecodeUTF8(space, space.wrap(s[pt : ps]))
w_u = space.wrap(unicodehelper.PyUnicode_DecodeUTF8(space, s[pt:ps]))
w_v = unicodehelper.PyUnicode_AsEncodedString(space, w_u, space.wrap(encoding))
v = space.str_w(w_v)
return v, ps
......
from pypy.interpreter import gateway
from pypy.module._codecs import interp_codecs
app = gateway.applevel(r'''
def PyUnicode_DecodeUnicodeEscape(data):
import _codecs
return _codecs.unicode_escape_decode(data)[0]
def PyUnicode_AsEncodedString(space, w_data, w_encoding):
return interp_codecs.encode(space, w_data, w_encoding)
def PyUnicode_DecodeRawUnicodeEscape(data):
import _codecs
return _codecs.raw_unicode_escape_decode(data)[0]
def PyUnicode_DecodeUTF8(data):
import _codecs
return _codecs.utf_8_decode(data)[0]
def PyUnicode_AsEncodedString(data, encoding):
import _codecs
return _codecs.encode(data, encoding)
def PyUnicode_EncodeUTF8(data):
import _codecs
return _codecs.utf_8_encode(data)[0]
''')
PyUnicode_DecodeUnicodeEscape = app.interphook('PyUnicode_DecodeUnicodeEscape')
PyUnicode_DecodeRawUnicodeEscape = app.interphook('PyUnicode_DecodeRawUnicodeEscape')
PyUnicode_DecodeUTF8 = app.interphook('PyUnicode_DecodeUTF8')
PyUnicode_AsEncodedString = app.interphook('PyUnicode_AsEncodedString')
PyUnicode_EncodeUTF8 = app.interphook('PyUnicode_EncodeUTF8')
# These functions take and return unwrapped rpython strings and unicodes
PyUnicode_DecodeUnicodeEscape = interp_codecs.make_raw_decoder('unicode_escape')
PyUnicode_DecodeRawUnicodeEscape = interp_codecs.make_raw_decoder('raw_unicode_escape')
PyUnicode_DecodeUTF8 = interp_codecs.make_raw_decoder('utf_8')
PyUnicode_EncodeUTF8 = interp_codecs.make_raw_encoder('utf_8')
......@@ -3,22 +3,43 @@ from pypy.rlib import runicode
from pypy.module._codecs import interp_codecs
class Module(MixedModule):
appleveldefs = {
'__doc__' : 'app_codecs.__doc__',
'__name__' : 'app_codecs.__name__',
'charmap_encode' : 'app_codecs.charmap_encode',
'escape_decode' : 'app_codecs.escape_decode',
'escape_encode' : 'app_codecs.escape_encode',
'raw_unicode_escape_decode' : 'app_codecs.raw_unicode_escape_decode',
'raw_unicode_escape_encode' : 'app_codecs.raw_unicode_escape_encode',
'unicode_escape_decode' : 'app_codecs.unicode_escape_decode',
'unicode_escape_encode' : 'app_codecs.unicode_escape_encode',
'unicode_internal_decode' : 'app_codecs.unicode_internal_decode',
'unicode_internal_encode' : 'app_codecs.unicode_internal_encode',
'utf_7_decode' : 'app_codecs.utf_7_decode',
'utf_7_encode' : 'app_codecs.utf_7_encode',
'charmap_build' : 'app_codecs.charmap_build'
}
"""
_codecs -- Provides access to the codec registry and the builtin
codecs.
This module should never be imported directly. The standard library
module "codecs" wraps this builtin module for use within Python.
The codec registry is accessible via:
register(search_function) -> None
lookup(encoding) -> (encoder, decoder, stream_reader, stream_writer)
The builtin Unicode codecs use the following interface:
<encoding>_encode(Unicode_object[,errors='strict']) ->
(string object, bytes consumed)
<encoding>_decode(char_buffer_obj[,errors='strict']) ->
(Unicode object, bytes consumed)
<encoding>_encode() interfaces also accept non-Unicode object as
input. The objects are then converted to Unicode using
PyUnicode_FromObject() prior to applying the conversion.
These <encoding>s are available: utf_8, unicode_escape,
raw_unicode_escape, unicode_internal, latin_1, ascii (7-bit),
mbcs (on win32).
Written by Marc-Andre Lemburg (mal@lemburg.com).
Copyright (c) Corporation for National Research Initiatives.
"""
appleveldefs = {}
interpleveldefs = {
'encode': 'interp_codecs.encode',
'decode': 'interp_codecs.decode',
......@@ -26,12 +47,15 @@ class Module(MixedModule):
'lookup_error': 'interp_codecs.lookup_error',
'register': 'interp_codecs.register_codec',
'register_error': 'interp_codecs.register_error',
'charmap_build' : 'interp_codecs.charmap_build',
# encoders and decoders
'ascii_decode' : 'interp_codecs.ascii_decode',
'ascii_encode' : 'interp_codecs.ascii_encode',
'latin_1_decode' : 'interp_codecs.latin_1_decode',
'latin_1_encode' : 'interp_codecs.latin_1_encode',
'utf_7_decode' : 'interp_codecs.utf_7_decode',
'utf_7_encode' : 'interp_codecs.utf_7_encode',
'utf_8_decode' : 'interp_codecs.utf_8_decode',
'utf_8_encode' : 'interp_codecs.utf_8_encode',
'utf_16_be_decode' : 'interp_codecs.utf_16_be_decode',
......@@ -44,6 +68,15 @@ class Module(MixedModule):
'charbuffer_encode': 'interp_codecs.buffer_encode',
'readbuffer_encode': 'interp_codecs.buffer_encode',
'charmap_decode' : 'interp_codecs.charmap_decode',
'charmap_encode' : 'interp_codecs.charmap_encode',
'escape_encode' : 'interp_codecs.escape_encode',
'escape_decode' : 'interp_codecs.escape_decode',
'unicode_escape_decode' : 'interp_codecs.unicode_escape_decode',
'unicode_escape_encode' : 'interp_codecs.unicode_escape_encode',
'raw_unicode_escape_decode' : 'interp_codecs.raw_unicode_escape_decode',
'raw_unicode_escape_encode' : 'interp_codecs.raw_unicode_escape_encode',
'unicode_internal_decode' : 'interp_codecs.unicode_internal_decode',
'unicode_internal_encode' : 'interp_codecs.unicode_internal_encode',
}
def __init__(self, space, *args):
......
This diff is collapsed.
This diff is collapsed.
import autopath
from pypy.conftest import gettestobjspace
from pypy.module._codecs.app_codecs import unicode_escape_encode,\
charmap_encode, unicode_escape_decode
class AppTestCodecs:
......@@ -14,26 +12,16 @@ class AppTestCodecs:
raises(TypeError, _codecs.register, 1)
def test_bigU_codecs(self):
import sys
oldmaxunicode = sys.maxunicode
if sys.maxunicode <= 0xffff:
return # this test cannot run on UCS2 builds
u = u'\U00010001\U00020002\U00030003\U00040004\U00050005'
for encoding in ('utf-8', 'utf-16', 'utf-16-le', 'utf-16-be',
'raw_unicode_escape',
'unicode_escape', 'unicode_internal'):
assert unicode(u.encode(encoding),encoding) == u
sys.maxunicode = oldmaxunicode
def test_ucs4(self):
import sys
oldmaxunicode = sys.maxunicode
if sys.maxunicode <= 0xffff:
sys.maxunicode = 0xffffffff
x = u'\U00100000'
y = x.encode("raw-unicode-escape").decode("raw-unicode-escape")
assert x == y
sys.maxunicode = oldmaxunicode
def test_named_unicode(self):
assert unicode('\\N{SPACE}','unicode-escape') == u" "
......@@ -118,12 +106,20 @@ class AppTestCodecs:
def test_charmap_decode(self):
from _codecs import charmap_decode
import sys
assert charmap_decode('', 'strict', 'blablabla') == ('', 0)
assert charmap_decode('xxx') == ('xxx', 3)
assert charmap_decode('xxx', 'strict', {ord('x'): u'XX'}) == ('XXXXXX', 3)
map = tuple([unichr(i) for i in range(256)])
assert charmap_decode('xxx\xff', 'strict', map) == (u'xxx\xff', 4)
raises(TypeError, charmap_decode, '\xff', "replace", {0xff: 0x10001})
def test_unicode_escape(self):
from _codecs import unicode_escape_encode, unicode_escape_decode
assert unicode_escape_encode(u'abc') == (u'abc'.encode('unicode_escape'), 3)
assert unicode_escape_decode('abc') == (u'abc'.decode('unicode_escape'), 3)
assert unicode_escape_decode('\\x61\\x62\\x63') == (u'abc', 12)
class AppTestPartialEvaluation:
......@@ -377,6 +373,9 @@ class AppTestPartialEvaluation:
def test_charmap_decode_1(self):
import codecs
assert codecs.charmap_encode(u'xxx') == ('xxx', 3)
assert codecs.charmap_encode(u'xxx', 'strict', {ord('x'): 'XX'}) == ('XXXXXX', 3)
res = codecs.charmap_decode("\x00\x01\x02", "replace", u"ab")
assert res == (u"ab\ufffd", 3)
res = codecs.charmap_decode("\x00\x01\x02", "replace", u"ab\ufffe")
......@@ -464,6 +463,9 @@ class AppTestPartialEvaluation:
assert '\xff'.decode('utf-7', 'ignore') == ''
assert '\x00'.decode('unicode-internal', 'ignore') == ''
def test_backslahreplace(self):
assert u'a\xac\u1234\u20ac\u8000'.encode('ascii', 'backslashreplace') == 'a\\xac\u1234\u20ac\u8000'
def test_badhandler(self):
import codecs
results = ( 42, u"foo", (1,2,3), (u"foo", 1, 3), (u"foo", None), (u"foo",), ("foo", 1, 3), ("foo", None), ("foo",) )
......@@ -527,9 +529,26 @@ class AppTestPartialEvaluation:
def test_charmap_encode(self):
assert 'xxx'.encode('charmap') == 'xxx'
import codecs
raises(TypeError, codecs.charmap_encode, u'\xff', "replace", {0xff: 300})
raises(UnicodeError, codecs.charmap_encode, u"\xff", "replace", {0xff: None})
def test_charmap_encode_replace(self):
charmap = dict([ (ord(c), 2*c.upper()) for c in "abcdefgh"])
charmap[ord("?")] = "XYZ"
import codecs
sin = u"abcDEF"
sout = codecs.charmap_encode(sin, "replace", charmap)[0]
assert sout == "AABBCCXYZXYZXYZ"
def test_charmap_decode_2(self):
assert 'foo'.decode('charmap') == 'foo'
def test_charmap_build(self):
import codecs
assert codecs.charmap_build(u'123456') == {49: 0, 50: 1, 51: 2,
52: 3, 53: 4, 54: 5}
def test_utf7_start_end_in_exception(self):
try:
'+IC'.decode('utf-7')
......@@ -537,6 +556,9 @@ class AppTestPartialEvaluation:
assert exc.start == 0
assert exc.end == 3
def test_utf7_surrogate(self):
raises(UnicodeDecodeError, '+3ADYAA-'.decode, 'utf-7')
def test_utf_16_encode_decode(self):
import codecs
x = u'123abc'
......@@ -546,6 +568,8 @@ class AppTestPartialEvaluation:
def test_unicode_escape(self):
assert u'\\'.encode('unicode-escape') == '\\\\'
assert '\\\\'.decode('unicode-escape') == u'\\'
assert u'\ud801'.encode('unicode-escape') == '\\ud801'
assert u'\u0013'.encode('unicode-escape') == '\\x13'
def test_mbcs(self):
import sys
......@@ -555,14 +579,3 @@ class AppTestPartialEvaluation:
assert u'caf\xe9'.encode('mbcs') == 'caf\xe9'
assert u'\u040a'.encode('mbcs') == '?' # some cyrillic letter
assert 'cafx\e9'.decode('mbcs') == u'cafx\e9'
class TestDirect:
def test_charmap_encode(self):
assert charmap_encode(u'xxx') == ('xxx', 3)
assert charmap_encode(u'xxx', 'strict', {ord('x'): 'XX'}) == ('XXXXXX', 6)
def test_unicode_escape(self):
assert unicode_escape_encode(u'abc') == (u'abc'.encode('unicode_escape'), 3)
assert unicode_escape_decode('abc') == (u'abc'.decode('unicode_escape'), 3)
assert unicode_escape_decode('\\x61\\x62\\x63') == (u'abc', 12)
......@@ -447,11 +447,11 @@ def unmarshal_pycode(space, u, tc):
register(TYPE_CODE, unmarshal_pycode)
def marshal_w__Unicode(space, w_unicode, m):
s = space.str_w(unicodehelper.PyUnicode_EncodeUTF8(space, w_unicode))
s = unicodehelper.PyUnicode_EncodeUTF8(space, space.unicode_w(w_unicode))
m.atom_str(TYPE_UNICODE, s)
def unmarshal_Unicode(space, u, tc):
return unicodehelper.PyUnicode_DecodeUTF8(space, space.wrap(u.get_str()))
return space.wrap(unicodehelper.PyUnicode_DecodeUTF8(space, u.get_str()))
register(TYPE_UNICODE, unmarshal_Unicode)
app = gateway.applevel(r'''
......
......@@ -12,6 +12,7 @@ from pypy.objspace.std.tupleobject import W_TupleObject
from pypy.rlib.rarithmetic import intmask, ovfcheck
from pypy.rlib.objectmodel import compute_hash
from pypy.rlib.rstring import string_repeat
from pypy.rlib.runicode import unicode_encode_unicode_escape
from pypy.module.unicodedata import unicodedb_4_1_0 as unicodedb
from pypy.tool.sourcetools import func_with_new_name
......@@ -892,101 +893,11 @@ def unicode_translate__Unicode_ANY(space, w_self, w_table):
space.wrap("character mapping must return integer, None or unicode"))
return W_UnicodeObject(u''.join(result))
# Move this into the _codecs module as 'unicodeescape_string (Remember to cater for quotes)'
def repr__Unicode(space, w_unicode):
hexdigits = "0123456789abcdef"
chars = w_unicode._value
size = len(chars)
singlequote = doublequote = False
for c in chars:
if c == u'\'':
singlequote = True
elif c == u'"':
doublequote = True
if singlequote and not doublequote:
quote = '"'
else:
quote = '\''
result = ['u', quote]
j = 0
while j<len(chars):
ch = chars[j]
code = ord(ch)
if code >= 0x10000:
# Resize if needed
result.extend(['\\', "U",
hexdigits[(code >> 28) & 0xf],
hexdigits[(code >> 24) & 0xf],
hexdigits[(code >> 20) & 0xf],
hexdigits[(code >> 16) & 0xf],
hexdigits[(code >> 12) & 0xf],
hexdigits[(code >> 8) & 0xf],
hexdigits[(code >> 4) & 0xf],
hexdigits[(code >> 0) & 0xf],
])
j += 1
continue
if code >= 0xD800 and code < 0xDC00:
if j < size - 1:
ch2 = chars[j+1]
code2 = ord(ch2)
if code2 >= 0xDC00 and code2 <= 0xDFFF:
code = (((code & 0x03FF) << 10) | (code2 & 0x03FF)) + 0x00010000
result.extend(['\\', "U",
hexdigits[(code >> 28) & 0xf],
hexdigits[(code >> 24) & 0xf],
hexdigits[(code >> 20) & 0xf],
hexdigits[(code >> 16) & 0xf],
hexdigits[(code >> 12) & 0xf],
hexdigits[(code >> 8) & 0xf],
hexdigits[(code >> 4) & 0xf],
hexdigits[(code >> 0) & 0xf],
])
j += 2
continue
if code >= 0x100:
result.extend(['\\', "u",
hexdigits[(code >> 12) & 0xf],
hexdigits[(code >> 8) & 0xf],
hexdigits[(code >> 4) & 0xf],
hexdigits[(code >> 0) & 0xf],
])
j += 1
continue
if code == ord('\\') or code == ord(quote):
result.append('\\')
result.append(chr(code))
j += 1
continue
if code == ord('\t'):
result.append('\\')
result.append('t')
j += 1
continue
if code == ord('\r'):
result.append('\\')
result.append('r')
j += 1
continue
if code == ord('\n'):
result.append('\\')
result.append('n')
j += 1
continue
if code < ord(' ') or code >= 0x7f:
result.extend(['\\', "x",
hexdigits[(code >> 4) & 0xf],
hexdigits[(code >> 0) & 0xf],
])
j += 1
continue
result.append(chr(code))
j += 1
result.append(quote)
return space.wrap(''.join(result))
s = unicode_encode_unicode_escape(chars, size, "strict", quotes=True)
return space.wrap(s)
def mod__Unicode_ANY(space, w_format, w_values):
return mod_format(space, w_format, w_values, do_unicode=True)
......
......@@ -56,6 +56,7 @@ class AbstractStringBuilder(object):
self.l.append(s)
def append_slice(self, s, start, end):
assert 0 <= start <= end <= len(s)
self.l.append(s[start:end])
def append_multiple_char(self, c, times):
......
This diff is collapsed.
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment