This instance will be upgraded to Heptapod 0.21.0rc3 on 2021-04-23 at 10:30 UTC+2 (a few minutes of down time)

some ascii fast paths of latin-1 encoding/decoding

parent 97476e67e885
Pipeline #18699 failed with stage
in 1 minute and 4 seconds
......@@ -11,6 +11,7 @@ from rpython.rlib import rutf8
from pypy.interpreter.unicodehelper import str_decode_utf8
from pypy.interpreter.unicodehelper import utf8_encode_ascii, str_decode_ascii
from pypy.interpreter.unicodehelper import utf8_encode_latin_1
from pypy.interpreter import unicodehelper as uh
from pypy.module._codecs.interp_codecs import CodecState
......@@ -91,3 +92,9 @@ def test_encode_decimal(space):
result = uh.unicode_encode_decimal(
u'12\u1234'.encode('utf8'), 'xmlcharrefreplace', handler)
assert result == '12ሴ'
def test_utf8_encode_latin1_ascii_prefix():
utf8 = b'abcde\xc3\xa4g'
b = utf8_encode_latin_1(utf8, None, None)
assert b == b'abcde\xe4g'
......@@ -155,14 +155,15 @@ def utf8_encode_latin_1(s, errors, errorhandler):
try:
rutf8.check_ascii(s)
return s
except rutf8.CheckError:
return _utf8_encode_latin_1_slowpath(s, errors, errorhandler)
except rutf8.CheckError, e:
return _utf8_encode_latin_1_slowpath(s, e.pos, errors, errorhandler)
def _utf8_encode_latin_1_slowpath(s, errors, errorhandler):
def _utf8_encode_latin_1_slowpath(s, first_non_ascii_char, errors, errorhandler):
size = len(s)
result = StringBuilder(size)
index = 0
pos = 0
result.append_slice(s, 0, first_non_ascii_char)
pos = first_non_ascii_char
while pos < size:
ch = rutf8.codepoint_at_pos(s, pos)
if ch <= 0xFF:
......
......@@ -217,6 +217,14 @@ class TestUnicodeObject:
uniupper, = unicodedb.toupper_full(ch)
assert chr(uniupper) == chr(ch).upper()
def test_latin1_encode_shortcut_ascii(self, monkeypatch):
from rpython.rlib import rutf8
from pypy.objspace.std.unicodeobject import encode_object
monkeypatch.setattr(rutf8, "check_ascii", None)
w_b = encode_object(self.space, self.space.newutf8("abc", 3), "latin-1", "strict")
assert self.space.bytes_w(w_b) == "abc"
class AppTestUnicodeStringStdOnly:
def test_compares(self):
assert u'a' == 'a'
......
......@@ -1216,6 +1216,9 @@ def encode_object(space, w_obj, encoding, errors):
if rutf8.has_surrogates(utf8):
utf8 = rutf8.reencode_utf8_with_surrogates(utf8)
return space.newbytes(utf8)
if (encoding == "latin1" or encoding == "latin-1" and
isinstance(w_obj, W_UnicodeObject) and w_obj.is_ascii()):
return space.newbytes(w_obj._utf8)
return encode(space, w_obj, encoding, errors)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment