Read about our upcoming Code of Conduct on this issue

Commit f02bbdda authored by Carl Friedrich Bolz-Tereick's avatar Carl Friedrich Bolz-Tereick
Browse files

fix #3108: the map based parser didn't deal with json dicts with repeated keys

correctly
parent 73f583091cbf
......@@ -342,7 +342,14 @@ class JSONDecoder(W_Root):
currmap = self.startmap
while True:
# parse a key: value
currmap = self.decode_key_map(i, currmap)
newmap = self.decode_key_map(i, currmap)
if newmap is None:
# We've seen a repeated value, switch to dict-based storage.
dict_w = self._switch_to_dict(currmap, values_w, nextindex)
# We re-parse the last key, to get the correct overwriting
# effect. Pointless to care for performance here.
return self.decode_object_dict(i, start, dict_w)
currmap = newmap
i = self.skip_whitespace(self.pos)
ch = self.ll_chars[i]
if ch != ':':
......@@ -610,6 +617,8 @@ class JSONDecoder(W_Root):
""" Given the current map currmap of an object, decode the next key at
position i. This returns the new map of the object. """
newmap = self._decode_key_map(i, currmap)
if newmap is None:
return None
currmap.observe_transition(newmap, self.startmap)
return newmap
......@@ -789,6 +798,11 @@ class MapBase(object):
self.nextmap_first._check_invariants()
def get_next(self, w_key, string, start, stop, terminator):
""" Returns the next map, given a wrapped key w_key, the json input
string with positions start and stop, as well as a terminator.
Returns None if the key already appears somewhere in the map chain.
"""
from pypy.objspace.std.dictmultiobject import unicode_hash, unicode_eq
if isinstance(self, JSONMap):
assert not self.state == MapBase.BLOCKED
......@@ -803,6 +817,8 @@ class MapBase(object):
if nextmap_first is None:
# first transition ever seen, don't initialize nextmap_all
next = self._make_next_map(w_key, string[start:stop])
if next is None:
return None
self.nextmap_first = next
else:
if self.nextmap_all is None:
......@@ -817,6 +833,8 @@ class MapBase(object):
# if we are at this point we didn't find the transition yet, so
# create a new one
next = self._make_next_map(w_key, string[start:stop])
if next is None:
return None
self.nextmap_all[w_key] = next
# one new leaf has been created
......@@ -859,6 +877,14 @@ class MapBase(object):
self.mark_useful(terminator)
def _make_next_map(self, w_key, key_repr):
# Check whether w_key is already part of the self.prev chain
# to prevent strangeness in the json dict implementation.
# This is slow, but it should be rare to call this function.
check = self
while isinstance(check, JSONMap):
if check.w_key._utf8 == w_key._utf8:
return None
check = check.prev
return JSONMap(self.space, self, w_key, key_repr)
def fill_dict(self, dict_w, values_w):
......
......@@ -74,6 +74,17 @@ class TestJson(object):
m3.fill_dict(d, [space.w_None, space.w_None, space.w_None])
assert list(d) == [w_a, w_b, w_c]
def test_repeated_key_get_next(self):
m = Terminator(self.space)
w_a = self.space.newutf8("a", 1)
w_b = self.space.newutf8("b", 1)
w_c = self.space.newutf8("c", 1)
m1 = m.get_next(w_a, '"a"', 0, 3, m)
m1 = m1.get_next(w_b, '"b"', 0, 3, m)
m1 = m1.get_next(w_c, '"c"', 0, 3, m)
m2 = m1.get_next(w_a, '"a"', 0, 3, m)
assert m2 is None
def test_decode_key_map(self):
m = Terminator(self.space)
......@@ -519,3 +530,11 @@ class AppTest(object):
exc = raises(ValueError, _pypyjson.loads, inputtext)
assert str(exc.value) == errmsg
def test_repeated_key(self):
import _pypyjson
a = '{"abc": "4", "k": 1, "k": 2}'
d = _pypyjson.loads(a)
assert d == {u"abc": u"4", u"k": 2}
a = '{"abc": "4", "k": 1, "k": 1.5, "c": null, "k": 2}'
d = _pypyjson.loads(a)
assert d == {u"abc": u"4", u"c": None, u"k": 2}
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment