Commit b52effc4 authored by Matti Picus's avatar Matti Picus
Browse files

merge py3.8-isidentical-ast-feature-version into py3.8

--HG--
branch : py3.8
Pipeline #25287 passed with stage
in 14 minutes and 6 seconds
......@@ -144,6 +144,10 @@ class ASTBuilder(object):
raise SyntaxError(msg, ast_node.lineno, ast_node.col_offset,
filename=self.compile_info.filename)
def check_feature(self, condition, version, msg, n):
if condition and self.compile_info.feature_version < version:
return self.error(msg, n)
def deprecation_warn(self, msg, n):
from pypy.module._warnings.interp_warnings import warn_explicit
space = self.space
......@@ -420,6 +424,12 @@ class ASTBuilder(object):
return self.space.w_None, False
def handle_for_stmt(self, for_node, is_async):
self.check_feature(
is_async,
version=5,
msg="Async for loops are only supported in Python 3.5 and greater",
n=for_node
)
target_node = for_node.get_child(1)
target_as_exprlist = self.handle_exprlist(target_node, ast.Store)
if target_node.num_children() == 1:
......@@ -488,6 +498,12 @@ class ASTBuilder(object):
return ast.withitem(test, target)
def handle_with_stmt(self, with_node, is_async):
self.check_feature(
is_async,
version=5,
msg="Async with statements are only supported in Python 3.5 and greater",
n=with_node
)
body = self.handle_suite(with_node.get_child(-1))
type_comment, has_type_comment = self.handle_type_comment(with_node.get_child(-2))
num_children = with_node.num_children() - has_type_comment
......@@ -522,6 +538,12 @@ class ASTBuilder(object):
body, decorators, classdef_node)
def handle_funcdef_impl(self, funcdef_node, is_async, decorators=None, posnode=None):
self.check_feature(
is_async,
version=5,
msg="Async functions are only supported in Python 3.5 and greater",
n=funcdef_node
)
if posnode is None:
posnode = funcdef_node
name_node = funcdef_node.get_child(1)
......@@ -537,6 +559,12 @@ class ASTBuilder(object):
suite += has_type_comment
body, possible_type_comment = self.handle_typed_suite(funcdef_node.get_child(suite))
if not self.space.is_none(possible_type_comment):
if not self.space.is_none(type_comment):
raise SyntaxError(
"Cannot have two type comments on def",
funcdef_node.get_lineno(),
funcdef_node.get_column()
)
type_comment = possible_type_comment
assert len(body)
if is_async:
......@@ -840,10 +868,22 @@ class ASTBuilder(object):
value_expr = self.handle_expr(value_child)
op_str = stmt.get_child(1).get_child(0).get_value()
operator = augassign_operator_map[op_str]
self.check_feature(
operator is ast.MatMult,
version=5,
msg="The '@' operator is only supported in Python 3.5 and greater",
n=stmt
)
return build(ast.AugAssign, target_expr, operator, value_expr,
stmt)
elif stmt.get_child(1).type == syms.annassign:
# Variable annotation (PEP 526), which may or may not include assignment.
self.check_feature(
condition=True,
version=6,
msg="Variable annotation syntax is only supported in Python 3.6 and greater",
n=stmt
)
target = stmt.get_child(0)
target_expr = self.handle_testlist(target)
simple = 0
......@@ -1070,11 +1110,23 @@ class ASTBuilder(object):
left = self.handle_expr(binop_node.get_child(0))
right = self.handle_expr(binop_node.get_child(2))
op = operator_map(binop_node.get_child(1).type)
self.check_feature(
op is ast.MatMult,
version=5,
msg="The '@' operator is only supported in Python 3.5 and greater",
n=binop_node
)
result = build(ast.BinOp, left, op, right, binop_node)
number_of_ops = (binop_node.num_children() - 1) / 2
for i in range(1, number_of_ops):
op_node = binop_node.get_child(i * 2 + 1)
op = operator_map(op_node.type)
self.check_feature(
op is ast.MatMult,
version=5,
msg="The '@' operator is only supported in Python 3.5 and greater",
n=binop_node
)
right_node = binop_node.get_child(i * 2 + 2)
sub_right = self.handle_expr(right_node)
result = build(ast.BinOp, result, op, sub_right, op_node)
......@@ -1098,8 +1150,13 @@ class ASTBuilder(object):
def handle_atom_expr(self, atom_node):
start = 0
num_ch = atom_node.num_children()
if atom_node.get_child(0).type == tokens.NAME:
# await
if atom_node.get_child(0).type == tokens.AWAIT:
self.check_feature(
condition=True,
version=5,
msg="Await expressions are only supported in Python 3.5 and greater",
n=atom_node
)
start = 1
start_node = atom_node.get_child(start)
atom_expr = self.handle_atom(start_node)
......@@ -1367,6 +1424,12 @@ class ASTBuilder(object):
return fstring.string_parse_literal(self, atom_node)
#
elif first_child_type == tokens.NUMBER:
self.check_feature(
"_" in first_child.get_value(),
version=6,
msg="Underscores in numeric literals are only supported in Python 3.6 and greater",
n=atom_node
)
num_value = self.parse_number(first_child.get_value())
return build(ast.Constant, num_value, self.space.w_None, atom_node)
elif first_child_type == tokens.ELLIPSIS:
......@@ -1432,7 +1495,7 @@ class ASTBuilder(object):
current_for = comp_node
while True:
count += 1
is_async = current_for.get_child(0).type == tokens.NAME
is_async = current_for.get_child(0).type == tokens.ASYNC
current_for = current_for.get_child(int(is_async))
assert current_for.type == syms.sync_comp_for
if current_for.num_children() == 5:
......@@ -1468,7 +1531,13 @@ class ASTBuilder(object):
fors_count = self.count_comp_fors(comp_node)
comps = []
for i in range(fors_count):
is_async = comp_node.get_child(0).type == tokens.NAME
is_async = comp_node.get_child(0).type == tokens.ASYNC
self.check_feature(
is_async,
version=6,
msg="Async comprehensions are only supported in Python 3.6 and greater",
n=comp_node
)
comp_node = comp_node.get_child(int(is_async))
assert comp_node.type == syms.sync_comp_for
for_node = comp_node.get_child(1)
......
......@@ -34,6 +34,7 @@ PyCF_SOURCE_IS_UTF8 = 0x0100
PyCF_DONT_IMPLY_DEDENT = 0x0200
PyCF_ONLY_AST = 0x0400
PyCF_IGNORE_COOKIE = 0x0800
PyCF_ASYNC_HACKS = 0x1000
PyCF_ACCEPT_NULL_BYTES = 0x10000000 # PyPy only, for compile()
PyCF_FOUND_ENCODING = 0x20000000 # PyPy only, for pytokenizer
PyCF_TYPE_COMMENTS = 0x40000000
......
......@@ -223,6 +223,12 @@ def fstring_find_expr(astbuilder, fstr, atom_node, rec):
# Check for the equal sign (debugging expr)
if s[i] == '=':
astbuilder.check_feature(
condition=True,
version=8,
msg="f-string: self documenting expressions are only supported in Python 3.8 and greater",
n=atom_node
)
i += 1
if i >= len(s):
unexpected_end_of_string(astbuilder, atom_node)
......@@ -452,6 +458,12 @@ def string_parse_literal(astbuilder, atom_node):
if not fmode and len(joined_pieces) == 1: # <= the common path
return joined_pieces[0] # ast.Constant or FormattedValue
astbuilder.check_feature(
fmode,
version=6,
msg="Format strings are only supported in Python 3.6 and greater",
n=atom_node
)
# with more than one piece, it is a combination of ast.Constant[str]
# and FormattedValue pieces --- if there is a bytes value, then we got
# an invalid mixture of bytes and unicode literals
......
......@@ -17,11 +17,13 @@ class TestAstBuilder:
def setup_class(cls):
cls.parser = pyparse.PythonParser(cls.space)
def get_ast(self, source, p_mode=None, flags=None):
def get_ast(self, source, p_mode=None, flags=None, with_async_hacks=False):
if p_mode is None:
p_mode = "exec"
if flags is None:
flags = consts.CO_FUTURE_WITH_STATEMENT
if with_async_hacks:
flags |= consts.PyCF_ASYNC_HACKS
info = pyparse.CompileInfo("<test>", p_mode, flags)
tree = self.parser.parse_source(source, info)
ast_node = ast_from_node(self.space, tree, info, self.parser)
......@@ -1426,8 +1428,9 @@ class TestAstBuilder:
assert isinstance(expr.right, ast.Name)
# imatmul is tested earlier search for @=
def test_asyncFunctionDef(self):
mod = self.get_ast("async def f():\n await something()")
@pytest.mark.parametrize('with_async_hacks', [False, True])
def test_asyncFunctionDef(self, with_async_hacks):
mod = self.get_ast("async def f():\n await something()", with_async_hacks=with_async_hacks)
assert isinstance(mod, ast.Module)
assert len(mod.body) == 1
asyncdef = mod.body[0]
......@@ -1446,8 +1449,9 @@ class TestAstBuilder:
assert func.id == 'something'
assert func.ctx == ast.Load
def test_asyncFor(self):
mod = self.get_ast("async def f():\n async for e in i: 1\n else: 2")
@pytest.mark.parametrize('with_async_hacks', [False, True])
def test_asyncFor(self, with_async_hacks):
mod = self.get_ast("async def f():\n async for e in i: 1\n else: 2", with_async_hacks=with_async_hacks)
assert isinstance(mod, ast.Module)
assert len(mod.body) == 1
asyncdef = mod.body[0]
......@@ -1466,8 +1470,9 @@ class TestAstBuilder:
assert isinstance(asyncfor.orelse[0], ast.Expr)
assert isinstance(asyncfor.orelse[0].value, ast.Constant)
def test_asyncWith(self):
mod = self.get_ast("async def f():\n async with a as b: 1")
@pytest.mark.parametrize('with_async_hacks', [False, True])
def test_asyncWith(self, with_async_hacks):
mod = self.get_ast("async def f():\n async with a as b: 1", with_async_hacks=with_async_hacks)
assert isinstance(mod, ast.Module)
assert len(mod.body) == 1
asyncdef = mod.body[0]
......@@ -1486,8 +1491,9 @@ class TestAstBuilder:
assert isinstance(asyncwith.body[0], ast.Expr)
assert isinstance(asyncwith.body[0].value, ast.Constant)
def test_asyncYield(self):
mod = self.get_ast("async def f():\n yield 5")
@pytest.mark.parametrize('with_async_hacks', [False, True])
def test_asyncYield(self, with_async_hacks):
mod = self.get_ast("async def f():\n yield 5", with_async_hacks=with_async_hacks)
assert isinstance(mod, ast.Module)
assert len(mod.body) == 1
asyncdef = mod.body[0]
......@@ -1500,8 +1506,9 @@ class TestAstBuilder:
assert isinstance(expr.value, ast.Yield)
assert isinstance(expr.value.value, ast.Constant)
def test_asyncComp(self):
mod = self.get_ast("async def f():\n [i async for b in c]")
@pytest.mark.parametrize('with_async_hacks', [False, True])
def test_asyncComp(self, with_async_hacks):
mod = self.get_ast("async def f():\n [i async for b in c]", with_async_hacks=with_async_hacks)
asyncdef = mod.body[0]
expr = asyncdef.body[0]
comp = expr.value.generators[0]
......@@ -1509,6 +1516,28 @@ class TestAstBuilder:
assert comp.iter.id == 'c'
assert comp.is_async is True
def test_without_async_hacks(self):
with pytest.raises(SyntaxError):
self.get_ast("await = 1", with_async_hacks=False)
mod = self.get_ast("await x()", with_async_hacks=False)
assert isinstance(mod.body[0].value, ast.Await)
mod = self.get_ast("async for x in y: pass", with_async_hacks=False)
assert isinstance(mod.body[0], ast.AsyncFor)
def test_with_async_hacks(self):
mod = self.get_ast("await = 1", with_async_hacks=True)
assert isinstance(mod.body[0], ast.Assign)
assert isinstance(mod.body[0].targets[0], ast.Name)
assert mod.body[0].targets[0].id == "await"
with pytest.raises(SyntaxError):
self.get_ast("await x()", with_async_hacks=True)
with pytest.raises(SyntaxError):
self.get_ast("await x()", with_async_hacks=True)
def test_decode_error_in_string_literal(self):
input = "u'\\x'"
exc = pytest.raises(SyntaxError, self.get_ast, input).value
......@@ -1576,14 +1605,6 @@ class TestAstBuilder:
func = self.get_first_typed_stmt(source)
assert eq_w(func.type_comment, w("() -> int"))
source = textwrap.dedent("""\
def foo(): # type: () -> str
# type: () -> int
pass
""")
func = self.get_first_typed_stmt(source)
assert eq_w(func.type_comment, w("() -> int"))
def test_type_comments_statements(self):
eq_w, w = self.space.eq_w, self.space.wrap
asyncdef = textwrap.dedent("""\
......@@ -1767,6 +1788,31 @@ class TestAstBuilder:
for i, arg in enumerate(all_args, 1)
])
def test_double_type_comment(self):
with pytest.raises(SyntaxError) as excinfo:
tree = self.get_ast(textwrap.dedent("""\
def foo(): # type: () -> int
# type: () -> str
return test
"""), flags=consts.PyCF_TYPE_COMMENTS)
assert excinfo.value.msg.startswith("Cannot have two type comments on def")
def test_invalid_type_comments(self):
def check_both_ways(source):
self.get_ast(source) # this is fine, no type_comments
with pytest.raises(SyntaxError):
self.get_ast(source, flags=consts.PyCF_TYPE_COMMENTS)
check_both_ways("pass # type: int\n")
check_both_ways("foo() # type: int\n")
check_both_ways("x += 1 # type: int\n")
check_both_ways("while True: # type: int\n continue\n")
check_both_ways("while True:\n continue # type: int\n")
check_both_ways("try: # type: int\n pass\nfinally:\n pass\n")
check_both_ways("try:\n pass\nfinally: # type: int\n pass\n")
check_both_ways("pass # type: ignorewhatever\n")
check_both_ways("pass # type: ignoreé\n")
def test_walrus(self):
mod = self.get_ast("(a := 1)")
expr = mod.body[0].value
......
......@@ -152,12 +152,13 @@ class PythonAstCompiler(PyCodeCompiler):
raise OperationError(self.space.w_ValueError,
self.space.newtext(e.message))
def compile_to_ast(self, source, filename, mode, flags):
info = pyparse.CompileInfo(filename, mode, flags)
def compile_to_ast(self, source, filename, mode, flags, feature_version):
info = pyparse.CompileInfo(filename, mode, flags, feature_version=feature_version)
return self._compile_to_ast(source, info)
def _compile_to_ast(self, source, info):
space = self.space
self.parser.reset()
try:
parse_tree = self.parser.parse_source(source, info)
mod = astbuilder.ast_from_node(space, parse_tree, info,
......
......@@ -17,7 +17,7 @@ decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE
decorators: decorator+
decorated: decorators (classdef | funcdef | async_funcdef)
async_funcdef: 'async' funcdef
async_funcdef: ASYNC funcdef
funcdef: 'def' NAME parameters ['->' test] ':' [TYPE_COMMENT] func_body_suite
parameters: '(' [typedargslist] ')'
......@@ -111,7 +111,7 @@ nonlocal_stmt: 'nonlocal' NAME (',' NAME)*
assert_stmt: 'assert' test [',' test]
compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | with_stmt | funcdef | classdef | decorated | async_stmt
async_stmt: 'async' (funcdef | with_stmt | for_stmt)
async_stmt: ASYNC (funcdef | with_stmt | for_stmt)
if_stmt: 'if' namedexpr_test ':' suite ('elif' namedexpr_test ':' suite)* ['else' ':' suite]
while_stmt: 'while' namedexpr_test ':' suite ['else' ':' suite]
for_stmt: 'for' exprlist 'in' testlist ':' [TYPE_COMMENT] suite ['else' ':' suite]
......@@ -147,7 +147,7 @@ arith_expr: term (('+'|'-') term)*
term: factor (('*'|'@'|'/'|'%'|'//') factor)*
factor: ('+'|'-'|'~') factor | power
power: atom_expr ['**' factor]
atom_expr: ['await'] atom trailer*
atom_expr: [AWAIT] atom trailer*
atom: ('(' [yield_expr|testlist_comp] ')' |
'[' [testlist_comp] ']' |
'{' [dictorsetmaker] '}' |
......@@ -185,7 +185,7 @@ argument: ( test [comp_for] |
comp_iter: comp_for | comp_if
sync_comp_for: 'for' exprlist 'in' or_test [comp_iter]
comp_for: ['async'] sync_comp_for
comp_for: [ASYNC] sync_comp_for
comp_if: 'if' test_nocond [comp_iter]
# not used in grammar, but may appear in "node" passed from Parser to Compiler
......
......@@ -463,3 +463,7 @@ class Parser(object):
self.stack.node_append_child(node)
else:
self.root = node
def reset(self):
"""Reset the state-bound data"""
from pypy.interpreter.error import OperationError
from pypy.interpreter.pyparser import future, parser, pytokenizer, pygram, error
from pypy.interpreter.astcompiler import consts
from pypy.module.sys.version import CPYTHON_VERSION
from rpython.rlib import rstring
CPYTHON_MINOR_VERSION = CPYTHON_VERSION[1]
def recode_to_utf8(space, bytes, encoding):
if encoding == 'utf-8':
......@@ -75,8 +77,12 @@ class CompileInfo(object):
"""
def __init__(self, filename, mode="exec", flags=0, future_pos=(0, 0),
hidden_applevel=False, optimize=0):
hidden_applevel=False, optimize=0, feature_version=-1):
assert optimize >= 0
if feature_version == -1:
feature_version = CPYTHON_MINOR_VERSION
if feature_version < 7:
flags |= consts.PyCF_ASYNC_HACKS
rstring.check_str0(filename)
self.filename = filename
self.mode = mode
......@@ -85,6 +91,7 @@ class CompileInfo(object):
self.optimize = optimize
self.last_future_import = future_pos
self.hidden_applevel = hidden_applevel
self.feature_version = feature_version
_targets = {
......@@ -103,6 +110,9 @@ class PythonParser(parser.Parser):
self.future_flags = future_flags
self.type_ignores = []
def reset(self):
self.type_ignores = []
def parse_source(self, bytessrc, compile_info):
"""Main entry point for parsing Python source.
......
......@@ -57,9 +57,16 @@ def handle_type_comment(token, flags, lnum, start, line):
):
return None
# A TYPE_IGNORE is "type: ignore" followed by the end of the token
# or anything ASCII and non-alphanumeric. */
# Leading whitespace is ignored
type_decl = sub_tokens[1].lstrip()
if type_decl.startswith(TYPE_IGNORE):
following_char = type_decl[len(TYPE_IGNORE):]
if type_decl.startswith(TYPE_IGNORE) and (
following_char == '' or
ord(following_char[0]) < 0x80 and not following_char[0].isalnum()
):
tok_type = tokens.TYPE_IGNORE
type_decl = type_decl[len(TYPE_IGNORE):]
else:
......@@ -132,6 +139,10 @@ def generate_tokens(lines, flags):
altindents = [0]
last_comment = ''
parenstack = []
async_hacks = flags & consts.PyCF_ASYNC_HACKS
async_def = False
async_def_nl = False
async_def_indent = 0
# make the annotator happy
endDFA = DUMMY_DFA
......@@ -228,6 +239,10 @@ def generate_tokens(lines, flags):
raise TokenIndentationError(err, line, lnum, column+1, token_list)
if altcolumn != altindents[-1]:
raise TabError(lnum, pos, line)
if async_def_nl and async_def_indent >= indents[-1]:
async_def = False
async_def_nl = False
async_def_indent = 0
else: # continued statement
if not line:
......@@ -261,6 +276,8 @@ def generate_tokens(lines, flags):
last_comment = ''
elif initial in '\r\n':
if not parenstack:
if async_def:
async_def_nl = True
tok = Token(tokens.NEWLINE, last_comment, lnum, start, line)
token_list.append(tok)
......@@ -319,6 +336,31 @@ def generate_tokens(lines, flags):
# be used in identifiers
raise TokenError("invalid character in identifier",
line, lnum, start + 1, token_list)
# inside 'async def' function or no async_hacks
# so recognize them unconditionally.
if not async_hacks or async_def:
if token == 'async':
token_list.append(Token(tokens.ASYNC, token, lnum, start, line, lnum, end))
elif token == 'await':
token_list.append(Token(tokens.AWAIT, token, lnum, start, line, lnum, end))
else:
token_list.append(Token(tokens.NAME, token, lnum, start, line, lnum, end))
elif token == 'async': # async token, look ahead
#ahead token
if pos < max:
async_end = pseudoDFA.recognize(line, pos)
assert async_end >= 3
async_start = async_end - 3
assert async_start >= 0
ahead_token = line[async_start:async_end]
if ahead_token == 'def':
async_def = True
async_def_indent = indents[-1]
token_list.append(Token(tokens.ASYNC, token, lnum, start, line, lnum, end))
else:
token_list.append(Token(tokens.NAME, token, lnum, start, line, lnum, end))
else:
token_list.append(Token(tokens.NAME, token, lnum, start, line, lnum, end))
else:
token_list.append(Token(tokens.NAME, token, lnum, start, line, lnum, end))
last_comment = ''
......
......@@ -13,9 +13,9 @@ class TestPythonParser:
def setup_class(self):
self.parser = pyparse.PythonParser(self.space)
def parse(self, source, mode="exec", info=None):
def parse(self, source, mode="exec", info=None, flags=0):
if info is None:
info = pyparse.CompileInfo("<test>", mode)
info = pyparse.CompileInfo("<test>", mode, flags=flags)
return self.parser.parse_source(source, info)
def test_with_and_as(self):
......@@ -261,6 +261,32 @@ if 1:
async with a:
pass''')
def test_async_await_hacks(self):
def parse(source):
return self.parse(source, flags=consts.PyCF_ASYNC_HACKS)
# legal syntax
parse("async def coro(): await func")
# legal syntax for 3.6<=
parse("async = 1")
parse("await = 1")
parse("def async(): pass")
parse("def await(): pass")
parse("""async def foo():
async for a in b:
pass""")
# illegal syntax for 3.6<=
with pytest.raises(SyntaxError):
parse("await x")
with pytest.raises(SyntaxError):
parse("async for a in b: pass")
with pytest.raises(SyntaxError):
parse("def foo(): async for a in b: pass")
with pytest.raises(SyntaxError):
parse("def foo(): async for a in b: pass")
def test_number_underscores(self):
VALID_UNDERSCORE_LITERALS = [
'0_0_0',
......
......@@ -30,8 +30,10 @@ in addition to any features explicitly specified.
# only allow default value of _feature_version for now
# we need to support the keyword argument, the ast module passes it (set to
# -1, usually)
if _feature_version != -1:
raise oefmt(space.w_ValueError, "_feature_version only accepts -1 for now")
if _feature_version >= 0 and (flags & consts.PyCF_ONLY_AST):
feature_version = _feature_version
else:
feature_version = -1
ec = space.getexecutioncontext()
if flags & ~(ec.compiler.compiler_flags | consts.PyCF_ONLY_AST |
......@@ -76,7 +78,7 @@ in addition to any features explicitly specified.
"string, bytes or AST", flags)
if only_ast:
node = ec.compiler.compile_to_ast(source, filename, mode, flags)
node = ec.compiler.compile_to_ast(source, filename, mode, flags, feature_version)
return node.to_object(space)
else: