Commit 70f3d8b3 authored by Carl Friedrich Bolz-Tereick's avatar Carl Friedrich Bolz-Tereick
Browse files

#3832 merge fix-intutils-ovf-bug:

- fix several crashes and miscompilation bugs all related to integer
  optimizations
- add a simple z3 bounded model checker for integer traces, which checks that
  the optimized trace and the input trace behave identically
- this operates on all unit tests in test_optimizeintbound
 also generates some random tests
Pipeline #59202 passed with stage
in 8 minutes and 30 seconds
"""
This conftest adds options used by test/test_random and
x86/test/test_zll_random.
"""
import random
def pytest_addoption(parser):
group = parser.getgroup('random test options')
group.addoption('--random-seed', action="store", type=int,
default=random.randrange(0, 10000),
dest="randomseed",
help="choose a fixed random seed")
group.addoption('--backend', action="store",
default='llgraph',
choices=['llgraph', 'cpu'],
dest="backend",
help="select the backend to run the functions with")
group.addoption('--block-length', action="store", type=int,
default=30,
dest="block_length",
help="insert up to this many operations in each test")
group.addoption('--n-vars', action="store", type=int,
default=10,
dest="n_vars",
help="supply this many randomly-valued arguments to "
"the function")
group.addoption('--repeat', action="store", type=int,
default=15,
dest="repeat",
help="run the test this many times"),
group.addoption('--output', '-O', action="store", type=str,
default="", dest="output",
help="dump output to a file")
......@@ -61,6 +61,8 @@ def constbox(v):
class OperationBuilder(object):
produce_failing_guards = True
def __init__(self, cpu, loop, vars):
self.cpu = cpu
if not hasattr(cpu, '_faildescr_keepalive'):
......@@ -482,9 +484,16 @@ class CastFloatToIntOperation(AbstractFloatOperation):
class GuardOperation(AbstractOperation):
def gen_guard(self, builder, r):
v = builder.get_bool_var(r)
op = ResOperation(self.opnum, [v])
passing = ((self.opnum == rop.GUARD_TRUE and getint(v)) or
(self.opnum == rop.GUARD_FALSE and not getint(v)))
if not builder.produce_failing_guards:
if getint(v):
opnum = rop.GUARD_TRUE
else:
opnum = rop.GUARD_FALSE
else:
opnum = self.opnum
op = ResOperation(opnum, [v])
passing = ((opnum == rop.GUARD_TRUE and getint(v)) or
(opnum == rop.GUARD_FALSE and not getint(v)))
return op, passing
def produce_into(self, builder, r):
......@@ -493,6 +502,7 @@ class GuardOperation(AbstractOperation):
op.setdescr(builder.getfaildescr())
op.setfailargs(builder.subset_of_intvars(r))
if not passing:
assert builder.produce_failing_guards
builder.should_fail_by = op
builder.guard_op = op
......@@ -509,8 +519,10 @@ class GuardPtrOperation(GuardOperation):
class GuardValueOperation(GuardOperation):
def gen_guard(self, builder, r):
v = r.choice(builder.intvars)
if r.random() > 0.8:
other = r.choice(builder.intvars)
if not builder.produce_failing_guards:
other = ConstInt(getint(v))
elif r.random() > 0.8:
other = ConstInt(getint(r.choice(builder.intvars)))
else:
if r.random() < 0.75:
value = getint(v)
......@@ -615,9 +627,10 @@ def Random():
import random
seed = pytest.config.option.randomseed
print
print 'Random seed value is %d.' % (seed,)
print 'Random start seed value is %d.' % (seed,)
print
r = random.Random(seed)
r = random.Random()
r.seed(seed)
def get_random_integer():
while True:
result = int(r.expovariate(0.05))
......@@ -669,17 +682,19 @@ class RandomLoop(object):
self.output = output
if startvars is None:
startvars = []
# vary the number of startvars, between 1 and n_vars
n_vars = r.randrange(pytest.config.option.n_vars) + 1
if cpu.supports_floats:
# pick up a single threshold for the whole 'inputargs', so
# that some loops have no or mostly no FLOATs while others
# have a lot of them
k = r.random()
# but make sure there is at least one INT
at_least_once = r.randrange(0, pytest.config.option.n_vars)
at_least_once = r.randrange(n_vars)
else:
k = -1
at_least_once = 0
for i in range(pytest.config.option.n_vars):
for i in range(n_vars):
if r.random() < k and i != at_least_once:
startvars.append(InputArgFloat(r.random_float_storage()))
else:
......@@ -730,7 +745,8 @@ class RandomLoop(object):
loop._targettoken))
def generate_ops(self, builder, r, loop, startvars, needs_a_label=False):
block_length = pytest.config.option.block_length
block_length = pytest.config.option.block_length // 2
block_length += int(r.expovariate(1.0 / block_length))
istart = 0
for i in range(block_length):
......
"""
This conftest adds options used by test/test_random and
x86/test/test_zll_random.
"""
import random
def pytest_addoption(parser):
group = parser.getgroup("JIT options")
group.addoption('--slow', action="store_true",
default=False, dest="run_slow_tests",
help="run all the compiled tests (instead of just a few)")
group = parser.getgroup('random test options')
group.addoption('--random-seed', action="store", type=int,
default=random.randrange(0, 10000),
dest="randomseed",
help="choose a fixed random seed")
group.addoption('--backend', action="store",
default='llgraph',
choices=['llgraph', 'cpu'],
dest="backend",
help="select the backend to run the functions with")
group.addoption('--block-length', action="store", type=int,
default=30,
dest="block_length",
help="insert up to this many operations in each test")
group.addoption('--n-vars', action="store", type=int,
default=10,
dest="n_vars",
help="supply this many randomly-valued arguments to "
"the function")
group.addoption('--repeat', action="store", type=int,
default=15,
dest="repeat",
help="run the test this many times"),
group.addoption('--output', '-O', action="store", type=str,
default="", dest="output",
help="dump output to a file")
group.addoption('--z3-timeout', action="store", type=int,
default=500, dest="z3timeout",
help="give timeout that Z3 should use for proving in milliseconds per condition")
......@@ -87,8 +87,7 @@ class OptIntBounds(Optimization):
b1 = self.getintbound(op.getarg(0))
b2 = self.getintbound(op.getarg(1))
b = b1.sub_bound(b2)
if b.bounded():
self.getintbound(op).intersect(b)
self.getintbound(op).intersect(b)
def optimize_INT_ADD(self, op):
arg1 = get_box_replacement(op.getarg(0))
......@@ -124,7 +123,7 @@ class OptIntBounds(Optimization):
prod_arg1, prod_arg2 = prod_arg2, prod_arg1
prod_v1, prod_v2 = prod_v2, prod_v1
if prod_v2.is_constant():
sum = intmask(arg2.getint() + prod_arg2.getint())
sum = intmask(v2.get_constant_int() + prod_v2.get_constant_int())
arg1 = prod_arg1
arg2 = ConstInt(sum)
op = self.replace_op_with(op, rop.INT_ADD, args=[arg1, arg2])
......@@ -136,22 +135,14 @@ class OptIntBounds(Optimization):
b2 = self.getintbound(op.getarg(1))
r = self.getintbound(op)
b = b1.add_bound(b2)
# NB: the result only gets its bound updated if b has an upper and a
# lower bound. This is important, to do the right thing in the presence
# of overflow. Example:
# y = x + 1 where x >= 0
# here it's tempting to give a bound of y >= 1, but that would be
# wrong, due to wraparound
if b.bounded():
r.intersect(b)
r.intersect(b)
def postprocess_INT_MUL(self, op):
b1 = self.getintbound(op.getarg(0))
b2 = self.getintbound(op.getarg(1))
r = self.getintbound(op)
b = b1.mul_bound(b2)
if b.bounded():
r.intersect(b)
r.intersect(b)
def postprocess_CALL_PURE_I(self, op):
# dispatch based on 'oopspecindex' to a method that handles
......@@ -183,10 +174,7 @@ class OptIntBounds(Optimization):
r = self.getintbound(op)
b = b1.lshift_bound(b2)
r.intersect(b)
# intbound.lshift_bound checks for an overflow and if the
# lshift can be proven not to overflow sets b.has_upper and
# b.has_lower
if b.bounded():
if b1.lshift_bound_cannot_overflow(b2):
# Synthesize the reverse op for optimize_default to reuse
self.pure_from_args(rop.INT_RSHIFT,
[op, arg1], arg0)
......@@ -250,22 +238,25 @@ class OptIntBounds(Optimization):
def optimize_INT_ADD_OVF(self, op):
b1 = self.getintbound(op.getarg(0))
b2 = self.getintbound(op.getarg(1))
resbound = b1.add_bound(b2)
if resbound.bounded():
if b1.add_bound_cannot_overflow(b2):
# Transform into INT_ADD. The following guard will be killed
# by optimize_GUARD_NO_OVERFLOW; if we see instead an
# optimize_GUARD_OVERFLOW, then InvalidLoop.
# NB: this case also takes care of int_add_ovf with 0 as on of the
# arguments: the result will be bounded, and then the optimization
# for int_add with 0 as argument will remove the op.
# NB: this case also takes care of int_add_ovf with 0 as one of the
# arguments
op = self.replace_op_with(op, rop.INT_ADD)
return self.emit(op)
def postprocess_INT_ADD_OVF(self, op):
b1 = self.getintbound(op.getarg(0))
b2 = self.getintbound(op.getarg(1))
resbound = b1.add_bound(b2)
# we can always give the result a bound. if the int_add_ovf is followed
# by a guard_no_overflow, then we know no overflow occurred, and the
# bound is correct. Otherwise, it must be followed by a guard_overflow
# and it is also fine to give the result a bound, because the result
# box must never be used in the rest of the trace
resbound = b1.add_bound_no_overflow(b2)
r = self.getintbound(op)
r.intersect(resbound)
......@@ -277,8 +268,7 @@ class OptIntBounds(Optimization):
if arg0.same_box(arg1):
self.make_constant_int(op, 0)
return None
resbound = b0.sub_bound(b1)
if resbound.bounded():
if b0.sub_bound_cannot_overflow(b1):
# this case takes care of int_sub_ovf(x, 0) as well
op = self.replace_op_with(op, rop.INT_SUB)
return self.emit(op)
......@@ -288,15 +278,14 @@ class OptIntBounds(Optimization):
arg1 = get_box_replacement(op.getarg(1))
b0 = self.getintbound(arg0)
b1 = self.getintbound(arg1)
resbound = b0.sub_bound(b1)
resbound = b0.sub_bound_no_overflow(b1)
r = self.getintbound(op)
r.intersect(resbound)
def optimize_INT_MUL_OVF(self, op):
b1 = self.getintbound(op.getarg(0))
b2 = self.getintbound(op.getarg(1))
resbound = b1.mul_bound(b2)
if resbound.bounded():
b0 = self.getintbound(op.getarg(0))
b1 = self.getintbound(op.getarg(1))
if b0.mul_bound_cannot_overflow(b1):
# this case also takes care of multiplication with 0 and 1
op = self.replace_op_with(op, rop.INT_MUL)
return self.emit(op)
......@@ -304,7 +293,7 @@ class OptIntBounds(Optimization):
def postprocess_INT_MUL_OVF(self, op):
b1 = self.getintbound(op.getarg(0))
b2 = self.getintbound(op.getarg(1))
resbound = b1.mul_bound(b2)
resbound = b1.mul_bound_no_overflow(b2)
r = self.getintbound(op)
r.intersect(resbound)
......@@ -506,7 +495,7 @@ class OptIntBounds(Optimization):
v1 = self.getintbound(op)
v2 = getptrinfo(op.getarg(0))
intbound = self.getintbound(op.getarg(1))
if intbound.has_lower and v2 is not None:
if v2 is not None:
lenbound = v2.getlenbound(vstring.mode_string)
if lenbound is not None:
lenbound.make_gt_const(intbound.lower)
......@@ -544,7 +533,7 @@ class OptIntBounds(Optimization):
b1.make_ge_const(0)
v2 = getptrinfo(op.getarg(0))
intbound = self.getintbound(op.getarg(1))
if intbound.has_lower and v2 is not None:
if v2 is not None:
lenbound = v2.getlenbound(vstring.mode_unicode)
if lenbound is not None:
lenbound.make_gt_const(intbound.lower)
......@@ -675,22 +664,36 @@ class OptIntBounds(Optimization):
def propagate_bounds_INT_EQ(self, op):
r = self.getintbound(op)
if r.equal(1):
b1 = self.getintbound(op.getarg(0))
b2 = self.getintbound(op.getarg(1))
if b1.intersect(b2):
self.propagate_bounds_backward(op.getarg(0))
if b2.intersect(b1):
self.propagate_bounds_backward(op.getarg(1))
self.make_eq(op.getarg(0), op.getarg(1))
elif r.equal(0):
self.make_ne(op.getarg(0), op.getarg(1))
def propagate_bounds_INT_NE(self, op):
r = self.getintbound(op)
if r.equal(0):
b1 = self.getintbound(op.getarg(0))
b2 = self.getintbound(op.getarg(1))
if b1.intersect(b2):
self.propagate_bounds_backward(op.getarg(0))
if b2.intersect(b1):
self.propagate_bounds_backward(op.getarg(1))
self.make_eq(op.getarg(0), op.getarg(1))
elif r.equal(0):
self.make_ne(op.getarg(0), op.getarg(1))
def make_eq(self, arg0, arg1):
b0 = self.getintbound(arg0)
b1 = self.getintbound(arg1)
if b0.intersect(b1):
self.propagate_bounds_backward(arg0)
if b1.intersect(b0):
self.propagate_bounds_backward(arg1)
def make_ne(self, arg0, arg1):
b0 = self.getintbound(arg0)
b1 = self.getintbound(arg1)
if b1.is_constant():
v1 = b1.get_constant_int()
if b0.make_ne_const(v1):
self.propagate_bounds_backward(arg0)
elif b0.is_constant():
v0 = b0.get_constant_int()
if b1.make_ne_const(v0):
self.propagate_bounds_backward(arg1)
def _propagate_int_is_true_or_zero(self, op, valnonzero, valzero):
if self.is_raw_ptr(op.getarg(0)):
......@@ -732,13 +735,16 @@ class OptIntBounds(Optimization):
b = r.add_bound(b2)
if b1.intersect(b):
self.propagate_bounds_backward(op.getarg(0))
b = r.sub_bound(b1).mul(-1)
b = r.sub_bound(b1).neg_bound()
if b2.intersect(b):
self.propagate_bounds_backward(op.getarg(1))
def propagate_bounds_INT_MUL(self, op):
b1 = self.getintbound(op.getarg(0))
b2 = self.getintbound(op.getarg(1))
if op.opnum != rop.INT_MUL_OVF and not b1.mul_bound_cannot_overflow(b2):
# we can only do divide if the operation didn't overflow
return
r = self.getintbound(op)
b = r.py_div_bound(b2)
if b1.intersect(b):
......@@ -750,6 +756,8 @@ class OptIntBounds(Optimization):
def propagate_bounds_INT_LSHIFT(self, op):
b1 = self.getintbound(op.getarg(0))
b2 = self.getintbound(op.getarg(1))
if not b1.lshift_bound_cannot_overflow(b2):
return
r = self.getintbound(op)
b = r.rshift_bound(b2)
if b1.intersect(b):
......
......@@ -27,11 +27,9 @@ def next_pow2_m1(n):
class IntBound(AbstractInfo):
_attrs_ = ('has_upper', 'has_lower', 'upper', 'lower')
_attrs_ = ('upper', 'lower')
def __init__(self, lower, upper):
self.has_upper = True
self.has_lower = True
self.upper = upper
self.lower = lower
# check for unexpected overflows:
......@@ -41,21 +39,16 @@ class IntBound(AbstractInfo):
# Returns True if the bound was updated
def make_le(self, other):
if other.has_upper:
return self.make_le_const(other.upper)
return False
return self.make_le_const(other.upper)
def make_le_const(self, other):
if not self.has_upper or other < self.upper:
self.has_upper = True
if other < self.upper:
self.upper = other
return True
return False
def make_lt(self, other):
if other.has_upper:
return self.make_lt_const(other.upper)
return False
return self.make_lt_const(other.upper)
def make_lt_const(self, other):
try:
......@@ -65,13 +58,10 @@ class IntBound(AbstractInfo):
return self.make_le_const(other)
def make_ge(self, other):
if other.has_lower:
return self.make_ge_const(other.lower)
return False
return self.make_ge_const(other.lower)
def make_ge_const(self, other):
if not self.has_lower or other > self.lower:
self.has_lower = True
if other > self.lower:
self.lower = other
return True
return False
......@@ -84,18 +74,23 @@ class IntBound(AbstractInfo):
return self.make_ge_const(other)
def make_eq_const(self, intval):
self.has_upper = True
self.has_lower = True
self.upper = intval
self.lower = intval
def make_gt(self, other):
if other.has_lower:
return self.make_gt_const(other.lower)
def make_ne_const(self, intval):
if self.lower < intval == self.upper:
self.upper -= 1
return True
if self.lower == intval < self.upper:
self.lower += 1
return True
return False
def make_gt(self, other):
return self.make_gt_const(other.lower)
def is_constant(self):
return self.has_upper and self.has_lower and self.lower == self.upper
return self.lower == self.upper
def get_constant_int(self):
assert self.is_constant()
......@@ -106,38 +101,23 @@ class IntBound(AbstractInfo):
return False
return self.lower == value
def bounded(self):
return self.has_lower and self.has_upper
def known_lt_const(self, other):
if self.has_upper:
return self.upper < other
return False
return self.upper < other
def known_le_const(self, other):
if self.has_upper:
return self.upper <= other
return False
return self.upper <= other
def known_gt_const(self, other):
if self.has_lower:
return self.lower > other
return False
return self.lower > other
def known_ge_const(self, other):
if self.has_upper:
return self.upper >= other
return False
return self.lower >= other
def known_lt(self, other):
if other.has_lower:
return self.known_lt_const(other.lower)
return False
return self.known_lt_const(other.lower)
def known_le(self, other):
if other.has_lower:
return self.known_le_const(other.lower)
return False
return self.known_le_const(other.lower)
def known_gt(self, other):
return other.known_lt(self)
......@@ -146,16 +126,15 @@ class IntBound(AbstractInfo):
return other.known_le(self)
def known_nonnegative(self):
return self.has_lower and 0 <= self.lower
return 0 <= self.lower
def intersect(self, other):
assert not self.known_gt(other) and not self.known_lt(other)
r = False
if other.has_lower:
if self.make_ge_const(other.lower):
r = True
if other.has_upper:
if self.make_le_const(other.upper):
r = True
if self.make_ge_const(other.lower):
r = True
if self.make_le_const(other.upper):
r = True
return r
def intersect_const(self, lower, upper):
......@@ -164,75 +143,98 @@ class IntBound(AbstractInfo):
r = True
return r
def add(self, offset):
res = self.clone()
def add_bound(self, other):
""" add two bounds. must be correct even in the presence of possible
overflows. """
try:
res.lower = ovfcheck(res.lower + offset)
lower = ovfcheck(self.lower + other.lower)
except OverflowError:
res.has_lower = False
return IntUnbounded()
try:
res.upper = ovfcheck(res.upper + offset)
upper = ovfcheck(self.upper + other.upper)
except OverflowError:
res.has_upper = False
return res
return IntUnbounded()
return IntBound(lower, upper)
def mul(self, value):
return self.mul_bound(IntBound(value, value))
def add_bound_cannot_overflow(self, other):
""" returns True if self + other can never overflow """
try:
ovfcheck(self.upper + other.upper)
ovfcheck(self.lower + other.lower)
except OverflowError:
return False
return True
def add_bound(self, other):
res = self.clone()
if other.has_upper:
try:
res.upper = ovfcheck(res.upper + other.upper)
except OverflowError:
res.has_upper = False
else:
res.has_upper = False
if other.has_lower:
try:
res.lower = ovfcheck(res.lower + other.lower)
except OverflowError:
res.has_lower = False
else:
res.has_lower = False