diff --git a/script.module.simpleeval/addon.xml b/script.module.simpleeval/addon.xml index 5eae274798..f8c25c379b 100644 --- a/script.module.simpleeval/addon.xml +++ b/script.module.simpleeval/addon.xml @@ -1,5 +1,5 @@ - + diff --git a/script.module.simpleeval/lib/simpleeval.py b/script.module.simpleeval/lib/simpleeval.py index 6de93f4cd4..f6a3ed6ef5 100644 --- a/script.module.simpleeval/lib/simpleeval.py +++ b/script.module.simpleeval/lib/simpleeval.py @@ -1,5 +1,5 @@ """ -SimpleEval - (C) 2013-2019 Daniel Fairhead +SimpleEval - (C) 2013-2023 Daniel Fairhead ------------------------------------- An short, easy to use, safe and reasonably extensible expression evaluator. @@ -49,8 +49,14 @@ - mommothazaz123 (Andrew Zhu) f"string" support, Python 3.8 support - lubieowoce (Uryga) various potential vulnerabilities - JCavallo (Jean Cavallo) names dict shouldn't be modified -- Birne94 (Daniel Birnstiel) for fixing leaking generators. +- Birne94 (Daniel Birnstiel) for fixing leaking generators, star expressions - patricksurry (Patrick Surry) or should return last value, even if falsy. +- shughes-uk (Samantha Hughes) python w/o 'site' should not fail to import. +- KOLANICH packaging / deployment / setup help & << + >> & other bit ops +- graingert (Thomas Grainger) packaging / deployment / setup help +- bozokopic (Bozo Kopic) Memory leak fix +- daxamin (Dax Amin) Better error for attempting to eval empty string +- smurfix (Matthias Urlichs) Allow clearing functions / operators / etc completely ------------------------------------- Basic Usage: @@ -93,9 +99,11 @@ import ast import operator as op import sys +import warnings from random import random PYTHON3 = sys.version_info[0] == 3 +PYTHON35 = PYTHON3 and sys.version_info > (3, 5) ######################################## # Module wide 'globals' @@ -103,8 +111,10 @@ MAX_STRING_LENGTH = 100000 MAX_COMPREHENSION_LENGTH = 10000 MAX_POWER = 4000000 # highest exponent -DISALLOW_PREFIXES = ['_', 'func_'] -DISALLOW_METHODS = ['format', 'format_map', 'mro'] +MAX_SHIFT = 10000 # highest << or >> (lshift / rshift) +MAX_SHIFT_BASE = int(sys.float_info.max) # highest on left side of << or >> +DISALLOW_PREFIXES = ["_", "func_"] +DISALLOW_METHODS = ["format", "format_map", "mro"] # Disallow functions: # This, strictly speaking, is not necessary. These /should/ never be accessable anyway, @@ -112,11 +122,18 @@ # people not be stupid. Allowing these functions opens up all sorts of holes - if any of # their functionality is required, then please wrap them up in a safe container. And think # very hard about it first. And don't say I didn't warn you. +# builtins is a dict in python >3.6 but a module before +DISALLOW_FUNCTIONS = {type, isinstance, eval, getattr, setattr, repr, compile, open} +if hasattr(__builtins__, "help") or ( + hasattr(__builtins__, "__contains__") and "help" in __builtins__ # type: ignore +): + # PyInstaller environment doesn't include this module. + DISALLOW_FUNCTIONS.add(help) -DISALLOW_FUNCTIONS = {type, isinstance, eval, getattr, setattr, help, repr, compile, open} if PYTHON3: - exec('DISALLOW_FUNCTIONS.add(exec)') # exec is not a function in Python2... + # exec is not a function in Python2... + exec("DISALLOW_FUNCTIONS.add(exec)") # pylint: disable=exec-used ######################################## @@ -124,34 +141,32 @@ class InvalidExpression(Exception): - """ Generic Exception """ + """Generic Exception""" pass class FunctionNotDefined(InvalidExpression): - """ sorry! That function isn't defined! """ + """sorry! That function isn't defined!""" def __init__(self, func_name, expression): - self.message = "Function '{0}' not defined," \ - " for expression '{1}'.".format(func_name, expression) - setattr(self, 'func_name', func_name) # bypass 2to3 confusion. + self.message = "Function '{0}' not defined," " for expression '{1}'.".format( + func_name, expression + ) + setattr(self, "func_name", func_name) # bypass 2to3 confusion. self.expression = expression - # pylint: disable=bad-super-call super(InvalidExpression, self).__init__(self.message) class NameNotDefined(InvalidExpression): - """ a name isn't defined. """ + """a name isn't defined.""" def __init__(self, name, expression): self.name = name - self.message = "'{0}' is not defined for expression '{1}'".format( - name, expression) + self.message = "'{0}' is not defined for expression '{1}'".format(name, expression) self.expression = expression - # pylint: disable=bad-super-call super(InvalidExpression, self).__init__(self.message) @@ -159,28 +174,53 @@ class AttributeDoesNotExist(InvalidExpression): """attribute does not exist""" def __init__(self, attr, expression): - self.message = \ - "Attribute '{0}' does not exist in expression '{1}'".format( - attr, expression) + self.message = "Attribute '{0}' does not exist in expression '{1}'".format( + attr, expression + ) + self.attr = attr + self.expression = expression + + super(InvalidExpression, self).__init__(self.message) + + +class OperatorNotDefined(InvalidExpression): + """operator does not exist""" + + def __init__(self, attr, expression): + self.message = "Operator '{0}' does not exist in expression '{1}'".format(attr, expression) self.attr = attr self.expression = expression + super(InvalidExpression, self).__init__(self.message) + class FeatureNotAvailable(InvalidExpression): - """ What you're trying to do is not allowed. """ + """What you're trying to do is not allowed.""" pass class NumberTooHigh(InvalidExpression): - """ Sorry! That number is too high. I don't want to spend the - next 10 years evaluating this expression! """ + """Sorry! That number is too high. I don't want to spend the + next 10 years evaluating this expression!""" pass class IterableTooLong(InvalidExpression): - """ That iterable is **way** too long, baby. """ + """That iterable is **way** too long, baby.""" + + pass + + +class AssignmentAttempted(UserWarning): + """Assignment not allowed in SimpleEval""" + + pass + + +class MultipleExpressions(UserWarning): + """Only the first expression parsed will be used""" pass @@ -190,61 +230,95 @@ class IterableTooLong(InvalidExpression): def random_int(top): - """ return a random int below """ + """return a random int below """ return int(random() * top) def safe_power(a, b): # pylint: disable=invalid-name - """ a limited exponent/to-the-power-of function, for safety reasons """ + """a limited exponent/to-the-power-of function, for safety reasons""" if abs(a) > MAX_POWER or abs(b) > MAX_POWER: - raise NumberTooHigh("Sorry! I don't want to evaluate {0} ** {1}" - .format(a, b)) - return a ** b + raise NumberTooHigh("Sorry! I don't want to evaluate {0} ** {1}".format(a, b)) + return a**b def safe_mult(a, b): # pylint: disable=invalid-name - """ limit the number of times an iterable can be repeated... """ + """limit the number of times an iterable can be repeated...""" - if hasattr(a, '__len__') and b * len(a) > MAX_STRING_LENGTH: - raise IterableTooLong('Sorry, I will not evalute something that long.') - if hasattr(b, '__len__') and a * len(b) > MAX_STRING_LENGTH: - raise IterableTooLong('Sorry, I will not evalute something that long.') + if hasattr(a, "__len__") and b * len(a) > MAX_STRING_LENGTH: + raise IterableTooLong("Sorry, I will not evalute something that long.") + if hasattr(b, "__len__") and a * len(b) > MAX_STRING_LENGTH: + raise IterableTooLong("Sorry, I will not evalute something that long.") return a * b def safe_add(a, b): # pylint: disable=invalid-name - """ iterable length limit again """ + """iterable length limit again""" - if hasattr(a, '__len__') and hasattr(b, '__len__'): + if hasattr(a, "__len__") and hasattr(b, "__len__"): if len(a) + len(b) > MAX_STRING_LENGTH: - raise IterableTooLong("Sorry, adding those two together would" - " make something too long.") + raise IterableTooLong( + "Sorry, adding those two together would" " make something too long." + ) return a + b +def safe_rshift(a, b): # pylint: disable=invalid-name + """rshift, but with input limits""" + if abs(b) > MAX_SHIFT or abs(a) > MAX_SHIFT_BASE: + raise NumberTooHigh("Sorry! I don't want to evaluate {0} >> {1}".format(a, b)) + return a >> b + + +def safe_lshift(a, b): # pylint: disable=invalid-name + """lshift, but with input limits""" + if abs(b) > MAX_SHIFT or abs(a) > MAX_SHIFT_BASE: + raise NumberTooHigh("Sorry! I don't want to evaluate {0} << {1}".format(a, b)) + return a << b + + ######################################## # Defaults for the evaluator: -DEFAULT_OPERATORS = {ast.Add: safe_add, ast.Sub: op.sub, ast.Mult: safe_mult, - ast.Div: op.truediv, ast.FloorDiv: op.floordiv, - ast.Pow: safe_power, ast.Mod: op.mod, - ast.Eq: op.eq, ast.NotEq: op.ne, - ast.Gt: op.gt, ast.Lt: op.lt, - ast.GtE: op.ge, ast.LtE: op.le, - ast.Not: op.not_, - ast.USub: op.neg, ast.UAdd: op.pos, - ast.In: lambda x, y: op.contains(y, x), - ast.NotIn: lambda x, y: not op.contains(y, x), - ast.Is: lambda x, y: x is y, - ast.IsNot: lambda x, y: x is not y, - } - -DEFAULT_FUNCTIONS = {"rand": random, "randint": random_int, - "int": int, "float": float, - "str": str if PYTHON3 else unicode} +DEFAULT_OPERATORS = { + ast.Add: safe_add, + ast.Sub: op.sub, + ast.Mult: safe_mult, + ast.Div: op.truediv, + ast.FloorDiv: op.floordiv, + ast.RShift: safe_rshift, + ast.LShift: safe_lshift, + ast.Pow: safe_power, + ast.Mod: op.mod, + ast.Eq: op.eq, + ast.NotEq: op.ne, + ast.Gt: op.gt, + ast.Lt: op.lt, + ast.GtE: op.ge, + ast.LtE: op.le, + ast.Not: op.not_, + ast.USub: op.neg, + ast.UAdd: op.pos, + ast.BitXor: op.xor, + ast.BitOr: op.or_, + ast.BitAnd: op.and_, + ast.Invert: op.invert, + ast.In: lambda x, y: op.contains(y, x), + ast.NotIn: lambda x, y: not op.contains(y, x), + ast.Is: lambda x, y: x is y, + ast.IsNot: lambda x, y: x is not y, +} + +DEFAULT_FUNCTIONS = { + "rand": random, + "randint": random_int, + "int": int, + "float": float, + # pylint: disable=undefined-variable + "str": str if PYTHON3 else unicode, # type: ignore +} DEFAULT_NAMES = {"True": True, "False": False, "None": None} @@ -256,23 +330,24 @@ def safe_add(a, b): # pylint: disable=invalid-name class SimpleEval(object): # pylint: disable=too-few-public-methods - """ A very simple expression parser. - >>> s = SimpleEval() - >>> s.eval("20 + 30 - ( 10 * 5)") - 0 - """ + """A very simple expression parser. + >>> s = SimpleEval() + >>> s.eval("20 + 30 - ( 10 * 5)") + 0 + """ + expr = "" def __init__(self, operators=None, functions=None, names=None): """ - Create the evaluator instance. Set up valid operators (+,-, etc) - functions (add, random, get_val, whatever) and names. """ + Create the evaluator instance. Set up valid operators (+,-, etc) + functions (add, random, get_val, whatever) and names.""" - if not operators: + if operators is None: operators = DEFAULT_OPERATORS.copy() - if not functions: + if functions is None: functions = DEFAULT_FUNCTIONS.copy() - if not names: + if names is None: names = DEFAULT_NAMES.copy() self.operators = operators @@ -280,6 +355,10 @@ def __init__(self, operators=None, functions=None, names=None): self.names = names self.nodes = { + ast.Expr: self._eval_expr, + ast.Assign: self._eval_assign, + ast.AugAssign: self._eval_aug_assign, + ast.Import: self._eval_import, ast.Num: self._eval_num, ast.Str: self._eval_str, ast.Name: self._eval_name, @@ -297,16 +376,18 @@ def __init__(self, operators=None, functions=None, names=None): } # py3k stuff: - if hasattr(ast, 'NameConstant'): + if hasattr(ast, "NameConstant"): self.nodes[ast.NameConstant] = self._eval_constant # py3.6, f-strings - if hasattr(ast, 'JoinedStr'): + if hasattr(ast, "JoinedStr"): self.nodes[ast.JoinedStr] = self._eval_joinedstr # f-string - self.nodes[ast.FormattedValue] = self._eval_formattedvalue # formatted value in f-string + self.nodes[ + ast.FormattedValue + ] = self._eval_formattedvalue # formatted value in f-string # py3.8 uses ast.Constant instead of ast.Num, ast.Str, ast.NameConstant - if hasattr(ast, 'Constant'): + if hasattr(ast, "Constant"): self.nodes[ast.Constant] = self._eval_constant # Defaults: @@ -317,31 +398,66 @@ def __init__(self, operators=None, functions=None, names=None): for f in self.functions.values(): if f in DISALLOW_FUNCTIONS: - raise FeatureNotAvailable('This function {} is a really bad idea.'.format(f)) + raise FeatureNotAvailable("This function {} is a really bad idea.".format(f)) + + def __del__(self): + self.nodes = None + @staticmethod + def parse(expr): + """parse an expression into a node tree""" - def eval(self, expr): - """ evaluate an expresssion, using the operators, functions and - names previously set up. """ + parsed = ast.parse(expr.strip()) - # set a copy of the expression aside, so we can give nice errors... + if not parsed.body: + raise InvalidExpression("Sorry, cannot evaluate empty string") + if len(parsed.body) > 1: + warnings.warn( + "'{}' contains multiple expressions. Only the first will be used.".format(expr), + MultipleExpressions, + ) + return parsed.body[0] + def eval(self, expr, previously_parsed=None): + """evaluate an expresssion, using the operators, functions and + names previously set up.""" + + # set a copy of the expression aside, so we can give nice errors... self.expr = expr - # and evaluate: - return self._eval(ast.parse(expr.strip()).body[0].value) + return self._eval(previously_parsed or self.parse(expr)) def _eval(self, node): - """ The internal evaluator used on each node in the parsed tree. """ + """The internal evaluator used on each node in the parsed tree.""" try: handler = self.nodes[type(node)] except KeyError: - raise FeatureNotAvailable("Sorry, {0} is not available in this " - "evaluator".format(type(node).__name__)) + raise FeatureNotAvailable( + "Sorry, {0} is not available in this " "evaluator".format(type(node).__name__) + ) return handler(node) + def _eval_expr(self, node): + return self._eval(node.value) + + def _eval_assign(self, node): + warnings.warn( + "Assignment ({}) attempted, but this is ignored".format(self.expr), AssignmentAttempted + ) + return self._eval(node.value) + + def _eval_aug_assign(self, node): + warnings.warn( + "Assignment ({}) attempted, but this is ignored".format(self.expr), AssignmentAttempted + ) + return self._eval(node.value) + + @staticmethod + def _eval_import(node): + raise FeatureNotAvailable("Sorry, 'import' is not allowed.") + @staticmethod def _eval_num(node): return node.n @@ -349,39 +465,48 @@ def _eval_num(node): @staticmethod def _eval_str(node): if len(node.s) > MAX_STRING_LENGTH: - raise IterableTooLong("String Literal in statement is too long!" - " ({0}, when {1} is max)".format( - len(node.s), MAX_STRING_LENGTH)) + raise IterableTooLong( + "String Literal in statement is too long!" + " ({0}, when {1} is max)".format(len(node.s), MAX_STRING_LENGTH) + ) return node.s @staticmethod def _eval_constant(node): - if hasattr(node.value, '__len__') and len(node.value) > MAX_STRING_LENGTH: - raise IterableTooLong("Literal in statement is too long!" - " ({0}, when {1} is max)".format(len(node.value), MAX_STRING_LENGTH)) + if hasattr(node.value, "__len__") and len(node.value) > MAX_STRING_LENGTH: + raise IterableTooLong( + "Literal in statement is too long!" + " ({0}, when {1} is max)".format(len(node.value), MAX_STRING_LENGTH) + ) return node.value def _eval_unaryop(self, node): - return self.operators[type(node.op)](self._eval(node.operand)) + try: + operator = self.operators[type(node.op)] + except KeyError: + raise OperatorNotDefined(node.op, self.expr) + return operator(self._eval(node.operand)) def _eval_binop(self, node): - return self.operators[type(node.op)](self._eval(node.left), - self._eval(node.right)) + try: + operator = self.operators[type(node.op)] + except KeyError: + raise OperatorNotDefined(node.op, self.expr) + return operator(self._eval(node.left), self._eval(node.right)) def _eval_boolop(self, node): + to_return = False if isinstance(node.op, ast.And): - vout = False for value in node.values: - vout = self._eval(value) - if not vout: - return vout - return vout + to_return = self._eval(value) + if not to_return: + break elif isinstance(node.op, ast.Or): for value in node.values: - vout = self._eval(value) - if vout: - return vout - return vout + to_return = self._eval(value) + if to_return: + break + return to_return def _eval_compare(self, node): right = self._eval(node.left) @@ -395,8 +520,7 @@ def _eval_compare(self, node): return to_return def _eval_ifexp(self, node): - return self._eval(node.body) if self._eval(node.test) \ - else self._eval(node.orelse) + return self._eval(node.body) if self._eval(node.test) else self._eval(node.orelse) def _eval_call(self, node): if isinstance(node.func, ast.Attribute): @@ -406,15 +530,14 @@ def _eval_call(self, node): func = self.functions[node.func.id] except KeyError: raise FunctionNotDefined(node.func.id, self.expr) - except AttributeError as e: - raise FeatureNotAvailable('Lambda Functions not implemented') + except AttributeError: + raise FeatureNotAvailable("Lambda Functions not implemented") if func in DISALLOW_FUNCTIONS: - raise FeatureNotAvailable('This function is forbidden') + raise FeatureNotAvailable("This function is forbidden") return func( - *(self._eval(a) for a in node.args), - **dict(self._eval(k) for k in node.keywords) + *(self._eval(a) for a in node.args), **dict(self._eval(k) for k in node.keywords) ) def _eval_keyword(self, node): @@ -427,14 +550,15 @@ def _eval_name(self, node): # that there is a true exression assigning to none # (the compiler rejects it, so you can't even # pass that to ast.parse) - if hasattr(self.names, '__getitem__'): + if hasattr(self.names, "__getitem__"): return self.names[node.id] - elif callable(self.names): + if callable(self.names): return self.names(node) - else: - raise InvalidExpression('Trying to use name (variable) "{0}"' - ' when no "names" defined for' - ' evaluator'.format(node.id)) + raise InvalidExpression( + 'Trying to use name (variable) "{0}"' + ' when no "names" defined for' + " evaluator".format(node.id) + ) except KeyError: if node.id in self.functions: @@ -445,10 +569,9 @@ def _eval_name(self, node): def _eval_subscript(self, node): container = self._eval(node.value) key = self._eval(node.slice) - try: - return container[key] - except KeyError: - raise + # Currently if there's a KeyError, that gets raised straight up. + # TODO: Should that be wrapped in an InvalidExpression? + return container[key] def _eval_attribute(self, node): for prefix in DISALLOW_PREFIXES: @@ -456,11 +579,12 @@ def _eval_attribute(self, node): raise FeatureNotAvailable( "Sorry, access to __attributes " " or func_ attributes is not available. " - "({0})".format(node.attr)) + "({0})".format(node.attr) + ) if node.attr in DISALLOW_METHODS: raise FeatureNotAvailable( - "Sorry, this method is not available. " - "({0})".format(node.attr)) + "Sorry, this method is not available. " "({0})".format(node.attr) + ) # eval node node_evaluated = self._eval(node.value) @@ -501,7 +625,7 @@ def _eval_joinedstr(self, node): if len(val) + length > MAX_STRING_LENGTH: raise IterableTooLong("Sorry, I will not evaluate something this long.") evaluated_values.append(val) - return ''.join(evaluated_values) + return "".join(evaluated_values) def _eval_formattedvalue(self, node): if node.format_spec: @@ -512,41 +636,58 @@ def _eval_formattedvalue(self, node): class EvalWithCompoundTypes(SimpleEval): """ - SimpleEval with additional Compound Types, and their respective - function editions. (list, tuple, dict, set). + SimpleEval with additional Compound Types, and their respective + function editions. (list, tuple, dict, set). """ + _max_count = 0 + def __init__(self, operators=None, functions=None, names=None): super(EvalWithCompoundTypes, self).__init__(operators, functions, names) - self.functions.update( - list=list, - tuple=tuple, - dict=dict, - set=set) - - self.nodes.update({ - ast.Dict: self._eval_dict, - ast.Tuple: self._eval_tuple, - ast.List: self._eval_list, - ast.Set: self._eval_set, - ast.ListComp: self._eval_comprehension, - ast.GeneratorExp: self._eval_comprehension, - }) - - def eval(self, expr): + self.functions.update(list=list, tuple=tuple, dict=dict, set=set) + + self.nodes.update( + { + ast.Dict: self._eval_dict, + ast.Tuple: self._eval_tuple, + ast.List: self._eval_list, + ast.Set: self._eval_set, + ast.ListComp: self._eval_comprehension, + ast.GeneratorExp: self._eval_comprehension, + } + ) + + def eval(self, expr, previously_parsed=None): + # reset _max_count for each eval run self._max_count = 0 - return super(EvalWithCompoundTypes, self).eval(expr) + return super(EvalWithCompoundTypes, self).eval(expr, previously_parsed) def _eval_dict(self, node): - return {self._eval(k): self._eval(v) - for (k, v) in zip(node.keys, node.values)} + result = {} - def _eval_tuple(self, node): - return tuple(self._eval(x) for x in node.elts) + for key, value in zip(node.keys, node.values): + if PYTHON35 and key is None: + # "{**x}" gets parsed as a key-value pair of (None, Name(x)) + result.update(self._eval(value)) + else: + result[self._eval(key)] = self._eval(value) + + return result def _eval_list(self, node): - return list(self._eval(x) for x in node.elts) + result = [] + + for item in node.elts: + if PYTHON3 and isinstance(item, ast.Starred): + result.extend(self._eval(item.value)) + else: + result.append(self._eval(item)) + + return result + + def _eval_tuple(self, node): + return tuple(self._eval(x) for x in node.elts) def _eval_set(self, node): return set(self._eval(x) for x in node.elts) @@ -560,7 +701,7 @@ def _eval_comprehension(self, node): def eval_names_extra(node): """ - Here we hide our extra scope for within this comprehension + Here we hide our extra scope for within this comprehension """ if node.id in extra_names: return extra_names[node.id] @@ -585,11 +726,11 @@ def do_generator(gi=0): self._max_count += 1 if self._max_count > MAX_COMPREHENSION_LENGTH: - raise IterableTooLong('Comprehension generates too many elements') + raise IterableTooLong("Comprehension generates too many elements") recurse_targets(g.target, i) if all(self._eval(iff) for iff in g.ifs): if len(node.generators) > gi + 1: - do_generator(gi+1) + do_generator(gi + 1) else: to_return.append(self._eval(node.elt)) @@ -602,8 +743,6 @@ def do_generator(gi=0): def simple_eval(expr, operators=None, functions=None, names=None): - """ Simply evaluate an expresssion """ - s = SimpleEval(operators=operators, - functions=functions, - names=names) + """Simply evaluate an expresssion""" + s = SimpleEval(operators=operators, functions=functions, names=names) return s.eval(expr)