From c5aafa1a8149e2fcc6f231acabe05c07ca9fa667 Mon Sep 17 00:00:00 2001 From: cocolato Date: Thu, 24 Oct 2024 21:51:06 +0800 Subject: [PATCH 1/3] Support passing custom filters with the same name as built-in flags --- mako/codegen.py | 31 +++++++++++++++++++++++++++---- mako/filters.py | 2 ++ mako/pyparser.py | 21 +++++++++++++++++++-- test/test_ast.py | 16 +++++++++++----- test/test_filters.py | 33 +++++++++++++++++++++++++++++++++ test/test_lexer.py | 2 +- 6 files changed, 93 insertions(+), 12 deletions(-) diff --git a/mako/codegen.py b/mako/codegen.py index b9fea93..ff94748 100644 --- a/mako/codegen.py +++ b/mako/codegen.py @@ -16,6 +16,7 @@ from mako import filters from mako import parsetree from mako import util +from mako.filters import CONFLICT_PREFIX from mako.pygen import PythonPrinter @@ -522,6 +523,8 @@ def write_variable_declares(self, identifiers, toplevel=False, limit=None): self.printer.writeline("loop = __M_loop = runtime.LoopStack()") for ident in to_write: + if ident.startswith(CONFLICT_PREFIX): + ident = ident.replace(CONFLICT_PREFIX, "") if ident in comp_idents: comp = comp_idents[ident] if comp.is_block: @@ -785,16 +788,36 @@ def locate_encode(name): else: return filters.DEFAULT_ESCAPES.get(name, name) - if "n" not in args: + filter_args = [] + conflict_n = "%sn" % CONFLICT_PREFIX + if conflict_n not in args: if is_expression: if self.compiler.pagetag: args = self.compiler.pagetag.filter_args.args + args - if self.compiler.default_filters and "n" not in args: + filter_args = self.compiler.pagetag.filter_args.args + if self.compiler.default_filters and conflict_n not in args: args = self.compiler.default_filters + args for e in args: - # if filter given as a function, get just the identifier portion - if e == "n": + if e == conflict_n: continue + if e.startswith(CONFLICT_PREFIX): + if e not in filter_args: + ident = e.replace(CONFLICT_PREFIX, "") + m = re.match(r"(.+?)(\(.*\))", e) + if m: + target = "%s(%s)" % (ident, target) + continue + target = "%s(%s) if %s is not UNDEFINED else %s(%s)" % ( + ident, + target, + ident, + locate_encode(ident), + target, + ) + continue + e = e.replace(CONFLICT_PREFIX, "") + + # if filter given as a function, get just the identifier portion m = re.match(r"(.+?)(\(.*\))", e) if m: ident, fargs = m.group(1, 2) diff --git a/mako/filters.py b/mako/filters.py index 2a4b438..47faa42 100644 --- a/mako/filters.py +++ b/mako/filters.py @@ -161,3 +161,5 @@ def htmlentityreplace_errors(ex): "str": "str", "n": "n", } + +CONFLICT_PREFIX = "__ALIAS_" diff --git a/mako/pyparser.py b/mako/pyparser.py index 714e004..8f91e72 100644 --- a/mako/pyparser.py +++ b/mako/pyparser.py @@ -18,6 +18,8 @@ from mako import compat from mako import exceptions from mako import util +from mako.filters import CONFLICT_PREFIX +from mako.filters import DEFAULT_ESCAPES # words that cannot be assigned to (notably # smaller than the total keys in __builtins__) @@ -196,9 +198,24 @@ def visit_Tuple(self, node): p.declared_identifiers ) lui = self.listener.undeclared_identifiers - self.listener.undeclared_identifiers = lui.union( - p.undeclared_identifiers + # self.listener.undeclared_identifiers = lui.union( + # p.undeclared_identifiers + # ) + undeclared_identifiers = lui.union(p.undeclared_identifiers) + conflict_identifiers = undeclared_identifiers.intersection( + DEFAULT_ESCAPES ) + if conflict_identifiers: + _map = {i: CONFLICT_PREFIX + i for i in conflict_identifiers} + # for k, v in _map.items(): + for i, arg in enumerate(self.listener.args): + if arg in _map: + self.listener.args[i] = _map[arg] + self.listener.undeclared_identifiers = ( + undeclared_identifiers ^ conflict_identifiers + ).union(_map.values()) + else: + self.listener.undeclared_identifiers = undeclared_identifiers class ParseFunc(_ast_util.NodeVisitor): diff --git a/test/test_ast.py b/test/test_ast.py index 84e2338..884c99d 100644 --- a/test/test_ast.py +++ b/test/test_ast.py @@ -285,16 +285,22 @@ def test_python_fragment(self): def test_argument_list(self): parsed = ast.ArgumentList( - "3, 5, 'hi', x+5, " "context.get('lala')", **exception_kwargs + "3, 5, 'hi', g+5, " "context.get('lala')", **exception_kwargs ) - eq_(parsed.undeclared_identifiers, {"x", "context"}) + eq_(parsed.undeclared_identifiers, {"g", "context"}) eq_( [x for x in parsed.args], - ["3", "5", "'hi'", "(x + 5)", "context.get('lala')"], + ["3", "5", "'hi'", "(g + 5)", "context.get('lala')"], ) - parsed = ast.ArgumentList("h", **exception_kwargs) - eq_(parsed.args, ["h"]) + parsed = ast.ArgumentList("m", **exception_kwargs) + eq_(parsed.args, ["m"]) + + def test_conflict_argument_list(self): + parsed = ast.ArgumentList( + "3, 5, 'hi', n+5, " "context.get('lala')", **exception_kwargs + ) + eq_(parsed.undeclared_identifiers, {"__ALIAS_n", "context"}) def test_function_decl(self): """test getting the arguments from a function""" diff --git a/test/test_filters.py b/test/test_filters.py index 726f5d7..2728361 100644 --- a/test/test_filters.py +++ b/test/test_filters.py @@ -453,3 +453,36 @@ def test_capture_ccall(self): # print t.render() assert flatten_result(t.render()) == "this is foo. body: ccall body" + + def test_conflict_filter_ident(self): + class h(object): + foo = str + + t = Template( + """ +X: + ${"asdf" | h.foo} +""" + ) + assert flatten_result(t.render(h=h)) == "X: asdf" + + def h(i): + return str(i) + "1" + + t = Template( + """ + ${123 | h} +""" + ) + assert flatten_result(t.render()) == "123" + assert flatten_result(t.render(h=h)) == "1231" + + t = Template( + """ + <%def name="foo()" filter="h"> + this is foo + ${foo()} +""" + ) + assert flatten_result(t.render()) == "this is foo" + assert flatten_result(t.render(h=h)) == "this is foo1" diff --git a/test/test_lexer.py b/test/test_lexer.py index 05aaa1f..c0a9a8e 100644 --- a/test/test_lexer.py +++ b/test/test_lexer.py @@ -1327,7 +1327,7 @@ def test_integration(self): Text(" \n", (14, 1)), ControlLine("for", "for x in j:", False, (15, 1)), Text(" Hello ", (16, 1)), - Expression("x", ["h"], (16, 23)), + Expression("x", ["__ALIAS_h"], (16, 23)), Text("\n", (16, 30)), ControlLine("for", "endfor", True, (17, 1)), Text(" \n", (18, 1)), From 8945ad27dfdbd3630c71adfd23f6f751210932a0 Mon Sep 17 00:00:00 2001 From: cocolato Date: Thu, 24 Oct 2024 22:07:39 +0800 Subject: [PATCH 2/3] add change log --- doc/build/unreleased/140.rst | 10 ++++++++++ mako/pyparser.py | 4 ---- 2 files changed, 10 insertions(+), 4 deletions(-) create mode 100644 doc/build/unreleased/140.rst diff --git a/doc/build/unreleased/140.rst b/doc/build/unreleased/140.rst new file mode 100644 index 0000000..d73b625 --- /dev/null +++ b/doc/build/unreleased/140.rst @@ -0,0 +1,10 @@ +.. change:: + :tags: bug, lexer, codegen + :tickets: 140 + + During the lexical analysis phase, add an additional + prefix for undeclared identifiers that have the same name + as built-in flags, and determine the final filter to be used + during the code generation phase based on the context + provided by the user. + Pull request by Hai Zhu. \ No newline at end of file diff --git a/mako/pyparser.py b/mako/pyparser.py index 8f91e72..3786c80 100644 --- a/mako/pyparser.py +++ b/mako/pyparser.py @@ -198,16 +198,12 @@ def visit_Tuple(self, node): p.declared_identifiers ) lui = self.listener.undeclared_identifiers - # self.listener.undeclared_identifiers = lui.union( - # p.undeclared_identifiers - # ) undeclared_identifiers = lui.union(p.undeclared_identifiers) conflict_identifiers = undeclared_identifiers.intersection( DEFAULT_ESCAPES ) if conflict_identifiers: _map = {i: CONFLICT_PREFIX + i for i in conflict_identifiers} - # for k, v in _map.items(): for i, arg in enumerate(self.listener.args): if arg in _map: self.listener.args[i] = _map[arg] From 5e64e05c9370f82528654dcacee0d162889ce94a Mon Sep 17 00:00:00 2001 From: cocolato Date: Sun, 27 Oct 2024 23:47:53 +0800 Subject: [PATCH 3/3] Revise the variable names and reduce the redundant code --- mako/codegen.py | 63 ++++++++++++++++++++++++---------------------- mako/filters.py | 3 ++- mako/pyparser.py | 12 ++++++--- test/test_ast.py | 18 +++++++++++-- test/test_lexer.py | 2 +- 5 files changed, 60 insertions(+), 38 deletions(-) diff --git a/mako/codegen.py b/mako/codegen.py index ff94748..c3804e6 100644 --- a/mako/codegen.py +++ b/mako/codegen.py @@ -16,7 +16,7 @@ from mako import filters from mako import parsetree from mako import util -from mako.filters import CONFLICT_PREFIX +from mako.filters import DEFAULT_ESCAPE_PREFIX from mako.pygen import PythonPrinter @@ -27,6 +27,7 @@ # context itself TOPLEVEL_DECLARED = {"UNDEFINED", "STOP_RENDERING"} RESERVED_NAMES = {"context", "loop"}.union(TOPLEVEL_DECLARED) +DEFAULT_ESCAPED_N = "%sn" % DEFAULT_ESCAPE_PREFIX def compile( # noqa @@ -523,8 +524,7 @@ def write_variable_declares(self, identifiers, toplevel=False, limit=None): self.printer.writeline("loop = __M_loop = runtime.LoopStack()") for ident in to_write: - if ident.startswith(CONFLICT_PREFIX): - ident = ident.replace(CONFLICT_PREFIX, "") + ident = ident.replace(DEFAULT_ESCAPE_PREFIX, "") if ident in comp_idents: comp = comp_idents[ident] if comp.is_block: @@ -788,45 +788,48 @@ def locate_encode(name): else: return filters.DEFAULT_ESCAPES.get(name, name) - filter_args = [] - conflict_n = "%sn" % CONFLICT_PREFIX - if conflict_n not in args: + filter_args = set() + if DEFAULT_ESCAPED_N not in args: if is_expression: if self.compiler.pagetag: args = self.compiler.pagetag.filter_args.args + args - filter_args = self.compiler.pagetag.filter_args.args - if self.compiler.default_filters and conflict_n not in args: + filter_args = set(self.compiler.pagetag.filter_args.args) + if ( + self.compiler.default_filters + and DEFAULT_ESCAPED_N not in args + ): args = self.compiler.default_filters + args for e in args: - if e == conflict_n: + if e == DEFAULT_ESCAPED_N: continue - if e.startswith(CONFLICT_PREFIX): - if e not in filter_args: - ident = e.replace(CONFLICT_PREFIX, "") - m = re.match(r"(.+?)(\(.*\))", e) - if m: - target = "%s(%s)" % (ident, target) - continue - target = "%s(%s) if %s is not UNDEFINED else %s(%s)" % ( - ident, - target, - ident, - locate_encode(ident), - target, - ) - continue - e = e.replace(CONFLICT_PREFIX, "") + + if e.startswith(DEFAULT_ESCAPE_PREFIX): + render_e = e.replace(DEFAULT_ESCAPE_PREFIX, "") + is_default_filter = True + else: + render_e = e + is_default_filter = False # if filter given as a function, get just the identifier portion m = re.match(r"(.+?)(\(.*\))", e) if m: - ident, fargs = m.group(1, 2) - f = locate_encode(ident) - e = f + fargs + if not is_default_filter: + ident, fargs = m.group(1, 2) + f = locate_encode(ident) + render_e = f + fargs + target = "%s(%s)" % (render_e, target) + elif is_default_filter and e not in filter_args: + target = "%s(%s) if %s is not UNDEFINED else %s(%s)" % ( + render_e, + target, + render_e, + locate_encode(render_e), + target, + ) else: - e = locate_encode(e) + e = locate_encode(render_e) assert e is not None - target = "%s(%s)" % (e, target) + target = "%s(%s)" % (e, target) return target def visitExpression(self, node): diff --git a/mako/filters.py b/mako/filters.py index 47faa42..33575e0 100644 --- a/mako/filters.py +++ b/mako/filters.py @@ -162,4 +162,5 @@ def htmlentityreplace_errors(ex): "n": "n", } -CONFLICT_PREFIX = "__ALIAS_" + +DEFAULT_ESCAPE_PREFIX = "__DEFAULT_ESCAPE_" diff --git a/mako/pyparser.py b/mako/pyparser.py index 3786c80..f5ed367 100644 --- a/mako/pyparser.py +++ b/mako/pyparser.py @@ -18,7 +18,7 @@ from mako import compat from mako import exceptions from mako import util -from mako.filters import CONFLICT_PREFIX +from mako.filters import DEFAULT_ESCAPE_PREFIX from mako.filters import DEFAULT_ESCAPES # words that cannot be assigned to (notably @@ -203,13 +203,17 @@ def visit_Tuple(self, node): DEFAULT_ESCAPES ) if conflict_identifiers: - _map = {i: CONFLICT_PREFIX + i for i in conflict_identifiers} + _map = { + i: DEFAULT_ESCAPE_PREFIX + i for i in conflict_identifiers + } for i, arg in enumerate(self.listener.args): if arg in _map: self.listener.args[i] = _map[arg] self.listener.undeclared_identifiers = ( - undeclared_identifiers ^ conflict_identifiers - ).union(_map.values()) + undeclared_identifiers.symmetric_difference( + conflict_identifiers + ).union(_map.values()) + ) else: self.listener.undeclared_identifiers = undeclared_identifiers diff --git a/test/test_ast.py b/test/test_ast.py index 884c99d..079c411 100644 --- a/test/test_ast.py +++ b/test/test_ast.py @@ -298,9 +298,23 @@ def test_argument_list(self): def test_conflict_argument_list(self): parsed = ast.ArgumentList( - "3, 5, 'hi', n+5, " "context.get('lala')", **exception_kwargs + "x-2, h*2, '(u)', n+5, trim, entity, unicode, decode, str, other", + **exception_kwargs, + ) + eq_( + parsed.undeclared_identifiers, + { + "__DEFAULT_ESCAPE_trim", + "__DEFAULT_ESCAPE_h", + "__DEFAULT_ESCAPE_decode", + "__DEFAULT_ESCAPE_unicode", + "__DEFAULT_ESCAPE_x", + "__DEFAULT_ESCAPE_str", + "__DEFAULT_ESCAPE_entity", + "__DEFAULT_ESCAPE_n", + "other", + }, ) - eq_(parsed.undeclared_identifiers, {"__ALIAS_n", "context"}) def test_function_decl(self): """test getting the arguments from a function""" diff --git a/test/test_lexer.py b/test/test_lexer.py index c0a9a8e..28ba3cc 100644 --- a/test/test_lexer.py +++ b/test/test_lexer.py @@ -1327,7 +1327,7 @@ def test_integration(self): Text(" \n", (14, 1)), ControlLine("for", "for x in j:", False, (15, 1)), Text(" Hello ", (16, 1)), - Expression("x", ["__ALIAS_h"], (16, 23)), + Expression("x", ["__DEFAULT_ESCAPE_h"], (16, 23)), Text("\n", (16, 30)), ControlLine("for", "endfor", True, (17, 1)), Text(" \n", (18, 1)),