From aff1f9cbb1197ead4e88b5adc49eea2795d42710 Mon Sep 17 00:00:00 2001 From: Matthieu Lempereur Date: Tue, 30 Nov 2021 23:28:28 +0100 Subject: [PATCH 1/5] enh(twig): update keywords list --- CHANGES.md | 1 + src/languages/twig.js | 12 ++++++------ 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index a0b1160598..2ea6d81802 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -2,6 +2,7 @@ Grammars: +- enh(twig) update keywords list - fix(python) def, class keywords detected mid-identifier (#3381) [Josh Goebel][] - fix(python) Fix recognition of numeric literals followed by keywords without whitespace (#2985) [Richard Gibson][] - enh(swift) add SE-0290 unavailability condition (#3382) [Bradley Mackey][] diff --git a/src/languages/twig.js b/src/languages/twig.js index 4ea7bd041a..294ee34732 100644 --- a/src/languages/twig.js +++ b/src/languages/twig.js @@ -13,7 +13,7 @@ export default function(hljs) { begin: '\\(', end: '\\)' }; - var FUNCTION_NAMES = 'attribute block constant cycle date dump include ' + + var FUNCTION_NAMES = 'attribute block constant country_timezones cycle date dump html_classes include ' + 'max min parent random range source template_from_string'; var FUNCTIONS = { @@ -28,16 +28,16 @@ export default function(hljs) { var FILTER = { begin: /\|[A-Za-z_]+:?/, keywords: - 'abs batch capitalize column convert_encoding date date_modify default ' + - 'escape filter first format inky_to_html inline_css join json_encode keys last ' + - 'length lower map markdown merge nl2br number_format raw reduce replace ' + - 'reverse round slice sort spaceless split striptags title trim upper url_encode', + 'abs batch capitalize column convert_encoding country_name currency_name currency_symbol data_uri date date_modify default ' + + 'escape filter first format format_currency format_date format_datetime format_number format_time html_to_markdown inky_to_html inline_css join json_encode keys language_name last ' + + 'length locale_name lower map markdown markdown_to_html merge nl2br number_format raw reduce replace ' + + 'reverse round slice slug sort spaceless split striptags timezone_name title u|0 trim upper url_encode', contains: [ FUNCTIONS ] }; - var TAGS = 'apply autoescape block deprecated do embed extends filter flush for from ' + + var TAGS = 'apply autoescape block cache deprecated do embed extends filter flush for from ' + 'if import include macro sandbox set use verbatim with'; TAGS = TAGS + ' ' + TAGS.split(' ').map(function(t){return 'end' + t}).join(' '); From d1a5253086a3a9483bf6bed05ce857742b7a84d0 Mon Sep 17 00:00:00 2001 From: MrYamous Date: Wed, 1 Dec 2021 10:34:19 +0100 Subject: [PATCH 2/5] add credits in changes.md --- CHANGES.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 2ea6d81802..2d05be881b 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -2,7 +2,7 @@ Grammars: -- enh(twig) update keywords list +- enh(twig) update keywords list (#3415) [Matthieu Lempereur][] - fix(python) def, class keywords detected mid-identifier (#3381) [Josh Goebel][] - fix(python) Fix recognition of numeric literals followed by keywords without whitespace (#2985) [Richard Gibson][] - enh(swift) add SE-0290 unavailability condition (#3382) [Bradley Mackey][] @@ -32,6 +32,7 @@ Themes: [Björn Ebbinghaus]: https://github.com/MrEbbinghaus [Josh Goebel]: https://github.com/joshgoebel [Samia Ali]: https://github.com/samiaab1990 +[Matthieu Lempereur]: https://github.com/MrYamous ## Version 11.3.1 From c7304b3279e9216d15a7c72eb4e89e82ce2f5371 Mon Sep 17 00:00:00 2001 From: Josh Goebel Date: Wed, 1 Dec 2021 13:16:36 -0500 Subject: [PATCH 3/5] chore(twig) major twig refatoring, add new functionality --- src/languages/twig.js | 216 ++++++++++++++---- .../clojure/globals_definition.expect.txt | 2 +- .../markup/clojure/symbols-numbers.expect.txt | 2 +- test/markup/fsharp/bang-keywords.expect.txt | 2 +- .../fsharp/computation-expressions.expect.txt | 2 +- test/markup/fsharp/types.expect.txt | 2 +- .../twig/filter_with_underscore.expect.txt | 2 +- test/markup/twig/template_tags.expect.txt | 12 +- 8 files changed, 189 insertions(+), 51 deletions(-) diff --git a/src/languages/twig.js b/src/languages/twig.js index 294ee34732..93acc1944c 100644 --- a/src/languages/twig.js +++ b/src/languages/twig.js @@ -8,67 +8,205 @@ Category: template */ export default function(hljs) { - var PARAMS = { - className: 'params', - begin: '\\(', end: '\\)' + const regex = hljs.regex; + const FUNCTION_NAMES = [ + "attribute", + "block", + "constant", + "country_timezones", + "cycle", + "date", + "dump", + "html_classes", + "include", + "max", + "min", + "parent", + "random", + "range", + "source", + "template_from_string" + ]; + + const FILTERS = [ + "abs", + "batch", + "capitalize", + "column", + "convert_encoding", + "country_name", + "currency_name", + "currency_symbol", + "data_uri", + "date", + "date_modify", + "default", + "escape", + "filter", + "first", + "format", + "format_currency", + "format_date", + "format_datetime", + "format_number", + "format_time", + "html_to_markdown", + "inky_to_html", + "inline_css", + "join", + "json_encode", + "keys", + "language_name", + "last", + "length", + "locale_name", + "lower", + "map", + "markdown", + "markdown_to_html", + "merge", + "nl2br", + "number_format", + "raw", + "reduce", + "replace", + "reverse", + "round", + "slice", + "slug", + "sort", + "spaceless", + "split", + "striptags", + "timezone_name", + "title", + "u|0", + "trim", + "upper", + "url_encode" + ]; + + let TAG_NAMES = [ + "apply", + "autoescape", + "block", + "cache", + "deprecated", + "do", + "embed", + "extends", + "filter", + "flush", + "for", + "from", + "if", + "import", + "include", + "macro", + "sandbox", + "set", + "use", + "verbatim", + "with" + ]; + + TAG_NAMES = TAG_NAMES.concat(TAG_NAMES.map(t => `end${t}`)); + + const STRING = { + scope: 'string', + variants: [ + { + begin: /'/, + end: /'/ + }, + { + begin: /"/, + end: /"/ + }, + ] }; - var FUNCTION_NAMES = 'attribute block constant country_timezones cycle date dump html_classes include ' + - 'max min parent random range source template_from_string'; + const NUMBER = { + scope: "number", + match: /\d+/ + }; - var FUNCTIONS = { - beginKeywords: FUNCTION_NAMES, - keywords: {name: FUNCTION_NAMES}, - relevance: 0, + const PARAMS = { + begin: /\(/, + end: /\)/, + excludeBegin: true, + excludeEnd: true, contains: [ - PARAMS + STRING, + NUMBER ] }; - var FILTER = { - begin: /\|[A-Za-z_]+:?/, - keywords: - 'abs batch capitalize column convert_encoding country_name currency_name currency_symbol data_uri date date_modify default ' + - 'escape filter first format format_currency format_date format_datetime format_number format_time html_to_markdown inky_to_html inline_css join json_encode keys language_name last ' + - 'length locale_name lower map markdown markdown_to_html merge nl2br number_format raw reduce replace ' + - 'reverse round slice slug sort spaceless split striptags timezone_name title u|0 trim upper url_encode', + + const FUNCTIONS = { + beginKeywords: FUNCTION_NAMES.join(" "), + keywords: { name: FUNCTION_NAMES }, + relevance: 0, + contains: [ PARAMS ] + }; + + const FILTER = { + match: /\|(?=[A-Za-z_]+:?)/, + beginScope: "punctuation", contains: [ - FUNCTIONS + { + match: /[A-Za-z_]+:?/, + keywords: FILTERS + }, ] }; - var TAGS = 'apply autoescape block cache deprecated do embed extends filter flush for from ' + - 'if import include macro sandbox set use verbatim with'; + const tagNamed = (tagnames) => { + return { + beginScope: { + 1: 'template-tag', + 3: 'name' + }, + endScope: 'template-tag', + begin: [ + /\{%/, + /\s*/, + regex.either(...tagnames) + ], + end: /%\}/, + keywords: "in", + contains: [ + FILTER, + FUNCTIONS, + STRING, + NUMBER + ] + }; + }; - TAGS = TAGS + ' ' + TAGS.split(' ').map(function(t){return 'end' + t}).join(' '); + const CUSTOM_TAG_RE = /[a-z_]+/; + const ALL_TAGS = [].concat(TAG_NAMES).concat(CUSTOM_TAG_RE); + const TAG = tagNamed(ALL_TAGS); return { name: 'Twig', - aliases: ['craftcms'], + aliases: [ 'craftcms' ], case_insensitive: true, subLanguage: 'xml', contains: [ hljs.COMMENT(/\{#/, /#\}/), + TAG, { - className: 'template-tag', - begin: /\{%/, end: /%\}/, + className: 'template-variable', + begin: /\{\{/, + end: /\}\}/, contains: [ - { - className: 'name', - begin: /\w+/, - keywords: TAGS, - starts: { - endsWithParent: true, - contains: [FILTER, FUNCTIONS], - relevance: 0 - } - } + 'self', + FILTER, + FUNCTIONS, + STRING, + NUMBER ] - }, - { - className: 'template-variable', - begin: /\{\{/, end: /\}\}/, - contains: ['self', FILTER, FUNCTIONS] } ] }; diff --git a/test/markup/clojure/globals_definition.expect.txt b/test/markup/clojure/globals_definition.expect.txt index 832e283cd4..efc8a15c9a 100644 --- a/test/markup/clojure/globals_definition.expect.txt +++ b/test/markup/clojure/globals_definition.expect.txt @@ -69,4 +69,4 @@ string" ;; create a couple shapes and get their area (def myCircle (Circle. 10)) -(def mySquare (Square. 5 11)) \ No newline at end of file +(def mySquare (Square. 5 11)) diff --git a/test/markup/clojure/symbols-numbers.expect.txt b/test/markup/clojure/symbols-numbers.expect.txt index 7813d22876..2808223dcc 100644 --- a/test/markup/clojure/symbols-numbers.expect.txt +++ b/test/markup/clojure/symbols-numbers.expect.txt @@ -1,4 +1,4 @@ (def +x [(a 1) +2 -3.0 y-5]) (System/getProperty "java.vm.version") (.getEnclosingClass java.util.Map$Entry) -(java.util.Map$Entry. .getEnclosingClass) +(java.util.Map$Entry. .getEnclosingClass) \ No newline at end of file diff --git a/test/markup/fsharp/bang-keywords.expect.txt b/test/markup/fsharp/bang-keywords.expect.txt index c47d939e94..f681697fdf 100644 --- a/test/markup/fsharp/bang-keywords.expect.txt +++ b/test/markup/fsharp/bang-keywords.expect.txt @@ -1 +1 @@ -let! (result2 : byte[]) = stream.AsyncRead(bufferSize) \ No newline at end of file +let! (result2 : byte[]) = stream.AsyncRead(bufferSize) diff --git a/test/markup/fsharp/computation-expressions.expect.txt b/test/markup/fsharp/computation-expressions.expect.txt index 76cb4486bd..af296f3f2c 100644 --- a/test/markup/fsharp/computation-expressions.expect.txt +++ b/test/markup/fsharp/computation-expressions.expect.txt @@ -20,4 +20,4 @@ return result } -let result = work |> Async.RunSynchronously \ No newline at end of file +let result = work |> Async.RunSynchronously diff --git a/test/markup/fsharp/types.expect.txt b/test/markup/fsharp/types.expect.txt index 0ad8b0c6a7..e48881af06 100644 --- a/test/markup/fsharp/types.expect.txt +++ b/test/markup/fsharp/types.expect.txt @@ -140,4 +140,4 @@ | CaseK of ``var with spaces``: string | CaseL of ``var with spaces``: ``type with spaces`` | CaseM of v1 : ``type with spaces`` - | CaseN of ``type with spaces`` \ No newline at end of file + | CaseN of ``type with spaces`` diff --git a/test/markup/twig/filter_with_underscore.expect.txt b/test/markup/twig/filter_with_underscore.expect.txt index 72c93fd877..9d2928531a 100644 --- a/test/markup/twig/filter_with_underscore.expect.txt +++ b/test/markup/twig/filter_with_underscore.expect.txt @@ -1 +1 @@ -{{ "string with spaces"|url_encode }} \ No newline at end of file +{{ "string with spaces"|url_encode }} \ No newline at end of file diff --git a/test/markup/twig/template_tags.expect.txt b/test/markup/twig/template_tags.expect.txt index 410fb24347..0127a33510 100644 --- a/test/markup/twig/template_tags.expect.txt +++ b/test/markup/twig/template_tags.expect.txt @@ -1,12 +1,12 @@ -{% if posts|length %} - {% for article in articles %} +{% if posts|length %} + {% for article in articles %} <div> - {{ article.title|upper() }} + {{ article.title|upper() }} {# outputs 'WELCOME' #} </div> - {% endfor %} -{% endif %} + {% endfor %} +{% endif %} -{% set user = json_encode(user) %} +{% set user = json_encode(user) %} \ No newline at end of file From f257a7c71fa07fd4e05b13f743c83c3bcd8d1e46 Mon Sep 17 00:00:00 2001 From: Josh Goebel Date: Wed, 1 Dec 2021 13:32:45 -0500 Subject: [PATCH 4/5] fixup --- src/languages/twig.js | 9 ++++++--- tools/checkAutoDetect.js | 2 +- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/src/languages/twig.js b/src/languages/twig.js index 93acc1944c..6da7491894 100644 --- a/src/languages/twig.js +++ b/src/languages/twig.js @@ -153,6 +153,7 @@ export default function(hljs) { const FILTER = { match: /\|(?=[A-Za-z_]+:?)/, beginScope: "punctuation", + relevance: 0, contains: [ { match: /[A-Za-z_]+:?/, @@ -161,12 +162,13 @@ export default function(hljs) { ] }; - const tagNamed = (tagnames) => { + const tagNamed = (tagnames, {relevance}) => { return { beginScope: { 1: 'template-tag', 3: 'name' }, + relevance: relevance || 2, endScope: 'template-tag', begin: [ /\{%/, @@ -185,8 +187,8 @@ export default function(hljs) { }; const CUSTOM_TAG_RE = /[a-z_]+/; - const ALL_TAGS = [].concat(TAG_NAMES).concat(CUSTOM_TAG_RE); - const TAG = tagNamed(ALL_TAGS); + const TAG = tagNamed(TAG_NAMES, { relevance: 2 }); + const CUSTOM_TAG = tagNamed([ CUSTOM_TAG_RE ], { relevance: 1 }); return { name: 'Twig', @@ -196,6 +198,7 @@ export default function(hljs) { contains: [ hljs.COMMENT(/\{#/, /#\}/), TAG, + CUSTOM_TAG, { className: 'template-variable', begin: /\{\{/, diff --git a/tools/checkAutoDetect.js b/tools/checkAutoDetect.js index 0a725985a8..0da66c8dfd 100755 --- a/tools/checkAutoDetect.js +++ b/tools/checkAutoDetect.js @@ -2,7 +2,7 @@ 'use strict'; const fs = require('fs'); -const hljs = require('../build.js'); +const hljs = require('../build/lib/index.js'); const path = require('path'); const utility = require('../test/utility.js'); const Table = require('cli-table'); From 5408af882fdf794edde97fce5f7b688f35ce32c2 Mon Sep 17 00:00:00 2001 From: MrYamous Date: Tue, 7 Dec 2021 21:01:18 +0100 Subject: [PATCH 5/5] order in twig keywords list --- src/languages/twig.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/languages/twig.js b/src/languages/twig.js index 6da7491894..3e4a26257d 100644 --- a/src/languages/twig.js +++ b/src/languages/twig.js @@ -80,8 +80,8 @@ export default function(hljs) { "striptags", "timezone_name", "title", - "u|0", "trim", + "u|0", "upper", "url_encode" ];