Rust: Improvements (PrismJS#2332)

This makes several improvements to Rust. This includes support for multiline strings, better closures, attributes, macros, and comments, an update for the keyword list, and more.

components/prism-rust.js

@@ -1,68 +1,100 @@
-/* TODO
-	Add support for Markdown notation inside doc comments
-	Add support for nested block comments...
-	Match closure params even when not followed by dash or brace
-	Add better support for macro definition
-*/
+(function (Prism) {
 
-Prism.languages.rust = {
-	'comment': [
-		{
-			pattern: /(^|[^\\])\/\*[\s\S]*?\*\//,
-			lookbehind: true
-		},
-		{
-			pattern: /(^|[^\\:])\/\/.*/,
-			lookbehind: true
-		}
-	],
-	'string': [
-		{
-			pattern: /b?r(#*)"(?:\\.|(?!"\1)[^\\\r\n])*"\1/,
+	var multilineComment = /\/\*(?:[^*/]|\*(?!\/)|\/(?!\*)|<self>)*\*\//.source;
+	for (var i = 0; i < 2; i++) {
+		// support 4 levels of nested comments
+		multilineComment = multilineComment.replace(/<self>/g, function () { return multilineComment; });
+	}
+	multilineComment = multilineComment.replace(/<self>/g, function () { return /[^\s\S]/.source; });
+
+
+	Prism.languages.rust = {
+		'comment': [
+			{
+				pattern: RegExp(/(^|[^\\])/.source + multilineComment),
+				lookbehind: true,
+				greedy: true
+			},
+			{
+				pattern: /(^|[^\\:])\/\/.*/,
+				lookbehind: true,
+				greedy: true
+			}
+		],
+		'string': {
+			pattern: /b?"(?:\\[\s\S]|[^\\"])*"|b?r(#*)"(?:[^"]|"(?!\1))*"\1/,
 			greedy: true
 		},
-		{
-			pattern: /b?"(?:\\.|[^\\\r\n"])*"/,
-			greedy: true
-		}
-	],
-	'char': {
-		pattern: /b?'(?:\\(?:x[0-7][\da-fA-F]|u{(?:[\da-fA-F]_*){1,6}|.)|[^\\\r\n\t'])'/,
-		alias: 'string'
-	},
-	'lifetime-annotation': {
-		pattern: /'[^\s>']+/,
-		alias: 'symbol'
-	},
-	'keyword': /\b(?:abstract|alignof|as|async|await|be|box|break|const|continue|crate|do|dyn|else|enum|extern|false|final|fn|for|if|impl|in|let|loop|match|mod|move|mut|offsetof|once|override|priv|pub|pure|ref|return|sizeof|static|self|Self|struct|super|true|trait|type|typeof|union|unsafe|unsized|use|virtual|where|while|yield)\b/,
+		'char': {
+			pattern: /b?'(?:\\(?:x[0-7][\da-fA-F]|u{(?:[\da-fA-F]_*){1,6}|.)|[^\\\r\n\t'])'/,
+			greedy: true,
+			alias: 'string'
+		},
+		'attribute': {
+			pattern: /#!?\[[^[\]]*\]/,
+			greedy: true,
+			alias: 'attr-name',
+			inside: {
+				'string': null // see below
+			}
+		},
+
+		// Closure params should not be confused with bitwise OR |
+		'closure-params': {
+			pattern: /([=(,:]\s*|\bmove\s*)\|[^|]*\||\|[^|]*\|(?=\s*(?:\{|->))/,
+			lookbehind: true,
+			greedy: true,
+			inside: {
+				'closure-punctuation': {
+					pattern: /^\||\|$/,
+					alias: 'punctuation'
+				},
+				rest: null // see below
+			}
+		},
 
-	'attribute': {
-		pattern: /#!?\[.+?\]/,
-		greedy: true,
-		alias: 'attr-name'
-	},
+		'lifetime-annotation': {
+			pattern: /'\w+/,
+			alias: 'symbol'
+		},
+
+		'fragment-specifier': {
+			pattern: /(\$\w+:)[a-z]+/,
+			lookbehind: true,
+			alias: 'punctuation'
+		},
+		'variable': /\$\w+/,
+
+		'function-definition': {
+			pattern: /(\bfn\s*)\w+/,
+			lookbehind: true,
+			alias: 'function'
+		},
+		'keyword': [
+			// https://github.com/rust-lang/reference/blob/master/src/keywords.md
+			/\b(?:abstract|as|async|await|become|box|break|const|continue|crate|do|dyn|else|enum|extern|final|fn|for|if|impl|in|let|loop|macro|match|mod|move|mut|override|priv|pub|ref|return|self|Self|static|struct|super|trait|try|type|typeof|union|unsafe|unsized|use|virtual|where|while|yield)\b/,
+			// primitives
+			// https://doc.rust-lang.org/stable/rust-by-example/primitives.html
+			/\b(?:[ui](?:8|16|32|64|128|size)|f(?:32|64)|bool|char)\b/
+		],
+
+		// functions can technically start with an upper-case letter, but this will introduce a lot of false positives
+		// and Rust's naming conventions recommend snake_case anyway.
+		// https://doc.rust-lang.org/1.0.0/style/style/naming/README.html
+		'function': /\b[a-z_]\w*(?=\s*(?:::\s*<(?:[^<>]|<(?:[^<>]|<[^<>]*>)*>)*>\s*)?\()/,
+		'macro': {
+			pattern: /\w+!/,
+			alias: 'property'
+		},
 
-	'function': [
-		/\w+(?=\s*\()/,
-		// Macros can use parens or brackets
-		/\w+!(?=\s*\(|\[)/
-	],
-	'macro-rules': {
-		pattern: /\w+!/,
-		alias: 'function'
-	},
+		// Hex, oct, bin, dec numbers with visual separators and type suffix
+		'number': /\b(?:0x[\dA-Fa-f](?:_?[\dA-Fa-f])*|0o[0-7](?:_?[0-7])*|0b[01](?:_?[01])*|(?:\d(?:_?\d)*)?\.?\d(?:_?\d)*(?:[Ee][+-]?\d+)?)(?:_?(?:[iu](?:8|16|32|64|size)?|f32|f64))?\b/,
+		'boolean': /\b(?:false|true)\b/,
+		'punctuation': /->|\.\.=|\.{1,3}|::|[{}[\];(),:]/,
+		'operator': /[-+*\/%!^]=?|=[=>]?|&[&=]?|\|[|=]?|<<?=?|>>?=?|[@?]/
+	};
 
-	// Hex, oct, bin, dec numbers with visual separators and type suffix
-	'number': /\b(?:0x[\dA-Fa-f](?:_?[\dA-Fa-f])*|0o[0-7](?:_?[0-7])*|0b[01](?:_?[01])*|(?:\d(?:_?\d)*)?\.?\d(?:_?\d)*(?:[Ee][+-]?\d+)?)(?:_?(?:[iu](?:8|16|32|64)?|f32|f64))?\b/,
+	Prism.languages.rust['closure-params'].inside.rest = Prism.languages.rust;
+	Prism.languages.rust['attribute'].inside['string'] = Prism.languages.rust['string'];
 
-	// Closure params should not be confused with bitwise OR |
-	'closure-params': {
-		pattern: /\|[^|]*\|(?=\s*[{-])/,
-		inside: {
-			'punctuation': /[|:,]/,
-			'operator': /[&*]/
-		}
-	},
-	'punctuation': /->|\.\.=|\.{1,3}|::|[{}[\];(),:]/,
-	'operator': /[-+*\/%!^]=?|=[=>]?|&[&=]?|\|[|=]?|<<?=?|>>?=?|[@?]/
-};
+}(Prism));

tests/languages/rust/attribute_feature.test

@@ -1,13 +1,35 @@
 #[test]
 #![warn(unstable)]
+#[doc(hidden)]
+#[unstable(
+	feature = "thread_local_internals",
+	reason = "recently added to create a key",
+	issue = "none"
+)]
 
 ----------------------------------------------------
 
 [
-	["attribute", "#[test]"],
-	["attribute", "#![warn(unstable)]"]
+	["attribute", [
+		"#[test]"
+	]],
+	["attribute", [
+		"#![warn(unstable)]"
+	]],
+	["attribute", [
+		"#[doc(hidden)]"
+	]],
+	["attribute", [
+		"#[unstable(\r\n\tfeature = ",
+		["string", "\"thread_local_internals\""],
+		",\r\n\treason = ",
+		["string", "\"recently added to create a key\""],
+		",\r\n\tissue = ",
+		["string", "\"none\""],
+		"\r\n)]"
+	]]
 ]
 
 ----------------------------------------------------
 
-Checks for attributes.
+Checks for attributes.

tests/languages/rust/boolean_feature.test

@@ -0,0 +1,13 @@
+false
+true
+
+----------------------------------------------------
+
+[
+	["boolean", "false"],
+	["boolean", "true"]
+]
+
+----------------------------------------------------
+
+Checks for booleans.

tests/languages/rust/closure-params_feature.test

@@ -1,30 +1,134 @@
 |x: int, y: int| -> int {}
 || {}
 
+vec1.iter().any(|&x| x == 2);
+foo(123, || x * x);
+
+let add_one_v2 = |x: u32| -> u32 { x + 1 };
+let add_one_v3 = |x|             { x + 1 };
+let add_one_v4 = |x|               x + 1  ;
+move || println!("This is a: {}", text)
+
 ----------------------------------------------------
 
 [
 	["closure-params", [
-		["punctuation", "|"],
+		["closure-punctuation", "|"],
 		"x",
 		["punctuation", ":"],
 		" int",
 		["punctuation", ","],
 		" y",
 		["punctuation", ":"],
 		" int",
-		["punctuation", "|"]
+		["closure-punctuation", "|"]
+	]],
+	["punctuation", "->"],
+	" int ",
+	["punctuation", "{"],
+	["punctuation", "}"],
+
+	["closure-params", [
+		["closure-punctuation", "|"],
+		["closure-punctuation", "|"]
+	]],
+	["punctuation", "{"],
+	["punctuation", "}"],
+
+	"\r\n\r\nvec1",
+	["punctuation", "."],
+	["function", "iter"],
+	["punctuation", "("],
+	["punctuation", ")"],
+	["punctuation", "."],
+	["function", "any"],
+	["punctuation", "("],
+	["closure-params", [
+		["closure-punctuation", "|"],
+		["operator", "&"],
+		"x",
+		["closure-punctuation", "|"]
+	]],
+	" x ",
+	["operator", "=="],
+	["number", "2"],
+	["punctuation", ")"],
+	["punctuation", ";"],
+
+	["function", "foo"],
+	["punctuation", "("],
+	["number", "123"],
+	["punctuation", ","],
+	["closure-params", [
+		["closure-punctuation", "|"],
+		["closure-punctuation", "|"]
+	]],
+	" x ",
+	["operator", "*"],
+	" x",
+	["punctuation", ")"],
+	["punctuation", ";"],
+
+	["keyword", "let"],
+	" add_one_v2 ",
+	["operator", "="],
+	["closure-params", [
+		["closure-punctuation", "|"],
+		"x",
+		["punctuation", ":"],
+		["keyword", "u32"],
+		["closure-punctuation", "|"]
 	]],
 	["punctuation", "->"],
-	" int ", ["punctuation", "{"], ["punctuation", "}"],
+	["keyword", "u32"],
+	["punctuation", "{"],
+	" x ",
+	["operator", "+"],
+	["number", "1"],
+	["punctuation", "}"],
+	["punctuation", ";"],
+
+	["keyword", "let"],
+	" add_one_v3 ",
+	["operator", "="],
+	["closure-params", [
+		["closure-punctuation", "|"],
+		"x",
+		["closure-punctuation", "|"]
+	]],
+	["punctuation", "{"],
+	" x ",
+	["operator", "+"],
+	["number", "1"],
+	["punctuation", "}"],
+	["punctuation", ";"],
+
+	["keyword", "let"],
+	" add_one_v4 ",
+	["operator", "="],
+	["closure-params", [
+		["closure-punctuation", "|"],
+		"x",
+		["closure-punctuation", "|"]
+	]],
+	"               x ",
+	["operator", "+"],
+	["number", "1"],
+	["punctuation", ";"],
 
+	["keyword", "move"],
 	["closure-params", [
-		["punctuation", "|"],
-		["punctuation", "|"]
+		["closure-punctuation", "|"],
+		["closure-punctuation", "|"]
 	]],
-	["punctuation", "{"], ["punctuation", "}"]
+	["macro", "println!"],
+	["punctuation", "("],
+	["string", "\"This is a: {}\""],
+	["punctuation", ","],
+	" text",
+	["punctuation", ")"]
 ]
 
 ----------------------------------------------------
 
-Checks for closure params.
+Checks for closure params.

tests/languages/rust/comment_feature.test

@@ -4,15 +4,22 @@
 /* foo
 bar */
 
+/*   /* */  /** */  /*! */  */
+/*!  /* */  /** */  /*! */  */
+/**  /* */  /** */  /*! */  */
+
 ----------------------------------------------------
 
 [
 	["comment", "//"],
 	["comment", "// foobar"],
 	["comment", "/**/"],
-	["comment", "/* foo\r\nbar */"]
+	["comment", "/* foo\r\nbar */"],
+	["comment", "/*   /* */  /** */  /*! */  */"],
+	["comment", "/*!  /* */  /** */  /*! */  */"],
+	["comment", "/**  /* */  /** */  /*! */  */"]
 ]
 
 ----------------------------------------------------
 
-Checks for comments.
+Checks for comments.