Drop dedent-on-enter for "return" statements. (#6939)

(for #6813)

We'll work on adding support back in #6564.

news/2 Fixes/6813.md

@@ -0,0 +1,2 @@
+Drop dedent-on-enter for "return" statements.  It will be addressed in:
+  https://github.com/microsoft/vscode-python/issues/6564

src/client/common/utils/regexp.ts

@@ -0,0 +1,22 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+'use strict';
+
+/* Generate a RegExp from a "verbose" pattern.
+ *
+ * All whitespace in the pattern is removed, including newlines.  This
+ * allows the pattern to be much more readable by allowing it to span
+ * multiple lines and to separate tokens with insignificant whitespace.
+ * The functionality is similar to the VERBOSE ("x") flag in Python's
+ * regular expressions.
+ *
+ * Note that significant whitespace in the pattern must be explicitly
+ * indicated by "\s".  Also, unlike with regular expression literals,
+ * backslashes must be escaped.  Conversely, forward slashes do not
+ * need to be escaped.
+ */
+export function verboseRegExp(pattern: string): RegExp {
+    pattern = pattern.replace(/\s+?/g, '');
+    return RegExp(pattern);
+}

src/client/extension.ts

@@ -40,7 +40,7 @@ import { registerTypes as appRegisterTypes } from './application/serviceRegistry
 import { IApplicationDiagnostics } from './application/types';
 import { DebugService } from './common/application/debugService';
 import { IApplicationShell, ICommandManager, IWorkspaceService } from './common/application/types';
-import { Commands, isTestExecution, PYTHON, STANDARD_OUTPUT_CHANNEL } from './common/constants';
+import { Commands, isTestExecution, PYTHON, PYTHON_LANGUAGE, STANDARD_OUTPUT_CHANNEL } from './common/constants';
 import { registerTypes as registerDotNetTypes } from './common/dotnet/serviceRegistry';
 import { registerTypes as installerRegisterTypes } from './common/installer/serviceRegistry';
 import { traceError } from './common/logger';
@@ -85,7 +85,7 @@ import { registerTypes as interpretersRegisterTypes } from './interpreter/servic
 import { ServiceContainer } from './ioc/container';
 import { ServiceManager } from './ioc/serviceManager';
 import { IServiceContainer, IServiceManager } from './ioc/types';
-import { setLanguageConfiguration } from './language/languageConfiguration';
+import { getLanguageConfiguration } from './language/languageConfiguration';
 import { LinterCommands } from './linters/linterCommands';
 import { registerTypes as lintersRegisterTypes } from './linters/serviceRegistry';
 import { ILintingEngine } from './linters/types';
@@ -176,7 +176,7 @@ async function activateUnsafe(context: ExtensionContext): Promise<IExtensionApi>
     const linterProvider = new LinterProvider(context, serviceManager);
     context.subscriptions.push(linterProvider);
 
-    setLanguageConfiguration();
+    languages.setLanguageConfiguration(PYTHON_LANGUAGE, getLanguageConfiguration());
 
     if (pythonSettings && pythonSettings.formatting && pythonSettings.formatting.provider !== 'internalConsole') {
         const formatProvider = new PythonFormattingEditProvider(context, serviceContainer);

src/client/language/languageConfiguration.ts

@@ -2,33 +2,111 @@
 // Licensed under the MIT License.
 'use strict';
 
-import { IndentAction, languages } from 'vscode';
-import { PYTHON_LANGUAGE } from '../common/constants';
+import { IndentAction, LanguageConfiguration } from 'vscode';
+import { verboseRegExp } from '../common/utils/regexp';
 
-export const MULTILINE_SEPARATOR_INDENT_REGEX = /^(?!\s+\\)[^#\n]+\\$/;
+// tslint:disable:no-multiline-string
 
-export function setLanguageConfiguration() {
-    // Enable indentAction
-    languages.setLanguageConfiguration(PYTHON_LANGUAGE, {
+// tslint:disable-next-line:max-func-body-length
+export function getLanguageConfiguration(): LanguageConfiguration {
+    return {
         onEnterRules: [
+            // multi-line separator
             {
-                beforeText: /^\s*(?:def|class|for|if|elif|else|while|try|with|finally|except|async)\b.*:\s*/,
-                action: { indentAction: IndentAction.Indent }
-            },
-            {
-                beforeText: MULTILINE_SEPARATOR_INDENT_REGEX,
-                action: { indentAction: IndentAction.Indent }
+                beforeText: verboseRegExp(`
+                    ^
+                    (?! \\s+ \\\\ )
+                    [^#\n]+
+                    \\\\
+                    $
+                `),
+                action: {
+                    indentAction: IndentAction.Indent
+                }
             },
+            // continue comments
             {
                 beforeText: /^\s*#.*/,
                 afterText: /.+$/,
-                action: { indentAction: IndentAction.None, appendText: '# ' }
+                action: {
+                    indentAction: IndentAction.None,
+                    appendText: '# '
+                }
+            },
+            // indent on enter (block-beginning statements)
+            {
+                /**
+                 * This does not handle all cases. However, it does handle nearly all usage.
+                 * Here's what it does not cover:
+                 * - the statement is split over multiple lines (and hence the ":" is on a different line)
+                 * - the code block is inlined (after the ":")
+                 * - there are multiple statements on the line (separated by semicolons)
+                 * Also note that `lambda` is purposefully excluded.
+                 */
+                beforeText: verboseRegExp(`
+                    ^
+                    \\s*
+                    (?:
+                        (?:
+                            (?:
+                                class |
+                                def |
+                                async \\s+ def |
+                                except |
+                                for |
+                                if |
+                                elif |
+                                while |
+                                with
+                            )
+                            \\b .*
+                        ) |
+                        else |
+                        try |
+                        finally
+                    )
+                    \\s*
+                    [:]
+                    \\s*
+                    (?: [#] .* )?
+                    $
+                `),
+                action: {
+                    indentAction: IndentAction.Indent
+                }
             },
+            // outdent on enter (block-ending statements)
             {
-                beforeText: /^\s+(continue|break|return)\b.*/,
-                afterText: /\s+$/,
-                action: { indentAction: IndentAction.Outdent }
+                beforeText: verboseRegExp(`
+                    ^
+                    (?:
+                        (?:
+                            \\s*
+                            (?:
+                                pass |
+                                raise \\s+ [^#\\s] [^#]*
+                            )
+                        ) |
+                        (?:
+                            \\s+
+                            (?:
+                                raise |
+                                break |
+                                continue
+                            )
+                        )
+                    )
+                    \\s*
+                    (?: [#] .* )?
+                    $
+                `),
+                action: {
+                    indentAction: IndentAction.Outdent
+                }
             }
+            // Note that we do not currently have an auto-dedent
+            // solution for "elif", "else", "except", and "finally".
+            // We had one but had to remove it (see issue #6886).
         ]
-    });
+    };
 }

src/test/common/utils/regexp.unit.test.ts

@@ -0,0 +1,75 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+'use strict';
+
+// tslint:disable:no-multiline-string
+
+import { expect } from 'chai';
+
+import {
+    verboseRegExp
+} from '../../../client/common/utils/regexp';
+
+suite('Utils for regular expressions - verboseRegExp()', () => {
+    test('whitespace removed in multiline pattern (example of typical usage)', () => {
+        const regex = verboseRegExp(`
+            ^
+            (?:
+                spam \\b .*
+            ) |
+            (?:
+                eggs \\b .*
+            )
+            $
+        `);
+
+        expect(regex.source).to.equal('^(?:spam\\b.*)|(?:eggs\\b.*)$', 'mismatch');
+    });
+
+    const whitespaceTests = [
+        ['spam eggs', 'spameggs'],
+        [`spam
+          eggs`, 'spameggs'],
+        // empty
+        ['  ', '(?:)'],
+        [`
+         `, '(?:)']
+    ];
+    for (const [pat, expected] of whitespaceTests) {
+        test(`whitespace removed ("${pat}")`, () => {
+            const regex = verboseRegExp(pat);
+
+            expect(regex.source).to.equal(expected, 'mismatch');
+        });
+    }
+
+    const noopPatterns = [
+        '^(?:spam\\b.*)$',
+        'spam',
+        '^spam$',
+        'spam$',
+        '^spam'
+    ];
+    for (const pat of noopPatterns) {
+        test(`pattern not changed ("${pat}")`, () => {
+            const regex = verboseRegExp(pat);
+
+            expect(regex.source).to.equal(pat, 'mismatch');
+        });
+    }
+
+    const emptyPatterns = [
+        '',
+        `
+        `,
+        '  '
+    ];
+    for (const pat of emptyPatterns) {
+        test(`no pattern ("${pat}")`, () => {
+            const regex = verboseRegExp(pat);
+
+            expect(regex.source).to.equal('(?:)', 'mismatch');
+        });
+    }
+});