diff --git a/package.json b/package.json
index 8c24290a941..13298af5247 100644
--- a/package.json
+++ b/package.json
@@ -511,6 +511,11 @@
           "description": "Uses a hack to move around folds properly",
           "default": false
         },
+        "vim.mouseSelectionGoesIntoVisualMode": {
+          "type": "boolean",
+          "description": "Does dragging with the mouse put you into visual mode",
+          "default": true
+        },
         "vim.enableNeovim": {
           "type": "boolean",
           "description": "Use neovim on backend. (only works for Ex commands right now). You should restart VScode after enable/disabling this for the changes to take effect. NOTE: Neovim must be installed (v0.2.0) and neovimPath must be set the executable in order for this setting to work. Otherwise, vscodevim will crash.",
diff --git a/src/configuration/configuration.ts b/src/configuration/configuration.ts
index d35356689bd..3b42664156b 100644
--- a/src/configuration/configuration.ts
+++ b/src/configuration/configuration.ts
@@ -292,6 +292,7 @@ class ConfigurationClass {
    */
   visualstar = false;
 
+  mouseSelectionGoesIntoVisualMode = true;
   /**
    * Uses a hack to fix moving around folds.
    */
diff --git a/src/mode/modeHandler.ts b/src/mode/modeHandler.ts
index 2443b16659c..45cb6af73fc 100644
--- a/src/mode/modeHandler.ts
+++ b/src/mode/modeHandler.ts
@@ -746,7 +746,11 @@ export class ModeHandler implements vscode.Disposable {
           this._vimState.lastClickWasPastEol = false;
         }
 
-        if (!this._vimState.getModeObject(this).isVisualMode) {
+        if (
+          !this._vimState.getModeObject(this).isVisualMode &&
+          (this._vimState.getModeObject(this).name !== ModeName.Insert ||
+            Configuration.mouseSelectionGoesIntoVisualMode)
+        ) {
           this._vimState.currentMode = ModeName.Visual;
           this.setCurrentModeByName(this._vimState);