feat: Improve Linting performance (#23865)

## Description This PR introduces a new architecture, making evaluation and linting independent. <img width="500" alt="Screenshot 2023-07-04 at 17 24 40" src="https://github.com/appsmithorg/appsmith/assets/46670083/00b1eab9-cd79-4442-b51a-5345c2d6c4da"> In the previous architecture, one dependency graph was used to hold the relationship between entities in the application and subsequently, the "evaluation order" and "paths to lint" were generated. Although similar, the dependency graph required for evaluation and linting differ. For example, trigger fields should not depend on any other entity/entity path in the eval's dependency graph since they are not reactive. This is not the case for the linting dependency graph. ## Performance - This PR introduces "lint only" actions. These actions trigger linting, but not evaluation. For example, UPDATE_JS_ACTION_BODY_INIT (which is fired immediately after a user edits the body of a JS Object). Since linting fires without waiting for a successful update on the server, **response time decreases by 40%** (from 2s to 1.2s). - Reduction in time taken to generate paths requiring linting. <img width="715" alt="Screenshot 2023-07-04 at 18 10 52" src="https://github.com/appsmithorg/appsmith/assets/46670083/d73a4bfc-de73-4fa7-bdca-af1e5d8ce8a1"> #### PR fixes following issue(s) Fixes #23447 Fixes #23166 Fixes #24194 Fixes #23720 Fixes #23868 Fixes #21895 Latest DP: https://appsmith-r3f9e325p-get-appsmith.vercel.app/ #### Type of change - Chore (housekeeping or task changes that don't impact user perception) ## Testing > #### How Has This Been Tested? - [x] Manual - [ ] Jest - [ ] Cypress > > #### Test Plan #23865 (comment) > > #### Issues raised during DP testing #23865 (comment) response: #23865 (comment) > > > ## Checklist: #### Dev activity - [ ] My code follows the style guidelines of this project - [ ] I have performed a self-review of my own code - [ ] I have commented my code, particularly in hard-to-understand areas - [ ] I have made corresponding changes to the documentation - [ ] My changes generate no new warnings - [ ] I have added tests that prove my fix is effective or that my feature works - [ ] New and existing unit tests pass locally with my changes - [ ] PR is being merged under a feature flag #### QA activity: - [ ] [Speedbreak features](https://github.com/appsmithorg/TestSmith/wiki/Test-plan-implementation#speedbreaker-features-to-consider-for-every-change) have been covered - [ ] Test plan covers all impacted features and [areas of interest](https://github.com/appsmithorg/TestSmith/wiki/Guidelines-for-test-plans/_edit#areas-of-interest) - [x] Test plan has been peer reviewed by project stakeholders and other QA members - [x] Manually tested functionality on DP - [ ] We had an implementation alignment call with stakeholders post QA Round 2 - [ ] Cypress test cases have been added and approved by SDET/manual QA - [ ] Added `Test Plan Approved` label after Cypress tests were reviewed - [ ] Added `Test Plan Approved` label after JUnit tests were reviewed --------- Co-authored-by: arunvjn <[email protected]> Co-authored-by: Ivan Akulov <[email protected]>
appsmithorg · Jul 5, 2023 · e6f2dca · e6f2dca
1 parent 1ab630f
commit e6f2dca
Show file tree

Hide file tree

Showing 59 changed files with 2,087 additions and 438 deletions.
diff --git a/app/client/cypress/e2e/Regression/ClientSide/BugTests/Bug24194_Spec.ts b/app/client/cypress/e2e/Regression/ClientSide/BugTests/Bug24194_Spec.ts
@@ -0,0 +1,39 @@
+import {
+  agHelper,
+  entityExplorer,
+  jsEditor,
+  locators,
+  propPane,
+} from "../../../../support/Objects/ObjectsCore";
+
+describe("Responsiveness of linting", () => {
+  before(() => {
+    entityExplorer.DragDropWidgetNVerify("buttonwidget", 300, 300);
+  });
+  it("Should update linting when entity is added/renamed", () => {
+    const JS_OBJECT = `export default {
+        myFun1: () => {
+          return "";
+        },
+        myFun2: ()=>{
+         return ""
+        }
+      }`;
+    propPane.UpdatePropertyFieldValue("Tooltip", "{{JSObject1.myFun1}}");
+    agHelper.AssertElementExist(locators._lintErrorElement);
+    jsEditor.CreateJSObject(JS_OBJECT, {
+      paste: true,
+      completeReplace: true,
+      toRun: false,
+      shouldCreateNewJSObj: true,
+    });
+
+    entityExplorer.SelectEntityByName("Button1", "Widgets");
+    agHelper.AssertElementAbsence(locators._lintErrorElement);
+    agHelper.RefreshPage();
+    entityExplorer.SelectEntityByName("JSObject1", "Queries/JS");
+    jsEditor.RenameJSObjFromPane("JSObject2");
+    entityExplorer.SelectEntityByName("Button1", "Widgets");
+    agHelper.AssertElementAbsence(locators._lintErrorElement);
+  });
+});
diff --git a/app/client/cypress/e2e/Regression/ClientSide/Linting/BasicLint_spec.ts b/app/client/cypress/e2e/Regression/ClientSide/Linting/BasicLint_spec.ts
@@ -366,4 +366,62 @@ describe("Linting", () => {
       agHelper.AssertElementExist(locators._lintErrorElement);
     },
   );
+  it("10. Should not clear unrelated lint errors", () => {
+    const JS_OBJECT_WITH_MULTPLE_ERRORS = `export default {
+      myFun1: () => {
+        return error1;
+      },
+      myFun2: ()=>{
+       return error2
+      }
+    }`;
+    const JS_OBJECT_WITH_MYFUN2_EDITED = `export default {
+      myFun1: () => {
+        return error1;
+      },
+      myFun2: ()=>{
+       return "error cleared"
+      }
+    }`;
+
+    jsEditor.CreateJSObject(JS_OBJECT_WITH_MULTPLE_ERRORS, {
+      paste: true,
+      completeReplace: true,
+      toRun: false,
+      shouldCreateNewJSObj: true,
+      prettify: false,
+    });
+    agHelper.AssertElementExist(locators._lintErrorElement);
+
+    jsEditor.EditJSObj(JS_OBJECT_WITH_MYFUN2_EDITED, false);
+
+    agHelper.AssertElementExist(locators._lintErrorElement);
+  });
+  it("11. Shows correct lint error when js object has duplicate keys", () => {
+    const JS_OBJECT_WITH_DUPLICATE_KEYS = `export default {
+        myVar1: [],
+        myVar2: {},
+        myFun1 () {
+            //	write code here
+            //	this.myVar1 = [1,2,3]
+        
+        },
+        async myFun1 () {
+            //	use async-await or promises
+            //	await storeValue('varName', 'hello world') 
+        }
+    }`;
+
+    jsEditor.CreateJSObject(JS_OBJECT_WITH_DUPLICATE_KEYS, {
+      paste: true,
+      completeReplace: true,
+      toRun: false,
+      shouldCreateNewJSObj: true,
+      prettify: false,
+    });
+
+    agHelper
+      .AssertElementExist(locators._lintErrorElement)
+      .should("have.length", 1);
+  });
 });
diff --git a/app/client/cypress/e2e/Regression/ClientSide/Performance/LinkRelPreload_Spec.js b/app/client/cypress/e2e/Regression/ClientSide/Performance/LinkRelPreload_Spec.js
@@ -27,30 +27,30 @@ import emptyDSL from "../../../../fixtures/emptyDSL.json";
 // started failing for you, it’s likely you import()ed some new chunks that the edit or the view mode uses.
 // To fix the test, see preloading instructions in public/index.html.
 
-describe("html should include <link rel='preload'>s for all code-split javascript", function () {
+describe("html should include preload metadata for all code-split javascript", function () {
   before(() => {
     cy.addDsl(emptyDSL);
   });
 
-  it("1. In edit & View mode", function () {
-    testLinkRelPreloads("edit-mode");
+  it("1. In edit mode", function () {
+    testPreloadMetadata("edit-mode");
   });
 
   // Note: this must be a separate test from the previous one,
   // as we’re relying on Cypress resetting intercepts between tests.
   it("2. In view mode", function () {
-    cy.reload();
+    reloadAndTogglePreloading(true);
 
     // Ensure the app editor is fully loaded
     cy.get("#sidebar").should("be.visible");
 
     _.deployMode.DeployApp();
 
-    testLinkRelPreloads("view-mode");
+    testPreloadMetadata("view-mode");
   });
 });
 
-function testLinkRelPreloads(viewOrEditMode) {
+function testPreloadMetadata(viewOrEditMode) {
   // Disable network caching in Chromium, per https://docs.cypress.io/api/commands/intercept#cyintercept-and-request-caching
   // and https://github.com/cypress-io/cypress/issues/14459#issuecomment-768616195
   Cypress.automation("remote:debugger:protocol", {
@@ -66,7 +66,7 @@ function testLinkRelPreloads(viewOrEditMode) {
 
   // Intercept all JS network requests and collect them
   cy.intercept(/\/static\/js\/.+\.js/, (req) => {
-    // Ignore
+    // Don’t collect:
     // - requests to worker files
     // - requests to icons
     // - request to the main bundle
@@ -83,11 +83,11 @@ function testLinkRelPreloads(viewOrEditMode) {
 
   // Make all web workers empty. This prevents web workers from loading additional chunks,
   // as we need to collect only chunks from the main thread
-  cy.intercept(/\/static\/js\/.+Worker\..+\.js/, { body: "" }).as(
-    "workerRequests",
-  );
+  cy.intercept(/\/static\/js\/.+Worker\..+\.js/, { body: "" }).as("worker");
 
-  cy.reload();
+  // Reload without preloading, as we want to collect only chunks
+  // actually requested by the current route
+  reloadAndTogglePreloading(false);
 
   cy.waitForNetworkIdle("/static/js/*.js", 5000, { timeout: 60 * 1000 });
 
@@ -127,20 +127,32 @@ function testLinkRelPreloads(viewOrEditMode) {
       ),
     );
 
-    const requestsString = `[${
+    const actuallyLoadedFiles = `[${
       requestsToCompare.length
     } items] ${requestsToCompare.sort().join(", ")}`;
-    const linksString = `[${linksToCompare.length} items] ${linksToCompare
+    const preloadedFiles = `[${linksToCompare.length} items] ${linksToCompare
       .sort()
       .join(", ")}`;
 
     // Comparing strings instead of deep-equalling arrays because this is the only way
     // to see which chunks are actually missing: https://github.com/cypress-io/cypress/issues/4084
-    cy.wrap(requestsString).should("equal", linksString);
+    cy.wrap(actuallyLoadedFiles).should("equal", preloadedFiles);
   });
 }
 
 /** Removes all duplicated items from the array */
 function unique(arr) {
   return Array.from(new Set(arr));
 }
+
+function reloadAndTogglePreloading(chunkPreloadingEnabled) {
+  cy.url().then((currentURL) => {
+    let url = new URL(currentURL);
+    if (chunkPreloadingEnabled) {
+      url.searchParams.set("disableChunkPreload", "true");
+    } else {
+      url.searchParams.delete("disableChunkPreload");
+    }
+    cy.visit(url.toString());
+  });
+}
diff --git a/app/client/packages/ast/index.ts b/app/client/packages/ast/index.ts
@@ -25,7 +25,12 @@ import {
 import { ECMA_VERSION, SourceType, NodeTypes } from "./src/constants";
 
 // JSObjects
-import type { TParsedJSProperty, JSPropertyPosition } from "./src/jsObject";
+import type {
+  TParsedJSProperty,
+  JSPropertyPosition,
+  JSVarProperty,
+  JSFunctionProperty,
+} from "./src/jsObject";
 import { parseJSObject, isJSFunctionProperty } from "./src/jsObject";
 
 // action creator
@@ -73,6 +78,8 @@ export type {
   TParsedJSProperty,
   JSPropertyPosition,
   PeekOverlayExpressionIdentifierOptions,
+  JSVarProperty,
+  JSFunctionProperty,
 };
 
 export {

diff --git a/app/client/packages/ast/src/jsObject/index.ts b/app/client/packages/ast/src/jsObject/index.ts
@@ -35,20 +35,20 @@ export interface JSPropertyPosition {
   keyEndColumn: number;
 }
 
-interface baseJSProperty {
+interface BaseJSProperty {
   key: string;
   value: string;
   type: string;
   position: Partial<JSPropertyPosition>;
   rawContent: string;
 }
 
-type JSFunctionProperty = baseJSProperty & {
+export type JSFunctionProperty = BaseJSProperty & {
   arguments: functionParam[];
   // If function uses the "async" keyword
   isMarkedAsync: boolean;
 };
-type JSVarProperty = baseJSProperty;
+export type JSVarProperty = BaseJSProperty;
 
 export type TParsedJSProperty = JSVarProperty | JSFunctionProperty;
 

diff --git a/app/client/src/actions/evaluationActions.ts b/app/client/src/actions/evaluationActions.ts
@@ -3,7 +3,7 @@ import {
   ReduxActionErrorTypes,
   ReduxActionTypes,
 } from "@appsmith/constants/ReduxActionConstants";
-import _ from "lodash";
+import { intersection, union } from "lodash";
 import type { DataTree } from "entities/DataTree/dataTreeFactory";
 import type { DependencyMap } from "utils/DynamicBindingUtils";
 import type { Diff } from "deep-diff";
@@ -30,8 +30,14 @@ export const LINT_REDUX_ACTIONS = {
   [ReduxActionTypes.UPDATE_LAYOUT]: true,
   [ReduxActionTypes.UPDATE_WIDGET_PROPERTY]: true,
   [ReduxActionTypes.UPDATE_WIDGET_NAME_SUCCESS]: true,
-  [ReduxActionTypes.UPDATE_JS_ACTION_BODY_SUCCESS]: true,
+  [ReduxActionTypes.UPDATE_JS_ACTION_BODY_INIT]: true, // "lint only" action
   [ReduxActionTypes.META_UPDATE_DEBOUNCED_EVAL]: true,
+  [ReduxActionTypes.FETCH_JS_ACTIONS_FOR_PAGE_SUCCESS]: true,
+  [ReduxActionTypes.FETCH_ACTIONS_FOR_PAGE_SUCCESS]: true,
+  [ReduxActionTypes.INSTALL_LIBRARY_SUCCESS]: true,
+  [ReduxActionTypes.UNINSTALL_LIBRARY_SUCCESS]: true,
+  [ReduxActionTypes.BUFFERED_ACTION]: true,
+  [ReduxActionTypes.BATCH_UPDATES_SUCCESS]: true,
 };
 
 export const LOG_REDUX_ACTIONS = {
@@ -94,41 +100,46 @@ export const EVALUATE_REDUX_ACTIONS = [
   ReduxActionTypes.UPDATE_SELECTED_APP_THEME_SUCCESS,
   ReduxActionTypes.CHANGE_SELECTED_APP_THEME_SUCCESS,
   ReduxActionTypes.SET_PREVIEW_APP_THEME,
+
+  // Custom Library
+  ReduxActionTypes.INSTALL_LIBRARY_SUCCESS,
+  ReduxActionTypes.UNINSTALL_LIBRARY_SUCCESS,
+  // Buffer
+  ReduxActionTypes.BUFFERED_ACTION,
 ];
 // Topics used for datasource and query form evaluations
 export const FORM_EVALUATION_REDUX_ACTIONS = [
   ReduxActionTypes.INIT_FORM_EVALUATION,
   ReduxActionTypes.RUN_FORM_EVALUATION,
 ];
-export const shouldProcessBatchedAction = (action: ReduxAction<unknown>) => {
-  if (
-    action.type === ReduxActionTypes.BATCH_UPDATES_SUCCESS &&
-    Array.isArray(action.payload)
-  ) {
-    const batchedActionTypes = action.payload.map(
-      (batchedAction) => batchedAction.type,
-    );
-    return (
-      _.intersection(EVALUATE_REDUX_ACTIONS, batchedActionTypes).length > 0
-    );
-  }
-  return true;
+
+export const shouldTriggerEvaluation = (action: ReduxAction<unknown>) => {
+  return (
+    shouldProcessAction(action) && EVALUATE_REDUX_ACTIONS.includes(action.type)
+  );
+};
+export const shouldTriggerLinting = (action: ReduxAction<unknown>) => {
+  return shouldProcessAction(action) && !!LINT_REDUX_ACTIONS[action.type];
 };
 
-export function shouldLint(action: ReduxAction<unknown>) {
+export const getAllActionTypes = (action: ReduxAction<unknown>) => {
   if (
     action.type === ReduxActionTypes.BATCH_UPDATES_SUCCESS &&
     Array.isArray(action.payload)
   ) {
     const batchedActionTypes = action.payload.map(
-      (batchedAction) => batchedAction.type,
-    );
-    return batchedActionTypes.some(
-      (actionType) => LINT_REDUX_ACTIONS[actionType],
+      (batchedAction) => batchedAction.type as string,
     );
+    return batchedActionTypes;
   }
-  return LINT_REDUX_ACTIONS[action.type];
-}
+  return [action.type];
+};
+
+export const shouldProcessAction = (action: ReduxAction<unknown>) => {
+  const actionTypes = getAllActionTypes(action);
+
+  return intersection(EVAL_AND_LINT_REDUX_ACTIONS, actionTypes).length > 0;
+};
 
 export function shouldLog(action: ReduxAction<unknown>) {
   if (
@@ -199,3 +210,18 @@ export const startFormEvaluations = (
     },
   };
 };
+
+// These actions require the entire tree to be re-evaluated
+const FORCE_EVAL_ACTIONS = {
+  [ReduxActionTypes.INSTALL_LIBRARY_SUCCESS]: true,
+  [ReduxActionTypes.UNINSTALL_LIBRARY_SUCCESS]: true,
+};
+
+export const shouldForceEval = (action: ReduxAction<unknown>) => {
+  return !!FORCE_EVAL_ACTIONS[action.type];
+};
+
+export const EVAL_AND_LINT_REDUX_ACTIONS = union(
+  EVALUATE_REDUX_ACTIONS,
+  Object.keys(LINT_REDUX_ACTIONS),
+);
diff --git a/app/client/src/ce/constants/ReduxActionConstants.tsx b/app/client/src/ce/constants/ReduxActionConstants.tsx
@@ -839,6 +839,7 @@ const ActionTypes = {
   DATASOURCE_DISCARD_ACTION: "DATASOURCE_DISCARD_ACTION",
   SET_ONE_CLICK_BINDING_OPTIONS_VISIBILITY:
     "SET_ONE_CLICK_BINDING_OPTIONS_VISIBILITY",
+  BUFFERED_ACTION: "BUFFERED_ACTION",
 };
 
 export const ReduxActionTypes = {