chore: add etag in consolidated api controller

app/client/craco.build.config.js

@@ -14,9 +14,8 @@ const plugins = [];
 plugins.push(
   new WorkboxPlugin.InjectManifest({
     swSrc: "./src/serviceWorker.ts",
-    mode: "development",
+    mode: "production",
     swDest: "./pageService.js",
-    maximumFileSizeToCacheInBytes: 11 * 1024 * 1024,
     exclude: [
       // Don’t cache source maps and PWA manifests.
       // (These are the default values of the `exclude` option: https://developer.chrome.com/docs/workbox/reference/workbox-build/#type-WebpackPartial,
@@ -32,9 +31,8 @@ plugins.push(
       // one by one (as the service worker does it) keeps the network busy for a long time
       // and delays the service worker installation
       /\/*\.svg$/,
+      /\.(js|css|html|png|jpg|jpeg|gif)$/, // Exclude JS, CSS, HTML, and image files
     ],
-    // Don’t cache-bust JS and CSS chunks
-    dontCacheBustURLsMatching: /\.[0-9a-zA-Z]{8}\.chunk\.(js|css)$/,
   }),
 );
 

app/client/cypress/limited-tests.txt

@@ -1,7 +1,5 @@
 # To run only limited tests - give the spec names in below format:
 #cypress/e2e/Regression/ClientSide/VisualTests/JSEditorIndent_spec.js
-# For running all specs - uncomment below:
-#cypress/e2e/**/**/*
-cypress/e2e/Regression/ClientSide/Anvil/Widgets/*
+cypress/e2e/Regression/ClientSide/ExplorerTests/Page_Load_Spec.js
 
 #ci-test-limit uses this file to run minimum of specs. Do not run entire suite with this command.

app/client/cypress/support/Objects/FeatureFlags.ts

@@ -36,7 +36,8 @@ export const getConsolidatedDataApi = (
       if (
         res.statusCode === 200 ||
         res.statusCode === 401 ||
-        res.statusCode === 500
+        res.statusCode === 500 ||
+        res.statusCode === 304
       ) {
         const originalResponse = res?.body;
         const updatedResponse = produce(originalResponse, (draft: any) => {
@@ -86,7 +87,7 @@ export const featureFlagInterceptForLicenseFlags = () => {
 
   cy.intercept("GET", "/api/v1/consolidated-api/*?*", (req) => {
     req.reply((res: any) => {
-      if (res.statusCode === 200) {
+      if (res.statusCode === 200 || res.statusCode === 304) {
         const originalResponse = res?.body;
         const updatedResponse = produce(originalResponse, (draft: any) => {
           draft.data.featureFlags.data = {};

app/client/cypress/support/Pages/AggregateHelper.ts

@@ -1193,7 +1193,7 @@ export class AggregateHelper {
       this.locator._specificToast("Cannot read properties of undefined"),
     );
     this.assertHelper.AssertDocumentReady();
-    this.Sleep(4000); //for page to load for CI runs
+    cy.wait(4000); //for page to load for CI runs
     networkCallAlias &&
       this.assertHelper.AssertNetworkStatus("@" + networkCallAlias); //getWorkspace for Edit page!
   }

app/client/src/api/services/ConsolidatedPageLoadApi/api.ts

@@ -8,13 +8,21 @@ export const getConsolidatedPageLoadDataView = async (
 ) => {
   const viewUrl = ConsolidatedApiUtils.getViewUrl(params);
 
-  return api.get<InitConsolidatedApi>(viewUrl);
+  return api.get<InitConsolidatedApi>(viewUrl, {
+    headers: {
+      "x-initiated-from": "main-thread",
+    },
+  });
 };
 
 export const getConsolidatedPageLoadDataEdit = async (
   params: ConsolidatedApiParams,
 ) => {
   const editUrl = ConsolidatedApiUtils.getEditUrl(params);
 
-  return api.get<InitConsolidatedApi>(editUrl);
+  return api.get<InitConsolidatedApi>(editUrl, {
+    headers: {
+      "x-initiated-from": "main-thread",
+    },
+  });
 };

app/client/src/serviceWorker.ts

@@ -1,11 +1,6 @@
-import { precacheAndRoute } from "workbox-precaching";
-import { clientsClaim, setCacheNameDetails, skipWaiting } from "workbox-core";
+import { clientsClaim, skipWaiting } from "workbox-core";
 import { registerRoute, Route } from "workbox-routing";
-import {
-  CacheFirst,
-  NetworkOnly,
-  StaleWhileRevalidate,
-} from "workbox-strategies";
+import { NetworkOnly } from "workbox-strategies";
 import {
   cachedApiUrlRegex,
   getApplicationParamsFromUrl,
@@ -14,32 +9,11 @@ import {
 } from "ee/utils/serviceWorkerUtils";
 import type { RouteHandlerCallback } from "workbox-core/types";
 
-setCacheNameDetails({
-  prefix: "appsmith",
-  suffix: "",
-  precache: "precache-v1",
-  runtime: "runtime",
-  googleAnalytics: "appsmith-ga",
-});
-
-const regexMap = {
-  appViewPage: new RegExp(/api\/v1\/pages\/\w+\/view$/),
-  static3PAssets: new RegExp(
-    /(tiny.cloud|googleapis|gstatic|cloudfront).*.(js|css|woff2)/,
-  ),
-  shims: new RegExp(/shims\/.*.js/),
-  profile: new RegExp(/v1\/(users\/profile|workspaces)/),
-};
-
-/* eslint-disable no-restricted-globals */
-// Note: if you need to filter out some files from precaching,
+// eslint-disable-next-line @typescript-eslint/no-unused-vars, @typescript-eslint/no-explicit-any
+const wbManifest = (self as any).__WB_MANIFEST;
 
-// do that in craco.build.config.js → workbox webpack plugin options
-// TODO: Fix this the next time the file is edited
-// eslint-disable-next-line @typescript-eslint/no-explicit-any
-const toPrecache = (self as any).__WB_MANIFEST;
-
-precacheAndRoute(toPrecache);
+// Delete the old pre-fetch cache. All static files are now cached by cache control headers.
+caches.delete("appsmith-precache-v1");
 
 self.__WB_DISABLE_DEV_LOGS = true;
 skipWaiting();
@@ -75,23 +49,6 @@ const htmlRouteHandlerCallback: RouteHandlerCallback = async ({
   return networkHandler.handle({ event, request });
 };
 
-// This route's caching seems too aggressive.
-// TODO(abhinav): Figure out if this is really necessary.
-// Maybe add the assets locally?
-registerRoute(({ url }) => {
-  return (
-    regexMap.shims.test(url.pathname) || regexMap.static3PAssets.test(url.href)
-  );
-}, new CacheFirst());
-
-registerRoute(({ url }) => {
-  return regexMap.profile.test(url.pathname);
-}, new NetworkOnly());
-
-registerRoute(({ url }) => {
-  return regexMap.appViewPage.test(url.pathname);
-}, new StaleWhileRevalidate());
-
 registerRoute(
   new Route(({ request, sameOrigin }) => {
     return sameOrigin && request.destination === "document";

app/client/start-https.sh

@@ -294,6 +294,14 @@ $(if [[ $use_https == 1 ]]; then echo "
 
         location /api {
             proxy_pass $backend;
+
+            gzip off; # Etag stripped from upstream if gzip is off. 
+            # Ref1: https://forum.nginx.org/read.php?2,242807,242810#msg-242810
+            # Ref2: https://www.ruby-forum.com/t/reverse-proxy-deleting-etag-header-from-response/246209/2
+            # Delete the Cache-Control header set in the server block above.
+            add_header Cache-Control '' always;
+            # Proxy pass the Cache-Control header from the upstream.
+            proxy_pass_header Cache-Control;
         }
 
         location /oauth2 {

app/server/appsmith-interfaces/src/main/java/com/appsmith/external/constants/spans/ce/ConsolidatedApiSpanNamesCE.java

@@ -31,4 +31,5 @@ public class ConsolidatedApiSpanNamesCE {
     public static final String DATASOURCES_SPAN = "datasources";
     public static final String FORM_CONFIG_SPAN = "form_config";
     public static final String MOCK_DATASOURCES_SPAN = "mock_datasources";
+    public static final String ETAG_SPAN = CONSOLIDATED_API_PREFIX + VIEW + "compute_etag";
 }

app/server/appsmith-server/pom.xml

@@ -412,6 +412,11 @@
             <version>2.14.2.Final</version>
             <scope>test</scope>
         </dependency>
+        <dependency>
+            <groupId>com.fasterxml.jackson.datatype</groupId>
+            <artifactId>jackson-datatype-jsr310</artifactId>
+            <version>2.17.0</version>
+        </dependency>
     </dependencies>
 
     <repositories>

app/server/appsmith-server/src/main/java/com/appsmith/server/controllers/ConsolidatedAPIController.java

@@ -11,9 +11,12 @@
 import com.fasterxml.jackson.annotation.JsonView;
 import io.micrometer.observation.ObservationRegistry;
 import lombok.extern.slf4j.Slf4j;
+import org.springframework.http.HttpHeaders;
 import org.springframework.http.HttpStatus;
+import org.springframework.http.ResponseEntity;
 import org.springframework.util.StringUtils;
 import org.springframework.web.bind.annotation.GetMapping;
+import org.springframework.web.bind.annotation.RequestHeader;
 import org.springframework.web.bind.annotation.RequestMapping;
 import org.springframework.web.bind.annotation.RequestParam;
 import org.springframework.web.bind.annotation.RestController;
@@ -24,6 +27,7 @@
 
 import static com.appsmith.external.constants.spans.ConsolidatedApiSpanNames.CONSOLIDATED_API_ROOT_EDIT;
 import static com.appsmith.external.constants.spans.ConsolidatedApiSpanNames.CONSOLIDATED_API_ROOT_VIEW;
+import static org.apache.commons.lang3.StringUtils.isBlank;
 
 @Slf4j
 @RestController
@@ -78,13 +82,13 @@ public Mono<ResponseDTO<ConsolidatedAPIResponseDTO>> getAllDataForFirstPageLoadF
 
     @JsonView(Views.Public.class)
     @GetMapping("/view")
-    public Mono<ResponseDTO<ConsolidatedAPIResponseDTO>> getAllDataForFirstPageLoadForViewMode(
+    public Mono<ResponseEntity<ResponseDTO<ConsolidatedAPIResponseDTO>>> getAllDataForFirstPageLoadForViewMode(
             @RequestParam(required = false) String applicationId,
             @RequestParam(required = false) String defaultPageId,
             @RequestParam(required = false, defaultValue = "branch") RefType refType,
             @RequestParam(required = false) String refName,
-            @RequestParam(required = false) String branchName) {
-
+            @RequestParam(required = false) String branchName,
+            @RequestHeader(required = false, name = "if-none-match") String ifNoneMatch) {
         if (!StringUtils.hasLength(refName)) {
             refName = branchName;
         }
@@ -100,8 +104,37 @@ public Mono<ResponseDTO<ConsolidatedAPIResponseDTO>> getAllDataForFirstPageLoadF
         return consolidatedAPIService
                 .getConsolidatedInfoForPageLoad(
                         defaultPageId, applicationId, refType, refName, ApplicationMode.PUBLISHED)
-                .map(consolidatedAPIResponseDTO ->
-                        new ResponseDTO<>(HttpStatus.OK.value(), consolidatedAPIResponseDTO, null))
+                .map(consolidatedAPIResponseDTO -> {
+                    long startTime = System.currentTimeMillis();
+
+                    String responseHash = consolidatedAPIService.computeConsolidatedAPIResponseEtag(
+                            consolidatedAPIResponseDTO, defaultPageId, applicationId);
+                    long endTime = System.currentTimeMillis();
+                    long duration = endTime - startTime;
+                    log.debug("Time taken to compute ETag: {} ms", duration);
+
+                    // if defaultPageId and applicationId are both null, then don't compute ETag
+                    if (isBlank(responseHash)) {
+                        ResponseDTO<ConsolidatedAPIResponseDTO> responseDTO =
+                                new ResponseDTO<>(HttpStatus.OK.value(), consolidatedAPIResponseDTO, null);
+                        return new ResponseEntity<>(responseDTO, HttpStatus.OK);
+                    }
+
+                    HttpHeaders headers = new HttpHeaders();
+                    headers.add("ETag", responseHash);
+                    headers.add("Cache-Control", "private, must-revalidate");
+
+                    if (ifNoneMatch != null && ifNoneMatch.equals(responseHash)) {
+                        ResponseDTO<ConsolidatedAPIResponseDTO> responseDTO =
+                                new ResponseDTO<>(HttpStatus.NOT_MODIFIED.value(), null, null);
+                        return new ResponseEntity<>(responseDTO, headers, HttpStatus.NOT_MODIFIED);
+                    }
+
+                    ResponseDTO<ConsolidatedAPIResponseDTO> responseDTO =
+                            new ResponseDTO<>(HttpStatus.OK.value(), consolidatedAPIResponseDTO, null);
+
+                    return new ResponseEntity<>(responseDTO, headers, HttpStatus.OK);
+                })
                 .tag("pageId", Objects.toString(defaultPageId))
                 .tag("applicationId", Objects.toString(applicationId))
                 .tag("refType", Objects.toString(refType))

app/server/appsmith-server/src/main/java/com/appsmith/server/services/ConsolidatedAPIServiceImpl.java

@@ -1,5 +1,6 @@
 package com.appsmith.server.services;
 
+import com.appsmith.external.helpers.ObservationHelper;
 import com.appsmith.server.actioncollections.base.ActionCollectionService;
 import com.appsmith.server.applications.base.ApplicationService;
 import com.appsmith.server.datasources.base.DatasourceService;
@@ -35,7 +36,8 @@ public ConsolidatedAPIServiceImpl(
             DatasourceService datasourceService,
             MockDataService mockDataService,
             ObservationRegistry observationRegistry,
-            CacheableRepositoryHelper cacheableRepositoryHelper) {
+            CacheableRepositoryHelper cacheableRepositoryHelper,
+            ObservationHelper observationHelper) {
         super(
                 sessionUserService,
                 userService,
@@ -53,6 +55,7 @@ public ConsolidatedAPIServiceImpl(
                 datasourceService,
                 mockDataService,
                 observationRegistry,
-                cacheableRepositoryHelper);
+                cacheableRepositoryHelper,
+                observationHelper);
     }
 }

app/server/appsmith-server/src/main/java/com/appsmith/server/services/ce/ConsolidatedAPIServiceCE.java

@@ -9,4 +9,7 @@ public interface ConsolidatedAPIServiceCE {
 
     Mono<ConsolidatedAPIResponseDTO> getConsolidatedInfoForPageLoad(
             String defaultPageId, String applicationId, RefType refType, String refName, ApplicationMode mode);
+
+    String computeConsolidatedAPIResponseEtag(
+            ConsolidatedAPIResponseDTO consolidatedAPIResponseDTO, String defaultPageId, String applicationId);
 }