Use the Summary API to improve the schema synchronization performance

packages/graph-explorer-proxy-server/node-server.js

@@ -32,7 +32,7 @@ const getCredentials = async () => {
 };
 
 dotenv.config({ path: "../graph-explorer/.env" });
-  
+
 (async () => {
   let creds = await getCredentials();
   let requestSig;
@@ -64,23 +64,23 @@ dotenv.config({ path: "../graph-explorer/.env" });
     await getAuthHeaders(language, req, reqObjects[0], reqObjects[2]);
 
     return new Promise((resolve, reject) => {
-        
+
       const wrapper = (n) => {
         fetch(url, { headers: req.headers, })
           .then(async res => {
 
             if (!res.ok) {
-	            console.log("Response not ok");
+              console.log("Response not ok");
               const error = res.status
               return Promise.reject(error);
             } else {
-	            console.log("Response ok");
+              console.log("Response ok");
               resolve(res);
             }
           })
           .catch(async (err) => {
             console.log("Attempt Credential Refresh");
-	          creds = await getCredentials();
+            creds = await getCredentials();
             if (creds === undefined) {
               reject("Credentials undefined after credential refresh. Check that you have proper acccess")
             }
@@ -119,7 +119,7 @@ dotenv.config({ path: "../graph-explorer/.env" });
   }
 
   async function getAuthHeaders(language, req, endpoint_url, requestSig) {
-    let authHeaders 
+    let authHeaders
     if (language == "sparql") {
       authHeaders = await requestSig.requestAuthHeaders(
         endpoint_url.port,
@@ -147,9 +147,9 @@ dotenv.config({ path: "../graph-explorer/.env" });
     let data;
     try {
       response = await retryFetch(`${req.headers["graph-db-connection-url"]}/sparql?query=` +
-      encodeURIComponent(req.query.query) +
-      "&format=json", undefined, undefined, req, "sparql").then((res) => res)
-      
+        encodeURIComponent(req.query.query) +
+        "&format=json", undefined, undefined, req, "sparql").then((res) => res)
+
       data = await response.json();
       res.send(data);
     } catch (error) {
@@ -163,7 +163,35 @@ dotenv.config({ path: "../graph-explorer/.env" });
     let data;
     try {
       response = await retryFetch(`${req.headers["graph-db-connection-url"]}/?gremlin=` +
-      encodeURIComponent(req.query.gremlin), undefined, undefined, req, "gremlin").then((res) => res)
+        encodeURIComponent(req.query.gremlin), undefined, undefined, req, "gremlin").then((res) => res)
+
+      data = await response.json();
+      res.send(data);
+    } catch (error) {
+      next(error);
+      console.log(error);
+    }
+  });
+
+  app.get("/pg/statistics/summary", async (req, res, next) => {
+    let response;
+    let data;
+    try {
+      response = await retryFetch(`${req.headers["graph-db-connection-url"]}/pg/statistics/summary`, undefined, undefined, req, "gremlin").then((res) => res)
+
+      data = await response.json();
+      res.send(data);
+    } catch (error) {
+      next(error);
+      console.log(error);
+    }
+  });
+
+  app.get("/rdf/statistics/summary", async (req, res, next) => {
+    let response;
+    let data;
+    try {
+      response = await retryFetch(`${req.headers["graph-db-connection-url"]}/rdf/statistics/summary`, undefined, undefined, req, "gremlin").then((res) => res)
 
       data = await response.json();
       res.send(data);

packages/graph-explorer/src/connector/AbstractConnector.ts

@@ -8,6 +8,7 @@ import type {
 
 export type QueryOptions = {
   abortSignal?: AbortSignal;
+  disableCache?: boolean;
 };
 
 export type VertexSchemaResponse = Pick<
@@ -18,21 +19,36 @@ export type VertexSchemaResponse = Pick<
   | "displayNameAttribute"
   | "longDisplayNameAttribute"
 > & {
+  total?: number;
+};
+
+export type CountsByTypeRequest = {
+  label: string;
+};
+export type CountsByTypeResponse = {
   total: number;
 };
 
 export type EdgeSchemaResponse = Pick<
   EdgeTypeConfig,
   "type" | "displayLabel" | "attributes"
 > & {
-  total: number;
+  total?: number;
 };
 
 export type SchemaResponse = {
+  /**
+   * Total number of vertices.
+   */
+  totalVertices?: number;
   /**
    * List of vertices definitions.
    */
   vertices: VertexSchemaResponse[];
+  /**
+   * Total number of edges.
+   */
+  totalEdges?: number;
   /**
    * List of edges definitions.
    */
@@ -189,8 +205,6 @@ export abstract class AbstractConnector {
 
   protected abstract readonly basePath: string;
 
-  private readonly _requestCache: Map<string, CacheItem> = new Map();
-
   public constructor(connection: ConfigurationWithConnection["connection"]) {
     if (!connection?.url) {
       throw new Error("Invalid configuration. Missing 'connection.url'");
@@ -204,6 +218,14 @@ export abstract class AbstractConnector {
    */
   public abstract fetchSchema(options?: QueryOptions): Promise<SchemaResponse>;
 
+  /**
+   * Count the number of vertices of a given type
+   */
+  public abstract fetchVertexCountsByType(
+    req: CountsByTypeRequest,
+    options?: QueryOptions
+  ): Promise<CountsByTypeResponse>;
+
   /**
    * Fetch all directly connected neighbors of a given source
    * filtered by vertex or edge types and/or vertex attributes.
@@ -233,15 +255,23 @@ export abstract class AbstractConnector {
   /**
    * This method performs requests and cache their responses.
    */
-  protected async request<TResult>(
+  protected async requestQueryTemplate<TResult = any>(
     queryTemplate: string,
-    options?: { signal?: AbortSignal }
-  ): Promise<any> {
-    const url = this._connection.url.replace(/\/$/, "");
+    options?: { signal?: AbortSignal; disableCache?: boolean }
+  ): Promise<TResult> {
     const encodedQuery = encodeURIComponent(queryTemplate);
-    const uri = `${url}${this.basePath}${encodedQuery}&format=json`;
+    const uri = `${this.basePath}${encodedQuery}&format=json`;
+    return this.request<TResult>(uri, options);
+  }
+
+  protected async request<TResult = any>(
+    uri: string,
+    options?: { signal?: AbortSignal; disableCache?: boolean }
+  ): Promise<TResult> {
+    const url = this._connection.url.replace(/\/$/, "");
+    const currentUri = `${url}${uri}`;
 
-    const cachedResponse = await this._getFromCache(uri);
+    const cachedResponse = await this._getFromCache(currentUri);
     if (
       cachedResponse &&
       cachedResponse.updatedAt +
@@ -251,7 +281,7 @@ export abstract class AbstractConnector {
       return cachedResponse.data;
     }
 
-    return this._requestAndCache<TResult>(uri, {
+    return this._requestAndCache<TResult>(currentUri, {
       signal: options?.signal,
       headers: this._getAuthHeaders(),
     });
@@ -269,10 +299,16 @@ export abstract class AbstractConnector {
     return headers;
   }
 
-  private async _requestAndCache<TResult>(url: string, init?: RequestInit) {
+  private async _requestAndCache<TResult = any>(
+    url: string,
+    init?: RequestInit,
+    options?: Pick<QueryOptions, "disableCache">
+  ) {
     const response = await fetch(url, init);
     const data = await response.json();
-    this._setToCache(url, { data, updatedAt: new Date().getTime() });
+    if (options?.disableCache !== true) {
+      this._setToCache(url, { data, updatedAt: new Date().getTime() });
+    }
     return data as TResult;
   }
 

packages/graph-explorer/src/connector/gremlin/GremlinConnector.ts

@@ -1,4 +1,6 @@
 import type {
+  CountsByTypeRequest,
+  CountsByTypeResponse,
   KeywordSearchRequest,
   KeywordSearchResponse,
   NeighborsCountRequest,
@@ -12,13 +14,36 @@ import { AbstractConnector } from "../AbstractConnector";
 import fetchNeighbors from "./queries/fetchNeighbors";
 import fetchNeighborsCount from "./queries/fetchNeighborsCount";
 import fetchSchema from "./queries/fetchSchema";
+import fetchVertexTypeCounts from "./queries/fetchVertexTypeCounts";
 import keywordSearch from "./queries/keywordSearch";
+import { GraphSummary } from "./types";
 
 export default class GremlinConnector extends AbstractConnector {
   protected readonly basePath = "/?gremlin=";
+  private readonly _summaryPath = "/pg/statistics/summary?mode=detailed";
 
-  fetchSchema(options?: QueryOptions): Promise<SchemaResponse> {
-    return fetchSchema(this._gremlinFetch(options));
+  async fetchSchema(options?: QueryOptions): Promise<SchemaResponse> {
+    const ops = { ...options, disableCache: true };
+    let summary: GraphSummary | undefined;
+    try {
+      const response = await this.request<{
+        payload: { graphSummary: GraphSummary };
+      }>(this._summaryPath, ops);
+      summary = response.payload.graphSummary;
+    } catch (e) {
+      if (import.meta.env.DEV) {
+        console.error("[Summary API]", e);
+      }
+    }
+
+    return fetchSchema(this._gremlinFetch(ops), summary);
+  }
+
+  fetchVertexCountsByType(
+    req: CountsByTypeRequest,
+    options: QueryOptions | undefined
+  ): Promise<CountsByTypeResponse> {
+    return fetchVertexTypeCounts(this._gremlinFetch(options), req);
   }
 
   fetchNeighbors(
@@ -44,8 +69,9 @@ export default class GremlinConnector extends AbstractConnector {
 
   private _gremlinFetch<TResult>(options?: QueryOptions) {
     return async (queryTemplate: string) => {
-      return super.request<TResult>(queryTemplate, {
+      return super.requestQueryTemplate<TResult>(queryTemplate, {
         signal: options?.abortSignal,
+        disableCache: options?.disableCache,
       });
     };
   }