From 651ebe0a7ff273c1b6b6d719e53080a751a68d19 Mon Sep 17 00:00:00 2001 From: nestoralvarezd Date: Thu, 20 Apr 2023 09:03:25 -0400 Subject: [PATCH 1/4] Use Summary API to sync the schema --- .../node-server.js | 48 ++++++++--- .../src/connector/AbstractConnector.ts | 60 +++++++++++--- .../src/connector/gremlin/GremlinConnector.ts | 32 +++++++- .../connector/gremlin/queries/fetchSchema.ts | 77 ++++++++++++++---- .../gremlin/queries/fetchVertexTypeCounts.ts | 33 ++++++++ .../templates/vertexTypeCountTemplate.ts | 8 ++ .../src/connector/gremlin/types.ts | 15 ++++ .../src/connector/sparql/SPARQLConnector.ts | 34 +++++++- .../sparql/queries/fetchClassCounts.ts | 30 +++++++ .../connector/sparql/queries/fetchSchema.ts | 80 +++++++++++++++---- .../templates/classWithCountsTemplates.ts | 10 +++ .../src/connector/sparql/types.ts | 10 +++ .../ConfigurationProvider.tsx | 2 + .../src/core/ConfigurationProvider/types.ts | 6 +- .../graph-explorer/src/hooks/useSchemaSync.ts | 24 +----- .../src/hooks/useUpdateSchema.ts | 64 +++++++++++++++ .../src/hooks/useUpdateVertexTypeCounts.ts | 61 ++++++++++++++ .../workspaces/DataExplorer/DataExplorer.tsx | 4 + 18 files changed, 517 insertions(+), 81 deletions(-) create mode 100644 packages/graph-explorer/src/connector/gremlin/queries/fetchVertexTypeCounts.ts create mode 100644 packages/graph-explorer/src/connector/gremlin/templates/vertexTypeCountTemplate.ts create mode 100644 packages/graph-explorer/src/connector/sparql/queries/fetchClassCounts.ts create mode 100644 packages/graph-explorer/src/connector/sparql/templates/classWithCountsTemplates.ts create mode 100644 packages/graph-explorer/src/hooks/useUpdateSchema.ts create mode 100644 packages/graph-explorer/src/hooks/useUpdateVertexTypeCounts.ts diff --git a/packages/graph-explorer-proxy-server/node-server.js b/packages/graph-explorer-proxy-server/node-server.js index 4c20fe9c3..c25cabb32 100644 --- a/packages/graph-explorer-proxy-server/node-server.js +++ b/packages/graph-explorer-proxy-server/node-server.js @@ -32,7 +32,7 @@ const getCredentials = async () => { }; dotenv.config({ path: "../graph-explorer/.env" }); - + (async () => { let creds = await getCredentials(); let requestSig; @@ -64,23 +64,23 @@ dotenv.config({ path: "../graph-explorer/.env" }); await getAuthHeaders(language, req, reqObjects[0], reqObjects[2]); return new Promise((resolve, reject) => { - + const wrapper = (n) => { fetch(url, { headers: req.headers, }) .then(async res => { if (!res.ok) { - console.log("Response not ok"); + console.log("Response not ok"); const error = res.status return Promise.reject(error); } else { - console.log("Response ok"); + console.log("Response ok"); resolve(res); } }) .catch(async (err) => { console.log("Attempt Credential Refresh"); - creds = await getCredentials(); + creds = await getCredentials(); if (creds === undefined) { reject("Credentials undefined after credential refresh. Check that you have proper acccess") } @@ -119,7 +119,7 @@ dotenv.config({ path: "../graph-explorer/.env" }); } async function getAuthHeaders(language, req, endpoint_url, requestSig) { - let authHeaders + let authHeaders if (language == "sparql") { authHeaders = await requestSig.requestAuthHeaders( endpoint_url.port, @@ -147,9 +147,9 @@ dotenv.config({ path: "../graph-explorer/.env" }); let data; try { response = await retryFetch(`${req.headers["graph-db-connection-url"]}/sparql?query=` + - encodeURIComponent(req.query.query) + - "&format=json", undefined, undefined, req, "sparql").then((res) => res) - + encodeURIComponent(req.query.query) + + "&format=json", undefined, undefined, req, "sparql").then((res) => res) + data = await response.json(); res.send(data); } catch (error) { @@ -163,7 +163,35 @@ dotenv.config({ path: "../graph-explorer/.env" }); let data; try { response = await retryFetch(`${req.headers["graph-db-connection-url"]}/?gremlin=` + - encodeURIComponent(req.query.gremlin), undefined, undefined, req, "gremlin").then((res) => res) + encodeURIComponent(req.query.gremlin), undefined, undefined, req, "gremlin").then((res) => res) + + data = await response.json(); + res.send(data); + } catch (error) { + next(error); + console.log(error); + } + }); + + app.get("/pg/statistics/summary", async (req, res, next) => { + let response; + let data; + try { + response = await retryFetch(`${req.headers["graph-db-connection-url"]}/pg/statistics/summary`, undefined, undefined, req, "gremlin").then((res) => res) + + data = await response.json(); + res.send(data); + } catch (error) { + next(error); + console.log(error); + } + }); + + app.get("/rdf/statistics/summary", async (req, res, next) => { + let response; + let data; + try { + response = await retryFetch(`${req.headers["graph-db-connection-url"]}/rdf/statistics/summary`, undefined, undefined, req, "gremlin").then((res) => res) data = await response.json(); res.send(data); diff --git a/packages/graph-explorer/src/connector/AbstractConnector.ts b/packages/graph-explorer/src/connector/AbstractConnector.ts index 5f823a6fa..e361bac5a 100644 --- a/packages/graph-explorer/src/connector/AbstractConnector.ts +++ b/packages/graph-explorer/src/connector/AbstractConnector.ts @@ -8,6 +8,7 @@ import type { export type QueryOptions = { abortSignal?: AbortSignal; + disableCache?: boolean; }; export type VertexSchemaResponse = Pick< @@ -18,6 +19,13 @@ export type VertexSchemaResponse = Pick< | "displayNameAttribute" | "longDisplayNameAttribute" > & { + total?: number; +}; + +export type CountsByTypeRequest = { + label: string; +}; +export type CountsByTypeResponse = { total: number; }; @@ -25,14 +33,22 @@ export type EdgeSchemaResponse = Pick< EdgeTypeConfig, "type" | "displayLabel" | "attributes" > & { - total: number; + total?: number; }; export type SchemaResponse = { + /** + * Total number of vertices. + */ + totalVertices?: number; /** * List of vertices definitions. */ vertices: VertexSchemaResponse[]; + /** + * Total number of edges. + */ + totalEdges?: number; /** * List of edges definitions. */ @@ -189,8 +205,6 @@ export abstract class AbstractConnector { protected abstract readonly basePath: string; - private readonly _requestCache: Map = new Map(); - public constructor(connection: ConfigurationWithConnection["connection"]) { if (!connection?.url) { throw new Error("Invalid configuration. Missing 'connection.url'"); @@ -204,6 +218,14 @@ export abstract class AbstractConnector { */ public abstract fetchSchema(options?: QueryOptions): Promise; + /** + * Count the number of vertices of a given type + */ + public abstract fetchVertexCountsByType( + req: CountsByTypeRequest, + options?: QueryOptions + ): Promise; + /** * Fetch all directly connected neighbors of a given source * filtered by vertex or edge types and/or vertex attributes. @@ -233,15 +255,23 @@ export abstract class AbstractConnector { /** * This method performs requests and cache their responses. */ - protected async request( + protected async requestQueryTemplate( queryTemplate: string, - options?: { signal?: AbortSignal } - ): Promise { - const url = this._connection.url.replace(/\/$/, ""); + options?: { signal?: AbortSignal; disableCache?: boolean } + ): Promise { const encodedQuery = encodeURIComponent(queryTemplate); - const uri = `${url}${this.basePath}${encodedQuery}&format=json`; + const uri = `${this.basePath}${encodedQuery}&format=json`; + return this.request(uri, options); + } + + protected async request( + uri: string, + options?: { signal?: AbortSignal; disableCache?: boolean } + ): Promise { + const url = this._connection.url.replace(/\/$/, ""); + const currentUri = `${url}${uri}`; - const cachedResponse = await this._getFromCache(uri); + const cachedResponse = await this._getFromCache(currentUri); if ( cachedResponse && cachedResponse.updatedAt + @@ -251,7 +281,7 @@ export abstract class AbstractConnector { return cachedResponse.data; } - return this._requestAndCache(uri, { + return this._requestAndCache(currentUri, { signal: options?.signal, headers: this._getAuthHeaders(), }); @@ -269,10 +299,16 @@ export abstract class AbstractConnector { return headers; } - private async _requestAndCache(url: string, init?: RequestInit) { + private async _requestAndCache( + url: string, + init?: RequestInit, + options?: Pick + ) { const response = await fetch(url, init); const data = await response.json(); - this._setToCache(url, { data, updatedAt: new Date().getTime() }); + if (options?.disableCache !== true) { + this._setToCache(url, { data, updatedAt: new Date().getTime() }); + } return data as TResult; } diff --git a/packages/graph-explorer/src/connector/gremlin/GremlinConnector.ts b/packages/graph-explorer/src/connector/gremlin/GremlinConnector.ts index ce8bd5130..4ff77aea3 100644 --- a/packages/graph-explorer/src/connector/gremlin/GremlinConnector.ts +++ b/packages/graph-explorer/src/connector/gremlin/GremlinConnector.ts @@ -1,4 +1,6 @@ import type { + CountsByTypeRequest, + CountsByTypeResponse, KeywordSearchRequest, KeywordSearchResponse, NeighborsCountRequest, @@ -12,13 +14,36 @@ import { AbstractConnector } from "../AbstractConnector"; import fetchNeighbors from "./queries/fetchNeighbors"; import fetchNeighborsCount from "./queries/fetchNeighborsCount"; import fetchSchema from "./queries/fetchSchema"; +import fetchVertexTypeCounts from "./queries/fetchVertexTypeCounts"; import keywordSearch from "./queries/keywordSearch"; +import { GraphSummary } from "./types"; export default class GremlinConnector extends AbstractConnector { protected readonly basePath = "/?gremlin="; + private readonly _summaryPath = "/pg/statistics/summary?mode=detailed"; - fetchSchema(options?: QueryOptions): Promise { - return fetchSchema(this._gremlinFetch(options)); + async fetchSchema(options?: QueryOptions): Promise { + const ops = { ...options, disableCache: true }; + let summary: GraphSummary | undefined; + try { + const response = await this.request<{ + payload: { graphSummary: GraphSummary }; + }>(this._summaryPath, ops); + summary = response.payload.graphSummary; + } catch (e) { + if (import.meta.env.DEV) { + console.error("[Summary API]", e); + } + } + + return fetchSchema(this._gremlinFetch(ops), summary); + } + + fetchVertexCountsByType( + req: CountsByTypeRequest, + options: QueryOptions | undefined + ): Promise { + return fetchVertexTypeCounts(this._gremlinFetch(options), req); } fetchNeighbors( @@ -44,8 +69,9 @@ export default class GremlinConnector extends AbstractConnector { private _gremlinFetch(options?: QueryOptions) { return async (queryTemplate: string) => { - return super.request(queryTemplate, { + return super.requestQueryTemplate(queryTemplate, { signal: options?.abortSignal, + disableCache: options?.disableCache, }); }; } diff --git a/packages/graph-explorer/src/connector/gremlin/queries/fetchSchema.ts b/packages/graph-explorer/src/connector/gremlin/queries/fetchSchema.ts index 577f448a7..6cc389c03 100644 --- a/packages/graph-explorer/src/connector/gremlin/queries/fetchSchema.ts +++ b/packages/graph-explorer/src/connector/gremlin/queries/fetchSchema.ts @@ -5,7 +5,7 @@ import edgesSchemaTemplate from "../templates/edgesSchemaTemplate"; import vertexLabelsTemplate from "../templates/vertexLabelsTemplate"; import verticesSchemaTemplate from "../templates/verticesSchemaTemplate"; import type { GEdge, GInt64, GVertex } from "../types"; -import { GremlinFetch } from "../types"; +import { GraphSummary, GremlinFetch } from "../types"; type RawVertexLabelsResponse = { requestId: string; @@ -106,12 +106,12 @@ const TYPE_MAP = { "g:Float": "Number", }; -const fetchVerticesSchema = async ( - gremlinFetch: GremlinFetch +const fetchVerticesAttributes = async ( + gremlinFetch: GremlinFetch, + labels: Array, + countsByLabel: Record ): Promise => { - const allLabels = await fetchVertexLabels(gremlinFetch); const vertices: SchemaResponse["vertices"] = []; - const labels = Object.keys(allLabels); if (labels.length === 0) { return vertices; @@ -133,7 +133,7 @@ const fetchVerticesSchema = async ( vertices.push({ type: label, displayLabel: sanitizeText(label), - total: allLabels[label], + total: countsByLabel[label], attributes: Object.entries(properties || {}).map(([name, prop]) => { const value = prop[0]?.["@value"].value; return { @@ -151,6 +151,15 @@ const fetchVerticesSchema = async ( return vertices; }; +const fetchVerticesSchema = async ( + gremlinFetch: GremlinFetch +): Promise => { + const countsByLabel = await fetchVertexLabels(gremlinFetch); + const labels = Object.keys(countsByLabel); + + return fetchVerticesAttributes(gremlinFetch, labels, countsByLabel); +}; + const fetchEdgeLabels = async ( gremlinFetch: GremlinFetch ): Promise> => { @@ -166,13 +175,12 @@ const fetchEdgeLabels = async ( return labelsWithCounts; }; -const fetchEdgesSchema = async ( - gremlinFetch: GremlinFetch +const fetchEdgesAttributes = async ( + gremlinFetch: GremlinFetch, + labels: Array, + countsByLabel: Record ): Promise => { - const allLabels = await fetchEdgeLabels(gremlinFetch); const edges: SchemaResponse["edges"] = []; - const labels = Object.keys(allLabels); - if (labels.length === 0) { return edges; } @@ -191,7 +199,7 @@ const fetchEdgesSchema = async ( edges.push({ type: label, displayLabel: sanitizeText(label), - total: allLabels[label], + total: countsByLabel[label], attributes: Object.entries(properties || {}).map(([name, prop]) => { const value = prop["@value"].value; return { @@ -206,6 +214,15 @@ const fetchEdgesSchema = async ( return edges; }; +const fetchEdgesSchema = async ( + gremlinFetch: GremlinFetch +): Promise => { + const countsByLabel = await fetchEdgeLabels(gremlinFetch); + const labels = Object.keys(countsByLabel); + + return fetchEdgesAttributes(gremlinFetch, labels, countsByLabel); +}; + /** * Fetch the database shape. * It follows this process: @@ -218,13 +235,43 @@ const fetchEdgesSchema = async ( * nodes/edges with the same label contains an exact set of attributes. */ const fetchSchema = async ( - gremlinFetch: GremlinFetch + gremlinFetch: GremlinFetch, + summary?: GraphSummary ): Promise => { - const vertices = await fetchVerticesSchema(gremlinFetch); - const edges = await fetchEdgesSchema(gremlinFetch); + if (!summary) { + const vertices = await fetchVerticesSchema(gremlinFetch); + const totalVertices = vertices.reduce((total, vertex) => { + return total + (vertex.total ?? 0); + }, 0); + + const edges = await fetchEdgesSchema(gremlinFetch); + const totalEdges = edges.reduce((total, edge) => { + return total + (edge.total ?? 0); + }, 0); + + return { + totalVertices, + vertices, + totalEdges, + edges, + }; + } + + const vertices = await fetchVerticesAttributes( + gremlinFetch, + summary.nodeLabels, + {} + ); + const edges = await fetchEdgesAttributes( + gremlinFetch, + summary.edgeLabels, + {} + ); return { + totalVertices: summary.numNodes, vertices, + totalEdges: summary.numEdges, edges, }; }; diff --git a/packages/graph-explorer/src/connector/gremlin/queries/fetchVertexTypeCounts.ts b/packages/graph-explorer/src/connector/gremlin/queries/fetchVertexTypeCounts.ts new file mode 100644 index 000000000..cfa1fe1a3 --- /dev/null +++ b/packages/graph-explorer/src/connector/gremlin/queries/fetchVertexTypeCounts.ts @@ -0,0 +1,33 @@ +import { + CountsByTypeRequest, + CountsByTypeResponse, +} from "../../AbstractConnector"; +import vertexTypeCountTemplate from "../templates/vertexTypeCountTemplate"; +import { GInt64, GremlinFetch } from "../types"; + +type RawCountsByTypeResponse = { + requestId: string; + status: { + message: string; + code: number; + }; + result: { + data: { + "@type": "g:List"; + "@value": Array; + }; + }; +}; + +const fetchVertexTypeCounts = async ( + gremlinFetch: GremlinFetch, + req: CountsByTypeRequest +): Promise => { + const template = vertexTypeCountTemplate(req.label); + const response = await gremlinFetch(template); + return { + total: response.result.data["@value"][0]["@value"], + }; +}; + +export default fetchVertexTypeCounts; diff --git a/packages/graph-explorer/src/connector/gremlin/templates/vertexTypeCountTemplate.ts b/packages/graph-explorer/src/connector/gremlin/templates/vertexTypeCountTemplate.ts new file mode 100644 index 000000000..4f24b5aa9 --- /dev/null +++ b/packages/graph-explorer/src/connector/gremlin/templates/vertexTypeCountTemplate.ts @@ -0,0 +1,8 @@ +/** + * It returns a Gremlin template to number of vertices of a particular label + */ +const vertexTypeCountTemplate = (label: string) => { + return `g.V().hasLabel("${label}").count()`; +}; + +export default vertexTypeCountTemplate; diff --git a/packages/graph-explorer/src/connector/gremlin/types.ts b/packages/graph-explorer/src/connector/gremlin/types.ts index d717c40cc..0f080e4aa 100644 --- a/packages/graph-explorer/src/connector/gremlin/types.ts +++ b/packages/graph-explorer/src/connector/gremlin/types.ts @@ -71,3 +71,18 @@ export type GEdgeList = { export type GremlinFetch = ( queryTemplate: string ) => Promise; + +export type GraphSummary = { + numNodes: number; + numEdges: number; + numNodeLabels: number; + numEdgeLabels: number; + nodeLabels: Array; + edgeLabels: Array; + numNodeProperties: number; + numEdgeProperties: number; + nodeProperties: Record; + edgeProperties: Record; + totalNodePropertyValues: number; + totalEdgePropertyValues: number; +}; diff --git a/packages/graph-explorer/src/connector/sparql/SPARQLConnector.ts b/packages/graph-explorer/src/connector/sparql/SPARQLConnector.ts index 1703f4f96..22fdeee8f 100644 --- a/packages/graph-explorer/src/connector/sparql/SPARQLConnector.ts +++ b/packages/graph-explorer/src/connector/sparql/SPARQLConnector.ts @@ -1,4 +1,6 @@ import type { + CountsByTypeRequest, + CountsByTypeResponse, KeywordSearchResponse, NeighborsCountRequest, NeighborsCountResponse, @@ -12,20 +14,43 @@ import { QueryOptions, } from "../AbstractConnector"; import fetchBlankNodeNeighbors from "./queries/fetchBlankNodeNeighbors"; +import fetchClassCounts from "./queries/fetchClassCounts"; import fetchNeighbors from "./queries/fetchNeighbors"; import fetchNeighborsCount from "./queries/fetchNeighborsCount"; import fetchSchema from "./queries/fetchSchema"; import keywordSearch from "./queries/keywordSearch"; import keywordSearchBlankNodesIdsTemplate from "./templates/keywordSearch/keywordSearchBlankNodesIdsTemplate"; import oneHopNeighborsBlankNodesIdsTemplate from "./templates/oneHopNeighbors/oneHopNeighborsBlankNodesIdsTemplate"; -import { BlankNodesMap, SPARQLNeighborsRequest } from "./types"; +import { BlankNodesMap, GraphSummary, SPARQLNeighborsRequest } from "./types"; export default class SPARQLConnector extends AbstractConnector { protected readonly basePath = "/sparql?query="; + private readonly _summaryPath = "/rdf/statistics/summary?mode=detailed"; + private _blankNodes: BlankNodesMap = new Map(); - fetchSchema(options?: QueryOptions): Promise { - return fetchSchema(this._sparqlFetch(options)); + async fetchSchema(options?: QueryOptions): Promise { + const ops = { ...options, disableCache: true }; + let summary: GraphSummary | undefined; + try { + const response = await this.request<{ + payload: { graphSummary: GraphSummary }; + }>(this._summaryPath, ops); + summary = response.payload.graphSummary; + } catch (e) { + if (import.meta.env.DEV) { + console.error("[Summary API]", e); + } + } + + return fetchSchema(this._sparqlFetch(ops), summary); + } + + fetchVertexCountsByType( + req: CountsByTypeRequest, + options?: QueryOptions + ): Promise { + return fetchClassCounts(this._sparqlFetch(options), req); } async fetchNeighbors( @@ -122,8 +147,9 @@ export default class SPARQLConnector extends AbstractConnector { private _sparqlFetch(options?: QueryOptions) { return async (queryTemplate: string) => { - return super.request(queryTemplate, { + return super.requestQueryTemplate(queryTemplate, { signal: options?.abortSignal, + disableCache: options?.disableCache, }); }; } diff --git a/packages/graph-explorer/src/connector/sparql/queries/fetchClassCounts.ts b/packages/graph-explorer/src/connector/sparql/queries/fetchClassCounts.ts new file mode 100644 index 000000000..43b5beb62 --- /dev/null +++ b/packages/graph-explorer/src/connector/sparql/queries/fetchClassCounts.ts @@ -0,0 +1,30 @@ +import { + CountsByTypeRequest, + CountsByTypeResponse, +} from "../../AbstractConnector"; +import classWithCountsTemplates from "../templates/classWithCountsTemplates"; +import { RawValue, SparqlFetch } from "../types"; + +type RawCountsByTypeResponse = { + results: { + bindings: [ + { + instancesCount: RawValue; + } + ]; + }; +}; + +const fetchClassCounts = async ( + sparqlFetch: SparqlFetch, + req: CountsByTypeRequest +): Promise => { + const template = classWithCountsTemplates(req.label); + const response = await sparqlFetch(template); + + return { + total: Number(response.results.bindings[0].instancesCount.value), + }; +}; + +export default fetchClassCounts; diff --git a/packages/graph-explorer/src/connector/sparql/queries/fetchSchema.ts b/packages/graph-explorer/src/connector/sparql/queries/fetchSchema.ts index fc3abd0cd..8fb12cf3b 100644 --- a/packages/graph-explorer/src/connector/sparql/queries/fetchSchema.ts +++ b/packages/graph-explorer/src/connector/sparql/queries/fetchSchema.ts @@ -2,7 +2,7 @@ import { SchemaResponse } from "../../AbstractConnector"; import classesWithCountsTemplates from "../templates/classesWithCountsTemplates"; import predicatesByClassTemplate from "../templates/predicatesByClassTemplate"; import predicatesWithCountsTemplate from "../templates/predicatesWithCountsTemplate"; -import { RawValue, SparqlFetch } from "../types"; +import { GraphSummary, RawValue, SparqlFetch } from "../types"; type RawClassesWCountsResponse = { results: { @@ -57,17 +57,16 @@ const skosNote = "http://www.w3.org/2004/02/skos/core#note"; const skosDefinition = "http://www.w3.org/2004/02/skos/core#definition"; const displayDescCandidates = [rdfsComment, skosNote, skosDefinition]; -const fetchClassesSchema = async (sparqlFetch: SparqlFetch) => { - const classesTemplate = classesWithCountsTemplates(); - const classesCounts = await sparqlFetch( - classesTemplate - ); - +const fetchPredicatesByClass = async ( + sparqlFetch: SparqlFetch, + classes: Array, + countsByClass: Record +) => { const vertices: SchemaResponse["vertices"] = []; await Promise.all( - classesCounts.results.bindings.map(async classResult => { + classes.map(async classResult => { const classPredicatesTemplate = predicatesByClassTemplate({ - class: classResult.class.value, + class: classResult, }); const predicatesResponse = await sparqlFetch< RawPredicatesSamplesResponse @@ -82,9 +81,9 @@ const fetchClassesSchema = async (sparqlFetch: SparqlFetch) => { })); vertices.push({ - type: classResult.class.value, + type: classResult, displayLabel: "", - total: Number(classResult.instancesCount.value), + total: countsByClass[classResult], displayNameAttribute: attributes.find(attr => displayNameCandidates.includes(attr.name)) ?.name || "id", @@ -99,6 +98,24 @@ const fetchClassesSchema = async (sparqlFetch: SparqlFetch) => { return vertices; }; +const fetchClassesSchema = async (sparqlFetch: SparqlFetch) => { + const classesTemplate = classesWithCountsTemplates(); + const classesCounts = await sparqlFetch( + classesTemplate + ); + + const classes: Array = []; + const countsByClass: Record = {}; + classesCounts.results.bindings.forEach(classResult => { + classes.push(classResult.class.value); + countsByClass[classResult.class.value] = Number( + classResult.instancesCount.value + ); + }); + + return fetchPredicatesByClass(sparqlFetch, classes, countsByClass); +}; + const fetchPredicatesWithCounts = async ( sparqlFetch: SparqlFetch ): Promise> => { @@ -138,13 +155,48 @@ const fetchPredicatesSchema = async (sparqlFetch: SparqlFetch) => { * 4. Generate prefixes using the received URIs */ const fetchSchema = async ( - sparqlFetch: SparqlFetch + sparqlFetch: SparqlFetch, + summary?: GraphSummary ): Promise => { - const vertices = await fetchClassesSchema(sparqlFetch); - const edges = await fetchPredicatesSchema(sparqlFetch); + if (!summary) { + const vertices = await fetchClassesSchema(sparqlFetch); + const totalVertices = vertices.reduce((total, vertex) => { + return total + (vertex.total ?? 0); + }, 0); + + const edges = await fetchPredicatesSchema(sparqlFetch); + const totalEdges = edges.reduce((total, edge) => { + return total + (edge.total ?? 0); + }, 0); + + return { + totalVertices, + vertices, + totalEdges, + edges, + }; + } + + const vertices = await fetchPredicatesByClass( + sparqlFetch, + summary.classes, + {} + ); + const edges = summary.predicates.flatMap(pred => { + return Object.entries(pred).map(([type, count]) => { + return { + type, + displayLabel: "", + total: count, + attributes: [], + }; + }); + }); return { + totalVertices: summary.numDistinctSubjects, vertices, + totalEdges: summary.numQuads, edges, }; }; diff --git a/packages/graph-explorer/src/connector/sparql/templates/classWithCountsTemplates.ts b/packages/graph-explorer/src/connector/sparql/templates/classWithCountsTemplates.ts new file mode 100644 index 000000000..08e4d995c --- /dev/null +++ b/packages/graph-explorer/src/connector/sparql/templates/classWithCountsTemplates.ts @@ -0,0 +1,10 @@ +// It returns the number of instances of the given class +const classWithCountsTemplates = (className: string) => { + return ` + SELECT (COUNT(?start) AS ?instancesCount) { + ?start a <${className}> + } + `; +}; + +export default classWithCountsTemplates; diff --git a/packages/graph-explorer/src/connector/sparql/types.ts b/packages/graph-explorer/src/connector/sparql/types.ts index 68e4ef4ca..e1d7aaab3 100644 --- a/packages/graph-explorer/src/connector/sparql/types.ts +++ b/packages/graph-explorer/src/connector/sparql/types.ts @@ -149,4 +149,14 @@ export type BlankNodeItem = { edges: Array; }; }; + export type BlankNodesMap = Map; + +export type GraphSummary = { + numDistinctSubjects: number; + numDistinctPredicates: number; + numQuads: number; + numClasses: number; + classes: Array; + predicates: Array>; +}; diff --git a/packages/graph-explorer/src/core/ConfigurationProvider/ConfigurationProvider.tsx b/packages/graph-explorer/src/core/ConfigurationProvider/ConfigurationProvider.tsx index 5d848d421..45f0d2755 100644 --- a/packages/graph-explorer/src/core/ConfigurationProvider/ConfigurationProvider.tsx +++ b/packages/graph-explorer/src/core/ConfigurationProvider/ConfigurationProvider.tsx @@ -102,7 +102,9 @@ const ConfigurationProvider = ({ return { ...(configuration || {}), + totalVertices: configuration.schema?.totalVertices ?? 0, vertexTypes: configuration.schema?.vertices?.map(vt => vt.type) || [], + totalEdges: configuration.schema?.totalEdges ?? 0, edgeTypes: configuration.schema?.edges?.map(et => et.type) || [], getVertexTypeConfig, getVertexTypeAttributes, diff --git a/packages/graph-explorer/src/core/ConfigurationProvider/types.ts b/packages/graph-explorer/src/core/ConfigurationProvider/types.ts index 303666160..f2b422398 100644 --- a/packages/graph-explorer/src/core/ConfigurationProvider/types.ts +++ b/packages/graph-explorer/src/core/ConfigurationProvider/types.ts @@ -117,7 +117,7 @@ export type ConnectionConfig = { * Choose between gremlin or sparQL engines. * By default, it uses gremlin */ - queryEngine?: "gremlin" | "sparql"; + queryEngine?: "gremlin" | "sparql" | "open-cypher"; /** * If the service is Neptune, * all requests should be sent through the nodejs proxy-server. @@ -168,7 +168,9 @@ export type RawConfiguration = { * Database schema: types, names, labels, icons, ... */ schema?: { + totalVertices: number; vertices: Array; + totalEdges: number; edges: Array; lastUpdate?: Date; triedToSync?: boolean; @@ -185,7 +187,9 @@ export type RawConfiguration = { }; export type ConfigurationContextProps = RawConfiguration & { + totalVertices: number; vertexTypes: Array; + totalEdges: number; edgeTypes: Array; getVertexTypeConfig(vertexType: string): VertexTypeConfig | undefined; getVertexTypeAttributes(vertexTypes: string[]): Array; diff --git a/packages/graph-explorer/src/hooks/useSchemaSync.ts b/packages/graph-explorer/src/hooks/useSchemaSync.ts index 9065ad2f3..0a12c0d4e 100644 --- a/packages/graph-explorer/src/hooks/useSchemaSync.ts +++ b/packages/graph-explorer/src/hooks/useSchemaSync.ts @@ -1,11 +1,10 @@ import { useCallback, useRef } from "react"; -import { useRecoilCallback } from "recoil"; import { useNotification } from "../components/NotificationProvider"; import { SchemaResponse } from "../connector/AbstractConnector"; import useConfiguration from "../core/ConfigurationProvider/useConfiguration"; import useConnector from "../core/ConnectorProvider/useConnector"; -import { schemaAtom } from "../core/StateProvider/schema"; import usePrefixesUpdater from "./usePrefixesUpdater"; +import useUpdateSchema from "./useUpdateSchema"; const useSchemaSync = (onSyncChange?: (isSyncing: boolean) => void) => { const config = useConfiguration(); @@ -15,26 +14,7 @@ const useSchemaSync = (onSyncChange?: (isSyncing: boolean) => void) => { const { enqueueNotification, clearNotification } = useNotification(); const notificationId = useRef(null); - const updateSchemaState = useRecoilCallback( - ({ set }) => (id: string, schema?: SchemaResponse) => { - set(schemaAtom, prevSchemaMap => { - const updatedSchema = new Map(prevSchemaMap); - const prevSchema = prevSchemaMap.get(id); - - updatedSchema.set(id, { - vertices: schema?.vertices || prevSchema?.vertices || [], - edges: schema?.edges || prevSchema?.edges || [], - prefixes: prevSchema?.prefixes || [], - lastUpdate: !schema ? prevSchema?.lastUpdate : new Date(), - triedToSync: true, - lastSyncFail: !schema && !!prevSchema, - }); - return updatedSchema; - }); - }, - [] - ); - + const updateSchemaState = useUpdateSchema(); return useCallback( async (abortSignal?: AbortSignal) => { if (!config || !connector.explorer) { diff --git a/packages/graph-explorer/src/hooks/useUpdateSchema.ts b/packages/graph-explorer/src/hooks/useUpdateSchema.ts new file mode 100644 index 000000000..04bff6f65 --- /dev/null +++ b/packages/graph-explorer/src/hooks/useUpdateSchema.ts @@ -0,0 +1,64 @@ +import { useRecoilCallback } from "recoil"; +import { SchemaResponse } from "../connector/AbstractConnector"; +import { schemaAtom } from "../core/StateProvider/schema"; + +const useUpdateSchema = () => { + return useRecoilCallback( + ({ set }) => ( + id: string, + schema?: + | Partial + | ((prevSchema?: SchemaResponse) => Partial) + ) => { + set(schemaAtom, prevSchemaMap => { + const updatedSchema = new Map(prevSchemaMap); + const prevSchema = prevSchemaMap.get(id); + + const currentSchema = + typeof schema === "function" ? schema(prevSchema) : schema; + + // Preserve vertices counts + const vertices = ( + currentSchema?.vertices || + prevSchema?.vertices || + [] + ).map(vertex => { + const prevVertex = prevSchema?.vertices.find( + v => v.type === vertex.type + ); + return { + ...vertex, + total: vertex.total ?? prevVertex?.total, + }; + }); + + // Preserve edges counts + const edges = (currentSchema?.edges || prevSchema?.edges || []).map( + edge => { + const prevEdge = prevSchema?.edges.find(e => e.type === edge.type); + return { + ...edge, + total: edge.total ?? prevEdge?.total, + }; + } + ); + + updatedSchema.set(id, { + totalVertices: + currentSchema?.totalVertices || prevSchema?.totalVertices || 0, + vertices, + totalEdges: currentSchema?.totalEdges || prevSchema?.totalEdges || 0, + edges, + prefixes: prevSchema?.prefixes || [], + lastUpdate: !currentSchema ? prevSchema?.lastUpdate : new Date(), + triedToSync: true, + lastSyncFail: !currentSchema && !!prevSchema, + }); + return updatedSchema; + }); + }, + [] + ); +}; + +export default useUpdateSchema; diff --git a/packages/graph-explorer/src/hooks/useUpdateVertexTypeCounts.ts b/packages/graph-explorer/src/hooks/useUpdateVertexTypeCounts.ts new file mode 100644 index 000000000..4563d3e39 --- /dev/null +++ b/packages/graph-explorer/src/hooks/useUpdateVertexTypeCounts.ts @@ -0,0 +1,61 @@ +import { useMemo } from "react"; +import { useQuery } from "react-query"; +import { useConfiguration } from "../core"; +import useConnector from "../core/ConnectorProvider/useConnector"; +import useUpdateSchema from "./useUpdateSchema"; + +const useUpdateVertexTypeCounts = (vertexType?: string) => { + const config = useConfiguration(); + const connector = useConnector(); + + const vertexConfig = useMemo(() => { + if (!vertexType) { + return; + } + + return config?.getVertexTypeConfig(vertexType); + }, [config, vertexType]); + + const updateSchemaState = useUpdateSchema(); + useQuery( + ["fetchCountsByType", vertexConfig?.type], + () => { + if (vertexConfig?.total != null || vertexConfig?.type == null) { + return; + } + + return connector.explorer?.fetchVertexCountsByType({ + label: vertexConfig?.type, + }); + }, + { + enabled: vertexConfig?.total == null && vertexConfig?.type != null, + onSuccess: response => { + if (!config?.id || !response) { + return; + } + + updateSchemaState(config.id, prevSchema => { + const vertexSchema = prevSchema?.vertices.find( + vertex => vertex.type === vertexType + ); + if (!vertexSchema) { + return { ...(prevSchema || {}) }; + } + + vertexSchema.total = response.total; + return { + vertices: [ + ...(prevSchema?.vertices.filter( + vertex => vertex.type !== vertexType + ) || []), + vertexSchema, + ], + }; + }); + }, + } + ); +}; + +export default useUpdateVertexTypeCounts; diff --git a/packages/graph-explorer/src/workspaces/DataExplorer/DataExplorer.tsx b/packages/graph-explorer/src/workspaces/DataExplorer/DataExplorer.tsx index 1d379832f..1c3c265b4 100644 --- a/packages/graph-explorer/src/workspaces/DataExplorer/DataExplorer.tsx +++ b/packages/graph-explorer/src/workspaces/DataExplorer/DataExplorer.tsx @@ -45,6 +45,7 @@ import useFetchNode from "../../hooks/useFetchNode"; import usePrefixesUpdater from "../../hooks/usePrefixesUpdater"; import useTextTransform from "../../hooks/useTextTransform"; import useTranslations from "../../hooks/useTranslations"; +import useUpdateVertexTypeCounts from "../../hooks/useUpdateVertexTypeCounts"; import TopBarWithLogo from "../common/TopBarWithLogo"; import defaultStyles from "./DataExplorer.styles"; @@ -69,6 +70,9 @@ const DataExplorer = ({ classNamePrefix = "ft" }: ConnectionsProps) => { const fetchNode = useFetchNode(); const [entities] = useEntities({ disableFilters: true }); + // Automatically updates counts if needed + useUpdateVertexTypeCounts(vertexType); + const vertexConfig = useMemo(() => { if (!vertexType) { return; From 0aa5625268a35fd6b4cbac7230eedafd6f0c2d37 Mon Sep 17 00:00:00 2001 From: nestoralvarezd Date: Thu, 27 Apr 2023 17:06:14 +0100 Subject: [PATCH 2/4] Update counts badges in ConnectionData --- .../src/core/ConfigurationProvider/types.ts | 2 +- .../src/core/StateProvider/configuration.ts | 2 ++ packages/graph-explorer/src/core/StateProvider/schema.ts | 2 ++ packages/graph-explorer/src/hooks/useEntitiesCounts.ts | 8 ++++++++ .../src/modules/ConnectionDetail/ConnectionData.tsx | 6 ++---- 5 files changed, 15 insertions(+), 5 deletions(-) diff --git a/packages/graph-explorer/src/core/ConfigurationProvider/types.ts b/packages/graph-explorer/src/core/ConfigurationProvider/types.ts index f2b422398..9ff6c1b80 100644 --- a/packages/graph-explorer/src/core/ConfigurationProvider/types.ts +++ b/packages/graph-explorer/src/core/ConfigurationProvider/types.ts @@ -117,7 +117,7 @@ export type ConnectionConfig = { * Choose between gremlin or sparQL engines. * By default, it uses gremlin */ - queryEngine?: "gremlin" | "sparql" | "open-cypher"; + queryEngine?: "gremlin" | "sparql"; /** * If the service is Neptune, * all requests should be sent through the nodejs proxy-server. diff --git a/packages/graph-explorer/src/core/StateProvider/configuration.ts b/packages/graph-explorer/src/core/StateProvider/configuration.ts index 90b090133..199ce03d0 100644 --- a/packages/graph-explorer/src/core/StateProvider/configuration.ts +++ b/packages/graph-explorer/src/core/StateProvider/configuration.ts @@ -99,6 +99,8 @@ export const mergedConfigurationSelector = selector({ : undefined, triedToSync: currentSchema?.triedToSync, lastSyncFail: currentSchema?.lastSyncFail, + totalVertices: currentSchema?.totalVertices ?? 0, + totalEdges: currentSchema?.totalEdges ?? 0, }, }; }, diff --git a/packages/graph-explorer/src/core/StateProvider/schema.ts b/packages/graph-explorer/src/core/StateProvider/schema.ts index 1e5ce0be9..5c9668f49 100644 --- a/packages/graph-explorer/src/core/StateProvider/schema.ts +++ b/packages/graph-explorer/src/core/StateProvider/schema.ts @@ -13,6 +13,8 @@ export type SchemaInference = { lastUpdate?: Date; triedToSync?: boolean; lastSyncFail?: boolean; + totalVertices?: number; + totalEdges?: number; }; export const schemaAtom = atom>({ diff --git a/packages/graph-explorer/src/hooks/useEntitiesCounts.ts b/packages/graph-explorer/src/hooks/useEntitiesCounts.ts index 3d81c38fb..1512e465c 100644 --- a/packages/graph-explorer/src/hooks/useEntitiesCounts.ts +++ b/packages/graph-explorer/src/hooks/useEntitiesCounts.ts @@ -5,6 +5,10 @@ const useEntitiesCounts = () => { const config = useConfiguration(); const totalNodes = useMemo(() => { + if (config?.totalVertices != null) { + return config?.totalVertices; + } + if (!config?.vertexTypes.length) { return null; } @@ -24,6 +28,10 @@ const useEntitiesCounts = () => { }, [config]); const totalEdges = useMemo(() => { + if (config?.totalEdges != null) { + return config?.totalEdges; + } + if (!config?.edgeTypes.length) { return null; } diff --git a/packages/graph-explorer/src/modules/ConnectionDetail/ConnectionData.tsx b/packages/graph-explorer/src/modules/ConnectionDetail/ConnectionData.tsx index 7a158fdad..2af0535f4 100644 --- a/packages/graph-explorer/src/modules/ConnectionDetail/ConnectionData.tsx +++ b/packages/graph-explorer/src/modules/ConnectionDetail/ConnectionData.tsx @@ -47,11 +47,9 @@ const ConnectionData = ({ classNamePrefix = "ft" }: VertexDetailProps) => { {textTransform(displayLabel)}
- {t("connection-detail.nodes")}:{" "} - {vtConfig?.total ? ( + {t("connection-detail.nodes")}: {vtConfig?.total == null && "~"} + {vtConfig?.total != null && ( - ) : ( - "Unknown" )}
From d607b12f9002dac82527fe15ff8749b0ab39e686 Mon Sep 17 00:00:00 2001 From: nestoralvarezd Date: Thu, 4 May 2023 17:05:48 +0100 Subject: [PATCH 3/4] Remove individual counts from ConnectionData --- .../src/modules/ConnectionDetail/ConnectionData.tsx | 6 ------ 1 file changed, 6 deletions(-) diff --git a/packages/graph-explorer/src/modules/ConnectionDetail/ConnectionData.tsx b/packages/graph-explorer/src/modules/ConnectionDetail/ConnectionData.tsx index 2af0535f4..26de7c874 100644 --- a/packages/graph-explorer/src/modules/ConnectionDetail/ConnectionData.tsx +++ b/packages/graph-explorer/src/modules/ConnectionDetail/ConnectionData.tsx @@ -46,12 +46,6 @@ const ConnectionData = ({ classNamePrefix = "ft" }: VertexDetailProps) => {
{textTransform(displayLabel)}
-
- {t("connection-detail.nodes")}: {vtConfig?.total == null && "~"} - {vtConfig?.total != null && ( - - )} -
), icon: ( From a9e82822e5dab9185a7a966102333b253d9d9bee Mon Sep 17 00:00:00 2001 From: nestoralvarezd Date: Thu, 4 May 2023 17:18:00 +0100 Subject: [PATCH 4/4] Remove individual counts from ConnectionData --- .../src/modules/ConnectionDetail/ConnectionData.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/graph-explorer/src/modules/ConnectionDetail/ConnectionData.tsx b/packages/graph-explorer/src/modules/ConnectionDetail/ConnectionData.tsx index 26de7c874..f6e6f6efd 100644 --- a/packages/graph-explorer/src/modules/ConnectionDetail/ConnectionData.tsx +++ b/packages/graph-explorer/src/modules/ConnectionDetail/ConnectionData.tsx @@ -79,7 +79,7 @@ const ConnectionData = ({ classNamePrefix = "ft" }: VertexDetailProps) => { }); return items; - }, [config, pfx, textTransform, t, navigate]); + }, [config, pfx, textTransform, navigate]); const [search, setSearch] = useState("");