diff --git a/Changelog.md b/Changelog.md index 6c9db7884..79f73eccf 100644 --- a/Changelog.md +++ b/Changelog.md @@ -3,6 +3,21 @@ ## Upcoming +## Release 1.2.0 + +This release includes the following feature enhancements and bug fixes: + +**Features** +- Significantly reduced size of Docker image (https://github.com/aws/graph-explorer/pull/104) +- Improved schema synchronization performance via Summary API integration (https://github.com/aws/graph-explorer/pull/80) +- Improved error messaging when no/insufficient IAM role is found (https://github.com/aws/graph-explorer/pull/81) +- Updated Connections UI documentation for single server changes (https://github.com/aws/graph-explorer/pull/59) +- Added manual trigger for ECR updates (https://github.com/aws/graph-explorer/pull/68) + +**Bug fixes** +- Fixed incorrect display of non-string IDs for Gremlin (https://github.com/aws/graph-explorer/pull/60) +- Fixed a database synchronization error caused by white spaces in labels for Gremlin requests (https://github.com/aws/graph-explorer/pull/84) + ## Release 1.1.0 This release includes the following feature enhancements and bug fixes: diff --git a/Dockerfile b/Dockerfile index 166ac6a1d..cdf20e00c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,12 +3,9 @@ FROM amazonlinux:2 WORKDIR / COPY . /graph-explorer/ WORKDIR /graph-explorer -RUN yum install -y curl -RUN curl -sL https://rpm.nodesource.com/setup_16.x | bash - -RUN yum install -y nodejs -RUN yum install -y openssl -RUN npm install -g pnpm -RUN pnpm install +# Keeping all the RUN commands on a single line reduces the number of layers and, +# as a result, significantly reduces the final image size. +RUN curl -sL https://rpm.nodesource.com/setup_16.x | bash - && yum install -y nodejs openssl && npm install -g pnpm && pnpm install && rm -rf /var/cache/yum WORKDIR /graph-explorer/ ENV HOME=/graph-explorer RUN pnpm build diff --git a/package.json b/package.json index 7d80c3e3b..b7b007ed2 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "graph-explorer", - "version": "1.1.0", + "version": "1.2.0", "description": "Graph Explorer", "packageManager": "pnpm@7.9.3", "engines": { diff --git a/packages/graph-explorer-proxy-server/node-server.js b/packages/graph-explorer-proxy-server/node-server.js index a5b48d77a..188fb7a26 100644 --- a/packages/graph-explorer-proxy-server/node-server.js +++ b/packages/graph-explorer-proxy-server/node-server.js @@ -32,7 +32,7 @@ const getCredentials = async () => { }; dotenv.config({ path: "../graph-explorer/.env" }); - + (async () => { let creds = await getCredentials(); let requestSig; @@ -64,23 +64,23 @@ dotenv.config({ path: "../graph-explorer/.env" }); await getAuthHeaders(language, req, reqObjects[0], reqObjects[2]); return new Promise((resolve, reject) => { - + const wrapper = (n) => { fetch(url, { headers: req.headers, }) .then(async res => { if (!res.ok) { - console.log("Response not ok"); + console.log("Response not ok"); const error = res.status return Promise.reject(error); } else { - console.log("Response ok"); + console.log("Response ok"); resolve(res); } }) .catch(async (err) => { console.log("Attempt Credential Refresh"); - creds = await getCredentials(); + creds = await getCredentials(); if (creds === undefined) { reject("Credentials undefined after credential refresh. Check that you have proper acccess") } @@ -119,7 +119,7 @@ dotenv.config({ path: "../graph-explorer/.env" }); } async function getAuthHeaders(language, req, endpoint_url, requestSig) { - let authHeaders + let authHeaders if (language == "sparql") { authHeaders = await requestSig.requestAuthHeaders( endpoint_url.port, @@ -147,9 +147,9 @@ dotenv.config({ path: "../graph-explorer/.env" }); let data; try { response = await retryFetch(`${req.headers["graph-db-connection-url"]}/sparql?query=` + - encodeURIComponent(req.query.query) + - "&format=json", undefined, undefined, req, "sparql").then((res) => res) - + encodeURIComponent(req.query.query) + + "&format=json", undefined, undefined, req, "sparql").then((res) => res) + data = await response.json(); res.send(data); } catch (error) { @@ -163,7 +163,35 @@ dotenv.config({ path: "../graph-explorer/.env" }); let data; try { response = await retryFetch(`${req.headers["graph-db-connection-url"]}/?gremlin=` + - encodeURIComponent(req.query.gremlin), undefined, undefined, req, "gremlin").then((res) => res) + encodeURIComponent(req.query.gremlin), undefined, undefined, req, "gremlin").then((res) => res) + + data = await response.json(); + res.send(data); + } catch (error) { + next(error); + console.log(error); + } + }); + + app.get("/pg/statistics/summary", async (req, res, next) => { + let response; + let data; + try { + response = await retryFetch(`${req.headers["graph-db-connection-url"]}/pg/statistics/summary`, undefined, undefined, req, "gremlin").then((res) => res) + + data = await response.json(); + res.send(data); + } catch (error) { + next(error); + console.log(error); + } + }); + + app.get("/rdf/statistics/summary", async (req, res, next) => { + let response; + let data; + try { + response = await retryFetch(`${req.headers["graph-db-connection-url"]}/rdf/statistics/summary`, undefined, undefined, req, "gremlin").then((res) => res) data = await response.json(); res.send(data); diff --git a/packages/graph-explorer-proxy-server/package.json b/packages/graph-explorer-proxy-server/package.json index b6b8e9fd5..55adf61c2 100644 --- a/packages/graph-explorer-proxy-server/package.json +++ b/packages/graph-explorer-proxy-server/package.json @@ -1,6 +1,6 @@ { "name": "graph-explorer-proxy-server", - "version": "1.1.0", + "version": "1.2.0", "description": "Server to facilitate communication between the browser and the supported graph database.", "main": "node-server.js", "scripts": { diff --git a/packages/graph-explorer/package.json b/packages/graph-explorer/package.json index 68d33dcca..940dca671 100644 --- a/packages/graph-explorer/package.json +++ b/packages/graph-explorer/package.json @@ -1,6 +1,6 @@ { "name": "graph-explorer", - "version": "1.1.0", + "version": "1.2.0", "description": "Graph Explorer", "packageManager": "pnpm@7.9.3", "engines": { diff --git a/packages/graph-explorer/src/components/Graph/hooks/useRunLayout.ts b/packages/graph-explorer/src/components/Graph/hooks/useRunLayout.ts index bc9b98665..82728d76a 100755 --- a/packages/graph-explorer/src/components/Graph/hooks/useRunLayout.ts +++ b/packages/graph-explorer/src/components/Graph/hooks/useRunLayout.ts @@ -62,12 +62,15 @@ function useUpdateLayout({ previousNodesRef.current = nodes; previousLayoutRef.current = layout; } + // nodes variable is not a dependency because + // it is kept in the reference and the hook will run using + // graphStructureVersion as trigger. + // eslint-disable-next-line react-hooks/exhaustive-deps }, [ cy, layout, additionalLayoutsConfig, useAnimation, - nodes, onLayoutUpdated, graphStructureVersion, mounted, diff --git a/packages/graph-explorer/src/connector/AbstractConnector.ts b/packages/graph-explorer/src/connector/AbstractConnector.ts index 5f823a6fa..e361bac5a 100644 --- a/packages/graph-explorer/src/connector/AbstractConnector.ts +++ b/packages/graph-explorer/src/connector/AbstractConnector.ts @@ -8,6 +8,7 @@ import type { export type QueryOptions = { abortSignal?: AbortSignal; + disableCache?: boolean; }; export type VertexSchemaResponse = Pick< @@ -18,6 +19,13 @@ export type VertexSchemaResponse = Pick< | "displayNameAttribute" | "longDisplayNameAttribute" > & { + total?: number; +}; + +export type CountsByTypeRequest = { + label: string; +}; +export type CountsByTypeResponse = { total: number; }; @@ -25,14 +33,22 @@ export type EdgeSchemaResponse = Pick< EdgeTypeConfig, "type" | "displayLabel" | "attributes" > & { - total: number; + total?: number; }; export type SchemaResponse = { + /** + * Total number of vertices. + */ + totalVertices?: number; /** * List of vertices definitions. */ vertices: VertexSchemaResponse[]; + /** + * Total number of edges. + */ + totalEdges?: number; /** * List of edges definitions. */ @@ -189,8 +205,6 @@ export abstract class AbstractConnector { protected abstract readonly basePath: string; - private readonly _requestCache: Map = new Map(); - public constructor(connection: ConfigurationWithConnection["connection"]) { if (!connection?.url) { throw new Error("Invalid configuration. Missing 'connection.url'"); @@ -204,6 +218,14 @@ export abstract class AbstractConnector { */ public abstract fetchSchema(options?: QueryOptions): Promise; + /** + * Count the number of vertices of a given type + */ + public abstract fetchVertexCountsByType( + req: CountsByTypeRequest, + options?: QueryOptions + ): Promise; + /** * Fetch all directly connected neighbors of a given source * filtered by vertex or edge types and/or vertex attributes. @@ -233,15 +255,23 @@ export abstract class AbstractConnector { /** * This method performs requests and cache their responses. */ - protected async request( + protected async requestQueryTemplate( queryTemplate: string, - options?: { signal?: AbortSignal } - ): Promise { - const url = this._connection.url.replace(/\/$/, ""); + options?: { signal?: AbortSignal; disableCache?: boolean } + ): Promise { const encodedQuery = encodeURIComponent(queryTemplate); - const uri = `${url}${this.basePath}${encodedQuery}&format=json`; + const uri = `${this.basePath}${encodedQuery}&format=json`; + return this.request(uri, options); + } + + protected async request( + uri: string, + options?: { signal?: AbortSignal; disableCache?: boolean } + ): Promise { + const url = this._connection.url.replace(/\/$/, ""); + const currentUri = `${url}${uri}`; - const cachedResponse = await this._getFromCache(uri); + const cachedResponse = await this._getFromCache(currentUri); if ( cachedResponse && cachedResponse.updatedAt + @@ -251,7 +281,7 @@ export abstract class AbstractConnector { return cachedResponse.data; } - return this._requestAndCache(uri, { + return this._requestAndCache(currentUri, { signal: options?.signal, headers: this._getAuthHeaders(), }); @@ -269,10 +299,16 @@ export abstract class AbstractConnector { return headers; } - private async _requestAndCache(url: string, init?: RequestInit) { + private async _requestAndCache( + url: string, + init?: RequestInit, + options?: Pick + ) { const response = await fetch(url, init); const data = await response.json(); - this._setToCache(url, { data, updatedAt: new Date().getTime() }); + if (options?.disableCache !== true) { + this._setToCache(url, { data, updatedAt: new Date().getTime() }); + } return data as TResult; } diff --git a/packages/graph-explorer/src/connector/gremlin/GremlinConnector.ts b/packages/graph-explorer/src/connector/gremlin/GremlinConnector.ts index ce8bd5130..6a70b12b3 100644 --- a/packages/graph-explorer/src/connector/gremlin/GremlinConnector.ts +++ b/packages/graph-explorer/src/connector/gremlin/GremlinConnector.ts @@ -1,4 +1,6 @@ import type { + CountsByTypeRequest, + CountsByTypeResponse, KeywordSearchRequest, KeywordSearchResponse, NeighborsCountRequest, @@ -12,40 +14,71 @@ import { AbstractConnector } from "../AbstractConnector"; import fetchNeighbors from "./queries/fetchNeighbors"; import fetchNeighborsCount from "./queries/fetchNeighborsCount"; import fetchSchema from "./queries/fetchSchema"; +import fetchVertexTypeCounts from "./queries/fetchVertexTypeCounts"; import keywordSearch from "./queries/keywordSearch"; +import { GraphSummary } from "./types"; export default class GremlinConnector extends AbstractConnector { protected readonly basePath = "/?gremlin="; + private readonly _summaryPath = "/pg/statistics/summary?mode=detailed"; - fetchSchema(options?: QueryOptions): Promise { - return fetchSchema(this._gremlinFetch(options)); + // Stores the id type before casting it to string + private _rawIdTypeMap: Map = new Map(); + + async fetchSchema(options?: QueryOptions): Promise { + const ops = { ...options, disableCache: true }; + let summary: GraphSummary | undefined; + try { + const response = await this.request<{ + payload: { graphSummary: GraphSummary }; + }>(this._summaryPath, ops); + summary = response.payload.graphSummary; + } catch (e) { + if (import.meta.env.DEV) { + console.error("[Summary API]", e); + } + } + + return fetchSchema(this._gremlinFetch(ops), summary); + } + + fetchVertexCountsByType( + req: CountsByTypeRequest, + options: QueryOptions | undefined + ): Promise { + return fetchVertexTypeCounts(this._gremlinFetch(options), req); } fetchNeighbors( req: NeighborsRequest, options: QueryOptions | undefined ): Promise { - return fetchNeighbors(this._gremlinFetch(options), req); + return fetchNeighbors(this._gremlinFetch(options), req, this._rawIdTypeMap); } fetchNeighborsCount( req: NeighborsCountRequest, options?: QueryOptions ): Promise { - return fetchNeighborsCount(this._gremlinFetch(options), req); + return fetchNeighborsCount( + this._gremlinFetch(options), + req, + this._rawIdTypeMap + ); } keywordSearch( req: KeywordSearchRequest, options?: QueryOptions ): Promise { - return keywordSearch(this._gremlinFetch(options), req); + return keywordSearch(this._gremlinFetch(options), req, this._rawIdTypeMap); } private _gremlinFetch(options?: QueryOptions) { return async (queryTemplate: string) => { - return super.request(queryTemplate, { + return super.requestQueryTemplate(queryTemplate, { signal: options?.abortSignal, + disableCache: options?.disableCache, }); }; } diff --git a/packages/graph-explorer/src/connector/gremlin/mappers/mapApiVertex.ts b/packages/graph-explorer/src/connector/gremlin/mappers/mapApiVertex.ts index aa027c8cf..1793872c4 100644 --- a/packages/graph-explorer/src/connector/gremlin/mappers/mapApiVertex.ts +++ b/packages/graph-explorer/src/connector/gremlin/mappers/mapApiVertex.ts @@ -2,7 +2,11 @@ import type { Vertex } from "../../../@types/entities"; import type { NeighborsCountResponse } from "../../AbstractConnector"; import type { GVertex } from "../types"; import parsePropertiesValues from "./parsePropertiesValues"; +<<<<<<< HEAD import toStringId from './toStringId' +======= +import toStringId from "./toStringId"; +>>>>>>> 0072316067b7c8f9e3d7d7464b956a3415649f61 const mapApiVertex = ( apiVertex: GVertex, diff --git a/packages/graph-explorer/src/connector/gremlin/mappers/toStringId.ts b/packages/graph-explorer/src/connector/gremlin/mappers/toStringId.ts index 70357e8db..2aa55a174 100644 --- a/packages/graph-explorer/src/connector/gremlin/mappers/toStringId.ts +++ b/packages/graph-explorer/src/connector/gremlin/mappers/toStringId.ts @@ -1,10 +1,35 @@ +<<<<<<< HEAD import { GInt64 } from "../types"; const toStringId = (id: string | GInt64): string => { +======= +import { GInt64, JanusID } from "../types"; + +const isJanusID = (id: any): id is JanusID => { + return ( + typeof id === "object" && + "@type" in id && + typeof id["@type"] === "string" && + "@value" in id && + typeof id["@value"] === "object" && + "relationId" in id["@value"] && + typeof id["@value"]["relationId"] === "string" + ); +}; + +const toStringId = (id: string | GInt64 | JanusID): string => { +>>>>>>> 0072316067b7c8f9e3d7d7464b956a3415649f61 if (typeof id === "string") { return id; } +<<<<<<< HEAD +======= + if (isJanusID(id)) { + return id["@value"]["relationId"]; + } + +>>>>>>> 0072316067b7c8f9e3d7d7464b956a3415649f61 return String(id["@value"]); }; diff --git a/packages/graph-explorer/src/connector/gremlin/queries/fetchNeighbors.test.ts b/packages/graph-explorer/src/connector/gremlin/queries/fetchNeighbors.test.ts index 4af669f01..383bb2d65 100644 --- a/packages/graph-explorer/src/connector/gremlin/queries/fetchNeighbors.test.ts +++ b/packages/graph-explorer/src/connector/gremlin/queries/fetchNeighbors.test.ts @@ -101,7 +101,7 @@ describe("Gremlin > fetchNeighbors", () => { const response = await fetchNeighbors(mockGremlinFetch(), { vertexId: "2018", vertexType: "airport", - }); + }, new Map()); expect(response).toMatchObject({ vertices: expectedVertices.map(v => ({ @@ -259,7 +259,7 @@ describe("Gremlin > fetchNeighbors", () => { vertexType: "airport", filterByVertexTypes: ["airport"], filterCriteria: [{ name: "code", value: "TF", operator: "LIKE" }], - }); + }, new Map()); expect(response).toMatchObject({ vertices: expectedVertices.map(v => ({ diff --git a/packages/graph-explorer/src/connector/gremlin/queries/fetchNeighbors.ts b/packages/graph-explorer/src/connector/gremlin/queries/fetchNeighbors.ts index dfeb6267c..b60634467 100644 --- a/packages/graph-explorer/src/connector/gremlin/queries/fetchNeighbors.ts +++ b/packages/graph-explorer/src/connector/gremlin/queries/fetchNeighbors.ts @@ -4,6 +4,7 @@ import type { } from "../../AbstractConnector"; import mapApiEdge from "../mappers/mapApiEdge"; import mapApiVertex from "../mappers/mapApiVertex"; +import toStringId from "../mappers/toStringId"; import oneHopTemplate from "../templates/oneHopTemplate"; import type { GEdgeList, GVertexList } from "../types"; import { GremlinFetch } from "../types"; @@ -27,9 +28,11 @@ type RawOneHopRequest = { const fetchNeighbors = async ( gremlinFetch: GremlinFetch, - req: NeighborsRequest + req: NeighborsRequest, + rawIds: Map ): Promise => { - const gremlinTemplate = oneHopTemplate(req); + const idType = rawIds.get(req.vertexId) ?? "string"; + const gremlinTemplate = oneHopTemplate({...req, idType}); const data = await gremlinFetch(gremlinTemplate); const verticesResponse = diff --git a/packages/graph-explorer/src/connector/gremlin/queries/fetchNeighborsCount.test.ts b/packages/graph-explorer/src/connector/gremlin/queries/fetchNeighborsCount.test.ts index 846add1c9..57ba714a4 100644 --- a/packages/graph-explorer/src/connector/gremlin/queries/fetchNeighborsCount.test.ts +++ b/packages/graph-explorer/src/connector/gremlin/queries/fetchNeighborsCount.test.ts @@ -8,7 +8,7 @@ describe("Gremlin > fetchNeighborsCount", () => { it("Should return neighbors counts for node 2018", async () => { const response = await fetchNeighborsCount(mockGremlinFetch(), { vertexId: "123", - }); + }, new Map()); expect(response).toMatchObject({ totalCount: 18, diff --git a/packages/graph-explorer/src/connector/gremlin/queries/fetchNeighborsCount.ts b/packages/graph-explorer/src/connector/gremlin/queries/fetchNeighborsCount.ts index 8280d1db1..e45642bb2 100644 --- a/packages/graph-explorer/src/connector/gremlin/queries/fetchNeighborsCount.ts +++ b/packages/graph-explorer/src/connector/gremlin/queries/fetchNeighborsCount.ts @@ -25,9 +25,11 @@ type RawNeighborsCountResponse = { const fetchNeighborsCount = async ( gremlinFetch: GremlinFetch, - req: NeighborsCountRequest + req: NeighborsCountRequest, + rawIds: Map ): Promise => { - const gremlinTemplate = neighborsCountTemplate(req); + const idType = rawIds.get(req.vertexId) ?? "string"; + const gremlinTemplate = neighborsCountTemplate({ ...req, idType }); const data = await gremlinFetch(gremlinTemplate); const pairs = data.result.data["@value"]?.[0]?.["@value"] || []; diff --git a/packages/graph-explorer/src/connector/gremlin/queries/fetchSchema.ts b/packages/graph-explorer/src/connector/gremlin/queries/fetchSchema.ts index 577f448a7..6cc389c03 100644 --- a/packages/graph-explorer/src/connector/gremlin/queries/fetchSchema.ts +++ b/packages/graph-explorer/src/connector/gremlin/queries/fetchSchema.ts @@ -5,7 +5,7 @@ import edgesSchemaTemplate from "../templates/edgesSchemaTemplate"; import vertexLabelsTemplate from "../templates/vertexLabelsTemplate"; import verticesSchemaTemplate from "../templates/verticesSchemaTemplate"; import type { GEdge, GInt64, GVertex } from "../types"; -import { GremlinFetch } from "../types"; +import { GraphSummary, GremlinFetch } from "../types"; type RawVertexLabelsResponse = { requestId: string; @@ -106,12 +106,12 @@ const TYPE_MAP = { "g:Float": "Number", }; -const fetchVerticesSchema = async ( - gremlinFetch: GremlinFetch +const fetchVerticesAttributes = async ( + gremlinFetch: GremlinFetch, + labels: Array, + countsByLabel: Record ): Promise => { - const allLabels = await fetchVertexLabels(gremlinFetch); const vertices: SchemaResponse["vertices"] = []; - const labels = Object.keys(allLabels); if (labels.length === 0) { return vertices; @@ -133,7 +133,7 @@ const fetchVerticesSchema = async ( vertices.push({ type: label, displayLabel: sanitizeText(label), - total: allLabels[label], + total: countsByLabel[label], attributes: Object.entries(properties || {}).map(([name, prop]) => { const value = prop[0]?.["@value"].value; return { @@ -151,6 +151,15 @@ const fetchVerticesSchema = async ( return vertices; }; +const fetchVerticesSchema = async ( + gremlinFetch: GremlinFetch +): Promise => { + const countsByLabel = await fetchVertexLabels(gremlinFetch); + const labels = Object.keys(countsByLabel); + + return fetchVerticesAttributes(gremlinFetch, labels, countsByLabel); +}; + const fetchEdgeLabels = async ( gremlinFetch: GremlinFetch ): Promise> => { @@ -166,13 +175,12 @@ const fetchEdgeLabels = async ( return labelsWithCounts; }; -const fetchEdgesSchema = async ( - gremlinFetch: GremlinFetch +const fetchEdgesAttributes = async ( + gremlinFetch: GremlinFetch, + labels: Array, + countsByLabel: Record ): Promise => { - const allLabels = await fetchEdgeLabels(gremlinFetch); const edges: SchemaResponse["edges"] = []; - const labels = Object.keys(allLabels); - if (labels.length === 0) { return edges; } @@ -191,7 +199,7 @@ const fetchEdgesSchema = async ( edges.push({ type: label, displayLabel: sanitizeText(label), - total: allLabels[label], + total: countsByLabel[label], attributes: Object.entries(properties || {}).map(([name, prop]) => { const value = prop["@value"].value; return { @@ -206,6 +214,15 @@ const fetchEdgesSchema = async ( return edges; }; +const fetchEdgesSchema = async ( + gremlinFetch: GremlinFetch +): Promise => { + const countsByLabel = await fetchEdgeLabels(gremlinFetch); + const labels = Object.keys(countsByLabel); + + return fetchEdgesAttributes(gremlinFetch, labels, countsByLabel); +}; + /** * Fetch the database shape. * It follows this process: @@ -218,13 +235,43 @@ const fetchEdgesSchema = async ( * nodes/edges with the same label contains an exact set of attributes. */ const fetchSchema = async ( - gremlinFetch: GremlinFetch + gremlinFetch: GremlinFetch, + summary?: GraphSummary ): Promise => { - const vertices = await fetchVerticesSchema(gremlinFetch); - const edges = await fetchEdgesSchema(gremlinFetch); + if (!summary) { + const vertices = await fetchVerticesSchema(gremlinFetch); + const totalVertices = vertices.reduce((total, vertex) => { + return total + (vertex.total ?? 0); + }, 0); + + const edges = await fetchEdgesSchema(gremlinFetch); + const totalEdges = edges.reduce((total, edge) => { + return total + (edge.total ?? 0); + }, 0); + + return { + totalVertices, + vertices, + totalEdges, + edges, + }; + } + + const vertices = await fetchVerticesAttributes( + gremlinFetch, + summary.nodeLabels, + {} + ); + const edges = await fetchEdgesAttributes( + gremlinFetch, + summary.edgeLabels, + {} + ); return { + totalVertices: summary.numNodes, vertices, + totalEdges: summary.numEdges, edges, }; }; diff --git a/packages/graph-explorer/src/connector/gremlin/queries/fetchVertexTypeCounts.ts b/packages/graph-explorer/src/connector/gremlin/queries/fetchVertexTypeCounts.ts new file mode 100644 index 000000000..cfa1fe1a3 --- /dev/null +++ b/packages/graph-explorer/src/connector/gremlin/queries/fetchVertexTypeCounts.ts @@ -0,0 +1,33 @@ +import { + CountsByTypeRequest, + CountsByTypeResponse, +} from "../../AbstractConnector"; +import vertexTypeCountTemplate from "../templates/vertexTypeCountTemplate"; +import { GInt64, GremlinFetch } from "../types"; + +type RawCountsByTypeResponse = { + requestId: string; + status: { + message: string; + code: number; + }; + result: { + data: { + "@type": "g:List"; + "@value": Array; + }; + }; +}; + +const fetchVertexTypeCounts = async ( + gremlinFetch: GremlinFetch, + req: CountsByTypeRequest +): Promise => { + const template = vertexTypeCountTemplate(req.label); + const response = await gremlinFetch(template); + return { + total: response.result.data["@value"][0]["@value"], + }; +}; + +export default fetchVertexTypeCounts; diff --git a/packages/graph-explorer/src/connector/gremlin/queries/keywordSearch.test.ts b/packages/graph-explorer/src/connector/gremlin/queries/keywordSearch.test.ts index 5a1693ce6..6ee238d47 100644 --- a/packages/graph-explorer/src/connector/gremlin/queries/keywordSearch.test.ts +++ b/packages/graph-explorer/src/connector/gremlin/queries/keywordSearch.test.ts @@ -8,7 +8,7 @@ describe("Gremlin > keywordSearch", () => { it("Should return 1 random node", async () => { const keywordResponse = await keywordSearch(mockGremlinFetch(), { limit: 1, - }); + }, new Map()); expect(keywordResponse).toMatchObject({ vertices: [ @@ -44,7 +44,7 @@ describe("Gremlin > keywordSearch", () => { searchTerm: "SFA", vertexTypes: ["airport"], searchByAttributes: ["code"], - }); + }, new Map()); expect(keywordResponse).toMatchObject({ vertices: [ diff --git a/packages/graph-explorer/src/connector/gremlin/queries/keywordSearch.ts b/packages/graph-explorer/src/connector/gremlin/queries/keywordSearch.ts index 6daffaae5..1e1c0ef20 100644 --- a/packages/graph-explorer/src/connector/gremlin/queries/keywordSearch.ts +++ b/packages/graph-explorer/src/connector/gremlin/queries/keywordSearch.ts @@ -5,9 +5,10 @@ import type { } from "../../AbstractConnector"; import isErrorResponse from "../../utils/isErrorResponse"; import mapApiVertex from "../mappers/mapApiVertex"; +import toStringId from "../mappers/toStringId"; import keywordSearchTemplate from "../templates/keywordSearchTemplate"; import type { GVertexList } from "../types"; -import { GremlinFetch } from "../types"; +import { GInt64, GremlinFetch } from "../types"; type RawKeySearchResponse = { requestId: string; @@ -20,9 +21,18 @@ type RawKeySearchResponse = { }; }; +const idType = (id: string | GInt64) => { + if (typeof id === "string") { + return "string"; + } + + return "number"; +}; + const keywordSearch = async ( gremlinFetch: GremlinFetch, - req: KeywordSearchRequest + req: KeywordSearchRequest, + rawIds: Map ): Promise => { const gremlinTemplate = keywordSearchTemplate(req); const data = await gremlinFetch( @@ -34,6 +44,7 @@ const keywordSearch = async ( } const vertices = data.result.data["@value"].map(value => { + rawIds.set(toStringId(value["@value"].id), idType(value["@value"].id)); return mapApiVertex(value); }); diff --git a/packages/graph-explorer/src/connector/gremlin/templates/neighborsCountTemplate.ts b/packages/graph-explorer/src/connector/gremlin/templates/neighborsCountTemplate.ts index faeab1e44..b86cb3e34 100644 --- a/packages/graph-explorer/src/connector/gremlin/templates/neighborsCountTemplate.ts +++ b/packages/graph-explorer/src/connector/gremlin/templates/neighborsCountTemplate.ts @@ -14,8 +14,14 @@ import type { NeighborsCountRequest } from "../../AbstractConnector"; const neighborsCountTemplate = ({ vertexId, limit = 500, -}: NeighborsCountRequest) => { - let template = `g.V("${vertexId}").both()`; + idType = "string", +}: NeighborsCountRequest & { idType?: "string" | "number" }) => { + let template = ""; + if (idType === "number") { + template = `g.V(${vertexId}L).both()`; + } else { + template = `g.V("${vertexId}").both()`; + } if (limit > 0) { template += `.limit(${limit})`; diff --git a/packages/graph-explorer/src/connector/gremlin/templates/oneHopTemplate.ts b/packages/graph-explorer/src/connector/gremlin/templates/oneHopTemplate.ts index 88566bc3e..3d16b2bf6 100644 --- a/packages/graph-explorer/src/connector/gremlin/templates/oneHopTemplate.ts +++ b/packages/graph-explorer/src/connector/gremlin/templates/oneHopTemplate.ts @@ -132,9 +132,19 @@ const oneHopTemplate = ({ filterCriteria = [], limit = 10, offset = 0, -}: Omit): string => { + idType = "string", +}: Omit & { + idType?: "string" | "number"; +}): string => { const range = `.range(${offset}, ${offset + limit})`; - let template = `g.V("${vertexId}").project("vertices", "edges")`; + let template = ""; + if (idType === "number") { + template = `g.V(${vertexId}L)`; + } else { + template = `g.V("${vertexId}")`; + } + + template += `.project("vertices", "edges")`; const hasLabelContent = filterByVertexTypes .flatMap(type => type.split("::")) diff --git a/packages/graph-explorer/src/connector/gremlin/templates/vertexTypeCountTemplate.ts b/packages/graph-explorer/src/connector/gremlin/templates/vertexTypeCountTemplate.ts new file mode 100644 index 000000000..4f24b5aa9 --- /dev/null +++ b/packages/graph-explorer/src/connector/gremlin/templates/vertexTypeCountTemplate.ts @@ -0,0 +1,8 @@ +/** + * It returns a Gremlin template to number of vertices of a particular label + */ +const vertexTypeCountTemplate = (label: string) => { + return `g.V().hasLabel("${label}").count()`; +}; + +export default vertexTypeCountTemplate; diff --git a/packages/graph-explorer/src/connector/gremlin/types.ts b/packages/graph-explorer/src/connector/gremlin/types.ts index a9c0a0e78..a529ca64f 100644 --- a/packages/graph-explorer/src/connector/gremlin/types.ts +++ b/packages/graph-explorer/src/connector/gremlin/types.ts @@ -18,6 +18,13 @@ export type GDate = { "@value": number; }; +export type JanusID = { + "@type": "janusgraph:RelationIdentifier", + "@value": { + "relationId": string + } +} + export type GVertexProperty = { "@type": "g:VertexProperty"; "@value": { @@ -48,7 +55,11 @@ export type GVertex = { export type GEdge = { "@type": "g:Edge"; "@value": { +<<<<<<< HEAD id: string | GInt64; +======= + id: string | GInt64 | JanusID; +>>>>>>> 0072316067b7c8f9e3d7d7464b956a3415649f61 label: string; inVLabel: string; inV: string | GInt64; @@ -71,3 +82,18 @@ export type GEdgeList = { export type GremlinFetch = ( queryTemplate: string ) => Promise; + +export type GraphSummary = { + numNodes: number; + numEdges: number; + numNodeLabels: number; + numEdgeLabels: number; + nodeLabels: Array; + edgeLabels: Array; + numNodeProperties: number; + numEdgeProperties: number; + nodeProperties: Record; + edgeProperties: Record; + totalNodePropertyValues: number; + totalEdgePropertyValues: number; +}; diff --git a/packages/graph-explorer/src/connector/sparql/SPARQLConnector.ts b/packages/graph-explorer/src/connector/sparql/SPARQLConnector.ts index 1703f4f96..22fdeee8f 100644 --- a/packages/graph-explorer/src/connector/sparql/SPARQLConnector.ts +++ b/packages/graph-explorer/src/connector/sparql/SPARQLConnector.ts @@ -1,4 +1,6 @@ import type { + CountsByTypeRequest, + CountsByTypeResponse, KeywordSearchResponse, NeighborsCountRequest, NeighborsCountResponse, @@ -12,20 +14,43 @@ import { QueryOptions, } from "../AbstractConnector"; import fetchBlankNodeNeighbors from "./queries/fetchBlankNodeNeighbors"; +import fetchClassCounts from "./queries/fetchClassCounts"; import fetchNeighbors from "./queries/fetchNeighbors"; import fetchNeighborsCount from "./queries/fetchNeighborsCount"; import fetchSchema from "./queries/fetchSchema"; import keywordSearch from "./queries/keywordSearch"; import keywordSearchBlankNodesIdsTemplate from "./templates/keywordSearch/keywordSearchBlankNodesIdsTemplate"; import oneHopNeighborsBlankNodesIdsTemplate from "./templates/oneHopNeighbors/oneHopNeighborsBlankNodesIdsTemplate"; -import { BlankNodesMap, SPARQLNeighborsRequest } from "./types"; +import { BlankNodesMap, GraphSummary, SPARQLNeighborsRequest } from "./types"; export default class SPARQLConnector extends AbstractConnector { protected readonly basePath = "/sparql?query="; + private readonly _summaryPath = "/rdf/statistics/summary?mode=detailed"; + private _blankNodes: BlankNodesMap = new Map(); - fetchSchema(options?: QueryOptions): Promise { - return fetchSchema(this._sparqlFetch(options)); + async fetchSchema(options?: QueryOptions): Promise { + const ops = { ...options, disableCache: true }; + let summary: GraphSummary | undefined; + try { + const response = await this.request<{ + payload: { graphSummary: GraphSummary }; + }>(this._summaryPath, ops); + summary = response.payload.graphSummary; + } catch (e) { + if (import.meta.env.DEV) { + console.error("[Summary API]", e); + } + } + + return fetchSchema(this._sparqlFetch(ops), summary); + } + + fetchVertexCountsByType( + req: CountsByTypeRequest, + options?: QueryOptions + ): Promise { + return fetchClassCounts(this._sparqlFetch(options), req); } async fetchNeighbors( @@ -122,8 +147,9 @@ export default class SPARQLConnector extends AbstractConnector { private _sparqlFetch(options?: QueryOptions) { return async (queryTemplate: string) => { - return super.request(queryTemplate, { + return super.requestQueryTemplate(queryTemplate, { signal: options?.abortSignal, + disableCache: options?.disableCache, }); }; } diff --git a/packages/graph-explorer/src/connector/sparql/queries/fetchClassCounts.ts b/packages/graph-explorer/src/connector/sparql/queries/fetchClassCounts.ts new file mode 100644 index 000000000..43b5beb62 --- /dev/null +++ b/packages/graph-explorer/src/connector/sparql/queries/fetchClassCounts.ts @@ -0,0 +1,30 @@ +import { + CountsByTypeRequest, + CountsByTypeResponse, +} from "../../AbstractConnector"; +import classWithCountsTemplates from "../templates/classWithCountsTemplates"; +import { RawValue, SparqlFetch } from "../types"; + +type RawCountsByTypeResponse = { + results: { + bindings: [ + { + instancesCount: RawValue; + } + ]; + }; +}; + +const fetchClassCounts = async ( + sparqlFetch: SparqlFetch, + req: CountsByTypeRequest +): Promise => { + const template = classWithCountsTemplates(req.label); + const response = await sparqlFetch(template); + + return { + total: Number(response.results.bindings[0].instancesCount.value), + }; +}; + +export default fetchClassCounts; diff --git a/packages/graph-explorer/src/connector/sparql/queries/fetchSchema.ts b/packages/graph-explorer/src/connector/sparql/queries/fetchSchema.ts index fc3abd0cd..8fb12cf3b 100644 --- a/packages/graph-explorer/src/connector/sparql/queries/fetchSchema.ts +++ b/packages/graph-explorer/src/connector/sparql/queries/fetchSchema.ts @@ -2,7 +2,7 @@ import { SchemaResponse } from "../../AbstractConnector"; import classesWithCountsTemplates from "../templates/classesWithCountsTemplates"; import predicatesByClassTemplate from "../templates/predicatesByClassTemplate"; import predicatesWithCountsTemplate from "../templates/predicatesWithCountsTemplate"; -import { RawValue, SparqlFetch } from "../types"; +import { GraphSummary, RawValue, SparqlFetch } from "../types"; type RawClassesWCountsResponse = { results: { @@ -57,17 +57,16 @@ const skosNote = "http://www.w3.org/2004/02/skos/core#note"; const skosDefinition = "http://www.w3.org/2004/02/skos/core#definition"; const displayDescCandidates = [rdfsComment, skosNote, skosDefinition]; -const fetchClassesSchema = async (sparqlFetch: SparqlFetch) => { - const classesTemplate = classesWithCountsTemplates(); - const classesCounts = await sparqlFetch( - classesTemplate - ); - +const fetchPredicatesByClass = async ( + sparqlFetch: SparqlFetch, + classes: Array, + countsByClass: Record +) => { const vertices: SchemaResponse["vertices"] = []; await Promise.all( - classesCounts.results.bindings.map(async classResult => { + classes.map(async classResult => { const classPredicatesTemplate = predicatesByClassTemplate({ - class: classResult.class.value, + class: classResult, }); const predicatesResponse = await sparqlFetch< RawPredicatesSamplesResponse @@ -82,9 +81,9 @@ const fetchClassesSchema = async (sparqlFetch: SparqlFetch) => { })); vertices.push({ - type: classResult.class.value, + type: classResult, displayLabel: "", - total: Number(classResult.instancesCount.value), + total: countsByClass[classResult], displayNameAttribute: attributes.find(attr => displayNameCandidates.includes(attr.name)) ?.name || "id", @@ -99,6 +98,24 @@ const fetchClassesSchema = async (sparqlFetch: SparqlFetch) => { return vertices; }; +const fetchClassesSchema = async (sparqlFetch: SparqlFetch) => { + const classesTemplate = classesWithCountsTemplates(); + const classesCounts = await sparqlFetch( + classesTemplate + ); + + const classes: Array = []; + const countsByClass: Record = {}; + classesCounts.results.bindings.forEach(classResult => { + classes.push(classResult.class.value); + countsByClass[classResult.class.value] = Number( + classResult.instancesCount.value + ); + }); + + return fetchPredicatesByClass(sparqlFetch, classes, countsByClass); +}; + const fetchPredicatesWithCounts = async ( sparqlFetch: SparqlFetch ): Promise> => { @@ -138,13 +155,48 @@ const fetchPredicatesSchema = async (sparqlFetch: SparqlFetch) => { * 4. Generate prefixes using the received URIs */ const fetchSchema = async ( - sparqlFetch: SparqlFetch + sparqlFetch: SparqlFetch, + summary?: GraphSummary ): Promise => { - const vertices = await fetchClassesSchema(sparqlFetch); - const edges = await fetchPredicatesSchema(sparqlFetch); + if (!summary) { + const vertices = await fetchClassesSchema(sparqlFetch); + const totalVertices = vertices.reduce((total, vertex) => { + return total + (vertex.total ?? 0); + }, 0); + + const edges = await fetchPredicatesSchema(sparqlFetch); + const totalEdges = edges.reduce((total, edge) => { + return total + (edge.total ?? 0); + }, 0); + + return { + totalVertices, + vertices, + totalEdges, + edges, + }; + } + + const vertices = await fetchPredicatesByClass( + sparqlFetch, + summary.classes, + {} + ); + const edges = summary.predicates.flatMap(pred => { + return Object.entries(pred).map(([type, count]) => { + return { + type, + displayLabel: "", + total: count, + attributes: [], + }; + }); + }); return { + totalVertices: summary.numDistinctSubjects, vertices, + totalEdges: summary.numQuads, edges, }; }; diff --git a/packages/graph-explorer/src/connector/sparql/templates/classWithCountsTemplates.ts b/packages/graph-explorer/src/connector/sparql/templates/classWithCountsTemplates.ts new file mode 100644 index 000000000..08e4d995c --- /dev/null +++ b/packages/graph-explorer/src/connector/sparql/templates/classWithCountsTemplates.ts @@ -0,0 +1,10 @@ +// It returns the number of instances of the given class +const classWithCountsTemplates = (className: string) => { + return ` + SELECT (COUNT(?start) AS ?instancesCount) { + ?start a <${className}> + } + `; +}; + +export default classWithCountsTemplates; diff --git a/packages/graph-explorer/src/connector/sparql/types.ts b/packages/graph-explorer/src/connector/sparql/types.ts index 68e4ef4ca..e1d7aaab3 100644 --- a/packages/graph-explorer/src/connector/sparql/types.ts +++ b/packages/graph-explorer/src/connector/sparql/types.ts @@ -149,4 +149,14 @@ export type BlankNodeItem = { edges: Array; }; }; + export type BlankNodesMap = Map; + +export type GraphSummary = { + numDistinctSubjects: number; + numDistinctPredicates: number; + numQuads: number; + numClasses: number; + classes: Array; + predicates: Array>; +}; diff --git a/packages/graph-explorer/src/core/ConfigurationProvider/ConfigurationProvider.tsx b/packages/graph-explorer/src/core/ConfigurationProvider/ConfigurationProvider.tsx index 5d848d421..45f0d2755 100644 --- a/packages/graph-explorer/src/core/ConfigurationProvider/ConfigurationProvider.tsx +++ b/packages/graph-explorer/src/core/ConfigurationProvider/ConfigurationProvider.tsx @@ -102,7 +102,9 @@ const ConfigurationProvider = ({ return { ...(configuration || {}), + totalVertices: configuration.schema?.totalVertices ?? 0, vertexTypes: configuration.schema?.vertices?.map(vt => vt.type) || [], + totalEdges: configuration.schema?.totalEdges ?? 0, edgeTypes: configuration.schema?.edges?.map(et => et.type) || [], getVertexTypeConfig, getVertexTypeAttributes, diff --git a/packages/graph-explorer/src/core/ConfigurationProvider/types.ts b/packages/graph-explorer/src/core/ConfigurationProvider/types.ts index 303666160..21b2868b1 100644 --- a/packages/graph-explorer/src/core/ConfigurationProvider/types.ts +++ b/packages/graph-explorer/src/core/ConfigurationProvider/types.ts @@ -3,6 +3,8 @@ import { VertexPreferences, } from "../StateProvider/userPreferences"; +import { OntologyListArrayType } from "../../modules/AP-OntologyTab/OntologyListTypes"; + export type AttributeConfig = { /** * Name of the attribute in the DB schema @@ -168,7 +170,9 @@ export type RawConfiguration = { * Database schema: types, names, labels, icons, ... */ schema?: { + totalVertices: number; vertices: Array; + totalEdges: number; edges: Array; lastUpdate?: Date; triedToSync?: boolean; @@ -182,10 +186,13 @@ export type RawConfiguration = { * Mark as created from a file */ __fileBase?: boolean; + OntologyData?:OntologyListArrayType }; export type ConfigurationContextProps = RawConfiguration & { + totalVertices: number; vertexTypes: Array; + totalEdges: number; edgeTypes: Array; getVertexTypeConfig(vertexType: string): VertexTypeConfig | undefined; getVertexTypeAttributes(vertexTypes: string[]): Array; diff --git a/packages/graph-explorer/src/core/StateProvider/configuration.ts b/packages/graph-explorer/src/core/StateProvider/configuration.ts index 90b090133..199ce03d0 100644 --- a/packages/graph-explorer/src/core/StateProvider/configuration.ts +++ b/packages/graph-explorer/src/core/StateProvider/configuration.ts @@ -99,6 +99,8 @@ export const mergedConfigurationSelector = selector({ : undefined, triedToSync: currentSchema?.triedToSync, lastSyncFail: currentSchema?.lastSyncFail, + totalVertices: currentSchema?.totalVertices ?? 0, + totalEdges: currentSchema?.totalEdges ?? 0, }, }; }, diff --git a/packages/graph-explorer/src/core/StateProvider/schema.ts b/packages/graph-explorer/src/core/StateProvider/schema.ts index 1e5ce0be9..5c9668f49 100644 --- a/packages/graph-explorer/src/core/StateProvider/schema.ts +++ b/packages/graph-explorer/src/core/StateProvider/schema.ts @@ -13,6 +13,8 @@ export type SchemaInference = { lastUpdate?: Date; triedToSync?: boolean; lastSyncFail?: boolean; + totalVertices?: number; + totalEdges?: number; }; export const schemaAtom = atom>({ diff --git a/packages/graph-explorer/src/hooks/useEntitiesCounts.ts b/packages/graph-explorer/src/hooks/useEntitiesCounts.ts index 3d81c38fb..1512e465c 100644 --- a/packages/graph-explorer/src/hooks/useEntitiesCounts.ts +++ b/packages/graph-explorer/src/hooks/useEntitiesCounts.ts @@ -5,6 +5,10 @@ const useEntitiesCounts = () => { const config = useConfiguration(); const totalNodes = useMemo(() => { + if (config?.totalVertices != null) { + return config?.totalVertices; + } + if (!config?.vertexTypes.length) { return null; } @@ -24,6 +28,10 @@ const useEntitiesCounts = () => { }, [config]); const totalEdges = useMemo(() => { + if (config?.totalEdges != null) { + return config?.totalEdges; + } + if (!config?.edgeTypes.length) { return null; } diff --git a/packages/graph-explorer/src/hooks/useSchemaSync.ts b/packages/graph-explorer/src/hooks/useSchemaSync.ts index 9065ad2f3..0a12c0d4e 100644 --- a/packages/graph-explorer/src/hooks/useSchemaSync.ts +++ b/packages/graph-explorer/src/hooks/useSchemaSync.ts @@ -1,11 +1,10 @@ import { useCallback, useRef } from "react"; -import { useRecoilCallback } from "recoil"; import { useNotification } from "../components/NotificationProvider"; import { SchemaResponse } from "../connector/AbstractConnector"; import useConfiguration from "../core/ConfigurationProvider/useConfiguration"; import useConnector from "../core/ConnectorProvider/useConnector"; -import { schemaAtom } from "../core/StateProvider/schema"; import usePrefixesUpdater from "./usePrefixesUpdater"; +import useUpdateSchema from "./useUpdateSchema"; const useSchemaSync = (onSyncChange?: (isSyncing: boolean) => void) => { const config = useConfiguration(); @@ -15,26 +14,7 @@ const useSchemaSync = (onSyncChange?: (isSyncing: boolean) => void) => { const { enqueueNotification, clearNotification } = useNotification(); const notificationId = useRef(null); - const updateSchemaState = useRecoilCallback( - ({ set }) => (id: string, schema?: SchemaResponse) => { - set(schemaAtom, prevSchemaMap => { - const updatedSchema = new Map(prevSchemaMap); - const prevSchema = prevSchemaMap.get(id); - - updatedSchema.set(id, { - vertices: schema?.vertices || prevSchema?.vertices || [], - edges: schema?.edges || prevSchema?.edges || [], - prefixes: prevSchema?.prefixes || [], - lastUpdate: !schema ? prevSchema?.lastUpdate : new Date(), - triedToSync: true, - lastSyncFail: !schema && !!prevSchema, - }); - return updatedSchema; - }); - }, - [] - ); - + const updateSchemaState = useUpdateSchema(); return useCallback( async (abortSignal?: AbortSignal) => { if (!config || !connector.explorer) { diff --git a/packages/graph-explorer/src/hooks/useUpdateSchema.ts b/packages/graph-explorer/src/hooks/useUpdateSchema.ts new file mode 100644 index 000000000..04bff6f65 --- /dev/null +++ b/packages/graph-explorer/src/hooks/useUpdateSchema.ts @@ -0,0 +1,64 @@ +import { useRecoilCallback } from "recoil"; +import { SchemaResponse } from "../connector/AbstractConnector"; +import { schemaAtom } from "../core/StateProvider/schema"; + +const useUpdateSchema = () => { + return useRecoilCallback( + ({ set }) => ( + id: string, + schema?: + | Partial + | ((prevSchema?: SchemaResponse) => Partial) + ) => { + set(schemaAtom, prevSchemaMap => { + const updatedSchema = new Map(prevSchemaMap); + const prevSchema = prevSchemaMap.get(id); + + const currentSchema = + typeof schema === "function" ? schema(prevSchema) : schema; + + // Preserve vertices counts + const vertices = ( + currentSchema?.vertices || + prevSchema?.vertices || + [] + ).map(vertex => { + const prevVertex = prevSchema?.vertices.find( + v => v.type === vertex.type + ); + return { + ...vertex, + total: vertex.total ?? prevVertex?.total, + }; + }); + + // Preserve edges counts + const edges = (currentSchema?.edges || prevSchema?.edges || []).map( + edge => { + const prevEdge = prevSchema?.edges.find(e => e.type === edge.type); + return { + ...edge, + total: edge.total ?? prevEdge?.total, + }; + } + ); + + updatedSchema.set(id, { + totalVertices: + currentSchema?.totalVertices || prevSchema?.totalVertices || 0, + vertices, + totalEdges: currentSchema?.totalEdges || prevSchema?.totalEdges || 0, + edges, + prefixes: prevSchema?.prefixes || [], + lastUpdate: !currentSchema ? prevSchema?.lastUpdate : new Date(), + triedToSync: true, + lastSyncFail: !currentSchema && !!prevSchema, + }); + return updatedSchema; + }); + }, + [] + ); +}; + +export default useUpdateSchema; diff --git a/packages/graph-explorer/src/hooks/useUpdateVertexTypeCounts.ts b/packages/graph-explorer/src/hooks/useUpdateVertexTypeCounts.ts new file mode 100644 index 000000000..4563d3e39 --- /dev/null +++ b/packages/graph-explorer/src/hooks/useUpdateVertexTypeCounts.ts @@ -0,0 +1,61 @@ +import { useMemo } from "react"; +import { useQuery } from "react-query"; +import { useConfiguration } from "../core"; +import useConnector from "../core/ConnectorProvider/useConnector"; +import useUpdateSchema from "./useUpdateSchema"; + +const useUpdateVertexTypeCounts = (vertexType?: string) => { + const config = useConfiguration(); + const connector = useConnector(); + + const vertexConfig = useMemo(() => { + if (!vertexType) { + return; + } + + return config?.getVertexTypeConfig(vertexType); + }, [config, vertexType]); + + const updateSchemaState = useUpdateSchema(); + useQuery( + ["fetchCountsByType", vertexConfig?.type], + () => { + if (vertexConfig?.total != null || vertexConfig?.type == null) { + return; + } + + return connector.explorer?.fetchVertexCountsByType({ + label: vertexConfig?.type, + }); + }, + { + enabled: vertexConfig?.total == null && vertexConfig?.type != null, + onSuccess: response => { + if (!config?.id || !response) { + return; + } + + updateSchemaState(config.id, prevSchema => { + const vertexSchema = prevSchema?.vertices.find( + vertex => vertex.type === vertexType + ); + if (!vertexSchema) { + return { ...(prevSchema || {}) }; + } + + vertexSchema.total = response.total; + return { + vertices: [ + ...(prevSchema?.vertices.filter( + vertex => vertex.type !== vertexType + ) || []), + vertexSchema, + ], + }; + }); + }, + } + ); +}; + +export default useUpdateVertexTypeCounts; diff --git a/packages/graph-explorer/src/modules/AP-OntologySearch/OntologySearch.tsx b/packages/graph-explorer/src/modules/AP-OntologySearch/OntologySearch.tsx index 7b56871b3..eb193c7cb 100644 --- a/packages/graph-explorer/src/modules/AP-OntologySearch/OntologySearch.tsx +++ b/packages/graph-explorer/src/modules/AP-OntologySearch/OntologySearch.tsx @@ -78,7 +78,7 @@ const KeywordSearch = ({ ontologyOptions, selectedOntologyName, onOntologyChange, - ontologySearchSCTIDArray, + fetchedData, } = useOntologySearch({ @@ -219,6 +219,15 @@ const KeywordSearch = ({ handleOnClose(); }; + + // fxn for button to select entire list + const setAllActive =()=> { + + setEntities + } + + + const currentTotal = useMemo(() => { if (!config?.vertexTypes.length) { return null; @@ -383,8 +392,9 @@ const KeywordSearch = ({ } - onPress={()=>{}} - >eeeeee + onPress={setAllActive} + >set all active +
{/*console.log(ontologySearchResults)*/}
diff --git a/packages/graph-explorer/src/modules/AP-OntologySearch/useOntologySearch.ts b/packages/graph-explorer/src/modules/AP-OntologySearch/useOntologySearch.ts index f2ebcb942..f4b0748d2 100644 --- a/packages/graph-explorer/src/modules/AP-OntologySearch/useOntologySearch.ts +++ b/packages/graph-explorer/src/modules/AP-OntologySearch/useOntologySearch.ts @@ -32,7 +32,6 @@ const useOntologySearch = ({ isOpen }: { isOpen: boolean }) => { const [selectedOntologyName, setSelectedOntologyName] = useState('') const [fetchedData, setFetchedData] = useState([]); - const ontologySearchSCTIDArray:string[] = [] const vertexOptions = useMemo(() => { const vertexOps = @@ -134,7 +133,6 @@ const useOntologySearch = ({ isOpen }: { isOpen: boolean }) => { const { data, isFetching } = useQuery( [ "keyword-search", - ontologySearchSCTIDArray, ontologyResultArray, debouncedSearchTerm, vertexTypes, @@ -219,25 +217,6 @@ const useOntologySearch = ({ isOpen }: { isOpen: boolean }) => { setFetchedData(sctidValues) - if (typeof sctidValues !== undefined){ - //ontologySearchSCTIDArray.length=0 - for (const ids of sctidValues){ - ontologySearchSCTIDArray.push(ids.data.attributes.FSN.toString()) - - - } - // here this will result in an array of the sctid values we need, everything works up to this point - // console.log('here is your array of just fsn names') - // console.log(ontologySearchSCTIDArray) - - // below does not seem to work - for (const item of ontologySearchSCTIDArray) { - setSearchTerm(item) - } - return ontologySearchSCTIDArray - - } - }, []); @@ -268,7 +247,7 @@ const useOntologySearch = ({ isOpen }: { isOpen: boolean }) => { onAttributeOptionChange, searchResults: data?.vertices || [], - ontologySearchSCTIDArray, + }; }; diff --git a/packages/graph-explorer/src/modules/AvailableConnections/AvailableConnections.tsx b/packages/graph-explorer/src/modules/AvailableConnections/AvailableConnections.tsx index ef15daaa9..b3cd55ec4 100644 --- a/packages/graph-explorer/src/modules/AvailableConnections/AvailableConnections.tsx +++ b/packages/graph-explorer/src/modules/AvailableConnections/AvailableConnections.tsx @@ -83,6 +83,13 @@ const AvailableConnections = ({ displayLabel: fileContent.displayLabel, connection: fileContent.connection, }); + + // below adds the locally stored ontologies from the imported connection data + console.log('here is file content') + console.log(JSON.stringify(fileContent.OntologyData)) + console.log(fileContent.OntologyData) + + localStorage.setItem('OntologyData', JSON.stringify(fileContent.OntologyData)) return updatedConfig; }); set(schemaAtom, prevSchema => { diff --git a/packages/graph-explorer/src/modules/ConnectionDetail/ConnectionData.tsx b/packages/graph-explorer/src/modules/ConnectionDetail/ConnectionData.tsx index 7a158fdad..f6e6f6efd 100644 --- a/packages/graph-explorer/src/modules/ConnectionDetail/ConnectionData.tsx +++ b/packages/graph-explorer/src/modules/ConnectionDetail/ConnectionData.tsx @@ -46,14 +46,6 @@ const ConnectionData = ({ classNamePrefix = "ft" }: VertexDetailProps) => {
{textTransform(displayLabel)}
-
- {t("connection-detail.nodes")}:{" "} - {vtConfig?.total ? ( - - ) : ( - "Unknown" - )} -
), icon: ( @@ -87,7 +79,7 @@ const ConnectionData = ({ classNamePrefix = "ft" }: VertexDetailProps) => { }); return items; - }, [config, pfx, textTransform, t, navigate]); + }, [config, pfx, textTransform, navigate]); const [search, setSearch] = useState(""); diff --git a/packages/graph-explorer/src/utils/isValidConfigurationFile.ts b/packages/graph-explorer/src/utils/isValidConfigurationFile.ts index 9c2589a69..312035937 100644 --- a/packages/graph-explorer/src/utils/isValidConfigurationFile.ts +++ b/packages/graph-explorer/src/utils/isValidConfigurationFile.ts @@ -50,7 +50,7 @@ const isValidConfigurationFile = ( data: any ): data is Pick< ConfigurationContextProps, - "id" | "displayLabel" | "connection" | "schema" + "id" | "displayLabel" | "connection" | "schema" | "OntologyData" > => { if (!data.id || !data.connection || !data.schema) { return false; diff --git a/packages/graph-explorer/src/utils/saveConfigurationToFile.ts b/packages/graph-explorer/src/utils/saveConfigurationToFile.ts index 5163512d4..3ba11bdbc 100644 --- a/packages/graph-explorer/src/utils/saveConfigurationToFile.ts +++ b/packages/graph-explorer/src/utils/saveConfigurationToFile.ts @@ -18,6 +18,7 @@ const saveConfigurationToFile = (config: ConfigurationContextProps) => { })), lastUpdate: config.schema?.lastUpdate?.toISOString(), }, + OntologyData:JSON.parse(localStorage.OntologyData) }; const fileToSave = new Blob([JSON.stringify(exportableConfig)], { diff --git a/packages/graph-explorer/src/workspaces/DataExplorer/DataExplorer.tsx b/packages/graph-explorer/src/workspaces/DataExplorer/DataExplorer.tsx index 1d379832f..1c3c265b4 100644 --- a/packages/graph-explorer/src/workspaces/DataExplorer/DataExplorer.tsx +++ b/packages/graph-explorer/src/workspaces/DataExplorer/DataExplorer.tsx @@ -45,6 +45,7 @@ import useFetchNode from "../../hooks/useFetchNode"; import usePrefixesUpdater from "../../hooks/usePrefixesUpdater"; import useTextTransform from "../../hooks/useTextTransform"; import useTranslations from "../../hooks/useTranslations"; +import useUpdateVertexTypeCounts from "../../hooks/useUpdateVertexTypeCounts"; import TopBarWithLogo from "../common/TopBarWithLogo"; import defaultStyles from "./DataExplorer.styles"; @@ -69,6 +70,9 @@ const DataExplorer = ({ classNamePrefix = "ft" }: ConnectionsProps) => { const fetchNode = useFetchNode(); const [entities] = useEntities({ disableFilters: true }); + // Automatically updates counts if needed + useUpdateVertexTypeCounts(vertexType); + const vertexConfig = useMemo(() => { if (!vertexType) { return;