Merge pull request #1810 from Inist-CNRS/feat/precomputed-filtered

Use routine to filter precomputed data
Inist-CNRS · Nov 29, 2023 · d0338bd · d0338bd
2 parents c00024c + ec64028
commit d0338bd
Show file tree

Hide file tree

Showing 18 changed files with 498 additions and 191 deletions.
diff --git a/config.json b/config.json
@@ -27,6 +27,7 @@
         "filter",
         "get-fields",
         "graph-by",
+        "graph-by-precomputed",
         "group-and-sum-with",
         "hello-world",
         "istex-facet",
@@ -42,6 +43,7 @@
         "syndication-from",
         "syndication",
         "total-of",
+        "tree-by-precomputed",
         "tree-by",
         "unwind-by"
     ],
@@ -101,10 +103,5 @@
     "precomputedBaseUrlForDevelopment": "{public url of the server accessible from webhooks (ie ngrok)}",
     "precomputedAnswerMode": "retrieve",
     "activateBullDashboard": false,
-    "themes": [
-        "void",
-        "nougat",
-        "inist",
-        "istex"
-    ]
+    "themes": ["void", "nougat", "inist", "istex"]
 }
diff --git a/packages/ezsLodex/src/buildContext.js b/packages/ezsLodex/src/buildContext.js
@@ -1,23 +1,22 @@
 import mongoDatabase from './mongoDatabase';
 import getPublishedDatasetFilter from './getPublishedDatasetFilter';
 
-const findAll = async (collection) => collection
-    .find({})
-    .sort({ position: 1, cover: 1 })
-    .toArray();
+const findAll = async collection =>
+    collection
+        .find({})
+        .sort({ position: 1, cover: 1 })
+        .toArray();
 
-const findSearchableNames = async (collection) => {
+const findSearchableNames = async collection => {
     const searchableFields = await collection
         .find({ searchable: true })
         .toArray();
 
     return searchableFields.map(({ name }) => name);
 };
 
-const findFacetNames = async (collection) => {
-    const searchableFields = await collection
-        .find({ isFacet: true })
-        .toArray();
+const findFacetNames = async collection => {
+    const searchableFields = await collection.find({ isFacet: true }).toArray();
 
     return searchableFields.map(({ name }) => name);
 };
@@ -31,65 +30,73 @@ const findFacetNames = async (collection) => {
  * @param {String}  [host] to set host (usefull to build some links)
  * @returns {Object}
  */
-export const createFunction = () => async function LodexBuildContext(data, feed) {
-    if (this.isLast()) {
-        return feed.close();
-    }
+export const createFunction = () =>
+    async function LodexBuildContext(data, feed) {
+        if (this.isLast()) {
+            return feed.close();
+        }
 
-    const connectionStringURI = this.getParam('connectionStringURI', 'mongodb://ezmaster_db:27017');
-    const db = await mongoDatabase(connectionStringURI);
-    const collection = db.collection('field');
+        const precomputedName = this.getParam('precomputedName');
 
-    const host = this.getParam('host');
-    const {
-        uri,
-        maxSize,
-        skip,
-        maxValue,
-        minValue,
-        match,
-        orderBy = '_id/asc',
-        invertedFacets = [],
-        $query,
-        field,
-        ...facets
-    } = data;
-    const searchableFieldNames = await findSearchableNames(collection);
-    const facetFieldNames = await findFacetNames(collection);
-    const fields = await findAll(collection);
-    const filter = getPublishedDatasetFilter({
-        uri,
-        match,
-        invertedFacets,
-        facets,
-        ...$query,
-        searchableFieldNames,
-        facetFieldNames,
-    });
+        const connectionStringURI = this.getParam(
+            'connectionStringURI',
+            'mongodb://ezmaster_db:27017',
+        );
 
-    if (filter.$and && !filter.$and.length) {
-        delete filter.$and;
-    }
-    // context is the intput for LodexReduceQuery & LodexRunQuery & LodexDocuments
-    const context = {
-        // /*
-        // to build the MongoDB Query
-        filter,
-        field,
-        fields,
-        // Default parameters for ALL scripts
-        maxSize,
-        maxValue,
-        minValue,
-        orderBy,
-        skip,
-        uri,
-        host,
-        // to allow script to connect to MongoDB
-        connectionStringURI,
+        const db = await mongoDatabase(connectionStringURI);
+        const collection = db.collection('field');
+
+        const host = this.getParam('host');
+        const {
+            uri,
+            maxSize,
+            skip,
+            maxValue,
+            minValue,
+            match,
+            orderBy = '_id/asc',
+            invertedFacets = [],
+            $query,
+            field,
+            ...facets
+        } = data;
+        const searchableFieldNames = await findSearchableNames(collection);
+        const facetFieldNames = await findFacetNames(collection);
+        const fields = await findAll(collection);
+        const filter = getPublishedDatasetFilter({
+            uri,
+            match,
+            invertedFacets,
+            facets,
+            ...$query,
+            searchableFieldNames,
+            facetFieldNames,
+        });
+
+        if (filter.$and && !filter.$and.length) {
+            delete filter.$and;
+        }
+        // context is the intput for LodexReduceQuery & LodexRunQuery & LodexDocuments
+        const context = {
+            // /*
+            // to build the MongoDB Query
+            filter,
+            field,
+            fields,
+            // Default parameters for ALL scripts
+            maxSize,
+            maxValue,
+            minValue,
+            orderBy,
+            skip,
+            uri,
+            host,
+            // to allow script to connect to MongoDB
+            connectionStringURI,
+            precomputedName,
+        };
+        feed.send(context);
     };
-    feed.send(context);
-};
 
 export default {
     buildContext: createFunction(),

diff --git a/packages/ezsLodex/src/filterPrecomputed.js b/packages/ezsLodex/src/filterPrecomputed.js
@@ -0,0 +1,73 @@
+import mongoDatabase from './mongoDatabase';
+import { Readable } from 'stream';
+
+/**
+ * Take `Object` containing a MongoDB query and throw the result
+ *
+ * The input object must contain a `connectionStringURI` property, containing
+ * the connection string to MongoDB.
+ *
+ * @name LodexFilterPrecomputed
+ * @param {Object}  [referer]      data injected into every result object
+ * @returns {Object}
+ */
+export const createFunction = () =>
+    async function LodexFilterPrecomputed(data, feed) {
+        const {
+            connectionStringURI,
+            filter: rawFilter,
+            precomputedName,
+        } = this.getEnv();
+        const collectionName = 'precomputed';
+        const isFilter =
+            rawFilter &&
+            Object.keys(rawFilter).filter(
+                key => key != 'removedAt' && rawFilter[key] != null,
+            ).length > 0;
+
+        const db = await mongoDatabase(connectionStringURI);
+        const collection = db.collection(collectionName);
+        const precomputedData = (
+            await collection.findOne({
+                name: precomputedName,
+            })
+        ).data;
+
+        if (!isFilter) {
+            //With unfiltered precomputed data, we can send the whole precomputed Data directly
+            precomputedData.map(item => feed.write(item));
+            feed.end();
+            return feed.close();
+        }
+
+        const filteredItems = precomputedData.filter(
+            item => item.origin[0] === (data || {}).uri,
+        );
+
+        const { ezs } = this;
+        const total = await filteredItems.length;
+        if (total === 0) {
+            return feed.send({ total: 0 });
+        }
+        const path = ['total'];
+        const value = [total];
+
+        if (this.isLast()) {
+            return feed.close();
+        }
+
+        const readableStream = new Readable({
+            read() {
+                filteredItems.map(item => this.push(item));
+                this.push(null);
+            },
+            objectMode: true,
+        });
+
+        readableStream.pipe(ezs('assign', { path, value }));
+        await feed.flow(readableStream);
+    };
+
+export default {
+    filterPrecomputed: createFunction(),
+};
diff --git a/packages/ezsLodex/src/index.js b/packages/ezsLodex/src/index.js
@@ -14,6 +14,7 @@ import JSONLDCompacter from './JSONLDCompacter';
 import JSONLDString from './JSONLDString';
 import JSONLDObject from './JSONLDObject';
 import disabled from './disabled';
+import filterPrecomputed from './filterPrecomputed';
 import runQuery from './runQuery';
 import reduceQuery from './reduceQuery';
 import formatOutput from './formatOutput';
@@ -55,6 +56,7 @@ const funcs = {
     injectSyndicationFrom,
     injectCountFrom,
     labelizeFieldID,
+    filterPrecomputed,
     runQuery,
     reduceQuery,
     formatOutput,
@@ -72,6 +74,7 @@ const funcs = {
     LodexGetFields: getFields.getFields,
     LodexGetCharacteristics: getCharacteristics.getCharacteristics,
     LodexDocuments: runQuery.runQuery,
+    LodexFilterPrecomputed: filterPrecomputed.filterPrecomputed,
     LodexRunQuery: runQuery.runQuery,
     LodexReduceQuery: reduceQuery.reduceQuery,
     LodexOutput: formatOutput.formatOutput,