support open-ended datetime intervals, fix camelcase violations

.eslintrc.json

@@ -9,13 +9,13 @@
     "mocha": true
   },
   "rules": {
+    "no-restricted-syntax": ["off", "BinaryExpression[operator='of']"],
     "object-curly-newline": 0,
     "semi": [
       2,
       "never"
     ],
     "quote-props": 0,
-    "camelcase": 0,
     "func-names": 0,
     "no-param-reassign": "off",
     "no-prototype-builtins": "off",

CHANGELOG.md

@@ -12,6 +12,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 
 ### Fixed
 
+- open-ended datetime intervals using either empty string or '..' now work
+
 ### Changed
 - Upgrade to Node 14
 - Elasticsearch version update 7.9 -> 7.10

fixtures/common.js

@@ -11,6 +11,7 @@ const properties = {
   }
 }
 
+// eslint-disable-next-line camelcase
 const dynamic_templates = [
   {
     descriptions: {

src/lambdas/ingest/index.js

@@ -8,8 +8,8 @@ const logger = console
 module.exports.handler = async function handler(event) {
   logger.debug(`Event: ${JSON.stringify(event)}`)
   if (event.create_indices) {
-    await esClient.create_index('collections')
-    //await satlib.es.create_index('items')
+    await esClient.createIndex('collections')
+    //await satlib.es.createIndex('items')
   }
 
   // start with message as is

src/lib/api.js

@@ -263,16 +263,16 @@ const addItemLinks = function (results, endpoint) {
 }
 
 const collectionsToCatalogLinks = function (results, endpoint) {
-  const stac_version = process.env.STAC_VERSION
-  const stac_id = process.env.STAC_ID || 'stac-server'
-  const stac_title = process.env.STAC_TITLE || 'A STAC API'
-  const stac_description = process.env.STAC_DESCRIPTION || 'A STAC API running on stac-server'
+  const stacVersion = process.env.STAC_VERSION
+  const catalogId = process.env.STAC_ID || 'stac-server'
+  const catalogTitle = process.env.STAC_TITLE || 'A STAC API'
+  const catalogDescription = process.env.STAC_DESCRIPTION || 'A STAC API running on stac-server'
   const catalog = {
-    stac_version,
+    stac_version: stacVersion,
     type: 'Catalog',
-    id: stac_id,
-    title: stac_title,
-    description: stac_description
+    id: catalogId,
+    title: catalogTitle,
+    description: catalogDescription
   }
 
   catalog.links = results.map((result) => {
@@ -399,15 +399,15 @@ const searchItems = async function (collectionId, queryParameters, backend, endp
       [key]: parameters[key]
     }), {})
 
-  let new_endpoint = `${endpoint}/search`
+  let newEndpoint = `${endpoint}/search`
   if (collectionId) {
     searchParameters.collections = [collectionId]
-    new_endpoint = `${endpoint}/collections/${collectionId}/items`
+    newEndpoint = `${endpoint}/collections/${collectionId}/items`
   }
   logger.debug(`Search parameters: ${JSON.stringify(searchParameters)}`)
   const results = await backend.search(searchParameters, page, limit)
   const { 'results': itemsResults, 'context': itemsMeta } = results
-  const pageLinks = buildPageLinks(itemsMeta, searchParameters, new_endpoint, httpMethod)
+  const pageLinks = buildPageLinks(itemsMeta, searchParameters, newEndpoint, httpMethod)
   const items = addItemLinks(itemsResults, endpoint)
   const response = wrapResponseInFeatureCollection(itemsMeta, items, pageLinks)
   return response

src/lib/es.js

@@ -43,20 +43,28 @@ function buildDatetimeQuery(parameters) {
   let dateQuery
   const { datetime } = parameters
   if (datetime) {
-    const dataRange = datetime.split('/')
-    if (dataRange.length === 2) {
+    if (!datetime.includes('/')) {
       dateQuery = {
-        range: {
-          'properties.datetime': {
-            gte: dataRange[0],
-            lte: dataRange[1]
-          }
+        term: {
+          'properties.datetime': datetime
         }
       }
     } else {
-      dateQuery = {
-        term: {
-          'properties.datetime': datetime
+      const [start, end, ...rest] = datetime.split('/')
+      if (rest.length) {
+        dateQuery = new Error('datetime value is invalid, contains more than one forward slash')
+      } else if ((!start && !end) || (start === '..' && start === end)) {
+        dateQuery = new Error(
+          'datetime value is invalid, at least one end of the interval must be closed'
+        )
+      } else {
+        const datetimeRange = {}
+        if (start && start !== '..') datetimeRange.gte = start
+        if (end && end !== '..') datetimeRange.lte = end
+        dateQuery = {
+          range: {
+            'properties.datetime': datetimeRange
+          }
         }
       }
     }
@@ -123,7 +131,9 @@ function buildQuery(parameters) {
   }
 
   const datetimeQuery = buildDatetimeQuery(parameters)
-  if (datetimeQuery) {
+  if (datetimeQuery instanceof Error) {
+    throw datetimeQuery
+  } else if (datetimeQuery) {
     must.push(datetimeQuery)
   }
 
@@ -344,5 +354,6 @@ module.exports = {
   getCollections,
   search,
   editPartialItem,
-  constructSearchParams
+  constructSearchParams,
+  buildDatetimeQuery
 }

src/lib/esClient.js

@@ -5,8 +5,8 @@ const { createAWSConnection, awsCredsifyAll } = require('@acuris/aws-es-connecti
 const elasticsearch = require('@elastic/elasticsearch')
 const logger = console //require('./logger')
 
-const collections_mapping = require('../../fixtures/collections')()
-const items_mapping = require('../../fixtures/items')()
+const collectionsMapping = require('../../fixtures/collections')()
+const itemsMapping = require('../../fixtures/items')()
 
 let _esClient
 
@@ -54,10 +54,10 @@ async function esClient() {
   return _esClient
 }
 
-async function create_index(index) {
+async function createIndex(index) {
   const client = await esClient()
   const exists = await client.indices.exists({ index })
-  const mapping = (index === 'collections' ? collections_mapping : items_mapping)
+  const mapping = (index === 'collections' ? collectionsMapping : itemsMapping)
   if (!exists.body) {
     logger.info(`${index} does not exist, creating...`)
     try {
@@ -73,5 +73,5 @@ async function create_index(index) {
 
 module.exports = {
   client: esClient,
-  create_index
+  createIndex
 }

src/lib/esStream.js

@@ -61,7 +61,7 @@ class ElasticSearchWritableStream extends _stream.Writable {
 
       // if this was a collection, then add a new index with collection name
       if (index === COLLECTIONS_INDEX) {
-        await esClient.create_index(id)
+        await esClient.createIndex(id)
       }
 
       next()

tests/system/setup-es.js

@@ -19,7 +19,7 @@ const main = async () => {
 
   await esClient.indices.delete({ index: '*' })
 
-  await client.create_index('collections')
+  await client.createIndex('collections')
 
   const fixtureFiles = [
     'catalog.json',

tests/unit/test-es.js

@@ -21,3 +21,69 @@ test('search id parameter doesnt override other parameters', async (t) => {
     'query contains datetime filter'
   )
 })
+
+test('invalid search datetime parameter intervals fail', async (t) => {
+  const invalidDatetimeIntervals = [
+    '1985-04-12T23:20:50.52-01:00/1986-04-12T23:20:50.52-01:00/'
+  ]
+
+  await invalidDatetimeIntervals.map(async (datetime) => {
+    const dtQuery = await es.buildDatetimeQuery({ datetime: datetime })
+    t.assert(dtQuery instanceof Error, 'invalid datetime interval should fail')
+  })
+})
+
+/* eslint max-len: 0 */
+test('search datetime parameter intervals are correctly parsed', async (t) => {
+  const datetimes = [
+    ['1985-04-12T23:20:50.52-01:00/1986-04-12T23:20:50.52-01:00', '1985-04-12T23:20:50.52-01:00', '1986-04-12T23:20:50.52-01:00'],
+    ['../1985-04-12T23:20:50.52Z', undefined, '1985-04-12T23:20:50.52Z'],
+    ['1985-04-12T23:20:50.52Z/..', '1985-04-12T23:20:50.52Z', undefined],
+    ['1985-04-12T23:20:50.52Z/', '1985-04-12T23:20:50.52Z', undefined],
+    ['../1985-04-12T23:20:50.52Z', undefined, '1985-04-12T23:20:50.52Z'],
+    ['/1985-04-12T23:20:50.52Z', undefined, '1985-04-12T23:20:50.52Z'],
+    ['1985-04-12T23:20:50.52Z/1986-04-12T23:20:50.52Z', '1985-04-12T23:20:50.52Z', '1986-04-12T23:20:50.52Z'],
+    ['1985-04-12T23:20:50.52+01:00/1986-04-12T23:20:50.52+01:00', '1985-04-12T23:20:50.52+01:00', '1986-04-12T23:20:50.52+01:00'],
+    ['1985-04-12T23:20:50.52-01:00/1986-04-12T23:20:50.52-01:00', '1985-04-12T23:20:50.52-01:00', '1986-04-12T23:20:50.52-01:00']
+  ]
+
+  await datetimes.map(async ([datetime, start, end]) => {
+    const dtQuery = await es.buildDatetimeQuery({ datetime: datetime })
+    t.is(dtQuery.range['properties.datetime'].gte, start, 'datetime interval start')
+    t.is(dtQuery.range['properties.datetime'].lte, end, 'datetime interval end')
+  })
+})
+
+test('search datetime parameter instants are correctly parsed', async (t) => {
+  const validDatetimes = [
+    '1985-04-12T23:20:50.52Z',
+    '1996-12-19T16:39:57-00:00',
+    '1996-12-19T16:39:57+00:00',
+    '1996-12-19T16:39:57-08:00',
+    '1996-12-19T16:39:57+08:00',
+    '1937-01-01T12:00:27.87+01:00',
+    '1985-04-12T23:20:50.52Z',
+    '1937-01-01T12:00:27.8710+01:00',
+    '1937-01-01T12:00:27.8+01:00',
+    '1937-01-01T12:00:27.8Z',
+    '2020-07-23T00:00:00.000+03:00',
+    '2020-07-23T00:00:00+03:00',
+    '1985-04-12t23:20:50.000z',
+    '2020-07-23T00:00:00Z',
+    '2020-07-23T00:00:00.0Z',
+    '2020-07-23T00:00:00.01Z',
+    '2020-07-23T00:00:00.012Z',
+    '2020-07-23T00:00:00.0123Z',
+    '2020-07-23T00:00:00.01234Z',
+    '2020-07-23T00:00:00.012345Z',
+    '2020-07-23T00:00:00.0123456Z',
+    '2020-07-23T00:00:00.01234567Z',
+    '2020-07-23T00:00:00.012345678Z',
+    '1985-04-12' // date only is not required by STAC, but accepted here
+  ]
+
+  await validDatetimes.map(async (datetime) => {
+    const dtQuery = await es.buildDatetimeQuery({ datetime: datetime })
+    t.is(dtQuery.term['properties.datetime'], datetime, 'datetime instant parses correctly')
+  })
+})