Maintain created time when reingesting an item

.eslintrc.json

@@ -10,6 +10,7 @@
   },
   "rules": {
     "dot-notation": "off",
+    "implicit-arrow-linebreak": "off",
     "no-restricted-syntax": [
       "off",
       "BinaryExpression[operator='of']"

.vscode/settings.json

@@ -1,7 +1,7 @@
 {
   "editor.formatOnSave": true,
   "editor.rulers": [
-    80
+    90
   ],
   "eslint.format.enable": true,
   "eslint.packageManager": "npm",

CHANGELOG.md

@@ -19,6 +19,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 - Open-ended datetime intervals using either empty string or '..' now work
 - Correct content types are now returned
 - Searching for a nonexistent collection returns empty results
+- Re-ingesting an Item maintains the existing value of properties.created and sets properties.updated to now
 
 ### Changed
 

package.json

@@ -56,6 +56,7 @@
     "@tsconfig/node14": "^1.0.1",
     "@types/compression": "^1.7.2",
     "@types/js-yaml": "^3.0.0",
+    "@types/luxon": "^2.0.9",
     "@types/memorystream": "^0.3.0",
     "@types/node": "^14.18.12",
     "@types/sinon": "^10.0.11",
@@ -71,6 +72,7 @@
     "eslint-plugin-jsdoc": "^3.5.0",
     "eslint-plugin-jsx-a11y": "^6.0.3",
     "eslint-plugin-react": "^7.7.0",
+    "luxon": "^2.3.1",
     "nock": "^12.0.3",
     "nodemon": "^2.0.15",
     "prettier": "^2.0.5",

src/lib/es.js

@@ -360,9 +360,27 @@ async function search(parameters, page = 1, limit = 10) {
   return response
 }
 
+const getItem = async (collectionId, itemId) => {
+  const searchResponse = await search({
+    collections: [collectionId],
+    id: itemId
+  })
+
+  return searchResponse.results[0]
+}
+
+const getItemCreated = async (collectionId, itemId) => {
+  const item = await getItem(collectionId, itemId)
+  if (!item) return undefined
+  if (!item.properties) return undefined
+  return item.properties.created
+}
+
 module.exports = {
   getCollection,
   getCollections,
+  getItem,
+  getItemCreated,
   isIndexNotFoundError,
   search,
   editPartialItem,

src/lib/esStream.js

@@ -2,6 +2,7 @@ const _stream = require('stream')
 const through2 = require('through2')
 const logger = console //require('./logger')
 const esClient = require('./esClient')
+const { getItemCreated } = require('./es')
 
 const COLLECTIONS_INDEX = process.env.COLLECTIONS_INDEX || 'collections'
 
@@ -97,7 +98,7 @@ async function stream() {
   try {
     const client = await esClient.client()
 
-    const toEs = through2.obj({ objectMode: true }, (data, encoding, next) => {
+    const toEs = through2.obj({ objectMode: true }, async (data, encoding, next) => {
       let index = ''
       logger.debug(`Data: ${JSON.stringify(data)}`)
       if (data && data.hasOwnProperty('extent')) {
@@ -115,8 +116,12 @@ async function stream() {
       const esDataObject = { ...data, links }
 
       if (data.hasOwnProperty('properties')) {
-        esDataObject.properties.created = new Date().toISOString()
-        esDataObject.properties.updated = new Date().toISOString()
+        const now = (new Date()).toISOString()
+
+        const created = (await getItemCreated(data.collection, data.id)) || now
+
+        esDataObject.properties.created = created
+        esDataObject.properties.updated = now
       }
 
       // create ES record
@@ -130,6 +135,7 @@ async function stream() {
           doc_as_upsert: true
         }
       }
+
       next(null, record)
     })
 

tests/helpers/api-client.js

@@ -1,4 +1,6 @@
-const got = require('got')
+// @ts-check
+
+const { default: got } = require('got')
 
 const apiClient = got.extend({
   prefixUrl: 'http://localhost:3000/',
@@ -12,10 +14,28 @@ const apiClient = got.extend({
 const getCollectionIds = async () => {
   const response = await apiClient.get('collections')
 
+  // @ts-expect-error We need to be validating this response
   return response.collections.map((c) => c.id)
 }
 
+/**
+ * @param {string} collectionId
+ * @returns {Promise<unknown>}
+ */
+const getCollection = async (collectionId) =>
+  apiClient.get(`collections/${collectionId}`)
+
+/**
+ * @param {string} collectionId
+ * @param {string} itemId
+ * @returns {Promise<unknown>}
+ */
+const getItem = async (collectionId, itemId) =>
+  apiClient.get(`collections/${collectionId}/items/${itemId}`)
+
 module.exports = {
   apiClient,
+  getCollection,
+  getItem,
   getCollectionIds
 }

tests/system/test-api-get-collection.js

@@ -1,5 +1,3 @@
-// @ts-check
-
 const { default: anyTest } = require('ava')
 const { apiClient } = require('../helpers/api-client')
 const { deleteAllIndices } = require('../helpers/es')
@@ -49,6 +47,7 @@ test('GET /collections/:collectionId returns a collection', async (t) => {
 
   const response = await apiClient.get(`collections/${collectionId}`)
 
+  // @ts-expect-error We need to validate these responses
   t.is(response.id, collectionId)
 })
 

tests/system/test-api-get-collections.js

@@ -1,5 +1,3 @@
-// @ts-check
-
 const { default: anyTest } = require('ava')
 const { apiClient } = require('../helpers/api-client')
 const { ingestItem } = require('../helpers/ingest')

tests/system/test-api-get-conformance.js

@@ -1,5 +1,3 @@
-// @ts-check
-
 const { default: test } = require('ava')
 const { apiClient } = require('../helpers/api-client')
 

tests/system/test-api-get-item.js

@@ -1,5 +1,3 @@
-// @ts-check
-
 const { default: anyTest } = require('ava')
 const { apiClient } = require('../helpers/api-client')
 const { deleteAllIndices } = require('../helpers/es')

tests/system/test-api-get-items.js

@@ -1,5 +1,3 @@
-// @ts-check
-
 const { default: anyTest } = require('ava')
 const { apiClient } = require('../helpers/api-client')
 const { deleteAllIndices } = require('../helpers/es')

tests/system/test-api-get-root.js

@@ -1,5 +1,3 @@
-// @ts-check
-
 const { default: anyTest } = require('ava')
 const { apiClient } = require('../helpers/api-client')
 const { deleteAllIndices } = require('../helpers/es')

tests/system/test-api-get-search.js

@@ -1,5 +1,3 @@
-// @ts-check
-
 const { default: anyTest } = require('ava')
 const { apiClient } = require('../helpers/api-client')
 const { deleteAllIndices } = require('../helpers/es')

tests/system/test-api-post-search.js

@@ -1,5 +1,3 @@
-// @ts-check
-
 const { default: anyTest } = require('ava')
 const { promisify } = require('util')
 const fs = require('fs')

tests/system/test-ingest.js

@@ -2,14 +2,16 @@
 
 const { default: anyTest } = require('ava')
 const nock = require('nock')
-const { getCollectionIds } = require('../helpers/api-client')
+const { DateTime } = require('luxon')
+const { getCollectionIds, getItem } = require('../helpers/api-client')
 const { handler } = require('../../src/lambdas/ingest')
 const { loadFixture, randomId } = require('../helpers/utils')
 const { nullLoggerContext } = require('../helpers/context')
 const { refreshIndices, deleteAllIndices } = require('../helpers/es')
 const { sqsTriggerLambda, purgeQueue } = require('../helpers/sqs')
 const awsClients = require('../../src/lib/aws-clients')
 const systemTests = require('../helpers/system-tests')
+const { ingestItem } = require('../helpers/ingest')
 
 /**
  * @template T
@@ -135,3 +137,55 @@ test('The ingest lambda supports ingesting a collection sourced from http', asyn
 
   t.true(collectionIds.includes(collection.id))
 })
+
+test('Reingesting an item maintains the `created` value and updates `updated`', async (t) => {
+  const { ingestQueueUrl, ingestTopicArn } = t.context
+  if (ingestQueueUrl === undefined) throw new Error('ingestQueueUrl undefined')
+  if (ingestTopicArn === undefined) throw new Error('ingestTopicArn undefined')
+
+  const collection = await loadFixture(
+    'landsat-8-l1-collection.json',
+    { id: randomId('collection') }
+  )
+
+  await ingestItem({
+    ingestTopicArn,
+    ingestQueueUrl,
+    item: collection
+  })
+
+  const item = await loadFixture(
+    'stac/LC80100102015082LGN00.json',
+    {
+      id: randomId('item'),
+      collection: collection.id
+    }
+  )
+
+  await ingestItem({
+    ingestQueueUrl,
+    ingestTopicArn,
+    item
+  })
+
+  const originalItem = await getItem(collection.id, item.id)
+  // @ts-expect-error Need to validate these responses
+  const originalCreated = DateTime.fromISO(originalItem.properties.created)
+  // @ts-expect-error Need to validate these responses
+  const originalUpdated = DateTime.fromISO(originalItem.properties.updated)
+
+  await ingestItem({
+    ingestQueueUrl,
+    ingestTopicArn,
+    item
+  })
+
+  const updatedItem = await getItem(collection.id, item.id)
+  // @ts-expect-error Need to validate these responses
+  const updatedCreated = DateTime.fromISO(updatedItem.properties.created)
+  // @ts-expect-error Need to validate these responses
+  const updatedUpdated = DateTime.fromISO(updatedItem.properties.updated)
+
+  t.is(updatedCreated.toISO(), originalCreated.toISO())
+  t.true(updatedUpdated.toISO() > originalUpdated.toISO())
+})