Initial commit

.eslintignore

@@ -0,0 +1,2 @@
+node_modules
+dist

.eslintrc

@@ -0,0 +1,32 @@
+{
+  "root": true,
+  "parser": "@typescript-eslint/parser",
+  "parserOptions": {
+    "project": "./tsconfig.json"
+  },
+  "plugins": ["@typescript-eslint", "prettier"],
+  "extends": [
+    "eslint:recommended",
+    "plugin:@typescript-eslint/eslint-recommended",
+    "plugin:@typescript-eslint/recommended",
+    "prettier"
+  ],
+  "env": {
+    "jest": true
+  },
+  "rules": {
+    "no-console": 1, // Means warning
+    "prettier/prettier": 2, // Means error
+    "@typescript-eslint/ban-ts-comment": "off",
+    "@typescript-eslint/no-floating-promises": ["error"],
+    "@typescript-eslint/no-misused-promises": ["error"],
+    "@typescript-eslint/promise-function-async": ["error"],
+    "@typescript-eslint/require-await": ["error"],
+    // note you must disable the base rule as it can report incorrect errors
+    "no-return-await": "off",
+    "@typescript-eslint/return-await": ["error"],
+    // Don't allow awaiting non-Promises
+    "@typescript-eslint/await-thenable": "error"
+  },
+  "ignorePatterns": ["dist", "cdk.out"]
+}

.github/workflows/ci.yml

@@ -0,0 +1,33 @@
+name: Build - CI
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+
+      - name: Use Node.js 16
+        uses: actions/setup-node@v3
+        with:
+          node-version: 16
+
+      - name: Install Modules
+        run: npm ci
+
+      - name: Build
+        run: npm run build
+
+      - name: Lint
+        run: npm run lint
+
+      - name: Test
+        run: npm run test

.github/workflows/docs.yml

@@ -0,0 +1,31 @@
+name: Publish Docs
+
+on:
+  release:
+    types: [published]
+
+  workflow_dispatch:
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+
+      - name: Use Node.js 16
+        uses: actions/setup-node@v3
+        with:
+          node-version: 16
+
+      - name: Install Modules
+        run: npm ci
+
+      - name: Build Docs
+        run: npm run build:docs
+
+      - name: Deploy to GitHub Pages
+        uses: peaceiris/actions-gh-pages@v3
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          publish_dir: ./docs

.github/workflows/publish.yml

@@ -0,0 +1,40 @@
+name: Package and Publish
+
+on:
+  release:
+    types: [published]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+
+      - name: Use Node.js 16
+        uses: actions/setup-node@v3
+        with:
+          node-version: 16
+
+      - name: Use the Release Tag Version
+        run: |
+          npm version from-git --allow-same-version --no-git-tag-version
+
+      - name: Install Modules
+        run: npm ci
+
+      - name: Build
+        run: npm run build
+
+      - name: Lint
+        run: npm run lint
+
+      - name: Test
+        run: npm run test
+
+      - name: Publish with CLI to Code Artifact
+        run: |
+          npm publish --access public --ignore-scripts --dry-run

.gitignore

@@ -0,0 +1,9 @@
+.idea/
+node_modules
+dist/
+*.tsbuildinfo
+*.log
+.nyc_output/
+coverage/
+.DS_Store
+docs/

.nvmrc

@@ -0,0 +1 @@
+v16.17.1

.prettierrc

@@ -0,0 +1,6 @@
+{
+  "semi": true,
+  "trailingComma": "all",
+  "singleQuote": true,
+  "printWidth": 100
+}

.vscode/settings.json

@@ -0,0 +1,50 @@
+{
+  "files.exclude": {
+    // "**/node_modules/*": true,
+    // "cdk.out": true
+    // "node_modules": true,
+    // "**/node_modules": true
+  },
+  "editor.defaultFormatter": "esbenp.prettier-vscode",
+  "editor.formatOnSave": true,
+  "editor.insertSpaces": true,
+  "editor.tabSize": 2,
+  "files.associations": {
+    "Dockerfile*": "dockerfile"
+  },
+  "[typescript]": {
+    "editor.defaultFormatter": "esbenp.prettier-vscode"
+  },
+  "yaml.customTags": [
+    "!And",
+    "!And sequence",
+    "!If",
+    "!If sequence",
+    "!Not",
+    "!Not sequence",
+    "!Equals",
+    "!Equals sequence",
+    "!Or",
+    "!Or sequence",
+    "!FindInMap",
+    "!FindInMap sequence",
+    "!Base64",
+    "!Join",
+    "!Join sequence",
+    "!Cidr",
+    "!Ref",
+    "!Sub",
+    "!Sub sequence",
+    "!GetAtt",
+    "!GetAZs",
+    "!ImportValue",
+    "!ImportValue sequence",
+    "!Select",
+    "!Select sequence",
+    "!Split",
+    "!Split sequence"
+  ],
+  "[xml]": {
+    "editor.defaultFormatter": "DotJoshJohnson.xml"
+  }
+}

CODE_OF_CONDUCT.md

@@ -0,0 +1,5 @@
+# Contributor Code of Conduct
+
+The Shutterstock team is committed to fostering a welcoming community.
+
+This project is governed by Shutterstock’s [Code of Conduct](https://github.com/shutterstock/code-of-conduct). All contributors and participants agree to abide by its terms.

ISSUE_TEMPLATE.md

@@ -0,0 +1,8 @@
+### Expected behavior
+
+### Actual behavior
+
+### Steps to reproduce the behavior
+
+### Additional specs (e.g. browser, version, etc.)
+

LICENSE.md

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+=====================
+Copyright (c) `2022` `Shutterstock, Inc.`
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
+of the Software, and to permit persons to whom the Software is furnished to do
+so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -0,0 +1,73 @@
+[![License: MIT](https://img.shields.io/badge/License-MIT-green.svg)](https://opensource.org/licenses/MIT) [![API Docs](https://img.shields.io/badge/API%20Docs-View%20Here-blue)](https://tech.shutterstock.com/kinesis-helpers/) [![Build - CI](https://github.com/shutterstock/kinesis-helpers/actions/workflows/ci.yml/badge.svg)](https://github.com/shutterstock/kinesis-helpers/actions/workflows/ci.yml) [![Package and Publish](https://github.com/shutterstock/kinesis-helpers/actions/workflows/publish.yml/badge.svg)](https://github.com/shutterstock/kinesis-helpers/actions/workflows/publish.yml) [![Publish Docs](https://github.com/shutterstock/kinesis-helpers/actions/workflows/docs.yml/badge.svg)](https://github.com/shutterstock/kinesis-helpers/actions/workflows/docs.yml)
+
+# Overview
+
+Helper classes for the AWS v3 SDK for JS Kinesis Client (`@aws-sdk/client-kinesis`).
+
+`KinesisRetrier` will retry failed records with a "successful" `PutRecordsCommand` that has items marked as failed within the response, which then need to be retried.
+
+`KinesisBackgroundWriter` will accept writes immediately if there is a concurrent writer slot available, and will block the caller until a slot becomes available if all slots are in use.
+
+# Getting Started
+
+## Installation
+
+The package is available on npm as [@shutterstock/kinesis-helpers](https://www.npmjs.com/package/@shutterstock/kinesis-helpers)
+
+`npm i @shutterstock/kinesis-helpers`
+
+## Importing
+
+```typescript
+import { KinesisBackgroundWriter, KinesisRetrier } from '@shutterstock/kinesis-helpers';
+```
+
+## API Documentation
+
+After installing the package, you might want to look at our [API Documentation](https://tech.shutterstock.com/kinesis-helpers/) to learn about all the features available.
+
+# Features
+
+- [KinesisRetrier](https://tech.shutterstock.com/kinesis-helpers/classes/KinesisRetrier.html)
+  - Kinesis batch put retrier
+  - Batch puts return a list of items that were throttled
+  - The batch is marked as succeeded (200 status code) even though items failed
+  - As a result, the built-in AWS SDK retry logic will not retry the items in the batch that were throttle
+- [KinesisBackgroundWriter](https://tech.shutterstock.com/kinesis-helpers/classes/KinesisBackgroundWriter.html)
+  - Accepts writes immediately if there is a concurrent writer slot available
+  - Blocks the caller until a slot becomes available if all slots are in use
+  - Collects and exposes errors so the caller can detect if errors have been happening
+
+# Contributing
+
+## Setting up Build Environment
+
+- `nvm use`
+- `npm i`
+- `npm run build`
+- `npm run lint`
+- `npm run test`
+
+## Running Examples
+
+### kinesis-retrier
+
+1. Create Kinesis Data Stream using AWS Console or any other method
+   1. Default name is `kinesis-helpers-test-stream`
+   2. 1 shard is sufficient
+   3. 1 day retention is sufficient
+   4. No encryption is sufficient
+   5. On-demand throughput is sufficient
+2. `npm run example:kinesis-retrier`
+   1. If the stream name was changed: `KINESIS_STREAM_NAME=my-stream-name npm run example:kinesis-retrier`
+
+### kinesis-background-writer
+
+1. Create Kinesis Data Stream using AWS Console or any other method
+   1. Default name is `kinesis-helpers-test-stream`
+   2. 1 shard is sufficient
+   3. 1 day retention is sufficient
+   4. No encryption is sufficient
+   5. On-demand throughput is sufficient
+2. `npm run example:kinesis-background-writer`
+   1. If the stream name was changed: `KINESIS_STREAM_NAME=my-stream-name npm run example:kinesis-background-writer`

examples/kinesis-background-writer.ts

@@ -0,0 +1,49 @@
+import * as kinesis from '@aws-sdk/client-kinesis';
+import { KinesisBackgroundWriter, KinesisRetrier } from '@shutterstock/kinesis-helpers';
+
+const kinesisClient = new kinesis.KinesisClient({});
+const { KINESIS_STREAM_NAME = 'kinesis-helpers-test-stream', RECORDS_TO_WRITE = '1000000' } =
+  process.env;
+const RECORDS_TO_WRITE_NUM = parseInt(RECORDS_TO_WRITE, 10);
+const RECORDS_PER_BATCH = 500;
+
+async function main() {
+  // Use a KinesisRetrier so that we do not get throttling exceptions
+  // within the background batch writes
+  const kinesisRetrier = new KinesisRetrier({
+    kinesisClient,
+  });
+
+  const backgroundWriter = new KinesisBackgroundWriter({
+    kinesisClient: kinesisRetrier,
+    concurrency: 4,
+  });
+
+  const records: kinesis.PutRecordsCommandInput = {
+    StreamName: KINESIS_STREAM_NAME,
+    Records: [],
+  };
+
+  // Thanks TypeScript?  I guess? The value is assigned above but
+  // we are getting "possibly undefined" in the loop below
+  records.Records = [];
+
+  for (let i = 0; i < RECORDS_PER_BATCH; i++) {
+    records.Records.push({
+      Data: Buffer.from('123', 'utf-8'),
+      PartitionKey: '123',
+    });
+  }
+
+  // Send a whole lot of records so we start getting throttled within the batches
+  for (let i = 0; i < RECORDS_TO_WRITE_NUM; i += RECORDS_PER_BATCH) {
+    await backgroundWriter.send(new kinesis.PutRecordsCommand(records));
+  }
+
+  // Need to wait until the backgroundWriter is idle (has finished any pending requests)
+  await backgroundWriter.onIdle();
+
+  // TODO: If there were any errors, log them
+}
+
+void main();