Add client side encryption before Send files to S3

.gitignore

@@ -4,6 +4,7 @@
 /dist
 /tmp
 /out-tsc
+**/dist
 
 # dependencies
 /node_modules

README.md

@@ -28,15 +28,16 @@ This will make the necessary changes to nx.json in your workspace to use nx-aws-
 
 There are two ways to set-up plugin options, using `nx.json` or `Environment variables`. Here is a list of all possible options:
 
-| Parameter         | Description                                                                                         | Environment variable / .env     | `nx.json`            | Example                        |
-| ----------------- | --------------------------------------------------------------------------------------------------- | ------------------------------- | -------------------- | ------------------------------ |
-| Access Key Id     | Access Key Id.                                                                                      | `NXCACHE_AWS_ACCESS_KEY_ID`     | `awsAccessKeyId`     | my-id                          |
-| Secret Access Key | Secret Access Key.                                                                                  | `NXCACHE_AWS_SECRET_ACCESS_KEY` | `awsSecretAccessKey` | my-key                         |
-| Profile           | Configuration profile to use (applied only if Access Key Id and Secret Access Key are not set).     | `NXCACHE_AWS_PROFILE`           | `awsProfile`         | profile-1                      |
-| Endpoint          | Fully qualified endpoint of the web service if a custom endpoint is needed (e.g. when using MinIO). | `NXCACHE_AWS_ENDPOINT`          | `awsEndpoint`        | http://custom.de-eu.myhost.com |
-| Region            | Region to which this client will send requests.                                                     | `NXCACHE_AWS_REGION`            | `awsRegion`          | eu-central-1                   |
-| Bucket            | Bucket name where cache files are stored or retrieved (can contain sub-paths as well).              | `NXCACHE_AWS_BUCKET`            | `awsBucket`          | bucket-name/sub-path           |
-| Force Path Style  | Whether to force path style URLs for S3 objects (e.g. when using MinIO).                            | `NXCACHE_AWS_FORCE_PATH_STYLE`  | `awsForcePathStyle`  | true                           |
+| Parameter         | Description                                                                                         | Environment variable / .env                               | `nx.json`            | Example                                      |
+| ----------------- | --------------------------------------------------------------------------------------------------- | --------------------------------------------------------- | -------------------- | -------------------------------------------- |
+| Access Key Id     | Access Key Id.                                                                                      | `NXCACHE_AWS_ACCESS_KEY_ID`                               | `awsAccessKeyId`     | my-id                                        |
+| Secret Access Key | Secret Access Key.                                                                                  | `NXCACHE_AWS_SECRET_ACCESS_KEY`                           | `awsSecretAccessKey` | my-key                                       |
+| Profile           | Configuration profile to use (applied only if Access Key Id and Secret Access Key are not set).     | `NXCACHE_AWS_PROFILE`                                     | `awsProfile`         | profile-1                                    |
+| Endpoint          | Fully qualified endpoint of the web service if a custom endpoint is needed (e.g. when using MinIO). | `NXCACHE_AWS_ENDPOINT`                                    | `awsEndpoint`        | http://custom.de-eu.myhost.com               |
+| Region            | Region to which this client will send requests.                                                     | `NXCACHE_AWS_REGION`                                      | `awsRegion`          | eu-central-1                                 |
+| Bucket            | Bucket name where cache files are stored or retrieved (can contain sub-paths as well).              | `NXCACHE_AWS_BUCKET`                                      | `awsBucket`          | bucket-name/sub-path                         |
+| Force Path Style  | Whether to force path style URLs for S3 objects (e.g. when using MinIO).                            | `NXCACHE_AWS_FORCE_PATH_STYLE`                            | `awsForcePathStyle`  | true                                         |
+| Enrypt File Key   | Encrypt file in client before send to S3 server. 32 bytes in base64                                 | `NXCACHE_AWS_ENCRYPTION_KEY` or `NX_CLOUD_ENCRYPTION_KEY` | `encryptionFileKey`  | PcZrGOSda3zwWh9yYTJB5bnHORgXf3dphj55tPI74O0= |
 
 > **Important:** `Environment variables` take precedence over `nx.json` options (introduced in v3.0.0)!
 
@@ -55,6 +56,7 @@ There are two ways to set-up plugin options, using `nx.json` or `Environment var
       "awsBucket": "bucket-name/sub-path",
       "awsRegion": "eu-central-1",
       "awsForcePathStyle": true,
+      "encryptionFileKey": "Pbfk58EpcK7IxTxWwSXNsTAKmzhJQE+99vkpGftyJg8="
     }
   }
 }
@@ -66,6 +68,16 @@ There are two ways to set-up plugin options, using `nx.json` or `Environment var
 - `.env.local`
 - `.env`
 
+## Encryption
+
+If you want you can encrypt file before send to s3 server (Client side). Use ENV
+NX_CLOUD_ENCRYPTION_KEY or NXCACHE_CLOUD_ENCRYPTION_KEY
+
+```sh
+# Generate new key for encryption.
+openssl rand -base64 32
+```
+
 ## Disabling S3 cache
 
 Remote cache can be disabled in favor of local cache using an environment variable
@@ -92,7 +104,7 @@ AWS SDK v3 is used under the hood with a support for [SSO login](https://docs.aw
 
 Run `yarn nx build nx-aws-cache` to build the plugin. The build artifacts will be stored in the `dist/` directory. Use the `--prod` flag for a production build.
 
-## Running unit tests
+## Running unit/integration tests
 
 Run `yarn nx test nx-aws-cache` to execute the unit tests via [Jest](https://jestjs.io).
 

package.json

@@ -55,6 +55,7 @@
   "dependencies": {
     "@aws-sdk/client-s3": "3.474.0",
     "@aws-sdk/credential-providers": "3.474.0",
+    "@aws-sdk/lib-storage": "^3.474.0",
     "@aws-sdk/property-provider": "^3.374.0",
     "@swc/helpers": "~0.5.3",
     "dotenv": "16.3.1",
@@ -63,6 +64,7 @@
   "devDependencies": {
     "@actions/core": "1.10.1",
     "@actions/github": "5.1.1",
+    "@aws-sdk/util-stream-node": "^3.374.0",
     "@nx/devkit": "16.2.2",
     "@nx/eslint-plugin": "16.2.2",
     "@nx/jest": "16.2.2",
@@ -75,6 +77,7 @@
     "@types/tar": "6.1.6",
     "@typescript-eslint/eslint-plugin": "5.18.0",
     "@typescript-eslint/parser": "5.18.0",
+    "aws-sdk-client-mock": "^3.0.0",
     "eslint": "8.13.0",
     "eslint-config-prettier": "8.5.0",
     "eslint-plugin-import": "2.25.4",

packages/nx-aws-cache/src/generators/init/generator.spec.ts

@@ -15,11 +15,11 @@ describe('init generator', () => {
     appTree = createTreeWithEmptyWorkspace();
   });
 
-  it('should add @nx-aws-plugin/nx-aws-cache to nx.json', () => {
+  it('should add @nx-aws-plugin/nx-aws-cache to nx.json', async () => {
     let nxJson = readJson(appTree, 'nx.json');
     expect(nxJson.tasksRunnerOptions.default.runner).toBe('nx/tasks-runners/default');
 
-    generator(appTree, options);
+    await generator(appTree, options);
 
     nxJson = readJson(appTree, 'nx.json');
 
@@ -28,11 +28,11 @@ describe('init generator', () => {
     expect(nxJson.tasksRunnerOptions.default.options.awsBucket).toBe('bucket-name');
   });
 
-  it('should add @nx-aws-plugin/nx-aws-cache with no aws options to nx.json', () => {
+  it('should add @nx-aws-plugin/nx-aws-cache with no aws options to nx.json', async () => {
     let nxJson = readJson(appTree, 'nx.json');
     expect(nxJson.tasksRunnerOptions.default.runner).toBe('nx/tasks-runners/default');
 
-    generator(appTree, {});
+    await generator(appTree, {});
 
     nxJson = readJson(appTree, 'nx.json');
 

packages/nx-aws-cache/src/generators/init/generator.ts

@@ -49,6 +49,7 @@ function updateNxJson(tree: Tree, options: InitGeneratorSchema): void {
           ...(options.awsRegion ? { awsRegion: options.awsRegion } : {}),
           ...(options.awsBucket ? { awsBucket: options.awsBucket } : {}),
           ...(options.awsForcePathStyle ? { awsForcePathStyle: options.awsForcePathStyle } : {}),
+          ...(options.encryptionFileKey ? { encryptionFileKey: options.encryptionFileKey } : {}),
         },
       },
     };

packages/nx-aws-cache/src/generators/init/schema.d.ts

@@ -6,4 +6,5 @@ export interface InitGeneratorSchema {
   awsRegion?: string;
   awsBucket?: string;
   awsForcePathStyle?: boolean;
+  encryptionFileKey?: string;
 }

packages/nx-aws-cache/src/generators/init/schema.json

@@ -38,6 +38,11 @@
       "type": "boolean",
       "description": "Whether to force path style URLs for S3 objects (e.g. when using MinIO)",
       "x-prompt": "Whether to force path style URLs for S3 objects (e.g. when using MinIO)"
+    },
+    "encryptionFileKey": {
+      "type": "string",
+      "description": "Encrypt file in client before send to S3 server. 32 bytes in base64",
+      "x-prompt": "Encrypt file in client before send to S3 server. 32 bytes in base64"
     }
   }
 }

packages/nx-aws-cache/src/tasks-runner/aws-cache.spec.ts

@@ -0,0 +1,92 @@
+import * as fs from 'fs';
+import * as os from 'os';
+import { randomUUID } from 'crypto';
+import * as path from 'path';
+import { mockClient } from 'aws-sdk-client-mock';
+import { GetObjectCommand, S3Client } from '@aws-sdk/client-s3';
+import { sdkStreamMixin } from '@aws-sdk/util-stream-node';
+import { AwsCache } from './aws-cache';
+import { Logger } from './logger';
+import { MessageReporter } from './message-reporter';
+import { Encrypt, EncryptConfig } from './encryptor';
+
+// eslint-disable-next-line max-lines-per-function
+describe('Test aws put and get unencrypted file', () => {
+  let awsCache: AwsCache;
+  const s3Mock = mockClient(S3Client);
+  const hash = randomUUID();
+  const cacheDirectory = path.join(os.tmpdir(), 'aws-cache');
+  const cacheDirectorySave = path.join(os.tmpdir(), 'aws-cache-decompress');
+  const fileContent = 'console.log(123)';
+  let filePath = '';
+
+  const config = {
+    encryptionFileKey: 'Pbfk58EpcK7IxTxWwSXNsTAKmzhJQE+99vkpGftyJg8=',
+    awsAccessKeyId: 'minio',
+    awsSecretAccessKey: 'minio123',
+    awsBucket: 'test',
+    awsEndpoint: 'http://127.0.0.1:9000',
+    awsForcePathStyle: true,
+    awsRegion: 'us-east-1',
+  };
+
+  beforeEach(() => {
+    fs.mkdirSync(cacheDirectory, {
+      recursive: true,
+    });
+    fs.mkdirSync(cacheDirectorySave, {
+      recursive: true,
+    });
+
+    const fileDir = path.join(cacheDirectory, `${hash}/outputs`);
+
+    fs.mkdirSync(fileDir, { recursive: true });
+    filePath = path.join(fileDir, 'test.js');
+    fs.writeFileSync(filePath, fileContent);
+  });
+
+  afterEach(() => {
+    jest.resetAllMocks();
+    s3Mock.reset();
+  });
+
+  it('Should save encrypted data in s3 file, and read an unencrypted', async () => {
+    awsCache = new AwsCache(config, new MessageReporter(new Logger()));
+
+    await awsCache.store(hash, cacheDirectory);
+
+    const tgzFilePath = path.join(cacheDirectory, `${hash}.tar.gz`);
+    const tgzFileStream = fs.createReadStream(tgzFilePath);
+    const sdkStream = sdkStreamMixin(
+      tgzFileStream.pipe(new Encrypt(new EncryptConfig(config.encryptionFileKey))),
+    );
+    s3Mock.on(GetObjectCommand).resolves({ Body: sdkStream });
+
+    await awsCache.retrieve(hash, cacheDirectorySave);
+
+    const extractedFilePath = path.join(cacheDirectorySave, `${hash}/outputs/test.js`);
+    expect(fs.existsSync(extractedFilePath)).toBeTruthy();
+    expect(fs.readFileSync(extractedFilePath).toString()).toBe(fileContent);
+  });
+
+  it('Should save in unencrypted s3 file, and read an unencrypted', async () => {
+    const configWithoutEncryption = {
+      ...config,
+      encryptionFileKey: '',
+    };
+    awsCache = new AwsCache(configWithoutEncryption, new MessageReporter(new Logger()));
+
+    await awsCache.store(hash, cacheDirectory);
+
+    const tgzFilePath = path.join(cacheDirectory, `${hash}.tar.gz`);
+    const tgzFileStream = fs.createReadStream(tgzFilePath);
+    const sdkStream = sdkStreamMixin(tgzFileStream);
+
+    s3Mock.on(GetObjectCommand).resolves({ Body: sdkStream });
+
+    await awsCache.retrieve(hash, cacheDirectorySave);
+    const extractedFilePath = path.join(cacheDirectorySave, `${hash}/outputs/test.js`);
+    expect(fs.existsSync(extractedFilePath)).toBeTruthy();
+    expect(fs.readFileSync(extractedFilePath).toString()).toBe(fileContent);
+  });
+});

packages/nx-aws-cache/src/tasks-runner/aws-cache.ts

@@ -2,23 +2,24 @@ import { createReadStream, createWriteStream, writeFile } from 'fs';
 import { join, dirname } from 'path';
 import { pipeline, Readable } from 'stream';
 import { promisify } from 'util';
-
 import * as clientS3 from '@aws-sdk/client-s3';
 import { fromNodeProviderChain } from '@aws-sdk/credential-providers';
 import { CredentialsProviderError } from '@aws-sdk/property-provider';
 import { RemoteCache } from '@nx/workspace/src/tasks-runner/default-tasks-runner';
 import { create, extract } from 'tar';
-
 import { AwsNxCacheOptions } from './models/aws-nx-cache-options.model';
 import { Logger } from './logger';
 import { MessageReporter } from './message-reporter';
+import { Encrypt, Decrypt, EncryptConfig } from './encryptor';
+import { Upload } from '@aws-sdk/lib-storage';
 
 export class AwsCache implements RemoteCache {
   private readonly bucket: string;
   private readonly path: string;
   private readonly s3: clientS3.S3Client;
   private readonly logger = new Logger();
   private readonly uploadQueue: Array<Promise<boolean>> = [];
+  private readonly encryptConfig: EncryptConfig | undefined;
 
   public constructor(options: AwsNxCacheOptions, private messages: MessageReporter) {
     const awsBucket = options.awsBucket ?? '';
@@ -51,6 +52,10 @@ export class AwsCache implements RemoteCache {
       clientConfig.forcePathStyle = true;
     }
 
+    if (options?.encryptionFileKey) {
+      this.encryptConfig = new EncryptConfig(options.encryptionFileKey);
+    }
+
     this.s3 = new clientS3.S3Client(clientConfig);
   }
 
@@ -72,10 +77,8 @@ export class AwsCache implements RemoteCache {
       await this.s3.config.credentials();
     } catch (err) {
       this.messages.error = err as Error;
-
       return false;
     }
-
     if (this.messages.error) {
       return false;
     }
@@ -113,7 +116,6 @@ export class AwsCache implements RemoteCache {
     }
 
     const resultPromise = this.createAndUploadFile(hash, cacheDirectory);
-
     this.uploadQueue.push(resultPromise);
 
     return resultPromise;
@@ -126,9 +128,15 @@ export class AwsCache implements RemoteCache {
   private async createAndUploadFile(hash: string, cacheDirectory: string): Promise<boolean> {
     try {
       const tgzFilePath = this.getTgzFilePath(hash, cacheDirectory);
-
       await this.createTgzFile(tgzFilePath, hash, cacheDirectory);
-      await this.uploadFile(hash, tgzFilePath);
+      const sourceFileStream = createReadStream(tgzFilePath);
+
+      await this.uploadFile(
+        hash,
+        this.encryptConfig
+          ? sourceFileStream.pipe(new Encrypt(this.encryptConfig))
+          : sourceFileStream,
+      );
 
       return true;
     } catch (err) {
@@ -170,29 +178,41 @@ export class AwsCache implements RemoteCache {
     }
   }
 
-  private async uploadFile(hash: string, tgzFilePath: string): Promise<void> {
-    const tgzFileName = this.getTgzFileName(hash);
-    const params: clientS3.PutObjectCommand = new clientS3.PutObjectCommand({
-      Bucket: this.bucket,
-      Key: this.getS3Key(tgzFileName),
-      Body: createReadStream(tgzFilePath),
-    });
+  private getS3Key(tgzFileName: string) {
+    return join(this.path, tgzFileName);
+  }
 
+  /**
+   * When uploading a file with a transform stream, the final ContentLength is unknown so it has to be uploaded as multipart.
+   *
+   * @param hash
+   * @param file
+   * @private
+   */
+  private async uploadFile(hash: string, file: Readable) {
     try {
       this.logger.debug(`Storage Cache: Uploading ${hash}`);
 
-      await this.s3.send(params);
+      const tgzFileName = this.getTgzFileName(hash);
+
+      const upload = new Upload({
+        client: this.s3,
+        params: {
+          Bucket: this.bucket,
+          Key: this.getS3Key(tgzFileName),
+          Body: file,
+        },
+      });
 
+      const response = await upload.done();
       this.logger.debug(`Storage Cache: Stored ${hash}`);
+
+      return response;
     } catch (err) {
       throw new Error(`Storage Cache: Upload error - ${err}`);
     }
   }
 
-  private getS3Key(tgzFileName: string) {
-    return join(this.path, tgzFileName);
-  }
-
   private async downloadFile(hash: string, tgzFilePath: string): Promise<void> {
     const pipelinePromise = promisify(pipeline),
       tgzFileName = this.getTgzFileName(hash),
@@ -205,8 +225,11 @@ export class AwsCache implements RemoteCache {
     try {
       const commandOutput = await this.s3.send(params);
       const fileStream = commandOutput.Body as Readable;
-
-      await pipelinePromise(fileStream, writeFileToLocalDir);
+      if (this.encryptConfig) {
+        await pipelinePromise(fileStream, new Decrypt(this.encryptConfig), writeFileToLocalDir);
+      } else {
+        await pipelinePromise(fileStream, writeFileToLocalDir);
+      }
     } catch (err) {
       throw new Error(`Storage Cache: Download error - ${err}`);
     }
@@ -221,7 +244,6 @@ export class AwsCache implements RemoteCache {
 
     try {
       await this.s3.send(params);
-
       return true;
     } catch (err) {
       if ((err as Error).name === 'NotFound') {
@@ -254,7 +276,6 @@ export class AwsCache implements RemoteCache {
 
   private filterTgzContent(filePath: string): boolean {
     const dir = dirname(filePath);
-
     const excludedPaths = [
       /**
        * The 'source' file is used by NX for integrity check purposes, but isn't utilized by custom cache providers.