Add client side encryption before Send files to S3

README.md

@@ -29,14 +29,16 @@ This will make the necessary changes to nx.json in your workspace to use nx-aws-
 There are two ways to set-up plugin options, using `nx.json` or `Environment variables`. Here is a list of all possible options:
 
 | Parameter         | Description                                                                                         | Environment variable / .env     | `nx.json`            | Example                        |
-| ----------------- | --------------------------------------------------------------------------------------------------- | ------------------------------- | -------------------- | ------------------------------ |
+|-------------------|-----------------------------------------------------------------------------------------------------|---------------------------------|----------------------|--------------------------------|
 | Access Key Id     | Access Key Id.                                                                                      | `NXCACHE_AWS_ACCESS_KEY_ID`     | `awsAccessKeyId`     | my-id                          |
 | Secret Access Key | Secret Access Key.                                                                                  | `NXCACHE_AWS_SECRET_ACCESS_KEY` | `awsSecretAccessKey` | my-key                         |
 | Profile           | Configuration profile to use (applied only if Access Key Id and Secret Access Key are not set).     | `NXCACHE_AWS_PROFILE`           | `awsProfile`         | profile-1                      |
 | Endpoint          | Fully qualified endpoint of the web service if a custom endpoint is needed (e.g. when using MinIO). | `NXCACHE_AWS_ENDPOINT`          | `awsEndpoint`        | http://custom.de-eu.myhost.com |
 | Region            | Region to which this client will send requests.                                                     | `NXCACHE_AWS_REGION`            | `awsRegion`          | eu-central-1                   |
 | Bucket            | Bucket name where cache files are stored or retrieved (can contain sub-paths as well).              | `NXCACHE_AWS_BUCKET`            | `awsBucket`          | bucket-name/sub-path           |
 | Force Path Style  | Whether to force path style URLs for S3 objects (e.g. when using MinIO).                            | `NXCACHE_AWS_FORCE_PATH_STYLE`  | `awsForcePathStyle`  | true                           |
+| Enrypt File Key   | Encrypt file in client before send to S3 server. 32 bytes in base64                                 | `NXCACHE_AWS_ENCRYPTION_KEY` or `NX_CLOUD_ENCRYPTION_KEY` | `encryptionFileKey` | PcZrGOSda3zwWh9yYTJB5bnHORgXf3dphj55tPI74O0= |
+
 
 > **Important:** `Environment variables` take precedence over `nx.json` options (introduced in v3.0.0)!
 
@@ -55,6 +57,7 @@ There are two ways to set-up plugin options, using `nx.json` or `Environment var
       "awsBucket": "bucket-name/sub-path",
       "awsRegion": "eu-central-1",
       "awsForcePathStyle": true,
+      "encryptionFileKey": "Pbfk58EpcK7IxTxWwSXNsTAKmzhJQE+99vkpGftyJg8="
     }
   }
 }
@@ -66,6 +69,13 @@ There are two ways to set-up plugin options, using `nx.json` or `Environment var
 - `.env.local`
 - `.env`
 
+## Encryption
+If you want you can encrypt file before send to s3 server (Client side). Use ENV
+NX_CLOUD_ENCRYPTION_KEY or NXCACHE_CLOUD_ENCRYPTION_KEY
+```sh
+# Generate new key for encryption.
+openssl rand -base64 32
+```
 ## Disabling S3 cache
 
 Remote cache can be disabled in favor of local cache using an environment variable

e2e/nx-aws-cache-e2e/docker-compose.yml

@@ -0,0 +1,12 @@
+version: '3'
+services:
+
+  minio:
+    image: bitnami/minio
+    ports:
+      - '9000:9000'
+      - '9001:9001'
+    environment:
+      MINIO_ROOT_USER: minio
+      MINIO_ROOT_PASSWORD: minio123
+      MINIO_DEFAULT_BUCKETS: test

e2e/nx-aws-cache-e2e/project.json

@@ -10,7 +10,17 @@
         "jestConfig": "e2e/nx-aws-cache-e2e/jest.config.ts",
         "runInBand": true
       },
-      "dependsOn": ["nx-aws-cache:build"]
+      "dependsOn": ["nx-aws-cache:build", "docker-compose"]
+    },
+    "docker-compose": {
+      "executor": "nx:run-commands",
+      "options": {
+        "commands": [
+          "docker-compose down && docker-compose up -d && sleep 5"
+        ],
+        "cwd": "e2e/nx-aws-cache-e2e",
+        "parallel": false
+      }
     }
   },
   "tags": [],

e2e/nx-aws-cache-e2e/tests/aws-cache.test.ts

@@ -0,0 +1,94 @@
+import { AwsCache } from '../../../packages/nx-aws-cache/src/tasks-runner/aws-cache';
+import { MessageReporter } from '../../../packages/nx-aws-cache/src/tasks-runner/message-reporter';
+import { Logger } from '../../../packages/nx-aws-cache/src/tasks-runner/logger';
+import * as fs from 'fs';
+import * as os from 'os';
+import * as path from 'path';
+import { GetObjectCommand, S3Client } from '@aws-sdk/client-s3';
+import { createWriteStream } from 'fs';
+import { Decrypt, EncryptConfig } from '../../../packages/nx-aws-cache/src/tasks-runner/encryptor';
+import { pipeline, Readable } from 'stream';
+import { promisify } from 'util';
+
+describe('Test aws put and get encrypted file', () => {
+  let awsCache: AwsCache;
+  let hash = new Date().getTime().toString()
+  let cacheDirectory = path.join(os.tmpdir(),`aws-cache`);
+  let cacheDirectorySave = path.join(os.tmpdir(),`aws-cache-decompress`)
+  let fileContent = "console.log(123)"
+  let filePath = ""
+
+  const config = {
+    encryptionFileKey: "Pbfk58EpcK7IxTxWwSXNsTAKmzhJQE+99vkpGftyJg8=",
+    awsAccessKeyId: 'minio',
+    awsSecretAccessKey: 'minio123',
+    awsBucket: 'test',
+    awsEndpoint: 'http://127.0.0.1:9000',
+    awsForcePathStyle: true,
+    awsRegion: 'us-east-1'
+  };
+
+  beforeEach(() => {
+      awsCache = new AwsCache(config, new MessageReporter(new Logger()))
+
+    fs.mkdirSync(cacheDirectory, {
+      recursive: true
+    })
+    fs.mkdirSync(cacheDirectorySave,{
+      recursive: true
+    })
+    const fileDir = path.join(cacheDirectory, `${hash}/outputs`);
+    fs.mkdirSync(fileDir, {recursive: true})
+    filePath = path.join(fileDir, `test.js`);
+    fs.writeFileSync(filePath, fileContent)
+  })
+
+  it('Should be defined', () => {
+    expect(awsCache).toBeDefined()
+  })
+
+  it('Should save in s3 file, and read an unencrypted', async() => {
+    await awsCache.store(hash, cacheDirectory);
+    await awsCache.retrieve(hash, cacheDirectorySave)
+    const extractedFilePath = path.join(cacheDirectorySave, `${hash}/outputs/test.js`);
+    expect(fs.existsSync(extractedFilePath)).toBeTruthy()
+    expect(fs.readFileSync(extractedFilePath).toString()).toBe(fileContent);
+  })
+
+
+  it('Check that file is encrypted in s3 at rest', async() => {
+    await awsCache.store(hash, cacheDirectory);
+
+    // Check that file is encrypted in s3
+    const s3 = new S3Client({
+      credentials:{
+        accessKeyId: config.awsAccessKeyId,
+        secretAccessKey: config.awsSecretAccessKey
+      },
+      endpoint: config.awsEndpoint,
+      forcePathStyle: config.awsForcePathStyle,
+      region: config.awsRegion
+    });
+    const response = await s3.send(new GetObjectCommand({
+      Bucket: config.awsBucket,
+      Key: `${hash}.tar.gz`
+    }))
+    const decryptedPath = path.join(os.tmpdir(), `${hash}.decr`)
+    const fileOutput = createWriteStream(decryptedPath)
+    if (!response?.Body) throw new Error("Cant get file from s3");
+
+    const pipelinePromise = promisify(pipeline);
+    try{
+      await pipelinePromise(
+        response.Body as Readable,
+        new Decrypt(new EncryptConfig(config.encryptionFileKey)),
+        fileOutput
+      )
+    }catch(err){
+      if (err instanceof Error) throw new Error(`Cant decrypt file from s3, ${err?.message}`)
+      else throw err;
+    }
+
+  })
+
+})

package.json

@@ -31,6 +31,7 @@
     "test": "nx test",
     "lint": "nx workspace-lint && nx lint",
     "e2e": "nx e2e",
+    "generate-encryption-key": "",
     "affected:apps": "nx affected:apps",
     "affected:libs": "nx affected:libs",
     "affected:build": "nx affected:build",
@@ -55,6 +56,7 @@
   "dependencies": {
     "@aws-sdk/client-s3": "3.420.0",
     "@aws-sdk/credential-providers": "3.418.0",
+    "@aws-sdk/lib-storage": "^3.470.0",
     "@aws-sdk/property-provider": "^3.374.0",
     "@swc/helpers": "~0.5.2",
     "dotenv": "16.3.1",

packages/nx-aws-cache/src/tasks-runner/aws-cache.ts

@@ -2,39 +2,39 @@ import { createReadStream, createWriteStream, writeFile } from 'fs';
 import { join, dirname } from 'path';
 import { pipeline, Readable } from 'stream';
 import { promisify } from 'util';
-
 import * as clientS3 from '@aws-sdk/client-s3';
 import { fromNodeProviderChain } from '@aws-sdk/credential-providers';
 import { CredentialsProviderError } from '@aws-sdk/property-provider';
 import { RemoteCache } from '@nx/workspace/src/tasks-runner/default-tasks-runner';
 import { create, extract } from 'tar';
-
 import { AwsNxCacheOptions } from './models/aws-nx-cache-options.model';
 import { Logger } from './logger';
 import { MessageReporter } from './message-reporter';
+import { Encrypt, Decrypt, EncryptConfig } from './encryptor';
+import { Upload } from '@aws-sdk/lib-storage';
 
 export class AwsCache implements RemoteCache {
   private readonly bucket: string;
   private readonly path: string;
   private readonly s3: clientS3.S3Client;
   private readonly logger = new Logger();
   private readonly uploadQueue: Array<Promise<boolean>> = [];
+  private readonly encryptConfig: EncryptConfig | null = null;
 
   public constructor(options: AwsNxCacheOptions, private messages: MessageReporter) {
     const awsBucket = options.awsBucket ?? '';
     const bucketTokens = awsBucket.split('/');
     this.bucket = bucketTokens.shift() as string;
     this.path = bucketTokens.join('/');
+    this.encryptConfig = null;
 
     const clientConfig: clientS3.S3ClientConfig = {};
 
-    if (options.awsRegion) {
+    if (options.awsRegion)
       clientConfig.region = options.awsRegion;
-    }
 
-    if (options.awsEndpoint) {
+    if (options.awsEndpoint)
       clientConfig.endpoint = options.awsEndpoint;
-    }
 
     if (options.awsAccessKeyId && options.awsSecretAccessKey) {
       clientConfig.credentials = {
@@ -47,23 +47,23 @@ export class AwsCache implements RemoteCache {
       );
     }
 
-    if (options.awsForcePathStyle) {
+    if (options.awsForcePathStyle)
       clientConfig.forcePathStyle = true;
-    }
+
+    if (options?.encryptionFileKey)
+      this.encryptConfig = new EncryptConfig(options.encryptionFileKey);
 
     this.s3 = new clientS3.S3Client(clientConfig);
   }
 
   public checkConfig(options: AwsNxCacheOptions): void {
     const missingOptions: Array<string> = [];
 
-    if (!options.awsBucket) {
+    if (!options.awsBucket)
       missingOptions.push('NXCACHE_AWS_BUCKET | awsBucket');
-    }
 
-    if (missingOptions.length > 0) {
+    if (missingOptions.length > 0)
       throw new Error(`Missing AWS options: \n\n${missingOptions.join('\n')}`);
-    }
   }
 
   // eslint-disable-next-line max-statements
@@ -72,10 +72,8 @@ export class AwsCache implements RemoteCache {
       await this.s3.config.credentials();
     } catch (err) {
       this.messages.error = err as Error;
-
       return false;
     }
-
     if (this.messages.error) {
       return false;
     }
@@ -113,7 +111,6 @@ export class AwsCache implements RemoteCache {
     }
 
     const resultPromise = this.createAndUploadFile(hash, cacheDirectory);
-
     this.uploadQueue.push(resultPromise);
 
     return resultPromise;
@@ -126,9 +123,15 @@ export class AwsCache implements RemoteCache {
   private async createAndUploadFile(hash: string, cacheDirectory: string): Promise<boolean> {
     try {
       const tgzFilePath = this.getTgzFilePath(hash, cacheDirectory);
-
       await this.createTgzFile(tgzFilePath, hash, cacheDirectory);
-      await this.uploadFile(hash, tgzFilePath);
+      const sourceFileStream = createReadStream(tgzFilePath);
+
+      if (this.encryptConfig) {
+        const encrypt = new Encrypt(this.encryptConfig);
+        await this.uploadFileMultipart(hash, sourceFileStream.pipe(encrypt));
+      } else {
+        await this.uploadFile(hash, sourceFileStream);
+      }
 
       return true;
     } catch (err) {
@@ -170,12 +173,16 @@ export class AwsCache implements RemoteCache {
     }
   }
 
-  private async uploadFile(hash: string, tgzFilePath: string): Promise<void> {
+  private getS3Key(tgzFileName: string) {
+    return join(this.path, tgzFileName);
+  }
+
+  private async uploadFile(hash: string, sourceFile: string | Readable): Promise<void> {
     const tgzFileName = this.getTgzFileName(hash);
     const params: clientS3.PutObjectCommand = new clientS3.PutObjectCommand({
       Bucket: this.bucket,
       Key: this.getS3Key(tgzFileName),
-      Body: createReadStream(tgzFilePath),
+      Body: typeof sourceFile === 'string' ? createReadStream(sourceFile) : sourceFile,
     });
 
     try {
@@ -189,8 +196,27 @@ export class AwsCache implements RemoteCache {
     }
   }
 
-  private getS3Key(tgzFileName: string) {
-    return join(this.path, tgzFileName);
+  /**
+   * When upload file with transform stream you don't know what will be the final ContentLength,
+   * so you have to upload that file as multipart upload
+   *
+   * @param hash
+   * @param file
+   * @private
+   */
+  private uploadFileMultipart(hash: string, file: Readable) {
+    const tgzFileName = this.getTgzFileName(hash);
+
+    const upload = new Upload({
+      client: this.s3,
+      params: {
+        Bucket: this.bucket,
+        Key: this.getS3Key(tgzFileName),
+        Body: file,
+      },
+    });
+
+    return upload.done();
   }
 
   private async downloadFile(hash: string, tgzFilePath: string): Promise<void> {
@@ -202,11 +228,16 @@ export class AwsCache implements RemoteCache {
         Key: this.getS3Key(tgzFileName),
       });
 
+    // eslint-disable-next-line max-lines
     try {
       const commandOutput = await this.s3.send(params);
       const fileStream = commandOutput.Body as Readable;
-
-      await pipelinePromise(fileStream, writeFileToLocalDir);
+      if (this.encryptConfig) {
+        const decryptStream = new Decrypt(this.encryptConfig);
+        await pipelinePromise(fileStream, decryptStream, writeFileToLocalDir);
+      } else {
+        await pipelinePromise(fileStream, writeFileToLocalDir);
+      }
     } catch (err) {
       throw new Error(`Storage Cache: Download error - ${err}`);
     }
@@ -224,7 +255,7 @@ export class AwsCache implements RemoteCache {
 
       return true;
     } catch (err) {
-      if ((err as Error).name === 'NotFound') {
+      if ((err as Error['name']) === 'NotFound') {
         return false;
       } else if (err instanceof CredentialsProviderError) {
         return false;