Add client side encryption before Send files to S3

README.md

@@ -29,14 +29,16 @@ This will make the necessary changes to nx.json in your workspace to use nx-aws-
 There are two ways to set-up plugin options, using `nx.json` or `Environment variables`. Here is a list of all possible options:
 
 | Parameter         | Description                                                                                         | Environment variable / .env     | `nx.json`            | Example                        |
-| ----------------- | --------------------------------------------------------------------------------------------------- | ------------------------------- | -------------------- | ------------------------------ |
+|-------------------|-----------------------------------------------------------------------------------------------------|---------------------------------|----------------------|--------------------------------|
 | Access Key Id     | Access Key Id.                                                                                      | `NXCACHE_AWS_ACCESS_KEY_ID`     | `awsAccessKeyId`     | my-id                          |
 | Secret Access Key | Secret Access Key.                                                                                  | `NXCACHE_AWS_SECRET_ACCESS_KEY` | `awsSecretAccessKey` | my-key                         |
 | Profile           | Configuration profile to use (applied only if Access Key Id and Secret Access Key are not set).     | `NXCACHE_AWS_PROFILE`           | `awsProfile`         | profile-1                      |
 | Endpoint          | Fully qualified endpoint of the web service if a custom endpoint is needed (e.g. when using MinIO). | `NXCACHE_AWS_ENDPOINT`          | `awsEndpoint`        | http://custom.de-eu.myhost.com |
 | Region            | Region to which this client will send requests.                                                     | `NXCACHE_AWS_REGION`            | `awsRegion`          | eu-central-1                   |
 | Bucket            | Bucket name where cache files are stored or retrieved (can contain sub-paths as well).              | `NXCACHE_AWS_BUCKET`            | `awsBucket`          | bucket-name/sub-path           |
 | Force Path Style  | Whether to force path style URLs for S3 objects (e.g. when using MinIO).                            | `NXCACHE_AWS_FORCE_PATH_STYLE`  | `awsForcePathStyle`  | true                           |
+| Enrypt File Key   | Encrypt file in client before send to S3 server. 32 bytes in base64                                 | `NXCACHE_AWS_ENCRYPTION_KEY` or `NX_CLOUD_ENCRYPTION_KEY` | `encryptionFileKey` | PcZrGOSda3zwWh9yYTJB5bnHORgXf3dphj55tPI74O0= |
+
 
 > **Important:** `Environment variables` take precedence over `nx.json` options (introduced in v3.0.0)!
 
@@ -55,6 +57,7 @@ There are two ways to set-up plugin options, using `nx.json` or `Environment var
       "awsBucket": "bucket-name/sub-path",
       "awsRegion": "eu-central-1",
       "awsForcePathStyle": true,
+      "encryptionFileKey": "Pbfk58EpcK7IxTxWwSXNsTAKmzhJQE+99vkpGftyJg8="
     }
   }
 }
@@ -66,6 +69,13 @@ There are two ways to set-up plugin options, using `nx.json` or `Environment var
 - `.env.local`
 - `.env`
 
+## Encryption
+If you want you can encrypt file before send to s3 server (Client side). Use ENV
+NX_CLOUD_ENCRYPTION_KEY or NXCACHE_CLOUD_ENCRYPTION_KEY
+```sh
+# Generate new key for encryption.
+openssl rand -base64 32
+```
 ## Disabling S3 cache
 
 Remote cache can be disabled in favor of local cache using an environment variable

e2e/nx-aws-cache-e2e/docker-compose.yml

@@ -0,0 +1,11 @@
+version: '3'
+services:
+  minio:
+    image: bitnami/minio
+    ports:
+      - '9000:9000'
+      - '9001:9001'
+    environment:
+      MINIO_ROOT_USER: minio
+      MINIO_ROOT_PASSWORD: minio123
+      MINIO_DEFAULT_BUCKETS: test

e2e/nx-aws-cache-e2e/project.json

@@ -10,7 +10,17 @@
         "jestConfig": "e2e/nx-aws-cache-e2e/jest.config.ts",
         "runInBand": true
       },
-      "dependsOn": ["nx-aws-cache:build"]
+      "dependsOn": ["nx-aws-cache:build", "docker-compose"]
+    },
+    "docker-compose": {
+      "executor": "nx:run-commands",
+      "options": {
+        "commands": [
+          "docker-compose down && docker-compose up -d && sleep 5"
+        ],
+        "cwd": "e2e/nx-aws-cache-e2e",
+        "parallel": false
+      }
     }
   },
   "tags": [],

e2e/nx-aws-cache-e2e/tests/aws-cache.test.ts

@@ -0,0 +1,93 @@
+import { AwsCache } from '../../../packages/nx-aws-cache/src/tasks-runner/aws-cache';
+import { MessageReporter } from '../../../packages/nx-aws-cache/src/tasks-runner/message-reporter';
+import { Logger } from '../../../packages/nx-aws-cache/src/tasks-runner/logger';
+import * as fs from 'fs';
+import * as os from 'os';
+import * as path from 'path';
+import { GetObjectCommand, S3Client } from '@aws-sdk/client-s3';
+import { createWriteStream } from 'fs';
+import { Decrypt, EncryptConfig } from '../../../packages/nx-aws-cache/src/tasks-runner/encryptor';
+import { pipeline, Readable } from 'stream';
+import { promisify } from 'util';
+
+describe('Test aws put and get encrypted file', () => {
+  let awsCache: AwsCache;
+  let hash = new Date().getTime().toString();
+  let cacheDirectory = path.join(os.tmpdir(), `aws-cache`);
+  let cacheDirectorySave = path.join(os.tmpdir(), `aws-cache-decompress`);
+  let fileContent = 'console.log(123)';
+  let filePath = '';
+
+  const config = {
+    encryptionFileKey: 'Pbfk58EpcK7IxTxWwSXNsTAKmzhJQE+99vkpGftyJg8=',
+    awsAccessKeyId: 'minio',
+    awsSecretAccessKey: 'minio123',
+    awsBucket: 'test',
+    awsEndpoint: 'http://127.0.0.1:9000',
+    awsForcePathStyle: true,
+    awsRegion: 'us-east-1',
+  };
+
+  beforeEach(() => {
+    awsCache = new AwsCache(config, new MessageReporter(new Logger()));
+
+    fs.mkdirSync(cacheDirectory, {
+      recursive: true,
+    });
+    fs.mkdirSync(cacheDirectorySave, {
+      recursive: true,
+    });
+    const fileDir = path.join(cacheDirectory, `${hash}/outputs`);
+    fs.mkdirSync(fileDir, { recursive: true });
+    filePath = path.join(fileDir, `test.js`);
+    fs.writeFileSync(filePath, fileContent);
+  });
+
+  it('Should be defined', () => {
+    expect(awsCache).toBeDefined();
+  });
+
+  it('Should save in s3 file, and read an unencrypted', async () => {
+    await awsCache.store(hash, cacheDirectory);
+    await awsCache.retrieve(hash, cacheDirectorySave);
+    const extractedFilePath = path.join(cacheDirectorySave, `${hash}/outputs/test.js`);
+    expect(fs.existsSync(extractedFilePath)).toBeTruthy();
+    expect(fs.readFileSync(extractedFilePath).toString()).toBe(fileContent);
+  });
+
+  it('Check that file is encrypted in s3 at rest', async () => {
+    await awsCache.store(hash, cacheDirectory);
+
+    // Check that file is encrypted in s3
+    const s3 = new S3Client({
+      credentials: {
+        accessKeyId: config.awsAccessKeyId,
+        secretAccessKey: config.awsSecretAccessKey,
+      },
+      endpoint: config.awsEndpoint,
+      forcePathStyle: config.awsForcePathStyle,
+      region: config.awsRegion,
+    });
+    const response = await s3.send(
+      new GetObjectCommand({
+        Bucket: config.awsBucket,
+        Key: `${hash}.tar.gz`,
+      }),
+    );
+    const decryptedPath = path.join(os.tmpdir(), `${hash}.decr`);
+    const fileOutput = createWriteStream(decryptedPath);
+    if (!response?.Body) throw new Error('Cant get file from s3');
+
+    const pipelinePromise = promisify(pipeline);
+    try {
+      await pipelinePromise(
+        response.Body as Readable,
+        new Decrypt(new EncryptConfig(config.encryptionFileKey)),
+        fileOutput,
+      );
+    } catch (err) {
+      if (err instanceof Error) throw new Error(`Cant decrypt file from s3, ${err?.message}`);
+      else throw err;
+    }
+  });
+});

nx.json

@@ -7,7 +7,7 @@
     "default": {
       "runner": "nx-cloud",
       "options": {
-        "cacheableOperations": ["build", "test", "lint", "e2e"],
+        "cacheableOperations": ["build", "test", "lint", "e2e", "docker-compose"],
         "parallel": 1,
         "accessToken": "NGIxYzhjYjMtYTBjNC00MDk3LTg0NGMtZDZhNDliZTc3MTMzfHJlYWQtd3JpdGU="
       }

package.json

@@ -31,6 +31,7 @@
     "test": "nx test",
     "lint": "nx workspace-lint && nx lint",
     "e2e": "nx e2e",
+    "generate-encryption-key": "openssl rand -base64 32",
     "affected:apps": "nx affected:apps",
     "affected:libs": "nx affected:libs",
     "affected:build": "nx affected:build",
@@ -55,6 +56,7 @@
   "dependencies": {
     "@aws-sdk/client-s3": "3.420.0",
     "@aws-sdk/credential-providers": "3.418.0",
+    "@aws-sdk/lib-storage": "^3.470.0",
     "@aws-sdk/property-provider": "^3.374.0",
     "@swc/helpers": "~0.5.2",
     "dotenv": "16.3.1",

packages/nx-aws-cache/src/generators/init/generator.ts

@@ -49,6 +49,7 @@ function updateNxJson(tree: Tree, options: InitGeneratorSchema): void {
           ...(options.awsRegion ? { awsRegion: options.awsRegion } : {}),
           ...(options.awsBucket ? { awsBucket: options.awsBucket } : {}),
           ...(options.awsForcePathStyle ? { awsForcePathStyle: options.awsForcePathStyle } : {}),
+          ...(options.encryptionFileKey ? { encryptionFileKey: options.encryptionFileKey } : {}),
         },
       },
     };

packages/nx-aws-cache/src/generators/init/schema.d.ts

@@ -6,4 +6,5 @@ export interface InitGeneratorSchema {
   awsRegion?: string;
   awsBucket?: string;
   awsForcePathStyle?: boolean;
+  encryptionFileKey?: string;
 }

packages/nx-aws-cache/src/generators/init/schema.json

@@ -38,6 +38,11 @@
       "type": "boolean",
       "description": "Whether to force path style URLs for S3 objects (e.g. when using MinIO)",
       "x-prompt": "Whether to force path style URLs for S3 objects (e.g. when using MinIO)"
+    },
+    "encryptionFileKey": {
+      "type": "string",
+      "description": "Encrypt file in client before send to S3 server. 32 bytes in base64",
+      "x-prompt": "Encrypt file in client before send to S3 server. 32 bytes in base64"
     }
   }
 }