Portal: Fetching and delivering App resources (js/css) improved

   * Fetching resources from remote servers and slow network devices has now a proper timeout set,
     because non-responding servers could potentially lead to a memory leak due to an increasing number of socket/file handles
   * The content-length header is now always correctly set
   * Added config properties to set timeout and max sockets

CHANGELOG.md

@@ -3,6 +3,20 @@
 
 ## [unreleased]
 
+ * Portal: Fetching and delivering App resources (js/css) improved
+   * Fetching resources from remote servers and slow network devices has now a proper timeout set, 
+     because non-responding servers could potentially lead to a memory leak due to an increasing number of socket/file handles
+   * The content-length header is now always correctly set
+   * Properties like timeout and max sockets can now be set like this in the server config:
+     ```json
+      "Mashroom Portal WebApp": {
+        "resourceFetchConfig": {
+          "fetchTimeoutMs": 3000,
+          "httpMaxSocketsPerHost": 10,
+          "httpRejectUnauthorized": true
+        }
+      }
+     ```
  * mashroom-utils refactoring: Added an index file that should be used exclusively to import utils
    **BREAKING CHANGE**: If you have used mashroom-utils in your custom plugins you have to change the imports
  * LDAP Security Provider: Fixed escaping of special characters in the DN. Didn't work if the same special character occurred multiple times.

package.json

@@ -52,6 +52,11 @@
         "nodemon": "^3.0.2",
         "@nrwl/nx-cloud": "^16.5.2"
     },
+    "overrides": {
+        "node-fetch": "^2.7.0",
+        "debug": "^4.3.4",
+        "body-parser@": "^1.20.2"
+    },
     "scripts": {
         "setup": "npm install && npm run build:core",
         "type-check": "lerna run type-check && flow check",

packages/core/mashroom-json-schemas/schemas/mashroom-server-config.json

@@ -625,6 +625,25 @@
                             "type": "string",
                             "description": "If you need unique resource version hashes per server instance provide here a string (Default: null)"
                         },
+                        "resourceFetchConfig": {
+                            "type": "object",
+                            "properties": {
+                                "fetchTimeoutMs": {
+                                    "type": "number",
+                                    "description": "Timeout for fetching (Default: 3000)"
+                                },
+                                "httpMaxSocketsPerHost": {
+                                    "type": "number",
+                                    "description": "Max sockets per host for fetching resources from Remote Apps (Default: 10)"
+                                },
+                                "httpRejectUnauthorized": {
+                                    "type": "boolean",
+                                    "description": "Reject resources from servers with invalid certificates (Default: true)"
+                                }
+                            },
+                            "additionalProperties": false,
+                            "description": "Optional config for resource fetching (App and plugin resources like js/css files)"
+                        },
                         "defaultProxyConfig": {
                             "type": "object",
                             "properties": {

packages/core/mashroom-json-schemas/type-definitions/mashroom-server-config.d.ts

@@ -429,6 +429,23 @@ export interface Plugins {
      * If you need unique resource version hashes per server instance provide here a string (Default: null)
      */
     versionHashSalt?: string;
+    /**
+     * Optional config for resource fetching (App and plugin resources like js/css files)
+     */
+    resourceFetchConfig?: {
+      /**
+       * Timeout for fetching (Default: 3000)
+       */
+      fetchTimeoutMs?: number;
+      /**
+       * Max sockets per host for fetching resources from Remote Apps (Default: 10)
+       */
+      httpMaxSocketsPerHost?: number;
+      /**
+       * Reject resources from servers with invalid certificates (Default: true)
+       */
+      httpRejectUnauthorized?: boolean;
+    };
     /**
      * Optional default http proxy config for portal apps
      */

packages/core/mashroom-utils/src/ResourceFetchError.ts

@@ -0,0 +1,11 @@
+import {mergeStackTrace} from './error-utils';
+
+export default class ResourceFetchError extends Error {
+    constructor(message: string, rootCause?: Error) {
+        super(message);
+        this.name = 'ResourceFetchError';
+        if (rootCause) {
+            this.stack = mergeStackTrace(this.stack ?? '', rootCause.stack);
+        }
+    }
+}

packages/core/mashroom-utils/src/ResourceNotFoundError.ts

@@ -0,0 +1,7 @@
+
+export default class ResourceNotFoundError extends Error {
+    constructor(message?: string) {
+        super(message);
+        this.name = 'ResourceNotFoundError';
+    }
+}

packages/core/mashroom-utils/src/ResourceTypeNotSupportedError.ts

@@ -0,0 +1,7 @@
+
+export default class ResourceTypeNotSupportedError extends Error {
+    constructor(message?: string) {
+        super(message);
+        this.name = 'UnsupportedResourceTypeError';
+    }
+}

packages/core/mashroom-utils/src/error-utils.ts

@@ -0,0 +1,19 @@
+
+export const mergeStackTrace = (stackTrace: string, rootCauseStackTrace: string | null | undefined): string => {
+    if (!rootCauseStackTrace) {
+        return stackTrace;
+    }
+
+    const entriesToMerge = stackTrace.split('\n');
+    const baseEntries = rootCauseStackTrace.split('\n');
+
+    const newEntries: Array<string> = [];
+    entriesToMerge.forEach((entry) => {
+        if (baseEntries.includes(entry)) {
+            return;
+        }
+        newEntries.push(entry);
+    });
+
+    return [...newEntries, ...baseEntries].join('\n');
+};

packages/core/mashroom-utils/src/index.ts

@@ -10,6 +10,7 @@ export * as messagingUtils from './messaging-utils';
 export * as modelUtils from './model-utils';
 export * as readonlyUtils from './readonly-utils';
 export * as requestUtils from './request-utils';
+export * as resourceUtils from './resource-utils';
 export * as tlsUtils from './tls-utils';
 export * as tsNodeUtils from './ts-node-utils';
 export * as userAgentUtils from './user-agent-utils';

packages/core/mashroom-utils/src/resource-utils.ts

@@ -0,0 +1,158 @@
+
+import {createReadStream} from 'fs';
+import {stat} from 'fs/promises';
+import {fileURLToPath} from 'url';
+import {Readable} from 'stream';
+import http from 'http';
+import https from 'https';
+import ResourceFetchError from './ResourceFetchError';
+import ResourceTypeNotSupportedError from './ResourceTypeNotSupportedError';
+import ResourceNotFoundError from './ResourceNotFoundError';
+
+import type {IncomingMessage, RequestOptions, Agent as HttpAgent} from 'http';
+import type {Agent as HttpsAgent} from 'https';
+
+type GetResourceOptions = {
+    readonly abortSignal: AbortSignal | null | undefined;
+    readonly httpAgent?: HttpAgent;
+    readonly httpsAgent?: HttpsAgent;
+}
+
+type Resource = {
+    readonly size: number | null;
+    readonly contentType: string | null;
+    readonly lastModified: Date | null;
+    readonly stream: NodeJS.ReadableStream;
+}
+
+const createFileStream = async (filePath: string, options: GetResourceOptions): Promise<Resource> => {
+    try {
+        const {size, mtime} = await stat(filePath);
+        const stream = createReadStream(filePath, {
+            autoClose: true,
+            signal: options.abortSignal || undefined,
+        });
+        return {
+            size,
+            contentType: null,
+            lastModified: mtime,
+            stream,
+        };
+    } catch (e: any) {
+        const code = (e as NodeJS.ErrnoException).code;
+        if (code === 'ENOENT') {
+            throw new ResourceNotFoundError(filePath);
+        }
+        throw new ResourceFetchError(`Error fetching ${filePath}`, e);
+    }
+};
+
+const createHttpStream = async (url: string, options: GetResourceOptions): Promise<Resource> => {
+    const isHttps = url.startsWith('https://');
+
+    const requestOptions: RequestOptions = {
+        agent: isHttps ?
+            options.httpsAgent :
+            options.httpAgent,
+    };
+
+    let response: IncomingMessage;
+    try {
+        response = await new Promise((resolve, reject) => {
+            const request = (isHttps ? https : http).get(url, requestOptions);
+            request.on('response', (response) => {
+                resolve(response);
+            });
+            request.on('error', (err) => {
+               reject(err);
+            });
+            options?.abortSignal?.addEventListener('abort', () => {
+                if (response) {
+                    response.destroy(new ResourceFetchError(`Aborted (timeout): ${url}`));
+                } else {
+                    request.destroy();
+                }
+            }, true);
+        });
+    } catch (e: any) {
+        if (options?.abortSignal?.aborted) {
+           throw new ResourceFetchError(`Aborted (timeout): ${url}`);
+        }
+        throw new ResourceFetchError(`Error fetching ${url}`, e);
+    }
+
+    if (!response.statusCode || response.statusCode >= 299) {
+        if (response.statusCode === 404) {
+            throw new ResourceNotFoundError(url);
+        } else {
+            throw new ResourceFetchError(`Error fetching ${url}. Status code: ${response.statusCode}`);
+        }
+    }
+
+    const contentLengthHeader = response.headers['content-length'];
+    const contentTypeHeader = response.headers['content-type'];
+    const lastModifiedHeader = response.headers['last-modified'];
+
+    return {
+        size: contentLengthHeader ? parseInt(contentLengthHeader) : null,
+        contentType: contentTypeHeader ?? null,
+        lastModified: lastModifiedHeader ? new Date(lastModifiedHeader) : null,
+        stream: response,
+    };
+};
+
+const createDataURIString = async (uri: string): Promise<Resource> => {
+    const data = uri.substring('data:'.length);
+    const type = data.substring(0, data.indexOf(','));
+    const [contentType, encoding] = type.split(';');
+    const body = data.substring(type.length + 1);
+    const buffer = Buffer.from(decodeURIComponent(body), encoding === 'base64' ? 'base64' : 'utf8');
+
+    return {
+        size: buffer.length,
+        contentType,
+        lastModified: null,
+        stream: Readable.from(buffer),
+    };
+};
+
+/*
+ * Return given resources as stream. Can be a local file, http/s or a data URI.
+ *
+ * We deliberately don't deal with compression or caching because that should be done by reverse proxies.
+ * HTTP redirects are ignored.
+ */
+export const getResourceAsStream = async (uri: string, options: GetResourceOptions): Promise<Resource> => {
+    if (uri.startsWith('file://')) {
+        return createFileStream(fileURLToPath(uri), options);
+    } else if (uri.startsWith('http://') || uri.startsWith('https://')) {
+       return createHttpStream(uri, options);
+    } else if (uri.startsWith('data:')) {
+        return createDataURIString(uri);
+    } else if (uri.indexOf('://') === -1) {
+        // File resource without protocol
+        return createFileStream(uri, options);
+    }
+
+    throw new ResourceTypeNotSupportedError(uri);
+};
+
+/*
+ * Return given resource as string.
+ */
+export const getResourceAsString = async (uri: string, options: GetResourceOptions): Promise<string> => {
+    const {stream, contentType} = await getResourceAsStream(uri, options);
+    let encoding: BufferEncoding = 'utf-8';
+    if (contentType && contentType.indexOf('charset=') !== -1) {
+        const charset = contentType.split('charset=')[1].trim();
+        if (Buffer.isEncoding(charset)) {
+            encoding = charset as BufferEncoding;
+        }
+    }
+    return new Promise((resolve, reject) => {
+        const chunks: Array<Buffer> = [];
+        stream.on('data', (chunk) => chunks.push(chunk as Buffer));
+        stream.on('error', (error) => reject(error));
+        stream.on('end', () => resolve(Buffer.concat(chunks).toString(encoding)));
+    });
+};

packages/core/mashroom-utils/test/data/test.txt

@@ -0,0 +1 @@
+Hello World!