[FEATURE] Properties File Escaping

lib/processors/stringEscaper.js

@@ -0,0 +1,59 @@
+const escapeUnicode = require("escape-unicode");
+
+/**
+ * @see https://ascii.cl/
+ * @type {number}
+ */
+const NUMBER_OF_ASCII_CHARACTERS = 127;
+
+// use memoization for escapeUnicode function for performance
+const memoizeEscapeUnicodeMap = {};
+const memoizeEscapeUnicode = function(sChar) {
+	if (memoizeEscapeUnicodeMap[sChar]) {
+		return memoizeEscapeUnicodeMap[sChar];
+	}
+	memoizeEscapeUnicodeMap[sChar] = escapeUnicode(sChar);
+	return memoizeEscapeUnicodeMap[sChar];
+};
+
+/**
+ * Escapes non ASCII characters with unicode characters.
+ *
+ * @see https://ascii.cl/
+ * @see https://tools.ietf.org/html/rfc5137#section-6.1
+ *
+ *
+ * @param {string} string input string with non ascii characters, e.g. L♥VE
+ * @returns {{string: (string), modified: boolean}} output string with all non ascii characters being escaped by unicode sequence, e.g. L\u2665VE
+ */
+const escapeNonAscii = function(string) {
+	let result = "";
+	let modified = false;
+	for (let i = 0; i < string.length; i++) {
+		const char = string[i];
+		// check for non ascii characters
+		if (string.charCodeAt(i) > NUMBER_OF_ASCII_CHARACTERS) {
+			result += memoizeEscapeUnicode(char);
+			modified = true;
+		} else {
+			result += char;
+		}
+	}
+	return {
+		modified,
+		string: result
+	};
+};
+
+module.exports = function({resources}) {
+	return Promise.all(resources.map((resource) => {
+		return resource.getString().then((resourceString) => {
+			const escaped = escapeNonAscii(resourceString);
+			// only modify the resource's string if it was changed
+			if (escaped.modified) {
+				resource.setString(escaped.string);
+			}
+			return resource;
+		});
+	}));
+};

lib/tasks/escapeNonAsciiCharacters.js

@@ -0,0 +1,43 @@
+const stringEscaper = require("../processors/stringEscaper");
+
+/**
+ * Encodings
+ *
+ * @type {{UTF_8: string, ISO_8859_1: string}}
+ */
+const ENCODING_TYPES = {
+	UTF_8: "UTF-8",
+	ISO_8859_1: "ISO-8859-1"
+};
+
+/**
+ * Task to escape non ascii characters in properties files resources.
+ *
+ * @public
+ * @alias module:@ui5/builder.tasks.escapePropertiesFiles
+ * @param {Object} parameters Parameters
+ * @param {module:@ui5/fs.DuplexCollection} parameters.workspace DuplexCollection to read and write files
+ * @param {Object} parameters.options Options
+ * @param {string} parameters.options.pattern Pattern to locate the files to be processed
+ * @param {string} [parameters.options.sourceEncoding] source file encoding: "UTF-8" or "ISO-8859-1". Defaults to "ISO-8859-1"
+ * @returns {Promise<undefined>} Promise resolving with <code>undefined</code> once data has been written
+ */
+module.exports = function({workspace, options}) {
+	if (options.sourceEncoding && !Object.values(ENCODING_TYPES).includes(options.sourceEncoding)) {
+		return Promise.reject(new Error(`Invalid encoding specified: '${options.sourceEncoding}'. Must be one of ${Object.values(ENCODING_TYPES)}`));
+	}
+	if (options.sourceEncoding && options.sourceEncoding === ENCODING_TYPES.UTF_8) {
+		return Promise.resolve();
+	}
+	return workspace.byGlob(options.pattern)
+		.then((allResources) => {
+			return stringEscaper({
+				resources: allResources
+			});
+		})
+		.then((processedResources) => {
+			return Promise.all(processedResources.map((resource) => {
+				return workspace.write(resource);
+			}));
+		});
+};

lib/tasks/taskRepository.js

@@ -2,6 +2,7 @@ const tasks = {
 	replaceCopyright: require("./replaceCopyright"),
 	replaceVersion: require("./replaceVersion"),
 	createDebugFiles: require("./createDebugFiles"),
+	escapeNonAsciiCharacters: require("./escapeNonAsciiCharacters"),
 	executeJsdocSdkTransformation: require("./jsdoc/executeJsdocSdkTransformation"),
 	generateApiIndex: require("./jsdoc/generateApiIndex"),
 	generateJsdoc: require("./jsdoc/generateJsdoc"),

lib/types/application/ApplicationBuilder.js

@@ -8,6 +8,7 @@ const tasks = { // can't require index.js due to circular dependency
 	generateStandaloneAppBundle: require("../../tasks/bundlers/generateStandaloneAppBundle"),
 	generateBundle: require("../../tasks/bundlers/generateBundle"),
 	generateCachebusterInfo: require("../../tasks/generateCachebusterInfo"),
+	escapeNonAsciiCharacters: require("../../tasks/escapeNonAsciiCharacters"),
 	buildThemes: require("../../tasks/buildThemes"),
 	createDebugFiles: require("../../tasks/createDebugFiles"),
 	generateVersionInfo: require("../../tasks/generateVersionInfo"),
@@ -26,6 +27,16 @@ class ApplicationBuilder extends AbstractBuilder {
 					"Also see: https://github.com/SAP/ui5-builder#application");
 		}
 
+		this.addTask("escapeNonAsciiCharacters", () => {
+			return tasks.escapeNonAsciiCharacters({
+				workspace: resourceCollections.workspace,
+				options: {
+					sourceEncoding: project.resources && project.resources.propertiesFileEncoding,
+					pattern: "/**/*.properties"
+				}
+			});
+		});
+
 		this.addTask("replaceCopyright", () => {
 			return tasks.replaceCopyright({
 				workspace: resourceCollections.workspace,

lib/types/library/LibraryBuilder.js

@@ -6,6 +6,7 @@ const tasks = { // can't require index.js due to circular dependency
 	generateLibraryPreload: require("../../tasks/bundlers/generateLibraryPreload"),
 	generateManifestBundle: require("../../tasks/bundlers/generateManifestBundle"),
 	generateStandaloneAppBundle: require("../../tasks/bundlers/generateStandaloneAppBundle"),
+	escapeNonAsciiCharacters: require("../../tasks/escapeNonAsciiCharacters"),
 	buildThemes: require("../../tasks/buildThemes"),
 	createDebugFiles: require("../../tasks/createDebugFiles"),
 	generateJsdoc: require("../../tasks/jsdoc/generateJsdoc"),
@@ -25,6 +26,16 @@ class LibraryBuilder extends AbstractBuilder {
 				"Also see: https://github.com/SAP/ui5-builder#library");
 		}
 
+		this.addTask("escapeNonAsciiCharacters", () => {
+			return tasks.escapeNonAsciiCharacters({
+				workspace: resourceCollections.workspace,
+				options: {
+					sourceEncoding: project.resources && project.resources.propertiesFileEncoding,
+					pattern: "/**/*.properties"
+				}
+			});
+		});
+
 		this.addTask("replaceCopyright", () => {
 			const replaceCopyright = tasks.replaceCopyright;
 			return replaceCopyright({

package.json

@@ -99,6 +99,7 @@
 		"@ui5/fs": "^1.1.2",
 		"@ui5/logger": "^1.0.1",
 		"cheerio": "^0.22.0",
+		"escape-unicode": "^0.2.0",
 		"escodegen": "^1.11.1",
 		"escope": "^3.6.0",
 		"esprima": "^4.0.1",

test/expected/build/application.b/dest/i18n/i18n.properties

@@ -1 +1,2 @@
-title=app-i18n
+title=app-i18n
+fame=Stra\u00dfe

test/expected/build/application.b/dest/i18n/i18n_de.properties

@@ -1 +1,2 @@
-title=app-i18n_de
+title=app-i18n_de
+fame=Stra\u00dfe

test/expected/build/application.b/dest/manifest-bundle/i18n/i18n.properties

@@ -1 +1,2 @@
-title=app-i18n
+title=app-i18n
+fame=Stra�e

test/expected/build/application.b/dest/manifest-bundle/i18n/i18n_de.properties

@@ -1 +1,2 @@
-title=app-i18n_de
+title=app-i18n_de
+fame=Stra�e

test/expected/build/application.b/standalone/i18n.properties

@@ -1 +1,2 @@
-title=app-i18n-wrong
+title=app-i18n-wrong
+fame=Stra�e

test/expected/build/application.b/standalone/i18n/i18n.properties

@@ -1 +1,2 @@
-title=app-i18n
+title=app-i18n
+fame=Stra�e

test/expected/build/application.b/standalone/i18n/i18n_de.properties

@@ -1 +1,2 @@
-title=app-i18n_de
+title=app-i18n_de
+fame=Stra�e

test/expected/build/application.b/standalone/i18n/l10n.properties

@@ -1 +1,2 @@
-title=app-i18n-wrong
+title=app-i18n-wrong
+fame=Stra�e

test/expected/build/application.b/standalone/resources/sap-ui-custom.js

@@ -5,10 +5,10 @@ jQuery.sap.registerPreloadedModules({
 	"id1/embedded/i18n/i18n_de.properties":'title=embedded-i18n_de',
 	"id1/embedded/i18n_fr.properties":'title=embedded-i18n_fr-wrong',
 	"id1/embedded/manifest.json":'{"_version":"1.1.0","sap.app":{"_version":"1.1.0","id":"id1.embedded","type":"component","applicationVersion":{"version":"1.2.2"},"embeddedBy":"../","title":"{{title}}"}}',
-	"id1/i18n.properties":'title=app-i18n-wrong',
-	"id1/i18n/i18n.properties":'title=app-i18n',
-	"id1/i18n/i18n_de.properties":'title=app-i18n_de',
-	"id1/i18n/l10n.properties":'title=app-i18n-wrong',
+	"id1/i18n.properties":'title=app-i18n-wrong\nfame=Straße',
+	"id1/i18n/i18n.properties":'title=app-i18n\nfame=Straße',
+	"id1/i18n/i18n_de.properties":'title=app-i18n_de\nfame=Straße',
+	"id1/i18n/l10n.properties":'title=app-i18n-wrong\nfame=Straße',
 	"id1/manifest.json":'{"_version":"1.1.0","sap.app":{"_version":"1.1.0","id":"id1","type":"application","applicationVersion":{"version":"1.2.2"},"embeds":["embedded"],"title":"{{title}}"}}',
 	"sap/ui/core/Core.js":function(){
 }

test/fixtures/application.b/webapp/i18n.properties

@@ -1 +1,2 @@
-title=app-i18n-wrong
+title=app-i18n-wrong
+fame=Stra�e

test/fixtures/application.b/webapp/i18n/i18n.properties

@@ -1 +1,2 @@
-title=app-i18n
+title=app-i18n
+fame=Stra�e

test/fixtures/application.b/webapp/i18n/i18n_de.properties

@@ -1 +1,2 @@
-title=app-i18n_de
+title=app-i18n_de
+fame=Stra�e

test/fixtures/application.b/webapp/i18n/l10n.properties

@@ -1 +1,2 @@
-title=app-i18n-wrong
+title=app-i18n-wrong
+fame=Stra�e

test/lib/processors/stringEscaper.js

@@ -0,0 +1,76 @@
+const test = require("ava");
+
+const stringEscaper = require("../../../lib/processors/stringEscaper");
+
+/**
+ * Executes string escaping. Returns <code>undefined</code> if nothing was escaped.
+ *
+ * @param {string} input string
+ * @returns {Promise<string|undefined>} escaped string if non-ascii characters present, <code>undefined</code> otherwise
+ */
+const escape = async function(input) {
+	let result = undefined;
+	const resource = {
+		getString: () => Promise.resolve(input),
+		setString: (actual) => {
+			result = actual;
+		}
+	};
+	await stringEscaper({resources: [resource]});
+	return result;
+};
+
+test("Replace symbol characters", async (t) => {
+	t.plan(1);
+
+	const input = `L♥VE is everywhere`;
+	const expected = `L\\u2665VE is everywhere`;
+	const output = await escape(input);
+	t.deepEqual(output, expected, "Correct file content should be set");
+});
+
+test("Replace chinese characters", async (t) => {
+	t.plan(1);
+
+	const input = `These are 人物 characters`;
+	const expected = "These are \\u4eba\\u7269 characters";
+	const output = await escape(input);
+	t.deepEqual(output, expected, "Correct file content should be set");
+});
+
+test("Replace umlaut characters", async (t) => {
+	t.plan(1);
+
+	const input = `Achso Ähem`;
+	const expected = "Achso \\u00c4hem";
+	const output = await escape(input);
+	t.deepEqual(output, expected, "Correct file content should be set");
+});
+
+test("Replace constructed characters", async (t) => {
+	t.plan(1);
+
+	const input = `Oh ẛ̣ that's ẛ̣ yes`;
+	const expected = "Oh \\u1e9b\\u0323 that's \\u1e9b\\u0323 yes";
+	const output = await escape(input);
+	t.deepEqual(output, expected, "Correct file content should be set");
+});
+
+
+test("Replace multiple times same character", async (t) => {
+	t.plan(1);
+
+	const input = `♥H L♥VE AND HARM♥NY ♥MG`;
+	const expected = "\\u2665H L\\u2665VE AND HARM\\u2665NY \\u2665MG";
+	const output = await escape(input);
+	t.deepEqual(output, expected, "Correct file content should be set");
+});
+
+test("No Replace of characters", async (t) => {
+	t.plan(1);
+
+	const input = `ONE LOVE`;
+	const expected = undefined;
+	const output = await escape(input);
+	t.deepEqual(output, expected, "Correct file content should not be modified");
+});