Upgrade Emojibase and Add emoticon variations

README.md

@@ -1,10 +1,11 @@
 # emojibase-bindings
+
 Kotlin, Swift and Web bindings for emojibase json.
 
 ### Requirements
 
-+ Node LTS
-+ Yarn Classic
+- Node LTS
+- Yarn Classic
 
 ### Installation
 
@@ -14,6 +15,13 @@ Run `./scripts/setup.sh` to install dependencies for this project.
 
 Run `./scripts/generateJson.sh` to generate the emojibase.json assets
 
+## Updating Emoji Version
+
+1. Update the emojibase `emojibase` version in a [package.json](package.json) to one that support the required emoji version.
+2. To update the emoji version web uses change `MAX_EMOJI_VERSION_WEB` in [src/emoji.ts](src/emoji.ts)(Element Web imports this file directly).
+   - **caveat**: This version should be changed in unison with the twemoji verison so that all emojis displayed in the picker can actually be rendered.
+3. To update the emoji version iOS or Android use change `MAX_EMOJI_VERSION_ANDROID` or `MAX_EMOJI_VERSION_IOS` respectively in [scripts/generateJson.sh](scripts/generateJson.sh)(This script generates the `emojibase.json` file packaged with the iOS and Android packages)
+
 ## Releasing
 
 Use "[Run workflow](https://github.com/matrix-org/emojibase-bindings/actions/workflows/release.yaml)".

package.json

@@ -43,7 +43,7 @@
     "typescript": "^5.1.6"
   },
   "dependencies": {
-    "emojibase": "^15.0.0",
-    "emojibase-data": "^15.0.0"
+    "emojibase": "^15.3.1",
+    "emojibase-data": "^15.3.1"
   }
 }

platforms/android/library/src/androidTest/java/io/element/android/emojibasebindings/ExampleInstrumentedTest.kt

@@ -39,6 +39,12 @@ class ExampleInstrumentedTest {
 
     @Test
     fun testAllEmojisHaveShortcodes() {
-        assert(store.allEmojis.all { it.shortcodes.isNotEmpty() })
+        assert(store.allEmojis.all{ it.shortcodes.isNotEmpty() })
+    }
+
+    @Test
+    fun testSupportsEmoji151() {
+        // Check 🙂‍↔️ emoji is present
+        assertEquals(store.allEmojis.firstOrNull{ it.hexcode == "1F642-200D-2194-FE0F" }?.label, "head shaking horizontally")
     }
 }

platforms/ios/EmojibaseTests/EmojibaseTests.swift

@@ -43,4 +43,10 @@ final class EmojibaseTests: XCTestCase {
         //All emojis have a shortcode
         XCTAssertTrue(store.allEmojis.allSatisfy({ $0.shortcodes.first != nil }))
     }
+
+    func testSupportsEmoji151() async throws {
+        let store = try XCTUnwrap(store)
+        // Check 🙂‍↔️ emoji is present
+        XCTAssertEqual(store.allEmojis.first(where: {$0.hexcode == "1F642-200D-2194-FE0F"})?.label, "head shaking horizontally")
+    }
 }

scripts/generateJson.sh

@@ -2,11 +2,19 @@
 
 set -e
 
-yarn start
+MAX_EMOJI_VERSION_ANDROID=15.1
+MAX_EMOJI_VERSION_IOS=15.1
 
 EMOJIJSON_FILE="build/emojibase.json"
-SWIFT_PATH="platforms/ios/Emojibase/Resources"
 KOTLIN_PATH="platforms/android/library/src/main/assets"
+SWIFT_PATH="platforms/ios/Emojibase/Resources"
 
-cp "$EMOJIJSON_FILE" "$SWIFT_PATH"
+# generate android json
+export MAX_EMOJI_VERSION=$MAX_EMOJI_VERSION_ANDROID
+yarn start
 mkdir -p "$KOTLIN_PATH" && cp "$EMOJIJSON_FILE" "$KOTLIN_PATH"
+
+# generate ios json
+export MAX_EMOJI_VERSION=$MAX_EMOJI_VERSION_IOS
+yarn start
+cp "$EMOJIJSON_FILE" "$SWIFT_PATH"

src/emoji.ts

@@ -1,5 +1,5 @@
 /*
-Copyright 2019 The Matrix.org Foundation C.I.C.
+Copyright 2019-2024 The Matrix.org Foundation C.I.C.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -16,13 +16,16 @@ limitations under the License.
 
 import EMOJIBASE from "emojibase-data/en/compact.json";
 import SHORTCODES from "emojibase-data/en/shortcodes/iamcal.json";
-import { CompactEmoji } from "emojibase";
+import VERSIONS from "emojibase-data/versions/emoji.json";
+import { CompactEmoji, generateEmoticonPermutations } from "emojibase";
 
 export interface Emoji extends Omit<CompactEmoji, "shortcodes"> {
   // We generate a shortcode based on the label if none exist in the dataset
   shortcodes: string[];
 }
 
+const MAX_EMOJI_VERSION_WEB = 15.0;
+
 // The unicode is stored without the variant selector
 const UNICODE_TO_EMOJI = new Map<string, Emoji>(); // not exported as gets for it are handled by getEmojiFromUnicode
 export const EMOTICON_TO_EMOJI = new Map<string, Emoji>();
@@ -67,8 +70,29 @@ export const DATA_BY_CATEGORY: Record<string, Emoji[]> = {
   flags: [],
 };
 
+// The MAX emoji version to be included in exported data.
+// This can be present as an env variable when it is being run for the mobile context.
+// We support setting a max emoji version so that we can upgrade emojibase for its library
+// features but not force a version on all platforms. Web for example needs to be upgraded
+// in unison with twemoji.
+const MAX_EMOJI_VERSION: number =
+  parseFloat(<string>process.env.MAX_EMOJI_VERSION) || MAX_EMOJI_VERSION_WEB;
+console.log(`emojibase MAX_EMOJI_VERSION ${MAX_EMOJI_VERSION}`);
+// The compact version of emojibase does not have version stored on the emoji object
+// but there does existing a map of version to emoji.
+// Create a map of emoji to version so that we can lookup an emoji version and filter.
+const EMOJI_TO_VERSION = new Map<string, number>();
+for (const [versionString, emojis] of Object.entries(VERSIONS)) {
+  const version = parseFloat(versionString);
+  emojis.forEach((emoji) => EMOJI_TO_VERSION.set(emoji, version));
+}
+
 // Store various mappings from unicode/emoticon/shortcode to the Emoji objects
-export const EMOJI: Emoji[] = EMOJIBASE.map((emojiData) => {
+export const EMOJI: Emoji[] = EMOJIBASE.filter((emojiData) => {
+  // filter emojis that are less than or equal to MAX_EMOJI_VERSION
+  const version = EMOJI_TO_VERSION.get(emojiData.hexcode);
+  return version && version <= MAX_EMOJI_VERSION;
+}).map((emojiData) => {
   // If there's ever a gap in shortcode coverage, we fudge it by
   // filling it in with the emoji's CLDR annotation
   const shortcodeData = SHORTCODES[emojiData.hexcode] ?? [
@@ -81,7 +105,10 @@ export const EMOJI: Emoji[] = EMOJIBASE.map((emojiData) => {
     shortcodes:
       typeof shortcodeData === "string" ? [shortcodeData] : shortcodeData,
   };
+  return emoji;
+});
 
+EMOJI.forEach((emoji) => {
   // We manually include regional indicators in the symbols group, since
   // Emojibase intentionally leaves them uncategorized
   const categoryId =
@@ -100,16 +127,14 @@ export const EMOJI: Emoji[] = EMOJIBASE.map((emojiData) => {
   // 'emoji'. We therefore strip any variation chars from strings
   // both when building the map and when looking up.
   UNICODE_TO_EMOJI.set(stripVariation(emoji.unicode), emoji);
-
   if (emoji.emoticon) {
     // Add mapping from emoticon to Emoji object
-    // eslint-disable-next-line @typescript-eslint/no-unused-expressions
-    Array.isArray(emoji.emoticon)
-      ? emoji.emoticon.forEach((x) => EMOTICON_TO_EMOJI.set(x, emoji))
-      : EMOTICON_TO_EMOJI.set(emoji.emoticon, emoji);
+    const emoticons = [emoji.emoticon]
+      //flatten, in case `emoji.emoticon` is an array
+      .flat()
+      .flatMap((x) => generateEmoticonPermutations(x));
+    emoticons.forEach((x) => EMOTICON_TO_EMOJI.set(x, emoji));
   }
-
-  return emoji;
 });
 
 /**

test/emoji-test.ts

@@ -1,3 +1,19 @@
+/*
+Copyright 2024 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
 import {
   DATA_BY_CATEGORY,
   EMOJI,
@@ -24,10 +40,18 @@ describe("Emojis", () => {
   it("that we can get an emoji by unicode", async () => {
     expect(getEmojiFromUnicode("🙂")?.hexcode).toBe("1F642");
   });
+
+  it("that emojis with version greater than MAX_EMOJI_VERSION_WEB are not included", async () => {
+    expect(getEmojiFromUnicode("🙂‍↔️")?.hexcode).toBeUndefined();
+  });
 });
 
 describe("Emoticons", () => {
   it("should return an emoji", async () => {
     expect(EMOTICON_TO_EMOJI.get(":)")?.hexcode).toBe("1F642");
   });
+
+  it("should include emoticon variations", async () => {
+    expect(EMOTICON_TO_EMOJI.get("=-]")?.emoticon).toBe(":)");
+  });
 });

tsconfig.json

@@ -1,6 +1,6 @@
 {
   "compilerOptions": {
-    "target": "es2016",                                  /* Set the JavaScript language version for emitted JavaScript and include compatible library declarations. */
+    "target": "es2022",                                  /* Set the JavaScript language version for emitted JavaScript and include compatible library declarations. */
     "module": "commonjs",                                /* Specify what module code is generated. */
     "esModuleInterop": true,                             /* Emit additional JavaScript to ease support for importing CommonJS modules. This enables 'allowSyntheticDefaultImports' for type compatibility. */
     "forceConsistentCasingInFileNames": true,            /* Ensure that casing is correct in imports. */

yarn.lock

@@ -2209,7 +2209,7 @@ diff@^4.0.1:
   version "4.0.2"
   resolved "https://registry.yarnpkg.com/diff/-/diff-4.0.2.tgz#60f3aecb89d5fae520c11aa19efc2bb982aade7d"
   integrity sha512-58lmxKSA4BNyLz+HHMUzlOEpg09FV+ev6ZMe3vJihgdxzgcwZ8VoEEPmALCZG9LmqfVoNMMKpttIYTVG6uDY7A==
-
+  
 doctrine@^2.1.0:
   version "2.1.0"
   resolved "https://registry.yarnpkg.com/doctrine/-/doctrine-2.1.0.tgz#5cd01fc101621b42c4cd7f5d1a66243716d3f39d"
@@ -2246,15 +2246,15 @@ emoji-regex@^8.0.0:
   resolved "https://registry.yarnpkg.com/emoji-regex/-/emoji-regex-8.0.0.tgz#e818fd69ce5ccfcb404594f842963bf53164cc37"
   integrity sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==
 
-emojibase-data@^15.0.0:
-  version "15.0.0"
-  resolved "https://registry.yarnpkg.com/emojibase-data/-/emojibase-data-15.0.0.tgz#d1f5467f3080688b9605103d0abdcd54bbc76419"
-  integrity sha512-hqrLNhEeBejKOQp5ArJcofkzV3qZBcp8czXj8nIKUGpBVc50NddNGwir4yAYxn3oNgrSj/lYdB9XxIVAKTkong==
+emojibase-data@^15.3.1:
+  version "15.3.2"
+  resolved "https://registry.yarnpkg.com/emojibase-data/-/emojibase-data-15.3.2.tgz#2742246bfe14f16a7829b42ca156dec09934cf85"
+  integrity sha512-TpDyTDDTdqWIJixV5sTA6OQ0P0JfIIeK2tFRR3q56G9LK65ylAZ7z3KyBXokpvTTJ+mLUXQXbLNyVkjvnTLE+A==
 
-emojibase@^15.0.0:
-  version "15.0.0"
-  resolved "https://registry.yarnpkg.com/emojibase/-/emojibase-15.0.0.tgz#f41b7773ec9a8a332373c18628ff4471255bd769"
-  integrity sha512-bvSIs98sHaVnyKPmW+obRjo49MFx0g+rhfSz6mTePAagEZSlDPosq0b6AcSJa5gt48z3VP2ooXclyBs8vIkpGA==
+emojibase@^15.3.1:
+  version "15.3.1"
+  resolved "https://registry.yarnpkg.com/emojibase/-/emojibase-15.3.1.tgz#7f6ff5482486f23e59a457de64e974bd35f3c9a3"
+  integrity sha512-GNsjHnG2J3Ktg684Fs/vZR/6XpOSkZPMAv85EHrr6br2RN2cJNwdS4am/3YSK3y+/gOv2kmoK3GGdahXdMxg2g==
 
 entities@^4.4.0:
   version "4.5.0"