Fixed #12905

Works around elvanto/litemoji#36 by only calling LitEmoji::unicodeToShortcode() for 4-byte character sequences
craftcms · Mar 16, 2023 · 263ead7 · 263ead7
1 parent 0f04477
commit 263ead7
Show file tree

Hide file tree

Showing 5 changed files with 90 additions and 10 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -6,6 +6,8 @@
 - Sections created via `entrify` commands no longer get a “Primary entry page” preview target by default, unless it was sourced from a category group with URLs. ([#12897](https://github.com/craftcms/cms/issues/12897))
 - `entrify` commands and the `sections/create` command now prompt for the initial entry type name and handle. ([#12894](https://github.com/craftcms/cms/discussions/12894))
 - Added `craft\helpers\FileHelper::uniqueName()`.
+- Added `craft\helpers\StringHelper::emojiToShortcodes()`.
+- Added `craft\helpers\StringHelper::shortcodesToEmoji()`.
 - Fixed an error that occurred when uploading an asset with a filename over 250 characters long. ([#12889](https://github.com/craftcms/cms/issues/12889))
 - Fixed an error that could occur when preparing licensing alerts, if any licenses were invalid. ([#12899](https://github.com/craftcms/cms/issues/12899))
 - Fixed a bug where it wasn’t possible to drag nested Neo blocks. ([#12896](https://github.com/craftcms/cms/issues/12896))
@@ -15,6 +17,7 @@
 - Fixed an error that could occur when rebuilding the project config, if there were any custom source definitions for element types that weren’t Composer-installed. ([#12881](https://github.com/craftcms/cms/issues/12881))
 - Fixed a PHP error that occurred if a field type stored enum values. ([#12297](https://github.com/craftcms/cms/issues/12297))
 - Fixed an error that could occur when generating transforms for images stored in Google Cloud Storage. ([#12878](https://github.com/craftcms/cms/issues/12878))
+- Fixed a bug where some unicode characters were getting removed by LitEmoji. ([#12905](https://github.com/craftcms/cms/issues/12905))
 
 ## 4.4.2 - 2023-03-14
 

diff --git a/src/fields/PlainText.php b/src/fields/PlainText.php
@@ -14,7 +14,7 @@
 use craft\base\SortableFieldInterface;
 use craft\fields\conditions\TextFieldConditionRule;
 use craft\helpers\Db;
-use LitEmoji\LitEmoji;
+use craft\helpers\StringHelper;
 use yii\db\Schema;
 
 /**
@@ -116,7 +116,7 @@ public function init(): void
         parent::init();
 
         if (isset($this->placeholder)) {
-            $this->placeholder = LitEmoji::shortcodeToUnicode($this->placeholder);
+            $this->placeholder = StringHelper::shortcodesToEmoji($this->placeholder);
         }
     }
 
@@ -127,7 +127,7 @@ public function getSettings(): array
     {
         $settings = parent::getSettings();
         if (isset($settings['placeholder'])) {
-            $settings['placeholder'] = LitEmoji::unicodeToShortcode($settings['placeholder']);
+            $settings['placeholder'] = StringHelper::emojiToShortcodes($settings['placeholder']);
         }
         return $settings;
     }
@@ -198,7 +198,7 @@ public function getContentColumnType(): string
     public function normalizeValue(mixed $value, ?ElementInterface $element = null): mixed
     {
         if ($value !== null) {
-            $value = LitEmoji::shortcodeToUnicode($value);
+            $value = StringHelper::shortcodesToEmoji($value);
             $value = trim(preg_replace('/\R/u', "\n", $value));
         }
 
@@ -238,7 +238,7 @@ public function getElementValidationRules(): array
     public function serializeValue(mixed $value, ?ElementInterface $element = null): mixed
     {
         if ($value !== null) {
-            $value = LitEmoji::unicodeToShortcode($value);
+            $value = StringHelper::emojiToShortcodes($value);
         }
         return $value;
     }
@@ -248,8 +248,7 @@ public function serializeValue(mixed $value, ?ElementInterface $element = null):
      */
     protected function searchKeywords(mixed $value, ElementInterface $element): string
     {
-        $value = (string)$value;
-        return LitEmoji::unicodeToShortcode($value);
+        return StringHelper::emojiToShortcodes((string)$value);
     }
 
     /**

diff --git a/src/fields/Table.php b/src/fields/Table.php
@@ -17,6 +17,7 @@
 use craft\helpers\Cp;
 use craft\helpers\DateTimeHelper;
 use craft\helpers\Json;
+use craft\helpers\StringHelper;
 use craft\validators\ColorValidator;
 use craft\validators\HandleValidator;
 use craft\validators\UrlValidator;
@@ -25,7 +26,6 @@
 use DateTime;
 use GraphQL\Type\Definition\InputObjectType;
 use GraphQL\Type\Definition\Type;
-use LitEmoji\LitEmoji;
 use yii\db\Schema;
 use yii\validators\EmailValidator;
 
@@ -448,7 +448,7 @@ public function serializeValue(mixed $value, ?ElementInterface $element = null):
                 $value = $row[$colId];
 
                 if (is_string($value) && in_array($this->columns[$colId]['type'], ['singleline', 'multiline'], true)) {
-                    $value = LitEmoji::unicodeToShortcode($value);
+                    $value = StringHelper::emojiToShortcodes($value);
                 }
 
                 $serializedRow[$colId] = parent::serializeValue($value ?? null);
@@ -558,7 +558,7 @@ private function _normalizeCellValue(string $type, mixed $value): mixed
             case 'multiline':
             case 'singleline':
                 if ($value !== null) {
-                    $value = LitEmoji::shortcodeToUnicode($value);
+                    $value = StringHelper::shortcodesToEmoji($value);
                     return trim(preg_replace('/\R/u', "\n", $value));
                 }
                 // no break

diff --git a/src/helpers/StringHelper.php b/src/helpers/StringHelper.php
@@ -10,6 +10,7 @@
 use Craft;
 use HTMLPurifier_Config;
 use IteratorAggregate;
+use LitEmoji\LitEmoji;
 use Normalizer;
 use Stringy\Stringy as BaseStringy;
 use voku\helper\ASCII;
@@ -1944,4 +1945,37 @@ public static function idnToUtf8Email(string $email): string
 
         return $combined;
     }
+
+    /**
+     * Converts emoji to shortcodes.
+     *
+     * @param string $str
+     * @return string
+     * @since 4.4.3
+     */
+    public static function emojiToShortcodes(string $str): string
+    {
+        // Add delimiters around all 4-byte chars
+        $dl = '__MB4_DL__';
+        $dr = '__MB4_DR__';
+        $str = static::replaceMb4($str, fn($char) => sprintf('%s%s%s', $dl, $char, $dr));
+
+        // Strip out consecutive delimiters
+        $str = str_replace(sprintf('%s%s', $dr, $dl), '', $str);
+
+        // Replace all 4-byte sequences individually
+        return preg_replace_callback("/$dl(.+?)$dr/", fn($m) => LitEmoji::unicodeToShortcode($m[1]), $str);
+    }
+
+    /**
+     * Converts shortcodes to emoji.
+     *
+     * @param string $str
+     * @return string
+     * @since 4.4.3
+     */
+    public static function shortcodesToEmoji(string $str): string
+    {
+        return LitEmoji::shortcodeToUnicode($str);
+    }
 }
diff --git a/tests/unit/helpers/StringHelperTest.php b/tests/unit/helpers/StringHelperTest.php
@@ -1551,6 +1551,28 @@ public function testIdnToUtf8Email(string $expected, string $string): void
         self::assertSame($expected, $actual);
     }
 
+    /**
+     * @dataProvider emojiToShortcodesDataProvider
+     *
+     * @param string $expected
+     * @param string $str
+     */
+    public function testEmojiToShortcodes(string $expected, string $str)
+    {
+        self::assertSame($expected, StringHelper::emojiToShortcodes($str));
+    }
+
+    /**
+     * @dataProvider shortcodesToEmojiDataProvider
+     *
+     * @param string $expected
+     * @param string $str
+     */
+    public function testShortcodesToEmoji(string $expected, string $str)
+    {
+        self::assertSame($expected, StringHelper::shortcodesToEmoji($str));
+    }
+
     /**
      * @return array
      */
@@ -4186,4 +4208,26 @@ public function idnToUtf8EmailDataProvider(): array
             ['aaa@äö.ee', '[email protected]'],
         ];
     }
+
+    /**
+     * @return array
+     */
+    public function emojiToShortcodesDataProvider(): array
+    {
+        return [
+            ['Baby you light my :fire:! :smiley:', 'Baby you light my 🔥! 😃'],
+            ['Test — em – en - dashes :hand_with_index_and_middle_fingers_crossed:', 'Test — em – en - dashes 🤞'],
+        ];
+    }
+
+    /**
+     * @return array
+     */
+    public function shortcodesToEmojiDataProvider(): array
+    {
+        return [
+            ['Baby you light my 🔥! 😃', 'Baby you light my :fire:! :smiley:'],
+            ['Test — em – en - dashes 🤞', 'Test — em – en - dashes :hand_with_index_and_middle_fingers_crossed:'],
+        ];
+    }
 }