Angular translation via babel (singular only)

An approach to i18n for Angular HTML templates using babel. This patch allows only singular translation. Plurals and context will be addressed in subsequent patches. Based on Thai Tran's work at https://review.openstack.org/#/c/187321 This patch implements a custom extractor in horizon/utils/babel_extract_angular.py. pybabel really wants this to be a package installed into site-packages. I have not done this so as to avoid introducing a dependency on another change to global requirements just yet. I will upload a package to pypi in parallel so that we can start using it in the future. For now, I have had to modify tools/with_venv.sh to add horizon to the PYTHONPATH. This is less than ideal but seemed to be the least worst option. Steps to verify that it works: 1. Create a new html file in horizon/static or openstack_dashboard/static 2. Add something like <div translate>hello world!</div> to the html To test interpolation add something like: <div translate>hello {$name$}!</div> 3. Run ./run_tests.sh --makemessages 4. Inspect your djangojs.po file, it should be there. The following steps are optional: 5. Run ./run_tests.sh --pseudo de 6. Run ./run_tests.sh --compilemessages 7. Go to http://localhost:8000/settings 8. Change Language to Deutsch 9. Go to http://localhost:8000/i18n/js/horizon+openstack_dashboard 10. Your translated text should be there as well. Note: There are now unit tests as well. Partially-Implements: blueprint angular-translate-makemessages Depends on: If4352aba01ce0ca6337f590c0689d8fe9f0867c8 Change-Id: I26d4fef8646d5f038918cd3376dbbdbef37eb603
leonzz · Jul 2, 2015 · 9b36822 · 9b36822
1 parent 94542b6
commit 9b36822
Show file tree

Hide file tree

Showing 5 changed files with 263 additions and 1 deletion.
diff --git a/babel-djangojs.cfg b/babel-djangojs.cfg
@@ -1 +1,14 @@
+[extractors]
+# We use a custom extractor to find translatable strings in AngularJS
+# templates. The extractor is included in horizon.utils for now.
+# See http://babel.pocoo.org/docs/messages/#referencing-extraction-methods for
+# details on how this works.
+angular = horizon.utils.babel_extract_angular:extract_angular
+
 [javascript: **.js]
+
+# We need to look into all static folders for HTML files.
+# The **/static ensures that we also search within
+# /openstack_dashboard/dashboards/XYZ/static which will ensure
+# that plugins are also translated.
+[angular: **/static/**.html]
diff --git a/horizon/test/tests/utils.py b/horizon/test/tests/utils.py
@@ -14,13 +14,15 @@
 
 import datetime
 import os
+from StringIO import StringIO
 
 from django.core.exceptions import ValidationError  # noqa
 import django.template
 from django.template import defaultfilters
 
 from horizon import forms
 from horizon.test import helpers as test
+from horizon.utils.babel_extract_angular import extract_angular
 from horizon.utils import filters
 # we have to import the filter in order to register it
 from horizon.utils.filters import parse_isotime  # noqa
@@ -474,3 +476,152 @@ def test_normalize(self):
 
         self.assertEqual(units.normalize(1, 'unknown_unit'),
                          (1, 'unknown_unit'))
+
+
+default_keys = []
+
+
+class ExtractAngularTestCase(test.TestCase):
+
+    def test_extract_no_tags(self):
+        buf = StringIO('<html></html>')
+
+        messages = list(extract_angular(buf, default_keys, [], {}))
+        self.assertEqual([], messages)
+
+    def test_simple_string(self):
+        buf = StringIO(
+            """<html><translate>hello world!</translate>'
+            <div translate>hello world!</div></html>"""
+        )
+
+        messages = list(extract_angular(buf, default_keys, [], {}))
+        self.assertEqual(
+            [
+                (1, u'gettext', 'hello world!', []),
+                (2, u'gettext', 'hello world!', [])
+            ],
+            messages)
+
+    def test_interpolation(self):
+        buf = StringIO(
+            """<html>
+            <translate>hello {$name$}!</translate>
+            <div translate>hello {$name$}!</div>
+            </html>
+            """
+        )
+
+        messages = list(extract_angular(buf, default_keys, [], {}))
+        self.assertEqual(
+            [
+                (2, u'gettext', 'hello %(name)!', []),
+                (3, u'gettext', 'hello %(name)!', [])
+            ], messages)
+
+    def test_interpolation_func_call(self):
+        buf = StringIO(
+            """<html><div translate>hello {$func(name)$}!</div>
+            '<translate>hello {$func(name)$}!</translate>"""
+        )
+
+        messages = list(extract_angular(buf, default_keys, [], {}))
+        self.assertEqual(
+            [
+                (1, u'gettext', 'hello %(func(name))!', []),
+                (2, u'gettext', 'hello %(func(name))!', [])
+            ],
+            messages)
+
+    def test_interpolation_list(self):
+        buf = StringIO(
+            """<html><div translate>hello {$name[1]$}!</div>
+            <translate>hello {$name[1]$}!</translate></html>"""
+        )
+
+        messages = list(extract_angular(buf, default_keys, [], {}))
+        self.assertEqual(
+            [
+                (1, 'gettext', 'hello %(name[1])!', []),
+                (2, 'gettext', 'hello %(name[1])!', [])
+            ],
+            messages)
+
+    def test_interpolation_dict(self):
+        buf = StringIO(
+            """<html><div translate>hello {$name['key']$}!</div>
+            <translate>hello {$name['key']$}!</translate></html>"""
+        )
+
+        messages = list(extract_angular(buf, default_keys, [], {}))
+        self.assertEqual(
+            [
+                (1, 'gettext', r"hello %(name['key'])!", []),
+                (2, 'gettext', r"hello %(name['key'])!", [])
+            ],
+            messages)
+
+    def test_interpolation_dict_double_quote(self):
+        buf = StringIO(
+            """<html><div translate>hello {$name["key"]$}!</div>
+            <translate>hello {$name["key"]$}!</translate></html>""")
+
+        messages = list(extract_angular(buf, default_keys, [], {}))
+        self.assertEqual(
+            [
+                (1, 'gettext', r'hello %(name["key"])!', []),
+                (2, 'gettext', r'hello %(name["key"])!', [])
+            ],
+            messages)
+
+    def test_interpolation_object(self):
+        buf = StringIO(
+            """<html><div translate>hello {$name.attr$}!</div>
+            <translate>hello {$name.attr$}!</translate></html>""")
+
+        messages = list(extract_angular(buf, default_keys, [], {}))
+        self.assertEqual(
+            [
+                (1, 'gettext', 'hello %(name.attr)!', []),
+                (2, 'gettext', 'hello %(name.attr)!', [])
+            ],
+            messages)
+
+    def test_interpolation_spaces(self):
+        """Spaces are not valid in interpolation expressions, but we don't
+        currently complain about them
+        """
+        buf = StringIO("""<html><div translate>hello {$name attr$}!</div>
+        <translate>hello {$name attr$}!</translate></html>""")
+
+        messages = list(extract_angular(buf, default_keys, [], {}))
+        self.assertEqual(
+            [
+                (1, 'gettext', 'hello {$name attr$}!', []),
+                (2, 'gettext', 'hello {$name attr$}!', [])
+            ],
+            messages)
+
+    def test_attr_value(self):
+        """We should not translate tags that have translate as the value of an
+        attribute.
+        """
+        buf = StringIO('<html><div id="translate">hello world!</div>')
+
+        messages = list(extract_angular(buf, [], [], {}))
+        self.assertEqual([], messages)
+
+    def test_attr_value_plus_directive(self):
+        """Unless they also have a translate directive.
+        """
+        buf = StringIO(
+            '<html><div id="translate" translate>hello world!</div>')
+
+        messages = list(extract_angular(buf, [], [], {}))
+        self.assertEqual([(1, 'gettext', 'hello world!', [])], messages)
+
+    def test_translate_tag(self):
+        buf = StringIO('<html><translate>hello world!</translate>')
+
+        messages = list(extract_angular(buf, [], [], {}))
+        self.assertEqual([(1, 'gettext', 'hello world!', [])], messages)
diff --git a/horizon/utils/babel_extract_angular.py b/horizon/utils/babel_extract_angular.py
@@ -0,0 +1,92 @@
+# Copyright 2015, Rackspace, US, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+try:
+    from html.parser import HTMLParser
+except ImportError:
+    from HTMLParser import HTMLParser
+
+import re
+
+
+class AngularGettextHTMLParser(HTMLParser):
+    """Parse HTML to find translate directives.
+
+    Note: This will not cope with nested tags (which I don't think make any
+    sense)
+    """
+
+    def __init__(self):
+        try:
+            super(self.__class__, self).__init__()
+        except TypeError:
+            HTMLParser.__init__(self)
+
+        self.in_translate = False
+        self.data = ''
+        self.strings = []
+
+    def handle_starttag(self, tag, attrs):
+        if tag == 'translate' or \
+                (attrs and 'translate' in [attr[0] for attr in attrs]):
+                self.in_translate = True
+                self.line = self.getpos()[0]
+
+    def handle_data(self, data):
+        if self.in_translate:
+            self.data += data
+
+    def handle_endtag(self, tag):
+        if self.in_translate:
+            self.strings.append(
+                (self.line, u'gettext', self.interpolate(), [])
+            )
+            self.in_translate = False
+            self.data = ''
+
+    def interpolate(self):
+        interpolation_regex = r"""{\$([\w\."'\]\[\(\)]+)\$}"""
+        return re.sub(interpolation_regex, r'%(\1)', self.data)
+
+
+def extract_angular(fileobj, keywords, comment_tags, options):
+    """Extract messages from angular template (HTML) files that use the
+    angular-gettext translate directive as per
+    https://angular-gettext.rocketeer.be/ .
+
+    :param fileobj: the file-like object the messages should be extracted
+                    from
+    :param keywords: This is a standard parameter so it isaccepted but ignored.
+
+    :param comment_tags: This is a standard parameter so it is accepted but
+                        ignored.
+    :param options: Another standard parameter that is accepted but ignored.
+    :return: an iterator over ``(lineno, funcname, message, comments)``
+             tuples
+    :rtype: ``iterator``
+
+    This particular extractor is quite simple because it is intended to only
+    deal with angular templates which do not need comments, or the more
+    complicated forms of translations.
+
+    A later version will address pluralization.
+    """
+
+    parser = AngularGettextHTMLParser()
+
+    for line in fileobj:
+        parser.feed(line)
+
+    for string in parser.strings:
+        yield(string)
diff --git a/run_tests.sh b/run_tests.sh
@@ -15,7 +15,7 @@ function usage {
   echo "                           environment. Useful when dependencies have"
   echo "                           been added."
   echo "  -m, --manage             Run a Django management command."
-  echo "  --makemessages           Create/Update English translation files using babel."
+  echo "  --makemessages           Create/Update English translation files."
   echo "  --compilemessages        Compile all translation files."
   echo "  --check-only             Do not update translation files (--makemessages only)."
   echo "  --pseudo                 Pseudo translate a language."

diff --git a/tools/with_venv.sh b/tools/with_venv.sh
@@ -4,4 +4,10 @@ VENV_PATH=${VENV_PATH:-${TOOLS_PATH}}
 VENV_DIR=${VENV_NAME:-/../.venv}
 TOOLS=${TOOLS_PATH}
 VENV=${VENV:-${VENV_PATH}/${VENV_DIR}}
+HORIZON_DIR=${TOOLS%/tools}
+
+# This horrible mangling of the PYTHONPATH is required to get the
+# babel-angular-gettext extractor to work. To fix this the extractor needs to
+# be packaged on pypi and added to global requirements. That work is in progress.
+export PYTHONPATH="$HORIZON_DIR"
 source ${VENV}/bin/activate && "$@"