Future Implementations for classes - Measure, Money and Date

Jenkinsfile

@@ -27,7 +27,7 @@ pipeline {
     HY_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/03-12-24-0'
     MR_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/03-12-24-1'
     JA_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/10-17-24-1'
-    HI_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/11-29-24-1'
+    HI_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/01-15-25-0'
     DEFAULT_TN_CACHE='/home/jenkinsci/TestData/text_norm/ci/grammars/06-08-23-0'
   }
   stages {
@@ -104,7 +104,7 @@ pipeline {
     parallel {
         stage('L0: Hi TN grammars') {
             steps {
-                sh 'CUDA_VISIBLE_DEVICES="" python nemo_text_processing/text_normalization/normalize.py --lang=hi --text="१" --cache_dir ${HI_TN_CACHE}'
+                sh 'CUDA_VISIBLE_DEVICES="" python nemo_text_processing/text_normalization/normalize.py --text="१" --cache_dir ${HI_TN_CACHE}'
             }
         }
         stage('L0: Hi ITN grammars') {

nemo_text_processing/text_normalization/hi/data/date/year_suffix.tsv

@@ -0,0 +1,3 @@
+ई. पू.	ईसा पूर्व
+ई.	ईसवी
+तक  तक

nemo_text_processing/text_normalization/hi/data/measure/unit.tsv

@@ -141,14 +141,16 @@ month	महीना
 months	महीने
 ct	कैरेट
 pH	पीएच
+km/h	किलोमीटर प्रति घंटा
 km/hr	किलोमीटर प्रति घंटा
 km/min	किलोमीटर प्रति मिनट
+m/h	मीटर प्रति घंटा
 m/hr	मीटर प्रति घंटा
 mi/s	मील प्रति सेकंड
+mi/h	मील प्रति घंटा
 mi/hr	मील प्रति घंटा
 mi/min	मील प्रति मिनट
 ₹/ac	रुपए प्रति एकड़
 x	बाई
 X	बाई
 *	बाई
--	से

nemo_text_processing/text_normalization/hi/data/money/currency.tsv

@@ -1,10 +1,9 @@
 ₹	रुपए
-P	पैसे 
 £	पाउंड
 ₩	वॉन
 $	डॉलर
 ₺	लीरा
 ৳	टका
 ¥	येन
 ₦	नाइरा
-€	यूरो
+€	यूरो

nemo_text_processing/text_normalization/hi/data/money/major_minor_currencies.py

@@ -0,0 +1,11 @@
+major_minor_currencies = {
+    "रुपए": "पैसे",
+    "पाउंड": "पेंस",
+    "वॉन": "जिओन",
+    "डॉलर": "सेंट",
+    "लीरा": "कुरस",
+    "टका": "पैसे",
+    "येन": "सेन",
+    "नाइरा": "कोबो",
+    "यूरो": "सेंट",
+}

nemo_text_processing/text_normalization/hi/data/time/hours.tsv

@@ -1,3 +1,4 @@
+०	शून्य
 १	एक
 २	दो
 ३	तीन

nemo_text_processing/text_normalization/hi/graph_utils.py

@@ -21,6 +21,7 @@
 
 import pynini
 from pynini import Far
+from pynini.examples import plurals
 from pynini.export import export
 from pynini.lib import byte, pynutil, utf8
 
@@ -99,6 +100,30 @@ def generator_main(file_name: str, graphs: Dict[str, 'pynini.FstLike']):
     logging.info(f'Created {file_name}')
 
 
+def get_plurals(fst):
+    """
+    Given singular returns plurals
+
+    Args:
+        fst: Fst
+
+    Returns plurals to given singular forms
+    """
+    return SINGULAR_TO_PLURAL @ fst
+
+
+def get_singulars(fst):
+    """
+    Given plural returns singulars
+
+    Args:
+        fst: Fst
+
+    Returns singulars to given plural forms
+    """
+    return PLURAL_TO_SINGULAR @ fst
+
+
 def convert_space(fst) -> 'pynini.FstLike':
     """
     Converts space to nonbreaking space.
@@ -113,6 +138,44 @@ def convert_space(fst) -> 'pynini.FstLike':
     return fst @ pynini.cdrewrite(pynini.cross(NEMO_SPACE, NEMO_NON_BREAKING_SPACE), "", "", NEMO_SIGMA)
 
 
+def string_map_cased(input_file: str, input_case: str = INPUT_LOWER_CASED):
+    labels = load_labels(input_file)
+
+    if input_case == INPUT_CASED:
+        additional_labels = []
+        for written, spoken, *weight in labels:
+            written_capitalized = written[0].upper() + written[1:]
+            additional_labels.extend(
+                [
+                    [written_capitalized, spoken.capitalize()],  # first letter capitalized
+                    [
+                        written_capitalized,
+                        spoken.upper().replace(" AND ", " and "),
+                    ],  # # add pairs with the all letters capitalized
+                ]
+            )
+
+            spoken_no_space = spoken.replace(" ", "")
+            # add abbreviations without spaces (both lower and upper case), i.e. "BMW" not "B M W"
+            if len(spoken) == (2 * len(spoken_no_space) - 1):
+                logging.debug(f"This is weight {weight}")
+                if len(weight) == 0:
+                    additional_labels.extend(
+                        [[written, spoken_no_space], [written_capitalized, spoken_no_space.upper()]]
+                    )
+                else:
+                    additional_labels.extend(
+                        [
+                            [written, spoken_no_space, weight[0]],
+                            [written_capitalized, spoken_no_space.upper(), weight[0]],
+                        ]
+                    )
+        labels += additional_labels
+
+    whitelist = pynini.string_map(labels).invert().optimize()
+    return whitelist
+
+
 class GraphFst:
     """
     Base class for all grammar fsts.

nemo_text_processing/text_normalization/hi/taggers/cardinal.py

@@ -13,10 +13,10 @@
 # limitations under the License.
 
 import pynini
-from pynini.lib import pynutil
+from pynini.lib import pynutil, rewrite
 
-from nemo_text_processing.text_normalization.hi.graph_utils import GraphFst
-from nemo_text_processing.text_normalization.hi.utils import get_abs_path
+from nemo_text_processing.text_normalization.hi.graph_utils import GraphFst, insert_space
+from nemo_text_processing.text_normalization.hi.utils import apply_fst, get_abs_path
 
 
 class CardinalFst(GraphFst):

nemo_text_processing/text_normalization/hi/taggers/date.py

@@ -14,7 +14,6 @@
 
 import pynini
 from pynini.lib import pynutil
-
 from nemo_text_processing.text_normalization.hi.graph_utils import (
     NEMO_HI_DIGIT,
     NEMO_HI_NON_ZERO,
@@ -26,6 +25,7 @@
 
 days = pynini.string_file(get_abs_path("data/date/days.tsv"))
 months = pynini.string_file(get_abs_path("data/date/months.tsv"))
+year_suffix = pynini.string_file(get_abs_path("data/date/year_suffix.tsv"))
 
 
 class DateFst(GraphFst):
@@ -62,12 +62,17 @@ def __init__(self, cardinal: GraphFst):
 
         years_graph = pynutil.insert("year: \"") + graph_year + pynutil.insert("\"") + insert_space
 
-        graph_dd_mm = days_graph + delete_dash + months_graph
+        graph_dd_mm = days_graph + (delete_dash | pynini.accep("")) + months_graph
 
-        graph_mm_dd = months_graph + delete_dash + days_graph
+        graph_mm_dd = months_graph + (delete_dash | pynini.accep("")) + days_graph
 
         graph_mm_dd += pynutil.insert(" preserve_order: true ")
 
+        # Graph for era
+        era_graph = pynutil.insert("era: \"") + year_suffix + pynutil.insert("\"") + insert_space
+
+        range_graph = pynini.cross("-", "से")
+
         graph_dd_mm_yyyy = (
             days_graph + (delete_dash | delete_slash) + months_graph + (delete_dash | delete_slash) + years_graph
         )
@@ -78,7 +83,22 @@ def __init__(self, cardinal: GraphFst):
 
         graph_mm_dd_yyyy += pynutil.insert(" preserve_order: true ")
 
-        graph_mm_yyyy = months_graph + delete_dash + years_graph
+        graph_mm_yyyy = (
+            months_graph + (delete_dash | pynini.accep("")) + years_graph + pynutil.insert(" preserve_order: true ")
+        )
+
+        graph_year_suffix = era_graph
+
+        graph_range = (
+            pynutil.insert("text: \"")
+            + (cardinal.final_graph | graph_year)
+            + insert_space
+            + range_graph
+            + insert_space
+            + (cardinal.final_graph | graph_year)
+            + pynutil.insert("\"")
+            + pynutil.insert(" preserve_order: true ")
+        )
 
         # default assume dd_mm_yyyy
 
@@ -88,6 +108,8 @@ def __init__(self, cardinal: GraphFst):
             | pynutil.add_weight(graph_dd_mm_yyyy, -0.001)
             | graph_mm_dd_yyyy
             | graph_mm_yyyy
+            | pynutil.add_weight(graph_year_suffix, -0.001)
+            | pynutil.add_weight(graph_range, -0.005)
         )
 
         self.final_graph = final_graph.optimize()

nemo_text_processing/text_normalization/hi/taggers/fraction.py

@@ -15,7 +15,9 @@
 import pynini
 from pynini.lib import pynutil
 
-from nemo_text_processing.text_normalization.hi.graph_utils import GraphFst
+from nemo_text_processing.text_normalization.hi.graph_utils import NEMO_NOT_QUOTE, GraphFst, insert_space
+from nemo_text_processing.text_normalization.hi.taggers.cardinal import CardinalFst
+from nemo_text_processing.text_normalization.hi.utils import apply_fst
 
 
 class FractionFst(GraphFst):

nemo_text_processing/text_normalization/hi/taggers/measure.py

@@ -44,16 +44,20 @@ def __init__(self, cardinal: GraphFst, decimal: GraphFst):
         )
 
         # Define the unit handling
-        self.unit = pynutil.insert("units: \"") + unit_graph + pynutil.insert("\" ")
+        unit = pynutil.insert("units: \"") + unit_graph + pynutil.insert("\" ")
+
+        # Handling symbols like x, X, *
+        symbol_graph = pynini.string_map([("x", "बाई"), ("X", "बाई"), ("*", "बाई"),])
 
         graph_measurements = (
             pynutil.insert("decimal { ")
             + optional_graph_negative
             + decimal_graph
             + pynutil.insert(" }")
             + delete_space
-            + self.unit
+            + unit
         )
+
         graph_measurements |= (
             pynutil.insert("cardinal { ")
             + optional_graph_negative
@@ -62,7 +66,27 @@ def __init__(self, cardinal: GraphFst, decimal: GraphFst):
             + pynutil.insert("\"")
             + pynutil.insert(" }")
             + delete_space
-            + self.unit
+            + unit
+        )
+
+        # Handling cardinal clubbed with symbol as single token
+        graph_measurements |= (
+            pynutil.insert("cardinal { ")
+            + optional_graph_negative
+            + pynutil.insert("integer: \"")
+            + cardinal_graph
+            + pynutil.insert("\"")
+            + pynutil.insert(" }")
+            + pynutil.insert(" units: \"")
+            + symbol_graph
+            + pynutil.insert("\" ")
+            + pynutil.insert("} }")
+            + insert_space
+            + pynutil.insert("tokens { cardinal { ")
+            + optional_graph_negative
+            + pynutil.insert("integer: \"")
+            + cardinal_graph
+            + pynutil.insert("\"")
         )
 
         graph = graph_measurements

nemo_text_processing/text_normalization/hi/taggers/money.py

@@ -24,39 +24,35 @@
 class MoneyFst(GraphFst):
     """
     Finite state transducer for classifying money, suppletive aware, e.g.
-        ₹1 -> money { currency: "रुपए" integer_part: "एक" }
-        ₹1.2 -> money { currency: "रुपए" integer_part: "एक" fractional_part: "दो" }
-
+        ₹५० -> money { money { currency_maj: "रुपए" integer_part: "पचास" }
+        ₹५०.५० -> money { currency_maj: "रुपए" integer_part: "पचास" fractional_part: "पचास" currency_min: "centiles" }
+        ₹०.५० -> money { currency_maj: "रुपए" integer_part: "शून्य" fractional_part: "पचास" currency_min: "centiles" }
+    Note that the 'centiles' string is a placeholder to handle by the verbalizer by applying the corresponding minor currency denomination
+
     Args:
         cardinal: CardinalFst
         decimal: DecimalFst
         deterministic: if True will provide a single transduction option,
             for False multiple transduction are generated (used for audio-based normalization)
     """
 
-    def __init__(self, cardinal: GraphFst, decimal: GraphFst):
+    def __init__(self, cardinal: GraphFst):
         super().__init__(name="money", kind="classify")
 
         cardinal_graph = cardinal.final_graph
 
-        optional_graph_negative = pynini.closure(
-            pynutil.insert("negative: ") + pynini.cross("-", "\"true\"") + insert_space, 0, 1,
-        )
-        self.currency = pynutil.insert("currency: \"") + currency_graph + pynutil.insert("\" ")
-        self.interger = pynutil.insert("integer_part: \"") + cardinal_graph + pynutil.insert("\" ")
-        self.fraction = pynutil.insert("fractional_part: \"") + cardinal_graph + pynutil.insert("\" ")
-
-        graph_currencies = optional_graph_negative + self.currency + insert_space + self.interger
-        graph_currencies |= (
-            optional_graph_negative
-            + self.currency
-            + insert_space
-            + self.interger
-            + pynutil.delete(".")
-            + insert_space
-            + self.fraction
+        currency_major = pynutil.insert('currency_maj: "') + currency_graph + pynutil.insert('"')
+        integer = pynutil.insert('integer_part: "') + cardinal_graph + pynutil.insert('"')
+        fraction = pynutil.insert('fractional_part: "') + cardinal_graph + pynutil.insert('"')
+        currency_minor = pynutil.insert('currency_min: "') + pynutil.insert("centiles") + pynutil.insert('"')
+
+        graph_major_only = currency_major + insert_space + integer
+        graph_major_and_minor = (
+            currency_major + insert_space + integer + pynini.cross(".", " ") + fraction + insert_space + currency_minor
         )
-        graph = graph_currencies
-        self.graph = graph.optimize()
+
+        graph_currencies = graph_major_only | graph_major_and_minor
+
+        graph = graph_currencies.optimize()
         final_graph = self.add_tokens(graph)
         self.fst = final_graph

nemo_text_processing/text_normalization/hi/taggers/time.py

@@ -13,10 +13,10 @@
 # limitations under the License.
 
 import pynini
-from pynini.lib import pynutil
+from pynini.lib import pynutil, rewrite
 
 from nemo_text_processing.text_normalization.hi.graph_utils import GraphFst, insert_space
-from nemo_text_processing.text_normalization.hi.utils import get_abs_path
+from nemo_text_processing.text_normalization.hi.utils import apply_fst, get_abs_path
 
 hours_graph = pynini.string_file(get_abs_path("data/time/hours.tsv"))
 minutes_graph = pynini.string_file(get_abs_path("data/time/minutes.tsv"))