feat: add Marathi TN/ITN with raw-field tagging

README.md

@@ -47,6 +47,7 @@ ITN converts spoken expressions into written forms, for example `十二` → `12
 | Japanese | `ja` | Yes | Yes | `japanese` |
 | Kinyarwanda | `rw` | Yes | Yes | `kinyarwanda` |
 | Korean | `ko` | Yes | Yes | `korean` |
+| Marathi | `mr` | Yes | Yes | `marathi` |
 | Portuguese | `pt` | Yes | Yes | `portuguese` |
 | Russian | `ru` | Yes | Yes | `russian` |
 | Spanish | `es` | Yes | Yes | `spanish` |

itn/marathi/data/date/eras.tsv

@@ -0,0 +1,2 @@
+इ.स.पू.	इसवी सन पूर्व
+इ.स.	इसवी सन

itn/marathi/data/date/months.tsv

@@ -0,0 +1,12 @@
+जानेवारी
+फेब्रुवारी
+मार्च
+एप्रिल
+मे
+जून
+जुलै
+ऑगस्ट
+सप्टेंबर
+ऑक्टोबर
+नोव्हेंबर
+डिसेंबर

itn/marathi/data/numbers/digit.tsv

@@ -0,0 +1,9 @@
+१	एक
+२	दोन
+३	तीन
+४	चार
+५	पाच
+६	सहा
+७	सात
+८	आठ
+९	नऊ

itn/marathi/data/numbers/tens.tsv

@@ -0,0 +1,90 @@
+१०	दहा
+११	अकरा
+१२	बारा
+१३	तेरा
+१४	चौदा
+१५	पंधरा
+१६	सोळा
+१७	सतरा
+१८	अठरा
+१९	एकोणीस
+२०	वीस
+२१	एकवीस
+२२	बावीस
+२३	तेवीस
+२४	चोवीस
+२५	पंचवीस
+२६	सव्वीस
+२७	सत्तावीस
+२८	अठ्ठावीस
+२९	एकोणतीस
+३०	तीस
+३१	एकतीस
+३२	बत्तीस
+३३	तेहतीस
+३४	चौतीस
+३५	पस्तीस
+३६	छत्तीस
+३७	सदतीस
+३८	अडतीस
+३९	एकोणचाळीस
+४०	चाळीस
+४१	एकेचाळीस
+४२	बेचाळीस
+४३	त्रेचाळीस
+४४	चव्वेचाळीस
+४५	पंचेचाळीस
+४६	सेहचाळीस
+४७	सत्तेचाळीस
+४८	अठ्ठेचाळीस
+४९	एकोणपन्नास
+५०	पन्नास
+५१	एकावन्न
+५२	बावन्न
+५३	त्रेपन्न
+५४	चौपन्न
+५५	पंचावन्न
+५६	छप्पन
+५७	सत्तावन्न
+५८	अठ्ठावन्न
+५९	एकोणसाठ
+६०	साठ
+६१	एकसष्ट
+६२	बासष्ट
+६३	त्रेसष्ट
+६४	चौसष्ट
+६५	पासष्ट
+६६	सहासष्ठ
+६७	सदुसष्ट
+६८	अडुसष्ठ
+६९	एकोणसत्तर
+७०	सत्तर
+७१	एकाहत्तर
+७२	बाहत्तर
+७३	त्र्याहत्तर
+७४	चौऱ्याहत्तर
+७५	पंचाहत्तर
+७६	शहात्तर
+७७	सत्याहत्तर
+७८	अठ्ठ्यात्तर
+७९	एकोणऐंशी
+८०	ऐंशी
+८१	एक्याऐंशी
+८२	ब्याऐंशी
+८३	त्र्याऐंशी
+८४	चौऱ्याऐंशी
+८५	पंच्याऐंशी
+८६	सह्यांशी
+८७	सत्याऐंशी
+८८	अठ्ठ्याऐंशी
+८९	एकोणनव्वद
+९०	नव्वद
+९१	एक्याण्णव
+९२	ब्याण्णव
+९३	त्र्याण्णव
+९४	चौऱ्याण्णव
+९५	पंच्याण्णव
+९६	शहाण्णव
+९७	सत्त्याण्णव
+९८	अठ्ठ्याण्णव
+९९	नव्व्याण्णव

itn/marathi/data/numbers/zero.tsv

@@ -0,0 +1 @@
+०	शून्य

itn/marathi/inverse_normalizer.py

@@ -0,0 +1,56 @@
+# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved.
+# Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn)
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from pynini.lib.pynutil import add_weight
+
+from itn.marathi.rules.cardinal import Cardinal
+from itn.marathi.rules.date import Date
+from itn.marathi.rules.decimal import Decimal
+from itn.marathi.rules.punctuation import Punctuation
+from itn.marathi.rules.time import Time
+from itn.marathi.rules.word import Word
+from tn.processor import Processor, RuleSpec
+
+# Every Marathi rule emits its fields in written order, so no token needs
+# reordering before verbalization.
+TOKEN_ORDERS = {}
+
+
+class InverseNormalizer(Processor):
+
+    def __init__(self, cache_dir=None, overwrite_cache=False):
+        super().__init__(name="mr_inverse_normalizer", ordertype="itn", token_orders=TOKEN_ORDERS)
+        self.build_fst("mr_itn", cache_dir, overwrite_cache, {})
+
+    def build_tagger_and_verbalizer(self):
+        cardinal = Cardinal()
+        punctuation = Punctuation()
+        word = Word()
+        rules = (
+            RuleSpec(Time(), 1.05),
+            RuleSpec(Date(cardinal), 1.09),
+            RuleSpec(Decimal(cardinal), 1.08),
+            RuleSpec(cardinal, 1.1),
+            RuleSpec(word, 100),
+            RuleSpec(punctuation),
+        )
+        classify = self.tagger_union(rules)
+        # Punctuation splits off only when attached to another token.
+        punct = add_weight(punctuation.tagger, 1.1)
+        token = punct.star + classify + punct.star
+        self.tagger = self.DELETE_SPACE + token + (self.DELETE_EXTRA_SPACE + token).star + self.DELETE_SPACE
+        self.tagger |= word.add_tokens(word.tag_field("value", self.SPACE.plus))
+        verbalize = self.verbalizer_union(rules)
+        self.verbalizer = verbalize + (self.INSERT_SPACE + verbalize).star

itn/marathi/rules/cardinal.py

@@ -0,0 +1,76 @@
+# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved.
+# Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn)
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import pynini
+from pynini.lib.pynutil import add_weight, delete, insert
+
+from itn.marathi.rules.utils import DIGIT, NON_ZERO, load_pairs, spoken_map
+from tn.processor import Processor
+
+MINUS = "उणे"
+# Indian place values above the hundreds, each spanning two digits.
+SCALES = ("अब्ज", "कोटी", "लाख", "हजार")
+# Skipping a place value costs a little, so ``चार लाख`` prefers the decimal
+# quantity reading ``४ लाख`` over the cardinal ``४०००००``.
+SKIP_WEIGHT = 0.1
+
+
+class Cardinal(Processor):
+    """Spoken cardinals, e.g. ``उणे सातशे बावीस`` -> ``-७२२``.
+
+    The tagger keeps the spoken words (and the ``उणे`` sign word); the
+    verbalizer assembles Devanagari digits in Indian grouping (lakh, crore,
+    abja) and also reads ``एकतीसशे`` style hundreds counts.
+    """
+
+    def __init__(self):
+        super().__init__(name="cardinal", ordertype="itn", token_orders={})
+        self.build_graphs()
+        negative = self.tag_field("negative", pynini.accep(MINUS)) + delete(self.SPACE.plus) + insert(" ")
+        self.tagger = self.add_tokens(negative.ques + self.tag_field("integer", self.graph))
+        sign = self.verbalize_field("negative", pynini.cross(MINUS, "-")) + delete(" ")
+        self.verbalizer = self.delete_tokens(sign.ques + self.verbalize_field("integer", self.graph))
+
+    def build_graphs(self):
+        ds = delete(self.SPACE).star
+        digit = spoken_map(load_pairs("numbers/digit.tsv"))
+        tens = spoken_map(load_pairs("numbers/tens.tsv"))
+        zero = spoken_map(load_pairs("numbers/zero.tsv"))
+        self.digit = digit
+        self.zero = zero
+
+        two_nonzero = (tens | insert("०") + digit).optimize()
+        two = (two_nonzero | insert("००")).optimize()
+        hundreds = (digit + delete("शे") + ds | insert("०")) + two
+        # ``एकतीसशे`` (thirty-one hundred) fills the thousands and hundreds slots.
+        hundreds_count = insert("०") + tens + delete("शे") + ds + two
+        thousands_and_hundreds = []
+        higher = pynini.accep("")
+        for scale in SCALES:
+            slot = two_nonzero + ds + delete(scale) + ds | add_weight(insert("००"), SKIP_WEIGHT)
+            if scale == "हजार":
+                thousands_and_hundreds = slot + hundreds | hundreds_count
+            else:
+                higher += slot
+        nonzero = DIGIT.star + NON_ZERO + DIGIT.star
+        strip_zeros = delete("०").star + NON_ZERO + DIGIT.star
+
+        trimmed = self.NOT_SPACE | self.NOT_SPACE + self.VSIGMA + self.NOT_SPACE
+        trimmed = trimmed.optimize()
+        hundred = pynini.cross("शंभर", "१००")
+        graph = (higher + thousands_and_hundreds) @ strip_zeros | hundred | zero
+        self.graph = (trimmed @ graph).optimize()
+        # 1-999 (including ``शंभर``) counts a lakh/crore quantity: ``सतरा लाख``.
+        self.hundreds = (trimmed @ ((hundreds @ nonzero) @ strip_zeros | hundred)).optimize()

itn/marathi/rules/date.py

@@ -0,0 +1,63 @@
+# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved.
+# Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn)
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import pynini
+from pynini.lib.pynutil import add_weight, delete, insert
+
+from itn.marathi.rules.utils import load_pairs, number_words, spoken_map, to_devanagari
+from tn.processor import Processor
+
+# A lone number after a month reads as the day unless it cannot be one.
+YEAR_ONLY_WEIGHT = 0.01
+
+
+class Date(Processor):
+    """Spoken dates in either day-month or month-day order.
+
+    ``पंधरा फेब्रुवारी अठराशे सतरा`` -> ``१५ फेब्रुवारी १८१७``, ``जानेवारी
+    तेरा दोन हजार तेवीस`` -> ``जानेवारी १३ २०२३`` and ``इसवी सन पूर्व
+    चारशे`` -> ``इ.स.पू. ४००``. The tagger keeps the spoken day, month,
+    year and era in spoken order.
+    """
+
+    def __init__(self, cardinal):
+        super().__init__(name="date", ordertype="itn", token_orders={})
+        words = number_words()
+        self.day = spoken_map([(to_devanagari(day), words[day]) for day in range(1, 32)])
+        self.month = pynini.union(*[spoken for _, spoken in load_pairs("date/months.tsv")]).optimize()
+        self.year = cardinal.graph
+        self.era = spoken_map(load_pairs("date/eras.tsv"))
+        self.build_tagger()
+        self.build_verbalizer()
+
+    def build_tagger(self):
+        sep = delete(self.SPACE.plus) + insert(" ")
+        day = self.tag_field("day", self.day)
+        month = self.tag_field("month", self.month)
+        year = self.tag_field("year", self.year)
+        era = self.tag_field("era", self.era)
+        mdy = month + sep + (day | add_weight(year, YEAR_ONLY_WEIGHT) | day + sep + year)
+        dmy = day + sep + month + (sep + year).ques
+        self.tagger = self.add_tokens(mdy | dmy | era + sep + year)
+
+    def build_verbalizer(self):
+        sep = insert(" ")
+        day = self.verbalize_field("day", self.day, True)
+        month = self.verbalize_field("month", self.month, True)
+        year = self.verbalize_field("year", self.year, True)
+        mdy = self.verbalize_field("month", self.month) + sep + (day | year | day + sep + year)
+        dmy = self.verbalize_field("day", self.day) + sep + month + (sep + year).ques
+        era = self.verbalize_field("era", self.era) + sep + year
+        self.verbalizer = self.delete_tokens(mdy | dmy | era)

itn/marathi/rules/decimal.py

@@ -0,0 +1,62 @@
+# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved.
+# Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn)
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import pynini
+from pynini.lib.pynutil import delete, insert
+
+from itn.marathi.rules.cardinal import MINUS
+from tn.processor import Processor
+
+
+class Decimal(Processor):
+    """Spoken decimals, e.g. ``उणे बारा पूर्णांक एक पाच कोटी`` -> ``-१२.१५ कोटी``.
+
+    The tagger keeps the spoken sign, integer, fractional digits and quantity
+    and drops the spoken decimal point ``पूर्णांक``.
+    """
+
+    def __init__(self, cardinal):
+        super().__init__(name="decimal", ordertype="itn", token_orders={})
+        digit = (cardinal.digit | cardinal.zero).optimize()
+        self.fractional = ((digit + delete(self.SPACE.plus)).star + digit).optimize()
+        self.integer = cardinal.graph
+        self.quantity_integer = cardinal.hundreds
+        self.quantity = pynini.union("लाख", "कोटी", "अब्ज").optimize()
+        self.decimal_quantity = (self.quantity | "हजार").optimize()
+        self.build_tagger()
+        self.build_verbalizer()
+
+    def build_tagger(self):
+        spaces = delete(self.SPACE.plus)
+        negative = self.tag_field("negative", pynini.accep(MINUS)) + spaces + insert(" ")
+        point = delete("पूर्णांक") + spaces
+        integer = self.tag_field("integer_part", self.integer) + spaces + point + insert(" ")
+        fractional = self.tag_field("fractional_part", self.fractional)
+        decimal = integer + fractional | point + fractional
+        quantity = self.tag_field("integer_part", self.quantity_integer) + spaces + insert(" ")
+        quantity += self.tag_field("quantity", self.quantity)
+        quantity |= decimal + spaces + insert(" ") + self.tag_field("quantity", self.decimal_quantity)
+        self.tagger = self.add_tokens(negative.ques + (decimal | quantity))
+
+    def build_verbalizer(self):
+        sign = self.verbalize_field("negative", pynini.cross(MINUS, "-")) + delete(" ")
+        integer = self.verbalize_field("integer_part", self.integer)
+        fractional = insert(".") + self.verbalize_field("fractional_part", self.fractional, leading_space=True)
+        decimal = integer + fractional
+        decimal |= insert(".") + self.verbalize_field("fractional_part", self.fractional)
+        quantity = self.verbalize_field("integer_part", self.quantity_integer) + insert(" ")
+        quantity += self.verbalize_field("quantity", self.quantity, leading_space=True)
+        quantity |= decimal + insert(" ") + self.verbalize_field("quantity", self.decimal_quantity, leading_space=True)
+        self.verbalizer = self.delete_tokens(sign.ques + (decimal | quantity))