From ba2159b65bf74bc8e0b69907564b029b9addbeb4 Mon Sep 17 00:00:00 2001 From: pengzhendong <275331498@qq.com> Date: Sat, 31 Oct 2026 10:07:27 +0800 Subject: [PATCH 1/2] feat: add Marathi TN/ITN with raw-field tagging --- itn/marathi/__init__.py | 0 itn/marathi/data/date/eras.tsv | 2 + itn/marathi/data/date/months.tsv | 12 ++ itn/marathi/data/numbers/digit.tsv | 9 + itn/marathi/data/numbers/tens.tsv | 90 ++++++++ itn/marathi/data/numbers/zero.tsv | 1 + itn/marathi/inverse_normalizer.py | 56 +++++ itn/marathi/rules/__init__.py | 0 itn/marathi/rules/cardinal.py | 76 +++++++ itn/marathi/rules/date.py | 63 ++++++ itn/marathi/rules/decimal.py | 62 ++++++ itn/marathi/rules/punctuation.py | 29 +++ itn/marathi/rules/time.py | 69 ++++++ itn/marathi/rules/utils.py | 62 ++++++ itn/marathi/rules/word.py | 28 +++ itn/marathi/test/__init__.py | 0 itn/marathi/test/data/test_cases_cardinal.txt | 33 +++ itn/marathi/test/data/test_cases_date.txt | 29 +++ itn/marathi/test/data/test_cases_decimal.txt | 29 +++ itn/marathi/test/data/test_cases_time.txt | 29 +++ itn/marathi/test/data/test_cases_word.txt | 11 + itn/marathi/test/normalizer_test.py | 164 +++++++++++++++ tn/marathi/__init__.py | 0 tn/marathi/data/date/eras.tsv | 2 + tn/marathi/data/date/months.tsv | 12 ++ tn/marathi/data/numbers/digit.tsv | 9 + tn/marathi/data/numbers/tens.tsv | 90 ++++++++ tn/marathi/data/numbers/zero.tsv | 1 + tn/marathi/normalizer.py | 55 +++++ tn/marathi/rules/__init__.py | 0 tn/marathi/rules/cardinal.py | 76 +++++++ tn/marathi/rules/date.py | 70 +++++++ tn/marathi/rules/decimal.py | 59 ++++++ tn/marathi/rules/punctuation.py | 28 +++ tn/marathi/rules/time.py | 86 ++++++++ tn/marathi/rules/utils.py | 76 +++++++ tn/marathi/rules/word.py | 28 +++ tn/marathi/test/__init__.py | 0 tn/marathi/test/normalizer_test.py | 197 ++++++++++++++++++ 39 files changed, 1643 insertions(+) create mode 100644 itn/marathi/__init__.py create mode 100644 itn/marathi/data/date/eras.tsv create mode 100644 itn/marathi/data/date/months.tsv create mode 100644 itn/marathi/data/numbers/digit.tsv create mode 100644 itn/marathi/data/numbers/tens.tsv create mode 100644 itn/marathi/data/numbers/zero.tsv create mode 100644 itn/marathi/inverse_normalizer.py create mode 100644 itn/marathi/rules/__init__.py create mode 100644 itn/marathi/rules/cardinal.py create mode 100644 itn/marathi/rules/date.py create mode 100644 itn/marathi/rules/decimal.py create mode 100644 itn/marathi/rules/punctuation.py create mode 100644 itn/marathi/rules/time.py create mode 100644 itn/marathi/rules/utils.py create mode 100644 itn/marathi/rules/word.py create mode 100644 itn/marathi/test/__init__.py create mode 100644 itn/marathi/test/data/test_cases_cardinal.txt create mode 100644 itn/marathi/test/data/test_cases_date.txt create mode 100644 itn/marathi/test/data/test_cases_decimal.txt create mode 100644 itn/marathi/test/data/test_cases_time.txt create mode 100644 itn/marathi/test/data/test_cases_word.txt create mode 100644 itn/marathi/test/normalizer_test.py create mode 100644 tn/marathi/__init__.py create mode 100644 tn/marathi/data/date/eras.tsv create mode 100644 tn/marathi/data/date/months.tsv create mode 100644 tn/marathi/data/numbers/digit.tsv create mode 100644 tn/marathi/data/numbers/tens.tsv create mode 100644 tn/marathi/data/numbers/zero.tsv create mode 100644 tn/marathi/normalizer.py create mode 100644 tn/marathi/rules/__init__.py create mode 100644 tn/marathi/rules/cardinal.py create mode 100644 tn/marathi/rules/date.py create mode 100644 tn/marathi/rules/decimal.py create mode 100644 tn/marathi/rules/punctuation.py create mode 100644 tn/marathi/rules/time.py create mode 100644 tn/marathi/rules/utils.py create mode 100644 tn/marathi/rules/word.py create mode 100644 tn/marathi/test/__init__.py create mode 100644 tn/marathi/test/normalizer_test.py diff --git a/itn/marathi/__init__.py b/itn/marathi/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/itn/marathi/data/date/eras.tsv b/itn/marathi/data/date/eras.tsv new file mode 100644 index 00000000..517186b0 --- /dev/null +++ b/itn/marathi/data/date/eras.tsv @@ -0,0 +1,2 @@ +इ.स.पू. इसवी सन पूर्व +इ.स. इसवी सन diff --git a/itn/marathi/data/date/months.tsv b/itn/marathi/data/date/months.tsv new file mode 100644 index 00000000..f39b90d2 --- /dev/null +++ b/itn/marathi/data/date/months.tsv @@ -0,0 +1,12 @@ +जानेवारी +फेब्रुवारी +मार्च +एप्रिल +मे +जून +जुलै +ऑगस्ट +सप्टेंबर +ऑक्टोबर +नोव्हेंबर +डिसेंबर \ No newline at end of file diff --git a/itn/marathi/data/numbers/digit.tsv b/itn/marathi/data/numbers/digit.tsv new file mode 100644 index 00000000..f737ade2 --- /dev/null +++ b/itn/marathi/data/numbers/digit.tsv @@ -0,0 +1,9 @@ +१ एक +२ दोन +३ तीन +४ चार +५ पाच +६ सहा +७ सात +८ आठ +९ नऊ \ No newline at end of file diff --git a/itn/marathi/data/numbers/tens.tsv b/itn/marathi/data/numbers/tens.tsv new file mode 100644 index 00000000..654244b3 --- /dev/null +++ b/itn/marathi/data/numbers/tens.tsv @@ -0,0 +1,90 @@ +१० दहा +११ अकरा +१२ बारा +१३ तेरा +१४ चौदा +१५ पंधरा +१६ सोळा +१७ सतरा +१८ अठरा +१९ एकोणीस +२० वीस +२१ एकवीस +२२ बावीस +२३ तेवीस +२४ चोवीस +२५ पंचवीस +२६ सव्वीस +२७ सत्तावीस +२८ अठ्ठावीस +२९ एकोणतीस +३० तीस +३१ एकतीस +३२ बत्तीस +३३ तेहतीस +३४ चौतीस +३५ पस्तीस +३६ छत्तीस +३७ सदतीस +३८ अडतीस +३९ एकोणचाळीस +४० चाळीस +४१ एकेचाळीस +४२ बेचाळीस +४३ त्रेचाळीस +४४ चव्वेचाळीस +४५ पंचेचाळीस +४६ सेहचाळीस +४७ सत्तेचाळीस +४८ अठ्ठेचाळीस +४९ एकोणपन्नास +५० पन्नास +५१ एकावन्न +५२ बावन्न +५३ त्रेपन्न +५४ चौपन्न +५५ पंचावन्न +५६ छप्पन +५७ सत्तावन्न +५८ अठ्ठावन्न +५९ एकोणसाठ +६० साठ +६१ एकसष्ट +६२ बासष्ट +६३ त्रेसष्ट +६४ चौसष्ट +६५ पासष्ट +६६ सहासष्ठ +६७ सदुसष्ट +६८ अडुसष्ठ +६९ एकोणसत्तर +७० सत्तर +७१ एकाहत्तर +७२ बाहत्तर +७३ त्र्याहत्तर +७४ चौऱ्याहत्तर +७५ पंचाहत्तर +७६ शहात्तर +७७ सत्याहत्तर +७८ अठ्ठ्यात्तर +७९ एकोणऐंशी +८० ऐंशी +८१ एक्याऐंशी +८२ ब्याऐंशी +८३ त्र्याऐंशी +८४ चौऱ्याऐंशी +८५ पंच्याऐंशी +८६ सह्यांशी +८७ सत्याऐंशी +८८ अठ्ठ्याऐंशी +८९ एकोणनव्वद +९० नव्वद +९१ एक्याण्णव +९२ ब्याण्णव +९३ त्र्याण्णव +९४ चौऱ्याण्णव +९५ पंच्याण्णव +९६ शहाण्णव +९७ सत्त्याण्णव +९८ अठ्ठ्याण्णव +९९ नव्व्याण्णव \ No newline at end of file diff --git a/itn/marathi/data/numbers/zero.tsv b/itn/marathi/data/numbers/zero.tsv new file mode 100644 index 00000000..0735899f --- /dev/null +++ b/itn/marathi/data/numbers/zero.tsv @@ -0,0 +1 @@ +० शून्य diff --git a/itn/marathi/inverse_normalizer.py b/itn/marathi/inverse_normalizer.py new file mode 100644 index 00000000..5ac0c394 --- /dev/null +++ b/itn/marathi/inverse_normalizer.py @@ -0,0 +1,56 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from pynini.lib.pynutil import add_weight + +from itn.marathi.rules.cardinal import Cardinal +from itn.marathi.rules.date import Date +from itn.marathi.rules.decimal import Decimal +from itn.marathi.rules.punctuation import Punctuation +from itn.marathi.rules.time import Time +from itn.marathi.rules.word import Word +from tn.processor import Processor, RuleSpec + +# Every Marathi rule emits its fields in written order, so no token needs +# reordering before verbalization. +TOKEN_ORDERS = {} + + +class InverseNormalizer(Processor): + + def __init__(self, cache_dir=None, overwrite_cache=False): + super().__init__(name="mr_inverse_normalizer", ordertype="itn", token_orders=TOKEN_ORDERS) + self.build_fst("mr_itn", cache_dir, overwrite_cache, {}) + + def build_tagger_and_verbalizer(self): + cardinal = Cardinal() + punctuation = Punctuation() + word = Word() + rules = ( + RuleSpec(Time(), 1.05), + RuleSpec(Date(cardinal), 1.09), + RuleSpec(Decimal(cardinal), 1.08), + RuleSpec(cardinal, 1.1), + RuleSpec(word, 100), + RuleSpec(punctuation), + ) + classify = self.tagger_union(rules) + # Punctuation splits off only when attached to another token. + punct = add_weight(punctuation.tagger, 1.1) + token = punct.star + classify + punct.star + self.tagger = self.DELETE_SPACE + token + (self.DELETE_EXTRA_SPACE + token).star + self.DELETE_SPACE + self.tagger |= word.add_tokens(word.tag_field("value", self.SPACE.plus)) + verbalize = self.verbalizer_union(rules) + self.verbalizer = verbalize + (self.INSERT_SPACE + verbalize).star diff --git a/itn/marathi/rules/__init__.py b/itn/marathi/rules/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/itn/marathi/rules/cardinal.py b/itn/marathi/rules/cardinal.py new file mode 100644 index 00000000..27796971 --- /dev/null +++ b/itn/marathi/rules/cardinal.py @@ -0,0 +1,76 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib.pynutil import add_weight, delete, insert + +from itn.marathi.rules.utils import DIGIT, NON_ZERO, load_pairs, spoken_map +from tn.processor import Processor + +MINUS = "उणे" +# Indian place values above the hundreds, each spanning two digits. +SCALES = ("अब्ज", "कोटी", "लाख", "हजार") +# Skipping a place value costs a little, so ``चार लाख`` prefers the decimal +# quantity reading ``४ लाख`` over the cardinal ``४०००००``. +SKIP_WEIGHT = 0.1 + + +class Cardinal(Processor): + """Spoken cardinals, e.g. ``उणे सातशे बावीस`` -> ``-७२२``. + + The tagger keeps the spoken words (and the ``उणे`` sign word); the + verbalizer assembles Devanagari digits in Indian grouping (lakh, crore, + abja) and also reads ``एकतीसशे`` style hundreds counts. + """ + + def __init__(self): + super().__init__(name="cardinal", ordertype="itn", token_orders={}) + self.build_graphs() + negative = self.tag_field("negative", pynini.accep(MINUS)) + delete(self.SPACE.plus) + insert(" ") + self.tagger = self.add_tokens(negative.ques + self.tag_field("integer", self.graph)) + sign = self.verbalize_field("negative", pynini.cross(MINUS, "-")) + delete(" ") + self.verbalizer = self.delete_tokens(sign.ques + self.verbalize_field("integer", self.graph)) + + def build_graphs(self): + ds = delete(self.SPACE).star + digit = spoken_map(load_pairs("numbers/digit.tsv")) + tens = spoken_map(load_pairs("numbers/tens.tsv")) + zero = spoken_map(load_pairs("numbers/zero.tsv")) + self.digit = digit + self.zero = zero + + two_nonzero = (tens | insert("०") + digit).optimize() + two = (two_nonzero | insert("००")).optimize() + hundreds = (digit + delete("शे") + ds | insert("०")) + two + # ``एकतीसशे`` (thirty-one hundred) fills the thousands and hundreds slots. + hundreds_count = insert("०") + tens + delete("शे") + ds + two + thousands_and_hundreds = [] + higher = pynini.accep("") + for scale in SCALES: + slot = two_nonzero + ds + delete(scale) + ds | add_weight(insert("००"), SKIP_WEIGHT) + if scale == "हजार": + thousands_and_hundreds = slot + hundreds | hundreds_count + else: + higher += slot + nonzero = DIGIT.star + NON_ZERO + DIGIT.star + strip_zeros = delete("०").star + NON_ZERO + DIGIT.star + + trimmed = self.NOT_SPACE | self.NOT_SPACE + self.VSIGMA + self.NOT_SPACE + trimmed = trimmed.optimize() + hundred = pynini.cross("शंभर", "१००") + graph = (higher + thousands_and_hundreds) @ strip_zeros | hundred | zero + self.graph = (trimmed @ graph).optimize() + # 1-999 (including ``शंभर``) counts a lakh/crore quantity: ``सतरा लाख``. + self.hundreds = (trimmed @ ((hundreds @ nonzero) @ strip_zeros | hundred)).optimize() diff --git a/itn/marathi/rules/date.py b/itn/marathi/rules/date.py new file mode 100644 index 00000000..cbea3a49 --- /dev/null +++ b/itn/marathi/rules/date.py @@ -0,0 +1,63 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib.pynutil import add_weight, delete, insert + +from itn.marathi.rules.utils import load_pairs, number_words, spoken_map, to_devanagari +from tn.processor import Processor + +# A lone number after a month reads as the day unless it cannot be one. +YEAR_ONLY_WEIGHT = 0.01 + + +class Date(Processor): + """Spoken dates in either day-month or month-day order. + + ``पंधरा फेब्रुवारी अठराशे सतरा`` -> ``१५ फेब्रुवारी १८१७``, ``जानेवारी + तेरा दोन हजार तेवीस`` -> ``जानेवारी १३ २०२३`` and ``इसवी सन पूर्व + चारशे`` -> ``इ.स.पू. ४००``. The tagger keeps the spoken day, month, + year and era in spoken order. + """ + + def __init__(self, cardinal): + super().__init__(name="date", ordertype="itn", token_orders={}) + words = number_words() + self.day = spoken_map([(to_devanagari(day), words[day]) for day in range(1, 32)]) + self.month = pynini.union(*[spoken for _, spoken in load_pairs("date/months.tsv")]).optimize() + self.year = cardinal.graph + self.era = spoken_map(load_pairs("date/eras.tsv")) + self.build_tagger() + self.build_verbalizer() + + def build_tagger(self): + sep = delete(self.SPACE.plus) + insert(" ") + day = self.tag_field("day", self.day) + month = self.tag_field("month", self.month) + year = self.tag_field("year", self.year) + era = self.tag_field("era", self.era) + mdy = month + sep + (day | add_weight(year, YEAR_ONLY_WEIGHT) | day + sep + year) + dmy = day + sep + month + (sep + year).ques + self.tagger = self.add_tokens(mdy | dmy | era + sep + year) + + def build_verbalizer(self): + sep = insert(" ") + day = self.verbalize_field("day", self.day, True) + month = self.verbalize_field("month", self.month, True) + year = self.verbalize_field("year", self.year, True) + mdy = self.verbalize_field("month", self.month) + sep + (day | year | day + sep + year) + dmy = self.verbalize_field("day", self.day) + sep + month + (sep + year).ques + era = self.verbalize_field("era", self.era) + sep + year + self.verbalizer = self.delete_tokens(mdy | dmy | era) diff --git a/itn/marathi/rules/decimal.py b/itn/marathi/rules/decimal.py new file mode 100644 index 00000000..518d8ce4 --- /dev/null +++ b/itn/marathi/rules/decimal.py @@ -0,0 +1,62 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib.pynutil import delete, insert + +from itn.marathi.rules.cardinal import MINUS +from tn.processor import Processor + + +class Decimal(Processor): + """Spoken decimals, e.g. ``उणे बारा पूर्णांक एक पाच कोटी`` -> ``-१२.१५ कोटी``. + + The tagger keeps the spoken sign, integer, fractional digits and quantity + and drops the spoken decimal point ``पूर्णांक``. + """ + + def __init__(self, cardinal): + super().__init__(name="decimal", ordertype="itn", token_orders={}) + digit = (cardinal.digit | cardinal.zero).optimize() + self.fractional = ((digit + delete(self.SPACE.plus)).star + digit).optimize() + self.integer = cardinal.graph + self.quantity_integer = cardinal.hundreds + self.quantity = pynini.union("लाख", "कोटी", "अब्ज").optimize() + self.decimal_quantity = (self.quantity | "हजार").optimize() + self.build_tagger() + self.build_verbalizer() + + def build_tagger(self): + spaces = delete(self.SPACE.plus) + negative = self.tag_field("negative", pynini.accep(MINUS)) + spaces + insert(" ") + point = delete("पूर्णांक") + spaces + integer = self.tag_field("integer_part", self.integer) + spaces + point + insert(" ") + fractional = self.tag_field("fractional_part", self.fractional) + decimal = integer + fractional | point + fractional + quantity = self.tag_field("integer_part", self.quantity_integer) + spaces + insert(" ") + quantity += self.tag_field("quantity", self.quantity) + quantity |= decimal + spaces + insert(" ") + self.tag_field("quantity", self.decimal_quantity) + self.tagger = self.add_tokens(negative.ques + (decimal | quantity)) + + def build_verbalizer(self): + sign = self.verbalize_field("negative", pynini.cross(MINUS, "-")) + delete(" ") + integer = self.verbalize_field("integer_part", self.integer) + fractional = insert(".") + self.verbalize_field("fractional_part", self.fractional, leading_space=True) + decimal = integer + fractional + decimal |= insert(".") + self.verbalize_field("fractional_part", self.fractional) + quantity = self.verbalize_field("integer_part", self.quantity_integer) + insert(" ") + quantity += self.verbalize_field("quantity", self.quantity, leading_space=True) + quantity |= decimal + insert(" ") + self.verbalize_field("quantity", self.decimal_quantity, leading_space=True) + self.verbalizer = self.delete_tokens(sign.ques + (decimal | quantity)) diff --git a/itn/marathi/rules/punctuation.py b/itn/marathi/rules/punctuation.py new file mode 100644 index 00000000..708f3530 --- /dev/null +++ b/itn/marathi/rules/punctuation.py @@ -0,0 +1,29 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini + +from tn.processor import Processor + + +class Punctuation(Processor): + """ASCII and Devanagari punctuation split off a neighbouring token, e.g. ``चार।``.""" + + def __init__(self): + super().__init__(name="punctuation", ordertype="itn", token_orders={}) + marks = "!#$%&'()*+,-./:;<=>?@^_`{|}~।॥‘’“”" + graph = pynini.union(*[pynini.escape(mark) for mark in marks]).optimize() + self.tagger = self.add_tokens(self.tag_field("value", graph)) + self.verbalizer = self.delete_tokens(self.verbalize_field("value", graph)) diff --git a/itn/marathi/rules/time.py b/itn/marathi/rules/time.py new file mode 100644 index 00000000..18c8ac55 --- /dev/null +++ b/itn/marathi/rules/time.py @@ -0,0 +1,69 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib.pynutil import delete, insert + +from itn.marathi.rules.utils import number_words, spoken_map, to_devanagari +from tn.processor import Processor + +# Quarter words before the hour and the minutes they stand for. +QUARTERS = {"सव्वा": "१५", "साडे": "३०", "पावणे": "४५"} +# ``दीड`` (half past one) and ``अडीच`` (half past two) name a whole time. +HALVES = {"दीड": "०१:३०", "अडीच": "०२:३०"} + + +class Time(Processor): + """Spoken clock times on the 12-hour dial. + + ``अकरा वाजून दोन मिनिटे`` -> ``११:०२`` (minutes past), ``अकराला पाच + मिनिटे`` -> ``१०:५५`` (minutes to; the ``-ला`` ending stays in the + hours field), ``साडे चार`` -> ``०४:३०`` and ``दीड`` -> ``०१:३०``. The + tagger drops ``वाजून`` and ``मिनिट(े)``. + """ + + def __init__(self): + super().__init__(name="time", ordertype="itn", token_orders={}) + words = number_words() + hours = spoken_map([(to_devanagari(hour, 2), words[hour]) for hour in range(1, 13)]) + to_hours = spoken_map([(to_devanagari((hour + 10) % 12 + 1, 2), words[hour] + "ला") for hour in range(1, 13)]) + before = spoken_map([(to_devanagari((hour + 10) % 12 + 1, 2), words[hour]) for hour in range(1, 13)]) + minutes = spoken_map([(to_devanagari(minute, 2), words[minute]) for minute in range(1, 60)]) + to_minutes = spoken_map([(to_devanagari(60 - minute, 2), words[minute]) for minute in range(1, 60)]) + halves = pynini.string_map(HALVES.items()).optimize() + + spaces = delete(self.SPACE.plus) + separator = spaces + insert(" ") + unit = spaces + delete(pynini.union("मिनिटे", "मिनिट")) + past = self.tag_field("hours", hours) + spaces + delete("वाजून") + separator + past += self.tag_field("minutes", minutes) + unit + to = self.tag_field("hours", to_hours) + separator + self.tag_field("minutes", to_minutes) + unit + quarters = [] + for word, value in QUARTERS.items(): + graph = before if word == "पावणे" else hours + quarters.append(self.tag_field("relation", pynini.accep(word)) + separator + self.tag_field("hours", graph)) + half = self.tag_field("hours", halves) + self.tagger = self.add_tokens(past | to | pynini.union(*quarters) | half) + + colon = insert(":") + past = self.verbalize_field("hours", hours) + colon + self.verbalize_field("minutes", minutes, True) + to = self.verbalize_field("hours", to_hours) + colon + self.verbalize_field("minutes", to_minutes, True) + quarters = [] + for word, value in QUARTERS.items(): + graph = before if word == "पावणे" else hours + relation = self.verbalize_field("relation", pynini.cross(word, "")) + quarters.append(relation + self.verbalize_field("hours", graph, True) + insert(":" + value)) + half = self.verbalize_field("hours", halves) + self.verbalizer = self.delete_tokens(past | to | pynini.union(*quarters) | half) diff --git a/itn/marathi/rules/utils.py b/itn/marathi/rules/utils.py new file mode 100644 index 00000000..996fe38c --- /dev/null +++ b/itn/marathi/rules/utils.py @@ -0,0 +1,62 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini + +from itn.utils import get_abs_path + +# Devanagari digits, the written form produced by Marathi ITN. +DIGITS = "०१२३४५६७८९" +DIGIT = pynini.union(*DIGITS).optimize() +NON_ZERO = pynini.union(*DIGITS[1:]).optimize() + + +def to_devanagari(number, width=1): + """Formats an integer with Devanagari digits, zero-padded to ``width``.""" + + return "".join(DIGITS[int(char)] for char in str(number).zfill(width)) + + +def load_pairs(rel_path): + """Loads ``writtenspoken`` rows of a Marathi data table in file order. + + A single-column row is read the same way on both sides. + """ + + pairs = [] + with open(get_abs_path("marathi/data/" + rel_path), encoding="utf-8") as stream: + for line in stream: + line = line.rstrip("\r\n") + if line: + columns = line.split("\t") + pairs.append((columns[0], columns[-1])) + return pairs + + +def number_words(): + """Maps 1-99 to their Marathi spoken words.""" + + words = {} + for name in ("digit", "tens"): + for written, spoken in load_pairs("numbers/{}.tsv".format(name)): + words[int(written.translate(str.maketrans(DIGITS, "0123456789")))] = spoken + return words + + +def spoken_map(pairs): + """Builds a ``spoken -> written`` transducer from ``(written, spoken)`` pairs.""" + + return pynini.union(*[pynini.cross(pynini.escape(spoken), pynini.escape(written)) + for written, spoken in pairs]).optimize() diff --git a/itn/marathi/rules/word.py b/itn/marathi/rules/word.py new file mode 100644 index 00000000..a5405c12 --- /dev/null +++ b/itn/marathi/rules/word.py @@ -0,0 +1,28 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from pynini.lib.pynutil import delete + +from tn.processor import Processor + + +class Word(Processor): + """Any other non-space run, kept verbatim.""" + + def __init__(self): + super().__init__(name="word", ordertype="itn", token_orders={}) + self.tagger = self.add_tokens(self.tag_field("value", self.NOT_SPACE.plus)) + graph = self.NOT_SPACE.plus | delete(self.SPACE.plus) + self.verbalizer = self.delete_tokens(self.verbalize_field("value", graph)) diff --git a/itn/marathi/test/__init__.py b/itn/marathi/test/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/itn/marathi/test/data/test_cases_cardinal.txt b/itn/marathi/test/data/test_cases_cardinal.txt new file mode 100644 index 00000000..e5ccff88 --- /dev/null +++ b/itn/marathi/test/data/test_cases_cardinal.txt @@ -0,0 +1,33 @@ +शून्य~० +उणे शंभर~-१०० +उणे सत्तावीस~-२७ +तीनशे एक~३०१ +सातशे~७०० +सातशे बावीस~७२२ +तीन हजार~३००० +आठ हजार तीन~८००३ +पाच हजार नव्वद~५०९० +उणे दोन हजार चाळीस~-२०४० +एक हजार सातशे अडुसष्ठ~१७६८ +नऊ हजार दोनशे~९२०० +तीन हजार एकशे~३१०० +एकतीसशे~३१०० +अठ्ठावन्नशे एक~५८०१ +उणे बावन्नशे~-५२०० +पंचाहत्तरशे सोळा~७५१६ +तेहतीस हजार~३३००० +सत्त्याण्णव हजार तीन~९७००३ +एक्याऐंशी हजार एक्याऐंशी~८१०८१ +सतरा हजार सातशे~१७७०० +उणे छप्पन हजार~-५६००० +उणे त्र्याऐंशी हजार सातशे पाच~-८३७०५ +चार लाख अठरा हजार~४१८००० +सात लाख आठ~७००००८ +सत्तावन्न लाख अठ्ठ्यात्तर हजार तीनशे नव्व्याण्णव~५७७८३९९ +उणे नऊ लाख दहा~-९०००१० +आठ कोटी एकतीस लाख बारा हजार नऊशे त्रेचाळीस~८३११२९४३ +एक्याण्णव कोटी सत्तर लाख~९१७०००००० +तेहतीस कोटी तेहतीस लाख बासष्ट हजार~३३३३६२००० +तेहतीस कोटी तेहतीस लाख पंचवीस~३३३३०००२५ +उणे तीन कोटी तेहतीस लाख~-३३३००००० +उणे तेहतीस कोटी तेहतीस लाख~-३३३३००००० \ No newline at end of file diff --git a/itn/marathi/test/data/test_cases_date.txt b/itn/marathi/test/data/test_cases_date.txt new file mode 100644 index 00000000..c4e0455d --- /dev/null +++ b/itn/marathi/test/data/test_cases_date.txt @@ -0,0 +1,29 @@ +दहा जून दोन हजार~१० जून २००० +जानेवारी तेरा दोन हजार तेवीस~जानेवारी १३ २०२३ +जानेवारी दहा~जानेवारी १० +जानेवारी दोन हजार~जानेवारी २००० +पंधरा फेब्रुवारी अठराशे सतरा~१५ फेब्रुवारी १८१७ +इसवी सन बाराशे सत्याहत्तर~इ.स. १२७७ +इसवी सन पूर्व एकोणीसशे नव्व्याण्णव~इ.स.पू. १९९९ +ऑक्टोबर तेवीस सातशे आठ~ऑक्टोबर २३ ७०८ +फेब्रुवारी एकोणतीस~फेब्रुवारी २९ +इसवी सन सतराशे~इ.स. १७०० +मार्च बावीस एकोणीसशे अठ्ठ्याण्णव~मार्च २२ १९९८ +एप्रिल एक अकराशे अकरा~एप्रिल १ ११११ +बावीस मे एकोणीसशे एकसष्ट~२२ मे १९६१ +एकतीस जुलै एक हजार~३१ जुलै १००० +ऑगस्ट सात~ऑगस्ट ७ +सप्टेंबर सत्तावीस दोन हजार सहा~सप्टेंबर २७ २००६ +नऊ ऑक्टोबर दोन हजार दोन~९ ऑक्टोबर २००२ +नोव्हेंबर सोळाशे सत्तेचाळीस~नोव्हेंबर १६४७ +एकतीस डिसेंबर एकोणीसशे नव्व्याण्णव~३१ डिसेंबर १९९९ +मार्च~मार्च +एप्रिल अठराशे एक~एप्रिल १८०१ +इसवी सन पूर्व चारशे~इ.स.पू. ४०० +इसवी सन दोन हजार दहा~इ.स. २०१० +जुलै पंधरा~जुलै १५ +वीस ऑगस्ट एकोणीसशे~२० ऑगस्ट १९०० +चौदा सप्टेंबर~१४ सप्टेंबर +ऑक्टोबर चार~ऑक्टोबर ४ +तेरा नोव्हेंबर दोन हजार तेवीस~१३ नोव्हेंबर २०२३ +डिसेंबर पंचवीस~डिसेंबर २५ \ No newline at end of file diff --git a/itn/marathi/test/data/test_cases_decimal.txt b/itn/marathi/test/data/test_cases_decimal.txt new file mode 100644 index 00000000..2ec11fef --- /dev/null +++ b/itn/marathi/test/data/test_cases_decimal.txt @@ -0,0 +1,29 @@ +तेहतीस पूर्णांक तीन~३३.३ +चारशे पूर्णांक शून्य पाच~४००.०५ +आठशे पूर्णांक शून्य तीन एक~८००.०३१ +दोन हजार पूर्णांक आठ एक दोन पाच~२०००.८१२५ +चार लाख~४ लाख +उणे बाहत्तर लाख~-७२ लाख +तेहतीस लाख सत्तावीस हजार आठशे वीस पूर्णांक नऊ~३३२७८२०.९ +एकेचाळीस पूर्णांक तीन लाख~४१.३ लाख +उणे एक्याण्णव पूर्णांक चार कोटी~-९१.४ कोटी +दहा पूर्णांक पाच सहा~१०.५६ +सात पूर्णांक सात~७.७ +पंचाहत्तर कोटी~७५ कोटी +उणे बारा पूर्णांक एक पाच कोटी~-१२.१५ कोटी +बावन्न पूर्णांक एक शून्य चार आठ पाच~५२.१०४८५ +उणे साठ हजार पूर्णांक नऊ नऊ नऊ शून्य~-६००००.९९९० +शून्य पूर्णांक आठ सात सहा~०.८७६ +उणे शून्य पूर्णांक एक दोन तीन~-०.१२३ +शंभर पूर्णांक नऊ~१००.९ +नव्व्याण्णव पूर्णांक नऊ नऊ नऊ~९९.९९९ +सतरा लाख~१७ लाख +उणे दोन पूर्णांक आठ एक दोन~-२.८१२ +सदुसष्ट पूर्णांक चार कोटी~६७.४ कोटी +उणे ऐंशी पूर्णांक एक सहा नऊ आठ लाख~-८०.१६९८ लाख +शंभर पूर्णांक एक दोन तीन चार पाच~१००.१२३४५ +शंभर कोटी~१०० कोटी +उणे सत्तर पूर्णांक पाच~-७०.५ +आठ पूर्णांक सहा लाख~८.६ लाख +पाच हजार दोनशे अडतीस पूर्णांक चार सहा~५२३८.४६ +चाळीस कोटी~४० कोटी \ No newline at end of file diff --git a/itn/marathi/test/data/test_cases_time.txt b/itn/marathi/test/data/test_cases_time.txt new file mode 100644 index 00000000..ea31ca0a --- /dev/null +++ b/itn/marathi/test/data/test_cases_time.txt @@ -0,0 +1,29 @@ +साडे चार~०४:३० +साडे अकरा~११:३० +सव्वा आठ~०८:१५ +सव्वा बारा~१२:१५ +पावणे सहा~०५:४५ +पावणे दहा~०९:४५ +अकराला पाच मिनिटे~१०:५५ +आठला सात मिनिटे~०७:५३ +अकरा वाजून दोन मिनिटे~११:०२ +अकराला दोन मिनिट~१०:५८ +बारा वाजून पाच मिनिटे~१२:०५ +एक वाजून सात मिनिटे~०१:०७ +दीड~०१:३० +अडीच~०२:३० +नऊ वाजून वीस मिनिटे~०९:२० +सातला बारा मिनिटे~०६:४८ +पावणे अकरा~१०:४५ +दहा वाजून दहा मिनिटे~१०:१० +एकला एक मिनिट~१२:५९ +बारा वाजून पन्नास मिनिटे~१२:५० +चार वाजून पंचेचाळीस मिनिटे~०४:४५ +पावणे पाच~०४:४५ +पावणे एक~१२:४५ +सव्वा एक~०१:१५ +दहाला एक मिनिट~०९:५९ +सहा वाजून एक मिनिट~०६:०१ +नऊ वाजून तीस मिनिटे~०९:३० +दहाला पंधरा मिनिटे~०९:४५ +दोन वाजून अठ्ठेचाळीस मिनिटे~०२:४८ \ No newline at end of file diff --git a/itn/marathi/test/data/test_cases_word.txt b/itn/marathi/test/data/test_cases_word.txt new file mode 100644 index 00000000..3b7ef04e --- /dev/null +++ b/itn/marathi/test/data/test_cases_word.txt @@ -0,0 +1,11 @@ +~ +, मी~, मी +म ~म +मी~मी +आम्ही~आम्ही +तिथे~तिथे +थोडे~थोडे +आई~आई +वडील~वडील +झाड~झाड +पिणे~पिणे \ No newline at end of file diff --git a/itn/marathi/test/normalizer_test.py b/itn/marathi/test/normalizer_test.py new file mode 100644 index 00000000..e037995e --- /dev/null +++ b/itn/marathi/test/normalizer_test.py @@ -0,0 +1,164 @@ +# Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import re +from pathlib import Path + +import pytest + +from itn.marathi.inverse_normalizer import InverseNormalizer + +PACKAGE = Path(__file__).resolve().parent.parent + + +def original_cases(): + cases = [] + for path in sorted((Path(__file__).parent / "data").glob("test_cases_*.txt")): + with path.open(encoding="utf-8", newline="") as stream: + for index, line in enumerate(stream, 1): + if line.endswith("\n"): + line = line[:-1] + spoken, written = line.split("~") + cases.append(pytest.param(spoken, written, id="{}-{}".format(path.stem, index))) + return cases + + +CASES = original_cases() + + +@pytest.fixture(scope="module") +def normalizer(shared_pipeline): + return shared_pipeline(InverseNormalizer) + + +def test_original_fixture_count(): + # No fixture file ends with a newline; every last line still counts. + assert len(CASES) == 131 + assert CASES[32].values == ("उणे तेहतीस कोटी तेहतीस लाख", "-३३३३०००००") + assert CASES[120].values == ("", "") + assert CASES[122].values == ("म ", "म") + assert CASES[130].values == ("पिणे", "पिणे") + + +@pytest.mark.parametrize("spoken, written", CASES) +def test_original(normalizer, spoken, written): + assert normalizer.normalize(spoken) == written + + +@pytest.mark.parametrize("spoken, written", CASES) +def test_original_mapping(normalizer, spoken, written): + result = normalizer.normalize_with_mapping(spoken, include_identity=True) + assert result.output_text == written + for mapping in result.mappings: + assert spoken[mapping.input_start:mapping.input_end] == mapping.input_text + assert written[mapping.output_start:mapping.output_end] == mapping.output_text + + +def test_tag_then_verbalize_matches_normalize(normalizer): + for param in CASES: + spoken, written = param.values + assert normalizer.verbalize(normalizer.tag(spoken)) == written + + +@pytest.mark.parametrize("spoken, kind, members, written", [ + ("उणे सातशे बावीस", "cardinal", {"negative": "उणे", "integer": "सातशे बावीस"}, "-७२२"), + ("दोन लाख एकतीसशे", "cardinal", {"integer": "दोन लाख एकतीसशे"}, "२०३१००"), + ("एक अब्ज दोन", "cardinal", {"integer": "एक अब्ज दोन"}, "१" + "०" * 8 + "२"), + ("उणे बारा पूर्णांक एक पाच कोटी", "decimal", { + "negative": "उणे", "integer_part": "बारा", "fractional_part": "एक पाच", "quantity": "कोटी", + }, "-१२.१५ कोटी"), + ("सतरा लाख", "decimal", {"integer_part": "सतरा", "quantity": "लाख"}, "१७ लाख"), + ("पूर्णांक पाच", "decimal", {"fractional_part": "पाच"}, ".५"), + ("पंधरा फेब्रुवारी अठराशे सतरा", "date", {"day": "पंधरा", "month": "फेब्रुवारी", "year": "अठराशे सतरा"}, + "१५ फेब्रुवारी १८१७"), + ("जानेवारी दहा", "date", {"month": "जानेवारी", "day": "दहा"}, "जानेवारी १०"), + ("जानेवारी दोन हजार", "date", {"month": "जानेवारी", "year": "दोन हजार"}, "जानेवारी २०००"), + ("इसवी सन पूर्व चारशे", "date", {"era": "इसवी सन पूर्व", "year": "चारशे"}, "इ.स.पू. ४००"), + ("अकरा वाजून दोन मिनिटे", "time", {"hours": "अकरा", "minutes": "दोन"}, "११:०२"), + ("अकराला पाच मिनिटे", "time", {"hours": "अकराला", "minutes": "पाच"}, "१०:५५"), + ("पावणे एक", "time", {"relation": "पावणे", "hours": "एक"}, "१२:४५"), + ("साडे चार", "time", {"relation": "साडे", "hours": "चार"}, "०४:३०"), + ("अडीच", "time", {"hours": "अडीच"}, "०२:३०"), + ("आई", "word", {"value": "आई"}, "आई"), +]) +def test_raw_fields(normalizer, spoken, kind, members, written): + tagged = normalizer.tag(spoken) + parser = normalizer.token_parser() + parser.parse(tagged) + assert len(parser.tokens) == 1 + assert parser.tokens[0].name == kind + assert parser.tokens[0].members == members + assert normalizer.verbalize(tagged) == written + assert normalizer.normalize(spoken) == written + + +@pytest.mark.parametrize("spoken, written", [ + ("वीस।", "२० ।"), + ("नव्व्याण्णव अब्ज नव्व्याण्णव कोटी नव्व्याण्णव लाख नव्व्याण्णव हजार नऊशे नव्व्याण्णव", "९९९९९९९९९९९"), + ("शून्य शून्य", "० ०"), + ("तीन पूर्णांक पाच हजार", "३.५ हजार"), + ("बारा वाजून एक मिनिट", "१२:०१"), + ("सव्वा बारा वाजता", "१२:१५ वाजता"), + ("मार्च पाच", "मार्च ५"), +]) +def test_additional_semantics(normalizer, spoken, written): + assert normalizer.normalize(spoken) == written + + +@pytest.mark.parametrize("spoken", ["चार लाख", "साडे चार", "जानेवारी दहा", "तीन हजार एकशे"]) +def test_nbest_mapping_synchronization(normalizer, spoken): + outputs = normalizer.normalize(spoken, nbest=3) + mapped = normalizer.normalize_with_mapping(spoken, nbest=3, include_identity=True) + assert [result.output_text for result in mapped] == outputs + assert len(outputs) == len(set(outputs)) + assert outputs[0] == normalizer.normalize(spoken) + for result in mapped: + for mapping in result.mappings: + assert spoken[mapping.input_start:mapping.input_end] == mapping.input_text + assert result.output_text[mapping.output_start:mapping.output_end] == mapping.output_text + + +def test_escaped_word_round_trips(normalizer): + spoken = 'मी\\a"b' + parser = normalizer.token_parser() + parser.parse(normalizer.tag(spoken)) + assert [token.members for token in parser.tokens] == [{"value": spoken}] + assert normalizer.normalize(spoken) == spoken + + +def test_empty_input(normalizer): + assert normalizer.normalize("") == "" + assert normalizer.normalize("", nbest=3) == [""] + assert normalizer.normalize_with_mapping("").output_text == "" + + +@pytest.mark.parametrize("spoken", [" ", "\t", "\n", " \t\n ", "\u00a0"]) +def test_whitespace_only_input(normalizer, spoken): + tagged = normalizer.tag(spoken) + parser = normalizer.token_parser() + parser.parse(tagged) + assert len(parser.tokens) == 1 + assert parser.tokens[0].name == "word" + assert parser.tokens[0].members == {"value": spoken} + assert normalizer.verbalize(tagged) == "" + assert normalizer.normalize(spoken, nbest=3) == [""] + mapped = normalizer.normalize_with_mapping(spoken, nbest=3, include_identity=True) + assert [result.output_text for result in mapped] == [""] + assert all(mapping.output_text == "" for mapping in mapped[0].mappings) + + +def test_standalone_from_tn(): + tn_import = re.compile(r"^\s*(from|import)\s+tn\.marathi\b", re.MULTILINE) + for path in PACKAGE.rglob("*.py"): + assert not tn_import.search(path.read_text(encoding="utf-8")), path diff --git a/tn/marathi/__init__.py b/tn/marathi/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tn/marathi/data/date/eras.tsv b/tn/marathi/data/date/eras.tsv new file mode 100644 index 00000000..517186b0 --- /dev/null +++ b/tn/marathi/data/date/eras.tsv @@ -0,0 +1,2 @@ +इ.स.पू. इसवी सन पूर्व +इ.स. इसवी सन diff --git a/tn/marathi/data/date/months.tsv b/tn/marathi/data/date/months.tsv new file mode 100644 index 00000000..f39b90d2 --- /dev/null +++ b/tn/marathi/data/date/months.tsv @@ -0,0 +1,12 @@ +जानेवारी +फेब्रुवारी +मार्च +एप्रिल +मे +जून +जुलै +ऑगस्ट +सप्टेंबर +ऑक्टोबर +नोव्हेंबर +डिसेंबर \ No newline at end of file diff --git a/tn/marathi/data/numbers/digit.tsv b/tn/marathi/data/numbers/digit.tsv new file mode 100644 index 00000000..f737ade2 --- /dev/null +++ b/tn/marathi/data/numbers/digit.tsv @@ -0,0 +1,9 @@ +१ एक +२ दोन +३ तीन +४ चार +५ पाच +६ सहा +७ सात +८ आठ +९ नऊ \ No newline at end of file diff --git a/tn/marathi/data/numbers/tens.tsv b/tn/marathi/data/numbers/tens.tsv new file mode 100644 index 00000000..654244b3 --- /dev/null +++ b/tn/marathi/data/numbers/tens.tsv @@ -0,0 +1,90 @@ +१० दहा +११ अकरा +१२ बारा +१३ तेरा +१४ चौदा +१५ पंधरा +१६ सोळा +१७ सतरा +१८ अठरा +१९ एकोणीस +२० वीस +२१ एकवीस +२२ बावीस +२३ तेवीस +२४ चोवीस +२५ पंचवीस +२६ सव्वीस +२७ सत्तावीस +२८ अठ्ठावीस +२९ एकोणतीस +३० तीस +३१ एकतीस +३२ बत्तीस +३३ तेहतीस +३४ चौतीस +३५ पस्तीस +३६ छत्तीस +३७ सदतीस +३८ अडतीस +३९ एकोणचाळीस +४० चाळीस +४१ एकेचाळीस +४२ बेचाळीस +४३ त्रेचाळीस +४४ चव्वेचाळीस +४५ पंचेचाळीस +४६ सेहचाळीस +४७ सत्तेचाळीस +४८ अठ्ठेचाळीस +४९ एकोणपन्नास +५० पन्नास +५१ एकावन्न +५२ बावन्न +५३ त्रेपन्न +५४ चौपन्न +५५ पंचावन्न +५६ छप्पन +५७ सत्तावन्न +५८ अठ्ठावन्न +५९ एकोणसाठ +६० साठ +६१ एकसष्ट +६२ बासष्ट +६३ त्रेसष्ट +६४ चौसष्ट +६५ पासष्ट +६६ सहासष्ठ +६७ सदुसष्ट +६८ अडुसष्ठ +६९ एकोणसत्तर +७० सत्तर +७१ एकाहत्तर +७२ बाहत्तर +७३ त्र्याहत्तर +७४ चौऱ्याहत्तर +७५ पंचाहत्तर +७६ शहात्तर +७७ सत्याहत्तर +७८ अठ्ठ्यात्तर +७९ एकोणऐंशी +८० ऐंशी +८१ एक्याऐंशी +८२ ब्याऐंशी +८३ त्र्याऐंशी +८४ चौऱ्याऐंशी +८५ पंच्याऐंशी +८६ सह्यांशी +८७ सत्याऐंशी +८८ अठ्ठ्याऐंशी +८९ एकोणनव्वद +९० नव्वद +९१ एक्याण्णव +९२ ब्याण्णव +९३ त्र्याण्णव +९४ चौऱ्याण्णव +९५ पंच्याण्णव +९६ शहाण्णव +९७ सत्त्याण्णव +९८ अठ्ठ्याण्णव +९९ नव्व्याण्णव \ No newline at end of file diff --git a/tn/marathi/data/numbers/zero.tsv b/tn/marathi/data/numbers/zero.tsv new file mode 100644 index 00000000..0735899f --- /dev/null +++ b/tn/marathi/data/numbers/zero.tsv @@ -0,0 +1 @@ +० शून्य diff --git a/tn/marathi/normalizer.py b/tn/marathi/normalizer.py new file mode 100644 index 00000000..ed24c08c --- /dev/null +++ b/tn/marathi/normalizer.py @@ -0,0 +1,55 @@ +# Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from pynini.lib.pynutil import add_weight + +from tn.marathi.rules.cardinal import Cardinal +from tn.marathi.rules.date import Date +from tn.marathi.rules.decimal import Decimal +from tn.marathi.rules.punctuation import Punctuation +from tn.marathi.rules.time import Time +from tn.marathi.rules.word import Word +from tn.processor import Processor, RuleSpec + +# Every Marathi rule emits its fields in spoken order, so no token needs +# reordering before verbalization. +TOKEN_ORDERS = {} + + +class Normalizer(Processor): + + def __init__(self, cache_dir=None, overwrite_cache=False): + super().__init__(name="mr_normalizer", token_orders=TOKEN_ORDERS) + self.build_fst("mr_tn", cache_dir, overwrite_cache, {}) + + def build_tagger_and_verbalizer(self): + cardinal = Cardinal() + punctuation = Punctuation() + word = Word() + rules = ( + RuleSpec(Time(), 1.05), + RuleSpec(Date(cardinal), 1.09), + RuleSpec(Decimal(cardinal), 1.08), + RuleSpec(cardinal, 1.1), + RuleSpec(word, 100), + RuleSpec(punctuation), + ) + classify = self.tagger_union(rules) + # Punctuation splits off only when attached to another token. + punct = add_weight(punctuation.tagger, 1.1) + token = punct.star + classify + punct.star + self.tagger = self.DELETE_SPACE + token + (self.DELETE_EXTRA_SPACE + token).star + self.DELETE_SPACE + self.tagger |= word.add_tokens(word.tag_field("value", self.SPACE.plus)) + verbalize = self.verbalizer_union(rules) + self.verbalizer = verbalize + (self.INSERT_SPACE + verbalize).star diff --git a/tn/marathi/rules/__init__.py b/tn/marathi/rules/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tn/marathi/rules/cardinal.py b/tn/marathi/rules/cardinal.py new file mode 100644 index 00000000..91c6f149 --- /dev/null +++ b/tn/marathi/rules/cardinal.py @@ -0,0 +1,76 @@ +# Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import utf8 +from pynini.lib.pynutil import delete, insert + +from tn.marathi.rules.utils import MR_DIGIT, MR_NON_ZERO, canonical, number_words, to_devanagari, written_map +from tn.processor import Processor + +# Indian place values above the hundreds, each spanning two digits. +SCALES = ("अब्ज", "कोटी", "लाख", "हजार") +WIDTH = 3 + 2 * len(SCALES) +MINUS = "उणे" + + +class Cardinal(Processor): + """Written cardinals, e.g. ``-७२२`` or ``-722`` -> ``उणे सातशे बावीस``. + + The tagger keeps the written digits (and the ``-`` sign); the verbalizer + reads them with Indian grouping up to ``अब्ज`` (10^9). Comma-grouped + numbers (``१,००,०००``) are read as one number, and numbers with leading + zeros or more than eleven digits are read digit by digit. + """ + + def __init__(self): + super().__init__("cardinal", token_orders={}) + self.build_graphs() + negative = self.tag_field("negative", pynini.accep("-")) + insert(" ") + self.tagger = self.add_tokens(negative.ques + self.tag_field("integer", self.graph)) + sign = self.verbalize_field("negative", pynini.cross("-", MINUS + " ")) + delete(" ") + self.verbalizer = self.delete_tokens(sign.ques + self.verbalize_field("integer", self.graph)) + + def build_graphs(self): + words = number_words() + self.digit = written_map([(to_devanagari(n), words[n]) for n in range(1, 10)]) + self.zero = written_map([(to_devanagari(0), words[0])]) + # Two-digit groups with at least one non-zero digit, e.g. ``०७`` or ``४२``. + self.two = written_map([(to_devanagari(n, 2), words[n]) for n in range(1, 100)]) + self.tens = written_map([(to_devanagari(n), words[n]) for n in range(10, 100)]) + hundreds = self.digit + insert("शे") + (delete("००") | insert(" ") + self.two) | delete("०") + self.two + self.hundreds = hundreds.optimize() + + fixed = pynini.accep("") + for scale in SCALES: + fixed += insert(" ") + self.two + insert(" " + scale) | delete("००") + fixed += insert(" ") + self.hundreds | delete("०००") + pad = pynini.union(*[insert("०" * (WIDTH - k)) + MR_NON_ZERO + MR_DIGIT ** (k - 1) for k in range(1, WIDTH + 1)]) + lstrip = delete(" ") + utf8.VALID_UTF8_CHAR.star + number = pad @ fixed @ lstrip + # ``शंभर`` is a hundred on its own; inside larger numbers it is ``एकशे``. + number = pynini.difference(MR_DIGIT.plus, "१००") @ number | pynini.cross("१००", "शंभर") + self.number = (number | self.zero).optimize() + + single = (self.digit | self.zero).optimize() + self.digits = (single + (insert(" ") + single).star).optimize() + long_or_padded = MR_DIGIT ** (WIDTH + 1) + MR_DIGIT.star | "०" + MR_DIGIT.plus + digitwise = long_or_padded @ self.digits + + comma = delete(",") + indian = MR_DIGIT.ques + MR_NON_ZERO + (comma + MR_DIGIT ** 2).star + comma + MR_DIGIT ** 3 + western = pynini.closure(MR_DIGIT, 0, 2) + MR_NON_ZERO + (comma + MR_DIGIT ** 3).plus + grouped = (indian | western) @ self.number + self.graph = canonical(self.number | grouped | digitwise) + self.integer = canonical(self.number | grouped) diff --git a/tn/marathi/rules/date.py b/tn/marathi/rules/date.py new file mode 100644 index 00000000..8ff71701 --- /dev/null +++ b/tn/marathi/rules/date.py @@ -0,0 +1,70 @@ +# Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib.pynutil import add_weight, delete, insert + +from tn.marathi.rules.utils import MR_DIGIT, MR_NON_ZERO, canonical, load_pairs, number_words, to_devanagari, written_map +from tn.processor import Processor + +# A lone number after a month reads as the day unless it cannot be one. +YEAR_ONLY_WEIGHT = 0.01 + + +class Date(Processor): + """Written dates in either day-month or month-day order. + + ``१५ फेब्रुवारी १८१७`` -> ``पंधरा फेब्रुवारी अठराशे सतरा``, ``जानेवारी १३ + २०२३`` -> ``जानेवारी तेरा दोन हजार तेवीस`` and ``इ.स.पू. ४००`` -> + ``इसवी सन पूर्व चारशे``. The tagger keeps the written day, month, year + and era; years 1100-1999 are read as hundreds (``अठराशे``), other years + as cardinals. + """ + + def __init__(self, cardinal): + super().__init__("date", token_orders={}) + words = number_words() + days = set() + for day in range(1, 32): + for width in (1, 2): + days.add((to_devanagari(day, width), words[day])) + self.day = canonical(written_map(sorted(days))) + self.month = pynini.union(*[written for written, _ in load_pairs("date/months.tsv")]).optimize() + self.era = written_map(load_pairs("date/eras.tsv")) + rest = delete("००") | insert(" ") + cardinal.two + hundreds = ("१" + MR_NON_ZERO) @ cardinal.tens + insert("शे") + rest + plain = pynini.difference(MR_NON_ZERO + pynini.closure(MR_DIGIT, 0, 3), "१" + MR_NON_ZERO + MR_DIGIT ** 2) + self.year = canonical(hundreds | plain @ cardinal.number) + self.build_tagger() + self.build_verbalizer() + + def build_tagger(self): + sep = delete(self.SPACE.plus) + insert(" ") + day = self.tag_field("day", self.day) + month = self.tag_field("month", self.month) + year = self.tag_field("year", self.year) + era = self.tag_field("era", self.era) + mdy = month + sep + (day | add_weight(year, YEAR_ONLY_WEIGHT) | day + sep + year) + dmy = day + sep + month + (sep + year).ques + self.tagger = self.add_tokens(mdy | dmy | era + sep + year) + + def build_verbalizer(self): + sep = insert(" ") + day = self.verbalize_field("day", self.day, True) + month = self.verbalize_field("month", self.month, True) + year = self.verbalize_field("year", self.year, True) + mdy = self.verbalize_field("month", self.month) + sep + (day | year | day + sep + year) + dmy = self.verbalize_field("day", self.day) + sep + month + (sep + year).ques + era = self.verbalize_field("era", self.era) + sep + year + self.verbalizer = self.delete_tokens(mdy | dmy | era) diff --git a/tn/marathi/rules/decimal.py b/tn/marathi/rules/decimal.py new file mode 100644 index 00000000..ca3a4c9a --- /dev/null +++ b/tn/marathi/rules/decimal.py @@ -0,0 +1,59 @@ +# Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib.pynutil import delete, insert + +from tn.marathi.rules.cardinal import MINUS +from tn.marathi.rules.utils import MR_DIGIT, canonical +from tn.processor import Processor + +POINT = "पूर्णांक" +QUANTITIES = ("हजार", "लाख", "कोटी", "अब्ज") + + +class Decimal(Processor): + """Written decimals, e.g. ``-१२.१५ कोटी`` -> ``उणे बारा पूर्णांक एक पाच कोटी``. + + The tagger keeps the written sign, integer digits, fractional digits and + quantity word and drops the decimal point; the verbalizer reads the + integer as a cardinal and the fraction digit by digit. + """ + + def __init__(self, cardinal): + super().__init__("decimal", token_orders={}) + self.integer = cardinal.integer + self.fractional = canonical(MR_DIGIT.plus @ cardinal.digits) + self.quantity = pynini.union(*QUANTITIES).optimize() + self.build_tagger() + self.build_verbalizer() + + def build_tagger(self): + negative = self.tag_field("negative", pynini.accep("-")) + insert(" ") + integer = self.tag_field("integer_part", self.integer) + fractional = self.tag_field("fractional_part", self.fractional) + decimal = (integer + insert(" ")).ques + delete(".") + fractional + quantity = delete(self.SPACE.plus) + insert(" ") + self.tag_field("quantity", self.quantity) + graph = decimal + quantity.ques | integer + quantity + self.tagger = self.add_tokens(negative.ques + graph) + + def build_verbalizer(self): + sign = self.verbalize_field("negative", pynini.cross("-", MINUS + " ")) + delete(" ") + integer = self.verbalize_field("integer_part", self.integer) + fractional = self.verbalize_field("fractional_part", self.fractional, leading_space=True) + decimal = integer + insert(" " + POINT + " ") + fractional + decimal |= insert(POINT + " ") + self.verbalize_field("fractional_part", self.fractional) + quantity = insert(" ") + self.verbalize_field("quantity", self.quantity, leading_space=True) + graph = decimal + quantity.ques | integer + quantity + self.verbalizer = self.delete_tokens(sign.ques + graph) diff --git a/tn/marathi/rules/punctuation.py b/tn/marathi/rules/punctuation.py new file mode 100644 index 00000000..113e8a94 --- /dev/null +++ b/tn/marathi/rules/punctuation.py @@ -0,0 +1,28 @@ +# Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini + +from tn.processor import Processor + + +class Punctuation(Processor): + """ASCII and Devanagari punctuation split off a neighbouring token, e.g. ``(५)``.""" + + def __init__(self): + super().__init__("punctuation", token_orders={}) + graph = pynini.union(*[pynini.escape(mark) for mark in "!#$%&'()*+,-./:;<=>?@^_`{|}~।॥‘’“”"]).optimize() + self.tagger = self.add_tokens(self.tag_field("value", graph)) + self.verbalizer = self.delete_tokens(self.verbalize_field("value", graph)) diff --git a/tn/marathi/rules/time.py b/tn/marathi/rules/time.py new file mode 100644 index 00000000..84f3e035 --- /dev/null +++ b/tn/marathi/rules/time.py @@ -0,0 +1,86 @@ +# Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib.pynutil import add_weight, delete, insert + +from tn.marathi.rules.utils import canonical, number_words, to_devanagari, written_map +from tn.processor import Processor + +# Minutes read as "minutes to the next hour" (``अकराला पाच मिनिटे``). +TO_MINUTES = range(51, 60) +# Readings with a quarter word before the hour. +QUARTERS = {15: "सव्वा ", 30: "साडे ", 45: "पावणे "} +# Half past one and half past two have their own words. +HALVES = {1: "दीड", 2: "अडीच"} +# The plain "hours past" reading stays available for n-best output. +ALTERNATIVE_WEIGHT = 0.5 + + +class Time(Processor): + """Written ``H:MM`` clock times read on the 12-hour dial. + + ``११:०२`` -> ``अकरा वाजून दोन मिनिटे``, ``१०:५५`` -> ``अकराला पाच + मिनिटे``, ``०४:३०`` -> ``साडे चार``, ``१२:४५`` -> ``पावणे एक`` and + ``०१:३०`` -> ``दीड``. The tagger keeps the written hours and minutes and + drops the colon. + """ + + def __init__(self): + super().__init__("time", token_orders={}) + words = number_words() + + def hours_map(reading, hours=range(24)): + pairs = set() + for hour in hours: + for width in (1, 2): + pairs.add((to_devanagari(hour, width), reading(hour % 12 or 12))) + return canonical(written_map(sorted(pairs))) + + def minutes_map(reading, minutes): + return canonical(written_map([(to_devanagari(minute, 2), reading(minute)) for minute in minutes])) + + def unit(count): + return words[count] + (" मिनिट" if count == 1 else " मिनिटे") + + def next_hour(hour): + return words[hour % 12 + 1] + + hour = hours_map(lambda hour: words[hour]) + paths = [ + (hour, minutes_map(lambda minute: " वाजता", [0])), + (hours_map(lambda hour: next_hour(hour) + "ला"), minutes_map(lambda minute: " " + unit(60 - minute), TO_MINUTES)), + ] + named = [hour for hour in range(24) if (hour % 12 or 12) in HALVES] + for minute, word in QUARTERS.items(): + if minute == 45: + hours = hours_map(lambda hour, word=word: word + next_hour(hour)) + elif minute == 30: + hours = hours_map(lambda hour, word=word: word + words[hour], set(range(24)) - set(named)) + hours |= hours_map(lambda hour: HALVES[hour], named) + else: + hours = hours_map(lambda hour, word=word: word + words[hour]) + paths.append((hours, minutes_map(lambda minute: "", [minute]))) + special = set(QUARTERS) | set(TO_MINUTES) + past = minutes_map(lambda minute: " वाजून " + unit(minute), [m for m in range(1, 60) if m not in special]) + alternative = minutes_map(lambda minute: " वाजून " + unit(minute), sorted(special)) + paths.append((hour, past)) + paths.append((hour, add_weight(alternative, ALTERNATIVE_WEIGHT))) + + hours = pynini.union(*[hours for hours, _ in paths]).optimize() + minutes = pynini.union(*[minutes for _, minutes in paths]).optimize() + tagger = self.tag_field("hours", hours) + delete(":") + insert(" ") + self.tag_field("minutes", minutes) + self.tagger = self.add_tokens(tagger) + verbalizer = [self.verbalize_field("hours", h) + self.verbalize_field("minutes", m, True) for h, m in paths] + self.verbalizer = self.delete_tokens(pynini.union(*verbalizer)) diff --git a/tn/marathi/rules/utils.py b/tn/marathi/rules/utils.py new file mode 100644 index 00000000..e228d3da --- /dev/null +++ b/tn/marathi/rules/utils.py @@ -0,0 +1,76 @@ +# Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import utf8 + +from tn.utils import get_abs_path + +# Devanagari digits are the canonical written form; ASCII digits are accepted +# as input and mapped onto them inside the verbalizer graphs. +DEVANAGARI_DIGITS = "०१२३४५६७८९" +MR_DIGIT = pynini.union(*DEVANAGARI_DIGITS).optimize() +MR_NON_ZERO = pynini.union(*DEVANAGARI_DIGITS[1:]).optimize() +ALL_DIGIT = pynini.union(MR_DIGIT, *"0123456789").optimize() +TO_DEVANAGARI = pynini.union(MR_DIGIT, *[pynini.cross(str(index), char) for index, char in enumerate(DEVANAGARI_DIGITS)]) +TO_DEVANAGARI = TO_DEVANAGARI.optimize() + + +def to_devanagari(number, width=1): + """Formats an integer with Devanagari digits, zero-padded to ``width``.""" + + return "".join(DEVANAGARI_DIGITS[int(char)] for char in str(number).zfill(width)) + + +def canonical(graph): + """Lets a graph over Devanagari digits also read ASCII (or mixed) digits. + + Non-digit characters pass through unchanged. + """ + + char = pynini.union(TO_DEVANAGARI, pynini.difference(utf8.VALID_UTF8_CHAR, ALL_DIGIT)) + return (char.optimize().star @ graph).optimize() + + +def load_pairs(rel_path): + """Loads ``writtenspoken`` rows of a Marathi data table in file order. + + A single-column row is read the same way on both sides. + """ + + pairs = [] + with open(get_abs_path("marathi/data/" + rel_path), encoding="utf-8") as stream: + for line in stream: + line = line.rstrip("\r\n") + if line: + columns = line.split("\t") + pairs.append((columns[0], columns[-1])) + return pairs + + +def number_words(): + """Maps 0-99 to their Marathi spoken words.""" + + words = {} + for name in ("zero", "digit", "tens"): + for written, spoken in load_pairs("numbers/{}.tsv".format(name)): + words[int(written.translate(str.maketrans(DEVANAGARI_DIGITS, "0123456789")))] = spoken + return words + + +def written_map(pairs): + """Builds a ``written -> spoken`` transducer from ``(written, spoken)`` pairs.""" + + return pynini.union(*[pynini.cross(pynini.escape(written), pynini.escape(spoken)) + for written, spoken in pairs]).optimize() diff --git a/tn/marathi/rules/word.py b/tn/marathi/rules/word.py new file mode 100644 index 00000000..b2643cdc --- /dev/null +++ b/tn/marathi/rules/word.py @@ -0,0 +1,28 @@ +# Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from pynini.lib.pynutil import delete + +from tn.processor import Processor + + +class Word(Processor): + """Any other non-space run, kept verbatim.""" + + def __init__(self): + super().__init__("word", token_orders={}) + self.tagger = self.add_tokens(self.tag_field("value", self.NOT_SPACE.plus)) + graph = self.NOT_SPACE.plus | delete(self.SPACE.plus) + self.verbalizer = self.delete_tokens(self.verbalize_field("value", graph)) diff --git a/tn/marathi/test/__init__.py b/tn/marathi/test/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tn/marathi/test/normalizer_test.py b/tn/marathi/test/normalizer_test.py new file mode 100644 index 00000000..291637e7 --- /dev/null +++ b/tn/marathi/test/normalizer_test.py @@ -0,0 +1,197 @@ +# Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import re +from pathlib import Path + +import pytest + +from tn.marathi.normalizer import Normalizer + +PACKAGE = Path(__file__).resolve().parent.parent +# NeMo has no Marathi TN fixtures; the ITN fixtures are read in reverse. +ITN_DATA = PACKAGE.parent.parent / "itn" / "marathi" / "test" / "data" + +# Reversed ITN pairs that cannot hold for a deterministic TN, with reasons. +EXCLUDED = { + "५८०१": "ITN accepts the colloquial hundreds count; TN reads cardinals with हजार", + "७५१६": "ITN accepts the colloquial hundreds count; TN reads cardinals with हजार", + "-५२००": "ITN accepts the colloquial hundreds count; TN reads cardinals with हजार", + "१०:५८": "fixture uses singular मिनिट for two minutes, unlike आठला सात मिनिटे", + "०६:४८": "fixture reads :48 as minutes-to, but ०२:४८ as minutes-past", + "०९:३०": "fixture reads :30 as वाजून तीस मिनिटे, but ०४:३० as साडे", + "म": "spoken side keeps an untrimmed trailing space", +} + + +def reversed_cases(): + candidates = {} + for path in sorted(ITN_DATA.glob("test_cases_*.txt")): + with path.open(encoding="utf-8", newline="") as stream: + for line in stream: + if line.endswith("\n"): + line = line[:-1] + spoken, written = line.split("~") + candidates.setdefault(written, []).append(spoken) + return [pytest.param(written, spoken, id=written) for written, spoken in candidates.items()] + + +CASES = reversed_cases() +CHECKED = [param for param in CASES if param.values[0] not in EXCLUDED] + + +@pytest.fixture(scope="module") +def normalizer(shared_pipeline): + return shared_pipeline(Normalizer) + + +def test_reversed_fixture_count(): + assert sum(len(param.values[1]) for param in CASES) == 131 + assert len(CASES) == 128 + assert len(CHECKED) == 121 + assert set(EXCLUDED) <= {param.values[0] for param in CASES} + + +@pytest.mark.parametrize("written, spoken", CHECKED) +def test_reversed_original(normalizer, written, spoken): + assert normalizer.normalize(written) in spoken + + +@pytest.mark.parametrize("written, spoken", CHECKED) +def test_reversed_original_mapping(normalizer, written, spoken): + result = normalizer.normalize_with_mapping(written, include_identity=True) + assert result.output_text in spoken + for mapping in result.mappings: + assert written[mapping.input_start:mapping.input_end] == mapping.input_text + assert result.output_text[mapping.output_start:mapping.output_end] == mapping.output_text + + +def test_tag_then_verbalize_matches_normalize(normalizer): + for param in CASES: + written = param.values[0] + assert normalizer.verbalize(normalizer.tag(written)) == normalizer.normalize(written) + + +@pytest.mark.parametrize("written, spoken", [ + ("५८०१", "पाच हजार आठशे एक"), + ("१०:५८", "अकराला दोन मिनिटे"), + ("०६:४८", "सहा वाजून अठ्ठेचाळीस मिनिटे"), + ("०९:३०", "साडे नऊ"), + ("म", "म"), +]) +def test_excluded_pairs_read_consistently(normalizer, written, spoken): + assert written in EXCLUDED + assert normalizer.normalize(written) == spoken + + +@pytest.mark.parametrize("written, kind, members, spoken", [ + ("-722", "cardinal", {"negative": "-", "integer": "722"}, "उणे सातशे बावीस"), + ("१,२३,४५६", "cardinal", {"integer": "१,२३,४५६"}, "एक लाख तेवीस हजार चारशे छप्पन"), + ("007", "cardinal", {"integer": "007"}, "शून्य शून्य सात"), + ("-१२.१५ कोटी", "decimal", { + "negative": "-", "integer_part": "१२", "fractional_part": "१५", "quantity": "कोटी", + }, "उणे बारा पूर्णांक एक पाच कोटी"), + ("4 लाख", "decimal", {"integer_part": "4", "quantity": "लाख"}, "चार लाख"), + ("15 फेब्रुवारी 1817", "date", {"day": "15", "month": "फेब्रुवारी", "year": "1817"}, + "पंधरा फेब्रुवारी अठराशे सतरा"), + ("मार्च ०५", "date", {"month": "मार्च", "day": "०५"}, "मार्च पाच"), + ("इ.स.पू. ४००", "date", {"era": "इ.स.पू.", "year": "४००"}, "इसवी सन पूर्व चारशे"), + ("11:02", "time", {"hours": "11", "minutes": "02"}, "अकरा वाजून दोन मिनिटे"), + ("१३:३०", "time", {"hours": "१३", "minutes": "३०"}, "दीड"), + ("२३:५९", "time", {"hours": "२३", "minutes": "५९"}, "बाराला एक मिनिट"), + ("१२:००", "time", {"hours": "१२", "minutes": "००"}, "बारा वाजता"), + ("आई", "word", {"value": "आई"}, "आई"), +]) +def test_raw_fields(normalizer, written, kind, members, spoken): + tagged = normalizer.tag(written) + parser = normalizer.token_parser() + parser.parse(tagged) + assert len(parser.tokens) == 1 + assert parser.tokens[0].name == kind + assert parser.tokens[0].members == members + assert normalizer.verbalize(tagged) == spoken + assert normalizer.normalize(written) == spoken + + +@pytest.mark.parametrize("written, spoken", [ + ("(५)", "( पाच )"), + ("५।", "पाच ।"), + ("१००", "शंभर"), + ("११००", "एक हजार एकशे"), + ("1,000,000", "दहा लाख"), + ("९९९९९९९९९९९", "नव्व्याण्णव अब्ज नव्व्याण्णव कोटी नव्व्याण्णव लाख नव्व्याण्णव हजार नऊशे नव्व्याण्णव"), + ("१२३४५६७८९०१२", "एक दोन तीन चार पाच सहा सात आठ नऊ शून्य एक दोन"), + (".५", "पूर्णांक पाच"), + ("३.५ हजार", "तीन पूर्णांक पाच हजार"), + ("२२ मे 2023", "बावीस मे दोन हजार तेवीस"), + ("३२ जून", "बत्तीस जून"), + ("00:55", "एकला पाच मिनिटे"), + ("7:15", "सव्वा सात"), + ("०२:३०", "अडीच"), + ("१४:४५", "पावणे तीन"), +]) +def test_additional_semantics(normalizer, written, spoken): + assert normalizer.normalize(written) == spoken + + +def test_quarter_times_keep_plain_reading_as_alternative(normalizer): + assert normalizer.normalize("०४:३०", nbest=2) == ["साडे चार", "चार वाजून तीस मिनिटे"] + + +@pytest.mark.parametrize("written", ["४ लाख", "०४:३०", "मार्च ५ १९९८", "१,००,०००", "-१२.५"]) +def test_nbest_mapping_synchronization(normalizer, written): + outputs = normalizer.normalize(written, nbest=3) + mapped = normalizer.normalize_with_mapping(written, nbest=3, include_identity=True) + assert [result.output_text for result in mapped] == outputs + assert len(outputs) == len(set(outputs)) + assert outputs[0] == normalizer.normalize(written) + for result in mapped: + for mapping in result.mappings: + assert written[mapping.input_start:mapping.input_end] == mapping.input_text + assert result.output_text[mapping.output_start:mapping.output_end] == mapping.output_text + + +def test_escaped_word_round_trips(normalizer): + written = 'मी\\a"b' + parser = normalizer.token_parser() + parser.parse(normalizer.tag(written)) + assert [token.members for token in parser.tokens] == [{"value": written}] + assert normalizer.normalize(written) == written + + +def test_empty_input(normalizer): + assert normalizer.normalize("") == "" + assert normalizer.normalize("", nbest=3) == [""] + assert normalizer.normalize_with_mapping("").output_text == "" + + +@pytest.mark.parametrize("written", [" ", "\t", "\n", " \t\n ", "\u00a0"]) +def test_whitespace_only_input(normalizer, written): + tagged = normalizer.tag(written) + parser = normalizer.token_parser() + parser.parse(tagged) + assert len(parser.tokens) == 1 + assert parser.tokens[0].name == "word" + assert parser.tokens[0].members == {"value": written} + assert normalizer.verbalize(tagged) == "" + assert normalizer.normalize(written, nbest=3) == [""] + mapped = normalizer.normalize_with_mapping(written, nbest=3, include_identity=True) + assert [result.output_text for result in mapped] == [""] + assert all(mapping.output_text == "" for mapping in mapped[0].mappings) + + +def test_standalone_from_itn(): + itn_import = re.compile(r"^\s*(from|import)\s+itn\.marathi\b", re.MULTILINE) + for path in PACKAGE.rglob("*.py"): + assert not itn_import.search(path.read_text(encoding="utf-8")), path From dd64916a7b12fec7e1de6427771bf4cf2eb13e24 Mon Sep 17 00:00:00 2001 From: pengzhendong <275331498@qq.com> Date: Sat, 31 Oct 2026 12:43:38 +0800 Subject: [PATCH 2/2] feat: register Marathi pipelines in CLI, packaging and README --- README.md | 7 ++++--- pyproject.toml | 2 ++ tn/chinese/test/cache_test.py | 4 ++++ tn/cli.py | 2 ++ tn/marathi/normalizer.py | 8 +++----- tn/marathi/rules/punctuation.py | 6 ++++-- tn/marathi/test/normalizer_test.py | 26 +++++++++++++++++++++++++- tn/test/cli_test.py | 20 +++++++++----------- 8 files changed, 53 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 20fc20d2..721ef428 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ WeTextProcessing converts between written and spoken text using weighted finite-state transducers (WFSTs), built with [Pynini](https://www.openfst.org/twiki/bin/view/GRM/Pynini) and [OpenFst](https://www.openfst.org/twiki/bin/view/FST/WebHome). - **TN and ITN** — normalize numbers, dates, times, currencies, units, and other structured expressions. -- **Multilingual rules** — Arabic, Armenian, Chinese, English, French, German, Hindi, Japanese, Korean, Portuguese, Russian, Spanish, Swedish, and Vietnamese pipelines with language-specific grammars. +- **Multilingual rules** — language-specific grammars for every language in [Supported languages](#supported-languages). - **Explainable output** — inspect raw tagged fields, trace input/output spans, and request unique n-best results. - **Reusable graphs** — automatic, configuration-aware FST caching and a separate C++ deployment runtime. @@ -46,6 +46,7 @@ ITN converts spoken expressions into written forms, for example `十二` → `12 | Hindi | `hi` | Yes | Yes | `hindi` | | Japanese | `ja` | Yes | Yes | `japanese` | | Korean | `ko` | Yes | Yes | `korean` | +| Marathi | `mr` | Yes | Yes | `marathi` | | Portuguese | `pt` | Yes | Yes | `portuguese` | | Russian | `ru` | Yes | Yes | `russian` | | Spanish | `es` | Yes | Yes | `spanish` | @@ -54,7 +55,7 @@ ITN converts spoken expressions into written forms, for example `十二` → `12 Chinese is the CLI default. Rule coverage and configurable options vary by language and direction. -Arabic, Armenian, French, German, Hindi, Korean, Portuguese, Russian, Spanish, Swedish, and Vietnamese support are included in this source checkout. Install from source to use them before the next package release. +The published package includes Chinese, English, and Japanese. The other languages are included in this source checkout; install from source to use them before the next package release. ## Quick start @@ -66,7 +67,7 @@ Install the published package: pip install WeTextProcessing ``` -For the latest rules, including Arabic, Armenian, French, German, Hindi, Korean, Portuguese, Russian, Spanish, Swedish, and Vietnamese, install from source: +For the latest rules, including every language beyond Chinese, English, and Japanese, install from source: ```bash git clone https://github.com/wenet-e2e/WeTextProcessing.git diff --git a/pyproject.toml b/pyproject.toml index c2efca0a..3e6870e4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -78,6 +78,7 @@ tn = [ "spanish/data/*.tsv", "spanish/data/*/*.tsv", "hindi/data/*/*.tsv", + "marathi/data/*/*.tsv", ] itn = [ "armenian/data/*.tsv", @@ -102,4 +103,5 @@ itn = [ "spanish/data/*.tsv", "spanish/data/*/*.tsv", "hindi/data/*/*.tsv", + "marathi/data/*/*.tsv", ] diff --git a/tn/chinese/test/cache_test.py b/tn/chinese/test/cache_test.py index 2d7ded86..0136c29a 100644 --- a/tn/chinese/test/cache_test.py +++ b/tn/chinese/test/cache_test.py @@ -12,6 +12,7 @@ from itn.hindi.inverse_normalizer import InverseNormalizer as HiInverseNormalizer from itn.japanese.inverse_normalizer import InverseNormalizer as JaInverseNormalizer from itn.korean.inverse_normalizer import InverseNormalizer as KoInverseNormalizer +from itn.marathi.inverse_normalizer import InverseNormalizer as MrInverseNormalizer from itn.portuguese.inverse_normalizer import InverseNormalizer as PtInverseNormalizer from itn.russian.inverse_normalizer import InverseNormalizer as RuInverseNormalizer from itn.spanish.inverse_normalizer import InverseNormalizer as EsInverseNormalizer @@ -26,6 +27,7 @@ from tn.hindi.normalizer import Normalizer as HiNormalizer from tn.japanese.normalizer import Normalizer as JaNormalizer from tn.korean.normalizer import Normalizer as KoNormalizer +from tn.marathi.normalizer import Normalizer as MrNormalizer from tn.portuguese.normalizer import Normalizer as PtNormalizer from tn.russian.normalizer import Normalizer as RuNormalizer from tn.spanish.normalizer import Normalizer as EsNormalizer @@ -96,6 +98,7 @@ def _build_small_graphs(processor): (EsNormalizer, "es_tn", "tn", {}, {"deterministic": True}), (EsNormalizer, "es_tn", "tn", {"deterministic": False}, {"deterministic": False}), (HiNormalizer, "hi_tn", "tn", {}, {}), + (MrNormalizer, "mr_tn", "tn", {}, {}), ( ZhInverseNormalizer, "zh_itn", @@ -143,6 +146,7 @@ def _build_small_graphs(processor): (EsInverseNormalizer, "es_itn", "itn", {}, {"cased": False}), (EsInverseNormalizer, "es_itn", "itn", {"cased": True}, {"cased": True}), (HiInverseNormalizer, "hi_itn", "itn", {}, {}), + (MrInverseNormalizer, "mr_itn", "itn", {}, {}), ], ) def test_all_pipeline_configs_are_part_of_manifest( diff --git a/tn/cli.py b/tn/cli.py index 6f7be9a4..f854e8e5 100644 --- a/tn/cli.py +++ b/tn/cli.py @@ -69,6 +69,7 @@ class PipelineSpec: (BooleanOption("deterministic", True, "use deterministic Spanish readings"),), ), "hi": PipelineSpec("tn.hindi.normalizer", "Normalizer"), + "mr": PipelineSpec("tn.marathi.normalizer", "Normalizer"), "ja": PipelineSpec( "tn.japanese.normalizer", "Normalizer", @@ -107,6 +108,7 @@ class PipelineSpec: (BooleanOption("cased", False, 'also accept capitalized spoken words, e.g. "Doce Dólares"'),), ), "hi": PipelineSpec("itn.hindi.inverse_normalizer", "InverseNormalizer"), + "mr": PipelineSpec("itn.marathi.inverse_normalizer", "InverseNormalizer"), "ja": PipelineSpec( "itn.japanese.inverse_normalizer", "InverseNormalizer", diff --git a/tn/marathi/normalizer.py b/tn/marathi/normalizer.py index ed24c08c..4c6c7f48 100644 --- a/tn/marathi/normalizer.py +++ b/tn/marathi/normalizer.py @@ -42,14 +42,12 @@ def build_tagger_and_verbalizer(self): RuleSpec(Date(cardinal), 1.09), RuleSpec(Decimal(cardinal), 1.08), RuleSpec(cardinal, 1.1), - RuleSpec(word, 100), - RuleSpec(punctuation), + RuleSpec(word, 100, verbalize=False), + RuleSpec(punctuation, verbalize=False), ) classify = self.tagger_union(rules) - # Punctuation splits off only when attached to another token. punct = add_weight(punctuation.tagger, 1.1) token = punct.star + classify + punct.star self.tagger = self.DELETE_SPACE + token + (self.DELETE_EXTRA_SPACE + token).star + self.DELETE_SPACE self.tagger |= word.add_tokens(word.tag_field("value", self.SPACE.plus)) - verbalize = self.verbalizer_union(rules) - self.verbalizer = verbalize + (self.INSERT_SPACE + verbalize).star + self.verbalizer = self.join_tokens(self.verbalizer_union(rules), (word.verbalizer, punctuation.verbalizer)) diff --git a/tn/marathi/rules/punctuation.py b/tn/marathi/rules/punctuation.py index 113e8a94..0d372c9d 100644 --- a/tn/marathi/rules/punctuation.py +++ b/tn/marathi/rules/punctuation.py @@ -14,6 +14,7 @@ # limitations under the License. import pynini +from pynini.lib.pynutil import add_weight from tn.processor import Processor @@ -24,5 +25,6 @@ class Punctuation(Processor): def __init__(self): super().__init__("punctuation", token_orders={}) graph = pynini.union(*[pynini.escape(mark) for mark in "!#$%&'()*+,-./:;<=>?@^_`{|}~।॥‘’“”"]).optimize() - self.tagger = self.add_tokens(self.tag_field("value", graph)) - self.verbalizer = self.delete_tokens(self.verbalize_field("value", graph)) + space = add_weight(self.SPACE, -0.0001).star + self.tagger = self.add_tokens(self.tag_field("value", space + graph + space)) + self.verbalizer = self.delete_tokens(self.verbalize_field("value", self.SPACE.star + graph + self.SPACE.star)) diff --git a/tn/marathi/test/normalizer_test.py b/tn/marathi/test/normalizer_test.py index 291637e7..cdb650c0 100644 --- a/tn/marathi/test/normalizer_test.py +++ b/tn/marathi/test/normalizer_test.py @@ -125,7 +125,7 @@ def test_raw_fields(normalizer, written, kind, members, spoken): @pytest.mark.parametrize("written, spoken", [ - ("(५)", "( पाच )"), + ("(५)", "(पाच)"), ("५।", "पाच ।"), ("१००", "शंभर"), ("११००", "एक हजार एकशे"), @@ -145,6 +145,30 @@ def test_additional_semantics(normalizer, written, spoken): assert normalizer.normalize(written) == spoken +@pytest.mark.parametrize("written, spoken, punctuation", [ + ("(5)", "(पाच)", ["(", ")"]), + ("5,", "पाच,", [","]), + ("5 !", "पाच !", [" !"]), + ("( 5 )", "( पाच )", ["( ", " )"]), +]) +def test_cardinal_punctuation_spacing(normalizer, written, spoken, punctuation): + assert normalizer.normalize(written) == spoken + tagged = normalizer.tag(written) + parser = normalizer.token_parser() + parser.parse(tagged) + assert [token.members for token in parser.tokens if token.name == "cardinal"] == [{"integer": "5"}] + assert [token.members for token in parser.tokens if token.name == "punctuation"] == [ + {"value": value} for value in punctuation + ] + assert normalizer.verbalize(tagged) == spoken + mapped = normalizer.normalize_with_mapping(written, include_identity=True) + assert mapped.output_text == spoken + assert [mapping.input_text for mapping in mapped.mappings if mapping.token_type == "punctuation"] == punctuation + for mapping in mapped.mappings: + assert written[mapping.input_start:mapping.input_end] == mapping.input_text + assert spoken[mapping.output_start:mapping.output_end] == mapping.output_text + + def test_quarter_times_keep_plain_reading_as_alternative(normalizer): assert normalizer.normalize("०४:३०", nbest=2) == ["साडे चार", "चार वाजून तीस मिनिटे"] diff --git a/tn/test/cli_test.py b/tn/test/cli_test.py index 9acb99d5..18554f3e 100644 --- a/tn/test/cli_test.py +++ b/tn/test/cli_test.py @@ -60,6 +60,7 @@ def verbalize(self, tagged): ("tn", "sv", "tn.swedish.normalizer", "Normalizer", ["--no-deterministic"], {"deterministic": False}), ("tn", "es", "tn.spanish.normalizer", "Normalizer", ["--no-deterministic"], {"deterministic": False}), ("tn", "hi", "tn.hindi.normalizer", "Normalizer", [], {}), + ("tn", "mr", "tn.marathi.normalizer", "Normalizer", [], {}), ( "itn", "zh", @@ -94,6 +95,7 @@ def verbalize(self, tagged): ("itn", "sv", "itn.swedish.inverse_normalizer", "InverseNormalizer", [], {}), ("itn", "es", "itn.spanish.inverse_normalizer", "InverseNormalizer", ["--cased"], {"cased": True}), ("itn", "hi", "itn.hindi.inverse_normalizer", "InverseNormalizer", [], {}), + ("itn", "mr", "itn.marathi.inverse_normalizer", "InverseNormalizer", [], {}), ], ) def test_create_processor_uses_language_capabilities( @@ -163,20 +165,14 @@ def import_module(name): parse_args(other_direction, ["--language", "xx"]) -@pytest.mark.parametrize("direction", ["tn", "itn"]) -@pytest.mark.parametrize("language", [ - "zh", "en", "ja", "ko", "de", "hy", "ar", "fr", "pt", "vi", "ru", "sv", "es", "hi", +@pytest.mark.parametrize("direction, language", [ + (direction, language) for direction, languages in cli.PIPELINES.items() for language in languages ]) def test_cli_defaults_and_forwarding_match_processor_signature(monkeypatch, direction, language): args = parse_args(direction, ["--language", language]) - package = { - "zh": "chinese", "en": "english", "ja": "japanese", "ko": "korean", "de": "german", "hy": "armenian", - "ar": "arabic", "fr": "french", "pt": "portuguese", "vi": "vietnamese", "ru": "russian", - "sv": "swedish", "es": "spanish", "hi": "hindi", - }[language] - module_suffix, class_name = ("normalizer", "Normalizer") if direction == "tn" else ( - "inverse_normalizer", "InverseNormalizer") - module = __import__("{}.{}.{}".format(direction, package, module_suffix), fromlist=[class_name]) + spec = cli.PIPELINES[direction][language] + module = __import__(spec.module, fromlist=[spec.class_name]) + class_name = spec.class_name parameters = inspect.signature(getattr(module, class_name)).parameters expected = {name: parameter.default for name, parameter in parameters.items()} assert {name: getattr(args, name) for name in expected} == expected @@ -223,6 +219,8 @@ def __init__(self, **kwargs): ("itn", "es", "--cased", "--enable-million"), ("tn", "hi", "--cache-dir", "--deterministic"), ("itn", "hi", "--cache-dir", "--enable-million"), + ("tn", "mr", "--cache-dir", "--deterministic"), + ("itn", "mr", "--cache-dir", "--enable-million"), ], ) def test_help_only_exposes_selected_pipeline_options(direction, language, present, absent):