diff --git a/README.md b/README.md index 966b76a4..80b05d2f 100644 --- a/README.md +++ b/README.md @@ -47,6 +47,7 @@ ITN converts spoken expressions into written forms, for example `十二` → `12 | Japanese | `ja` | Yes | Yes | `japanese` | | Kinyarwanda | `rw` | Yes | Yes | `kinyarwanda` | | Korean | `ko` | Yes | Yes | `korean` | +| Marathi | `mr` | Yes | Yes | `marathi` | | Portuguese | `pt` | Yes | Yes | `portuguese` | | Russian | `ru` | Yes | Yes | `russian` | | Spanish | `es` | Yes | Yes | `spanish` | diff --git a/itn/marathi/__init__.py b/itn/marathi/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/itn/marathi/data/date/eras.tsv b/itn/marathi/data/date/eras.tsv new file mode 100644 index 00000000..517186b0 --- /dev/null +++ b/itn/marathi/data/date/eras.tsv @@ -0,0 +1,2 @@ +इ.स.पू. इसवी सन पूर्व +इ.स. इसवी सन diff --git a/itn/marathi/data/date/months.tsv b/itn/marathi/data/date/months.tsv new file mode 100644 index 00000000..f39b90d2 --- /dev/null +++ b/itn/marathi/data/date/months.tsv @@ -0,0 +1,12 @@ +जानेवारी +फेब्रुवारी +मार्च +एप्रिल +मे +जून +जुलै +ऑगस्ट +सप्टेंबर +ऑक्टोबर +नोव्हेंबर +डिसेंबर \ No newline at end of file diff --git a/itn/marathi/data/numbers/digit.tsv b/itn/marathi/data/numbers/digit.tsv new file mode 100644 index 00000000..f737ade2 --- /dev/null +++ b/itn/marathi/data/numbers/digit.tsv @@ -0,0 +1,9 @@ +१ एक +२ दोन +३ तीन +४ चार +५ पाच +६ सहा +७ सात +८ आठ +९ नऊ \ No newline at end of file diff --git a/itn/marathi/data/numbers/tens.tsv b/itn/marathi/data/numbers/tens.tsv new file mode 100644 index 00000000..654244b3 --- /dev/null +++ b/itn/marathi/data/numbers/tens.tsv @@ -0,0 +1,90 @@ +१० दहा +११ अकरा +१२ बारा +१३ तेरा +१४ चौदा +१५ पंधरा +१६ सोळा +१७ सतरा +१८ अठरा +१९ एकोणीस +२० वीस +२१ एकवीस +२२ बावीस +२३ तेवीस +२४ चोवीस +२५ पंचवीस +२६ सव्वीस +२७ सत्तावीस +२८ अठ्ठावीस +२९ एकोणतीस +३० तीस +३१ एकतीस +३२ बत्तीस +३३ तेहतीस +३४ चौतीस +३५ पस्तीस +३६ छत्तीस +३७ सदतीस +३८ अडतीस +३९ एकोणचाळीस +४० चाळीस +४१ एकेचाळीस +४२ बेचाळीस +४३ त्रेचाळीस +४४ चव्वेचाळीस +४५ पंचेचाळीस +४६ सेहचाळीस +४७ सत्तेचाळीस +४८ अठ्ठेचाळीस +४९ एकोणपन्नास +५० पन्नास +५१ एकावन्न +५२ बावन्न +५३ त्रेपन्न +५४ चौपन्न +५५ पंचावन्न +५६ छप्पन +५७ सत्तावन्न +५८ अठ्ठावन्न +५९ एकोणसाठ +६० साठ +६१ एकसष्ट +६२ बासष्ट +६३ त्रेसष्ट +६४ चौसष्ट +६५ पासष्ट +६६ सहासष्ठ +६७ सदुसष्ट +६८ अडुसष्ठ +६९ एकोणसत्तर +७० सत्तर +७१ एकाहत्तर +७२ बाहत्तर +७३ त्र्याहत्तर +७४ चौऱ्याहत्तर +७५ पंचाहत्तर +७६ शहात्तर +७७ सत्याहत्तर +७८ अठ्ठ्यात्तर +७९ एकोणऐंशी +८० ऐंशी +८१ एक्याऐंशी +८२ ब्याऐंशी +८३ त्र्याऐंशी +८४ चौऱ्याऐंशी +८५ पंच्याऐंशी +८६ सह्यांशी +८७ सत्याऐंशी +८८ अठ्ठ्याऐंशी +८९ एकोणनव्वद +९० नव्वद +९१ एक्याण्णव +९२ ब्याण्णव +९३ त्र्याण्णव +९४ चौऱ्याण्णव +९५ पंच्याण्णव +९६ शहाण्णव +९७ सत्त्याण्णव +९८ अठ्ठ्याण्णव +९९ नव्व्याण्णव \ No newline at end of file diff --git a/itn/marathi/data/numbers/zero.tsv b/itn/marathi/data/numbers/zero.tsv new file mode 100644 index 00000000..0735899f --- /dev/null +++ b/itn/marathi/data/numbers/zero.tsv @@ -0,0 +1 @@ +० शून्य diff --git a/itn/marathi/inverse_normalizer.py b/itn/marathi/inverse_normalizer.py new file mode 100644 index 00000000..5ac0c394 --- /dev/null +++ b/itn/marathi/inverse_normalizer.py @@ -0,0 +1,56 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from pynini.lib.pynutil import add_weight + +from itn.marathi.rules.cardinal import Cardinal +from itn.marathi.rules.date import Date +from itn.marathi.rules.decimal import Decimal +from itn.marathi.rules.punctuation import Punctuation +from itn.marathi.rules.time import Time +from itn.marathi.rules.word import Word +from tn.processor import Processor, RuleSpec + +# Every Marathi rule emits its fields in written order, so no token needs +# reordering before verbalization. +TOKEN_ORDERS = {} + + +class InverseNormalizer(Processor): + + def __init__(self, cache_dir=None, overwrite_cache=False): + super().__init__(name="mr_inverse_normalizer", ordertype="itn", token_orders=TOKEN_ORDERS) + self.build_fst("mr_itn", cache_dir, overwrite_cache, {}) + + def build_tagger_and_verbalizer(self): + cardinal = Cardinal() + punctuation = Punctuation() + word = Word() + rules = ( + RuleSpec(Time(), 1.05), + RuleSpec(Date(cardinal), 1.09), + RuleSpec(Decimal(cardinal), 1.08), + RuleSpec(cardinal, 1.1), + RuleSpec(word, 100), + RuleSpec(punctuation), + ) + classify = self.tagger_union(rules) + # Punctuation splits off only when attached to another token. + punct = add_weight(punctuation.tagger, 1.1) + token = punct.star + classify + punct.star + self.tagger = self.DELETE_SPACE + token + (self.DELETE_EXTRA_SPACE + token).star + self.DELETE_SPACE + self.tagger |= word.add_tokens(word.tag_field("value", self.SPACE.plus)) + verbalize = self.verbalizer_union(rules) + self.verbalizer = verbalize + (self.INSERT_SPACE + verbalize).star diff --git a/itn/marathi/rules/__init__.py b/itn/marathi/rules/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/itn/marathi/rules/cardinal.py b/itn/marathi/rules/cardinal.py new file mode 100644 index 00000000..27796971 --- /dev/null +++ b/itn/marathi/rules/cardinal.py @@ -0,0 +1,76 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib.pynutil import add_weight, delete, insert + +from itn.marathi.rules.utils import DIGIT, NON_ZERO, load_pairs, spoken_map +from tn.processor import Processor + +MINUS = "उणे" +# Indian place values above the hundreds, each spanning two digits. +SCALES = ("अब्ज", "कोटी", "लाख", "हजार") +# Skipping a place value costs a little, so ``चार लाख`` prefers the decimal +# quantity reading ``४ लाख`` over the cardinal ``४०००००``. +SKIP_WEIGHT = 0.1 + + +class Cardinal(Processor): + """Spoken cardinals, e.g. ``उणे सातशे बावीस`` -> ``-७२२``. + + The tagger keeps the spoken words (and the ``उणे`` sign word); the + verbalizer assembles Devanagari digits in Indian grouping (lakh, crore, + abja) and also reads ``एकतीसशे`` style hundreds counts. + """ + + def __init__(self): + super().__init__(name="cardinal", ordertype="itn", token_orders={}) + self.build_graphs() + negative = self.tag_field("negative", pynini.accep(MINUS)) + delete(self.SPACE.plus) + insert(" ") + self.tagger = self.add_tokens(negative.ques + self.tag_field("integer", self.graph)) + sign = self.verbalize_field("negative", pynini.cross(MINUS, "-")) + delete(" ") + self.verbalizer = self.delete_tokens(sign.ques + self.verbalize_field("integer", self.graph)) + + def build_graphs(self): + ds = delete(self.SPACE).star + digit = spoken_map(load_pairs("numbers/digit.tsv")) + tens = spoken_map(load_pairs("numbers/tens.tsv")) + zero = spoken_map(load_pairs("numbers/zero.tsv")) + self.digit = digit + self.zero = zero + + two_nonzero = (tens | insert("०") + digit).optimize() + two = (two_nonzero | insert("००")).optimize() + hundreds = (digit + delete("शे") + ds | insert("०")) + two + # ``एकतीसशे`` (thirty-one hundred) fills the thousands and hundreds slots. + hundreds_count = insert("०") + tens + delete("शे") + ds + two + thousands_and_hundreds = [] + higher = pynini.accep("") + for scale in SCALES: + slot = two_nonzero + ds + delete(scale) + ds | add_weight(insert("००"), SKIP_WEIGHT) + if scale == "हजार": + thousands_and_hundreds = slot + hundreds | hundreds_count + else: + higher += slot + nonzero = DIGIT.star + NON_ZERO + DIGIT.star + strip_zeros = delete("०").star + NON_ZERO + DIGIT.star + + trimmed = self.NOT_SPACE | self.NOT_SPACE + self.VSIGMA + self.NOT_SPACE + trimmed = trimmed.optimize() + hundred = pynini.cross("शंभर", "१००") + graph = (higher + thousands_and_hundreds) @ strip_zeros | hundred | zero + self.graph = (trimmed @ graph).optimize() + # 1-999 (including ``शंभर``) counts a lakh/crore quantity: ``सतरा लाख``. + self.hundreds = (trimmed @ ((hundreds @ nonzero) @ strip_zeros | hundred)).optimize() diff --git a/itn/marathi/rules/date.py b/itn/marathi/rules/date.py new file mode 100644 index 00000000..cbea3a49 --- /dev/null +++ b/itn/marathi/rules/date.py @@ -0,0 +1,63 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib.pynutil import add_weight, delete, insert + +from itn.marathi.rules.utils import load_pairs, number_words, spoken_map, to_devanagari +from tn.processor import Processor + +# A lone number after a month reads as the day unless it cannot be one. +YEAR_ONLY_WEIGHT = 0.01 + + +class Date(Processor): + """Spoken dates in either day-month or month-day order. + + ``पंधरा फेब्रुवारी अठराशे सतरा`` -> ``१५ फेब्रुवारी १८१७``, ``जानेवारी + तेरा दोन हजार तेवीस`` -> ``जानेवारी १३ २०२३`` and ``इसवी सन पूर्व + चारशे`` -> ``इ.स.पू. ४००``. The tagger keeps the spoken day, month, + year and era in spoken order. + """ + + def __init__(self, cardinal): + super().__init__(name="date", ordertype="itn", token_orders={}) + words = number_words() + self.day = spoken_map([(to_devanagari(day), words[day]) for day in range(1, 32)]) + self.month = pynini.union(*[spoken for _, spoken in load_pairs("date/months.tsv")]).optimize() + self.year = cardinal.graph + self.era = spoken_map(load_pairs("date/eras.tsv")) + self.build_tagger() + self.build_verbalizer() + + def build_tagger(self): + sep = delete(self.SPACE.plus) + insert(" ") + day = self.tag_field("day", self.day) + month = self.tag_field("month", self.month) + year = self.tag_field("year", self.year) + era = self.tag_field("era", self.era) + mdy = month + sep + (day | add_weight(year, YEAR_ONLY_WEIGHT) | day + sep + year) + dmy = day + sep + month + (sep + year).ques + self.tagger = self.add_tokens(mdy | dmy | era + sep + year) + + def build_verbalizer(self): + sep = insert(" ") + day = self.verbalize_field("day", self.day, True) + month = self.verbalize_field("month", self.month, True) + year = self.verbalize_field("year", self.year, True) + mdy = self.verbalize_field("month", self.month) + sep + (day | year | day + sep + year) + dmy = self.verbalize_field("day", self.day) + sep + month + (sep + year).ques + era = self.verbalize_field("era", self.era) + sep + year + self.verbalizer = self.delete_tokens(mdy | dmy | era) diff --git a/itn/marathi/rules/decimal.py b/itn/marathi/rules/decimal.py new file mode 100644 index 00000000..518d8ce4 --- /dev/null +++ b/itn/marathi/rules/decimal.py @@ -0,0 +1,62 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib.pynutil import delete, insert + +from itn.marathi.rules.cardinal import MINUS +from tn.processor import Processor + + +class Decimal(Processor): + """Spoken decimals, e.g. ``उणे बारा पूर्णांक एक पाच कोटी`` -> ``-१२.१५ कोटी``. + + The tagger keeps the spoken sign, integer, fractional digits and quantity + and drops the spoken decimal point ``पूर्णांक``. + """ + + def __init__(self, cardinal): + super().__init__(name="decimal", ordertype="itn", token_orders={}) + digit = (cardinal.digit | cardinal.zero).optimize() + self.fractional = ((digit + delete(self.SPACE.plus)).star + digit).optimize() + self.integer = cardinal.graph + self.quantity_integer = cardinal.hundreds + self.quantity = pynini.union("लाख", "कोटी", "अब्ज").optimize() + self.decimal_quantity = (self.quantity | "हजार").optimize() + self.build_tagger() + self.build_verbalizer() + + def build_tagger(self): + spaces = delete(self.SPACE.plus) + negative = self.tag_field("negative", pynini.accep(MINUS)) + spaces + insert(" ") + point = delete("पूर्णांक") + spaces + integer = self.tag_field("integer_part", self.integer) + spaces + point + insert(" ") + fractional = self.tag_field("fractional_part", self.fractional) + decimal = integer + fractional | point + fractional + quantity = self.tag_field("integer_part", self.quantity_integer) + spaces + insert(" ") + quantity += self.tag_field("quantity", self.quantity) + quantity |= decimal + spaces + insert(" ") + self.tag_field("quantity", self.decimal_quantity) + self.tagger = self.add_tokens(negative.ques + (decimal | quantity)) + + def build_verbalizer(self): + sign = self.verbalize_field("negative", pynini.cross(MINUS, "-")) + delete(" ") + integer = self.verbalize_field("integer_part", self.integer) + fractional = insert(".") + self.verbalize_field("fractional_part", self.fractional, leading_space=True) + decimal = integer + fractional + decimal |= insert(".") + self.verbalize_field("fractional_part", self.fractional) + quantity = self.verbalize_field("integer_part", self.quantity_integer) + insert(" ") + quantity += self.verbalize_field("quantity", self.quantity, leading_space=True) + quantity |= decimal + insert(" ") + self.verbalize_field("quantity", self.decimal_quantity, leading_space=True) + self.verbalizer = self.delete_tokens(sign.ques + (decimal | quantity)) diff --git a/itn/marathi/rules/punctuation.py b/itn/marathi/rules/punctuation.py new file mode 100644 index 00000000..708f3530 --- /dev/null +++ b/itn/marathi/rules/punctuation.py @@ -0,0 +1,29 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini + +from tn.processor import Processor + + +class Punctuation(Processor): + """ASCII and Devanagari punctuation split off a neighbouring token, e.g. ``चार।``.""" + + def __init__(self): + super().__init__(name="punctuation", ordertype="itn", token_orders={}) + marks = "!#$%&'()*+,-./:;<=>?@^_`{|}~।॥‘’“”" + graph = pynini.union(*[pynini.escape(mark) for mark in marks]).optimize() + self.tagger = self.add_tokens(self.tag_field("value", graph)) + self.verbalizer = self.delete_tokens(self.verbalize_field("value", graph)) diff --git a/itn/marathi/rules/time.py b/itn/marathi/rules/time.py new file mode 100644 index 00000000..18c8ac55 --- /dev/null +++ b/itn/marathi/rules/time.py @@ -0,0 +1,69 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib.pynutil import delete, insert + +from itn.marathi.rules.utils import number_words, spoken_map, to_devanagari +from tn.processor import Processor + +# Quarter words before the hour and the minutes they stand for. +QUARTERS = {"सव्वा": "१५", "साडे": "३०", "पावणे": "४५"} +# ``दीड`` (half past one) and ``अडीच`` (half past two) name a whole time. +HALVES = {"दीड": "०१:३०", "अडीच": "०२:३०"} + + +class Time(Processor): + """Spoken clock times on the 12-hour dial. + + ``अकरा वाजून दोन मिनिटे`` -> ``११:०२`` (minutes past), ``अकराला पाच + मिनिटे`` -> ``१०:५५`` (minutes to; the ``-ला`` ending stays in the + hours field), ``साडे चार`` -> ``०४:३०`` and ``दीड`` -> ``०१:३०``. The + tagger drops ``वाजून`` and ``मिनिट(े)``. + """ + + def __init__(self): + super().__init__(name="time", ordertype="itn", token_orders={}) + words = number_words() + hours = spoken_map([(to_devanagari(hour, 2), words[hour]) for hour in range(1, 13)]) + to_hours = spoken_map([(to_devanagari((hour + 10) % 12 + 1, 2), words[hour] + "ला") for hour in range(1, 13)]) + before = spoken_map([(to_devanagari((hour + 10) % 12 + 1, 2), words[hour]) for hour in range(1, 13)]) + minutes = spoken_map([(to_devanagari(minute, 2), words[minute]) for minute in range(1, 60)]) + to_minutes = spoken_map([(to_devanagari(60 - minute, 2), words[minute]) for minute in range(1, 60)]) + halves = pynini.string_map(HALVES.items()).optimize() + + spaces = delete(self.SPACE.plus) + separator = spaces + insert(" ") + unit = spaces + delete(pynini.union("मिनिटे", "मिनिट")) + past = self.tag_field("hours", hours) + spaces + delete("वाजून") + separator + past += self.tag_field("minutes", minutes) + unit + to = self.tag_field("hours", to_hours) + separator + self.tag_field("minutes", to_minutes) + unit + quarters = [] + for word, value in QUARTERS.items(): + graph = before if word == "पावणे" else hours + quarters.append(self.tag_field("relation", pynini.accep(word)) + separator + self.tag_field("hours", graph)) + half = self.tag_field("hours", halves) + self.tagger = self.add_tokens(past | to | pynini.union(*quarters) | half) + + colon = insert(":") + past = self.verbalize_field("hours", hours) + colon + self.verbalize_field("minutes", minutes, True) + to = self.verbalize_field("hours", to_hours) + colon + self.verbalize_field("minutes", to_minutes, True) + quarters = [] + for word, value in QUARTERS.items(): + graph = before if word == "पावणे" else hours + relation = self.verbalize_field("relation", pynini.cross(word, "")) + quarters.append(relation + self.verbalize_field("hours", graph, True) + insert(":" + value)) + half = self.verbalize_field("hours", halves) + self.verbalizer = self.delete_tokens(past | to | pynini.union(*quarters) | half) diff --git a/itn/marathi/rules/utils.py b/itn/marathi/rules/utils.py new file mode 100644 index 00000000..996fe38c --- /dev/null +++ b/itn/marathi/rules/utils.py @@ -0,0 +1,62 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini + +from itn.utils import get_abs_path + +# Devanagari digits, the written form produced by Marathi ITN. +DIGITS = "०१२३४५६७८९" +DIGIT = pynini.union(*DIGITS).optimize() +NON_ZERO = pynini.union(*DIGITS[1:]).optimize() + + +def to_devanagari(number, width=1): + """Formats an integer with Devanagari digits, zero-padded to ``width``.""" + + return "".join(DIGITS[int(char)] for char in str(number).zfill(width)) + + +def load_pairs(rel_path): + """Loads ``writtenspoken`` rows of a Marathi data table in file order. + + A single-column row is read the same way on both sides. + """ + + pairs = [] + with open(get_abs_path("marathi/data/" + rel_path), encoding="utf-8") as stream: + for line in stream: + line = line.rstrip("\r\n") + if line: + columns = line.split("\t") + pairs.append((columns[0], columns[-1])) + return pairs + + +def number_words(): + """Maps 1-99 to their Marathi spoken words.""" + + words = {} + for name in ("digit", "tens"): + for written, spoken in load_pairs("numbers/{}.tsv".format(name)): + words[int(written.translate(str.maketrans(DIGITS, "0123456789")))] = spoken + return words + + +def spoken_map(pairs): + """Builds a ``spoken -> written`` transducer from ``(written, spoken)`` pairs.""" + + return pynini.union(*[pynini.cross(pynini.escape(spoken), pynini.escape(written)) + for written, spoken in pairs]).optimize() diff --git a/itn/marathi/rules/word.py b/itn/marathi/rules/word.py new file mode 100644 index 00000000..a5405c12 --- /dev/null +++ b/itn/marathi/rules/word.py @@ -0,0 +1,28 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from pynini.lib.pynutil import delete + +from tn.processor import Processor + + +class Word(Processor): + """Any other non-space run, kept verbatim.""" + + def __init__(self): + super().__init__(name="word", ordertype="itn", token_orders={}) + self.tagger = self.add_tokens(self.tag_field("value", self.NOT_SPACE.plus)) + graph = self.NOT_SPACE.plus | delete(self.SPACE.plus) + self.verbalizer = self.delete_tokens(self.verbalize_field("value", graph)) diff --git a/itn/marathi/test/__init__.py b/itn/marathi/test/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/itn/marathi/test/data/test_cases_cardinal.txt b/itn/marathi/test/data/test_cases_cardinal.txt new file mode 100644 index 00000000..e5ccff88 --- /dev/null +++ b/itn/marathi/test/data/test_cases_cardinal.txt @@ -0,0 +1,33 @@ +शून्य~० +उणे शंभर~-१०० +उणे सत्तावीस~-२७ +तीनशे एक~३०१ +सातशे~७०० +सातशे बावीस~७२२ +तीन हजार~३००० +आठ हजार तीन~८००३ +पाच हजार नव्वद~५०९० +उणे दोन हजार चाळीस~-२०४० +एक हजार सातशे अडुसष्ठ~१७६८ +नऊ हजार दोनशे~९२०० +तीन हजार एकशे~३१०० +एकतीसशे~३१०० +अठ्ठावन्नशे एक~५८०१ +उणे बावन्नशे~-५२०० +पंचाहत्तरशे सोळा~७५१६ +तेहतीस हजार~३३००० +सत्त्याण्णव हजार तीन~९७००३ +एक्याऐंशी हजार एक्याऐंशी~८१०८१ +सतरा हजार सातशे~१७७०० +उणे छप्पन हजार~-५६००० +उणे त्र्याऐंशी हजार सातशे पाच~-८३७०५ +चार लाख अठरा हजार~४१८००० +सात लाख आठ~७००००८ +सत्तावन्न लाख अठ्ठ्यात्तर हजार तीनशे नव्व्याण्णव~५७७८३९९ +उणे नऊ लाख दहा~-९०००१० +आठ कोटी एकतीस लाख बारा हजार नऊशे त्रेचाळीस~८३११२९४३ +एक्याण्णव कोटी सत्तर लाख~९१७०००००० +तेहतीस कोटी तेहतीस लाख बासष्ट हजार~३३३३६२००० +तेहतीस कोटी तेहतीस लाख पंचवीस~३३३३०००२५ +उणे तीन कोटी तेहतीस लाख~-३३३००००० +उणे तेहतीस कोटी तेहतीस लाख~-३३३३००००० \ No newline at end of file diff --git a/itn/marathi/test/data/test_cases_date.txt b/itn/marathi/test/data/test_cases_date.txt new file mode 100644 index 00000000..c4e0455d --- /dev/null +++ b/itn/marathi/test/data/test_cases_date.txt @@ -0,0 +1,29 @@ +दहा जून दोन हजार~१० जून २००० +जानेवारी तेरा दोन हजार तेवीस~जानेवारी १३ २०२३ +जानेवारी दहा~जानेवारी १० +जानेवारी दोन हजार~जानेवारी २००० +पंधरा फेब्रुवारी अठराशे सतरा~१५ फेब्रुवारी १८१७ +इसवी सन बाराशे सत्याहत्तर~इ.स. १२७७ +इसवी सन पूर्व एकोणीसशे नव्व्याण्णव~इ.स.पू. १९९९ +ऑक्टोबर तेवीस सातशे आठ~ऑक्टोबर २३ ७०८ +फेब्रुवारी एकोणतीस~फेब्रुवारी २९ +इसवी सन सतराशे~इ.स. १७०० +मार्च बावीस एकोणीसशे अठ्ठ्याण्णव~मार्च २२ १९९८ +एप्रिल एक अकराशे अकरा~एप्रिल १ ११११ +बावीस मे एकोणीसशे एकसष्ट~२२ मे १९६१ +एकतीस जुलै एक हजार~३१ जुलै १००० +ऑगस्ट सात~ऑगस्ट ७ +सप्टेंबर सत्तावीस दोन हजार सहा~सप्टेंबर २७ २००६ +नऊ ऑक्टोबर दोन हजार दोन~९ ऑक्टोबर २००२ +नोव्हेंबर सोळाशे सत्तेचाळीस~नोव्हेंबर १६४७ +एकतीस डिसेंबर एकोणीसशे नव्व्याण्णव~३१ डिसेंबर १९९९ +मार्च~मार्च +एप्रिल अठराशे एक~एप्रिल १८०१ +इसवी सन पूर्व चारशे~इ.स.पू. ४०० +इसवी सन दोन हजार दहा~इ.स. २०१० +जुलै पंधरा~जुलै १५ +वीस ऑगस्ट एकोणीसशे~२० ऑगस्ट १९०० +चौदा सप्टेंबर~१४ सप्टेंबर +ऑक्टोबर चार~ऑक्टोबर ४ +तेरा नोव्हेंबर दोन हजार तेवीस~१३ नोव्हेंबर २०२३ +डिसेंबर पंचवीस~डिसेंबर २५ \ No newline at end of file diff --git a/itn/marathi/test/data/test_cases_decimal.txt b/itn/marathi/test/data/test_cases_decimal.txt new file mode 100644 index 00000000..2ec11fef --- /dev/null +++ b/itn/marathi/test/data/test_cases_decimal.txt @@ -0,0 +1,29 @@ +तेहतीस पूर्णांक तीन~३३.३ +चारशे पूर्णांक शून्य पाच~४००.०५ +आठशे पूर्णांक शून्य तीन एक~८००.०३१ +दोन हजार पूर्णांक आठ एक दोन पाच~२०००.८१२५ +चार लाख~४ लाख +उणे बाहत्तर लाख~-७२ लाख +तेहतीस लाख सत्तावीस हजार आठशे वीस पूर्णांक नऊ~३३२७८२०.९ +एकेचाळीस पूर्णांक तीन लाख~४१.३ लाख +उणे एक्याण्णव पूर्णांक चार कोटी~-९१.४ कोटी +दहा पूर्णांक पाच सहा~१०.५६ +सात पूर्णांक सात~७.७ +पंचाहत्तर कोटी~७५ कोटी +उणे बारा पूर्णांक एक पाच कोटी~-१२.१५ कोटी +बावन्न पूर्णांक एक शून्य चार आठ पाच~५२.१०४८५ +उणे साठ हजार पूर्णांक नऊ नऊ नऊ शून्य~-६००००.९९९० +शून्य पूर्णांक आठ सात सहा~०.८७६ +उणे शून्य पूर्णांक एक दोन तीन~-०.१२३ +शंभर पूर्णांक नऊ~१००.९ +नव्व्याण्णव पूर्णांक नऊ नऊ नऊ~९९.९९९ +सतरा लाख~१७ लाख +उणे दोन पूर्णांक आठ एक दोन~-२.८१२ +सदुसष्ट पूर्णांक चार कोटी~६७.४ कोटी +उणे ऐंशी पूर्णांक एक सहा नऊ आठ लाख~-८०.१६९८ लाख +शंभर पूर्णांक एक दोन तीन चार पाच~१००.१२३४५ +शंभर कोटी~१०० कोटी +उणे सत्तर पूर्णांक पाच~-७०.५ +आठ पूर्णांक सहा लाख~८.६ लाख +पाच हजार दोनशे अडतीस पूर्णांक चार सहा~५२३८.४६ +चाळीस कोटी~४० कोटी \ No newline at end of file diff --git a/itn/marathi/test/data/test_cases_time.txt b/itn/marathi/test/data/test_cases_time.txt new file mode 100644 index 00000000..ea31ca0a --- /dev/null +++ b/itn/marathi/test/data/test_cases_time.txt @@ -0,0 +1,29 @@ +साडे चार~०४:३० +साडे अकरा~११:३० +सव्वा आठ~०८:१५ +सव्वा बारा~१२:१५ +पावणे सहा~०५:४५ +पावणे दहा~०९:४५ +अकराला पाच मिनिटे~१०:५५ +आठला सात मिनिटे~०७:५३ +अकरा वाजून दोन मिनिटे~११:०२ +अकराला दोन मिनिट~१०:५८ +बारा वाजून पाच मिनिटे~१२:०५ +एक वाजून सात मिनिटे~०१:०७ +दीड~०१:३० +अडीच~०२:३० +नऊ वाजून वीस मिनिटे~०९:२० +सातला बारा मिनिटे~०६:४८ +पावणे अकरा~१०:४५ +दहा वाजून दहा मिनिटे~१०:१० +एकला एक मिनिट~१२:५९ +बारा वाजून पन्नास मिनिटे~१२:५० +चार वाजून पंचेचाळीस मिनिटे~०४:४५ +पावणे पाच~०४:४५ +पावणे एक~१२:४५ +सव्वा एक~०१:१५ +दहाला एक मिनिट~०९:५९ +सहा वाजून एक मिनिट~०६:०१ +नऊ वाजून तीस मिनिटे~०९:३० +दहाला पंधरा मिनिटे~०९:४५ +दोन वाजून अठ्ठेचाळीस मिनिटे~०२:४८ \ No newline at end of file diff --git a/itn/marathi/test/data/test_cases_word.txt b/itn/marathi/test/data/test_cases_word.txt new file mode 100644 index 00000000..3b7ef04e --- /dev/null +++ b/itn/marathi/test/data/test_cases_word.txt @@ -0,0 +1,11 @@ +~ +, मी~, मी +म ~म +मी~मी +आम्ही~आम्ही +तिथे~तिथे +थोडे~थोडे +आई~आई +वडील~वडील +झाड~झाड +पिणे~पिणे \ No newline at end of file diff --git a/itn/marathi/test/normalizer_test.py b/itn/marathi/test/normalizer_test.py new file mode 100644 index 00000000..e037995e --- /dev/null +++ b/itn/marathi/test/normalizer_test.py @@ -0,0 +1,164 @@ +# Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import re +from pathlib import Path + +import pytest + +from itn.marathi.inverse_normalizer import InverseNormalizer + +PACKAGE = Path(__file__).resolve().parent.parent + + +def original_cases(): + cases = [] + for path in sorted((Path(__file__).parent / "data").glob("test_cases_*.txt")): + with path.open(encoding="utf-8", newline="") as stream: + for index, line in enumerate(stream, 1): + if line.endswith("\n"): + line = line[:-1] + spoken, written = line.split("~") + cases.append(pytest.param(spoken, written, id="{}-{}".format(path.stem, index))) + return cases + + +CASES = original_cases() + + +@pytest.fixture(scope="module") +def normalizer(shared_pipeline): + return shared_pipeline(InverseNormalizer) + + +def test_original_fixture_count(): + # No fixture file ends with a newline; every last line still counts. + assert len(CASES) == 131 + assert CASES[32].values == ("उणे तेहतीस कोटी तेहतीस लाख", "-३३३३०००००") + assert CASES[120].values == ("", "") + assert CASES[122].values == ("म ", "म") + assert CASES[130].values == ("पिणे", "पिणे") + + +@pytest.mark.parametrize("spoken, written", CASES) +def test_original(normalizer, spoken, written): + assert normalizer.normalize(spoken) == written + + +@pytest.mark.parametrize("spoken, written", CASES) +def test_original_mapping(normalizer, spoken, written): + result = normalizer.normalize_with_mapping(spoken, include_identity=True) + assert result.output_text == written + for mapping in result.mappings: + assert spoken[mapping.input_start:mapping.input_end] == mapping.input_text + assert written[mapping.output_start:mapping.output_end] == mapping.output_text + + +def test_tag_then_verbalize_matches_normalize(normalizer): + for param in CASES: + spoken, written = param.values + assert normalizer.verbalize(normalizer.tag(spoken)) == written + + +@pytest.mark.parametrize("spoken, kind, members, written", [ + ("उणे सातशे बावीस", "cardinal", {"negative": "उणे", "integer": "सातशे बावीस"}, "-७२२"), + ("दोन लाख एकतीसशे", "cardinal", {"integer": "दोन लाख एकतीसशे"}, "२०३१००"), + ("एक अब्ज दोन", "cardinal", {"integer": "एक अब्ज दोन"}, "१" + "०" * 8 + "२"), + ("उणे बारा पूर्णांक एक पाच कोटी", "decimal", { + "negative": "उणे", "integer_part": "बारा", "fractional_part": "एक पाच", "quantity": "कोटी", + }, "-१२.१५ कोटी"), + ("सतरा लाख", "decimal", {"integer_part": "सतरा", "quantity": "लाख"}, "१७ लाख"), + ("पूर्णांक पाच", "decimal", {"fractional_part": "पाच"}, ".५"), + ("पंधरा फेब्रुवारी अठराशे सतरा", "date", {"day": "पंधरा", "month": "फेब्रुवारी", "year": "अठराशे सतरा"}, + "१५ फेब्रुवारी १८१७"), + ("जानेवारी दहा", "date", {"month": "जानेवारी", "day": "दहा"}, "जानेवारी १०"), + ("जानेवारी दोन हजार", "date", {"month": "जानेवारी", "year": "दोन हजार"}, "जानेवारी २०००"), + ("इसवी सन पूर्व चारशे", "date", {"era": "इसवी सन पूर्व", "year": "चारशे"}, "इ.स.पू. ४००"), + ("अकरा वाजून दोन मिनिटे", "time", {"hours": "अकरा", "minutes": "दोन"}, "११:०२"), + ("अकराला पाच मिनिटे", "time", {"hours": "अकराला", "minutes": "पाच"}, "१०:५५"), + ("पावणे एक", "time", {"relation": "पावणे", "hours": "एक"}, "१२:४५"), + ("साडे चार", "time", {"relation": "साडे", "hours": "चार"}, "०४:३०"), + ("अडीच", "time", {"hours": "अडीच"}, "०२:३०"), + ("आई", "word", {"value": "आई"}, "आई"), +]) +def test_raw_fields(normalizer, spoken, kind, members, written): + tagged = normalizer.tag(spoken) + parser = normalizer.token_parser() + parser.parse(tagged) + assert len(parser.tokens) == 1 + assert parser.tokens[0].name == kind + assert parser.tokens[0].members == members + assert normalizer.verbalize(tagged) == written + assert normalizer.normalize(spoken) == written + + +@pytest.mark.parametrize("spoken, written", [ + ("वीस।", "२० ।"), + ("नव्व्याण्णव अब्ज नव्व्याण्णव कोटी नव्व्याण्णव लाख नव्व्याण्णव हजार नऊशे नव्व्याण्णव", "९९९९९९९९९९९"), + ("शून्य शून्य", "० ०"), + ("तीन पूर्णांक पाच हजार", "३.५ हजार"), + ("बारा वाजून एक मिनिट", "१२:०१"), + ("सव्वा बारा वाजता", "१२:१५ वाजता"), + ("मार्च पाच", "मार्च ५"), +]) +def test_additional_semantics(normalizer, spoken, written): + assert normalizer.normalize(spoken) == written + + +@pytest.mark.parametrize("spoken", ["चार लाख", "साडे चार", "जानेवारी दहा", "तीन हजार एकशे"]) +def test_nbest_mapping_synchronization(normalizer, spoken): + outputs = normalizer.normalize(spoken, nbest=3) + mapped = normalizer.normalize_with_mapping(spoken, nbest=3, include_identity=True) + assert [result.output_text for result in mapped] == outputs + assert len(outputs) == len(set(outputs)) + assert outputs[0] == normalizer.normalize(spoken) + for result in mapped: + for mapping in result.mappings: + assert spoken[mapping.input_start:mapping.input_end] == mapping.input_text + assert result.output_text[mapping.output_start:mapping.output_end] == mapping.output_text + + +def test_escaped_word_round_trips(normalizer): + spoken = 'मी\\a"b' + parser = normalizer.token_parser() + parser.parse(normalizer.tag(spoken)) + assert [token.members for token in parser.tokens] == [{"value": spoken}] + assert normalizer.normalize(spoken) == spoken + + +def test_empty_input(normalizer): + assert normalizer.normalize("") == "" + assert normalizer.normalize("", nbest=3) == [""] + assert normalizer.normalize_with_mapping("").output_text == "" + + +@pytest.mark.parametrize("spoken", [" ", "\t", "\n", " \t\n ", "\u00a0"]) +def test_whitespace_only_input(normalizer, spoken): + tagged = normalizer.tag(spoken) + parser = normalizer.token_parser() + parser.parse(tagged) + assert len(parser.tokens) == 1 + assert parser.tokens[0].name == "word" + assert parser.tokens[0].members == {"value": spoken} + assert normalizer.verbalize(tagged) == "" + assert normalizer.normalize(spoken, nbest=3) == [""] + mapped = normalizer.normalize_with_mapping(spoken, nbest=3, include_identity=True) + assert [result.output_text for result in mapped] == [""] + assert all(mapping.output_text == "" for mapping in mapped[0].mappings) + + +def test_standalone_from_tn(): + tn_import = re.compile(r"^\s*(from|import)\s+tn\.marathi\b", re.MULTILINE) + for path in PACKAGE.rglob("*.py"): + assert not tn_import.search(path.read_text(encoding="utf-8")), path diff --git a/pyproject.toml b/pyproject.toml index 1809e6fb..9f4acb7a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -78,6 +78,7 @@ tn = [ "spanish/data/*.tsv", "spanish/data/*/*.tsv", "hindi/data/*/*.tsv", + "marathi/data/*/*.tsv", "kinyarwanda/data/*.tsv", "kinyarwanda/data/*/*.tsv", ] @@ -104,6 +105,7 @@ itn = [ "spanish/data/*.tsv", "spanish/data/*/*.tsv", "hindi/data/*/*.tsv", + "marathi/data/*/*.tsv", "kinyarwanda/data/*.tsv", "kinyarwanda/data/*/*.tsv", ] diff --git a/tn/chinese/test/cache_test.py b/tn/chinese/test/cache_test.py index 5ccce03a..e87f740d 100644 --- a/tn/chinese/test/cache_test.py +++ b/tn/chinese/test/cache_test.py @@ -13,6 +13,7 @@ from itn.kinyarwanda.inverse_normalizer import InverseNormalizer as RwInverseNormalizer from itn.japanese.inverse_normalizer import InverseNormalizer as JaInverseNormalizer from itn.korean.inverse_normalizer import InverseNormalizer as KoInverseNormalizer +from itn.marathi.inverse_normalizer import InverseNormalizer as MrInverseNormalizer from itn.portuguese.inverse_normalizer import InverseNormalizer as PtInverseNormalizer from itn.russian.inverse_normalizer import InverseNormalizer as RuInverseNormalizer from itn.spanish.inverse_normalizer import InverseNormalizer as EsInverseNormalizer @@ -28,6 +29,7 @@ from tn.kinyarwanda.normalizer import Normalizer as RwNormalizer from tn.japanese.normalizer import Normalizer as JaNormalizer from tn.korean.normalizer import Normalizer as KoNormalizer +from tn.marathi.normalizer import Normalizer as MrNormalizer from tn.portuguese.normalizer import Normalizer as PtNormalizer from tn.russian.normalizer import Normalizer as RuNormalizer from tn.spanish.normalizer import Normalizer as EsNormalizer @@ -98,6 +100,7 @@ def _build_small_graphs(processor): (EsNormalizer, "es_tn", "tn", {}, {"deterministic": True}), (EsNormalizer, "es_tn", "tn", {"deterministic": False}, {"deterministic": False}), (HiNormalizer, "hi_tn", "tn", {}, {}), + (MrNormalizer, "mr_tn", "tn", {}, {}), (RwNormalizer, "rw_tn", "tn", {}, {}), ( ZhInverseNormalizer, @@ -146,6 +149,7 @@ def _build_small_graphs(processor): (EsInverseNormalizer, "es_itn", "itn", {}, {"cased": False}), (EsInverseNormalizer, "es_itn", "itn", {"cased": True}, {"cased": True}), (HiInverseNormalizer, "hi_itn", "itn", {}, {}), + (MrInverseNormalizer, "mr_itn", "itn", {}, {}), (RwInverseNormalizer, "rw_itn", "itn", {}, {}), ], ) diff --git a/tn/cli.py b/tn/cli.py index 2f2a879a..baef6922 100644 --- a/tn/cli.py +++ b/tn/cli.py @@ -69,6 +69,7 @@ class PipelineSpec: (BooleanOption("deterministic", True, "use deterministic Spanish readings"),), ), "hi": PipelineSpec("tn.hindi.normalizer", "Normalizer"), + "mr": PipelineSpec("tn.marathi.normalizer", "Normalizer"), "rw": PipelineSpec("tn.kinyarwanda.normalizer", "Normalizer"), "ja": PipelineSpec( "tn.japanese.normalizer", @@ -108,6 +109,7 @@ class PipelineSpec: (BooleanOption("cased", False, 'also accept capitalized spoken words, e.g. "Doce Dólares"'),), ), "hi": PipelineSpec("itn.hindi.inverse_normalizer", "InverseNormalizer"), + "mr": PipelineSpec("itn.marathi.inverse_normalizer", "InverseNormalizer"), "rw": PipelineSpec("itn.kinyarwanda.inverse_normalizer", "InverseNormalizer"), "ja": PipelineSpec( "itn.japanese.inverse_normalizer", diff --git a/tn/marathi/__init__.py b/tn/marathi/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tn/marathi/data/date/eras.tsv b/tn/marathi/data/date/eras.tsv new file mode 100644 index 00000000..517186b0 --- /dev/null +++ b/tn/marathi/data/date/eras.tsv @@ -0,0 +1,2 @@ +इ.स.पू. इसवी सन पूर्व +इ.स. इसवी सन diff --git a/tn/marathi/data/date/months.tsv b/tn/marathi/data/date/months.tsv new file mode 100644 index 00000000..f39b90d2 --- /dev/null +++ b/tn/marathi/data/date/months.tsv @@ -0,0 +1,12 @@ +जानेवारी +फेब्रुवारी +मार्च +एप्रिल +मे +जून +जुलै +ऑगस्ट +सप्टेंबर +ऑक्टोबर +नोव्हेंबर +डिसेंबर \ No newline at end of file diff --git a/tn/marathi/data/numbers/digit.tsv b/tn/marathi/data/numbers/digit.tsv new file mode 100644 index 00000000..f737ade2 --- /dev/null +++ b/tn/marathi/data/numbers/digit.tsv @@ -0,0 +1,9 @@ +१ एक +२ दोन +३ तीन +४ चार +५ पाच +६ सहा +७ सात +८ आठ +९ नऊ \ No newline at end of file diff --git a/tn/marathi/data/numbers/tens.tsv b/tn/marathi/data/numbers/tens.tsv new file mode 100644 index 00000000..654244b3 --- /dev/null +++ b/tn/marathi/data/numbers/tens.tsv @@ -0,0 +1,90 @@ +१० दहा +११ अकरा +१२ बारा +१३ तेरा +१४ चौदा +१५ पंधरा +१६ सोळा +१७ सतरा +१८ अठरा +१९ एकोणीस +२० वीस +२१ एकवीस +२२ बावीस +२३ तेवीस +२४ चोवीस +२५ पंचवीस +२६ सव्वीस +२७ सत्तावीस +२८ अठ्ठावीस +२९ एकोणतीस +३० तीस +३१ एकतीस +३२ बत्तीस +३३ तेहतीस +३४ चौतीस +३५ पस्तीस +३६ छत्तीस +३७ सदतीस +३८ अडतीस +३९ एकोणचाळीस +४० चाळीस +४१ एकेचाळीस +४२ बेचाळीस +४३ त्रेचाळीस +४४ चव्वेचाळीस +४५ पंचेचाळीस +४६ सेहचाळीस +४७ सत्तेचाळीस +४८ अठ्ठेचाळीस +४९ एकोणपन्नास +५० पन्नास +५१ एकावन्न +५२ बावन्न +५३ त्रेपन्न +५४ चौपन्न +५५ पंचावन्न +५६ छप्पन +५७ सत्तावन्न +५८ अठ्ठावन्न +५९ एकोणसाठ +६० साठ +६१ एकसष्ट +६२ बासष्ट +६३ त्रेसष्ट +६४ चौसष्ट +६५ पासष्ट +६६ सहासष्ठ +६७ सदुसष्ट +६८ अडुसष्ठ +६९ एकोणसत्तर +७० सत्तर +७१ एकाहत्तर +७२ बाहत्तर +७३ त्र्याहत्तर +७४ चौऱ्याहत्तर +७५ पंचाहत्तर +७६ शहात्तर +७७ सत्याहत्तर +७८ अठ्ठ्यात्तर +७९ एकोणऐंशी +८० ऐंशी +८१ एक्याऐंशी +८२ ब्याऐंशी +८३ त्र्याऐंशी +८४ चौऱ्याऐंशी +८५ पंच्याऐंशी +८६ सह्यांशी +८७ सत्याऐंशी +८८ अठ्ठ्याऐंशी +८९ एकोणनव्वद +९० नव्वद +९१ एक्याण्णव +९२ ब्याण्णव +९३ त्र्याण्णव +९४ चौऱ्याण्णव +९५ पंच्याण्णव +९६ शहाण्णव +९७ सत्त्याण्णव +९८ अठ्ठ्याण्णव +९९ नव्व्याण्णव \ No newline at end of file diff --git a/tn/marathi/data/numbers/zero.tsv b/tn/marathi/data/numbers/zero.tsv new file mode 100644 index 00000000..0735899f --- /dev/null +++ b/tn/marathi/data/numbers/zero.tsv @@ -0,0 +1 @@ +० शून्य diff --git a/tn/marathi/normalizer.py b/tn/marathi/normalizer.py new file mode 100644 index 00000000..4c6c7f48 --- /dev/null +++ b/tn/marathi/normalizer.py @@ -0,0 +1,53 @@ +# Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from pynini.lib.pynutil import add_weight + +from tn.marathi.rules.cardinal import Cardinal +from tn.marathi.rules.date import Date +from tn.marathi.rules.decimal import Decimal +from tn.marathi.rules.punctuation import Punctuation +from tn.marathi.rules.time import Time +from tn.marathi.rules.word import Word +from tn.processor import Processor, RuleSpec + +# Every Marathi rule emits its fields in spoken order, so no token needs +# reordering before verbalization. +TOKEN_ORDERS = {} + + +class Normalizer(Processor): + + def __init__(self, cache_dir=None, overwrite_cache=False): + super().__init__(name="mr_normalizer", token_orders=TOKEN_ORDERS) + self.build_fst("mr_tn", cache_dir, overwrite_cache, {}) + + def build_tagger_and_verbalizer(self): + cardinal = Cardinal() + punctuation = Punctuation() + word = Word() + rules = ( + RuleSpec(Time(), 1.05), + RuleSpec(Date(cardinal), 1.09), + RuleSpec(Decimal(cardinal), 1.08), + RuleSpec(cardinal, 1.1), + RuleSpec(word, 100, verbalize=False), + RuleSpec(punctuation, verbalize=False), + ) + classify = self.tagger_union(rules) + punct = add_weight(punctuation.tagger, 1.1) + token = punct.star + classify + punct.star + self.tagger = self.DELETE_SPACE + token + (self.DELETE_EXTRA_SPACE + token).star + self.DELETE_SPACE + self.tagger |= word.add_tokens(word.tag_field("value", self.SPACE.plus)) + self.verbalizer = self.join_tokens(self.verbalizer_union(rules), (word.verbalizer, punctuation.verbalizer)) diff --git a/tn/marathi/rules/__init__.py b/tn/marathi/rules/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tn/marathi/rules/cardinal.py b/tn/marathi/rules/cardinal.py new file mode 100644 index 00000000..91c6f149 --- /dev/null +++ b/tn/marathi/rules/cardinal.py @@ -0,0 +1,76 @@ +# Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import utf8 +from pynini.lib.pynutil import delete, insert + +from tn.marathi.rules.utils import MR_DIGIT, MR_NON_ZERO, canonical, number_words, to_devanagari, written_map +from tn.processor import Processor + +# Indian place values above the hundreds, each spanning two digits. +SCALES = ("अब्ज", "कोटी", "लाख", "हजार") +WIDTH = 3 + 2 * len(SCALES) +MINUS = "उणे" + + +class Cardinal(Processor): + """Written cardinals, e.g. ``-७२२`` or ``-722`` -> ``उणे सातशे बावीस``. + + The tagger keeps the written digits (and the ``-`` sign); the verbalizer + reads them with Indian grouping up to ``अब्ज`` (10^9). Comma-grouped + numbers (``१,००,०००``) are read as one number, and numbers with leading + zeros or more than eleven digits are read digit by digit. + """ + + def __init__(self): + super().__init__("cardinal", token_orders={}) + self.build_graphs() + negative = self.tag_field("negative", pynini.accep("-")) + insert(" ") + self.tagger = self.add_tokens(negative.ques + self.tag_field("integer", self.graph)) + sign = self.verbalize_field("negative", pynini.cross("-", MINUS + " ")) + delete(" ") + self.verbalizer = self.delete_tokens(sign.ques + self.verbalize_field("integer", self.graph)) + + def build_graphs(self): + words = number_words() + self.digit = written_map([(to_devanagari(n), words[n]) for n in range(1, 10)]) + self.zero = written_map([(to_devanagari(0), words[0])]) + # Two-digit groups with at least one non-zero digit, e.g. ``०७`` or ``४२``. + self.two = written_map([(to_devanagari(n, 2), words[n]) for n in range(1, 100)]) + self.tens = written_map([(to_devanagari(n), words[n]) for n in range(10, 100)]) + hundreds = self.digit + insert("शे") + (delete("००") | insert(" ") + self.two) | delete("०") + self.two + self.hundreds = hundreds.optimize() + + fixed = pynini.accep("") + for scale in SCALES: + fixed += insert(" ") + self.two + insert(" " + scale) | delete("००") + fixed += insert(" ") + self.hundreds | delete("०००") + pad = pynini.union(*[insert("०" * (WIDTH - k)) + MR_NON_ZERO + MR_DIGIT ** (k - 1) for k in range(1, WIDTH + 1)]) + lstrip = delete(" ") + utf8.VALID_UTF8_CHAR.star + number = pad @ fixed @ lstrip + # ``शंभर`` is a hundred on its own; inside larger numbers it is ``एकशे``. + number = pynini.difference(MR_DIGIT.plus, "१००") @ number | pynini.cross("१००", "शंभर") + self.number = (number | self.zero).optimize() + + single = (self.digit | self.zero).optimize() + self.digits = (single + (insert(" ") + single).star).optimize() + long_or_padded = MR_DIGIT ** (WIDTH + 1) + MR_DIGIT.star | "०" + MR_DIGIT.plus + digitwise = long_or_padded @ self.digits + + comma = delete(",") + indian = MR_DIGIT.ques + MR_NON_ZERO + (comma + MR_DIGIT ** 2).star + comma + MR_DIGIT ** 3 + western = pynini.closure(MR_DIGIT, 0, 2) + MR_NON_ZERO + (comma + MR_DIGIT ** 3).plus + grouped = (indian | western) @ self.number + self.graph = canonical(self.number | grouped | digitwise) + self.integer = canonical(self.number | grouped) diff --git a/tn/marathi/rules/date.py b/tn/marathi/rules/date.py new file mode 100644 index 00000000..8ff71701 --- /dev/null +++ b/tn/marathi/rules/date.py @@ -0,0 +1,70 @@ +# Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib.pynutil import add_weight, delete, insert + +from tn.marathi.rules.utils import MR_DIGIT, MR_NON_ZERO, canonical, load_pairs, number_words, to_devanagari, written_map +from tn.processor import Processor + +# A lone number after a month reads as the day unless it cannot be one. +YEAR_ONLY_WEIGHT = 0.01 + + +class Date(Processor): + """Written dates in either day-month or month-day order. + + ``१५ फेब्रुवारी १८१७`` -> ``पंधरा फेब्रुवारी अठराशे सतरा``, ``जानेवारी १३ + २०२३`` -> ``जानेवारी तेरा दोन हजार तेवीस`` and ``इ.स.पू. ४००`` -> + ``इसवी सन पूर्व चारशे``. The tagger keeps the written day, month, year + and era; years 1100-1999 are read as hundreds (``अठराशे``), other years + as cardinals. + """ + + def __init__(self, cardinal): + super().__init__("date", token_orders={}) + words = number_words() + days = set() + for day in range(1, 32): + for width in (1, 2): + days.add((to_devanagari(day, width), words[day])) + self.day = canonical(written_map(sorted(days))) + self.month = pynini.union(*[written for written, _ in load_pairs("date/months.tsv")]).optimize() + self.era = written_map(load_pairs("date/eras.tsv")) + rest = delete("००") | insert(" ") + cardinal.two + hundreds = ("१" + MR_NON_ZERO) @ cardinal.tens + insert("शे") + rest + plain = pynini.difference(MR_NON_ZERO + pynini.closure(MR_DIGIT, 0, 3), "१" + MR_NON_ZERO + MR_DIGIT ** 2) + self.year = canonical(hundreds | plain @ cardinal.number) + self.build_tagger() + self.build_verbalizer() + + def build_tagger(self): + sep = delete(self.SPACE.plus) + insert(" ") + day = self.tag_field("day", self.day) + month = self.tag_field("month", self.month) + year = self.tag_field("year", self.year) + era = self.tag_field("era", self.era) + mdy = month + sep + (day | add_weight(year, YEAR_ONLY_WEIGHT) | day + sep + year) + dmy = day + sep + month + (sep + year).ques + self.tagger = self.add_tokens(mdy | dmy | era + sep + year) + + def build_verbalizer(self): + sep = insert(" ") + day = self.verbalize_field("day", self.day, True) + month = self.verbalize_field("month", self.month, True) + year = self.verbalize_field("year", self.year, True) + mdy = self.verbalize_field("month", self.month) + sep + (day | year | day + sep + year) + dmy = self.verbalize_field("day", self.day) + sep + month + (sep + year).ques + era = self.verbalize_field("era", self.era) + sep + year + self.verbalizer = self.delete_tokens(mdy | dmy | era) diff --git a/tn/marathi/rules/decimal.py b/tn/marathi/rules/decimal.py new file mode 100644 index 00000000..ca3a4c9a --- /dev/null +++ b/tn/marathi/rules/decimal.py @@ -0,0 +1,59 @@ +# Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib.pynutil import delete, insert + +from tn.marathi.rules.cardinal import MINUS +from tn.marathi.rules.utils import MR_DIGIT, canonical +from tn.processor import Processor + +POINT = "पूर्णांक" +QUANTITIES = ("हजार", "लाख", "कोटी", "अब्ज") + + +class Decimal(Processor): + """Written decimals, e.g. ``-१२.१५ कोटी`` -> ``उणे बारा पूर्णांक एक पाच कोटी``. + + The tagger keeps the written sign, integer digits, fractional digits and + quantity word and drops the decimal point; the verbalizer reads the + integer as a cardinal and the fraction digit by digit. + """ + + def __init__(self, cardinal): + super().__init__("decimal", token_orders={}) + self.integer = cardinal.integer + self.fractional = canonical(MR_DIGIT.plus @ cardinal.digits) + self.quantity = pynini.union(*QUANTITIES).optimize() + self.build_tagger() + self.build_verbalizer() + + def build_tagger(self): + negative = self.tag_field("negative", pynini.accep("-")) + insert(" ") + integer = self.tag_field("integer_part", self.integer) + fractional = self.tag_field("fractional_part", self.fractional) + decimal = (integer + insert(" ")).ques + delete(".") + fractional + quantity = delete(self.SPACE.plus) + insert(" ") + self.tag_field("quantity", self.quantity) + graph = decimal + quantity.ques | integer + quantity + self.tagger = self.add_tokens(negative.ques + graph) + + def build_verbalizer(self): + sign = self.verbalize_field("negative", pynini.cross("-", MINUS + " ")) + delete(" ") + integer = self.verbalize_field("integer_part", self.integer) + fractional = self.verbalize_field("fractional_part", self.fractional, leading_space=True) + decimal = integer + insert(" " + POINT + " ") + fractional + decimal |= insert(POINT + " ") + self.verbalize_field("fractional_part", self.fractional) + quantity = insert(" ") + self.verbalize_field("quantity", self.quantity, leading_space=True) + graph = decimal + quantity.ques | integer + quantity + self.verbalizer = self.delete_tokens(sign.ques + graph) diff --git a/tn/marathi/rules/punctuation.py b/tn/marathi/rules/punctuation.py new file mode 100644 index 00000000..0d372c9d --- /dev/null +++ b/tn/marathi/rules/punctuation.py @@ -0,0 +1,30 @@ +# Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib.pynutil import add_weight + +from tn.processor import Processor + + +class Punctuation(Processor): + """ASCII and Devanagari punctuation split off a neighbouring token, e.g. ``(५)``.""" + + def __init__(self): + super().__init__("punctuation", token_orders={}) + graph = pynini.union(*[pynini.escape(mark) for mark in "!#$%&'()*+,-./:;<=>?@^_`{|}~।॥‘’“”"]).optimize() + space = add_weight(self.SPACE, -0.0001).star + self.tagger = self.add_tokens(self.tag_field("value", space + graph + space)) + self.verbalizer = self.delete_tokens(self.verbalize_field("value", self.SPACE.star + graph + self.SPACE.star)) diff --git a/tn/marathi/rules/time.py b/tn/marathi/rules/time.py new file mode 100644 index 00000000..84f3e035 --- /dev/null +++ b/tn/marathi/rules/time.py @@ -0,0 +1,86 @@ +# Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib.pynutil import add_weight, delete, insert + +from tn.marathi.rules.utils import canonical, number_words, to_devanagari, written_map +from tn.processor import Processor + +# Minutes read as "minutes to the next hour" (``अकराला पाच मिनिटे``). +TO_MINUTES = range(51, 60) +# Readings with a quarter word before the hour. +QUARTERS = {15: "सव्वा ", 30: "साडे ", 45: "पावणे "} +# Half past one and half past two have their own words. +HALVES = {1: "दीड", 2: "अडीच"} +# The plain "hours past" reading stays available for n-best output. +ALTERNATIVE_WEIGHT = 0.5 + + +class Time(Processor): + """Written ``H:MM`` clock times read on the 12-hour dial. + + ``११:०२`` -> ``अकरा वाजून दोन मिनिटे``, ``१०:५५`` -> ``अकराला पाच + मिनिटे``, ``०४:३०`` -> ``साडे चार``, ``१२:४५`` -> ``पावणे एक`` and + ``०१:३०`` -> ``दीड``. The tagger keeps the written hours and minutes and + drops the colon. + """ + + def __init__(self): + super().__init__("time", token_orders={}) + words = number_words() + + def hours_map(reading, hours=range(24)): + pairs = set() + for hour in hours: + for width in (1, 2): + pairs.add((to_devanagari(hour, width), reading(hour % 12 or 12))) + return canonical(written_map(sorted(pairs))) + + def minutes_map(reading, minutes): + return canonical(written_map([(to_devanagari(minute, 2), reading(minute)) for minute in minutes])) + + def unit(count): + return words[count] + (" मिनिट" if count == 1 else " मिनिटे") + + def next_hour(hour): + return words[hour % 12 + 1] + + hour = hours_map(lambda hour: words[hour]) + paths = [ + (hour, minutes_map(lambda minute: " वाजता", [0])), + (hours_map(lambda hour: next_hour(hour) + "ला"), minutes_map(lambda minute: " " + unit(60 - minute), TO_MINUTES)), + ] + named = [hour for hour in range(24) if (hour % 12 or 12) in HALVES] + for minute, word in QUARTERS.items(): + if minute == 45: + hours = hours_map(lambda hour, word=word: word + next_hour(hour)) + elif minute == 30: + hours = hours_map(lambda hour, word=word: word + words[hour], set(range(24)) - set(named)) + hours |= hours_map(lambda hour: HALVES[hour], named) + else: + hours = hours_map(lambda hour, word=word: word + words[hour]) + paths.append((hours, minutes_map(lambda minute: "", [minute]))) + special = set(QUARTERS) | set(TO_MINUTES) + past = minutes_map(lambda minute: " वाजून " + unit(minute), [m for m in range(1, 60) if m not in special]) + alternative = minutes_map(lambda minute: " वाजून " + unit(minute), sorted(special)) + paths.append((hour, past)) + paths.append((hour, add_weight(alternative, ALTERNATIVE_WEIGHT))) + + hours = pynini.union(*[hours for hours, _ in paths]).optimize() + minutes = pynini.union(*[minutes for _, minutes in paths]).optimize() + tagger = self.tag_field("hours", hours) + delete(":") + insert(" ") + self.tag_field("minutes", minutes) + self.tagger = self.add_tokens(tagger) + verbalizer = [self.verbalize_field("hours", h) + self.verbalize_field("minutes", m, True) for h, m in paths] + self.verbalizer = self.delete_tokens(pynini.union(*verbalizer)) diff --git a/tn/marathi/rules/utils.py b/tn/marathi/rules/utils.py new file mode 100644 index 00000000..e228d3da --- /dev/null +++ b/tn/marathi/rules/utils.py @@ -0,0 +1,76 @@ +# Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pynini +from pynini.lib import utf8 + +from tn.utils import get_abs_path + +# Devanagari digits are the canonical written form; ASCII digits are accepted +# as input and mapped onto them inside the verbalizer graphs. +DEVANAGARI_DIGITS = "०१२३४५६७८९" +MR_DIGIT = pynini.union(*DEVANAGARI_DIGITS).optimize() +MR_NON_ZERO = pynini.union(*DEVANAGARI_DIGITS[1:]).optimize() +ALL_DIGIT = pynini.union(MR_DIGIT, *"0123456789").optimize() +TO_DEVANAGARI = pynini.union(MR_DIGIT, *[pynini.cross(str(index), char) for index, char in enumerate(DEVANAGARI_DIGITS)]) +TO_DEVANAGARI = TO_DEVANAGARI.optimize() + + +def to_devanagari(number, width=1): + """Formats an integer with Devanagari digits, zero-padded to ``width``.""" + + return "".join(DEVANAGARI_DIGITS[int(char)] for char in str(number).zfill(width)) + + +def canonical(graph): + """Lets a graph over Devanagari digits also read ASCII (or mixed) digits. + + Non-digit characters pass through unchanged. + """ + + char = pynini.union(TO_DEVANAGARI, pynini.difference(utf8.VALID_UTF8_CHAR, ALL_DIGIT)) + return (char.optimize().star @ graph).optimize() + + +def load_pairs(rel_path): + """Loads ``writtenspoken`` rows of a Marathi data table in file order. + + A single-column row is read the same way on both sides. + """ + + pairs = [] + with open(get_abs_path("marathi/data/" + rel_path), encoding="utf-8") as stream: + for line in stream: + line = line.rstrip("\r\n") + if line: + columns = line.split("\t") + pairs.append((columns[0], columns[-1])) + return pairs + + +def number_words(): + """Maps 0-99 to their Marathi spoken words.""" + + words = {} + for name in ("zero", "digit", "tens"): + for written, spoken in load_pairs("numbers/{}.tsv".format(name)): + words[int(written.translate(str.maketrans(DEVANAGARI_DIGITS, "0123456789")))] = spoken + return words + + +def written_map(pairs): + """Builds a ``written -> spoken`` transducer from ``(written, spoken)`` pairs.""" + + return pynini.union(*[pynini.cross(pynini.escape(written), pynini.escape(spoken)) + for written, spoken in pairs]).optimize() diff --git a/tn/marathi/rules/word.py b/tn/marathi/rules/word.py new file mode 100644 index 00000000..b2643cdc --- /dev/null +++ b/tn/marathi/rules/word.py @@ -0,0 +1,28 @@ +# Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from pynini.lib.pynutil import delete + +from tn.processor import Processor + + +class Word(Processor): + """Any other non-space run, kept verbatim.""" + + def __init__(self): + super().__init__("word", token_orders={}) + self.tagger = self.add_tokens(self.tag_field("value", self.NOT_SPACE.plus)) + graph = self.NOT_SPACE.plus | delete(self.SPACE.plus) + self.verbalizer = self.delete_tokens(self.verbalize_field("value", graph)) diff --git a/tn/marathi/test/__init__.py b/tn/marathi/test/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tn/marathi/test/normalizer_test.py b/tn/marathi/test/normalizer_test.py new file mode 100644 index 00000000..cdb650c0 --- /dev/null +++ b/tn/marathi/test/normalizer_test.py @@ -0,0 +1,221 @@ +# Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import re +from pathlib import Path + +import pytest + +from tn.marathi.normalizer import Normalizer + +PACKAGE = Path(__file__).resolve().parent.parent +# NeMo has no Marathi TN fixtures; the ITN fixtures are read in reverse. +ITN_DATA = PACKAGE.parent.parent / "itn" / "marathi" / "test" / "data" + +# Reversed ITN pairs that cannot hold for a deterministic TN, with reasons. +EXCLUDED = { + "५८०१": "ITN accepts the colloquial hundreds count; TN reads cardinals with हजार", + "७५१६": "ITN accepts the colloquial hundreds count; TN reads cardinals with हजार", + "-५२००": "ITN accepts the colloquial hundreds count; TN reads cardinals with हजार", + "१०:५८": "fixture uses singular मिनिट for two minutes, unlike आठला सात मिनिटे", + "०६:४८": "fixture reads :48 as minutes-to, but ०२:४८ as minutes-past", + "०९:३०": "fixture reads :30 as वाजून तीस मिनिटे, but ०४:३० as साडे", + "म": "spoken side keeps an untrimmed trailing space", +} + + +def reversed_cases(): + candidates = {} + for path in sorted(ITN_DATA.glob("test_cases_*.txt")): + with path.open(encoding="utf-8", newline="") as stream: + for line in stream: + if line.endswith("\n"): + line = line[:-1] + spoken, written = line.split("~") + candidates.setdefault(written, []).append(spoken) + return [pytest.param(written, spoken, id=written) for written, spoken in candidates.items()] + + +CASES = reversed_cases() +CHECKED = [param for param in CASES if param.values[0] not in EXCLUDED] + + +@pytest.fixture(scope="module") +def normalizer(shared_pipeline): + return shared_pipeline(Normalizer) + + +def test_reversed_fixture_count(): + assert sum(len(param.values[1]) for param in CASES) == 131 + assert len(CASES) == 128 + assert len(CHECKED) == 121 + assert set(EXCLUDED) <= {param.values[0] for param in CASES} + + +@pytest.mark.parametrize("written, spoken", CHECKED) +def test_reversed_original(normalizer, written, spoken): + assert normalizer.normalize(written) in spoken + + +@pytest.mark.parametrize("written, spoken", CHECKED) +def test_reversed_original_mapping(normalizer, written, spoken): + result = normalizer.normalize_with_mapping(written, include_identity=True) + assert result.output_text in spoken + for mapping in result.mappings: + assert written[mapping.input_start:mapping.input_end] == mapping.input_text + assert result.output_text[mapping.output_start:mapping.output_end] == mapping.output_text + + +def test_tag_then_verbalize_matches_normalize(normalizer): + for param in CASES: + written = param.values[0] + assert normalizer.verbalize(normalizer.tag(written)) == normalizer.normalize(written) + + +@pytest.mark.parametrize("written, spoken", [ + ("५८०१", "पाच हजार आठशे एक"), + ("१०:५८", "अकराला दोन मिनिटे"), + ("०६:४८", "सहा वाजून अठ्ठेचाळीस मिनिटे"), + ("०९:३०", "साडे नऊ"), + ("म", "म"), +]) +def test_excluded_pairs_read_consistently(normalizer, written, spoken): + assert written in EXCLUDED + assert normalizer.normalize(written) == spoken + + +@pytest.mark.parametrize("written, kind, members, spoken", [ + ("-722", "cardinal", {"negative": "-", "integer": "722"}, "उणे सातशे बावीस"), + ("१,२३,४५६", "cardinal", {"integer": "१,२३,४५६"}, "एक लाख तेवीस हजार चारशे छप्पन"), + ("007", "cardinal", {"integer": "007"}, "शून्य शून्य सात"), + ("-१२.१५ कोटी", "decimal", { + "negative": "-", "integer_part": "१२", "fractional_part": "१५", "quantity": "कोटी", + }, "उणे बारा पूर्णांक एक पाच कोटी"), + ("4 लाख", "decimal", {"integer_part": "4", "quantity": "लाख"}, "चार लाख"), + ("15 फेब्रुवारी 1817", "date", {"day": "15", "month": "फेब्रुवारी", "year": "1817"}, + "पंधरा फेब्रुवारी अठराशे सतरा"), + ("मार्च ०५", "date", {"month": "मार्च", "day": "०५"}, "मार्च पाच"), + ("इ.स.पू. ४००", "date", {"era": "इ.स.पू.", "year": "४००"}, "इसवी सन पूर्व चारशे"), + ("11:02", "time", {"hours": "11", "minutes": "02"}, "अकरा वाजून दोन मिनिटे"), + ("१३:३०", "time", {"hours": "१३", "minutes": "३०"}, "दीड"), + ("२३:५९", "time", {"hours": "२३", "minutes": "५९"}, "बाराला एक मिनिट"), + ("१२:००", "time", {"hours": "१२", "minutes": "००"}, "बारा वाजता"), + ("आई", "word", {"value": "आई"}, "आई"), +]) +def test_raw_fields(normalizer, written, kind, members, spoken): + tagged = normalizer.tag(written) + parser = normalizer.token_parser() + parser.parse(tagged) + assert len(parser.tokens) == 1 + assert parser.tokens[0].name == kind + assert parser.tokens[0].members == members + assert normalizer.verbalize(tagged) == spoken + assert normalizer.normalize(written) == spoken + + +@pytest.mark.parametrize("written, spoken", [ + ("(५)", "(पाच)"), + ("५।", "पाच ।"), + ("१००", "शंभर"), + ("११००", "एक हजार एकशे"), + ("1,000,000", "दहा लाख"), + ("९९९९९९९९९९९", "नव्व्याण्णव अब्ज नव्व्याण्णव कोटी नव्व्याण्णव लाख नव्व्याण्णव हजार नऊशे नव्व्याण्णव"), + ("१२३४५६७८९०१२", "एक दोन तीन चार पाच सहा सात आठ नऊ शून्य एक दोन"), + (".५", "पूर्णांक पाच"), + ("३.५ हजार", "तीन पूर्णांक पाच हजार"), + ("२२ मे 2023", "बावीस मे दोन हजार तेवीस"), + ("३२ जून", "बत्तीस जून"), + ("00:55", "एकला पाच मिनिटे"), + ("7:15", "सव्वा सात"), + ("०२:३०", "अडीच"), + ("१४:४५", "पावणे तीन"), +]) +def test_additional_semantics(normalizer, written, spoken): + assert normalizer.normalize(written) == spoken + + +@pytest.mark.parametrize("written, spoken, punctuation", [ + ("(5)", "(पाच)", ["(", ")"]), + ("5,", "पाच,", [","]), + ("5 !", "पाच !", [" !"]), + ("( 5 )", "( पाच )", ["( ", " )"]), +]) +def test_cardinal_punctuation_spacing(normalizer, written, spoken, punctuation): + assert normalizer.normalize(written) == spoken + tagged = normalizer.tag(written) + parser = normalizer.token_parser() + parser.parse(tagged) + assert [token.members for token in parser.tokens if token.name == "cardinal"] == [{"integer": "5"}] + assert [token.members for token in parser.tokens if token.name == "punctuation"] == [ + {"value": value} for value in punctuation + ] + assert normalizer.verbalize(tagged) == spoken + mapped = normalizer.normalize_with_mapping(written, include_identity=True) + assert mapped.output_text == spoken + assert [mapping.input_text for mapping in mapped.mappings if mapping.token_type == "punctuation"] == punctuation + for mapping in mapped.mappings: + assert written[mapping.input_start:mapping.input_end] == mapping.input_text + assert spoken[mapping.output_start:mapping.output_end] == mapping.output_text + + +def test_quarter_times_keep_plain_reading_as_alternative(normalizer): + assert normalizer.normalize("०४:३०", nbest=2) == ["साडे चार", "चार वाजून तीस मिनिटे"] + + +@pytest.mark.parametrize("written", ["४ लाख", "०४:३०", "मार्च ५ १९९८", "१,००,०००", "-१२.५"]) +def test_nbest_mapping_synchronization(normalizer, written): + outputs = normalizer.normalize(written, nbest=3) + mapped = normalizer.normalize_with_mapping(written, nbest=3, include_identity=True) + assert [result.output_text for result in mapped] == outputs + assert len(outputs) == len(set(outputs)) + assert outputs[0] == normalizer.normalize(written) + for result in mapped: + for mapping in result.mappings: + assert written[mapping.input_start:mapping.input_end] == mapping.input_text + assert result.output_text[mapping.output_start:mapping.output_end] == mapping.output_text + + +def test_escaped_word_round_trips(normalizer): + written = 'मी\\a"b' + parser = normalizer.token_parser() + parser.parse(normalizer.tag(written)) + assert [token.members for token in parser.tokens] == [{"value": written}] + assert normalizer.normalize(written) == written + + +def test_empty_input(normalizer): + assert normalizer.normalize("") == "" + assert normalizer.normalize("", nbest=3) == [""] + assert normalizer.normalize_with_mapping("").output_text == "" + + +@pytest.mark.parametrize("written", [" ", "\t", "\n", " \t\n ", "\u00a0"]) +def test_whitespace_only_input(normalizer, written): + tagged = normalizer.tag(written) + parser = normalizer.token_parser() + parser.parse(tagged) + assert len(parser.tokens) == 1 + assert parser.tokens[0].name == "word" + assert parser.tokens[0].members == {"value": written} + assert normalizer.verbalize(tagged) == "" + assert normalizer.normalize(written, nbest=3) == [""] + mapped = normalizer.normalize_with_mapping(written, nbest=3, include_identity=True) + assert [result.output_text for result in mapped] == [""] + assert all(mapping.output_text == "" for mapping in mapped[0].mappings) + + +def test_standalone_from_itn(): + itn_import = re.compile(r"^\s*(from|import)\s+itn\.marathi\b", re.MULTILINE) + for path in PACKAGE.rglob("*.py"): + assert not itn_import.search(path.read_text(encoding="utf-8")), path diff --git a/tn/test/cli_test.py b/tn/test/cli_test.py index 60167201..c974e617 100644 --- a/tn/test/cli_test.py +++ b/tn/test/cli_test.py @@ -60,6 +60,7 @@ def verbalize(self, tagged): ("tn", "sv", "tn.swedish.normalizer", "Normalizer", ["--no-deterministic"], {"deterministic": False}), ("tn", "es", "tn.spanish.normalizer", "Normalizer", ["--no-deterministic"], {"deterministic": False}), ("tn", "hi", "tn.hindi.normalizer", "Normalizer", [], {}), + ("tn", "mr", "tn.marathi.normalizer", "Normalizer", [], {}), ("tn", "rw", "tn.kinyarwanda.normalizer", "Normalizer", [], {}), ( "itn", @@ -95,6 +96,7 @@ def verbalize(self, tagged): ("itn", "sv", "itn.swedish.inverse_normalizer", "InverseNormalizer", [], {}), ("itn", "es", "itn.spanish.inverse_normalizer", "InverseNormalizer", ["--cased"], {"cased": True}), ("itn", "hi", "itn.hindi.inverse_normalizer", "InverseNormalizer", [], {}), + ("itn", "mr", "itn.marathi.inverse_normalizer", "InverseNormalizer", [], {}), ("itn", "rw", "itn.kinyarwanda.inverse_normalizer", "InverseNormalizer", [], {}), ], ) @@ -219,6 +221,8 @@ def __init__(self, **kwargs): ("itn", "es", "--cased", "--enable-million"), ("tn", "hi", "--cache-dir", "--deterministic"), ("itn", "hi", "--cache-dir", "--enable-million"), + ("tn", "mr", "--cache-dir", "--deterministic"), + ("itn", "mr", "--cache-dir", "--enable-million"), ("tn", "rw", "--cache-dir", "--deterministic"), ("itn", "rw", "--cache-dir", "--enable-million"), ],