feat: add Kinyarwanda TN/ITN with raw-field tagging

README.md

@@ -13,7 +13,7 @@
 WeTextProcessing converts between written and spoken text using weighted finite-state transducers (WFSTs), built with [Pynini](https://www.openfst.org/twiki/bin/view/GRM/Pynini) and [OpenFst](https://www.openfst.org/twiki/bin/view/FST/WebHome).
 
 - **TN and ITN** — normalize numbers, dates, times, currencies, units, and other structured expressions.
-- **Multilingual rules** — Arabic, Armenian, Chinese, English, French, German, Hindi, Japanese, Korean, Portuguese, Russian, Spanish, Swedish, and Vietnamese pipelines with language-specific grammars.
+- **Multilingual rules** — language-specific grammars for every language in [Supported languages](#supported-languages).
 - **Explainable output** — inspect raw tagged fields, trace input/output spans, and request unique n-best results.
 - **Reusable graphs** — automatic, configuration-aware FST caching and a separate C++ deployment runtime.
 
@@ -45,6 +45,7 @@ ITN converts spoken expressions into written forms, for example `十二` → `12
 | German | `de` | Yes | Yes | `german` |
 | Hindi | `hi` | Yes | Yes | `hindi` |
 | Japanese | `ja` | Yes | Yes | `japanese` |
+| Kinyarwanda | `rw` | Yes | Yes | `kinyarwanda` |
 | Korean | `ko` | Yes | Yes | `korean` |
 | Portuguese | `pt` | Yes | Yes | `portuguese` |
 | Russian | `ru` | Yes | Yes | `russian` |
@@ -54,7 +55,7 @@ ITN converts spoken expressions into written forms, for example `十二` → `12
 
 Chinese is the CLI default. Rule coverage and configurable options vary by language and direction.
 
-Arabic, Armenian, French, German, Hindi, Korean, Portuguese, Russian, Spanish, Swedish, and Vietnamese support are included in this source checkout. Install from source to use them before the next package release.
+The published package includes Chinese, English, and Japanese. The other languages are included in this source checkout; install from source to use them before the next package release.
 
 ## Quick start
 
@@ -66,7 +67,7 @@ Install the published package:
 pip install WeTextProcessing
 ```
 
-For the latest rules, including Arabic, Armenian, French, German, Hindi, Korean, Portuguese, Russian, Spanish, Swedish, and Vietnamese, install from source:
+For the latest rules, including every language beyond Chinese, English, and Japanese, install from source:
 
 ```bash
 git clone https://github.com/wenet-e2e/WeTextProcessing.git

itn/kinyarwanda/data/numbers/digit.tsv

@@ -0,0 +1,9 @@
+1	rimwe
+2	kabiri
+3	gatatu
+4	kane
+5	gatanu
+6	gatandatu
+7	karindwi
+8	umunani
+9	icyenda

itn/kinyarwanda/data/numbers/digit_bi.tsv

@@ -0,0 +1,9 @@
+1	kimwe
+2	bibiri
+3	bitatu
+4	bine
+5	bitanu
+6	bitandatu
+7	birindwi
+8	umunani
+9	icyenda

itn/kinyarwanda/data/numbers/digit_e.tsv

@@ -0,0 +1,9 @@
+1	imwe
+2	ebyiri
+3	eshatu
+4	enye
+5	eshanu
+6	esheshatu
+7	zirindwi
+8	umunani
+9	icyenda

itn/kinyarwanda/data/numbers/hundreds.tsv

@@ -0,0 +1,9 @@
+1	ijana
+2	magana abiri
+3	magana atatu
+4	magana ane
+5	magana atanu
+6	magana atandatu
+7	magana arindwi
+8	magana inani
+9	magana cyenda

itn/kinyarwanda/data/numbers/tens.tsv

@@ -0,0 +1,8 @@
+2	makumyabiri
+3	mirongo itatu
+4	mirongo ine
+5	mirongo itanu
+6	mirongo itandatu
+7	mirongo irindwi
+8	mirongo inani
+9	mirongo icyenda

itn/kinyarwanda/data/numbers/zero.tsv

@@ -0,0 +1 @@
+0	zeru

itn/kinyarwanda/data/time/hours.tsv

@@ -0,0 +1,12 @@
+saba	1
+munani	2
+cyenda	3
+cumi	4
+cumi n'imwe	5
+cumi n'ebyiri	6
+moya	7
+mbiri	8
+tatu	9
+ine	10
+tanu	11
+sita	12

itn/kinyarwanda/data/time/minutes.tsv

@@ -0,0 +1,59 @@
+umunota umwe	01
+iminota ibiri	02
+iminota itatu	03
+iminota ine	04
+iminota itanu	05
+iminota itandatu	06
+iminota irindwi	07
+iminota umunani	08
+iminota icyenda	09
+iminota icumi	10
+iminota cumi n'umwe	11
+iminota cumi n'ibiri	12
+iminota cumi n'itatu	13
+iminota cumi n'ine	14
+iminota cumi n'itanu	15
+iminota cumi n'itandatu	16
+iminota cumi n'irindwi	17
+iminota cumi n'umunani	18
+iminota cumi n'icyenda	19
+iminota makumyabiri	20
+iminota makumyabiri na rimwe	21
+iminota makumyabiri n'ibiri	22
+iminota makumyabiri n'itatu	23
+iminota makumyabiri n'ine	24
+iminota makumyabiri n'itanu	25
+iminota makumyabiri n'itandatu	26
+iminota makumyabiri n'irindwi	27
+iminota makumyabiri n'umunani	28
+iminota makumyabiri n'icyenda	29
+iminota mirongo itatu	30
+iminota mirongo itatu n'umwe	31
+iminota mirongo itatu n'ibiri	32
+iminota mirongo itatu n'itatu	33
+iminota mirongo itatu n'ine	34
+iminota mirongo itatu n'itanu	35
+iminota mirongo itatu n'itandatu	36
+iminota mirongo itatu n'irindwi	37
+iminota mirongo itatu n'umunani	38
+iminota mirongo itatu n'icyenda	39
+iminota mirongo ine	40
+iminota mirongo ine n'umwe	41
+iminota mirongo ine n'ibiri	42
+iminota mirongo ine n'itatu	43
+iminota mirongo ine n'ine	44
+iminota mirongo ine n'itanu	45
+iminota mirongo ine n'itandatu	46
+iminota mirongo ine n'irindwi	47
+iminota mirongo ine n'umunani	48
+iminota mirongo ine n'icyenda	49
+iminota mirongo itanu	50
+iminota mirongo itanu n'umwe	51
+iminota mirongo itanu n'ibiri	52
+iminota mirongo itanu n'itatu	53
+iminota mirongo itanu n'ine	54
+iminota mirongo itanu n'itanu	55
+iminota mirongo itanu n'itandatu	56
+iminota mirongo itanu n'irindwi	57
+iminota mirongo itanu n'umunani	58
+iminota mirongo itanu n'icyenda	59

itn/kinyarwanda/data/whitelist.tsv

@@ -0,0 +1,18 @@
+sidisi	CDC
+arabisi	RBC
+ade eni	ADN
+bibisi	BBC
+heci ayivi	HIV
+diyeniyi	DNA
+araneyi	RNA
+pisiyara	PCR
+ayisiyu	ICU
+arati-pisiyara	RT-PCR
+hashiseyeri	HCR
+efu emi	FMI
+aradibi	RDB
+arabidi	RBD
+aradiyefu	RDF
+vi o aye	VOA
+emu araneyi	mRNA
+wawawa	www

itn/kinyarwanda/inverse_normalizer.py

@@ -0,0 +1,53 @@
+# Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn)
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from pynini.lib.pynutil import add_weight
+
+from itn.kinyarwanda.rules.cardinal import Cardinal
+from itn.kinyarwanda.rules.punctuation import Punctuation
+from itn.kinyarwanda.rules.time import Time
+from itn.kinyarwanda.rules.whitelist import WhiteList
+from itn.kinyarwanda.rules.word import Word
+from tn.processor import Processor, RuleSpec
+
+# Every Kinyarwanda rule emits its fields in written order, so no token needs
+# reordering before verbalization.
+TOKEN_ORDERS = {}
+
+
+class InverseNormalizer(Processor):
+
+    def __init__(self, cache_dir=None, overwrite_cache=False):
+        super().__init__(name="rw_inverse_normalizer", ordertype="itn", token_orders=TOKEN_ORDERS)
+        self.build_fst("rw_itn", cache_dir, overwrite_cache, {})
+
+    def build_tagger_and_verbalizer(self):
+        punctuation = Punctuation()
+        word = Word()
+        rules = (
+            RuleSpec(WhiteList(), 1.01),
+            RuleSpec(Time(), 1.05),
+            RuleSpec(Cardinal(), 1.1),
+            RuleSpec(word, 100),
+            RuleSpec(punctuation),
+        )
+        classify = self.tagger_union(rules)
+        # Punctuation splits off only when attached to another token.
+        punct = add_weight(punctuation.tagger, 1.1)
+        token = punct.star + classify + punct.star
+        self.tagger = self.DELETE_SPACE + token + (self.DELETE_EXTRA_SPACE + token).star + self.DELETE_SPACE
+        self.tagger |= word.add_tokens(word.tag_field("value", self.SPACE.plus))
+        verbalize = self.verbalizer_union(rules)
+        self.verbalizer = verbalize + (self.INSERT_SPACE + verbalize).star

itn/kinyarwanda/rules/cardinal.py

@@ -0,0 +1,131 @@
+# Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn)
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from itertools import product
+
+import pynini
+from pynini import accep, cross, difference, invert, union
+from pynini.lib.pynutil import add_weight, delete, insert
+
+from itn.kinyarwanda.rules.utils import string_map
+from tn.processor import Processor
+
+VOWELS = "aeiou"
+CONSONANTS = "bcdfghjklmnpqrstvwxyz"
+# Ten has a standalone form and a form that is followed by its units.
+TEN = "icumi"
+TEN_BEFORE_UNITS = "cumi"
+# Scale words: the bare word for exactly one, the class prefix otherwise, and
+# the hand-written explicit form for one (``miliyoni imwe``), if any.
+SCALES = (
+    ("tiriyoni", "tiriyoni ", "tiriyoni imwe"),
+    ("miliyoni", "miliyoni ", "miliyoni imwe"),
+    ("igihumbi", "ibihumbi ", None),
+)
+# The explicit forms for one are accepted but never preferred over the bare
+# scale word when several spellings are requested.
+EXPLICIT_ONE_WEIGHT = 0.01
+# TN readings are not unique: ``ibihumbi ijana na mirongo itatu`` is written
+# for both 130000 and 100030. Each part costs a little more the lower its
+# scale group, so the reading that keeps parts under the larger scale wins.
+GROUP_PART_WEIGHT = 0.001
+
+
+class Cardinal(Processor):
+    """Spoken cardinals, e.g. ``ijana na makumyabiri na gatatu`` -> ``123``.
+
+    Accepts the readings written by Kinyarwanda TN: units agree with what
+    they count (``kabiri`` alone, ``ibihumbi bibiri``, ``miliyoni ebyiri``)
+    and parts are joined with ``na`` before a consonant and ``n'`` before a
+    vowel; ``tiriyoni`` names 10^9. Besides the bare ``miliyoni`` and
+    ``tiriyoni`` for exactly one, the explicit ``miliyoni imwe`` and
+    ``tiriyoni imwe`` are accepted. Digit-by-digit readings that start with
+    ``zeru`` (``zeru zeru kabiri``) are written as a digit string.
+
+    The tagger keeps the spoken words (``integer`` or ``digits``); the
+    verbalizer writes the digits.
+    """
+
+    def __init__(self):
+        super().__init__(name="cardinal", ordertype="itn", token_orders={})
+        digit = string_map("kinyarwanda/data/numbers/digit.tsv")
+        zero = string_map("kinyarwanda/data/numbers/zero.tsv")
+        integer = invert(zero | self.build_number()).optimize()
+        any_digit = invert(digit | zero)
+        digits = invert(zero) + (delete(" ") + any_digit).plus
+
+        tagger = self.tag_field("integer", integer) | self.tag_field("digits", digits.optimize())
+        self.tagger = self.add_tokens(tagger)
+        verbalizer = self.verbalize_field("integer", integer) | self.verbalize_field("digits", digits.optimize())
+        self.verbalizer = self.delete_tokens(verbalizer)
+
+    def connect(self, graph):
+        """Prefixes a part with ``na`` or ``n'`` depending on its first letter."""
+
+        vowel_initial = union(*VOWELS) + self.VSIGMA
+        consonant_initial = union(*CONSONANTS) + self.VSIGMA
+        return (insert(" n'") + (graph @ vowel_initial)) | (insert(" na ") + (graph @ consonant_initial))
+
+    def group(self, units, weight):
+        """Maps a three-digit group other than ``000`` to its reading in a unit class.
+
+        Each spoken part of the group costs ``weight``.
+        """
+
+        hundreds = string_map("kinyarwanda/data/numbers/hundreds.tsv")
+        tens = string_map("kinyarwanda/data/numbers/tens.tsv")
+        options = []
+        for has_hundreds, has_tens, has_units in product((False, True), repeat=3):
+            if not (has_hundreds or has_tens or has_units):
+                continue
+            ten = cross("1", TEN_BEFORE_UNITS if has_units else TEN) | tens
+            graph = accep("")
+            started = False
+            for present, part in ((has_hundreds, hundreds), (has_tens, ten), (has_units, units)):
+                if not present:
+                    graph += delete("0")
+                    continue
+                part = add_weight(part, weight)
+                graph += self.connect(part) if started else part
+                started = True
+            options.append(graph)
+        return union(*options).optimize()
+
+    def build_number(self):
+        """Builds the digits -> words map; the caller inverts it."""
+
+        digit = string_map("kinyarwanda/data/numbers/digit.tsv")
+        digit_bi = string_map("kinyarwanda/data/numbers/digit_bi.tsv")
+        digit_e = string_map("kinyarwanda/data/numbers/digit_e.tsv")
+        not_one = difference(pynini.closure(self.DIGIT, 3, 3), "001").optimize()
+
+        groups = []
+        for index, ((one, prefix, explicit_one), units) in enumerate(zip(SCALES, (digit_e, digit_e, digit_bi))):
+            weight = index * GROUP_PART_WEIGHT
+            graph = add_weight(cross("001", one), weight) | (not_one @ (insert(prefix) + self.group(units, weight)))
+            if explicit_one is not None:
+                graph |= add_weight(cross("001", explicit_one), EXPLICIT_ONE_WEIGHT)
+            groups.append(graph.optimize())
+        groups.append(self.group(digit, len(SCALES) * GROUP_PART_WEIGHT))
+
+        nonzero = difference(self.DIGIT, "0")
+        pad = insert("00") + nonzero | insert("0") + nonzero + self.DIGIT | nonzero + self.DIGIT + self.DIGIT
+        options = []
+        for index, top in enumerate(groups):
+            graph = pad @ top
+            for lower in groups[index + 1:]:
+                graph += delete("000") | self.connect(lower)
+            options.append(graph)
+        return union(*options).optimize()

itn/kinyarwanda/rules/punctuation.py

@@ -0,0 +1,28 @@
+# Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# Copyright (c) 2026 Zhendong Peng (pzd17@tsinghua.org.cn)
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import pynini
+
+from tn.processor import Processor
+
+
+class Punctuation(Processor):
+    """ASCII punctuation split off a neighbouring token, e.g. ``gatanu,``."""
+
+    def __init__(self):
+        super().__init__(name="punctuation", ordertype="itn", token_orders={})
+        graph = pynini.union(*[pynini.escape(mark) for mark in "!#$%&'()*+,-./:;<=>?@^_`{|}~"]).optimize()
+        self.tagger = self.add_tokens(self.tag_field("value", graph))
+        self.verbalizer = self.delete_tokens(self.verbalize_field("value", graph))